Update model

eval_results.json

@@ -1,4 +1,4 @@
 {
-    "eval_accuracy": 0.6356837749481201,
-    "eval_loss": 1.9501113891601562
+    "eval_accuracy": 0.6502997875213623,
+    "eval_loss": 1.9498332738876343
 }

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00bb5b987a1bba701e3d03578ceed23e976c4b9e22db5d095c12495abd5e3a09
+oid sha256:35274b0f8d7d45dba0dec9b4043244035f8ee3d3c11465cb8828f53c433a50c8
 size 307750439

run_longt5-local-small-mc4.sh

@@ -26,21 +26,22 @@ python ../train/run_t5_mlm_flax_pmap.py \
     --dataset_name="${DATASET}" \
     --dataset_config_name="${DATASET_CONFIG}" \
     --max_seq_length="1024" \
-    --per_device_train_batch_size="32" \
-    --per_device_eval_batch_size="32" \
-    --gradient_accumulation_steps="4" \
+    --per_device_train_batch_size="64" \
+    --per_device_eval_batch_size="64" \
+    --gradient_accumulation_steps="1" \
     --mean_noise_span_length="3" \
+    --gradient_checkpointing="false" \
     --dtype="bfloat16" \
     --z_loss="1e-4" \
     --optim="adafactor" \
     --learning_rate="0.005" \
-    --lr_decay="linear" \
+    --lr_scheduler_type="linear" \
     --overwrite_output_dir \
     --num_train_epochs="6" \
-    --logging_steps="20" \
-    --save_steps="1000" \
-    --eval_steps="1000" \
-    --warmup_steps="300" \
+    --logging_steps="80" \
+    --save_steps="4000" \
+    --eval_steps="4000" \
+    --warmup_steps="1200" \
     --wandb_project="long-t5-local-small" \
     --wandb_job_type="pmap"
 

runs/{events.out.tfevents.1660513900.t1v-n-40c2b24e-w-0.1025593.0.v2 → Aug15_21-16-54_t1v-n-40c2b24e-w-0/events.out.tfevents.1660598268.t1v-n-40c2b24e-w-0.1128999.0.v2}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1863a6bfac6afc7708d3bb6e53d4f1b1eaa7d05a3adeabb256ed2e600622e52
-size 1086215
+oid sha256:2edc48086840f9785d0b3426ead313735f851c7705e4c146b669b1b9fa321282
+size 544983

training_state.json

@@ -1 +1 @@
-{"step": 168001}
+{"step": 84001}