after restart

events.out.tfevents.1628300146.t1v-n-1a0a7c50-w-0.1616525.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94e7c67609c3f27b9c3650ba91ed66f4f1a3106cf7c7c92c998bc28b399653d3
-size 52176063
+oid sha256:dc846e02789f45a7b1ada21713b56d4496b6dfa15c056619f2a5b46eb22ecdc0
+size 52250633

flax_model_backup350k.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7a1b3c986f8f90bfb67978326d950efc15e64ad9b6d5f4884bcde1013a65968
+size 1100762015

run_streaming.sh

@@ -3,6 +3,7 @@
     --model_type="t5" \
     --config_name="./" \
     --tokenizer_name="./" \
+    --model_name_or_path="./" \
     --dataset_name="pere/norwegian_colossal_corpus_v2_short100k" \
     --max_seq_length="512" \
     --weight_decay="0.01" \

run_t5_mlm_flax_streaming.py

@@ -552,16 +552,17 @@ if __name__ == "__main__":
     rng = jax.random.PRNGKey(training_args.seed)
     dropout_rngs = jax.random.split(rng, jax.local_device_count())
 
-    model = FlaxT5ForConditionalGeneration(config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype))
-
-    #if model_args.model_name_or_path:
-    #    model = FlaxT5ForConditionalGeneration.from_pretrained(
-    #        model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
-    #    )
-    #else:
-    #    model = FlaxT5ForConditionalGeneration.from_pretrained(
-    #        config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
-    #    )
+    #Pere changed 13 august
+    #model = FlaxT5ForConditionalGeneration(config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype))
+
+    if model_args.model_name_or_path:
+        model = FlaxT5ForConditionalGeneration.from_pretrained(
+            model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
+        )
+    else:
+        model = FlaxT5ForConditionalGeneration.from_pretrained(
+            config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
+        )
 
 
     # Data collator