Saving weights and logs of epoch 0

.gitattributes

@@ -25,3 +25,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qgen_train.csv filter=lfs diff=lfs merge=lfs -text
+qgen_training_data.csv filter=lfs diff=lfs merge=lfs -text
+qgen_val.csv filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 512,
+  "decoder_start_token_id": 0,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 8,
+  "num_heads": 6,
+  "num_layers": 8,
+  "pad_token_id": 0,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "transformers_version": "4.12.0.dev0",
+  "use_cache": true,
+  "vocab_size": 250112
+}

events.out.tfevents.1634611280.t1v-n-10409466-w-0.11492.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e182970a4bdaa09ab4bc39e24998573f2279663d8092199e97f5f27f509662e0
+size 40

events.out.tfevents.1634647562.t1v-n-10409466-w-0.4496.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f76d55580d4e2d1fe8679c9fe034df11554145300a05ce56d4d4839bf55ff6
+size 40

events.out.tfevents.1634681194.t1v-n-10409466-w-0.3720.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a33497fbf05f4540c98f5a8a9a4bdf2f88c6330c4bd969cd379a55f84b0d2e6
+size 2450039

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d1919cb1cc5091f578509943b09248e0a3f4cbdbdebf111c0d96fb2a47ff3d
+size 1200715307

mt5-base-qgen/events.out.tfevents.1634610755.t1v-n-10409466-w-0.9004.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e1acf82767bf97369c40271b7915dfe127686a6a3908c2d97b7626d45315e6d
+size 40

mt5-questions-dataset.log

@@ -0,0 +1,11 @@
+[{"stream_name":"stderr","time":34.74216129,"data":"/opt/conda/lib/python3.7/site-packages/papermill/iorw.py:50: FutureWarning: pyarrow.HadoopFileSystem is deprecated as of 2.0.0, please use pyarrow.fs.HadoopFileSystem instead.\n"}
+,{"stream_name":"stderr","time":34.74226826,"data":"  from pyarrow import HadoopFileSystem\n"}
+,{"stream_name":"stderr","time":37.894455456,"data":"/opt/conda/lib/python3.7/site-packages/traitlets/traitlets.py:2567: FutureWarning: --Exporter.preprocessors=[\"remove_papermill_header.RemovePapermillHeader\"] for containers is deprecated in traitlets 5.0. You can pass `--Exporter.preprocessors item` ... multiple times to add items to a list.\n"}
+,{"stream_name":"stderr","time":37.89452555,"data":"  FutureWarning,\n"}
+,{"stream_name":"stderr","time":37.894535347,"data":"[NbConvertApp] Converting notebook __notebook__.ipynb to notebook\n"}
+,{"stream_name":"stderr","time":37.920776986,"data":"[NbConvertApp] Writing 2546 bytes to __notebook__.ipynb\n"}
+,{"stream_name":"stderr","time":40.120591007,"data":"/opt/conda/lib/python3.7/site-packages/traitlets/traitlets.py:2567: FutureWarning: --Exporter.preprocessors=[\"nbconvert.preprocessors.ExtractOutputPreprocessor\"] for containers is deprecated in traitlets 5.0. You can pass `--Exporter.preprocessors item` ... multiple times to add items to a list.\n"}
+,{"stream_name":"stderr","time":40.12066086,"data":"  FutureWarning,\n"}
+,{"stream_name":"stderr","time":40.120670616,"data":"[NbConvertApp] Converting notebook __notebook__.ipynb to html\n"}
+,{"stream_name":"stderr","time":40.662991409,"data":"[NbConvertApp] Writing 276762 bytes to __results__.html\n"}
+]

qgen_train.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21a0e59e17a98496c830c4eaa5e7109e648ccc49905acd30cc24dc477233cd71
+size 337028529

qgen_training_data.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8045ce51c73fed96d10df60dae29f3dc9eb6344dd84046e65f38aa00edd0d258
+size 420218929

qgen_val.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:215fc1360575a907d9d0dc89fbe08e656e41c7ca76df813c12b95eb98fd68597
+size 83190420

run_summarization_flax.py

@@ -446,7 +446,7 @@ def main():
     # In Flax, for seq2seq models we need to pass `decoder_input_ids`
     # as the Flax models don't accept `labels`, we need to prepare the decoder_input_ids here
     # for that dynamically import the `shift_tokens_right` function from the model file
-    model_module = __import__(model.__module__, fromlist=["shift_tokens_tight"])
+    model_module = __import__("transformers.models.t5.modeling_flax_t5", fromlist=["shift_tokens_tight"])
     shift_tokens_right_fn = getattr(model_module, "shift_tokens_right")
 
     # Setting padding="max_length" as we need fixed length inputs for jitted functions

special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

start_qgen.sh

@@ -1,15 +1,21 @@
+#! /usr/bin/env bash
+
 python run_summarization_flax.py \
-	--output_dir ./mt5-base-qgen \
-	--model_name_or_path google/mt5-base \
-	--tokenizer_name google/mt5-base \
-    --train_file qgen_training_data.csv \
-    --text_column context \
-    --summary_column question \
-	--do_train 
-	--num_train_epochs 3 \
-	--learning_rate 5e-5 --warmup_steps 300 \
-	--per_device_train_batch_size 2 \
-	--per_device_eval_batch_size 2 \
-	--overwrite_output_dir \
-	--max_source_length 1024 --max_target_length 64 \
-	--push_to_hub
+	    --output_dir ./ \
+	        --model_name_or_path google/mt5-small \
+		    --tokenizer_name google/mt5-small \
+		        --train_file qgen_train.csv \
+			--validation_file qgen_val.csv \
+			    --text_column context \
+			        --summary_column question \
+				    --do_train \
+				    --do_eval \
+    --num_train_epochs 3 \
+    --eval_steps 2000 \
+    --logging_steps 100 \
+	        --learning_rate 5e-5 --warmup_steps 1000 \
+		    --per_device_train_batch_size 2 \
+		        --per_device_eval_batch_size 2 \
+			    --overwrite_output_dir \
+			        --max_source_length 1024 --max_target_length 64 \
+				    --push_to_hub

tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 0, "additional_special_tokens": null, "special_tokens_map_file": "/home/nicholas/.cache/huggingface/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276", "name_or_path": "google/mt5-small", "sp_model_kwargs": {}, "tokenizer_class": "T5Tokenizer"}