ul2

batch_lrtest_ul2scand.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+PROJECT_DIR=${HOME}"/models/t5-nynorsk-norbench"
+export PYTHONPATH=${PROJECT_DIR}
+echo "PROJECT_DIR is set to: ${PROJECT_DIR}"
+
+
+FINETUNE_STEPS=30000
+EVAL_PREFIX="norbench/eval_lr_translate_ul2_scand_"
+MODEL_BUCKET_DIR="gs://pere-north-t5x/finetuned/"
+
+CHECKPOINT_LIST=(
+  "nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
+  "nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
+  "nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
+  "nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
+  "nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
+  "nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
+)
+
+NAME_LIST=(
+  "north_t5_base_NCC_LR_0_00001"
+  "north_t5_base_NCC_LR_0_0001"
+  "north_t5_base_NCC_LR_0_0005"
+  "north_t5_base_NCC_LR_0_00075"
+  "north_t5_base_NCC_LR_0_001"
+  "north_t5_base_NCC_LR_0_002"
+)
+
+TASK_LIST=("translate_mt5" "translate_mt5" "translate_mt5" "translate_mt5" "translate_mt5" "translate_mt5")
+LR_LIST=(0.00001 0.0001 0.0005 0.00075 0.001 0.002)
+
+
+GIN_LIST=(
+  "finetune_translate_base_mt5_lr.gin"
+  "finetune_translate_base_mt5_lr.gin"
+  "finetune_translate_base_mt5_lr.gin"
+  "finetune_translate_base_mt5_lr.gin"
+  "finetune_translate_base_mt5_lr.gin"
+  "finetune_translate_base_mt5_lr.gin"
+  "finetune_translate_base_mt5_lr.gin"
+)
+
+START_LIST=(1500000 1500000 1500000 1500000 1500000 1500000)
+EXP_LIST=(171 172 173 174 175 176)
+
+
+VERSION_LIST=("1") 
+
+index=$(($1 + 0))
+
+if [ $index -lt 1 ] || [ $index -gt ${#CHECKPOINT_LIST[@]} ] || [ $# -ne 1 ]; then
+	echo "Error: You need to provide the number of the checkpoints below as a parameter."
+	for i in "${!CHECKPOINT_LIST[@]}"; do
+	        echo "$((i+1)). ${CHECKPOINT_LIST[i]}"
+	done	
+	
+	  exit 1
+fi
+
+for v in "${VERSION_LIST[@]}"; do
+	i=($index-1)
+    INITIAL_CHECKPOINT_PATH="gs://${CHECKPOINT_LIST[i]}"
+    
+    TRAIN_STEPS=$((START_LIST[i]+FINETUNE_STEPS))
+    GIN_FILE=${GIN_LIST[i]}
+    MIXTURE_OR_TASK_NAME=${TASK_LIST[i]}
+    LR=${LR_LIST[i]}
+    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP_LIST[i]}_${NAME_LIST[i]}_v${v}"
+
+    command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin.LR=${LR}  --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
+    echo "${command}"
+    # Uncomment the next line to run the command:
+    eval "${command}"
+done
+
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
+

finetune_translate_base_lr.gin

@@ -0,0 +1,33 @@
+from __gin__ import dynamic_registration
+import tasks
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include "t5x/examples/t5/t5_1_1/base.gin"
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 256, "targets": 256}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+  keep = 1  # number of checkpoints to keep
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+

tasks.py

@@ -194,9 +194,9 @@ seqio.TaskRegistry.add(
 )   
 
 seqio.TaskRegistry.add(
-    "translate_long_scand",
+    "translate_scand",
     source=seqio.TextLineDataSource(
-        split_to_filepattern=tsv_translate_long_path,
+        split_to_filepattern=tsv_translate_path,
         #num_input_examples=num_nq_examples
         ),
     preprocessors=[