ul2
Browse files- batch_lrtest_ul2scand.sh +81 -0
- finetune_translate_base_lr.gin +33 -0
- tasks.py +2 -2
batch_lrtest_ul2scand.sh
ADDED
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
PROJECT_DIR=${HOME}"/models/t5-nynorsk-norbench"
|
3 |
+
export PYTHONPATH=${PROJECT_DIR}
|
4 |
+
echo "PROJECT_DIR is set to: ${PROJECT_DIR}"
|
5 |
+
|
6 |
+
|
7 |
+
FINETUNE_STEPS=30000
|
8 |
+
EVAL_PREFIX="norbench/eval_lr_translate_ul2_scand_"
|
9 |
+
MODEL_BUCKET_DIR="gs://pere-north-t5x/finetuned/"
|
10 |
+
|
11 |
+
CHECKPOINT_LIST=(
|
12 |
+
"nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
|
13 |
+
"nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
|
14 |
+
"nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
|
15 |
+
"nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
|
16 |
+
"nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
|
17 |
+
"nb-t5x-us-central2/exp5-t5-base-ul2-scandvoc-full/checkpoint_1500000"
|
18 |
+
)
|
19 |
+
|
20 |
+
NAME_LIST=(
|
21 |
+
"north_t5_base_NCC_LR_0_00001"
|
22 |
+
"north_t5_base_NCC_LR_0_0001"
|
23 |
+
"north_t5_base_NCC_LR_0_0005"
|
24 |
+
"north_t5_base_NCC_LR_0_00075"
|
25 |
+
"north_t5_base_NCC_LR_0_001"
|
26 |
+
"north_t5_base_NCC_LR_0_002"
|
27 |
+
)
|
28 |
+
|
29 |
+
TASK_LIST=("translate_mt5" "translate_mt5" "translate_mt5" "translate_mt5" "translate_mt5" "translate_mt5")
|
30 |
+
LR_LIST=(0.00001 0.0001 0.0005 0.00075 0.001 0.002)
|
31 |
+
|
32 |
+
|
33 |
+
GIN_LIST=(
|
34 |
+
"finetune_translate_base_mt5_lr.gin"
|
35 |
+
"finetune_translate_base_mt5_lr.gin"
|
36 |
+
"finetune_translate_base_mt5_lr.gin"
|
37 |
+
"finetune_translate_base_mt5_lr.gin"
|
38 |
+
"finetune_translate_base_mt5_lr.gin"
|
39 |
+
"finetune_translate_base_mt5_lr.gin"
|
40 |
+
"finetune_translate_base_mt5_lr.gin"
|
41 |
+
)
|
42 |
+
|
43 |
+
START_LIST=(1500000 1500000 1500000 1500000 1500000 1500000)
|
44 |
+
EXP_LIST=(171 172 173 174 175 176)
|
45 |
+
|
46 |
+
|
47 |
+
VERSION_LIST=("1")
|
48 |
+
|
49 |
+
index=$(($1 + 0))
|
50 |
+
|
51 |
+
if [ $index -lt 1 ] || [ $index -gt ${#CHECKPOINT_LIST[@]} ] || [ $# -ne 1 ]; then
|
52 |
+
echo "Error: You need to provide the number of the checkpoints below as a parameter."
|
53 |
+
for i in "${!CHECKPOINT_LIST[@]}"; do
|
54 |
+
echo "$((i+1)). ${CHECKPOINT_LIST[i]}"
|
55 |
+
done
|
56 |
+
|
57 |
+
exit 1
|
58 |
+
fi
|
59 |
+
|
60 |
+
for v in "${VERSION_LIST[@]}"; do
|
61 |
+
i=($index-1)
|
62 |
+
INITIAL_CHECKPOINT_PATH="gs://${CHECKPOINT_LIST[i]}"
|
63 |
+
|
64 |
+
TRAIN_STEPS=$((START_LIST[i]+FINETUNE_STEPS))
|
65 |
+
GIN_FILE=${GIN_LIST[i]}
|
66 |
+
MIXTURE_OR_TASK_NAME=${TASK_LIST[i]}
|
67 |
+
LR=${LR_LIST[i]}
|
68 |
+
MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP_LIST[i]}_${NAME_LIST[i]}_v${v}"
|
69 |
+
|
70 |
+
command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin.LR=${LR} --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
|
71 |
+
echo "${command}"
|
72 |
+
# Uncomment the next line to run the command:
|
73 |
+
eval "${command}"
|
74 |
+
done
|
75 |
+
|
76 |
+
#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH} --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
|
77 |
+
#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH} --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
|
78 |
+
#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH} --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
|
79 |
+
#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH} --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
|
80 |
+
#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH} --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
|
81 |
+
|
finetune_translate_base_lr.gin
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from __gin__ import dynamic_registration
|
2 |
+
import tasks
|
3 |
+
|
4 |
+
import __main__ as train_script
|
5 |
+
from t5.data import mixtures
|
6 |
+
from t5x import models
|
7 |
+
from t5x import partitioning
|
8 |
+
from t5x import utils
|
9 |
+
|
10 |
+
include "t5x/examples/t5/t5_1_1/base.gin"
|
11 |
+
include "t5x/configs/runs/finetune.gin"
|
12 |
+
|
13 |
+
MIXTURE_OR_TASK_NAME = %gin.REQUIRED
|
14 |
+
TASK_FEATURE_LENGTHS = {"inputs": 256, "targets": 256}
|
15 |
+
INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
|
16 |
+
TRAIN_STEPS = %gin.REQUIRED # 1000000 pre-trained steps + 10000 fine-tuning steps.
|
17 |
+
USE_CACHED_TASKS = False
|
18 |
+
DROPOUT_RATE = 0.1
|
19 |
+
RANDOM_SEED = 0
|
20 |
+
|
21 |
+
#Fixing a small error
|
22 |
+
infer_eval/utils.DatasetConfig:
|
23 |
+
task_feature_lengths = %TASK_FEATURE_LENGTHS
|
24 |
+
|
25 |
+
#Saving every 1000 steps
|
26 |
+
utils.SaveCheckpointConfig:
|
27 |
+
period = 1000
|
28 |
+
keep = 1 # number of checkpoints to keep
|
29 |
+
|
30 |
+
# Might have to ba changed based on architecture
|
31 |
+
# partitioning.PjitPartitioner.num_partitions = 1
|
32 |
+
|
33 |
+
|
tasks.py
CHANGED
@@ -194,9 +194,9 @@ seqio.TaskRegistry.add(
|
|
194 |
)
|
195 |
|
196 |
seqio.TaskRegistry.add(
|
197 |
-
"
|
198 |
source=seqio.TextLineDataSource(
|
199 |
-
split_to_filepattern=
|
200 |
#num_input_examples=num_nq_examples
|
201 |
),
|
202 |
preprocessors=[
|
|
|
194 |
)
|
195 |
|
196 |
seqio.TaskRegistry.add(
|
197 |
+
"translate_scand",
|
198 |
source=seqio.TextLineDataSource(
|
199 |
+
split_to_filepattern=tsv_translate_path,
|
200 |
#num_input_examples=num_nq_examples
|
201 |
),
|
202 |
preprocessors=[
|