ready for llama3 r4

llama-factory/config/llama3-8b_lora_sft_bf16-p1_r4.yaml

@@ -0,0 +1,47 @@
+### model
+model_name_or_path: shenzhi-wang/Llama3-8B-Chinese-Chat
+#model_name_or_path: FlagAlpha/Llama3-Chinese-8B-Instruct
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_target: all
+# quantization_bit: 4                     # use 4-bit QLoRA
+loraplus_lr_ratio: 16.0                 # use LoRA+ with lambda=16.0
+# use_unsloth: true                       # use UnslothAI's LoRA optimization for 2x faster training
+upcast_layernorm: true
+
+### dataset
+dataset: alpaca_mgtv_p1
+template: llama3
+cutoff_len: 4096
+max_samples: 25000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### output
+output_dir: saves/llama3-8b/lora/sft_bf16_p1_full_r4
+logging_steps: 10
+save_steps: 35
+plot_loss: true
+# overwrite_output_dir: true
+
+### train
+per_device_train_batch_size: 16
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 2.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+val_size: 0.1
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 35
+
+report_to: wandb
+run_name: llama3_8b_p1_full_r4 # optional

results/mgtv-llama3_p2_r3_full_metrics.csv

@@ -2,3 +2,6 @@ epoch,model,accuracy,precision,recall,f1
 0.0,hfl/llama-3-chinese-8b-instruct-v3_torch.bfloat16_lf,0.25066666666666665,0.6852419041932336,0.25066666666666665,0.32636449818329016
 0.2,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-35_torch.bfloat16_lf,0.7283333333333334,0.7722393813259697,0.7283333333333334,0.7426450360790026
 0.4,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-70_torch.bfloat16_lf,0.741,0.7868300593752113,0.741,0.7514058688729928
+0.6,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-105_torch.bfloat16_lf,0.6223333333333333,0.7771706776754249,0.6223333333333333,0.6762790454549326
+0.8,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-140_torch.bfloat16_lf,0.7,0.7767966010489314,0.7,0.7298480873851099
+1.0,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-175_torch.bfloat16_lf,0.697,0.78712001874989,0.697,0.7309586130328194

scripts/eval-mgtv-llama3_8b.sh

@@ -21,19 +21,19 @@ export RESIZE_TOKEN_EMBEDDINGS=true
 export START_EPOCH=0
 export USING_LLAMA_FACTORY=true
 
-# export MODEL_NAME=shenzhi-wang/Llama3-8B-Chinese-Chat
-export MODEL_NAME=hfl/llama-3-chinese-8b-instruct-v3
+export MODEL_NAME=shenzhi-wang/Llama3-8B-Chinese-Chat
+# export MODEL_NAME=hfl/llama-3-chinese-8b-instruct-v3
 
 export MODEL_PREFIX=llama3-8b_lora_sft_bf16
 
-export LOGICAL_REASONING_RESULTS_PATH=results/$MODEL_PREFIX-p1_r3.csv
-export ADAPTER_PATH_BASE=llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full_r3
+export LOGICAL_REASONING_RESULTS_PATH=results/$MODEL_PREFIX-p1_r4.csv
+export ADAPTER_PATH_BASE=llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full_r4
 export USING_P1_PROMPT_TEMPLATE=true
 echo "Eval $MODEL_NAME with $ADAPTER_PATH_BASE"
 python llm_toolkit/eval_logical_reasoning_all_epochs.py
 
-export LOGICAL_REASONING_RESULTS_PATH=results/$MODEL_PREFIX-p2_r3.csv
-export ADAPTER_PATH_BASE=llama-factory/saves/llama3-8b/lora/sft_bf16_p2_full_r3
-export USING_P1_PROMPT_TEMPLATE=false
-echo "Eval $MODEL_NAME with $ADAPTER_PATH_BASE"
-python llm_toolkit/eval_logical_reasoning_all_epochs.py
+# export LOGICAL_REASONING_RESULTS_PATH=results/$MODEL_PREFIX-p2_r4.csv
+# export ADAPTER_PATH_BASE=llama-factory/saves/llama3-8b/lora/sft_bf16_p2_full_r4
+# export USING_P1_PROMPT_TEMPLATE=false
+# echo "Eval $MODEL_NAME with $ADAPTER_PATH_BASE"
+# python llm_toolkit/eval_logical_reasoning_all_epochs.py

scripts/tune-mgtv-llama3_8b.sh

@@ -20,19 +20,19 @@ grep MemTotal /proc/meminfo
 
 export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
 
-#export MODEL_NAME=shenzhi-wang/Llama3-8B-Chinese-Chat
-export MODEL_NAME=hfl/llama-3-chinese-8b-instruct-v3
+export MODEL_NAME=shenzhi-wang/Llama3-8B-Chinese-Chat
+# export MODEL_NAME=hfl/llama-3-chinese-8b-instruct-v3
 
 export MODEL_PREFIX=llama3-8b_lora_sft_bf16
 
-export CONFIG_FILE=config/$MODEL_PREFIX-p1_r3.yaml
+export CONFIG_FILE=config/$MODEL_PREFIX-p1_r4.yaml
 echo "Tuning $MODEL_NAME with $CONFIG_FILE"
 $BASEDIR/scripts/tune-lf.sh $CONFIG_FILE
 
 
-export CONFIG_FILE=config/$MODEL_PREFIX-p2_r3.yaml
-echo "Tuning $MODEL_NAME with $CONFIG_FILE"
-$BASEDIR/scripts/tune-lf.sh $CONFIG_FILE
+# export CONFIG_FILE=config/$MODEL_PREFIX-p2_r4.yaml
+# echo "Tuning $MODEL_NAME with $CONFIG_FILE"
+# $BASEDIR/scripts/tune-lf.sh $CONFIG_FILE
 
 
 $BASEDIR/scripts/eval-mgtv-llama3_8b.sh

scripts/tune-mgtv.sh

@@ -1 +1 @@
-tune-mgtv-internlm.sh
+tune-mgtv-llama3_8b.sh