llama3 finetune config

llama-factory/config/llama3-8b_lora_sft_bf16-p1.yaml

@@ -22,16 +22,16 @@ preprocessing_num_workers: 16
 ### output
 output_dir: saves/llama3-8b/lora/sft_bf16_p1_full
 logging_steps: 10
-save_steps: 88
+save_steps: 175
 plot_loss: true
 overwrite_output_dir: true
 # resume_from_checkpoint: true
 
 ### train
-per_device_train_batch_size: 32
+per_device_train_batch_size: 16
 gradient_accumulation_steps: 8
 learning_rate: 1.0e-4
-num_train_epochs: 6.0
+num_train_epochs: 3.0
 lr_scheduler_type: cosine
 warmup_ratio: 0.1
 bf16: true
@@ -41,7 +41,7 @@ ddp_timeout: 180000000
 val_size: 0.1
 per_device_eval_batch_size: 1
 eval_strategy: steps
-eval_steps: 88
+eval_steps: 175
 
 report_to: wandb
 run_name: llama3_8b_p1_full # optional

llama-factory/config/llama3-8b_lora_sft_bf16-p2.yaml

@@ -0,0 +1,47 @@
+### model
+model_name_or_path: FlagAlpha/Llama3-Chinese-8B-Instruct
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_target: all
+# quantization_bit: 4                     # use 4-bit QLoRA
+loraplus_lr_ratio: 16.0                 # use LoRA+ with lambda=16.0
+# use_unsloth: true                       # use UnslothAI's LoRA optimization for 2x faster training
+upcast_layernorm: true
+
+### dataset
+dataset: alpaca_mgtv_p2
+template: llama3
+cutoff_len: 4096
+max_samples: 25000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### output
+output_dir: saves/llama3-8b/lora/sft_bf16_p2_full
+logging_steps: 10
+save_steps: 175
+plot_loss: true
+overwrite_output_dir: true
+# resume_from_checkpoint: true
+
+### train
+per_device_train_batch_size: 16
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+val_size: 0.1
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 175
+
+report_to: wandb
+run_name: llama3_8b_p2_full # optional