raja-nectar commited on
Commit
6f2d0f3
1 Parent(s): 073576a

Create llama_factory_config.yaml

Browse files
Files changed (1) hide show
  1. llama_factory_config.yaml +42 -0
llama_factory_config.yaml ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Run on 8xh100 for 80/500 steps as loss was converged.
2
+
3
+ ### model
4
+ model_name_or_path: NeverSleep/Lumimaid-v0.2-70B
5
+
6
+ ### method
7
+ stage: sft
8
+ do_train: true
9
+ finetuning_type: lora
10
+ lora_target: all
11
+ deepspeed: examples/deepspeed/ds_z3_offload_config.json
12
+
13
+ ### dataset
14
+ dataset: bolt_sharegpt
15
+ template: llama3
16
+ cutoff_len: 2048
17
+ max_samples: 20000
18
+ overwrite_cache: true
19
+ preprocessing_num_workers: 16
20
+
21
+ ### output
22
+ output_dir: saves/lumi_70b_bolt_lora_1e4
23
+ logging_steps: 1
24
+ save_steps: 40
25
+ plot_loss: true
26
+ overwrite_output_dir: true
27
+
28
+ ### train
29
+ per_device_train_batch_size: 2
30
+ gradient_accumulation_steps: 8
31
+ learning_rate: 1.0e-4
32
+ num_train_epochs: 3.0
33
+ lr_scheduler_type: cosine
34
+ warmup_ratio: 0.01
35
+ bf16: true
36
+ ddp_timeout: 180000000
37
+
38
+ ### eval
39
+ val_size: 0.05
40
+ per_device_eval_batch_size: 1
41
+ eval_strategy: steps
42
+ eval_steps: 40