iekei commited on
Commit
9800c25
1 Parent(s): 42fd6ab

Upload training_config.yml with huggingface_hub

Browse files
Files changed (1) hide show
  1. training_config.yml +87 -0
training_config.yml ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ _component_: models.lora_mmllama3_8b
3
+ lora_attn_modules:
4
+ - q_proj
5
+ - v_proj
6
+ apply_lora_to_mlp: false
7
+ apply_lora_to_output: false
8
+ lora_rank: 8
9
+ lora_alpha: 16
10
+ perception_tokens: 2
11
+ use_clip: false
12
+ tokenizer:
13
+ _component_: models.a2a_tokenizer
14
+ path: models/tokenizer.model
15
+ checkpointer:
16
+ _component_: torchtune.utils.FullModelMetaCheckpointer
17
+ checkpoint_dir:
18
+ checkpoint_files:
19
+ -
20
+ adapter_checkpoint: null
21
+ recipe_checkpoint: null
22
+ output_dir: output_checkpoints/experiment_1
23
+ model_type: LLAMA3
24
+
25
+ resume_from_checkpoint: false
26
+ interim_checkpoint_steps: 20000
27
+ interim_gen_steps: null
28
+ max_new_tokens: 100
29
+ temperature: 0.6
30
+ top_k: 225
31
+ dataset:
32
+ _component_: ds.EvenBatcher
33
+ buffer_size: 1000
34
+ dataset:
35
+ _component_: ds.RoundRobinDataset
36
+ datasets:
37
+ - _component_: ds.OmegaVideoCaptionDataset
38
+ length: 3000
39
+ - _component_: ds.LlavaInstructDataset
40
+ dataset_path: ds/coco_llava_instruct/output.parquet
41
+ train_on_input: false
42
+ - _component_: ds.LlavaInstructDataset
43
+ dataset_path: ds/vision_flan/output.parquet
44
+ train_on_input: false
45
+ - _component_: ds.CaptionInstructDataset
46
+ dataset_path: ds/sam_llava/output.parquet
47
+ train_on_input: false
48
+ seed: null
49
+ shuffle: true
50
+ batch_size: 6
51
+ optimizer:
52
+ _component_: torch.optim.AdamW
53
+ weight_decay: 0.0
54
+ lr: 0.0
55
+
56
+ lr_scheduler:
57
+ _component_: torchtune.modules.get_cosine_schedule_with_warmup
58
+ num_warmup_steps: 00
59
+ loss:
60
+ _component_: torch.nn.CrossEntropyLoss
61
+
62
+ epochs: 6
63
+ max_steps_per_epoch: null
64
+ gradient_accumulation_steps: 32
65
+ compile: true
66
+ output_dir: /tmp/lora_finetune_output
67
+ metric_logger:
68
+ _component_: torchtune.utils.metric_logging.DiskLogger
69
+ log_dir: ${output_dir}
70
+ log_every_n_steps: null
71
+ device: cuda
72
+ dtype: bf16
73
+ enable_activation_checkpointing: false
74
+ profiler:
75
+ _component_: torchtune.utils.profiler
76
+ enabled: false
77
+ inference:
78
+ prompt_template: 'Video:
79
+
80
+ {video}
81
+
82
+ Caption the previous video.'
83
+ max_new_tokens: 300
84
+ temperature: 0.6
85
+ top_k: 300
86
+ quantizer: null
87
+ gradient-accumulation-steps: 32