Update README.md

README.md

@@ -60,274 +60,28 @@ Coming soon...
 
 ## Training Configuration
 
-<details><summary>Click here for Axolotl configs</summary>
+<details><summary>Click here for Mergekit and Axolotl configs</summary>
 
-Completion SFT
+MoE Merge
 
 ```yaml
-base_model: IntervitensInc/Llama-3.1-Minitron-4B-Width-Base-chatml
-model_type: AutoModelForCausalLM
-tokenizer_type: AutoTokenizer
-
-load_in_8bit: false
-load_in_4bit: false
-strict: false
-
-hub_model_id: jeiku/completion4B
-hub_strategy: "all_checkpoints"
-push_dataset_to_hub:
-hf_use_auth_token: true
-
-datasets:
-  - path: Mielikki/Erebus-87k
-    type: completion
-    field: body
-
-shuffle_merged_datasets: true
-val_set_size: 0.0025
-output_dir: ./outputs/out
-
-adapter:
-lora_r:
-lora_alpha:
-lora_dropout:
-lora_target_linear:
-
-sequence_len: 8192
-sample_packing: true
-eval_sample_packing: false
-pad_to_sequence_len: true
-
-plugins:
-  - axolotl.integrations.liger.LigerPlugin
-liger_rope: true
-liger_rms_norm: true
-liger_swiglu: true
-liger_fused_linear_cross_entropy: true
-
-wandb_project: EXP4B
-wandb_entity:
-wandb_watch:
-wandb_name: EXP4B
-wandb_log_model:
-
-gradient_accumulation_steps: 12
-micro_batch_size: 3
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.00001
-weight_decay: 0.05
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch: 4
-eval_table_size:
-eval_max_new_tokens: 128
-saves_per_epoch: 1
-
-debug:
-deepspeed: deepspeed_configs/zero3_bf16.json
-fsdp:
-fsdp_config:
-
-special_tokens:
-  pad_token: <|finetune_right_pad_id|>
-```
-
-Instruct SFT
-
-```yaml
-base_model: jeiku/completion4B
-model_type: AutoModelForCausalLM
-tokenizer_type: AutoTokenizer
-
-load_in_8bit: false
-load_in_4bit: false
-strict: false
-
-hub_model_id: jeiku/instructered4B
-hub_strategy: "all_checkpoints"
-push_dataset_to_hub:
-hf_use_auth_token: true
-
-datasets:
-  - path: FourOhFour/Instruct_Phase
-    type: sharegpt
-    conversation: chatml
-
-chat_template: chatml
-
-shuffle_merged_datasets: true
-val_set_size: 0.0025
-output_dir: ./outputs/out
-
-adapter:
-lora_r:
-lora_alpha:
-lora_dropout:
-lora_target_linear:
-
-sequence_len: 8192
-sample_packing: true
-eval_sample_packing: false
-pad_to_sequence_len: true
-
-plugins:
-  - axolotl.integrations.liger.LigerPlugin
-liger_rope: true
-liger_rms_norm: true
-liger_swiglu: true
-liger_fused_linear_cross_entropy: true
-
-wandb_project: EXP4B
-wandb_entity:
-wandb_watch:
-wandb_name: EXP4B
-wandb_log_model:
-
-gradient_accumulation_steps: 12
-micro_batch_size: 3
-num_epochs: 2
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.00001
-weight_decay: 0.05
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint: 
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch: 4
-eval_table_size:
-eval_max_new_tokens: 128
-saves_per_epoch: 2
-
-debug:
-deepspeed: deepspeed_configs/zero3_bf16.json
-fsdp:
-fsdp_config:
-
-special_tokens:
-  pad_token: <|finetune_right_pad_id|>
-```
-
-Roleplaying SFT
-
-```yaml
-base_model: jeiku/instructered4B
-model_type: AutoModelForCausalLM
-tokenizer_type: AutoTokenizer
-
-load_in_8bit: false
-load_in_4bit: false
-strict: false
-
-hub_model_id: jeiku/TheBest4B
-hub_strategy: "all_checkpoints"
-push_dataset_to_hub:
-hf_use_auth_token: true
-
-datasets:
-  - path: FourOhFour/RP_Phase
-    type: sharegpt
-    conversation: chatml
-
-chat_template: chatml
-
-shuffle_merged_datasets: true
-val_set_size: 0.0025
-output_dir: ./outputs/out
-
-adapter:
-lora_r:
-lora_alpha:
-lora_dropout:
-lora_target_linear:
-
-sequence_len: 8192
-sample_packing: true
-eval_sample_packing: false
-pad_to_sequence_len: true
-
-plugins:
-  - axolotl.integrations.liger.LigerPlugin
-liger_rope: true
-liger_rms_norm: true
-liger_swiglu: true
-liger_fused_linear_cross_entropy: true
-
-wandb_project: EXP4B
-wandb_entity:
-wandb_watch:
-wandb_name: EXP4B
-wandb_log_model:
-
-gradient_accumulation_steps: 12
-micro_batch_size: 3
-num_epochs: 2
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.00001
-weight_decay: 0.05
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint: 
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch: 4
-eval_table_size:
-eval_max_new_tokens: 128
-saves_per_epoch: 2
-
-debug:
-deepspeed: deepspeed_configs/zero3_bf16.json
-fsdp:
-fsdp_config:
-
-special_tokens:
-  pad_token: <|finetune_right_pad_id|>
+base_model: FourOhFour/Crispy_Crab_4B
+gate_mode: hidden
+dtype: bfloat16
+experts_per_token: 1
+experts:
+  - source_model: FourOhFour/Crispy_Crab_4B
+    positive_prompts:
+      - "You are a roleplaying powerhouse, reply as {{char}} in this ongoing conversation with {{user}}."
+  - source_model: FourOhFour/Zenith_4B
+    positive_prompts:
+      - "You are a helpful assistant designed to perform tasks for the user."
 ```
 
 KTO
 
 ```yaml
-base_model: FourOhFour/Crispy_Crab_4B
+base_model: jeiku/2x4Bmoe
 model_type: AutoModelForCausalLM
 tokenizer_type: AutoTokenizer
 
@@ -335,7 +89,7 @@ load_in_8bit: false
 load_in_4bit: false
 strict: false
 
-hub_model_id: jeiku/aura4bkto
+hub_model_id: jeiku/moekto
 hub_strategy: "all_checkpoints"
 push_dataset_to_hub:
 hf_use_auth_token: true
@@ -354,15 +108,24 @@ shuffle_merged_datasets: true
 val_set_size: 0.0
 output_dir: ./outputs/out
 
+adapter: lora
+lora_model_dir:
+
+lora_r: 32
+lora_alpha: 64
+lora_dropout: 0.05
+lora_target_linear: true
+lora_fan_in_fan_out:
+
 sequence_len: 8192
 sample_packing: false
 eval_sample_packing: false
 pad_to_sequence_len: false
 
-wandb_project: Aura-4B
+wandb_project: moekto
 wandb_entity:
 wandb_watch:
-wandb_name: Aura-4B
+wandb_name: moekto
 wandb_log_model:
 
 gradient_accumulation_steps: 16
@@ -372,7 +135,7 @@ max_steps: 500
 
 optimizer: adamw_8bit
 lr_scheduler: cosine
-learning_rate: 0.00001
+learning_rate: 0.0001
 weight_decay: 0.05
 
 train_on_inputs: false