base_model: mistralai/Mistral-7B-Instruct-v0.2
model_type: MistralForCausalLM
is_mistral_derived_model: true
datasets:
  - path: burkelibbey/colors
    type:
      field_instruction: color
      field_output: description
    conversation: chatml
chat_template: chatml
val_set_size: 0.05
output_dir: ./out
sequence_len: 512
gradient_accumulation_steps: 4
micro_batch_size: 2
learning_rate: 0.0002

load_in_4bit: true
adapter: qlora
lora_r: 16
lora_alpha: 8
lora_dropout: 0.05
lora_target_linear: true
gradient_checkpointing: true
warmup_steps: 10

tokenizer_type: LlamaTokenizer
num_epochs: 1
optimizer: paged_adamw_32bit
special_tokens:
  bos_token: "<s>"
  eos_token: "</s>"
  unk_token: "<unk>"