upcast_layernorm: true for 4bit qlora

llama-factory/config/qwen2_72b_lora_sft_4bit-p1.yaml

@@ -9,6 +9,7 @@ lora_target: all
 quantization_bit: 4                     # use 4-bit QLoRA
 loraplus_lr_ratio: 16.0                 # use LoRA+ with lambda=16.0
 # use_unsloth: true                       # use UnslothAI's LoRA optimization for 2x faster training
+upcast_layernorm: true
 
 ### dataset
 dataset: alpaca_mac

llama-factory/config/qwen2_72b_lora_sft_4bit-p2.yaml

@@ -9,6 +9,7 @@ lora_target: all
 quantization_bit: 4                     # use 4-bit QLoRA
 loraplus_lr_ratio: 16.0                 # use LoRA+ with lambda=16.0
 # use_unsloth: true                       # use UnslothAI's LoRA optimization for 2x faster training
+upcast_layernorm: true
 
 ### dataset
 dataset: alpaca_mac