Update HQQbagelmix_def.py

HQQbagelmix_def.py

@@ -1,13 +1,13 @@
 from hqq.engine.hf import HQQModelForCausalLM, AutoTokenizer
-model_path  = "/mnt/ancient240/Undi95_BagelMix-8x7B"
+model_path  = "Undi95/BagelMix-8x7B" #i used a directory here not hf's identifiers but in principle you can do that if your internet's fast and you trust the cache...
 model     = HQQModelForCausalLM.from_pretrained(model_path)
 
 #Quantize params
 from hqq.core.quantize import *
-#attn_prams     = BaseQuantizeConfig(nbits=4, group_size=64, quant_zero=True, quant_scale=True)
-attn_prams     = BaseQuantizeConfig(nbits=4, group_size=64, quant_zero=True, quant_scale=True)
+
+attn_prams     = BaseQuantizeConfig(nbits=4, group_size=64, quant_zero=True, quant_scale=True) #4g64
 attn_prams['scale_quant_params']['group_size'] = 256
-experts_params = BaseQuantizeConfig(nbits=2, group_size=16, quant_zero=True, quant_scale=True)
+experts_params = BaseQuantizeConfig(nbits=2, group_size=16, quant_zero=True, quant_scale=True) #2g16
 
 quant_config = {}
 #Attention
@@ -19,10 +19,10 @@ quant_config['self_attn.o_proj'] = attn_prams
 quant_config['block_sparse_moe.experts.w1'] = experts_params
 quant_config['block_sparse_moe.experts.w2'] = experts_params
 quant_config['block_sparse_moe.experts.w3'] = experts_params
-#print("configured")
+
 #Quantize
 save_path="models/BagelMix-8x7B-2g16-4g64-HQQ/"
 model.quantize_model(quant_config=quant_config)
 model.save_quantized(save_path)
 
-#brought to you by python -i
+#brought to you by python -i and a 200 GB swap device, code adapted from last upload