baichuan-inc
/

Baichuan2-13B-Chat

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

wuzhiying commited on Sep 6, 2023

Commit

35e3d5a

•

1 Parent(s): be60123

fix BitsAndBytesConfig 8bit issues

Files changed (1) hide show

modeling_baichuan.py +2 -1

modeling_baichuan.py CHANGED Viewed

@@ -534,7 +534,8 @@ class BaichuanForCausalLM(BaichuanPreTrainedModel):
         super().__init__(config, *model_args, **model_kwargs)
         self.model = BaichuanModel(config)
         self.lm_head = NormHead(config.hidden_size, config.vocab_size, bias=False)
-        if hasattr(config, "quantization_config") and config.quantization_config['load_in_4bit']:
             try:
                 from .quantizer import quantize_offline, init_model_weight_int4
             except ImportError:

         super().__init__(config, *model_args, **model_kwargs)
         self.model = BaichuanModel(config)
         self.lm_head = NormHead(config.hidden_size, config.vocab_size, bias=False)
+        #if hasattr(config, "quantization_config") and config.quantization_config['load_in_4bit']:
+        if hasattr(config, "quantization_config") and isinstance(config.quantization_config, dict) and config.quantization_config.get('load_in_4bit', False):
             try:
                 from .quantizer import quantize_offline, init_model_weight_int4
             except ImportError: