Update config.json

config.json

@@ -21,7 +21,7 @@
   "pretraining_tp": 1,
   "quantization_config": {
     "_load_in_4bit": false,
-    "_load_in_8bit": false,
+    "_load_in_8bit": true,
     "bnb_4bit_compute_dtype": "float32",  // Changed to float32 for CPU compatibility
     "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
@@ -31,7 +31,7 @@
     "llm_int8_skip_modules": null,
     "llm_int8_threshold": 6.0,
     "load_in_4bit": false,  // Disabled for CPU
-    "load_in_8bit": false,  // Disabled for CPU
+    "load_in_8bit": true,  // Disabled for CPU
     "quant_method": "bitsandbytes"
   },
   "rms_norm_eps": 1e-05,