Update config.json

config.json

@@ -26,4 +26,15 @@
   "untie_embeddings_and_output_weights": true,
   "use_cache": true,
   "vocab_size": 131072
+  "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.0.18",
+        "bits": 4.0,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
 }