Not use 8 bit quantization

app.py

@@ -5,12 +5,7 @@ from typing import Iterator
 import gradio as gr
 import spaces
 import torch
-from transformers import (
-    AutoModelForCausalLM,
-    BitsAndBytesConfig,
-    GemmaTokenizerFast,
-    TextIteratorStreamer,
-)
+from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer
 
 DESCRIPTION = """\
 # Gemma 2 9B IT
@@ -33,7 +28,7 @@ tokenizer = GemmaTokenizerFast.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
-    quantization_config=BitsAndBytesConfig(load_in_8bit=True),
+    torch_dtype=torch.bfloat16,
 )
 model.config.sliding_window = 4096
 model.eval()