Update app.py

app.py

@@ -26,16 +26,11 @@ model_id = "CohereForAI/c4ai-command-r-plus-4bit"
 
 tokenizer = AutoTokenizer.from_pretrained(model_id, token= token)
 
-# model = AutoModelForCausalLM.from_pretrained(model_id, token= token, torch_dtype=torch.bfloat16,    
-#                                              # attn_implementation="flash_attention_2",
-#                                              # low_cpu_mem_usage=True,
-#                                              llm_int8_enable_fp32_cpu_offload=True,
-#                                              device_map="auto"
-#                                             )
+# 
 
 
 
-@spaces.GPU(duration=180)
+@spaces.GPU(duration=300)
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -44,6 +39,13 @@ def respond(
     temperature,
     top_p,
 ):
+    model = AutoModelForCausalLM.from_pretrained(model_id, token= token, 
+                                                 # torch_dtype=torch.bfloat16,    
+                                             # attn_implementation="flash_attention_2",
+                                             # low_cpu_mem_usage=True,
+                                             # llm_int8_enable_fp32_cpu_offload=True,
+                                             device_map="auto"
+                                            )
     messages = [{"role": "user", "content": "Hello, how are you?"}]
     input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
 ## <BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>