Update app.py

app.py

@@ -8,7 +8,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStream
 
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
-MAX_INPUT_TOKEN_LENGTH = 4096
+MAX_INPUT_TOKEN_LENGTH = 65536
 
 DESCRIPTION = """\
 # CLEX-7B-Chat-16K
@@ -52,7 +52,7 @@ from modeling_llama import LlamaForCausalLM
 # config = CLEXLlamaConfig.from_pretrained(
 #         model_id
 #     )
-model = LlamaForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, trust_remote_code=True, low_cpu_mem_usage=True)
+model = LlamaForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, trust_remote_code=True, low_cpu_mem_usage=True, device_map="auto")
 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
 tokenizer.use_default_system_prompt = False