Update app.py
Browse files
app.py
CHANGED
|
@@ -14,8 +14,7 @@ os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:50'
|
|
| 14 |
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
| 15 |
model_name = "allenai/tulu-2-dpo-70b"
|
| 16 |
tokenizer = AutoTokenizer.from_pretrained("allenai/tulu-2-dpo-70b")
|
| 17 |
-
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
|
| 18 |
-
model = model.to(dtype=torch.bfloat16)
|
| 19 |
|
| 20 |
class XgenChatBot:
|
| 21 |
def __init__(self, model, tokenizer, system_message="You are 🌷Tulu, an AI language model created by Tonic-AI. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."):
|
|
|
|
| 14 |
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
| 15 |
model_name = "allenai/tulu-2-dpo-70b"
|
| 16 |
tokenizer = AutoTokenizer.from_pretrained("allenai/tulu-2-dpo-70b")
|
| 17 |
+
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, use_flash_attention_2=True, device_map="auto")
|
|
|
|
| 18 |
|
| 19 |
class XgenChatBot:
|
| 20 |
def __init__(self, model, tokenizer, system_message="You are 🌷Tulu, an AI language model created by Tonic-AI. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."):
|