Spaces:
Runtime error
Runtime error
small python fix
Browse files
detect-pretrain-code-contamination/src/run.py
CHANGED
@@ -44,7 +44,7 @@ def load_model(name1):
|
|
44 |
if name1 not in models:
|
45 |
model1 = AutoModelForCausalLM.from_pretrained(name1, return_dict=True, device_map='auto')
|
46 |
model1.eval()
|
47 |
-
if
|
48 |
tokenizer1 = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
|
49 |
else:
|
50 |
tokenizer1 = AutoTokenizer.from_pretrained(name1)
|
|
|
44 |
if name1 not in models:
|
45 |
model1 = AutoModelForCausalLM.from_pretrained(name1, return_dict=True, device_map='auto')
|
46 |
model1.eval()
|
47 |
+
if "mistral" in name1 or "Mistral" in name1: #Loading default mistral tokenizers as some tokenizers don't work out of the box.
|
48 |
tokenizer1 = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
|
49 |
else:
|
50 |
tokenizer1 = AutoTokenizer.from_pretrained(name1)
|