Clémentine
commited on
Commit
•
8137036
1
Parent(s):
a99162a
Fixed tokenizer message
Browse files
src/submission/check_validity.py
CHANGED
@@ -42,13 +42,15 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
|
|
42 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
43 |
if test_tokenizer:
|
44 |
try:
|
45 |
-
AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
46 |
except ValueError as e:
|
47 |
return (
|
48 |
False,
|
49 |
f"uses a tokenizer which is not in a transformers release: {e}",
|
50 |
None
|
51 |
)
|
|
|
|
|
52 |
return True, None, config
|
53 |
|
54 |
except ValueError:
|
|
|
42 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
43 |
if test_tokenizer:
|
44 |
try:
|
45 |
+
tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
46 |
except ValueError as e:
|
47 |
return (
|
48 |
False,
|
49 |
f"uses a tokenizer which is not in a transformers release: {e}",
|
50 |
None
|
51 |
)
|
52 |
+
except Exception as e:
|
53 |
+
return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
|
54 |
return True, None, config
|
55 |
|
56 |
except ValueError:
|