Commit
•
fc3d617
1
Parent(s):
5eb3aa1
Update tokenization_hyena.py
Browse files- tokenization_hyena.py +1 -1
tokenization_hyena.py
CHANGED
@@ -94,7 +94,7 @@ class HyenaDNATokenizer(PreTrainedTokenizer):
|
|
94 |
already_has_special_tokens=True,
|
95 |
)
|
96 |
|
97 |
-
result =
|
98 |
if token_ids_1 is not None:
|
99 |
result += ([0] * len(token_ids_1)) + [1]
|
100 |
return result
|
|
|
94 |
already_has_special_tokens=True,
|
95 |
)
|
96 |
|
97 |
+
result = ([0] * len(token_ids_0)) + [1]
|
98 |
if token_ids_1 is not None:
|
99 |
result += ([0] * len(token_ids_1)) + [1]
|
100 |
return result
|