kiddothe2b
commited on
Fix minor bug in auto-chunking tokenization
Browse files- tokenization_hat.py +1 -1
tokenization_hat.py
CHANGED
@@ -157,7 +157,7 @@ class HATTokenizer:
|
|
157 |
if isinstance(key_inputs_list[0], list):
|
158 |
key_inputs_list = [token for sentence in key_inputs_list for token in sentence]
|
159 |
else:
|
160 |
-
key_inputs_list = torch.stack(key_inputs_list)
|
161 |
if input_key in batch:
|
162 |
batch[input_key].append(key_inputs_list)
|
163 |
else:
|
|
|
157 |
if isinstance(key_inputs_list[0], list):
|
158 |
key_inputs_list = [token for sentence in key_inputs_list for token in sentence]
|
159 |
else:
|
160 |
+
key_inputs_list = torch.stack([token for sentence in key_inputs_list for token in sentence])
|
161 |
if input_key in batch:
|
162 |
batch[input_key].append(key_inputs_list)
|
163 |
else:
|