Spaces:
Runtime error
Runtime error
fix parsing so spaces in front of numbers don't go missing
Browse files
app.py
CHANGED
@@ -189,13 +189,11 @@ def token_generator(generator1, generator2, mapping_fn=None, fillvalue=None):
|
|
189 |
if not mapping_fn:
|
190 |
mapping_fn = lambda x: x
|
191 |
for output1, output2 in itertools.zip_longest(generator1, generator2, fillvalue=fillvalue):
|
192 |
-
tokens1 = re.findall(r'\s
|
193 |
-
tokens2 = re.findall(r'\s
|
194 |
-
# tokens1 = re.findall(r'(.*?)(\s|$)', mapping_fn(output1))
|
195 |
-
# tokens2 = re.findall(r'(.*?)(\s|$)', mapping_fn(output2))
|
196 |
|
197 |
for token1, token2 in itertools.zip_longest(tokens1, tokens2, fillvalue=''):
|
198 |
-
yield token1, token2
|
199 |
|
200 |
|
201 |
def chat(history1, history2, system_msg, state):
|
|
|
189 |
if not mapping_fn:
|
190 |
mapping_fn = lambda x: x
|
191 |
for output1, output2 in itertools.zip_longest(generator1, generator2, fillvalue=fillvalue):
|
192 |
+
tokens1 = re.findall(r'(.*?)(\s|$)', mapping_fn(output1))
|
193 |
+
tokens2 = re.findall(r'(.*?)(\s|$)', mapping_fn(output2))
|
|
|
|
|
194 |
|
195 |
for token1, token2 in itertools.zip_longest(tokens1, tokens2, fillvalue=''):
|
196 |
+
yield "".join(token1), "".join(token2)
|
197 |
|
198 |
|
199 |
def chat(history1, history2, system_msg, state):
|