Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files
app.py
CHANGED
|
@@ -3,7 +3,7 @@ from xsbpe.basic import BasicTokenizer
|
|
| 3 |
|
| 4 |
tk = BasicTokenizer()
|
| 5 |
print('Tokenizer initialized.')
|
| 6 |
-
tk.train(open('dune.txt').read(), 256 +
|
| 7 |
print('Training complete.')
|
| 8 |
|
| 9 |
def tokenize(text):
|
|
|
|
| 3 |
|
| 4 |
tk = BasicTokenizer()
|
| 5 |
print('Tokenizer initialized.')
|
| 6 |
+
tk.train(open('dune.txt').read(), 256 + 10000, verbose=False)
|
| 7 |
print('Training complete.')
|
| 8 |
|
| 9 |
def tokenize(text):
|