added tiktoken
Browse files- requirements.txt +2 -0
requirements.txt
CHANGED
|
@@ -6,3 +6,5 @@ joblib
|
|
| 6 |
matplotlib
|
| 7 |
seaborn
|
| 8 |
transformers
|
|
|
|
|
|
|
|
|
| 6 |
matplotlib
|
| 7 |
seaborn
|
| 8 |
transformers
|
| 9 |
+
tiktoken
|
| 10 |
+
SentencePiece
|