acecalisto3 commited on
Commit
0d3f166
1 Parent(s): f2e38a8

Create gemmacode/tokenizer.py

Browse files
Files changed (1) hide show
  1. gemmacode/tokenizer.py +5 -0
gemmacode/tokenizer.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ import transformers
2
+
3
+ class CodeTokenizer(transformers.AutoTokenizer):
4
+ def __init__(self, model_name):
5
+ super().__init__.from_pretrained(model_name)