ai2-adapt-dev
/

olmo-7b-peteish7-step928646

Text Generation

Inference Endpoints

Model card Files Files and versions Community

jacobmorrison commited on Aug 22

Commit

7b1a33d

•

1 Parent(s): 14297b9

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +2 -2

tokenizer_config.json CHANGED Viewed

@@ -230,9 +230,9 @@
   },
   "bos_token": null,
   "clean_up_tokenization_spaces": true,
-  "eos_token": "",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "",
   "tokenizer_class": "GPTNeoXTokenizer",
   "unk_token": null
 }

   },
   "bos_token": null,
   "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
   "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
   "tokenizer_class": "GPTNeoXTokenizer",
   "unk_token": null
 }