| license: mit | |
| tags: | |
| - tokenizers | |
| - transformers.js | |
| - experimental | |
| - auto tokenizer | |
| - llama | |
| - llama 2 | |
| - custom code | |
| library_name: transformers | |
| # Experimental tokenizer | |
| - Derived from the LLaMa-2 tokenizer | |
| - Inspired by the ChatML & Zephyr tokenizers |
| license: mit | |
| tags: | |
| - tokenizers | |
| - transformers.js | |
| - experimental | |
| - auto tokenizer | |
| - llama | |
| - llama 2 | |
| - custom code | |
| library_name: transformers | |
| # Experimental tokenizer | |
| - Derived from the LLaMa-2 tokenizer | |
| - Inspired by the ChatML & Zephyr tokenizers |