turing-motors
/

heron-chat-git-ja-stablelm-base-7b-v1

git_japanese_stablelm_alpha

text-generation

image-captioning

Model card Files Files and versions Community

Inoichan commited on May 2

Commit

7093f08

•

1 Parent(s): fa34885

Update README.md

Fix a tokenizer failure

Files changed (1) hide show

README.md +7 -0

README.md CHANGED Viewed

@@ -41,6 +41,13 @@ model.to(device)
 # prepare a processor
 processor = AutoProcessor.from_pretrained(MODEL_NAME)
 import requests
 from PIL import Image

 # prepare a processor
 processor = AutoProcessor.from_pretrained(MODEL_NAME)
+tokenizer = LlamaTokenizer.from_pretrained(
+    "novelai/nerdstash-tokenizer-v1",
+    padding_side="right",
+    additional_special_tokens=["▁▁"],
+)
+processor.tokenizer = tokenizer
 import requests
 from PIL import Image