File size: 905 Bytes
d6ea14a 505d21a e10ff17 505d21a e10ff17 505d21a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 |
---
license: apache-2.0
---
``` python
import ctranslate2
import transformers
from huggingface_hub import snapshot_download
model_dir = snapshot_download(repo_id="Praise2112/Mistral-7B-v0.1-int8-ct2")
generator = ctranslate2.Generator(model_dir, device="cuda", compute_type="int8") # GPU
# generator = ctranslate2.Generator(model_dir, device="cpu", compute_type="int8") #CPU
tokenizer = transformers.AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
# Unconditional generation.
start_tokens = [tokenizer.bos_token]
results = generator.generate_batch([start_tokens], max_length=30, sampling_topk=10)
print(tokenizer.decode(results[0].sequences_ids[0]))
# Conditional generation.
start_tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode("It is"))
results = generator.generate_batch([start_tokens], max_length=30, sampling_topk=10)
print(tokenizer.decode(results[0].sequences_ids[0]))
``` |