File size: 2,132 Bytes
bb1d05d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
---
base_model:
- mistralai/Mistral-Nemo-Instruct-2407
---
Ctranslate2 conversion of the model located at [mistralai/Mistral-Nemo-Instruct-2407](https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407)
Conversion script with graphical user interface can be downloaded [HERE](https://github.com/BBC-Esq/Ctranslate2-Converter)
## Tested with Ctranslate 4.4.0 and Torch 2.2.2
- NOTE: Ctranslate2 will soon release version 4.5.0, which will require greater than Torch 2.2.2.
## Example Usage:
```
import os
import sys
import ctranslate2
import gc
import torch
from transformers import AutoTokenizer
system_message = "You are a helpful person who answers questions."
user_message = "Hello, how are you today? I'd like you to write me a funny poem that is a parody of Milton's Paradise Lost if you are familiar with that famous epic poem?"
model_dir = r"D:\Scripts\bench_chat\models\mistralai--Mistral-Nemo-Instruct-2407-ct2-int8"
def build_prompt_mistral_nemo():
prompt = f"""<s>
[INST]{system_message}
{user_message}[/INST]"""
return prompt
def main():
model_name = os.path.basename(model_dir)
print(f"\033[32mLoading the model: {model_name}...\033[0m")
intra_threads = max(os.cpu_count() - 4, 4)
generator = ctranslate2.Generator(
model_dir,
device="cuda",
compute_type="int8",
intra_threads=intra_threads
)
tokenizer = AutoTokenizer.from_pretrained(model_dir, add_prefix_space=None)
prompt = build_prompt_mistral_nemo()
tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt))
results_batch = generator.generate_batch(
[tokens],
include_prompt_in_result=False,
max_batch_size=4096,
batch_type="tokens",
beam_size=1,
num_hypotheses=1,
max_length=512,
sampling_temperature=0.0,
)
output = tokenizer.decode(results_batch[0].sequences_ids[0])
print("\nGenerated response:")
print(output)
del generator
del tokenizer
torch.cuda.empty_cache()
gc.collect()
if __name__ == "__main__":
main()
``` |