minotaur-mpt-7b / examples /pipeline.py
winglian's picture
pipeline tools for testing
7b8728f
raw
history blame
725 Bytes
import sys
from transformers import AutoTokenizer, AutoModelForCausalLM
import transformers
import torch
model = "openaccess-ai-collective/minotaur-mpt-7b"
tokenizer = AutoTokenizer.from_pretrained(model)
pipeline = transformers.pipeline(
"text-generation",
model=model,
tokenizer=tokenizer,
torch_dtype=torch.bfloat16,
trust_remote_code=True,
device_map="auto",
)
prompt = "".join([l for l in sys.stdin]).strip()
sequences = pipeline(
prompt,
max_length=2048,
do_sample=True,
top_k=40,
top_p=0.95,
temperature=1.0,
num_beams=10,
num_return_sequences=1,
eos_token_id=tokenizer.eos_token_id,
)
for seq in sequences:
print(f"Result: {seq['generated_text']}")