Spaces:
Runtime error
Runtime error
import gradio as gr | |
from transformers import AutoModelForCausalLM, AutoTokenizer | |
model_name = "brdemorin/Phi3_80_steps_v2" | |
# Update the following parameters as necessary for your GPTQ model | |
gptq_parameters = { | |
"load_in_8bit": False, | |
"device_map": "auto", | |
"trust_remote_code": True, # This is often needed for custom models | |
} | |
# Load the model with GPTQ parameters | |
model = AutoModelForCausalLM.from_pretrained(model_name, **gptq_parameters) | |
tokenizer = AutoTokenizer.from_pretrained(model_name) | |
def chat(input_text): | |
inputs = tokenizer(input_text, return_tensors="pt") | |
outputs = model.generate(**inputs, max_new_tokens=50) | |
response = tokenizer.decode(outputs[0], skip_special_tokens=True) | |
return response | |
iface = gr.Interface(fn=chat, inputs="text", outputs="text", title="Chat with Phi3_80") | |
iface.launch() | |