File size: 1,299 Bytes
686678a 951adef 26a2377 6a968bc 951adef 6a968bc 951adef 686678a 6a968bc 26a2377 6a968bc 26a2377 951adef 686678a 6a968bc 951adef 686678a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
import spaces
import gradio as gr
# from airllm import HuggingFaceModelLoader, AutoModelForCausalLM
model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
pipeline = transformers.pipeline(
"text-generation",
model=model_id,
model_kwargs={"torch_dtype": torch.bfloat16},
device_map="auto",
)
@spaces.GPU
def generate_text(input_text):
output = pipeline(messages,
max_new_tokens=256,
)
# input_tokens = model.tokenizer(input_text,
# return_tensors="np",
# return_attention_mask=False,
# truncation=True,
# max_length=MAX_LENGTH,
# padding=False)
# output = model.generate(mx.array(input_tokens['input_ids']),
# max_new_tokens=20,
# use_cache=True,
# return_dict_in_generate=True)
return output
iface = gr.Interface(
fn=generate_text,
inputs=gr.Textbox(placeholder="Enter prompt..."),
outputs="text",
title="LLaMA 3 8B Text Generation"
)
iface.launch(server_name="0.0.0.0", server_port=7860) |