Spaces:
Sleeping
Sleeping
File size: 3,406 Bytes
34e2eaa 80d1253 34e2eaa 80d1253 34e2eaa 6e234f4 34e2eaa 80d1253 34e2eaa 80d1253 34e2eaa 49d2c83 34e2eaa 49d2c83 34e2eaa 80d1253 34e2eaa 80d1253 34e2eaa |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 |
import os
import gradio as gr
import copy
import time
import llama_cpp
from llama_cpp import Llama
from huggingface_hub import hf_hub_download
saiga = Llama(
model_path=hf_hub_download(
repo_id="FinancialSupport/saiga-7b-gguf",
filename="saiga-7b.Q4_K_M.gguf",
),
n_ctx=4086,
)
dante = Llama(
model_path=hf_hub_download(
repo_id="FinancialSupport/saiga-7b-gguf",
filename="saigaDante-7b.Q4_K_M.gguf",
),
n_ctx=4086,
)
karg = {
'input_prompt': input_prompt,
'temperature': 0.15,
'top_p': 0.1,
'top_k': 40,
'repeat_penalty': 1.1,
'max_tokens': 1024,
'stop': [
"[|Umano|]",
"[|Assistente|]",
],
'stream': True
}
history = []
def generate_text(message, history):
temp = ""
input_prompt = "Conversazione tra umano ed un assistente AI di nome saiga-7b\n"
for interaction in history:
input_prompt += "[|Umano|] " + interaction[0] + "\n"
input_prompt += "[|Assistente|]" + interaction[1]
input_prompt += "[|Umano|] " + message + "\n[|Assistente|]"
print(input_prompt)
output = saiga(**karg)
for out in output:
stream = copy.deepcopy(out)
temp += stream["choices"][0]["text"]
yield temp
history = ["init", input_prompt]
def generate_text_Dante(message, history):
temp = ""
input_prompt = "Conversazione tra umano ed un assistente AI di nome saiga-7b\n"
for interaction in history:
input_prompt += "[|Umano|] " + interaction[0] + "\n"
input_prompt += "[|Assistente|]" + interaction[1]
input_prompt += "[|Umano|] " + message + "\n[|Assistente|]"
print(input_prompt)
output = dante(**karg)
for out in output:
stream = copy.deepcopy(out)
temp += stream["choices"][0]["text"]
yield temp
history = ["init", input_prompt]
with gr.Blocks() as demo:
with gr.Tab('saiga'):
gr.ChatInterface(
generate_text,
title="saiga-7b running on CPU (quantized Q4_K)",
description="This is a quantized version of saiga-7b running on CPU (very slow). It is less powerful than the original version, but it can even run on the free tier of huggingface.",
examples=[
"Dammi 3 idee di ricette che posso fare con i pistacchi",
"Prepara un piano di esercizi da poter fare a casa",
"Scrivi una poesia sulla nuova AI chiamata cerbero-7b"
],
cache_examples=True,
retry_btn=None,
undo_btn="Delete Previous",
clear_btn="Clear",
)
with gr.Tab('Dante'):
gr.ChatInterface(
generate_text_Dante,
title="saigaDante-7b running on CPU (quantized Q4_K)",
description="This is a quantized version of saiga-7b with Dante LoRA attached running on CPU (very slow).",
examples=[
"Traduci in volgare fiorentino: tanto va la gatta al lardo che ci lascia lo zampino", #se trovi un esempio di traduzione valido mettilo!
"Traduci in volgare fiorentino: come preparo la pasta alla carbonara?",
"Traduci in volgare fiorentino: raccontami una fiaba su Firenze"
],
cache_examples=False,
retry_btn=None,
undo_btn="Delete Previous",
clear_btn="Clear",
)
demo.queue(concurrency_count=1, max_size=5)
demo.launch() |