Spaces:
Running
on
Zero
Running
on
Zero
| import os | |
| import gradio as gr | |
| import spaces | |
| from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer | |
| from threading import Thread | |
| # Set an environment variable | |
| HF_TOKEN = os.environ.get("HF_TOKEN", None) | |
| model_id = "rinna/llama-3-youko-8b-instruct" | |
| DESCRIPTION = """ | |
| <div> | |
| <p>🦊 <a href="https://huggingface.co/rinna/llama-3-youko-8b-instruct"><b>Llama 3 Youko 8B Instruct</b> (rinna/llama-3-youko-8b-instruct)</a>は、<a href="https://rinna.co.jp">rinna株式会社</a>が<a href=https://huggingface.co/meta-llama/Meta-Llama-3-8B>Meta Llama 3 8B</a>に日本語継続事前学習およびインストラクションチューニングを行った大規模言語モデルです．Llama 3 8Bの優れたパフォーマンスを日本語に引き継いでおり、日本語のチャットにおいて高い性能を示しています。</p> | |
| <p>🤖 このデモでは、Llama 3 Youko 8B Instructとチャットを行うことが可能です。</p> | |
| <p>📄 モデルの詳細については、<a href="https://rinna.co.jp/news/2024/07/20240725.html">プレスリリース</a>、および、<a href="https://rinnakk.github.io/research/benchmarks/lm/index.html">ベンチマーク</a>をご覧ください。お問い合わせは<a href="https://rinna.co.jp/inquiry/">こちら</a>までどうぞ。</p> | |
| </div> | |
| """ | |
| LICENSE = """ | |
| --- | |
| <div> | |
| <p>Built with Meta Llama 3</p> | |
| <p>License: <a href="https://llama.meta.com/llama3/license/">Meta Llama 3 Community License</a><p> | |
| <p>This space is implemented based on <a href="https://huggingface.co/spaces/ysharma/Chat_with_Meta_llama3_8b">ysharma/Chat_with_Meta_llama3_8b</a>.</p> | |
| </div> | |
| """ | |
| PLACEHOLDER = """ | |
| <div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;"> | |
| <img src="https://huggingface.co/rinna/llama-3-youko-8b/resolve/main/rinna.png" style="width: 80%; max-width: 550px; height: auto; opacity: 0.55; "> | |
| <h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">Llama 3 Youko</h1> | |
| </div> | |
| """ | |
| css = """ | |
| h1 { | |
| text-align: center; | |
| display: block; | |
| } | |
| #duplicate-button { | |
| margin: auto; | |
| color: white; | |
| background: #1565c0; | |
| border-radius: 100vh; | |
| } | |
| """ | |
| # Load the tokenizer and model | |
| tokenizer = AutoTokenizer.from_pretrained(model_id) | |
| model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto") | |
| terminators = [ | |
| tokenizer.eos_token_id, | |
| tokenizer.convert_tokens_to_ids("<|eot_id|>") | |
| ] | |
| def chat_llama3_8b(message: str, | |
| history: list, | |
| temperature: float, | |
| max_new_tokens: int | |
| ) -> str: | |
| """ | |
| Generate a streaming response using the llama3-8b model. | |
| Args: | |
| message (str): The input message. | |
| history (list): The conversation history used by ChatInterface. | |
| temperature (float): The temperature for generating the response. | |
| max_new_tokens (int): The maximum number of new tokens to generate. | |
| Returns: | |
| str: The generated response. | |
| """ | |
| conversation = [] | |
| conversation.append({"role": "system", "content": "あなたは誠実で優秀なアシスタントです。どうか、簡潔かつ正直に答えてください。"}) | |
| for user, assistant in history: | |
| conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}]) | |
| conversation.append({"role": "user", "content": message}) | |
| # Need to set add_generation_prompt=True to ensure the model generates the response. | |
| input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device) | |
| print(input_ids) | |
| print(tokenizer.decode(input_ids.tolist()[0])) | |
| streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True) | |
| generate_kwargs = dict( | |
| input_ids=input_ids, | |
| streamer=streamer, | |
| max_new_tokens=max_new_tokens, | |
| do_sample=True, | |
| temperature=temperature, | |
| repetition_penalty=1.1, | |
| eos_token_id=terminators, | |
| ) | |
| # This will enforce greedy generation (do_sample=False) when the temperature is passed 0, avoiding the crash. | |
| if temperature == 0: | |
| generate_kwargs['do_sample'] = False | |
| t = Thread(target=model.generate, kwargs=generate_kwargs) | |
| t.start() | |
| outputs = [] | |
| for text in streamer: | |
| outputs.append(text) | |
| yield "".join(outputs) | |
| # Gradio block | |
| chatbot=gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Gradio ChatInterface') | |
| with gr.Blocks(fill_height=True, css=css) as demo: | |
| gr.Markdown(DESCRIPTION) | |
| gr.ChatInterface( | |
| fn=chat_llama3_8b, | |
| chatbot=chatbot, | |
| fill_height=True, | |
| additional_inputs_accordion=gr.Accordion(label="⚙️ パラメータ", open=False, render=False), | |
| additional_inputs=[ | |
| gr.Slider(minimum=0, | |
| maximum=1, | |
| step=0.05, | |
| value=0.9, | |
| label="生成時におけるサンプリングの温度（ランダム性）", | |
| render=False), | |
| gr.Slider(minimum=128, | |
| maximum=4096, | |
| step=1, | |
| value=512, | |
| label="生成したい最大のトークン数", | |
| render=False), | |
| ], | |
| examples=[ | |
| ["日本で有名なものと言えば"], | |
| ["ネコ: 「お腹が減ったニャ」\nイヌ: 「\nで始まる物語を書いて"], | |
| ["C言語で素数を判定するコードを書いて"], | |
| ["人工知能とは何ですか"], | |
| ], | |
| cache_examples=False, | |
| ) | |
| gr.Markdown(LICENSE) | |
| if __name__ == "__main__": | |
| demo.launch() |