Upload 5 files

app.py

@@ -0,0 +1,108 @@
+import os
+import torch
+from transformers import AutoTokenizer, AutoConfig
+from optimum.intel.openvino import OVModelForCausalLM
+import openvino as ov
+import gradio as gr
+from gradio_helper import make_demo
+from llm_config import SUPPORTED_LLM_MODELS
+from pathlib import Path
+
+# Define model configuration
+model_language = "en"  # Example: set to English
+model_id = "qwen2.5-0.5b-instruct"  # Example model ID
+
+# Define model directories
+pt_model_id = SUPPORTED_LLM_MODELS[model_language][model_id]["model_id"]
+int4_model_dir = Path(model_id) / "INT4_compressed_weights"
+
+# Load tokenizer
+tok = AutoTokenizer.from_pretrained(int4_model_dir, trust_remote_code=True)
+
+# Ensure INT4 weights exist; if not, attempt conversion (locally before deployment)
+def check_and_convert_model():
+    if not (int4_model_dir / "openvino_model.xml").exists():
+        print("INT4 model weights not found. Attempting compression...")
+        convert_to_int4()
+
+def convert_to_int4():
+    """
+    Converts a model to INT4 precision using the optimum-cli tool.
+    This function should only be run locally or in an environment that supports shell commands.
+    """
+    # Define compression parameters
+    compression_configs = {
+        "qwen2.5-0.5b-instruct": {"sym": True, "group_size": 128, "ratio": 1.0},
+        "default": {"sym": False, "group_size": 128, "ratio": 0.8},
+    }
+
+    model_compression_params = compression_configs.get(model_id, compression_configs["default"])
+
+    # Check if the INT4 model already exists
+    if (int4_model_dir / "openvino_model.xml").exists():
+        print("INT4 model already exists.")
+        return  # Exit if the model is already converted
+
+    # Run model compression using `optimum-cli`
+    export_command_base = f"optimum-cli export openvino --model {pt_model_id} --task text-generation-with-past --weight-format int4"
+    int4_compression_args = f" --group-size {model_compression_params['group_size']} --ratio {model_compression_params['ratio']}"
+    if model_compression_params["sym"]:
+        int4_compression_args += " --sym"
+    
+    # You can add other custom compression arguments here (like AWQ)
+    export_command = export_command_base + int4_compression_args
+    print(f"Running compression command: {export_command}")
+
+    # Execute the export command (this is typically done locally, not in Hugging Face Spaces)
+    # For deployment, the model needs to be pre-compressed and uploaded
+    os.system(export_command)
+
+# Check if the INT4 model exists or needs conversion
+check_and_convert_model()
+
+# Initialize OpenVINO model
+core = ov.Core()
+ov_model = OVModelForCausalLM.from_pretrained(
+    str(int4_model_dir),
+    device="CPU",  # Adjust device as needed (e.g., "GPU" or "CPU")
+    config=AutoConfig.from_pretrained(str(int4_model_dir), trust_remote_code=True),
+    trust_remote_code=True,
+)
+
+def convert_history_to_token(history):
+    """
+    Convert the history of the conversation into tokens for the model.
+    """
+    input_ids = tok.encode(history[-1][0])  # Example tokenization
+    return torch.LongTensor([input_ids])
+
+def bot(history, temperature, top_p, top_k, repetition_penalty, conversation_id):
+    """
+    Bot logic to process conversation history and generate responses.
+    """
+    input_ids = convert_history_to_token(history)
+    streamer = TextIteratorStreamer(tok, timeout=3600.0, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        input_ids=input_ids,
+        max_new_tokens=256,
+        temperature=temperature,
+        do_sample=temperature > 0.0,
+        top_p=top_p,
+        top_k=top_k,
+        repetition_penalty=repetition_penalty,
+        streamer=streamer,
+    )
+    
+    # Generate response
+    ov_model.generate(**generate_kwargs)
+    
+    # Stream and update history with generated response
+    partial_text = ""
+    for new_text in streamer:
+        partial_text += new_text
+        history[-1][1] = partial_text
+        yield history
+
+# Gradio interface setup
+demo = make_demo(run_fn=bot, stop_fn=None, title="OpenVINO Chatbot", language="en")
+demo.launch(debug=True, share=True)

gradio_helper.py

@@ -0,0 +1,175 @@
+from typing import Callable, Literal
+import gradio as gr
+from uuid import uuid4
+
+
+chinese_examples = [
+    ["你好!"],
+    ["你是谁?"],
+    ["请介绍一下上海"],
+    ["请介绍一下英特尔公司"],
+    ["晚上睡不着怎么办？"],
+    ["给我讲一个年轻人奋斗创业最终取得成功的故事。"],
+    ["给这个故事起一个标题。"],
+]
+
+english_examples = [
+    ["Hello there! How are you doing?"],
+    ["What is OpenVINO?"],
+    ["Who are you?"],
+    ["Can you explain to me briefly what is Python programming language?"],
+    ["Explain the plot of Cinderella in a sentence."],
+    ["What are some common mistakes to avoid when writing code?"],
+    ["Write a 100-word blog post on “Benefits of Artificial Intelligence and OpenVINO“"],
+]
+
+japanese_examples = [
+    ["こんにちは！調子はどうですか?"],
+    ["OpenVINOとは何ですか?"],
+    ["あなたは誰ですか?"],
+    ["Pythonプログラミング言語とは何か簡単に説明してもらえますか?"],
+    ["シンデレラのあらすじを一文で説明してください。"],
+    ["コードを書くときに避けるべきよくある間違いは何ですか?"],
+    ["人工知能と「OpenVINOの利点」について100語程度のブログ記事を書いてください。"],
+]
+
+
+def get_uuid():
+    """
+    universal unique identifier for thread
+    """
+    return str(uuid4())
+
+
+def handle_user_message(message, history):
+    """
+    callback function for updating user messages in interface on submit button click
+
+    Params:
+      message: current message
+      history: conversation history
+    Returns:
+      None
+    """
+    # Append the user's message to the conversation history
+    return "", history + [[message, ""]]
+
+
+def make_demo(
+    run_fn: Callable,
+    stop_fn: Callable,
+    title: str = "OpenVINO Chatbot",
+    language: Literal["English", "Chinese", "Japanese"] = "English"
+):
+    # Define examples based on the selected language
+    examples = (
+        chinese_examples if language == "Chinese" 
+        else japanese_examples if language == "Japanese" 
+        else english_examples
+    )
+
+    with gr.Blocks(
+        theme=gr.themes.Soft(),
+        css=".disclaimer {font-variant-caps: all-small-caps;}"
+    ) as demo:
+        conversation_id = gr.State(get_uuid)  # Ensure get_uuid is defined elsewhere
+        gr.Markdown(f"<h1><center>{title}</center></h1>")
+        chatbot = gr.Chatbot(height=500)
+
+        # User message input
+        with gr.Row():
+            with gr.Column():
+                msg = gr.Textbox(
+                    label="Chat Message Box",
+                    placeholder="Chat Message Box",
+                    show_label=False,
+                    container=False,
+                )
+            with gr.Column():
+                submit = gr.Button("Submit")
+                stop = gr.Button("Stop")
+                clear = gr.Button("Clear")
+
+        # Advanced options for the chat
+        with gr.Row():
+            with gr.Accordion("Advanced Options:", open=False):
+                temperature = gr.Slider(
+                    label="Temperature",
+                    value=0.1,
+                    minimum=0.0,
+                    maximum=1.0,
+                    step=0.1,
+                    interactive=True,
+                    info="Higher values produce more diverse outputs",
+                )
+                top_p = gr.Slider(
+                    label="Top-p (nucleus sampling)",
+                    value=1.0,
+                    minimum=0.0,
+                    maximum=1.0,
+                    step=0.01,
+                    interactive=True,
+                    info=("Sample from the smallest possible set of tokens whose cumulative probability exceeds top_p. "
+                          "Set to 1 to disable and sample from all tokens."),
+                )
+                top_k = gr.Slider(
+                    label="Top-k",
+                    value=50,
+                    minimum=0,
+                    maximum=200,
+                    step=1,
+                    interactive=True,
+                    info="Sample from a shortlist of top-k tokens — 0 to disable and sample from all tokens.",
+                )
+                repetition_penalty = gr.Slider(
+                    label="Repetition Penalty",
+                    value=1.1,
+                    minimum=1.0,
+                    maximum=2.0,
+                    step=0.1,
+                    interactive=True,
+                    info="Penalize repetition — 1.0 to disable.",
+                )
+
+        # Example messages
+        gr.Examples(examples, inputs=msg, label="Click on any example and press the 'Submit' button")
+
+        # Submit message event
+        submit_event = msg.submit(
+            fn=handle_user_message,
+            inputs=[msg, chatbot],
+            outputs=[msg, chatbot],
+            queue=False,
+        ).then(
+            fn=run_fn,
+            inputs=[chatbot, temperature, top_p, top_k, repetition_penalty, conversation_id],
+            outputs=chatbot,
+            queue=True,
+        )
+
+        # Submit button click event
+        submit.click(
+            fn=handle_user_message,
+            inputs=[msg, chatbot],
+            outputs=[msg, chatbot],
+            queue=False,
+        ).then(
+            fn=run_fn,
+            inputs=[chatbot, temperature, top_p, top_k, repetition_penalty, conversation_id],
+            outputs=chatbot,
+            queue=True,
+        )
+
+        # Stop button functionality
+        stop.click(
+            fn=stop_fn,
+            inputs=None,
+            outputs=None,
+            cancels=[submit_event],  # Cancels the submission event
+            queue=False,
+        )
+
+        # Clear chat button functionality
+        clear.click(lambda: None, None, chatbot, queue=False)
+
+    return demo

llm_config.py

@@ -0,0 +1,785 @@
+DEFAULT_SYSTEM_PROMPT = """\
+You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+If a question does not make any sense or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\
+"""
+
+DEFAULT_SYSTEM_PROMPT_CHINESE = """\
+你是一个乐于助人、尊重他人以及诚实可靠的助手。在安全的情况下，始终尽可能有帮助地回答。 您的回答不应包含任何有害、不道德、种族主义、性别歧视、有毒、危险或非法的内容。请确保您的回答在社会上是公正的和积极的。
+如果一个问题没有任何意义或与事实不符，请解释原因，而不是回答错误的问题。如果您不知道问题的答案，请不要分享虚假信息。另外，答案请使用中文。\
+"""
+
+DEFAULT_SYSTEM_PROMPT_JAPANESE = """\
+あなたは親切で、礼儀正しく、誠実なアシスタントです。 常に安全を保ちながら、できるだけ役立つように答えてください。 回答には、有害、非倫理的、人種差別的、性差別的、有毒、危険、または違法なコンテンツを含めてはいけません。 回答は社会的に偏見がなく、本質的に前向きなものであることを確認してください。
+質問が意味をなさない場合、または事実に一貫性がない場合は、正しくないことに答えるのではなく、その理由を説明してください。 質問の答えがわからない場合は、誤った情報を共有しないでください。\
+"""
+
+DEFAULT_RAG_PROMPT = """\
+You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.\
+"""
+
+DEFAULT_RAG_PROMPT_CHINESE = """\
+基于以下已知信息，请简洁并专业地回答用户的问题。如果无法从中得到答案，请说 "根据已知信息无法回答该问题" 或 "没有提供足够的相关信息"。不允许在答案中添加编造成分。另外，答案请使用中文。\
+"""
+
+
+def red_pijama_partial_text_processor(partial_text, new_text):
+    if new_text == "<":
+        return partial_text
+
+    partial_text += new_text
+    return partial_text.split("<bot>:")[-1]
+
+
+def llama_partial_text_processor(partial_text, new_text):
+    new_text = new_text.replace("[INST]", "").replace("[/INST]", "")
+    partial_text += new_text
+    return partial_text
+
+
+def chatglm_partial_text_processor(partial_text, new_text):
+    new_text = new_text.strip()
+    new_text = new_text.replace("[[训练时间]]", "2023年")
+    partial_text += new_text
+    return partial_text
+
+
+def youri_partial_text_processor(partial_text, new_text):
+    new_text = new_text.replace("システム:", "")
+    partial_text += new_text
+    return partial_text
+
+
+def internlm_partial_text_processor(partial_text, new_text):
+    partial_text += new_text
+    return partial_text.split("<|im_end|>")[0]
+
+
+def phi_completion_to_prompt(completion):
+    return f"<|system|><|end|><|user|>{completion}<|end|><|assistant|>\n"
+
+
+def llama3_completion_to_prompt(completion):
+    return f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{completion}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+
+
+def qwen_completion_to_prompt(completion):
+    return f"<|im_start|>system\n<|im_end|>\n<|im_start|>user\n{completion}<|im_end|>\n<|im_start|>assistant\n"
+
+
+SUPPORTED_LLM_MODELS = {
+    "English": {
+        "qwen2.5-0.5b-instruct": {
+            "model_id": "Qwen/Qwen2.5-0.5B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "tiny-llama-1b-chat": {
+            "model_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+            "remote_code": False,
+            "start_message": f"<|system|>\n{DEFAULT_SYSTEM_PROMPT}</s>\n",
+            "history_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}</s> \n",
+            "current_message_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}",
+            "rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }</s>"""
+            + """
+            <|user|>
+            Question: {input} 
+            Context: {context} 
+            Answer: </s>
+            <|assistant|>""",
+        },
+        "llama-3.2-1b-instruct": {
+            "model_id": "meta-llama/Llama-3.2-1B-Instruct",
+            "start_message": DEFAULT_SYSTEM_PROMPT,
+            "stop_tokens": ["<|eot_id|>"],
+            "has_chat_template": True,
+            "start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
+            "history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
+            "current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
+            "rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
+            + """<|start_header_id|>user<|end_header_id|>
+            
+            
+            Question: {input}
+            Context: {context}
+            Answer:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+            
+            """,
+            "completion_to_prompt": llama3_completion_to_prompt,
+        },
+        "llama-3.2-3b-instruct": {
+            "model_id": "meta-llama/Llama-3.2-3B-Instruct",
+            "start_message": DEFAULT_SYSTEM_PROMPT,
+            "stop_tokens": ["<|eot_id|>"],
+            "has_chat_template": True,
+            "start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
+            "history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
+            "current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
+            "rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
+            + """<|start_header_id|>user<|end_header_id|>
+            
+            
+            Question: {input}
+            Context: {context}
+            Answer:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+            
+            """,
+            "completion_to_prompt": llama3_completion_to_prompt,
+        },
+        "qwen2.5-1.5b-instruct": {
+            "model_id": "Qwen/Qwen2.5-1.5B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "gemma-2b-it": {
+            "model_id": "google/gemma-2b-it",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT + ", ",
+            "history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
+            "current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
+            + """<start_of_turn>user{input}<end_of_turn><start_of_turn>context{context}<end_of_turn><start_of_turn>model""",
+        },
+        "gemma-2-2b-it": {
+            "model_id": "google/gemma-2-2b-it",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT + ", ",
+            "history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
+            "current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
+            + """<start_of_turn>user{input}<end_of_turn><start_of_turn>context{context}<end_of_turn><start_of_turn>model""",
+        },
+        "red-pajama-3b-chat": {
+            "model_id": "togethercomputer/RedPajama-INCITE-Chat-3B-v1",
+            "remote_code": False,
+            "start_message": "",
+            "history_template": "\n<human>:{user}\n<bot>:{assistant}",
+            "stop_tokens": [29, 0],
+            "partial_text_processor": red_pijama_partial_text_processor,
+            "current_message_template": "\n<human>:{user}\n<bot>:{assistant}",
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT }"""
+            + """
+            <human>: Question: {input} 
+            Context: {context} 
+            Answer: <bot>""",
+        },
+        "qwen2.5-3b-instruct": {
+            "model_id": "Qwen/Qwen2.5-3B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT + ", ",
+            "rag_prompt_template": f"""<|im_start|>system
+            {DEFAULT_RAG_PROMPT }<|im_end|>"""
+            + """
+            <|im_start|>user
+            Question: {input} 
+            Context: {context} 
+            Answer: <|im_end|>
+            <|im_start|>assistant
+            """,
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "qwen2.5-7b-instruct": {
+            "model_id": "Qwen/Qwen2.5-7B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT + ", ",
+            "rag_prompt_template": f"""<|im_start|>system
+            {DEFAULT_RAG_PROMPT }<|im_end|>"""
+            + """
+            <|im_start|>user
+            Question: {input} 
+            Context: {context} 
+            Answer: <|im_end|>
+            <|im_start|>assistant
+            """,
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "gemma-7b-it": {
+            "model_id": "google/gemma-7b-it",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT + ", ",
+            "history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
+            "current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
+            + """<start_of_turn>user{input}<end_of_turn><start_of_turn>context{context}<end_of_turn><start_of_turn>model""",
+        },
+        "gemma-2-9b-it": {
+            "model_id": "google/gemma-2-9b-it",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT + ", ",
+            "history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
+            "current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
+            + """<start_of_turn>user{input}<end_of_turn><start_of_turn>context{context}<end_of_turn><start_of_turn>model""",
+        },
+        "llama-2-chat-7b": {
+            "model_id": "meta-llama/Llama-2-7b-chat-hf",
+            "remote_code": False,
+            "start_message": f"<s>[INST] <<SYS>>\n{DEFAULT_SYSTEM_PROMPT }\n<</SYS>>\n\n",
+            "history_template": "{user}[/INST]{assistant}</s><s>[INST]",
+            "current_message_template": "{user} [/INST]{assistant}",
+            "tokenizer_kwargs": {"add_special_tokens": False},
+            "partial_text_processor": llama_partial_text_processor,
+            "rag_prompt_template": f"""[INST]Human: <<SYS>> {DEFAULT_RAG_PROMPT }<</SYS>>"""
+            + """
+            Question: {input} 
+            Context: {context} 
+            Answer: [/INST]""",
+        },
+        "llama-3-8b-instruct": {
+            "model_id": "meta-llama/Meta-Llama-3-8B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT,
+            "stop_tokens": ["<|eot_id|>", "<|end_of_text|>"],
+            "has_chat_template": True,
+            "start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
+            "history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
+            "current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
+            "rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
+            + """<|start_header_id|>user<|end_header_id|>
+            
+            
+            Question: {input}
+            Context: {context}
+            Answer:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+            
+            """,
+            "completion_to_prompt": llama3_completion_to_prompt,
+        },
+        "llama-3.1-8b-instruct": {
+            "model_id": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT,
+            "stop_tokens": ["<|eot_id|>", "<|end_of_text|>"],
+            "has_chat_template": True,
+            "start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
+            "history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
+            "current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
+            "rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
+            + """<|start_header_id|>user<|end_header_id|>
+            
+            
+            Question: {input}
+            Context: {context}
+            Answer:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+            
+            """,
+            "completion_to_prompt": llama3_completion_to_prompt,
+        },
+        "mistral-7b-instruct": {
+            "model_id": "mistralai/Mistral-7B-Instruct-v0.1",
+            "remote_code": False,
+            "start_message": f"<s>[INST] <<SYS>>\n{DEFAULT_SYSTEM_PROMPT }\n<</SYS>>\n\n",
+            "history_template": "{user}[/INST]{assistant}</s><s>[INST]",
+            "current_message_template": "{user} [/INST]{assistant}",
+            "tokenizer_kwargs": {"add_special_tokens": False},
+            "partial_text_processor": llama_partial_text_processor,
+            "rag_prompt_template": f"""<s> [INST] {DEFAULT_RAG_PROMPT } [/INST] </s>"""
+            + """ 
+            [INST] Question: {input} 
+            Context: {context} 
+            Answer: [/INST]""",
+        },
+        "zephyr-7b-beta": {
+            "model_id": "HuggingFaceH4/zephyr-7b-beta",
+            "remote_code": False,
+            "start_message": f"<|system|>\n{DEFAULT_SYSTEM_PROMPT}</s>\n",
+            "history_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}</s> \n",
+            "current_message_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}",
+            "rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }</s>"""
+            + """ 
+            <|user|>
+            Question: {input} 
+            Context: {context} 
+            Answer: </s>
+            <|assistant|>""",
+        },
+        "notus-7b-v1": {
+            "model_id": "argilla/notus-7b-v1",
+            "remote_code": False,
+            "start_message": f"<|system|>\n{DEFAULT_SYSTEM_PROMPT}</s>\n",
+            "history_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}</s> \n",
+            "current_message_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}",
+            "rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }</s>"""
+            + """
+            <|user|>
+            Question: {input} 
+            Context: {context} 
+            Answer: </s>
+            <|assistant|>""",
+        },
+        "neural-chat-7b-v3-3": {
+            "model_id": "Intel/neural-chat-7b-v3-3",
+            "remote_code": False,
+            "start_message": f"<s>[INST] <<SYS>>\n{DEFAULT_SYSTEM_PROMPT }\n<</SYS>>\n\n",
+            "history_template": "{user}[/INST]{assistant}</s><s>[INST]",
+            "current_message_template": "{user} [/INST]{assistant}",
+            "tokenizer_kwargs": {"add_special_tokens": False},
+            "partial_text_processor": llama_partial_text_processor,
+            "rag_prompt_template": f"""<s> [INST] {DEFAULT_RAG_PROMPT } [/INST] </s>"""
+            + """
+            [INST] Question: {input} 
+            Context: {context} 
+            Answer: [/INST]""",
+        },
+        "phi-3-mini-instruct": {
+            "model_id": "microsoft/Phi-3-mini-4k-instruct",
+            "remote_code": True,
+            "start_message": "<|system|>\n{DEFAULT_SYSTEM_PROMPT}<|end|>\n",
+            "history_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}<|end|>\n",
+            "current_message_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}",
+            "stop_tokens": ["<|end|>"],
+            "rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }<|end|>"""
+            + """
+            <|user|>
+            Question: {input} 
+            Context: {context} 
+            Answer: <|end|>
+            <|assistant|>""",
+            "completion_to_prompt": phi_completion_to_prompt,
+        },
+        "phi-3.5-mini-instruct": {
+            "model_id": "microsoft/Phi-3.5-mini-instruct",
+            "remote_code": True,
+            "start_message": "<|system|>\n{DEFAULT_SYSTEM_PROMPT}<|end|>\n",
+            "history_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}<|end|>\n",
+            "current_message_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}",
+            "stop_tokens": ["<|end|>"],
+            "rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }<|end|>"""
+            + """
+            <|user|>
+            Question: {input} 
+            Context: {context} 
+            Answer: <|end|>
+            <|assistant|>""",
+            "completion_to_prompt": phi_completion_to_prompt,
+        },
+        "qwen2.5-14b-instruct": {
+            "model_id": "Qwen/Qwen2.5-14B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT + ", ",
+            "rag_prompt_template": f"""<|im_start|>system
+            {DEFAULT_RAG_PROMPT }<|im_end|>"""
+            + """
+            <|im_start|>user
+            Question: {input} 
+            Context: {context} 
+            Answer: <|im_end|>
+            <|im_start|>assistant
+            """,
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+    },
+    "Chinese": {
+        "qwen2.5-0.5b-instruct": {
+            "model_id": "Qwen/Qwen2.5-0.5B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "qwen2.5-1.5b-instruct": {
+            "model_id": "Qwen/Qwen2.5-1.5B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "qwen2.5-3b-instruct": {
+            "model_id": "Qwen/Qwen2.5-3B-Instruct",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "qwen2.5-7b-instruct": {
+            "model_id": "Qwen/Qwen2.5-7B-Instruct",
+            "remote_code": False,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "rag_prompt_template": f"""<|im_start|>system
+            {DEFAULT_RAG_PROMPT_CHINESE }<|im_end|>"""
+            + """
+            <|im_start|>user
+            问题: {input} 
+            已知内容: {context} 
+            回答: <|im_end|>
+            <|im_start|>assistant
+            """,
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "qwen2.5-14b-instruct": {
+            "model_id": "Qwen/Qwen2.5-14B-Instruct",
+            "remote_code": False,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "rag_prompt_template": f"""<|im_start|>system
+            {DEFAULT_RAG_PROMPT_CHINESE }<|im_end|>"""
+            + """
+            <|im_start|>user
+            问题: {input} 
+            已知内容: {context} 
+            回答: <|im_end|>
+            <|im_start|>assistant
+            """,
+            "completion_to_prompt": qwen_completion_to_prompt,
+        },
+        "qwen-7b-chat": {
+            "model_id": "Qwen/Qwen-7B-Chat",
+            "remote_code": True,
+            "start_message": f"<|im_start|>system\n {DEFAULT_SYSTEM_PROMPT_CHINESE }<|im_end|>",
+            "history_template": "<|im_start|>user\n{user}<im_end><|im_start|>assistant\n{assistant}<|im_end|>",
+            "current_message_template": '"<|im_start|>user\n{user}<im_end><|im_start|>assistant\n{assistant}',
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "revision": "2abd8e5777bb4ce9c8ab4be7dbbd0fe4526db78d",
+            "rag_prompt_template": f"""<|im_start|>system
+            {DEFAULT_RAG_PROMPT_CHINESE }<|im_end|>"""
+            + """
+            <|im_start|>user
+            问题: {input} 
+            已知内容: {context} 
+            回答: <|im_end|>
+            <|im_start|>assistant
+            """,
+        },
+        "chatglm3-6b": {
+            "model_id": "THUDM/chatglm3-6b",
+            "remote_code": True,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "tokenizer_kwargs": {"add_special_tokens": False},
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT_CHINESE }"""
+            + """
+            问题: {input} 
+            已知内容: {context} 
+            回答: 
+            """,
+        },
+        "glm-4-9b-chat": {
+            "model_id": "THUDM/glm-4-9b-chat",
+            "remote_code": True,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "tokenizer_kwargs": {"add_special_tokens": False},
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT_CHINESE }"""
+            + """
+            问题: {input} 
+            已知内容: {context} 
+            回答: 
+            """,
+        },
+        "baichuan2-7b-chat": {
+            "model_id": "baichuan-inc/Baichuan2-7B-Chat",
+            "remote_code": True,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "tokenizer_kwargs": {"add_special_tokens": False},
+            "stop_tokens": ["<unk>", "</s>"],
+            "rag_prompt_template": f"""{DEFAULT_RAG_PROMPT_CHINESE }"""
+            + """
+            问题: {input} 
+            已知内容: {context} 
+            回答: 
+            """,
+        },
+        "minicpm-2b-dpo": {
+            "model_id": "openbmb/MiniCPM-2B-dpo-fp16",
+            "remote_code": True,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+        },
+        "internlm2-chat-1.8b": {
+            "model_id": "internlm/internlm2-chat-1_8b",
+            "remote_code": True,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "stop_tokens": ["</s>", "<|im_end|>"],
+            "partial_text_processor": internlm_partial_text_processor,
+        },
+        "qwen1.5-1.8b-chat": {
+            "model_id": "Qwen/Qwen1.5-1.8B-Chat",
+            "remote_code": False,
+            "start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
+            "stop_tokens": ["<|im_end|>", "<|endoftext|>"],
+            "rag_prompt_template": f"""<|im_start|>system
+            {DEFAULT_RAG_PROMPT_CHINESE }<|im_end|>"""
+            + """
+            <|im_start|>user
+            问题: {input} 
+            已知内容: {context} 
+            回答: <|im_end|>
+            <|im_start|>assistant
+            """,
+        },
+    },
+    "Japanese": {
+        "youri-7b-chat": {
+            "model_id": "rinna/youri-7b-chat",
+            "remote_code": False,
+            "start_message": f"設定: {DEFAULT_SYSTEM_PROMPT_JAPANESE}\n",
+            "history_template": "ユーザー: {user}\nシステム: {assistant}\n",
+            "current_message_template": "ユーザー: {user}\nシステム: {assistant}",
+            "tokenizer_kwargs": {"add_special_tokens": False},
+            "partial_text_processor": youri_partial_text_processor,
+        },
+    },
+}
+
+SUPPORTED_EMBEDDING_MODELS = {
+    "English": {
+        "bge-small-en-v1.5": {
+            "model_id": "BAAI/bge-small-en-v1.5",
+            "mean_pooling": False,
+            "normalize_embeddings": True,
+        },
+        "bge-large-en-v1.5": {
+            "model_id": "BAAI/bge-large-en-v1.5",
+            "mean_pooling": False,
+            "normalize_embeddings": True,
+        },
+        "bge-m3": {
+            "model_id": "BAAI/bge-m3",
+            "mean_pooling": False,
+            "normalize_embeddings": True,
+        },
+    },
+    "Chinese": {
+        "bge-small-zh-v1.5": {
+            "model_id": "BAAI/bge-small-zh-v1.5",
+            "mean_pooling": False,
+            "normalize_embeddings": True,
+        },
+        "bge-large-zh-v1.5": {
+            "model_id": "BAAI/bge-large-zh-v1.5",
+            "mean_pooling": False,
+            "normalize_embeddings": True,
+        },
+        "bge-m3": {
+            "model_id": "BAAI/bge-m3",
+            "mean_pooling": False,
+            "normalize_embeddings": True,
+        },
+    },
+}
+
+
+SUPPORTED_RERANK_MODELS = {
+    "bge-reranker-v2-m3": {"model_id": "BAAI/bge-reranker-v2-m3"},
+    "bge-reranker-large": {"model_id": "BAAI/bge-reranker-large"},
+    "bge-reranker-base": {"model_id": "BAAI/bge-reranker-base"},
+}
+
+compression_configs = {
+    "zephyr-7b-beta": {
+        "sym": True,
+        "group_size": 64,
+        "ratio": 0.6,
+    },
+    "mistral-7b": {
+        "sym": True,
+        "group_size": 64,
+        "ratio": 0.6,
+    },
+    "minicpm-2b-dpo": {
+        "sym": True,
+        "group_size": 64,
+        "ratio": 0.6,
+    },
+    "gemma-2b-it": {
+        "sym": True,
+        "group_size": 64,
+        "ratio": 0.6,
+    },
+    "notus-7b-v1": {
+        "sym": True,
+        "group_size": 64,
+        "ratio": 0.6,
+    },
+    "neural-chat-7b-v3-1": {
+        "sym": True,
+        "group_size": 64,
+        "ratio": 0.6,
+    },
+    "llama-2-chat-7b": {
+        "sym": True,
+        "group_size": 128,
+        "ratio": 0.8,
+    },
+    "llama-3-8b-instruct": {
+        "sym": True,
+        "group_size": 128,
+        "ratio": 0.8,
+    },
+    "gemma-7b-it": {
+        "sym": True,
+        "group_size": 128,
+        "ratio": 0.8,
+    },
+    "chatglm2-6b": {
+        "sym": True,
+        "group_size": 128,
+        "ratio": 0.72,
+    },
+    "qwen-7b-chat": {"sym": True, "group_size": 128, "ratio": 0.6},
+    "qwen2.5-7b-instruct": {"sym": True, "group_size": 128, "ratio": 1.0},
+    "qwen2.5-3b-instruct": {"sym": True, "group_size": 128, "ratio": 1.0},
+    "qwen2.5-14b-instruct": {"sym": True, "group_size": 128, "ratio": 1.0},
+    "qwen2.5-1.5b-instruct": {"sym": True, "group_size": 128, "ratio": 1.0},
+    "qwen2.5-0.5b-instruct": {"sym": True, "group_size": 128, "ratio": 1.0},
+    "red-pajama-3b-chat": {
+        "sym": False,
+        "group_size": 128,
+        "ratio": 0.5,
+    },
+    "llama-3.2-3b-instruct": {"sym": False, "group_size": 64, "ratio": 1.0, "dataset": "wikitext2", "awq": True, "all_layers": True, "scale_estimation": True},
+    "llama-3.2-1b-instruct": {"sym": False, "group_size": 64, "ratio": 1.0, "dataset": "wikitext2", "awq": True, "all_layers": True, "scale_estimation": True},
+    "default": {
+        "sym": False,
+        "group_size": 128,
+        "ratio": 0.8,
+    },
+}
+
+
+def get_optimum_cli_command(model_id, weight_format, output_dir, compression_options=None, enable_awq=False, trust_remote_code=False):
+    base_command = "optimum-cli export openvino --model {} --task text-generation-with-past --weight-format {}"
+    command = base_command.format(model_id, weight_format)
+    if compression_options:
+        compression_args = " --group-size {} --ratio {}".format(compression_options["group_size"], compression_options["ratio"])
+        if compression_options["sym"]:
+            compression_args += " --sym"
+        if enable_awq or compression_options.get("awq", False):
+            compression_args += " --awq --dataset wikitext2 --num-samples 128"
+            if compression_options.get("scale_estimation", False):
+                compression_args += " --scale-estimation"
+        if compression_options.get("all_layers", False):
+            compression_args += " --all-layers"
+
+        command = command + compression_args
+    if trust_remote_code:
+        command += "  --trust-remote-code"
+
+    command += " {}".format(output_dir)
+    return command
+
+
+default_language = "English"
+
+SUPPORTED_OPTIMIZATIONS = ["INT4", "INT4-AWQ", "INT8", "FP16"]
+
+
+def get_llm_selection_widget(languages=list(SUPPORTED_LLM_MODELS), models=SUPPORTED_LLM_MODELS[default_language], show_preconverted_checkbox=True):
+    import ipywidgets as widgets
+
+    lang_dropdown = widgets.Dropdown(options=languages or [])
+
+    # Define dependent drop down
+
+    model_dropdown = widgets.Dropdown(options=models)
+
+    def dropdown_handler(change):
+        global default_language
+        default_language = change.new
+        # If statement checking on dropdown value and changing options of the dependent dropdown accordingly
+        model_dropdown.options = SUPPORTED_LLM_MODELS[change.new]
+
+    lang_dropdown.observe(dropdown_handler, names="value")
+    compression_dropdown = widgets.Dropdown(options=SUPPORTED_OPTIMIZATIONS)
+    preconverted_checkbox = widgets.Checkbox(value=True)
+
+    form_items = []
+
+    if languages:
+        form_items.append(widgets.Box([widgets.Label(value="Language:"), lang_dropdown]))
+    form_items.extend(
+        [
+            widgets.Box([widgets.Label(value="Model:"), model_dropdown]),
+            widgets.Box([widgets.Label(value="Compression:"), compression_dropdown]),
+        ]
+    )
+    if show_preconverted_checkbox:
+        form_items.append(widgets.Box([widgets.Label(value="Use preconverted models:"), preconverted_checkbox]))
+
+    form = widgets.Box(
+        form_items,
+        layout=widgets.Layout(
+            display="flex",
+            flex_flow="column",
+            border="solid 1px",
+            # align_items='stretch',
+            width="30%",
+            padding="1%",
+        ),
+    )
+    return form, lang_dropdown, model_dropdown, compression_dropdown, preconverted_checkbox
+
+
+def convert_tokenizer(model_id, remote_code, model_dir):
+    import openvino as ov
+    from transformers import AutoTokenizer
+    from openvino_tokenizers import convert_tokenizer
+
+    hf_tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=remote_code)
+    ov_tokenizer, ov_detokenizer = convert_tokenizer(hf_tokenizer, with_detokenizer=True)
+    ov.save_model(ov_tokenizer, model_dir / "openvino_tokenizer.xml")
+    ov.save_model(ov_detokenizer, model_dir / "openvino_detokenizer.xml")
+
+
+def convert_and_compress_model(model_id, model_config, precision, use_preconverted=True):
+    from pathlib import Path
+    from IPython.display import Markdown, display
+    import subprocess  # nosec - disable B404:import-subprocess check
+    import platform
+
+    pt_model_id = model_config["model_id"]
+    pt_model_name = model_id.split("-")[0]
+    model_subdir = precision if precision == "FP16" else precision + "_compressed_weights"
+    model_dir = Path(pt_model_name) / model_subdir
+    remote_code = model_config.get("remote_code", False)
+    if (model_dir / "openvino_model.xml").exists():
+        print(f"✅ {precision} {model_id} model already converted and can be found in {model_dir}")
+
+        if not (model_dir / "openvino_tokenizer.xml").exists() or not (model_dir / "openvino_detokenizer.xml").exists():
+            convert_tokenizer(pt_model_id, remote_code, model_dir)
+        return model_dir
+    if use_preconverted:
+        OV_ORG = "OpenVINO"
+        pt_model_name = pt_model_id.split("/")[-1]
+        ov_model_name = pt_model_name + f"-{precision.lower()}-ov"
+        ov_model_hub_id = f"{OV_ORG}/{ov_model_name}"
+        import huggingface_hub as hf_hub
+
+        hub_api = hf_hub.HfApi()
+        if hub_api.repo_exists(ov_model_hub_id):
+            print(f"⌛Found preconverted {precision} {model_id}. Downloading model started. It may takes some time.")
+            hf_hub.snapshot_download(ov_model_hub_id, local_dir=model_dir)
+            print(f"✅ {precision} {model_id} model downloaded and can be found in {model_dir}")
+            return model_dir
+
+    model_compression_params = {}
+    if "INT4" in precision:
+        model_compression_params = compression_configs.get(model_id, compression_configs["default"])
+    weight_format = precision.split("-")[0].lower()
+    optimum_cli_command = get_optimum_cli_command(pt_model_id, weight_format, model_dir, model_compression_params, "AWQ" in precision, remote_code)
+    print(f"⌛ {model_id} conversion to {precision} started. It may takes some time.")
+    display(Markdown("**Export command:**"))
+    display(Markdown(f"`{optimum_cli_command}`"))
+    subprocess.run(optimum_cli_command.split(" "), shell=(platform.system() == "Windows"), check=True)
+    print(f"✅ {precision} {model_id} model converted and can be found in {model_dir}")
+    return model_dir
+
+
+def compare_model_size(model_dir):
+    fp16_weights = model_dir.parent / "FP16" / "openvino_model.bin"
+    int8_weights = model_dir.parent / "INT8_compressed_weights" / "openvino_model.bin"
+    int4_weights = model_dir.parent / "INT4_compressed_weights" / "openvino_model.bin"
+    int4_awq_weights = model_dir.parent / "INT4-AWQ_compressed_weights" / "openvino_model.bin"
+
+    if fp16_weights.exists():
+        print(f"Size of FP16 model is {fp16_weights.stat().st_size / 1024 / 1024:.2f} MB")
+    for precision, compressed_weights in zip(["INT8", "INT4", "INT4-AWQ"], [int8_weights, int4_weights, int4_awq_weights]):
+        if compressed_weights.exists():
+            print(f"Size of model with {precision} compressed weights is {compressed_weights.stat().st_size / 1024 / 1024:.2f} MB")
+        if compressed_weights.exists() and fp16_weights.exists():
+            print(f"Compression rate for {precision} model: {fp16_weights.stat().st_size / compressed_weights.stat().st_size:.3f}")

notebook_utils.py

@@ -0,0 +1,715 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[ ]:
+
+
+import os
+import platform
+import sys
+import threading
+import time
+import urllib.parse
+from os import PathLike
+from pathlib import Path
+from typing import List, NamedTuple, Optional, Tuple
+
+import numpy as np
+from openvino.runtime import Core, Type, get_version
+from IPython.display import HTML, Image, display
+
+import openvino as ov
+from openvino.runtime.passes import Manager, MatcherPass, WrapType, Matcher
+from openvino.runtime import opset10 as ops
+
+
+# ## Files
+#
+# Load an image, download a file, download an IR model, and create a progress bar to show download progress.
+
+# In[ ]:
+
+
+def device_widget(default="AUTO", exclude=None, added=None):
+    import openvino as ov
+    import ipywidgets as widgets
+
+    core = ov.Core()
+
+    supported_devices = core.available_devices + ["AUTO"]
+    exclude = exclude or []
+    if exclude:
+        for ex_device in exclude:
+            if ex_device in supported_devices:
+                supported_devices.remove(ex_device)
+
+    added = added or []
+    if added:
+        for add_device in added:
+            if add_device not in supported_devices:
+                supported_devices.append(add_device)
+
+    device = widgets.Dropdown(
+        options=supported_devices,
+        value=default,
+        description="Device:",
+        disabled=False,
+    )
+    return device
+
+
+def quantization_widget(default=True):
+    import ipywidgets as widgets
+
+    to_quantize = widgets.Checkbox(
+        value=default,
+        description="Quantization",
+        disabled=False,
+    )
+
+    return to_quantize
+
+
+def pip_install(*args):
+    import subprocess  # nosec - disable B404:import-subprocess check
+
+    cli_args = []
+    for arg in args:
+        cli_args.extend(str(arg).split(" "))
+    subprocess.run([sys.executable, "-m", "pip", "install", *cli_args], shell=(platform.system() == "Windows"), check=True)
+
+
+def load_image(path: str) -> np.ndarray:
+    """
+    Loads an image from `path` and returns it as BGR numpy array. `path`
+    should point to an image file, either a local filename or a url. The image is
+    not stored to the filesystem. Use the `download_file` function to download and
+    store an image.
+
+    :param path: Local path name or URL to image.
+    :return: image as BGR numpy array
+    """
+    import cv2
+    import requests
+
+    if path.startswith("http"):
+        # Set User-Agent to Mozilla because some websites block
+        # requests with User-Agent Python
+        response = requests.get(path, headers={"User-Agent": "Mozilla/5.0"})
+        array = np.asarray(bytearray(response.content), dtype="uint8")
+        image = cv2.imdecode(array, -1)  # Loads the image as BGR
+    else:
+        image = cv2.imread(path)
+    return image
+
+
+def download_file(
+    url: PathLike,
+    filename: PathLike = None,
+    directory: PathLike = None,
+    show_progress: bool = True,
+    silent: bool = False,
+    timeout: int = 10,
+) -> PathLike:
+    """
+    Download a file from a url and save it to the local filesystem. The file is saved to the
+    current directory by default, or to `directory` if specified. If a filename is not given,
+    the filename of the URL will be used.
+
+    :param url: URL that points to the file to download
+    :param filename: Name of the local file to save. Should point to the name of the file only,
+                     not the full path. If None the filename from the url will be used
+    :param directory: Directory to save the file to. Will be created if it doesn't exist
+                      If None the file will be saved to the current working directory
+    :param show_progress: If True, show an TQDM ProgressBar
+    :param silent: If True, do not print a message if the file already exists
+    :param timeout: Number of seconds before cancelling the connection attempt
+    :return: path to downloaded file
+    """
+    from tqdm.notebook import tqdm_notebook
+    import requests
+
+    filename = filename or Path(urllib.parse.urlparse(url).path).name
+    chunk_size = 16384  # make chunks bigger so that not too many updates are triggered for Jupyter front-end
+
+    filename = Path(filename)
+    if len(filename.parts) > 1:
+        raise ValueError(
+            "`filename` should refer to the name of the file, excluding the directory. "
+            "Use the `directory` parameter to specify a target directory for the downloaded file."
+        )
+
+    # create the directory if it does not exist, and add the directory to the filename
+    if directory is not None:
+        directory = Path(directory)
+        directory.mkdir(parents=True, exist_ok=True)
+        filename = directory / Path(filename)
+
+    try:
+        response = requests.get(url=url, headers={"User-agent": "Mozilla/5.0"}, stream=True)
+        response.raise_for_status()
+    except (
+        requests.exceptions.HTTPError
+    ) as error:  # For error associated with not-200 codes. Will output something like: "404 Client Error: Not Found for url: {url}"
+        raise Exception(error) from None
+    except requests.exceptions.Timeout:
+        raise Exception(
+            "Connection timed out. If you access the internet through a proxy server, please "
+            "make sure the proxy is set in the shell from where you launched Jupyter."
+        ) from None
+    except requests.exceptions.RequestException as error:
+        raise Exception(f"File downloading failed with error: {error}") from None
+
+    # download the file if it does not exist, or if it exists with an incorrect file size
+    filesize = int(response.headers.get("Content-length", 0))
+    if not filename.exists() or (os.stat(filename).st_size != filesize):
+        with tqdm_notebook(
+            total=filesize,
+            unit="B",
+            unit_scale=True,
+            unit_divisor=1024,
+            desc=str(filename),
+            disable=not show_progress,
+        ) as progress_bar:
+            with open(filename, "wb") as file_object:
+                for chunk in response.iter_content(chunk_size):
+                    file_object.write(chunk)
+                    progress_bar.update(len(chunk))
+                    progress_bar.refresh()
+    else:
+        if not silent:
+            print(f"'{filename}' already exists.")
+
+    response.close()
+
+    return filename.resolve()
+
+
+def download_ir_model(model_xml_url: str, destination_folder: PathLike = None) -> PathLike:
+    """
+    Download IR model from `model_xml_url`. Downloads model xml and bin file; the weights file is
+    assumed to exist at the same location and name as model_xml_url with a ".bin" extension.
+
+    :param model_xml_url: URL to model xml file to download
+    :param destination_folder: Directory where downloaded model xml and bin are saved. If None, model
+                               files are saved to the current directory
+    :return: path to downloaded xml model file
+    """
+    model_bin_url = model_xml_url[:-4] + ".bin"
+    model_xml_path = download_file(model_xml_url, directory=destination_folder, show_progress=False)
+    download_file(model_bin_url, directory=destination_folder)
+    return model_xml_path
+
+
+# ## Images
+
+# ### Convert Pixel Data
+#
+# Normalize image pixel values between 0 and 1, and convert images to RGB and BGR.
+
+# In[ ]:
+
+
+def normalize_minmax(data):
+    """
+    Normalizes the values in `data` between 0 and 1
+    """
+    if data.max() == data.min():
+        raise ValueError("Normalization is not possible because all elements of" f"`data` have the same value: {data.max()}.")
+    return (data - data.min()) / (data.max() - data.min())
+
+
+def to_rgb(image_data: np.ndarray) -> np.ndarray:
+    """
+    Convert image_data from BGR to RGB
+    """
+    import cv2
+
+    return cv2.cvtColor(image_data, cv2.COLOR_BGR2RGB)
+
+
+def to_bgr(image_data: np.ndarray) -> np.ndarray:
+    """
+    Convert image_data from RGB to BGR
+    """
+    import cv2
+
+    return cv2.cvtColor(image_data, cv2.COLOR_RGB2BGR)
+
+
+# ## Videos
+
+# ### Video Player
+#
+# Custom video player to fulfill FPS requirements. You can set target FPS and output size, flip the video horizontally or skip first N frames.
+
+# In[ ]:
+
+
+class VideoPlayer:
+    """
+    Custom video player to fulfill FPS requirements. You can set target FPS and output size,
+    flip the video horizontally or skip first N frames.
+
+    :param source: Video source. It could be either camera device or video file.
+    :param size: Output frame size.
+    :param flip: Flip source horizontally.
+    :param fps: Target FPS.
+    :param skip_first_frames: Skip first N frames.
+    """
+
+    def __init__(self, source, size=None, flip=False, fps=None, skip_first_frames=0, width=1280, height=720):
+        import cv2
+
+        self.cv2 = cv2  # This is done to access the package in class methods
+        self.__cap = cv2.VideoCapture(source)
+        # try HD by default to get better video quality
+        self.__cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
+        self.__cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
+
+        if not self.__cap.isOpened():
+            raise RuntimeError(f"Cannot open {'camera' if isinstance(source, int) else ''} {source}")
+        # skip first N frames
+        self.__cap.set(cv2.CAP_PROP_POS_FRAMES, skip_first_frames)
+        # fps of input file
+        self.__input_fps = self.__cap.get(cv2.CAP_PROP_FPS)
+        if self.__input_fps <= 0:
+            self.__input_fps = 60
+        # target fps given by user
+        self.__output_fps = fps if fps is not None else self.__input_fps
+        self.__flip = flip
+        self.__size = None
+        self.__interpolation = None
+        if size is not None:
+            self.__size = size
+            # AREA better for shrinking, LINEAR better for enlarging
+            self.__interpolation = cv2.INTER_AREA if size[0] < self.__cap.get(cv2.CAP_PROP_FRAME_WIDTH) else cv2.INTER_LINEAR
+        # first frame
+        _, self.__frame = self.__cap.read()
+        self.__lock = threading.Lock()
+        self.__thread = None
+        self.__stop = False
+
+    """
+    Start playing.
+    """
+
+    def start(self):
+        self.__stop = False
+        self.__thread = threading.Thread(target=self.__run, daemon=True)
+        self.__thread.start()
+
+    """
+    Stop playing and release resources.
+    """
+
+    def stop(self):
+        self.__stop = True
+        if self.__thread is not None:
+            self.__thread.join()
+        self.__cap.release()
+
+    def __run(self):
+        prev_time = 0
+        while not self.__stop:
+            t1 = time.time()
+            ret, frame = self.__cap.read()
+            if not ret:
+                break
+
+            # fulfill target fps
+            if 1 / self.__output_fps < time.time() - prev_time:
+                prev_time = time.time()
+                # replace by current frame
+                with self.__lock:
+                    self.__frame = frame
+
+            t2 = time.time()
+            # time to wait [s] to fulfill input fps
+            wait_time = 1 / self.__input_fps - (t2 - t1)
+            # wait until
+            time.sleep(max(0, wait_time))
+
+        self.__frame = None
+
+    """
+    Get current frame.
+    """
+
+    def next(self):
+        import cv2
+
+        with self.__lock:
+            if self.__frame is None:
+                return None
+            # need to copy frame, because can be cached and reused if fps is low
+            frame = self.__frame.copy()
+        if self.__size is not None:
+            frame = self.cv2.resize(frame, self.__size, interpolation=self.__interpolation)
+        if self.__flip:
+            frame = self.cv2.flip(frame, 1)
+        return frame
+
+
+# ## Visualization
+
+# ### Segmentation
+#
+# Define a SegmentationMap NamedTuple that keeps the labels and colormap for a segmentation project/dataset. Create CityScapesSegmentation and BinarySegmentation SegmentationMaps. Create a function to convert a segmentation map to an RGB image with a colormap, and to show the segmentation result as an overlay over the original image.
+
+# In[ ]:
+
+
+class Label(NamedTuple):
+    index: int
+    color: Tuple
+    name: Optional[str] = None
+
+
+# In[ ]:
+
+
+class SegmentationMap(NamedTuple):
+    labels: List
+
+    def get_colormap(self):
+        return np.array([label.color for label in self.labels])
+
+    def get_labels(self):
+        labelnames = [label.name for label in self.labels]
+        if any(labelnames):
+            return labelnames
+        else:
+            return None
+
+
+# In[ ]:
+
+
+cityscape_labels = [
+    Label(index=0, color=(128, 64, 128), name="road"),
+    Label(index=1, color=(244, 35, 232), name="sidewalk"),
+    Label(index=2, color=(70, 70, 70), name="building"),
+    Label(index=3, color=(102, 102, 156), name="wall"),
+    Label(index=4, color=(190, 153, 153), name="fence"),
+    Label(index=5, color=(153, 153, 153), name="pole"),
+    Label(index=6, color=(250, 170, 30), name="traffic light"),
+    Label(index=7, color=(220, 220, 0), name="traffic sign"),
+    Label(index=8, color=(107, 142, 35), name="vegetation"),
+    Label(index=9, color=(152, 251, 152), name="terrain"),
+    Label(index=10, color=(70, 130, 180), name="sky"),
+    Label(index=11, color=(220, 20, 60), name="person"),
+    Label(index=12, color=(255, 0, 0), name="rider"),
+    Label(index=13, color=(0, 0, 142), name="car"),
+    Label(index=14, color=(0, 0, 70), name="truck"),
+    Label(index=15, color=(0, 60, 100), name="bus"),
+    Label(index=16, color=(0, 80, 100), name="train"),
+    Label(index=17, color=(0, 0, 230), name="motorcycle"),
+    Label(index=18, color=(119, 11, 32), name="bicycle"),
+    Label(index=19, color=(255, 255, 255), name="background"),
+]
+
+CityScapesSegmentation = SegmentationMap(cityscape_labels)
+
+binary_labels = [
+    Label(index=0, color=(255, 255, 255), name="background"),
+    Label(index=1, color=(0, 0, 0), name="foreground"),
+]
+
+BinarySegmentation = SegmentationMap(binary_labels)
+
+
+# In[ ]:
+
+
+def segmentation_map_to_image(result: np.ndarray, colormap: np.ndarray, remove_holes: bool = False) -> np.ndarray:
+    """
+    Convert network result of floating point numbers to an RGB image with
+    integer values from 0-255 by applying a colormap.
+
+    :param result: A single network result after converting to pixel values in H,W or 1,H,W shape.
+    :param colormap: A numpy array of shape (num_classes, 3) with an RGB value per class.
+    :param remove_holes: If True, remove holes in the segmentation result.
+    :return: An RGB image where each pixel is an int8 value according to colormap.
+    """
+    import cv2
+
+    if len(result.shape) != 2 and result.shape[0] != 1:
+        raise ValueError(f"Expected result with shape (H,W) or (1,H,W), got result with shape {result.shape}")
+
+    if len(np.unique(result)) > colormap.shape[0]:
+        raise ValueError(
+            f"Expected max {colormap[0]} classes in result, got {len(np.unique(result))} "
+            "different output values. Please make sure to convert the network output to "
+            "pixel values before calling this function."
+        )
+    elif result.shape[0] == 1:
+        result = result.squeeze(0)
+
+    result = result.astype(np.uint8)
+
+    contour_mode = cv2.RETR_EXTERNAL if remove_holes else cv2.RETR_TREE
+    mask = np.zeros((result.shape[0], result.shape[1], 3), dtype=np.uint8)
+    for label_index, color in enumerate(colormap):
+        label_index_map = result == label_index
+        label_index_map = label_index_map.astype(np.uint8) * 255
+        contours, hierarchies = cv2.findContours(label_index_map, contour_mode, cv2.CHAIN_APPROX_SIMPLE)
+        cv2.drawContours(
+            mask,
+            contours,
+            contourIdx=-1,
+            color=color.tolist(),
+            thickness=cv2.FILLED,
+        )
+
+    return mask
+
+
+def segmentation_map_to_overlay(image, result, alpha, colormap, remove_holes=False) -> np.ndarray:
+    """
+    Returns a new image where a segmentation mask (created with colormap) is overlayed on
+    the source image.
+
+    :param image: Source image.
+    :param result: A single network result after converting to pixel values in H,W or 1,H,W shape.
+    :param alpha: Alpha transparency value for the overlay image.
+    :param colormap: A numpy array of shape (num_classes, 3) with an RGB value per class.
+    :param remove_holes: If True, remove holes in the segmentation result.
+    :return: An RGP image with segmentation mask overlayed on the source image.
+    """
+    import cv2
+
+    if len(image.shape) == 2:
+        image = np.repeat(np.expand_dims(image, -1), 3, 2)
+    mask = segmentation_map_to_image(result, colormap, remove_holes)
+    image_height, image_width = image.shape[:2]
+    mask = cv2.resize(src=mask, dsize=(image_width, image_height))
+    return cv2.addWeighted(mask, alpha, image, 1 - alpha, 0)
+
+
+# ### Network Results
+#
+# Show network result image, optionally together with the source image and a legend with labels.
+
+# In[ ]:
+
+
+def viz_result_image(
+    result_image: np.ndarray,
+    source_image: np.ndarray = None,
+    source_title: str = None,
+    result_title: str = None,
+    labels: List[Label] = None,
+    resize: bool = False,
+    bgr_to_rgb: bool = False,
+    hide_axes: bool = False,
+):
+    """
+    Show result image, optionally together with source images, and a legend with labels.
+
+    :param result_image: Numpy array of RGB result image.
+    :param source_image: Numpy array of source image. If provided this image will be shown
+                         next to the result image. source_image is expected to be in RGB format.
+                         Set bgr_to_rgb to True if source_image is in BGR format.
+    :param source_title: Title to display for the source image.
+    :param result_title: Title to display for the result image.
+    :param labels: List of labels. If provided, a legend will be shown with the given labels.
+    :param resize: If true, resize the result image to the same shape as the source image.
+    :param bgr_to_rgb: If true, convert the source image from BGR to RGB. Use this option if
+                       source_image is a BGR image.
+    :param hide_axes: If true, do not show matplotlib axes.
+    :return: Matplotlib figure with result image
+    """
+    import cv2
+    import matplotlib.pyplot as plt
+    from matplotlib.lines import Line2D
+
+    if bgr_to_rgb:
+        source_image = to_rgb(source_image)
+    if resize:
+        result_image = cv2.resize(result_image, (source_image.shape[1], source_image.shape[0]))
+
+    num_images = 1 if source_image is None else 2
+
+    fig, ax = plt.subplots(1, num_images, figsize=(16, 8), squeeze=False)
+    if source_image is not None:
+        ax[0, 0].imshow(source_image)
+        ax[0, 0].set_title(source_title)
+
+    ax[0, num_images - 1].imshow(result_image)
+    ax[0, num_images - 1].set_title(result_title)
+
+    if hide_axes:
+        for a in ax.ravel():
+            a.axis("off")
+    if labels:
+        colors = labels.get_colormap()
+        lines = [
+            Line2D(
+                [0],
+                [0],
+                color=[item / 255 for item in c.tolist()],
+                linewidth=3,
+                linestyle="-",
+            )
+            for c in colors
+        ]
+        plt.legend(
+            lines,
+            labels.get_labels(),
+            bbox_to_anchor=(1, 1),
+            loc="upper left",
+            prop={"size": 12},
+        )
+    plt.close(fig)
+    return fig
+
+
+# ### Live Inference
+
+# In[ ]:
+
+
+def show_array(frame: np.ndarray, display_handle=None):
+    """
+    Display array `frame`. Replace information at `display_handle` with `frame`
+    encoded as jpeg image. `frame` is expected to have data in BGR order.
+
+    Create a display_handle with: `display_handle = display(display_id=True)`
+    """
+    import cv2
+
+    _, frame = cv2.imencode(ext=".jpeg", img=frame)
+    if display_handle is None:
+        display_handle = display(Image(data=frame.tobytes()), display_id=True)
+    else:
+        display_handle.update(Image(data=frame.tobytes()))
+    return display_handle
+
+
+# ## Checks and Alerts
+#
+# Create an alert class to show stylized info/error/warning messages and a `check_device` function that checks whether a given device is available.
+
+# In[ ]:
+
+
+class NotebookAlert(Exception):
+    def __init__(self, message: str, alert_class: str):
+        """
+        Show an alert box with the given message.
+
+        :param message: The message to display.
+        :param alert_class: The class for styling the message. Options: info, warning, success, danger.
+        """
+        self.message = message
+        self.alert_class = alert_class
+        self.show_message()
+
+    def show_message(self):
+        display(HTML(f"""<div class="alert alert-{self.alert_class}">{self.message}"""))
+
+
+class DeviceNotFoundAlert(NotebookAlert):
+    def __init__(self, device: str):
+        """
+        Show a warning message about an unavailable device. This class does not check whether or
+        not the device is available, use the `check_device` function to check this. `check_device`
+        also shows the warning if the device is not found.
+
+        :param device: The unavailable device.
+        :return: A formatted alert box with the message that `device` is not available, and a list
+                 of devices that are available.
+        """
+        ie = Core()
+        supported_devices = ie.available_devices
+        self.message = f"Running this cell requires a {device} device, " "which is not available on this system. "
+        self.alert_class = "warning"
+        if len(supported_devices) == 1:
+            self.message += f"The following device is available: {ie.available_devices[0]}"
+        else:
+            self.message += "The following devices are available: " f"{', '.join(ie.available_devices)}"
+        super().__init__(self.message, self.alert_class)
+
+
+def check_device(device: str) -> bool:
+    """
+    Check if the specified device is available on the system.
+
+    :param device: Device to check. e.g. CPU, GPU
+    :return: True if the device is available, False if not. If the device is not available,
+             a DeviceNotFoundAlert will be shown.
+    """
+    ie = Core()
+    if device not in ie.available_devices:
+        DeviceNotFoundAlert(device)
+        return False
+    else:
+        return True
+
+
+def check_openvino_version(version: str) -> bool:
+    """
+    Check if the specified OpenVINO version is installed.
+
+    :param version: the OpenVINO version to check. Example: 2021.4
+    :return: True if the version is installed, False if not. If the version is not installed,
+             an alert message will be shown.
+    """
+    installed_version = get_version()
+    if version not in installed_version:
+        NotebookAlert(
+            f"This notebook requires OpenVINO {version}. "
+            f"The version on your system is: <i>{installed_version}</i>.<br>"
+            "Please run <span style='font-family:monospace'>pip install --upgrade -r requirements.txt</span> "
+            "in the openvino_env environment to install this version. "
+            "See the <a href='https://github.com/openvinotoolkit/openvino_notebooks'>"
+            "OpenVINO Notebooks README</a> for detailed instructions",
+            alert_class="danger",
+        )
+        return False
+    else:
+        return True
+
+
+packed_layername_tensor_dict_list = [{"name": "aten::mul/Multiply"}]
+
+
+class ReplaceTensor(MatcherPass):
+    def __init__(self, packed_layername_tensor_dict_list):
+        MatcherPass.__init__(self)
+        self.model_changed = False
+
+        param = WrapType("opset10.Multiply")
+
+        def callback(matcher: Matcher) -> bool:
+            root = matcher.get_match_root()
+            if root is None:
+                return False
+            for y in packed_layername_tensor_dict_list:
+                root_name = root.get_friendly_name()
+                if root_name.find(y["name"]) != -1:
+                    max_fp16 = np.array([[[[-np.finfo(np.float16).max]]]]).astype(np.float32)
+                    new_tenser = ops.constant(max_fp16, Type.f32, name="Constant_4431")
+                    root.set_arguments([root.input_value(0).node, new_tenser])
+                    packed_layername_tensor_dict_list.remove(y)
+
+            return True
+
+        self.register_matcher(Matcher(param, "ReplaceTensor"), callback)
+
+
+def optimize_bge_embedding(model_path, output_model_path):
+    """
+    optimize_bge_embedding used to optimize BGE model for NPU device
+
+    Arguments:
+        model_path {str} -- original BGE IR model path
+        output_model_path {str} -- Converted BGE IR model path
+    """
+    core = Core()
+    ov_model = core.read_model(model_path)
+    manager = Manager()
+    manager.register_pass(ReplaceTensor(packed_layername_tensor_dict_list))
+    manager.run_passes(ov_model)
+    ov.save_model(ov_model, output_model_path, compress_to_fp16=False)

requirements.txt

@@ -0,0 +1,14 @@
+openvino>=2024.2.0
+openvino-tokenizers[transformers]
+torch>=2.1
+datasets
+accelerate
+gradio>=4.19
+onnx<=1.16.1; sys_platform=='win32'
+einops
+transformers>=4.43.1
+transformers_stream_generator
+tiktoken
+bitsandbytes
+optimum-intel @ git+https://github.com/huggingface/optimum-intel.git
+nncf @ git+https://github.com/openvinotoolkit/nncf.git