app.py

import gradio as gr
import pandas as pd
import random
import time 

from info.train_a_model import (
    LLM_BENCHMARKS_TEXT)
from info.submit import (
    SUBMIT_TEXT)
from info.deployment import (
    DEPLOY_TEXT)
from info.programs import (
    PROGRAMS_TEXT)
from info.citation import(
    CITATION_TEXT)
from src.processing import filter_benchmarks_table, make_clickable

demo = gr.Blocks()

with demo:
    
    gr.HTML("""<h1 align="center" id="space-title">🤗Powered-by-Intel LLM Leaderboard 💻</h1>""")
    gr.Markdown("This leaderboard is designed to evaluate, score, and rank open-source large language \
        models that have been pre-trained or fine-tuned on Intel Hardware 🦾")
    gr.Markdown("Models submitted to the leaderboard are evaluated \
        on the Intel Developer Cloud ☁️")
    
    # TODO: Coming soon comparison tool
    #with gr.Accordion("🥊Large Language Model Boxing Ring 🥊", open=False):
    #    with gr.Row():
    #        chat_a = gr.Chatbot()
    #        chat_b = gr.Chatbot()
    #    msg = gr.Textbox()
    #    gr.ClearButton([msg, chat_a])
#
    #    def respond(message, chat_history):
    #        bot_message = random.choice(["How are you?", "I love you", "I'm very hungry"])
    #        chat_history.append((message, bot_message))
    #        time.sleep(2)
    #        return "", chat_history
#
    #    msg.submit(respond, inputs = [msg, chat_a],outputs = [msg, chat_a]) 
        
    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏆 LLM Benchmark", elem_id="llm-benchmark-table", id=0):
            with gr.Row():
                with gr.Column():
                    filter_hw = gr.CheckboxGroup(choices=["Gaudi","Xeon","GPU Max","Arc GPU","Core Ultra"],
                                     label="Select Training Platform*",
                                     elem_id="compute_platforms",
                                     value=["Gaudi","Xeon","GPU Max","Arc GPU","Core Ultra"])
                    filter_platform = gr.CheckboxGroup(choices=["Intel Developer Cloud","AWS","Azure","GCP","Local"],
                                     label="Training Infrastructure*",
                                     elem_id="training_infra",
                                     value=["Intel Developer Cloud","AWS","Azure","GCP","Local"])
                    filter_affiliation = gr.CheckboxGroup(choices=["No Affiliation","Intel Innovator","Intel Student Ambassador", "Intel Software Liftoff", "Intel Labs", "Other"],
                                     label="Intel Program Affiliation",
                                     elem_id="program_affiliation",
                                     value=["No Affiliation","Intel Innovator","Intel Student Ambassador", "Intel Software Liftoff", "Intel Labs", "Other"])
                    
                with gr.Column():
                    filter_size = gr.CheckboxGroup(choices=[1,3,5,7,13,35,60,70,100],
                                     label="Model Sizes (Billion of Parameters)",
                                     elem_id="parameter_size",
                                     value=[1,3,5,7,13,35,60,70,100])
                    filter_precision = gr.CheckboxGroup(choices=["fp8","fp16","bf16","int8","4bit"],
                                     label="Model Precision",
                                     elem_id="precision",
                                     value=["fp8","fp16","bf16","int8","4bit"])
                    filter_type = gr.CheckboxGroup(choices=["pretrained","fine-tuned","chat-models","merges/moerges"],
                                     label="Model Types",
                                     elem_id="model_types",
                                     value=["pretrained","fine-tuned","chat-models","merges/moerges"])
                    
            initial_df = pd.read_csv("leaderboard_status_030424.csv")
            
            gradio_df_display = gr.Dataframe()
            
            def update_df(hw_selected, platform_selected, affiliation_selected, size_selected, precision_selected, type_selected):
                filtered_df = filter_benchmarks_table(df=initial_df, hw_selected=hw_selected, platform_selected=platform_selected, 
                                                      affiliation_selected=affiliation_selected, size_selected=size_selected, 
                                                      precision_selected=precision_selected, type_selected=type_selected)
                return filtered_df
            
            filter_hw.change(fn=update_df, 
                             inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
                             outputs=[gradio_df_display])
            filter_platform.change(fn=update_df, 
                                inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
                                outputs=[gradio_df_display])
            filter_affiliation.change(fn=update_df, 
                                inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
                                outputs=[gradio_df_display])
            filter_size.change(fn=update_df, 
                               inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
                               outputs=[gradio_df_display])
            filter_precision.change(fn=update_df, 
                                inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
                                outputs=[gradio_df_display])
            filter_type.change(fn=update_df, 
                               inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
                               outputs=[gradio_df_display])
        
            
        with gr.TabItem("🧰 Train a Model", elem_id="getting-started", id=1):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
        with gr.TabItem("🚀 Deployment Tips", elem_id="deployment-tips", id=2):
            gr.Markdown(DEPLOY_TEXT, elem_classes="markdown-text")
        with gr.TabItem("👩‍💻 Developer Programs", elem_id="hardward-program", id=3):
            gr.Markdown(PROGRAMS_TEXT, elem_classes="markdown-text")
        with gr.TabItem("🏎️ Submit", elem_id="submit", id=4):
            gr.Markdown(SUBMIT_TEXT, elem_classes="markdown-text")
            with gr.Row():
                gr.Markdown("# Submit Model for Evaluation 🏎️", elem_classes="markdown-text")
            with gr.Row():
                with gr.Column():
                    model_name_textbox = gr.Textbox(label="Model name")
                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                    model_type = gr.Dropdown(
                        choices=["pretrained","fine-tuned","chat models","merges/moerges"],
                        label="Model type",
                        multiselect=False,
                        value="pretrained",
                        interactive=True,
                    )
                    
                    hw_type = gr.Dropdown(
                        choices=["Gaudi","Xeon","GPU Max","Arc GPU"],
                        label="Training Hardware",
                        multiselect=False,
                        value="Gaudi2",
                        interactive=True,
                    )
                    terms = gr.Checkbox(
                        label="Check if you have read and agreed to terms and conditions associated with submitting\
                            a model to the leaderboard.",
                        value=False,
                        interactive=True,
                    )
                with gr.Column():
                    precision = gr.Dropdown(
                        choices=["fp8","fp16","bf16","int8","4bit"],
                        label="Precision",
                        multiselect=False,
                        value="fp16",
                        interactive=True,
                    )
                    weight_type = gr.Dropdown(
                        choices=["Original", "Adapter", "Delta"],
                        label="Weights type",
                        multiselect=False,
                        value="Original",
                        interactive=True,
                    )
                    training_infra = gr.Dropdown(
                        choices=["IDC","AWS","Azure","GCP","Local"],
                        label="Training Infrastructure",
                        multiselect=False,
                        value="IDC",
                        interactive=True,
                    )
                    affiliation = gr.Dropdown(
                        choices=["No Affiliation","Innovator","Student Ambassador","Intel Liftoff", "Intel Labs", "Other"],
                        label="Affiliation with Intel",
                        multiselect=False,
                        value="Independent",
                        interactive=True,
                    )
                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
                
            #submit_button = gr.Button("Submit Eval")
            #submission_result = gr.Markdown()
            gr.Markdown("Community Submissions Coming soon!")
            
    with gr.Accordion("📙 Citation", open=False):
            citation =gr.Textbox(value = CITATION_TEXT,
                                 lines=6,
                                 label="Use the following to cite this content")
            
    
demo.launch()