app.py

import gradio as gr
import pandas as pd
from css_html_js import custom_css

TITLE = """<h1 align="center" id="space-title">🇲🇾 Malaysian RAG Embedding Leaderboard</h1>"""

INTRODUCTION_TEXT = """
📐 The 🇲🇾 Malaysian RAG Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook.

## Dataset

📈 We evaluate models based on 2 datasets,

1. Research paper keyword `melayu` using Crossref, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl
2. lom.agc.gov.my PDF files, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-lom-agc.jsonl
"""

close_source = [
    {
        'model': 'OpenAI ADA-002',
        'Crossref Melayu top-1': 0.3155939351340496,
        'Crossref Melayu top-3': 0.5120996083944171,
        'Crossref Melayu top-5': 0.5878100210864544,
        'Crossref Melayu top-10': 0.6721558389396526,
        'lom.agc.gov.my top-1': 0.19168533731640527,
        'lom.agc.gov.my top-3': 0.2827981080408265,
        'lom.agc.gov.my top-5': 0.322504356484939,
        'lom.agc.gov.my top-10': 0.36855862584017923,
    }
]

open_source = [
    {
        'model': '[mesolitica/llama2-embedding-600m-8k](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)',
        'Crossref Melayu top-1': 0.09549151521237072,
        'Crossref Melayu top-3': 0.1834521538307059,
        'Crossref Melayu top-5': 0.23375840947886334,
        'Crossref Melayu top-10': 0.3098704689225826,
        'lom.agc.gov.my top-1': 0.05215334826985312,
        'lom.agc.gov.my top-3': 0.09932785660941,
        'lom.agc.gov.my top-5': 0.12969878018421707,
        'lom.agc.gov.my top-10': 0.1797361214836943,
    },
    {
        'model': '[mesolitica/llama2-embedding-1b-8k](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)',
        'Crossref Melayu top-1': 0.06777788934631991,
        'Crossref Melayu top-3': 0.142584596847073,
        'Crossref Melayu top-5': 0.18817150316296816,
        'Crossref Melayu top-10': 0.25715433276433375,
        'lom.agc.gov.my top-1': 0.06870799103808813,
        'lom.agc.gov.my top-3': 0.1343042071197411,
        'lom.agc.gov.my top-5': 0.1717699775952203,
        'lom.agc.gov.my top-10': 0.23089370176748816,
    },
    {
        'model': '[mesolitica/mistral-embedding-191m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-191m-8k-contrastive)',
        'Crossref Melayu top-1': 0.08001654088700506,
        'Crossref Melayu top-3': 0.17378269409697095,
        'Crossref Melayu top-5': 0.232192701333609,
        'Crossref Melayu top-10': 0.32482166856197664,
        'lom.agc.gov.my top-1': 0.041075429424943986,
        'lom.agc.gov.my top-3': 0.09148618371919343,
        'lom.agc.gov.my top-5': 0.12758277321384118,
        'lom.agc.gov.my top-10': 0.18707991038088126,
    },
    {
        'model': '[mesolitica/mistral-embedding-349m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-349m-8k-contrastive)',
        'Crossref Melayu top-1': 0.09045797580895276,
        'Crossref Melayu top-3': 0.18742892587615012,
        'Crossref Melayu top-5': 0.2444949860436266,
        'Crossref Melayu top-10': 0.3398118474103174,
        'lom.agc.gov.my top-1': 0.039581777445855115,
        'lom.agc.gov.my top-3': 0.08849887976101568,
        'lom.agc.gov.my top-5': 0.12335075927308937,
        'lom.agc.gov.my top-10': 0.18558625840179238,
    },
    {
        'model': '[mesolitica/llama2-embedding-600m-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)',
        'Crossref Melayu top-1': 0.1260208828698439,
        'Crossref Melayu top-3': 0.2649643337123953,
        'Crossref Melayu top-5': 0.3535614597332782,
        'Crossref Melayu top-10': 0.4751369792205107,
        'lom.agc.gov.my top-1': 0.0660941000746826,
        'lom.agc.gov.my top-3': 0.13754045307443366,
        'lom.agc.gov.my top-5': 0.18869803335822755,
        'lom.agc.gov.my top-10': 0.2745830221558377,
    },
    {
        'model': '[mesolitica/llama2-embedding-1b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)',
        'Crossref Melayu top-1': 0.21958027499224647,
        'Crossref Melayu top-3': 0.42809883179985525,
        'Crossref Melayu top-5': 0.5379923498397602,
        'Crossref Melayu top-10': 0.6708363485991936,
        'lom.agc.gov.my top-1': 0.11899427433408016,
        'lom.agc.gov.my top-3': 0.239731142643764,
        'lom.agc.gov.my top-5': 0.3094349016679114,
        'lom.agc.gov.my top-10': 0.4134926562111028,
    },
    {
        'model': '[mesolitica/llama2-embedding-2b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-2b-8k)',
        'Crossref Melayu top-1': 0.2603122092422206,
        'Crossref Melayu top-3': 0.4910575829628864,
        'Crossref Melayu top-5': 0.6041558978600228,
        'Crossref Melayu top-10': 0.7295564974671767,
        'lom.agc.gov.my top-1': 0.15011202389843167,
        'lom.agc.gov.my top-3': 0.2803086880756784,
        'lom.agc.gov.my top-5': 0.35760517799352753,
        'lom.agc.gov.my top-10': 0.466268359472243,
    },
]

data = pd.DataFrame(close_source + open_source)

demo = gr.Blocks(css=custom_css)
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
    gr.DataFrame(data, datatype = 'markdown')

demo.launch()