huseinzol05's picture
Update app.py
7ac8402 verified
raw
history blame
14 kB
import gradio as gr
import pandas as pd
from css_html_js import custom_css
TITLE = """<h1 align="center" id="space-title">πŸ‡²πŸ‡Ύ Malaysian Embedding Leaderboard</h1>"""
INTRODUCTION_TEXT = """
πŸ“ The πŸ‡²πŸ‡Ύ Malaysian Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook.
## Dataset
πŸ“ˆ We evaluate models based on 6 datasets,
1. Research paper keyword `melayu` using Crossref, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl
2. lom.agc.gov.my PDF files, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-lom-agc.jsonl
3. b.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/b.cari.com.my-dataset-bge-test.jsonl
4. c.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/c.cari.com.my-dataset-bge-test.jsonl
5. Malay news, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/malay-news-dataset-bge-test.jsonl
6. Twitter, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/twitter-dataset-bge-test.jsonl
"""
close_source = [
{
'model': 'OpenAI ADA-002',
'Crossref Melayu top-1': 0.3155939351340496,
'Crossref Melayu top-3': 0.5120996083944171,
'Crossref Melayu top-5': 0.5878100210864544,
'Crossref Melayu top-10': 0.6721558389396526,
'lom.agc.gov.my top-1': 0.19168533731640527,
'lom.agc.gov.my top-3': 0.2827981080408265,
'lom.agc.gov.my top-5': 0.322504356484939,
'lom.agc.gov.my top-10': 0.36855862584017923,
'b.cari.com.my top-1': 0.31621790857858484,
'b.cari.com.my top-3': 0.6242955541640576,
'b.cari.com.my top-5': 0.6944270507201001,
'b.cari.com.my top-10': 0.7623669380087664,
'c.cari.com.my top-1': 0.08380430943129637,
'c.cari.com.my top-3': 0.21388202048746027,
'c.cari.com.my top-5': 0.27861179795125396,
'c.cari.com.my top-10': 0.3589720946661957,
'malay-news top-1': 0.1386895659334196,
'malay-news top-3': 0.2952593812492648,
'malay-news top-5': 0.3745441712739678,
'malay-news top-10': 0.4754734737089754,
'twitter top-1': 0.22942090082094518,
'twitter top-3': 0.4919014865764367,
'twitter top-5': 0.5930774351009541,
'twitter top-10': 0.7248724206789439,
}
]
open_source = [
{
'model': '[mesolitica/llama2-embedding-600m-8k](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)',
'Crossref Melayu top-1': 0.09549151521237072,
'Crossref Melayu top-3': 0.1834521538307059,
'Crossref Melayu top-5': 0.23375840947886334,
'Crossref Melayu top-10': 0.3098704689225826,
'lom.agc.gov.my top-1': 0.05215334826985312,
'lom.agc.gov.my top-3': 0.09932785660941,
'lom.agc.gov.my top-5': 0.12969878018421707,
'lom.agc.gov.my top-10': 0.1797361214836943,
'b.cari.com.my top-1': 0.3168440826549781,
'b.cari.com.my top-3': 0.6881653099561679,
'b.cari.com.my top-5': 0.7789605510331872,
'b.cari.com.my top-10': 0.8453350031308704,
'c.cari.com.my top-1': 0.08327446132108796,
'c.cari.com.my top-3': 0.18730130695867184,
'c.cari.com.my top-5': 0.23975626986930412,
'c.cari.com.my top-10': 0.3140233133168492,
'malay-news top-1': 0.14268909540054112,
'malay-news top-3': 0.27584990001176335,
'malay-news top-5': 0.3640748147276791,
'malay-news top-10': 0.47112104458299026,
'twitter top-1': 0.23496782782338585,
'twitter top-3': 0.5200798757488352,
'twitter top-5': 0.6416685156423342,
'twitter top-10': 0.785888617705791,
},
{
'model': '[mesolitica/llama2-embedding-1b-8k](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)',
'Crossref Melayu top-1': 0.06777788934631991,
'Crossref Melayu top-3': 0.142584596847073,
'Crossref Melayu top-5': 0.18817150316296816,
'Crossref Melayu top-10': 0.25715433276433375,
'lom.agc.gov.my top-1': 0.06870799103808813,
'lom.agc.gov.my top-3': 0.1343042071197411,
'lom.agc.gov.my top-5': 0.1717699775952203,
'lom.agc.gov.my top-10': 0.23089370176748816,
'b.cari.com.my top-1': 0.32623669380087666,
'b.cari.com.my top-3': 0.6947401377582968,
'b.cari.com.my top-5': 0.7902316844082655,
'b.cari.com.my top-10': 0.8603631809643081,
'c.cari.com.my top-1': 0.08689509007417874,
'c.cari.com.my top-3': 0.19966442953020133,
'c.cari.com.my top-5': 0.26086188625927237,
'c.cari.com.my top-10': 0.3430766513599435,
'malay-news top-1': 0.14610045876955652,
'malay-news top-3': 0.285260557581461,
'malay-news top-5': 0.3640748147276791,
'malay-news top-10': 0.4775908716621574,
'twitter top-1': 0.23053028622143332,
'twitter top-3': 0.5342800088750832,
'twitter top-5': 0.6498779676059463,
'twitter top-10': 0.7925449301087197,
},
{
'model': '[mesolitica/mistral-embedding-191m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-191m-8k-contrastive)',
'Crossref Melayu top-1': 0.08001654088700506,
'Crossref Melayu top-3': 0.17378269409697095,
'Crossref Melayu top-5': 0.232192701333609,
'Crossref Melayu top-10': 0.32482166856197664,
'lom.agc.gov.my top-1': 0.041075429424943986,
'lom.agc.gov.my top-3': 0.09148618371919343,
'lom.agc.gov.my top-5': 0.12758277321384118,
'lom.agc.gov.my top-10': 0.18707991038088126,
'b.cari.com.my top-1': 0.2564182842830307,
'b.cari.com.my top-3': 0.46900438321853477,
'b.cari.com.my top-5': 0.5309956167814652,
'b.cari.com.my top-10': 0.6036318096430808,
'c.cari.com.my top-1': 0.05298481102084069,
'c.cari.com.my top-3': 0.10005298481102085,
'c.cari.com.my top-5': 0.12433768986223949,
'c.cari.com.my top-10': 0.16990462734016248,
'malay-news top-1': 0.1102223267850841,
'malay-news top-3': 0.1672744383013763,
'malay-news top-5': 0.20091753911304552,
'malay-news top-10': 0.24973532525585226,
'twitter top-1': 0.18437985356112713,
'twitter top-3': 0.3536720656756157,
'twitter top-5': 0.422897714666075,
'twitter top-10': 0.5227424007100067,
},
{
'model': '[mesolitica/mistral-embedding-349m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-349m-8k-contrastive)',
'Crossref Melayu top-1': 0.09045797580895276,
'Crossref Melayu top-3': 0.18742892587615012,
'Crossref Melayu top-5': 0.2444949860436266,
'Crossref Melayu top-10': 0.3398118474103174,
'lom.agc.gov.my top-1': 0.039581777445855115,
'lom.agc.gov.my top-3': 0.08849887976101568,
'lom.agc.gov.my top-5': 0.12335075927308937,
'lom.agc.gov.my top-10': 0.18558625840179238,
'b.cari.com.my top-1': 0.24953036944270507,
'b.cari.com.my top-3': 0.4730745147150908,
'b.cari.com.my top-5': 0.5341264871634315,
'b.cari.com.my top-10': 0.5970569818409518,
'c.cari.com.my top-1': 0.057400211939244085,
'c.cari.com.my top-3': 0.10588131402331331,
'c.cari.com.my top-5': 0.13149063935005298,
'c.cari.com.my top-10': 0.17582126457082303,
'malay-news top-1': 0.11375132337372074,
'malay-news top-3': 0.17362663216092225,
'malay-news top-5': 0.20574050111751557,
'malay-news top-10': 0.26149864721797433,
'twitter top-1': 0.18105169735966276,
'twitter top-3': 0.3547814510761038,
'twitter top-5': 0.42666962502773464,
'twitter top-10': 0.5229642777901042,
},
{
'model': '[mesolitica/llama2-embedding-600m-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-600m-8k-contrastive)',
'Crossref Melayu top-1': 0.1260208828698439,
'Crossref Melayu top-3': 0.2649643337123953,
'Crossref Melayu top-5': 0.3535614597332782,
'Crossref Melayu top-10': 0.4751369792205107,
'lom.agc.gov.my top-1': 0.0660941000746826,
'lom.agc.gov.my top-3': 0.13754045307443366,
'lom.agc.gov.my top-5': 0.18869803335822755,
'lom.agc.gov.my top-10': 0.2745830221558377,
'b.cari.com.my top-1': 0.254539762053851,
'b.cari.com.my top-3': 0.4815278647463995,
'b.cari.com.my top-5': 0.5497808390732624,
'b.cari.com.my top-10': 0.628052598622417,
'c.cari.com.my top-1': 0.05183680678205581,
'c.cari.com.my top-3': 0.09766866831508302,
'c.cari.com.my top-5': 0.12398445778876722,
'c.cari.com.my top-10': 0.16566584245849522,
'malay-news top-1': 0.11657452064463004,
'malay-news top-3': 0.18986001646865075,
'malay-news top-5': 0.22338548406069875,
'malay-news top-10': 0.2812610281143395,
'twitter top-1': 0.1961393388063013,
'twitter top-3': 0.393831817173286,
'twitter top-5': 0.48080763257155534,
'twitter top-10': 0.5977368537830042,
},
{
'model': '[mesolitica/llama2-embedding-1b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-1b-8k-contrastive)',
'Crossref Melayu top-1': 0.21958027499224647,
'Crossref Melayu top-3': 0.42809883179985525,
'Crossref Melayu top-5': 0.5379923498397602,
'Crossref Melayu top-10': 0.6708363485991936,
'lom.agc.gov.my top-1': 0.11899427433408016,
'lom.agc.gov.my top-3': 0.239731142643764,
'lom.agc.gov.my top-5': 0.3094349016679114,
'lom.agc.gov.my top-10': 0.4134926562111028,
'b.cari.com.my top-1': 0.29805886036318097,
'b.cari.com.my top-3': 0.5795241077019412,
'b.cari.com.my top-5': 0.6581089542892924,
'b.cari.com.my top-10': 0.7366938008766437,
'c.cari.com.my top-1': 0.06720240197809961,
'c.cari.com.my top-3': 0.13678912045213706,
'c.cari.com.my top-5': 0.1750264924055104,
'c.cari.com.my top-10': 0.22818791946308725,
'malay-news top-1': 0.12892600870485826,
'malay-news top-3': 0.20785789907069757,
'malay-news top-5': 0.24808846018115516,
'malay-news top-10': 0.32466768615457003,
'twitter top-1': 0.2167739072553805,
'twitter top-3': 0.4453072997559352,
'twitter top-5': 0.5387175504770357,
'twitter top-10': 0.6685156423341468,
},
{
'model': '[mesolitica/llama2-embedding-2b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-2b-8k-contrastive)',
'Crossref Melayu top-1': 0.2671353251318102,
'Crossref Melayu top-3': 0.49116096350666805,
'Crossref Melayu top-5': 0.6091181639615424,
'Crossref Melayu top-10': 0.739791171301561,
'lom.agc.gov.my top-1': 0.1549663928304705,
'lom.agc.gov.my top-3': 0.29113766492407267,
'lom.agc.gov.my top-5': 0.3725416977844162,
'lom.agc.gov.my top-10': 0.4886731391585761,
'b.cari.com.my top-1': 0.30713838447088293,
'b.cari.com.my top-3': 0.6361928616155291,
'b.cari.com.my top-5': 0.7213525360050094,
'b.cari.com.my top-10': 0.8015028177833438,
'c.cari.com.my top-1': 0.07444365948428117,
'c.cari.com.my top-3': 0.15745319675026492,
'c.cari.com.my top-5': 0.19931119745672907,
'c.cari.com.my top-10': 0.26960438007771104,
'malay-news top-1': 0.13633690154099518,
'malay-news top-3': 0.23056111045759323,
'malay-news top-5': 0.28726032231502174,
'malay-news top-10': 0.3717209740030585,
'twitter top-1': 0.22498335921899268,
'twitter top-3': 0.48679831373419125,
'twitter top-5': 0.5850898602174396,
'twitter top-10': 0.7333037497226537,
},
{
'model': '[BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3)',
'Crossref Melayu top-1': 0.41331541403907784,
'Crossref Melayu top-3': 0.669802543161377,
'Crossref Melayu top-5': 0.7698749095420242,
'Crossref Melayu top-10': 0.8632275405768635,
'lom.agc.gov.my top-1': 0.35001244709982576,
'lom.agc.gov.my top-3': 0.5311177495643515,
'lom.agc.gov.my top-5': 0.6011949215832711,
'lom.agc.gov.my top-10': 0.6928055763007219,
'b.cari.com.my top-1': 0.3306199123356293,
'b.cari.com.my top-3': 0.7025673137132122,
'b.cari.com.my top-5': 0.7930494677520351,
'b.cari.com.my top-10': 0.8647463994990607,
'c.cari.com.my top-1': 0.08795478629459555,
'c.cari.com.my top-3': 0.23754856941010244,
'c.cari.com.my top-5': 0.33574708583539387,
'c.cari.com.my top-10': 0.4558459908159661,
'malay-news top-1': 0.1459828255499353,
'malay-news top-3': 0.3185507587342666,
'malay-news top-5': 0.4232443241971533,
'malay-news top-10': 0.5473473708975415,
'twitter top-1': 0.22919902374084758,
'twitter top-3': 0.5225205236299091,
'twitter top-5': 0.6443310406035057,
'twitter top-10': 0.7836698469048148,
}
]
data = pd.DataFrame(close_source + open_source)
demo = gr.Blocks(css=custom_css)
with demo:
gr.HTML(TITLE)
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
gr.DataFrame(data, datatype = 'markdown')
demo.launch()