metadata
license: cc-by-nc-4.0
language:
- ro
base_model:
- google/gemma-2-9b-it
datasets:
- OpenLLM-Ro/ro_sft_alpaca
- OpenLLM-Ro/ro_sft_alpaca_gpt4
- OpenLLM-Ro/ro_sft_dolly
- OpenLLM-Ro/ro_sft_selfinstruct_gpt4
- OpenLLM-Ro/ro_sft_norobots
- OpenLLM-Ro/ro_sft_orca
- OpenLLM-Ro/ro_sft_camel
- OpenLLM-Ro/ro_sft_oasst
- OpenLLM-Ro/ro_sft_ultrachat
model-index:
- name: OpenLLM-Ro/RoGemma2-9b-Instruct-2024-10-09
results:
- task:
type: text-generation
dataset:
name: RoMT-Bench
type: RoMT-Bench
metrics:
- name: Score
type: Score
value: 6.08
- task:
type: text-generation
dataset:
name: RoCulturaBench
type: RoCulturaBench
metrics:
- name: Score
type: Score
value: 4.2
- task:
type: text-generation
dataset:
name: Romanian_Academic_Benchmarks
type: Romanian_Academic_Benchmarks
metrics:
- name: Average accuracy
type: accuracy
value: 57.06
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_arc_challenge
type: OpenLLM-Ro/ro_arc_challenge
metrics:
- name: Average accuracy
type: accuracy
value: 56.2
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_mmlu
type: OpenLLM-Ro/ro_mmlu
metrics:
- name: Average accuracy
type: accuracy
value: 62.98
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_winogrande
type: OpenLLM-Ro/ro_winogrande
metrics:
- name: Average accuracy
type: accuracy
value: 71
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_hellaswag
type: OpenLLM-Ro/ro_hellaswag
metrics:
- name: Average accuracy
type: accuracy
value: 60.52
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_gsm8k
type: OpenLLM-Ro/ro_gsm8k
metrics:
- name: Average accuracy
type: accuracy
value: 37.86
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_truthfulqa
type: OpenLLM-Ro/ro_truthfulqa
metrics:
- name: Average accuracy
type: accuracy
value: 53.77
- task:
type: text-generation
dataset:
name: LaRoSeDa_binary
type: LaRoSeDa_binary
metrics:
- name: Average macro-f1
type: macro-f1
value: 96.19
- task:
type: text-generation
dataset:
name: LaRoSeDa_multiclass
type: LaRoSeDa_multiclass
metrics:
- name: Average macro-f1
type: macro-f1
value: 62.49
- task:
type: text-generation
dataset:
name: LaRoSeDa_binary_finetuned
type: LaRoSeDa_binary_finetuned
metrics:
- name: Average macro-f1
type: macro-f1
value: 98.93
- task:
type: text-generation
dataset:
name: LaRoSeDa_multiclass_finetuned
type: LaRoSeDa_multiclass_finetuned
metrics:
- name: Average macro-f1
type: macro-f1
value: 88.33
- task:
type: text-generation
dataset:
name: WMT_EN-RO
type: WMT_EN-RO
metrics:
- name: Average bleu
type: bleu
value: 25.74
- task:
type: text-generation
dataset:
name: WMT_RO-EN
type: WMT_RO-EN
metrics:
- name: Average bleu
type: bleu
value: 23.16
- task:
type: text-generation
dataset:
name: WMT_EN-RO_finetuned
type: WMT_EN-RO_finetuned
metrics:
- name: Average bleu
type: bleu
value: 28.43
- task:
type: text-generation
dataset:
name: WMT_RO-EN_finetuned
type: WMT_RO-EN_finetuned
metrics:
- name: Average bleu
type: bleu
value: 40.94
- task:
type: text-generation
dataset:
name: XQuAD
type: XQuAD
metrics:
- name: Average exact_match
type: exact_match
value: 51.37
- task:
type: text-generation
dataset:
name: XQuAD
type: XQuAD
metrics:
- name: Average f1
type: f1
value: 70.74
- task:
type: text-generation
dataset:
name: XQuAD_finetuned
type: XQuAD_finetuned
metrics:
- name: Average exact_match
type: exact_match
value: 50
- task:
type: text-generation
dataset:
name: XQuAD_finetuned
type: XQuAD_finetuned
metrics:
- name: Average f1
type: f1
value: 64.1
- task:
type: text-generation
dataset:
name: STS
type: STS
metrics:
- name: Average spearman
type: spearman
value: 77.15
- task:
type: text-generation
dataset:
name: STS
type: STS
metrics:
- name: Average pearson
type: pearson
value: 77.1
- task:
type: text-generation
dataset:
name: STS_finetuned
type: STS_finetuned
metrics:
- name: Average spearman
type: spearman
value: 89.45
- task:
type: text-generation
dataset:
name: STS_finetuned
type: STS_finetuned
metrics:
- name: Average pearson
type: pearson
value: 89.89
- task:
type: text-generation
dataset:
name: RoMT-Bench
type: RoMT-Bench
metrics:
- name: First turn
type: Score
value: 6.78
- name: Second turn
type: Score
value: 5.39
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_arc_challenge
type: OpenLLM-Ro/ro_arc_challenge
metrics:
- name: 0-shot
type: accuracy
value: 53.3
- name: 1-shot
type: accuracy
value: 54.93
- name: 3-shot
type: accuracy
value: 57.07
- name: 5-shot
type: accuracy
value: 57.33
- name: 10-shot
type: accuracy
value: 57.16
- name: 25-shot
type: accuracy
value: 57.41
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_mmlu
type: OpenLLM-Ro/ro_mmlu
metrics:
- name: 0-shot
type: accuracy
value: 59.2
- name: 1-shot
type: accuracy
value: 62.47
- name: 3-shot
type: accuracy
value: 64.97
- name: 5-shot
type: accuracy
value: 65.3
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_winogrande
type: OpenLLM-Ro/ro_winogrande
metrics:
- name: 0-shot
type: accuracy
value: 68.67
- name: 1-shot
type: accuracy
value: 71.03
- name: 3-shot
type: accuracy
value: 71.9
- name: 5-shot
type: accuracy
value: 72.38
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_hellaswag
type: OpenLLM-Ro/ro_hellaswag
metrics:
- name: 0-shot
type: accuracy
value: 62.29
- name: 1-shot
type: accuracy
value: 63.12
- name: 3-shot
type: accuracy
value: 61.34
- name: 5-shot
type: accuracy
value: 55.62
- name: 10-shot
type: accuracy
value: 60.25
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_gsm8k
type: OpenLLM-Ro/ro_gsm8k
metrics:
- name: 1-shot
type: accuracy
value: 36.77
- name: 3-shot
type: accuracy
value: 32.83
- name: 5-shot
type: accuracy
value: 43.97
- task:
type: text-generation
dataset:
name: LaRoSeDa_binary
type: LaRoSeDa_binary
metrics:
- name: 0-shot
type: macro-f1
value: 92.63
- name: 1-shot
type: macro-f1
value: 95.86
- name: 3-shot
type: macro-f1
value: 98.03
- name: 5-shot
type: macro-f1
value: 98.23
- task:
type: text-generation
dataset:
name: LaRoSeDa_multiclass
type: LaRoSeDa_multiclass
metrics:
- name: 0-shot
type: macro-f1
value: 38.51
- name: 1-shot
type: macro-f1
value: 69.7
- name: 3-shot
type: macro-f1
value: 71.38
- name: 5-shot
type: macro-f1
value: 70.37
- task:
type: text-generation
dataset:
name: WMT_EN-RO
type: WMT_EN-RO
metrics:
- name: 0-shot
type: bleu
value: 11.87
- name: 1-shot
type: bleu
value: 29.3
- name: 3-shot
type: bleu
value: 30.8
- name: 5-shot
type: bleu
value: 30.99
- task:
type: text-generation
dataset:
name: WMT_RO-EN
type: WMT_RO-EN
metrics:
- name: 0-shot
type: bleu
value: 1.03
- name: 1-shot
type: bleu
value: 22.25
- name: 3-shot
type: bleu
value: 32.75
- name: 5-shot
type: bleu
value: 36.61
- task:
type: text-generation
dataset:
name: XQuAD_EM
type: XQuAD_EM
metrics:
- name: 0-shot
type: exact_match
value: 52.6
- name: 1-shot
type: exact_match
value: 52.94
- name: 3-shot
type: exact_match
value: 49.66
- name: 5-shot
type: exact_match
value: 50.25
- task:
type: text-generation
dataset:
name: XQuAD_F1
type: XQuAD_F1
metrics:
- name: 0-shot
type: f1
value: 71.11
- name: 1-shot
type: f1
value: 71.67
- name: 3-shot
type: f1
value: 69.03
- name: 5-shot
type: f1
value: 71.15
- task:
type: text-generation
dataset:
name: STS_Spearman
type: STS_Spearman
metrics:
- name: 1-shot
type: spearman
value: 78.03
- name: 3-shot
type: spearman
value: 81.53
- name: 5-shot
type: spearman
value: 71.88
- task:
type: text-generation
dataset:
name: STS_Pearson
type: STS_Pearson
metrics:
- name: 1-shot
type: pearson
value: 79.09
- name: 3-shot
type: pearson
value: 80.89
- name: 5-shot
type: pearson
value: 71.33
Model Card for Model ID
RoGemma2 is a family of pretrained and fine-tuned generative text models for Romanian. This is the repository for the instruct 9B model. Links to other models can be found at the bottom of this page.
Model Details
Model Description
OpenLLM-Ro represents the first open-source effort to build a LLM specialized for Romanian. OpenLLM-Ro developed and publicly releases a collection of Romanian LLMs, both in the form of foundational model and instruct and chat variants.
- Developed by: OpenLLM-Ro
- Language(s): Romanian
- License: cc-by-nc-4.0
- Finetuned from model: gemma-2-9b-it
- Trained using: RoAlpaca, RoAlpacaGPT4, RoDolly, RoSelfInstruct, RoNoRobots, RoOrca, RoCamel, RoOpenAssistant, RoUltraChat
Model Sources
- Repository: https://github.com/OpenLLM-Ro/LLaMA-Factory
- Paper: https://arxiv.org/abs/2406.18266
Intended Use
Intended Use Cases
RoGemma2 is intented for research use in Romanian. Base models can be adapted for a variety of natural language tasks while instruction and chat tuned models are intended for assistant-like chat.
Out-of-Scope Use
Use in any manner that violates the license, any applicable laws or regluations, use in languages other than Romanian.
How to Get Started with the Model
Use the code below to get started with the model.
from transformers import AutoTokenizer, AutoModelForCausalLM
tokenizer = AutoTokenizer.from_pretrained("OpenLLM-Ro/RoGemma2-9b-Instruct-2024-10-09")
model = AutoModelForCausalLM.from_pretrained("OpenLLM-Ro/RoGemma2-9b-Instruct-2024-10-09")
instruction = "Ce jocuri de societate pot juca cu prietenii mei?"
chat = [
{"role": "user", "content": instruction},
]
prompt = tokenizer.apply_chat_template(chat, tokenize=False, system_message="")
inputs = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt")
outputs = model.generate(input_ids=inputs, max_new_tokens=128)
print(tokenizer.decode(outputs[0]))
Academic Benchmarks
| Model | |||||||
| gemma-2-9b-it | |||||||
| RoGemma2-9b-Instruct-2024-10-09 | |||||||
| RoGemma2-9b-Instruct-DPO-2024-10-09 |
Downstream tasks
| Model | (Macro F1) |
(Macro F1) |
(Macro F1) |
(Macro F1) |
(Bleu) |
(Bleu) |
(Bleu) |
(Bleu) |
| gemma-2-9b-it | ||||||||
| RoGemma2-9b-Instruct-2024-10-09 | ||||||||
| RoGemma2-9b-Instruct-DPO-2024-10-09 | ||||||||
| Model | ||||||||
| gemma-2-9b-it | ||||||||
| RoGemma2-9b-Instruct-2024-10-09 | ||||||||
| RoGemma2-9b-Instruct-DPO-2024-10-09 | ||||||||
MT-Bench
| Model | ||||
| gemma-2-9b-it | ||||
| RoGemma2-9b-Instruct-2024-10-09 | ||||
| RoGemma2-9b-Instruct-DPO-2024-10-09 |
RoCulturaBench
| Model | ||
| gemma-2-9b-it | ||
| RoGemma2-9b-Instruct-2024-10-09 | ||
| RoGemma2-9b-Instruct-DPO-2024-10-09 |
RoGemma2 Model Family
Citation
@misc{masala2024vorbecstiromanecsterecipetrain,
title={"Vorbe\c{s}ti Rom\^ane\c{s}te?" A Recipe to Train Powerful Romanian LLMs with English Instructions},
author={Mihai Masala and Denis C. Ilie-Ablachim and Alexandru Dima and Dragos Corlatescu and Miruna Zavelca and Ovio Olaru and Simina Terian-Dan and Andrei Terian-Dan and Marius Leordeanu and Horia Velicu and Marius Popescu and Mihai Dascalu and Traian Rebedea},
year={2024},
eprint={2406.18266},
archivePrefix={arXiv},
primaryClass={cs.CL},
url={https://arxiv.org/abs/2406.18266},
}