{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { "byteLimit": 2048000, "rowLimit": 10000 }, "inputWidgets": {}, "nuid": "d4ad56f5-dd6b-47e2-8b75-bdc3cb0d5acd", "showTitle": false, "title": "" } }, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { "byteLimit": 2048000, "rowLimit": 10000 }, "inputWidgets": {}, "nuid": "288100c1-33d1-4e46-abaf-9a5ea4f7eca5", "showTitle": false, "title": "" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "workding dir: /home/inflaton/code/projects/courses/llm-finetuning\n" ] } ], "source": [ "import os\n", "import sys\n", "from pathlib import Path\n", "\n", "workding_dir = str(Path.cwd().parent)\n", "os.chdir(workding_dir)\n", "sys.path.append(workding_dir)\n", "print(\"workding dir:\", workding_dir)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { "byteLimit": 2048000, "rowLimit": 10000 }, "inputWidgets": {}, "nuid": "396e9b1b-b8b6-4281-a574-e9decfd020f7", "showTitle": false, "title": "" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from dotenv import find_dotenv, load_dotenv\n", "\n", "found_dotenv = find_dotenv(\".env\")\n", "\n", "if len(found_dotenv) == 0:\n", " found_dotenv = find_dotenv(\".env.example\")\n", "print(f\"loading env vars from: {found_dotenv}\")\n", "load_dotenv(found_dotenv, override=True)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", "[nltk_data] Package wordnet is already up-to-date!\n", "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", "[nltk_data] Package punkt is already up-to-date!\n", "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", "[nltk_data] Package omw-1.4 is already up-to-date!\n" ] } ], "source": [ "from llm_toolkit.translation_utils import *" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Data Processing" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | chinese | \n", "english | \n", "unsloth/Qwen2-0.5B-Instruct | \n", "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct | \n", "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-0.5B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", "unsloth/Qwen2-7B-Instruct | \n", "unsloth/Qwen2-7B-Instruct(finetuned) | \n", "unsloth/Qwen2-7B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Tang held his gun, squinting his eyes with... | \n", "Old Geng lifted his rifle and narrowed his eye... | \n", "Old Geng took up his gun, squinted one of its ... | \n", "Old Geng raised the rifle, squeezed one tiny t... | \n", "Old Teng raised his gun and looked up at a pai... | \n", "Old Geng raised his rifle, squinted his eyes, ... | \n", "Old耿拿起枪,眯着眼睛一搂扳机就响了枪,金麻雀噼里啪啦的往下掉,铁砂子在柳枝间飞溅,发出“... | \n", "Old Geng raised his pistol, squinted, and fire... | \n", "Old Aigang raised his rifle, squinting one of ... | \n", "Old Geng raised his rifle and squinted into th... | \n", "Old Geng raised his gun, squinting one of his ... | \n", "Old Geng raised his rifle and squinted into th... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next morning when it was still dark, Liu G... | \n", "It was still not light when this little update... | \n", "By the time the next day dawned, Liu Lao got u... | \n", "Having been woken just before daybreak, Granni... | \n", "The next day at dawn, Liu Geowon got up early ... | \n", "Three or four hours before this, Grannie Liu h... | \n", "At dawn the next day, Liu Langlang got up to b... | \n", "But by some miracle of preparation, Grannie Li... | \n", "The next morning, before dawn, Old Liu rose to... | \n", "First thing next morning Grannie Liu rose befo... | \n", "The next morning, before dawn, Old Liu rose to... | \n", "First thing in the morning Grannie Liu rose an... | \n", "
\n", " | chinese | \n", "english | \n", "unsloth/Qwen2-0.5B-Instruct | \n", "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct | \n", "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-7B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", "unsloth/Qwen2-7B-Instruct | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-7B-Instruct(finetuned) | \n", "unsloth/mistral-7b-instruct-v0.3 | \n", "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Teng holds his gun up, his eyes narrowed a... | \n", "Old Geng raised his rifle and tilted his head ... | \n", "Old Jin raises his gun, squints one eye as he ... | \n", "Old Geng raised his pistol, squinted through t... | \n", "Old Geng raised his gun, squinting one of his ... | \n", "Old Geng raised his rifle and squinted into on... | \n", "The old man pulled out his gun, squinting one ... | \n", "Old Geng raised his rifle, squinting through t... | \n", "Old Geng raised his gun, squinted one of his t... | \n", "Lao Geng raised his gun, narrowed one of his t... | \n", "Old Geng raised his rifle and squinted into th... | \n", "Geng Da initiates firing, squinting to form a ... | \n", "Old Geng aimed and fired. A triangular slit op... | \n", "Old Geng raised his gun, narrowed one of his t... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next morning, Liu Geo woke up at five o'cl... | \n", "But not before noon did Grannie Liu rise up an... | \n", "At dawn the next day, Liu Langlang got up earl... | \n", "She got up about dawn with a purpose already e... | \n", "The next morning, before dawn, Old Liu rose to... | \n", "First thing next morning Grannie Liu rose earl... | \n", "The next day, when the sun had not yet risen, ... | \n", "Grannie Liu got up before daylight was even vi... | \n", "The next morning, before the dawn had fully br... | \n", "Before dawn next morning, Granny Liu got up to... | \n", "First thing in the morning Grannie Liu rose to... | \n", "The next day, when it was still dark, Liu Lao ... | \n", "Before dawn next day Grannie Liu got up and bu... | \n", "As soon as it was light, Grannie Liu got up an... | \n", "
\n", " | chinese | \n", "english | \n", "unsloth/Qwen2-0.5B-Instruct | \n", "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct | \n", "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", "unsloth/Qwen2-7B-Instruct | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-7B-Instruct(finetuned) | \n", "unsloth/mistral-7b-instruct-v0.3 | \n", "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Teng holds his gun up, his eyes narrowed a... | \n", "Old Geng raised his rifle and tilted his head ... | \n", "Old Jin raises his gun, squints one eye as he ... | \n", "Old Geng raised his pistol, squinted through t... | \n", "The old man pulled out his gun, squinting one ... | \n", "Old Geng raised his rifle, squinting through t... | \n", "Old Geng raised his gun, squinted one of his t... | \n", "Lao Geng raised his gun, narrowed one of his t... | \n", "Old Geng raised his rifle and squinted into th... | \n", "Geng Da initiates firing, squinting to form a ... | \n", "Old Geng aimed and fired. A triangular slit op... | \n", "Old Geng raised his gun, narrowed one of his t... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next morning, Liu Geo woke up at five o'cl... | \n", "But not before noon did Grannie Liu rise up an... | \n", "At dawn the next day, Liu Langlang got up earl... | \n", "She got up about dawn with a purpose already e... | \n", "The next day, when the sun had not yet risen, ... | \n", "Grannie Liu got up before daylight was even vi... | \n", "The next morning, before the dawn had fully br... | \n", "Before dawn next morning, Granny Liu got up to... | \n", "First thing in the morning Grannie Liu rose to... | \n", "The next day, when it was still dark, Liu Lao ... | \n", "Before dawn next day Grannie Liu got up and bu... | \n", "As soon as it was light, Grannie Liu got up an... | \n", "
2 | \n", "钱老板道:“是,是,多谢香主。” 在一张椅上坐了,续道:“属下将小郡主藏在猪肚里带进宫来,一... | \n", "'Thank you, Master,' said Butcher Qian, seatin... | \n", "Money Master said, 'Yes, yes, thank you for yo... | \n", "If you like,' said Butcher Qian sitting at a t... | \n", "Mr. Qian said, \"Yes, yes, thank you very much.... | \n", "Of course,' said Butcher Qian gratefully. 'Tha... | \n", "The boss said, \"Yes, thank you, Master. I sat ... | \n", "'No, no, thank you,'said Butcher Qian, sitting... | \n", "Mr. Qian said, \"Yes, yes, thank you for your c... | \n", "Mr. Qian said, \"Yes, yes, many thanks, Perfume... | \n", "Yes, Goong-goong, ' said Butcher Qian, sitting... | \n", "The boss said, \"Yes, yes, thank you, Madam. Si... | \n", "Many thanks, Master,' said Butcher Qian, and h... | \n", "Yes, yes, thank you, Master,' said Butcher Qia... | \n", "
3 | \n", "但已经晚了,物理学家静静地躺在地上,半睁的双眼看着从他的头颅上流出的血迹,疯狂的会场瞬间陷入... | \n", "But it was already too late. The physicist lay... | \n", "But it was too late; physicist lay lifelessly ... | \n", "But already too late: the physicist lay peacef... | \n", "But it was too late. Physicists lay quietly on... | \n", "But it was too late. The physicist was already... | \n", "But it was too late. The physicist lay still o... | \n", "But it was too late. The physicist lay quietly... | \n", "But it was too late. The physicist lay quietly... | \n", "But it was too late. The physicist lay quietly... | \n", "But it was too late. The physicist lay on the ... | \n", "The text is: \"But it's too late, the physicist... | \n", "But it was already late. The physicist lay sti... | \n", "But it was too late. The physicist lay on the ... | \n", "
4 | \n", "但这时,绍琳却做出了一件出人意料的事,与一位受迫害的教育部高干结了婚,当时那名高干还在干校住... | \n", "But then Shao did something that no one expect... | \n", "But this time, Rong Ling did something out of ... | \n", "However, Shen refused to make a surprising ann... | \n", "But at this time, Shen Lin made a surprising d... | \n", "But at that moment, Shao Lin did something une... | \n", "But at that time, Shao Lin did something unexp... | \n", "However, at that moment, Shao Lin took an unex... | \n", "But then, in a surprise move, she married a hi... | \n", "But, in a surprising move, she married a perse... | \n", "But then, Shao Lin surprised everyone by marry... | \n", "Shao Lin surprisingly married a high-ranking o... | \n", "But then Shao Lin did something unexpected: sh... | \n", "But at this time, Shao Lin did something unexp... | \n", "
\n", " | chinese | \n", "english | \n", "Qwen2-0.5B(flash-attn:true) | \n", "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", "Qwen2-1.5B(flash-attn:true) | \n", "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", "
---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old耿举起枪,眯着眼睛,枪声轰鸣,子弹砰砰砰地落在地上,一颗颗冰雹般的大鸟扑棱棱地落在柳树... | \n", "Old Geng raised his rifle and tilted his head,... | \n", "Old Geer lifted his gun, squinted one of his e... | \n", "Old Geng raised his gun, squinted, and emptied... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next day morning when the sun was still ri... | \n", "First thing that morning the old lady did rise... | \n", "The next morning, Liu Langlang got up early an... | \n", "In the predawn light she arose, dressed, and b... | \n", "
\n", " | chinese | \n", "english | \n", "Qwen2-0.5B(flash-attn:false) | \n", "Qwen2-0.5B(flash-attn:true) | \n", "Qwen2-0.5B(finetuned)(flash-attn:false) | \n", "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", "Qwen2-1.5B(flash-attn:false) | \n", "Qwen2-1.5B(flash-attn:true) | \n", "Qwen2-1.5B(finetuned)(flash-attn:false) | \n", "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", "
---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Teng raises his gun, closing his eyes with... | \n", "Old耿举起枪,眯着眼睛,枪声轰鸣,子弹砰砰砰地落在地上,一颗颗冰雹般的大鸟扑棱棱地落在柳树... | \n", "Old Geng raised his rifle and made a twist eye... | \n", "Old Geng raised his rifle and tilted his head,... | \n", "Old耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝... | \n", "Old Geer lifted his gun, squinted one of his e... | \n", "Old Geng raised his pistol, squinted through t... | \n", "Old Geng raised his gun, squinted, and emptied... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next day, after dawn, Liu Geong woke up at... | \n", "The next day morning when the sun was still ri... | \n", "First thing that made him up and go out of his... | \n", "First thing that morning the old lady did rise... | \n", "At dawn on the second day, Liu Laolao got up a... | \n", "The next morning, Liu Langlang got up early an... | \n", "But she didn't begin to wake her again until t... | \n", "In the predawn light she arose, dressed, and b... | \n", "
\n", " | chinese | \n", "english | \n", "unsloth/Qwen2-0.5B-Instruct | \n", "unsloth/Qwen2-0.5B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct | \n", "unsloth/Qwen2-1.5B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", "unsloth/Qwen2-7B-Instruct | \n", "unsloth/Qwen2-7B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-7B-Instruct(finetuned) | \n", "unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Tang held his gun, squinting his eyes with... | \n", "Old Teng raised his gun and looked up at a pai... | \n", "Old Geng lifted his rifle and narrowed his eye... | \n", "Old Geng raised his rifle, squinted his eyes, ... | \n", "Old Geng took up his gun, squinted one of its ... | \n", "Old耿拿起枪,眯着眼睛一搂扳机就响了枪,金麻雀噼里啪啦的往下掉,铁砂子在柳枝间飞溅,发出“... | \n", "Old Geng raised the rifle, squeezed one tiny t... | \n", "Old Geng raised his pistol, squinted, and fire... | \n", "Old Aigang raised his rifle, squinting one of ... | \n", "Old Geng raised his gun, squinting one of his ... | \n", "Old Geng raised his rifle and squinted into th... | \n", "Old Geng raised his rifle and squinted into th... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next morning when it was still dark, Liu G... | \n", "The next day at dawn, Liu Geowon got up early ... | \n", "It was still not light when this little update... | \n", "Three or four hours before this, Grannie Liu h... | \n", "By the time the next day dawned, Liu Lao got u... | \n", "At dawn the next day, Liu Langlang got up to b... | \n", "Having been woken just before daybreak, Granni... | \n", "But by some miracle of preparation, Grannie Li... | \n", "The next morning, before dawn, Old Liu rose to... | \n", "The next morning, before dawn, Old Liu rose to... | \n", "First thing next morning Grannie Liu rose befo... | \n", "First thing in the morning Grannie Liu rose an... | \n", "
\n", " | accuracy | \n", "meteor | \n", "bleu_1 | \n", "rouge_l | \n", "
---|---|---|---|---|
count | \n", "12.000000 | \n", "12.000000 | \n", "12.000000 | \n", "12.000000 | \n", "
mean | \n", "0.002427 | \n", "0.334524 | \n", "0.087780 | \n", "0.317040 | \n", "
std | \n", "0.002665 | \n", "0.052112 | \n", "0.031913 | \n", "0.046755 | \n", "
min | \n", "0.000000 | \n", "0.257613 | \n", "0.038502 | \n", "0.251768 | \n", "
25% | \n", "0.000662 | \n", "0.289797 | \n", "0.064345 | \n", "0.266434 | \n", "
50% | \n", "0.002207 | \n", "0.340443 | \n", "0.088830 | \n", "0.320191 | \n", "
75% | \n", "0.002648 | \n", "0.370605 | \n", "0.107558 | \n", "0.358062 | \n", "
max | \n", "0.008826 | \n", "0.411287 | \n", "0.138238 | \n", "0.380708 | \n", "
\n", " | model | \n", "meteor | \n", "bleu_1 | \n", "rouge_l | \n", "
---|---|---|---|---|
0 | \n", "Qwen2-0.5B-Instruct | \n", "0.266821 | \n", "0.050004 | \n", "0.264547 | \n", "
1 | \n", "Qwen2-0.5B-Instruct-bnb-4bit | \n", "0.257613 | \n", "0.038502 | \n", "0.251768 | \n", "
2 | \n", "Qwen2-0.5B-Instruct(finetuned) | \n", "0.290324 | \n", "0.065086 | \n", "0.267063 | \n", "
3 | \n", "Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", "0.288215 | \n", "0.062123 | \n", "0.262644 | \n", "
4 | \n", "Qwen2-1.5B-Instruct | \n", "0.335521 | \n", "0.082854 | \n", "0.328097 | \n", "
5 | \n", "Qwen2-1.5B-Instruct-bnb-4bit | \n", "0.312083 | \n", "0.071532 | \n", "0.312798 | \n", "
6 | \n", "Qwen2-1.5B-Instruct(finetuned) | \n", "0.355038 | \n", "0.097349 | \n", "0.322742 | \n", "
7 | \n", "Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", "0.345364 | \n", "0.094805 | \n", "0.317640 | \n", "
8 | \n", "Qwen2-7B-Instruct | \n", "0.370062 | \n", "0.106828 | \n", "0.359001 | \n", "
9 | \n", "Qwen2-7B-Instruct-bnb-4bit | \n", "0.372233 | \n", "0.109749 | \n", "0.357748 | \n", "
10 | \n", "Qwen2-7B-Instruct(finetuned) | \n", "0.411287 | \n", "0.138238 | \n", "0.379722 | \n", "
11 | \n", "Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", "0.409724 | \n", "0.136295 | \n", "0.380708 | \n", "
\n", " | model | \n", "meteor | \n", "train-time(mins) | \n", "eval-time(mins) | \n", "GPU | \n", "
---|---|---|---|---|---|
0 | \n", "Qwen2-0.5B-Instruct | \n", "0.266821 | \n", "62.99 | \n", "22.53 | \n", "RTX 4080 | \n", "
1 | \n", "Qwen2-0.5B-Instruct-bnb-4bit | \n", "0.257613 | \n", "85.05 | \n", "41.88 | \n", "RTX 4080 | \n", "
2 | \n", "Qwen2-0.5B-Instruct(finetuned) | \n", "0.290324 | \n", "0.00 | \n", "26.47 | \n", "RTX 4080 | \n", "
3 | \n", "Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", "0.288215 | \n", "0.00 | \n", "36.87 | \n", "RTX 4080 | \n", "
4 | \n", "Qwen2-1.5B-Instruct | \n", "0.335521 | \n", "92.74 | \n", "30.02 | \n", "RTX 4080 | \n", "
5 | \n", "Qwen2-1.5B-Instruct-bnb-4bit | \n", "0.312083 | \n", "139.92 | \n", "59.60 | \n", "RTX 4080 | \n", "
6 | \n", "Qwen2-1.5B-Instruct(finetuned) | \n", "0.355038 | \n", "0.00 | \n", "34.15 | \n", "RTX 4080 | \n", "
7 | \n", "Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", "0.345364 | \n", "0.00 | \n", "50.73 | \n", "RTX 4080 | \n", "
8 | \n", "Qwen2-7B-Instruct | \n", "0.370062 | \n", "97.77 | \n", "37.58 | \n", "L40 | \n", "
9 | \n", "Qwen2-7B-Instruct-bnb-4bit | \n", "0.372233 | \n", "103.40 | \n", "39.87 | \n", "L40 | \n", "
10 | \n", "Qwen2-7B-Instruct(finetuned) | \n", "0.411287 | \n", "0.00 | \n", "37.05 | \n", "L40 | \n", "
11 | \n", "Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", "0.409724 | \n", "0.00 | \n", "36.82 | \n", "L40 | \n", "
\n", " | chinese | \n", "english | \n", "unsloth/Qwen2-0.5B-Instruct | \n", "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct | \n", "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-7B-Instruct | \n", "unsloth/Qwen2-7B-Instruct(finetuned) | \n", "unsloth/mistral-7b-instruct-v0.3 | \n", "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Teng holds his gun up, his eyes narrowed a... | \n", "Old Geng raised his rifle and tilted his head ... | \n", "Old Jin raises his gun, squints one eye as he ... | \n", "Old Geng raised his pistol, squinted through t... | \n", "Old Geng raised his gun, squinted one of his t... | \n", "Old Geng raised his rifle and squinted into th... | \n", "Geng Da initiates firing, squinting to form a ... | \n", "Old Geng aimed and fired. A triangular slit op... | \n", "The old man pulled out his gun, squinting one ... | \n", "Old Geng raised his rifle, squinting through t... | \n", "Lao Geng raised his gun, narrowed one of his t... | \n", "Old Geng raised his gun, narrowed one of his t... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next morning, Liu Geo woke up at five o'cl... | \n", "But not before noon did Grannie Liu rise up an... | \n", "At dawn the next day, Liu Langlang got up earl... | \n", "She got up about dawn with a purpose already e... | \n", "The next morning, before the dawn had fully br... | \n", "First thing in the morning Grannie Liu rose to... | \n", "The next day, when it was still dark, Liu Lao ... | \n", "Before dawn next day Grannie Liu got up and bu... | \n", "The next day, when the sun had not yet risen, ... | \n", "Grannie Liu got up before daylight was even vi... | \n", "Before dawn next morning, Granny Liu got up to... | \n", "As soon as it was light, Grannie Liu got up an... | \n", "
\n", " | accuracy | \n", "meteor | \n", "bleu_1 | \n", "rouge_l | \n", "
---|---|---|---|---|
count | \n", "12.000000 | \n", "12.000000 | \n", "12.000000 | \n", "12.000000 | \n", "
mean | \n", "0.003383 | \n", "0.353735 | \n", "0.100411 | \n", "0.337371 | \n", "
std | \n", "0.003926 | \n", "0.055771 | \n", "0.037567 | \n", "0.050444 | \n", "
min | \n", "0.000000 | \n", "0.264533 | \n", "0.046980 | \n", "0.256093 | \n", "
25% | \n", "0.000662 | \n", "0.315699 | \n", "0.069623 | \n", "0.311106 | \n", "
50% | \n", "0.001324 | \n", "0.355950 | \n", "0.101607 | \n", "0.338239 | \n", "
75% | \n", "0.005516 | \n", "0.392410 | \n", "0.125756 | \n", "0.371022 | \n", "
max | \n", "0.012357 | \n", "0.457161 | \n", "0.170018 | \n", "0.424822 | \n", "
\n", " | model | \n", "accuracy | \n", "meteor | \n", "bleu_1 | \n", "rouge_l | \n", "all_metrics | \n", "
---|---|---|---|---|---|---|
0 | \n", "Qwen2-0.5B-Instruct | \n", "0.000000 | \n", "0.264533 | \n", "0.046980 | \n", "0.256093 | \n", "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", "
1 | \n", "Qwen2-0.5B-Instruct(finetuned) | \n", "0.001765 | \n", "0.286648 | \n", "0.063335 | \n", "0.261258 | \n", "{'accuracy': 0.00176522506619594, 'correct_ids... | \n", "
2 | \n", "Qwen2-1.5B-Instruct | \n", "0.000000 | \n", "0.310808 | \n", "0.071719 | \n", "0.310355 | \n", "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", "
3 | \n", "Qwen2-1.5B-Instruct(finetuned) | \n", "0.004413 | \n", "0.341287 | \n", "0.094396 | \n", "0.317709 | \n", "{'accuracy': 0.00441306266548985, 'correct_ids... | \n", "
4 | \n", "Qwen2-7B-Instruct | \n", "0.000883 | \n", "0.370613 | \n", "0.108818 | \n", "0.359339 | \n", "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", "
5 | \n", "Qwen2-7B-Instruct(finetuned) | \n", "0.006178 | \n", "0.401630 | \n", "0.134034 | \n", "0.379916 | \n", "{'accuracy': 0.00617828773168579, 'correct_ids... | \n", "
6 | \n", "mistral-7b-instruct-v0.3 | \n", "0.000883 | \n", "0.322159 | \n", "0.085008 | \n", "0.317920 | \n", "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", "
7 | \n", "mistral-7b-instruct-v0.3(finetuned) | \n", "0.005296 | \n", "0.387250 | \n", "0.125505 | \n", "0.358558 | \n", "{'accuracy': 0.00529567519858782, 'correct_ids... | \n", "
8 | \n", "Llama-3-8B-Instruct-Gradient-1048k | \n", "0.000000 | \n", "0.317330 | \n", "0.053289 | \n", "0.311356 | \n", "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", "
9 | \n", "Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", "0.012357 | \n", "0.392120 | \n", "0.126232 | \n", "0.368057 | \n", "{'accuracy': 0.01235657546337158, 'correct_ids... | \n", "
10 | \n", "Qwen2-72B-Instruct-bnb-4bit | \n", "0.000883 | \n", "0.393278 | \n", "0.125597 | \n", "0.383067 | \n", "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", "
11 | \n", "Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", "0.007944 | \n", "0.457161 | \n", "0.170018 | \n", "0.424822 | \n", "{'accuracy': 0.00794351279788173, 'correct_ids... | \n", "
\n", " | model | \n", "accuracy | \n", "meteor | \n", "bleu_1 | \n", "rouge_l | \n", "all_metrics | \n", "
---|---|---|---|---|---|---|
0 | \n", "Qwen2-0.5B(flash-attn:false) | \n", "0.000000 | \n", "0.259608 | \n", "0.044094 | \n", "0.252149 | \n", "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", "
1 | \n", "Qwen2-0.5B(flash-attn:true) | \n", "0.000000 | \n", "0.260415 | \n", "0.046013 | \n", "0.254703 | \n", "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", "
2 | \n", "Qwen2-0.5B(finetuned)(flash-attn:false) | \n", "0.002648 | \n", "0.290680 | \n", "0.060746 | \n", "0.265362 | \n", "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", "
3 | \n", "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", "0.002648 | \n", "0.287526 | \n", "0.064151 | \n", "0.265777 | \n", "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", "
4 | \n", "Qwen2-1.5B(flash-attn:false) | \n", "0.000883 | \n", "0.314090 | \n", "0.071057 | \n", "0.315755 | \n", "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", "
5 | \n", "Qwen2-1.5B(flash-attn:true) | \n", "0.000000 | \n", "0.311962 | \n", "0.072696 | \n", "0.310530 | \n", "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", "
6 | \n", "Qwen2-1.5B(finetuned)(flash-attn:false) | \n", "0.002648 | \n", "0.349020 | \n", "0.084510 | \n", "0.320888 | \n", "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", "
7 | \n", "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", "0.002648 | \n", "0.349413 | \n", "0.094641 | \n", "0.318565 | \n", "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", "
\n", " | model | \n", "meteor | \n", "bleu_1 | \n", "rouge_l | \n", "
---|---|---|---|---|
0 | \n", "Qwen2-0.5B(flash-attn:false) | \n", "0.259608 | \n", "0.044094 | \n", "0.252149 | \n", "
1 | \n", "Qwen2-0.5B(flash-attn:true) | \n", "0.260415 | \n", "0.046013 | \n", "0.254703 | \n", "
2 | \n", "Qwen2-0.5B(finetuned)(flash-attn:false) | \n", "0.290680 | \n", "0.060746 | \n", "0.265362 | \n", "
3 | \n", "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", "0.287526 | \n", "0.064151 | \n", "0.265777 | \n", "
4 | \n", "Qwen2-1.5B(flash-attn:false) | \n", "0.314090 | \n", "0.071057 | \n", "0.315755 | \n", "
5 | \n", "Qwen2-1.5B(flash-attn:true) | \n", "0.311962 | \n", "0.072696 | \n", "0.310530 | \n", "
6 | \n", "Qwen2-1.5B(finetuned)(flash-attn:false) | \n", "0.349020 | \n", "0.084510 | \n", "0.320888 | \n", "
7 | \n", "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", "0.349413 | \n", "0.094641 | \n", "0.318565 | \n", "
\n", " | chinese | \n", "english | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-560 | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-1120 | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-1680 | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-2240 | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-2800 | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-3360 | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-3920 | \n", "Qwen/Qwen2-0.5B-Instruct_checkpoint-4480 | \n", "... | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-560 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-1120 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-1680 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-2240 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-2800 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-3360 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-3920 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-4480 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-5040 | \n", "Qwen/Qwen2-1.5B-Instruct_checkpoint-5600 | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Trinket raised his gun and squinted his tr... | \n", "Old Geng raised his gun, his eyes narrowed. Th... | \n", "Old Geng held his gun up, half-closed, and coc... | \n", "Old Geng raised his gun, his triangular eye ha... | \n", "Old Geng took out his pistol, squinted over a ... | \n", "Old Geng held his rifle up and cocked it over ... | \n", "Old Geng held his gun to his chest, eyes on a ... | \n", "Old Geng took up his gun and raised a triangul... | \n", "... | \n", "Grannie Geng held up his gun with one eye, nar... | \n", "Old Geng raised his rifle and squinted at it t... | \n", "Old Geng took his gun off the table and raised... | \n", "Old Geng raised his rifle and squeezed the tri... | \n", "Old Geng took aim and squeezed the trigger; do... | \n", "Old Geng took a step forward, raised his pisto... | \n", "Old Geng raised his pistol, opened it up, and ... | \n", "Old Geng took a shot with his rifle. A spray o... | \n", "Old Geng took a step forward, raised his rifle... | \n", "Old Geng reached for his rifle, wedged it to h... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "In the morning she was up early for breakfast ... | \n", "In the morning, however, when the sun was just... | \n", "In the morning when the sun was just rising, G... | \n", "It was still dark before she got up for breakf... | \n", "In the early hours of the next day, when it wa... | \n", "By day's dawn her old lady had risen from bed ... | \n", "By the time the next morning was over, Grannie... | \n", "It was just now six o'clock that the old woman... | \n", "... | \n", "By morning of the next day, Old Liu got up ver... | \n", "At dawn the next day, Grannie Liu got up and w... | \n", "By midnight, Grannie Liu had risen from her be... | \n", "When she arose from her bed at daybreak the ne... | \n", "As soon as it was light outside, Grannie Liu r... | \n", "By daybreak she was up and dressed, having ins... | \n", "At daybreak the old woman got up and dressed h... | \n", "When she woke from her nap, Aunt Liu dressed h... | \n", "Then at daybreak the old woman was up and abou... | \n", "Grannie Liu got up very early the morning of t... | \n", "
2 rows × 22 columns
\n", "\n", " | chinese | \n", "english | \n", "unsloth/Qwen2-0.5B-Instruct | \n", "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-1.5B-Instruct | \n", "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", "unsloth/Qwen2-7B-Instruct | \n", "unsloth/Qwen2-7B-Instruct(finetuned) | \n", "unsloth/mistral-7b-instruct-v0.3 | \n", "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", "Old Geng picked up his shotgun, squinted, and ... | \n", "Old Teng holds his gun up, his eyes narrowed a... | \n", "Old Geng raised his rifle and tilted his head ... | \n", "Old Jin raises his gun, squints one eye as he ... | \n", "Old Geng raised his pistol, squinted through t... | \n", "Old Geng raised his gun, squinted one of his t... | \n", "Old Geng raised his rifle and squinted into th... | \n", "Geng Da initiates firing, squinting to form a ... | \n", "Old Geng aimed and fired. A triangular slit op... | \n", "The old man pulled out his gun, squinting one ... | \n", "Old Geng raised his rifle, squinting through t... | \n", "Lao Geng raised his gun, narrowed one of his t... | \n", "Old Geng raised his gun, narrowed one of his t... | \n", "
1 | \n", "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", "Next day Grannie Liu was up before dawn. As so... | \n", "The next morning, Liu Geo woke up at five o'cl... | \n", "But not before noon did Grannie Liu rise up an... | \n", "At dawn the next day, Liu Langlang got up earl... | \n", "She got up about dawn with a purpose already e... | \n", "The next morning, before the dawn had fully br... | \n", "First thing in the morning Grannie Liu rose to... | \n", "The next day, when it was still dark, Liu Lao ... | \n", "Before dawn next day Grannie Liu got up and bu... | \n", "The next day, when the sun had not yet risen, ... | \n", "Grannie Liu got up before daylight was even vi... | \n", "Before dawn next morning, Granny Liu got up to... | \n", "As soon as it was light, Grannie Liu got up an... | \n", "