diff --git "a/novel-translation/00_Data_Analysis.ipynb" "b/novel-translation/00_Data_Analysis.ipynb" new file mode 100644--- /dev/null +++ "b/novel-translation/00_Data_Analysis.ipynb" @@ -0,0 +1,7371 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d4ad56f5-dd6b-47e2-8b75-bdc3cb0d5acd", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "288100c1-33d1-4e46-abaf-9a5ea4f7eca5", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/llm-finetuning\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "396e9b1b-b8b6-4281-a574-e9decfd020f7", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + } + ], + "source": [ + "from llm_toolkit.translation_utils import *" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Processing" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " | chinese | \n", + "english | \n", + "unsloth/Qwen2-0.5B-Instruct | \n", + "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct | \n", + "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct | \n", + "unsloth/Qwen2-7B-Instruct(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Tang held his gun, squinting his eyes with... | \n", + "Old Geng lifted his rifle and narrowed his eye... | \n", + "Old Geng took up his gun, squinted one of its ... | \n", + "Old Geng raised the rifle, squeezed one tiny t... | \n", + "Old Teng raised his gun and looked up at a pai... | \n", + "Old Geng raised his rifle, squinted his eyes, ... | \n", + "Old耿拿起枪,眯着眼睛一搂扳机就响了枪,金麻雀噼里啪啦的往下掉,铁砂子在柳枝间飞溅,发出“... | \n", + "Old Geng raised his pistol, squinted, and fire... | \n", + "Old Aigang raised his rifle, squinting one of ... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "Old Geng raised his gun, squinting one of his ... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next morning when it was still dark, Liu G... | \n", + "It was still not light when this little update... | \n", + "By the time the next day dawned, Liu Lao got u... | \n", + "Having been woken just before daybreak, Granni... | \n", + "The next day at dawn, Liu Geowon got up early ... | \n", + "Three or four hours before this, Grannie Liu h... | \n", + "At dawn the next day, Liu Langlang got up to b... | \n", + "But by some miracle of preparation, Grannie Li... | \n", + "The next morning, before dawn, Old Liu rose to... | \n", + "First thing next morning Grannie Liu rose befo... | \n", + "The next morning, before dawn, Old Liu rose to... | \n", + "First thing in the morning Grannie Liu rose an... | \n", + "
\n", + " | chinese | \n", + "english | \n", + "unsloth/Qwen2-0.5B-Instruct | \n", + "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct | \n", + "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-7B-Instruct(finetuned) | \n", + "unsloth/mistral-7b-instruct-v0.3 | \n", + "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Teng holds his gun up, his eyes narrowed a... | \n", + "Old Geng raised his rifle and tilted his head ... | \n", + "Old Jin raises his gun, squints one eye as he ... | \n", + "Old Geng raised his pistol, squinted through t... | \n", + "Old Geng raised his gun, squinting one of his ... | \n", + "Old Geng raised his rifle and squinted into on... | \n", + "The old man pulled out his gun, squinting one ... | \n", + "Old Geng raised his rifle, squinting through t... | \n", + "Old Geng raised his gun, squinted one of his t... | \n", + "Lao Geng raised his gun, narrowed one of his t... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "Geng Da initiates firing, squinting to form a ... | \n", + "Old Geng aimed and fired. A triangular slit op... | \n", + "Old Geng raised his gun, narrowed one of his t... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next morning, Liu Geo woke up at five o'cl... | \n", + "But not before noon did Grannie Liu rise up an... | \n", + "At dawn the next day, Liu Langlang got up earl... | \n", + "She got up about dawn with a purpose already e... | \n", + "The next morning, before dawn, Old Liu rose to... | \n", + "First thing next morning Grannie Liu rose earl... | \n", + "The next day, when the sun had not yet risen, ... | \n", + "Grannie Liu got up before daylight was even vi... | \n", + "The next morning, before the dawn had fully br... | \n", + "Before dawn next morning, Granny Liu got up to... | \n", + "First thing in the morning Grannie Liu rose to... | \n", + "The next day, when it was still dark, Liu Lao ... | \n", + "Before dawn next day Grannie Liu got up and bu... | \n", + "As soon as it was light, Grannie Liu got up an... | \n", + "
\n", + " | chinese | \n", + "english | \n", + "unsloth/Qwen2-0.5B-Instruct | \n", + "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct | \n", + "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-7B-Instruct(finetuned) | \n", + "unsloth/mistral-7b-instruct-v0.3 | \n", + "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Teng holds his gun up, his eyes narrowed a... | \n", + "Old Geng raised his rifle and tilted his head ... | \n", + "Old Jin raises his gun, squints one eye as he ... | \n", + "Old Geng raised his pistol, squinted through t... | \n", + "The old man pulled out his gun, squinting one ... | \n", + "Old Geng raised his rifle, squinting through t... | \n", + "Old Geng raised his gun, squinted one of his t... | \n", + "Lao Geng raised his gun, narrowed one of his t... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "Geng Da initiates firing, squinting to form a ... | \n", + "Old Geng aimed and fired. A triangular slit op... | \n", + "Old Geng raised his gun, narrowed one of his t... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next morning, Liu Geo woke up at five o'cl... | \n", + "But not before noon did Grannie Liu rise up an... | \n", + "At dawn the next day, Liu Langlang got up earl... | \n", + "She got up about dawn with a purpose already e... | \n", + "The next day, when the sun had not yet risen, ... | \n", + "Grannie Liu got up before daylight was even vi... | \n", + "The next morning, before the dawn had fully br... | \n", + "Before dawn next morning, Granny Liu got up to... | \n", + "First thing in the morning Grannie Liu rose to... | \n", + "The next day, when it was still dark, Liu Lao ... | \n", + "Before dawn next day Grannie Liu got up and bu... | \n", + "As soon as it was light, Grannie Liu got up an... | \n", + "
2 | \n", + "钱老板道:“是,是,多谢香主。” 在一张椅上坐了,续道:“属下将小郡主藏在猪肚里带进宫来,一... | \n", + "'Thank you, Master,' said Butcher Qian, seatin... | \n", + "Money Master said, 'Yes, yes, thank you for yo... | \n", + "If you like,' said Butcher Qian sitting at a t... | \n", + "Mr. Qian said, \"Yes, yes, thank you very much.... | \n", + "Of course,' said Butcher Qian gratefully. 'Tha... | \n", + "The boss said, \"Yes, thank you, Master. I sat ... | \n", + "'No, no, thank you,'said Butcher Qian, sitting... | \n", + "Mr. Qian said, \"Yes, yes, thank you for your c... | \n", + "Mr. Qian said, \"Yes, yes, many thanks, Perfume... | \n", + "Yes, Goong-goong, ' said Butcher Qian, sitting... | \n", + "The boss said, \"Yes, yes, thank you, Madam. Si... | \n", + "Many thanks, Master,' said Butcher Qian, and h... | \n", + "Yes, yes, thank you, Master,' said Butcher Qia... | \n", + "
3 | \n", + "但已经晚了,物理学家静静地躺在地上,半睁的双眼看着从他的头颅上流出的血迹,疯狂的会场瞬间陷入... | \n", + "But it was already too late. The physicist lay... | \n", + "But it was too late; physicist lay lifelessly ... | \n", + "But already too late: the physicist lay peacef... | \n", + "But it was too late. Physicists lay quietly on... | \n", + "But it was too late. The physicist was already... | \n", + "But it was too late. The physicist lay still o... | \n", + "But it was too late. The physicist lay quietly... | \n", + "But it was too late. The physicist lay quietly... | \n", + "But it was too late. The physicist lay quietly... | \n", + "But it was too late. The physicist lay on the ... | \n", + "The text is: \"But it's too late, the physicist... | \n", + "But it was already late. The physicist lay sti... | \n", + "But it was too late. The physicist lay on the ... | \n", + "
4 | \n", + "但这时,绍琳却做出了一件出人意料的事,与一位受迫害的教育部高干结了婚,当时那名高干还在干校住... | \n", + "But then Shao did something that no one expect... | \n", + "But this time, Rong Ling did something out of ... | \n", + "However, Shen refused to make a surprising ann... | \n", + "But at this time, Shen Lin made a surprising d... | \n", + "But at that moment, Shao Lin did something une... | \n", + "But at that time, Shao Lin did something unexp... | \n", + "However, at that moment, Shao Lin took an unex... | \n", + "But then, in a surprise move, she married a hi... | \n", + "But, in a surprising move, she married a perse... | \n", + "But then, Shao Lin surprised everyone by marry... | \n", + "Shao Lin surprisingly married a high-ranking o... | \n", + "But then Shao Lin did something unexpected: sh... | \n", + "But at this time, Shao Lin did something unexp... | \n", + "
\n", + " | chinese | \n", + "english | \n", + "Qwen2-0.5B(flash-attn:true) | \n", + "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", + "Qwen2-1.5B(flash-attn:true) | \n", + "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", + "
---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old耿举起枪,眯着眼睛,枪声轰鸣,子弹砰砰砰地落在地上,一颗颗冰雹般的大鸟扑棱棱地落在柳树... | \n", + "Old Geng raised his rifle and tilted his head,... | \n", + "Old Geer lifted his gun, squinted one of his e... | \n", + "Old Geng raised his gun, squinted, and emptied... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next day morning when the sun was still ri... | \n", + "First thing that morning the old lady did rise... | \n", + "The next morning, Liu Langlang got up early an... | \n", + "In the predawn light she arose, dressed, and b... | \n", + "
\n", + " | chinese | \n", + "english | \n", + "Qwen2-0.5B(flash-attn:false) | \n", + "Qwen2-0.5B(flash-attn:true) | \n", + "Qwen2-0.5B(finetuned)(flash-attn:false) | \n", + "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", + "Qwen2-1.5B(flash-attn:false) | \n", + "Qwen2-1.5B(flash-attn:true) | \n", + "Qwen2-1.5B(finetuned)(flash-attn:false) | \n", + "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", + "
---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪��眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Teng raises his gun, closing his eyes with... | \n", + "Old耿举起枪,眯着眼睛,枪声轰鸣,子弹砰砰砰地落在地上,一颗颗冰雹般的大鸟扑棱棱地落在柳树... | \n", + "Old Geng raised his rifle and made a twist eye... | \n", + "Old Geng raised his rifle and tilted his head,... | \n", + "Old耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝... | \n", + "Old Geer lifted his gun, squinted one of his e... | \n", + "Old Geng raised his pistol, squinted through t... | \n", + "Old Geng raised his gun, squinted, and emptied... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next day, after dawn, Liu Geong woke up at... | \n", + "The next day morning when the sun was still ri... | \n", + "First thing that made him up and go out of his... | \n", + "First thing that morning the old lady did rise... | \n", + "At dawn on the second day, Liu Laolao got up a... | \n", + "The next morning, Liu Langlang got up early an... | \n", + "But she didn't begin to wake her again until t... | \n", + "In the predawn light she arose, dressed, and b... | \n", + "
\n", + " | chinese | \n", + "english | \n", + "unsloth/Qwen2-0.5B-Instruct | \n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct | \n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct | \n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-7B-Instruct(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Tang held his gun, squinting his eyes with... | \n", + "Old Teng raised his gun and looked up at a pai... | \n", + "Old Geng lifted his rifle and narrowed his eye... | \n", + "Old Geng raised his rifle, squinted his eyes, ... | \n", + "Old Geng took up his gun, squinted one of its ... | \n", + "Old耿拿起枪,眯着眼睛一搂扳机就响了枪,金麻雀噼里啪啦的往下掉,铁砂子在柳枝间飞溅,发出“... | \n", + "Old Geng raised the rifle, squeezed one tiny t... | \n", + "Old Geng raised his pistol, squinted, and fire... | \n", + "Old Aigang raised his rifle, squinting one of ... | \n", + "Old Geng raised his gun, squinting one of his ... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next morning when it was still dark, Liu G... | \n", + "The next day at dawn, Liu Geowon got up early ... | \n", + "It was still not light when this little update... | \n", + "Three or four hours before this, Grannie Liu h... | \n", + "By the time the next day dawned, Liu Lao got u... | \n", + "At dawn the next day, Liu Langlang got up to b... | \n", + "Having been woken just before daybreak, Granni... | \n", + "But by some miracle of preparation, Grannie Li... | \n", + "The next morning, before dawn, Old Liu rose to... | \n", + "The next morning, before dawn, Old Liu rose to... | \n", + "First thing next morning Grannie Liu rose befo... | \n", + "First thing in the morning Grannie Liu rose an... | \n", + "
\n", + " | accuracy | \n", + "meteor | \n", + "bleu_1 | \n", + "rouge_l | \n", + "
---|---|---|---|---|
count | \n", + "12.000000 | \n", + "12.000000 | \n", + "12.000000 | \n", + "12.000000 | \n", + "
mean | \n", + "0.002427 | \n", + "0.334524 | \n", + "0.087780 | \n", + "0.317040 | \n", + "
std | \n", + "0.002665 | \n", + "0.052112 | \n", + "0.031913 | \n", + "0.046755 | \n", + "
min | \n", + "0.000000 | \n", + "0.257613 | \n", + "0.038502 | \n", + "0.251768 | \n", + "
25% | \n", + "0.000662 | \n", + "0.289797 | \n", + "0.064345 | \n", + "0.266434 | \n", + "
50% | \n", + "0.002207 | \n", + "0.340443 | \n", + "0.088830 | \n", + "0.320191 | \n", + "
75% | \n", + "0.002648 | \n", + "0.370605 | \n", + "0.107558 | \n", + "0.358062 | \n", + "
max | \n", + "0.008826 | \n", + "0.411287 | \n", + "0.138238 | \n", + "0.380708 | \n", + "
\n", + " | model | \n", + "meteor | \n", + "bleu_1 | \n", + "rouge_l | \n", + "
---|---|---|---|---|
0 | \n", + "Qwen2-0.5B-Instruct | \n", + "0.266821 | \n", + "0.050004 | \n", + "0.264547 | \n", + "
1 | \n", + "Qwen2-0.5B-Instruct-bnb-4bit | \n", + "0.257613 | \n", + "0.038502 | \n", + "0.251768 | \n", + "
2 | \n", + "Qwen2-0.5B-Instruct(finetuned) | \n", + "0.290324 | \n", + "0.065086 | \n", + "0.267063 | \n", + "
3 | \n", + "Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", + "0.288215 | \n", + "0.062123 | \n", + "0.262644 | \n", + "
4 | \n", + "Qwen2-1.5B-Instruct | \n", + "0.335521 | \n", + "0.082854 | \n", + "0.328097 | \n", + "
5 | \n", + "Qwen2-1.5B-Instruct-bnb-4bit | \n", + "0.312083 | \n", + "0.071532 | \n", + "0.312798 | \n", + "
6 | \n", + "Qwen2-1.5B-Instruct(finetuned) | \n", + "0.355038 | \n", + "0.097349 | \n", + "0.322742 | \n", + "
7 | \n", + "Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", + "0.345364 | \n", + "0.094805 | \n", + "0.317640 | \n", + "
8 | \n", + "Qwen2-7B-Instruct | \n", + "0.370062 | \n", + "0.106828 | \n", + "0.359001 | \n", + "
9 | \n", + "Qwen2-7B-Instruct-bnb-4bit | \n", + "0.372233 | \n", + "0.109749 | \n", + "0.357748 | \n", + "
10 | \n", + "Qwen2-7B-Instruct(finetuned) | \n", + "0.411287 | \n", + "0.138238 | \n", + "0.379722 | \n", + "
11 | \n", + "Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", + "0.409724 | \n", + "0.136295 | \n", + "0.380708 | \n", + "
\n", + " | model | \n", + "meteor | \n", + "train-time(mins) | \n", + "eval-time(mins) | \n", + "GPU | \n", + "
---|---|---|---|---|---|
0 | \n", + "Qwen2-0.5B-Instruct | \n", + "0.266821 | \n", + "62.99 | \n", + "22.53 | \n", + "RTX 4080 | \n", + "
1 | \n", + "Qwen2-0.5B-Instruct-bnb-4bit | \n", + "0.257613 | \n", + "85.05 | \n", + "41.88 | \n", + "RTX 4080 | \n", + "
2 | \n", + "Qwen2-0.5B-Instruct(finetuned) | \n", + "0.290324 | \n", + "0.00 | \n", + "26.47 | \n", + "RTX 4080 | \n", + "
3 | \n", + "Qwen2-0.5B-Instruct-bnb-4bit(finetuned) | \n", + "0.288215 | \n", + "0.00 | \n", + "36.87 | \n", + "RTX 4080 | \n", + "
4 | \n", + "Qwen2-1.5B-Instruct | \n", + "0.335521 | \n", + "92.74 | \n", + "30.02 | \n", + "RTX 4080 | \n", + "
5 | \n", + "Qwen2-1.5B-Instruct-bnb-4bit | \n", + "0.312083 | \n", + "139.92 | \n", + "59.60 | \n", + "RTX 4080 | \n", + "
6 | \n", + "Qwen2-1.5B-Instruct(finetuned) | \n", + "0.355038 | \n", + "0.00 | \n", + "34.15 | \n", + "RTX 4080 | \n", + "
7 | \n", + "Qwen2-1.5B-Instruct-bnb-4bit(finetuned) | \n", + "0.345364 | \n", + "0.00 | \n", + "50.73 | \n", + "RTX 4080 | \n", + "
8 | \n", + "Qwen2-7B-Instruct | \n", + "0.370062 | \n", + "97.77 | \n", + "37.58 | \n", + "L40 | \n", + "
9 | \n", + "Qwen2-7B-Instruct-bnb-4bit | \n", + "0.372233 | \n", + "103.40 | \n", + "39.87 | \n", + "L40 | \n", + "
10 | \n", + "Qwen2-7B-Instruct(finetuned) | \n", + "0.411287 | \n", + "0.00 | \n", + "37.05 | \n", + "L40 | \n", + "
11 | \n", + "Qwen2-7B-Instruct-bnb-4bit(finetuned) | \n", + "0.409724 | \n", + "0.00 | \n", + "36.82 | \n", + "L40 | \n", + "
\n", + " | chinese | \n", + "english | \n", + "unsloth/Qwen2-0.5B-Instruct | \n", + "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct | \n", + "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct | \n", + "unsloth/Qwen2-7B-Instruct(finetuned) | \n", + "unsloth/mistral-7b-instruct-v0.3 | \n", + "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Teng holds his gun up, his eyes narrowed a... | \n", + "Old Geng raised his rifle and tilted his head ... | \n", + "Old Jin raises his gun, squints one eye as he ... | \n", + "Old Geng raised his pistol, squinted through t... | \n", + "Old Geng raised his gun, squinted one of his t... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "Geng Da initiates firing, squinting to form a ... | \n", + "Old Geng aimed and fired. A triangular slit op... | \n", + "The old man pulled out his gun, squinting one ... | \n", + "Old Geng raised his rifle, squinting through t... | \n", + "Lao Geng raised his gun, narrowed one of his t... | \n", + "Old Geng raised his gun, narrowed one of his t... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next morning, Liu Geo woke up at five o'cl... | \n", + "But not before noon did Grannie Liu rise up an... | \n", + "At dawn the next day, Liu Langlang got up earl... | \n", + "She got up about dawn with a purpose already e... | \n", + "The next morning, before the dawn had fully br... | \n", + "First thing in the morning Grannie Liu rose to... | \n", + "The next day, when it was still dark, Liu Lao ... | \n", + "Before dawn next day Grannie Liu got up and bu... | \n", + "The next day, when the sun had not yet risen, ... | \n", + "Grannie Liu got up before daylight was even vi... | \n", + "Before dawn next morning, Granny Liu got up to... | \n", + "As soon as it was light, Grannie Liu got up an... | \n", + "
\n", + " | accuracy | \n", + "meteor | \n", + "bleu_1 | \n", + "rouge_l | \n", + "
---|---|---|---|---|
count | \n", + "12.000000 | \n", + "12.000000 | \n", + "12.000000 | \n", + "12.000000 | \n", + "
mean | \n", + "0.003383 | \n", + "0.353735 | \n", + "0.100411 | \n", + "0.337371 | \n", + "
std | \n", + "0.003926 | \n", + "0.055771 | \n", + "0.037567 | \n", + "0.050444 | \n", + "
min | \n", + "0.000000 | \n", + "0.264533 | \n", + "0.046980 | \n", + "0.256093 | \n", + "
25% | \n", + "0.000662 | \n", + "0.315699 | \n", + "0.069623 | \n", + "0.311106 | \n", + "
50% | \n", + "0.001324 | \n", + "0.355950 | \n", + "0.101607 | \n", + "0.338239 | \n", + "
75% | \n", + "0.005516 | \n", + "0.392410 | \n", + "0.125756 | \n", + "0.371022 | \n", + "
max | \n", + "0.012357 | \n", + "0.457161 | \n", + "0.170018 | \n", + "0.424822 | \n", + "
\n", + " | model | \n", + "accuracy | \n", + "meteor | \n", + "bleu_1 | \n", + "rouge_l | \n", + "all_metrics | \n", + "
---|---|---|---|---|---|---|
0 | \n", + "Qwen2-0.5B-Instruct | \n", + "0.000000 | \n", + "0.264533 | \n", + "0.046980 | \n", + "0.256093 | \n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", + "
1 | \n", + "Qwen2-0.5B-Instruct(finetuned) | \n", + "0.001765 | \n", + "0.286648 | \n", + "0.063335 | \n", + "0.261258 | \n", + "{'accuracy': 0.00176522506619594, 'correct_ids... | \n", + "
2 | \n", + "Qwen2-1.5B-Instruct | \n", + "0.000000 | \n", + "0.310808 | \n", + "0.071719 | \n", + "0.310355 | \n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", + "
3 | \n", + "Qwen2-1.5B-Instruct(finetuned) | \n", + "0.004413 | \n", + "0.341287 | \n", + "0.094396 | \n", + "0.317709 | \n", + "{'accuracy': 0.00441306266548985, 'correct_ids... | \n", + "
4 | \n", + "Qwen2-7B-Instruct | \n", + "0.000883 | \n", + "0.370613 | \n", + "0.108818 | \n", + "0.359339 | \n", + "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", + "
5 | \n", + "Qwen2-7B-Instruct(finetuned) | \n", + "0.006178 | \n", + "0.401630 | \n", + "0.134034 | \n", + "0.379916 | \n", + "{'accuracy': 0.00617828773168579, 'correct_ids... | \n", + "
6 | \n", + "mistral-7b-instruct-v0.3 | \n", + "0.000883 | \n", + "0.322159 | \n", + "0.085008 | \n", + "0.317920 | \n", + "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", + "
7 | \n", + "mistral-7b-instruct-v0.3(finetuned) | \n", + "0.005296 | \n", + "0.387250 | \n", + "0.125505 | \n", + "0.358558 | \n", + "{'accuracy': 0.00529567519858782, 'correct_ids... | \n", + "
8 | \n", + "Llama-3-8B-Instruct-Gradient-1048k | \n", + "0.000000 | \n", + "0.317330 | \n", + "0.053289 | \n", + "0.311356 | \n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", + "
9 | \n", + "Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", + "0.012357 | \n", + "0.392120 | \n", + "0.126232 | \n", + "0.368057 | \n", + "{'accuracy': 0.01235657546337158, 'correct_ids... | \n", + "
10 | \n", + "Qwen2-72B-Instruct-bnb-4bit | \n", + "0.000883 | \n", + "0.393278 | \n", + "0.125597 | \n", + "0.383067 | \n", + "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", + "
11 | \n", + "Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", + "0.007944 | \n", + "0.457161 | \n", + "0.170018 | \n", + "0.424822 | \n", + "{'accuracy': 0.00794351279788173, 'correct_ids... | \n", + "
\n", + " | model | \n", + "accuracy | \n", + "meteor | \n", + "bleu_1 | \n", + "rouge_l | \n", + "all_metrics | \n", + "
---|---|---|---|---|---|---|
0 | \n", + "Qwen2-0.5B(flash-attn:false) | \n", + "0.000000 | \n", + "0.259608 | \n", + "0.044094 | \n", + "0.252149 | \n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", + "
1 | \n", + "Qwen2-0.5B(flash-attn:true) | \n", + "0.000000 | \n", + "0.260415 | \n", + "0.046013 | \n", + "0.254703 | \n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", + "
2 | \n", + "Qwen2-0.5B(finetuned)(flash-attn:false) | \n", + "0.002648 | \n", + "0.290680 | \n", + "0.060746 | \n", + "0.265362 | \n", + "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", + "
3 | \n", + "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", + "0.002648 | \n", + "0.287526 | \n", + "0.064151 | \n", + "0.265777 | \n", + "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", + "
4 | \n", + "Qwen2-1.5B(flash-attn:false) | \n", + "0.000883 | \n", + "0.314090 | \n", + "0.071057 | \n", + "0.315755 | \n", + "{'accuracy': 0.00088261253309797, 'correct_ids... | \n", + "
5 | \n", + "Qwen2-1.5B(flash-attn:true) | \n", + "0.000000 | \n", + "0.311962 | \n", + "0.072696 | \n", + "0.310530 | \n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor':... | \n", + "
6 | \n", + "Qwen2-1.5B(finetuned)(flash-attn:false) | \n", + "0.002648 | \n", + "0.349020 | \n", + "0.084510 | \n", + "0.320888 | \n", + "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", + "
7 | \n", + "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", + "0.002648 | \n", + "0.349413 | \n", + "0.094641 | \n", + "0.318565 | \n", + "{'accuracy': 0.00264783759929391, 'correct_ids... | \n", + "
\n", + " | model | \n", + "meteor | \n", + "bleu_1 | \n", + "rouge_l | \n", + "
---|---|---|---|---|
0 | \n", + "Qwen2-0.5B(flash-attn:false) | \n", + "0.259608 | \n", + "0.044094 | \n", + "0.252149 | \n", + "
1 | \n", + "Qwen2-0.5B(flash-attn:true) | \n", + "0.260415 | \n", + "0.046013 | \n", + "0.254703 | \n", + "
2 | \n", + "Qwen2-0.5B(finetuned)(flash-attn:false) | \n", + "0.290680 | \n", + "0.060746 | \n", + "0.265362 | \n", + "
3 | \n", + "Qwen2-0.5B(finetuned)(flash-attn:true) | \n", + "0.287526 | \n", + "0.064151 | \n", + "0.265777 | \n", + "
4 | \n", + "Qwen2-1.5B(flash-attn:false) | \n", + "0.314090 | \n", + "0.071057 | \n", + "0.315755 | \n", + "
5 | \n", + "Qwen2-1.5B(flash-attn:true) | \n", + "0.311962 | \n", + "0.072696 | \n", + "0.310530 | \n", + "
6 | \n", + "Qwen2-1.5B(finetuned)(flash-attn:false) | \n", + "0.349020 | \n", + "0.084510 | \n", + "0.320888 | \n", + "
7 | \n", + "Qwen2-1.5B(finetuned)(flash-attn:true) | \n", + "0.349413 | \n", + "0.094641 | \n", + "0.318565 | \n", + "
\n", + " | chinese | \n", + "english | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-560 | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-1120 | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-1680 | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-2240 | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-2800 | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-3360 | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-3920 | \n", + "Qwen/Qwen2-0.5B-Instruct_checkpoint-4480 | \n", + "... | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-560 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-1120 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-1680 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-2240 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-2800 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-3360 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-3920 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-4480 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-5040 | \n", + "Qwen/Qwen2-1.5B-Instruct_checkpoint-5600 | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Trinket raised his gun and squinted his tr... | \n", + "Old Geng raised his gun, his eyes narrowed. Th... | \n", + "Old Geng held his gun up, half-closed, and coc... | \n", + "Old Geng raised his gun, his triangular eye ha... | \n", + "Old Geng took out his pistol, squinted over a ... | \n", + "Old Geng held his rifle up and cocked it over ... | \n", + "Old Geng held his gun to his chest, eyes on a ... | \n", + "Old Geng took up his gun and raised a triangul... | \n", + "... | \n", + "Grannie Geng held up his gun with one eye, nar... | \n", + "Old Geng raised his rifle and squinted at it t... | \n", + "Old Geng took his gun off the table and raised... | \n", + "Old Geng raised his rifle and squeezed the tri... | \n", + "Old Geng took aim and squeezed the trigger; do... | \n", + "Old Geng took a step forward, raised his pisto... | \n", + "Old Geng raised his pistol, opened it up, and ... | \n", + "Old Geng took a shot with his rifle. A spray o... | \n", + "Old Geng took a step forward, raised his rifle... | \n", + "Old Geng reached for his rifle, wedged it to h... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "In the morning she was up early for breakfast ... | \n", + "In the morning, however, when the sun was just... | \n", + "In the morning when the sun was just rising, G... | \n", + "It was still dark before she got up for breakf... | \n", + "In the early hours of the next day, when it wa... | \n", + "By day's dawn her old lady had risen from bed ... | \n", + "By the time the next morning was over, Grannie... | \n", + "It was just now six o'clock that the old woman... | \n", + "... | \n", + "By morning of the next day, Old Liu got up ver... | \n", + "At dawn the next day, Grannie Liu got up and w... | \n", + "By midnight, Grannie Liu had risen from her be... | \n", + "When she arose from her bed at daybreak the ne... | \n", + "As soon as it was light outside, Grannie Liu r... | \n", + "By daybreak she was up and dressed, having ins... | \n", + "At daybreak the old woman got up and dressed h... | \n", + "When she woke from her nap, Aunt Liu dressed h... | \n", + "Then at daybreak the old woman was up and abou... | \n", + "Grannie Liu got up very early the morning of t... | \n", + "
2 rows × 22 columns
\n", + "\n", + " | chinese | \n", + "english | \n", + "unsloth/Qwen2-0.5B-Instruct | \n", + "unsloth/Qwen2-0.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-1.5B-Instruct | \n", + "unsloth/Qwen2-1.5B-Instruct(finetuned) | \n", + "unsloth/Qwen2-7B-Instruct | \n", + "unsloth/Qwen2-7B-Instruct(finetuned) | \n", + "unsloth/mistral-7b-instruct-v0.3 | \n", + "unsloth/mistral-7b-instruct-v0.3(finetuned) | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k | \n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit | \n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "老耿端起枪,眯缝起一只三角眼,一搂扳机响了枪,冰雹般的金麻雀劈哩啪啦往下落,铁砂子在柳枝间飞... | \n", + "Old Geng picked up his shotgun, squinted, and ... | \n", + "Old Teng holds his gun up, his eyes narrowed a... | \n", + "Old Geng raised his rifle and tilted his head ... | \n", + "Old Jin raises his gun, squints one eye as he ... | \n", + "Old Geng raised his pistol, squinted through t... | \n", + "Old Geng raised his gun, squinted one of his t... | \n", + "Old Geng raised his rifle and squinted into th... | \n", + "Geng Da initiates firing, squinting to form a ... | \n", + "Old Geng aimed and fired. A triangular slit op... | \n", + "The old man pulled out his gun, squinting one ... | \n", + "Old Geng raised his rifle, squinting through t... | \n", + "Lao Geng raised his gun, narrowed one of his t... | \n", + "Old Geng raised his gun, narrowed one of his t... | \n", + "
1 | \n", + "次日天未明时,刘老老便起来梳洗了, 又将板儿教了几句话; 五六岁的孩子,听见带了他进城逛去,... | \n", + "Next day Grannie Liu was up before dawn. As so... | \n", + "The next morning, Liu Geo woke up at five o'cl... | \n", + "But not before noon did Grannie Liu rise up an... | \n", + "At dawn the next day, Liu Langlang got up earl... | \n", + "She got up about dawn with a purpose already e... | \n", + "The next morning, before the dawn had fully br... | \n", + "First thing in the morning Grannie Liu rose to... | \n", + "The next day, when it was still dark, Liu Lao ... | \n", + "Before dawn next day Grannie Liu got up and bu... | \n", + "The next day, when the sun had not yet risen, ... | \n", + "Grannie Liu got up before daylight was even vi... | \n", + "Before dawn next morning, Granny Liu got up to... | \n", + "As soon as it was light, Grannie Liu got up an... | \n", + "