Spaces:
Build error
Build error
test translation
Browse files
novel-translation/08_eval-lf-py3.11.ipynb
CHANGED
@@ -1360,7 +1360,7 @@
|
|
1360 |
" print(f\"Epoch {i}\")\n",
|
1361 |
" adapter_path = f\"llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-{560 * i}\"\n",
|
1362 |
" os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n",
|
1363 |
-
" !python llm_toolkit/
|
1364 |
]
|
1365 |
},
|
1366 |
{
|
@@ -2517,7 +2517,44 @@
|
|
2517 |
" print(f\"Epoch {i}\")\n",
|
2518 |
" adapter_path = f\"llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-{560 * i}\"\n",
|
2519 |
" os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n",
|
2520 |
-
" !python llm_toolkit/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2521 |
]
|
2522 |
},
|
2523 |
{
|
@@ -2544,29 +2581,27 @@
|
|
2544 |
},
|
2545 |
{
|
2546 |
"cell_type": "code",
|
2547 |
-
"execution_count":
|
2548 |
"metadata": {},
|
2549 |
"outputs": [
|
2550 |
{
|
2551 |
"data": {
|
2552 |
"text/plain": [
|
2553 |
-
"'
|
2554 |
]
|
2555 |
},
|
2556 |
-
"execution_count":
|
2557 |
"metadata": {},
|
2558 |
"output_type": "execute_result"
|
2559 |
}
|
2560 |
],
|
2561 |
"source": [
|
2562 |
-
"translate_via_llm(\
|
2563 |
-
" \"按他的逻辑,我身上这个通红通红,直不愣登,长约一尺的东西就是罪恶的化身。\"\n",
|
2564 |
-
")"
|
2565 |
]
|
2566 |
},
|
2567 |
{
|
2568 |
"cell_type": "code",
|
2569 |
-
"execution_count":
|
2570 |
"metadata": {},
|
2571 |
"outputs": [],
|
2572 |
"source": [
|
@@ -2575,23 +2610,23 @@
|
|
2575 |
},
|
2576 |
{
|
2577 |
"cell_type": "code",
|
2578 |
-
"execution_count":
|
2579 |
"metadata": {},
|
2580 |
"outputs": [
|
2581 |
{
|
2582 |
"data": {
|
2583 |
"text/plain": [
|
2584 |
-
"{'
|
2585 |
]
|
2586 |
},
|
2587 |
-
"execution_count":
|
2588 |
"metadata": {},
|
2589 |
"output_type": "execute_result"
|
2590 |
}
|
2591 |
],
|
2592 |
"source": [
|
2593 |
"translate(\n",
|
2594 |
-
" \"
|
2595 |
" cache_dict=cache_dict,\n",
|
2596 |
")\n",
|
2597 |
"cache_dict"
|
|
|
1360 |
" print(f\"Epoch {i}\")\n",
|
1361 |
" adapter_path = f\"llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-{560 * i}\"\n",
|
1362 |
" os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n",
|
1363 |
+
" !python llm_toolkit/eval.py "
|
1364 |
]
|
1365 |
},
|
1366 |
{
|
|
|
2517 |
" print(f\"Epoch {i}\")\n",
|
2518 |
" adapter_path = f\"llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-{560 * i}\"\n",
|
2519 |
" os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n",
|
2520 |
+
" !python llm_toolkit/eval.py "
|
2521 |
+
]
|
2522 |
+
},
|
2523 |
+
{
|
2524 |
+
"cell_type": "code",
|
2525 |
+
"execution_count": null,
|
2526 |
+
"metadata": {},
|
2527 |
+
"outputs": [],
|
2528 |
+
"source": [
|
2529 |
+
"%%time\n",
|
2530 |
+
"\n",
|
2531 |
+
"os.environ[\"MODEL_NAME\"] = \"Qwen/Qwen2-7B-Instruct\" \n",
|
2532 |
+
"for i in range(1, num_train_epochs + 1):\n",
|
2533 |
+
" print(f\"Epoch {i}\")\n",
|
2534 |
+
" adapter_path = f\"llama-factory/saves/qwen2-7b/lora/sft/checkpoint-{560 * i}\"\n",
|
2535 |
+
" os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n",
|
2536 |
+
" !python llm_toolkit/eval.py "
|
2537 |
+
]
|
2538 |
+
},
|
2539 |
+
{
|
2540 |
+
"cell_type": "code",
|
2541 |
+
"execution_count": 10,
|
2542 |
+
"metadata": {},
|
2543 |
+
"outputs": [],
|
2544 |
+
"source": [
|
2545 |
+
"from langchain_openai import ChatOpenAI\n",
|
2546 |
+
"\n",
|
2547 |
+
"llm = ChatOpenAI(\n",
|
2548 |
+
" model=\"gpt-4o\",\n",
|
2549 |
+
" temperature=0,\n",
|
2550 |
+
" max_tokens=None,\n",
|
2551 |
+
" timeout=None,\n",
|
2552 |
+
" max_retries=2,\n",
|
2553 |
+
" # api_key=\"...\", # if you prefer to pass api key in directly instaed of using env vars\n",
|
2554 |
+
" base_url=\"http://localhost:8000/v1\",\n",
|
2555 |
+
" # organization=\"...\",\n",
|
2556 |
+
" # other params...\n",
|
2557 |
+
")"
|
2558 |
]
|
2559 |
},
|
2560 |
{
|
|
|
2581 |
},
|
2582 |
{
|
2583 |
"cell_type": "code",
|
2584 |
+
"execution_count": 28,
|
2585 |
"metadata": {},
|
2586 |
"outputs": [
|
2587 |
{
|
2588 |
"data": {
|
2589 |
"text/plain": [
|
2590 |
+
"'The body was found on the morning beach'"
|
2591 |
]
|
2592 |
},
|
2593 |
+
"execution_count": 28,
|
2594 |
"metadata": {},
|
2595 |
"output_type": "execute_result"
|
2596 |
}
|
2597 |
],
|
2598 |
"source": [
|
2599 |
+
"translate_via_llm(\"死者凌晨去的沙滩\")"
|
|
|
|
|
2600 |
]
|
2601 |
},
|
2602 |
{
|
2603 |
"cell_type": "code",
|
2604 |
+
"execution_count": 31,
|
2605 |
"metadata": {},
|
2606 |
"outputs": [],
|
2607 |
"source": [
|
|
|
2610 |
},
|
2611 |
{
|
2612 |
"cell_type": "code",
|
2613 |
+
"execution_count": 34,
|
2614 |
"metadata": {},
|
2615 |
"outputs": [
|
2616 |
{
|
2617 |
"data": {
|
2618 |
"text/plain": [
|
2619 |
+
"{'死者凌晨去的沙滩': 'The body was found on the morning beach'}"
|
2620 |
]
|
2621 |
},
|
2622 |
+
"execution_count": 34,
|
2623 |
"metadata": {},
|
2624 |
"output_type": "execute_result"
|
2625 |
}
|
2626 |
],
|
2627 |
"source": [
|
2628 |
"translate(\n",
|
2629 |
+
" \"死者凌晨去的沙滩\",\n",
|
2630 |
" cache_dict=cache_dict,\n",
|
2631 |
")\n",
|
2632 |
"cache_dict"
|