{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b12ae8a3-9e08-402c-894c-31697fad6c56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eab00695e2b240ffb58ab998c85c0e7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "160c80c1-0ca4-45df-8171-87cd3c88a223",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    DataCollatorWithPadding,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    ")\n",
    "from utils import ConsistentSentenceRegressor, get_metrics, load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "25800588-5d42-4524-9dc6-a6a0c180b8b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                  text  label\n",
      "512  カーキ色の服を着た男性が、口元にリンゴを当てています。[SEP]カーキ色の服を着た男性が、口...    0.0\n",
      "513    男性がグラウンドでボールを投げています。[SEP]白い髯を生やした男性がボールを投げています。    0.5\n",
      "514  椅子に座った子供が、手づかみで食事をしています。[SEP]椅子に座った子供が手づかみで、食事...    1.0\n",
      "515         プロペラ機が何台も駐機しています。[SEP]プロペラ機が何台も連なって飛んでいます。    0.0\n",
      "516  消火栓から水が勢いよく噴き出しています。[SEP]水が噴き出している消火栓の水を浴びるように...    0.5\n",
      "517  冷蔵庫のないキッチンにナイフとフォークが置かれています。[SEP]冷蔵庫の置かれたキッチンに...    0.0\n",
      "518  うみでサーフィンをしているひとがいます。[SEP]黒いウェットスーツを着た人がサーフボードに...    0.5\n",
      "519             池から白い鳥が飛び立っています。[SEP]森にある水の上を鳥が飛んでいます。    0.5\n",
      "520       丈夫なビーチパラソルが立っています。[SEP]ビーチパラソルの支柱が折れ曲がっています。    0.0\n",
      "521  白髪の男性が少女から花束を受け取っています。[SEP]花束を持った男性の前に多くの子供たちが...    0.5\n",
      "                                                text  label\n",
      "0    赤いひとつの傘に、二人の人が入っています。[SEP]歩道を歩く通行人が傘をさして歩いています。    0.5\n",
      "1              川を小さなボートが進んで行きます。[SEP]川を豪華客船が進んでいきます。    0.0\n",
      "2  ゲレンデのこぶでスキージャンプしています。[SEP]雪上でモーグルを楽しむ水色のウェアを着た女性。    0.5\n",
      "3       黒いお皿に乗っているピザをカットしています。[SEP]黒い皿の上にピザが盛られています。    1.0\n",
      "4    女性が目を細めて携帯電話で話をしています。[SEP]目を細めた女性が携帯電話で話をしています。    1.0\n",
      "5  バナナやパパイヤなどの果物が売られている。[SEP]台の上にはバナナなどの青果が並べられています。    0.5\n",
      "6  ヘッドライトを点灯させた白いバスが駐車場に止まっています。[SEP]ライトを点灯させているバ...    1.0\n",
      "7  水面の上に、カイトサーフィンの凧が揚がっています。[SEP]海の上に水上スポーツ用の凧が揚が...    0.5\n",
      "8        ホットドッグを野外で食べている人たちです。[SEP]家の中でホットドッグを食べている。    0.0\n",
      "9  草が生い茂っている所に、3頭のゾウがいます。[SEP]草むらの中に三頭のゾウが立っているとこ...    0.5\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f20c88d0f96c4c06a9a0ddf835e544e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/19561 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Asking to pad to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no padding.\n",
      "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97591288ca9d40fd91e7737b41828f63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/512 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"line-corporation/line-distilbert-base-japanese\")\n",
    "dataset = load_dataset('train-v1.1.json')\n",
    "tokenized_dataset = dataset.map(\n",
    "    lambda examples: tokenizer(examples[\"text\"], padding='max_length', truncation=True), batched=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bc83d4c-378c-4313-b641-8ead0c02f715",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:XRT configuration not detected. Defaulting to preview PJRT runtime. To silence this warning and continue using PJRT, explicitly set PJRT_DEVICE to a supported device or configure XRT. To disable default device selection, set PJRT_SELECT_DEFAULT_DEVICE=0\n",
      "WARNING:root:For more information about the status of PJRT, see https://github.com/pytorch/xla/blob/master/docs/pjrt.md\n",
      "WARNING:root:Defaulting to PJRT_DEVICE=CPU\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7971' max='30600' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 7971/30600 16:14 < 46:07, 8.18 it/s, Epoch 26.05/100]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.085583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.104100</td>\n",
       "      <td>0.081926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.104100</td>\n",
       "      <td>0.079540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.091900</td>\n",
       "      <td>0.078066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.087600</td>\n",
       "      <td>0.076963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.087600</td>\n",
       "      <td>0.075823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.084500</td>\n",
       "      <td>0.075087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.084500</td>\n",
       "      <td>0.075002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.083000</td>\n",
       "      <td>0.073672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.080900</td>\n",
       "      <td>0.073238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.080900</td>\n",
       "      <td>0.072717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.080200</td>\n",
       "      <td>0.072234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.080200</td>\n",
       "      <td>0.071684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.079700</td>\n",
       "      <td>0.072137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>0.071143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>0.070724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.076500</td>\n",
       "      <td>0.070303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.077400</td>\n",
       "      <td>0.069888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.077400</td>\n",
       "      <td>0.069760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.076200</td>\n",
       "      <td>0.069610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.076200</td>\n",
       "      <td>0.069183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.075600</td>\n",
       "      <td>0.069061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.075600</td>\n",
       "      <td>0.068791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.075600</td>\n",
       "      <td>0.068658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.075000</td>\n",
       "      <td>0.068027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.075000</td>\n",
       "      <td>0.068032</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = ConsistentSentenceRegressor(\n",
    "    freeze_bert=True)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\".\",\n",
    "    learning_rate=1e-5,\n",
    "    per_device_train_batch_size=64,\n",
    "    num_train_epochs=100,\n",
    "    weight_decay=0.02,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    eval_accumulation_steps=1,\n",
    "    save_strategy=\"epoch\",\n",
    "    load_best_model_at_end=True,\n",
    "    push_to_hub=True,\n",
    ")\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_dataset[\"train\"],\n",
    "    eval_dataset=tokenized_dataset[\"test\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "trainer.push_to_hub('factual-consistency-regression-ja')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6eb93f7-5a38-49a2-be0d-e42267e23a0a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3638c8d8-fc85-4caf-83a4-4fd2ad6fb95d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "pytorch-gpu.2-0.m112",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/pytorch-gpu.2-0:m112"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}