tmp/AI Search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8495bede-ab8f-416b-b5f2-6a76b1e63935",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Projects\\LLMs\\venv\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from sentence_transformers import SentenceTransformer, util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2b8cae6d-547b-4018-9f68-b0a45284b4b4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# model = SentenceTransformer('DMetaSoul/sbert-chinese-general-v2')\n",
    "model = SentenceTransformer('TintinMeimei/menglang_yongtulv_aimatch_v1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d3907a6f-f8ab-40fe-8702-c8cb81e189c6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def sim(text1, text2):\n",
    "    emb1 = model.encode(text1, convert_to_tensor=True)\n",
    "    emb2 = model.encode(text2, convert_to_tensor=True)\n",
    "    score = util.cos_sim(emb1, emb2)\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3cec9f05-4ea9-46f8-a393-950c67a0150a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text1 = '挂机空调'\n",
    "# text2 = '1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平板电视机、电风扇等家用电器制造。房间空气调节器能效优于《房间空气调节器能效限定值及能效等级》（GB 12021.3）标准1级能效水平；转速可控型房间空气调节器能效优于《转速可控型房间空气调节器能效限定值及能效等级》（GB 21455）标准1级能效水平；多联式空调（热泵）机组能效比优于《多联式空调（热泵）机组能效限定值及能源效率等级》（GB 21454）标准1级能效水平；家用电冰箱能效优于《家用电冰箱耗电量限定值及能效等级》（GB 12021.2）标准1级能效水平；电动洗衣机能效优于《电动洗衣机能效水效限定值及等级》（GB 12021.4）标准1级能效水平；电饭煲能效优于《电饭锅能效限定值及能效等级》（GB 12021.6）标准1级能效水平；平板电视机能效优于《平板电视能效限定值及能效等级》（GB 24850）标准1级能效水平；交流电风扇能效优于《交流电风扇能效限定值及能效等级》（GB 12021.9）标准1级能效水平。其他高效节能家用电器能效均优于相应国家强制性标准1级能效水平。'\n",
    "# text2 = '包括节能泵、节能型真空干燥设备、节能型真空炉等设备制造。清水离心泵能效指标优于《清水离心泵能效限定值及节能评价值》（GB 19762）标准中节能评价值水平；石油化工离心泵能效优于《石油化工离心泵能效限定值及能效等级》（GB 32284）标准中1级能效水平；潜水电泵能效优于《井用潜水电泵能效限定值及能效等级》（GB 32030）、《小型潜水电泵能效限定值及能效等级》（GB 32029）、《污水污物潜水电泵能效限定值及能效等级》（GB 32031）标准中1级能效水平。'\n",
    "text2 = '退耕还林'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d570bf57-2518-4306-a7ae-712e81199460",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.5000]], device='cuda:0')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sim(text1, text2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "040cc794-9bb0-4c22-986c-933ca55ee637",
   "metadata": {},
   "source": [
    "### Process Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d46e4e74-f7c2-4339-b009-4ba77f1b2f9a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>Y</th>\n",
       "      <th>Split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>中新制药厂空调末端送回风系统改造-询价公示</td>\n",
       "      <td>1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平...</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>中新制药厂空调末端送回风系统改造-询价公示</td>\n",
       "      <td>1.5.1 锅炉（窑炉）节能改造和能效提升\\n包括燃煤锅炉“以大代小”，采用先进燃煤锅炉、节...</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      X1                                                 X2  \\\n",
       "0  中新制药厂空调末端送回风系统改造-询价公示  1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平...   \n",
       "1  中新制药厂空调末端送回风系统改造-询价公示  1.5.1 锅炉（窑炉）节能改造和能效提升\\n包括燃煤锅炉“以大代小”，采用先进燃煤锅炉、节...   \n",
       "\n",
       "   Y  Split  \n",
       "0  1  train  \n",
       "1  0  train  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_data = pd.read_excel('AI匹配算法样本.xlsx', sheet_name='Sheet1', dtype=str)\n",
    "df_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "673ce0e0-2801-4bb3-8e5d-5c4aff3ac725",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:1: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n",
      "  train_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='train']\n",
      "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:2: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n",
      "  eval_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='eval']\n",
      "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:3: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n",
      "  test_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='test']\n"
     ]
    }
   ],
   "source": [
    "train_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='train']\n",
    "eval_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='eval']\n",
    "test_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='test']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5037803d-980d-48a1-a61d-528bb9508ce0",
   "metadata": {},
   "source": [
    "### Model 1 - Fine tune a Sentence Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "773429e9-57ce-418f-ad44-3c35d1b31a74",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sentence_transformers import InputExample, losses\n",
    "# from torch.utils.data import DataLoader\n",
    "\n",
    "# # Prepare data\n",
    "# train_data_sbert = []\n",
    "# eval_data_sbert = []\n",
    "# test_data_sbert = []\n",
    "\n",
    "# for item in train_data:\n",
    "#     label = 1.0 if float(item.get('Y')) == 1 else -1.0\n",
    "#     train_data_sbert.append(InputExample(texts=[item.get('X1'), item.get('X2')], label=label))\n",
    "# train_dataloader = DataLoader(train_data_sbert, shuffle=True, batch_size=2)\n",
    "# train_loss = losses.CosineSimilarityLoss(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ec1b68cb-bec3-4896-b196-ec31b1132ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sentence_transformers import evaluation\n",
    "# evaluator = evaluation.EmbeddingSimilarityEvaluator([item.get('X1') for item in eval_data], [item.get('X2') for item in eval_data], [1.0 if float(item.get('Y'))==1 else -1.0 for item in eval_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7c05c6ef-c5e7-416b-b797-9f8735ae5436",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=100, evaluator=evaluator, evaluation_steps=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7de1e5f0-4b83-4d34-8385-77cdaa0ef08f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.save('./tmp_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdd686b1-c654-4135-8989-05f23c914afa",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Model 2 - No Fine Tune + Threshold Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a0247889-577d-4a92-8c0f-9c923748df93",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def sim(text1, text2):\n",
    "    emb1 = model.encode(text1, convert_to_tensor=True)\n",
    "    emb2 = model.encode(text2, convert_to_tensor=True)\n",
    "    score = util.cos_sim(emb1, emb2)\n",
    "    return score\n",
    "\n",
    "def _acc_thres(scores, thres):\n",
    "    correct = 0\n",
    "    total = len(scores)\n",
    "    for score, truth in scores:\n",
    "        truth = float(truth)\n",
    "        pred = 1 if score >= thres else 0\n",
    "        if pred == truth:\n",
    "            correct += 1\n",
    "    return round(correct/total, 3)\n",
    "\n",
    "def model_train(train_data, eval_data):\n",
    "    score_train = []\n",
    "    score_eval = []\n",
    "    for item in tqdm(train_data):\n",
    "        score = sim(item['X1'], item['X2'])\n",
    "        score_train.append((score, item['Y']))\n",
    "    for item in tqdm(eval_data):\n",
    "        score = sim(item['X1'], item['X2'])\n",
    "        score_eval.append((score, item['Y']))\n",
    "    # find threshold that minize train error\n",
    "    score_train = sorted(score_train, reverse=True)\n",
    "    win_acc = -1\n",
    "    win_thres = -1\n",
    "    for thres in range(5, 100, 5):\n",
    "        thres = thres*0.01\n",
    "        acc = _acc_thres(score_train, thres)\n",
    "        if acc > win_acc:\n",
    "            win_acc = acc\n",
    "            win_thres = thres\n",
    "    eval_acc = _acc_thres(score_eval, win_thres)\n",
    "    return {'thres': win_thres, 'train_accuracy': win_acc, 'eval_accuracy': eval_acc}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4e943ef9-ad40-494e-9d53-db9ccbf48bb4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12256/12256 [13:54<00:00, 14.69it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4248/4248 [04:44<00:00, 14.94it/s]\n"
     ]
    }
   ],
   "source": [
    "r = model_train(train_data, eval_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9cd38cc9-fe71-45b9-a22e-977a2e787fb5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'thres': 0.25, 'train_accuracy': 0.831, 'eval_accuracy': 0.816}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "53622ff1-7465-4663-a9f0-0c18df37b93e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4468/4468 [04:58<00:00, 14.98it/s]\n"
     ]
    }
   ],
   "source": [
    "score_test = []\n",
    "for item in tqdm(test_data):\n",
    "    score = sim(item['X1'], item['X2'])\n",
    "    score_test.append((score, item['Y']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "47411f71-c774-4274-a1af-2a128589b559",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.815"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_acc_thres(score_test, r['thres'])\n",
    "#_acc_thres(score_test, 0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59b741bc-7a20-4ed0-bc9d-b82ec3edff34",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}