{ "cells": [ { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [], "source": [ "from langchain import PromptTemplate\n", "from langchain.chains import RetrievalQA\n", "from langchain.embeddings import HuggingFaceEmbeddings\n", "from langchain.vectorstores import Pinecone\n", "\n", "import pinecone\n", "from langchain.document_loaders import PyPDFLoader, DirectoryLoader\n", "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", "from langchain_core.prompts import PromptTemplate\n", "from langchain.llms import CTransformers\n", "from tqdm.autonotebook import tqdm" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "PINECONE_API_KEY = \"1bae0d8e-019e-4e87-8080-ecf523e5f25f\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def load_pdf(data):\n", " loader = DirectoryLoader(data, glob=\"*.pdf\", loader_cls=PyPDFLoader)\n", " documents = loader.load()\n", " return documents" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "extracted_data = load_pdf(\"data/\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# Data is extracted from the PDFs\n", "# Now form chunks out of it" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def text_split(extracted_data):\n", " splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=20)\n", " chunks = splitter.split_documents(extracted_data)\n", " return chunks\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "chunks = text_split(extracted_data)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Document(page_content='TheGALE\\nENCYCLOPEDIA\\nofMEDICINE\\nSECOND EDITION', metadata={'source': 'data\\\\Medical_book.pdf', 'page': 1})" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# len(chunks)\n", "chunks[0]\n", "# Chunks formation is done\n", "# Now, convert the chunks into embeddings\n", "# Then Store this embeddings to VectorDB pinecone" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def download_hugging_face_embeddings():\n", " embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n", " return embeddings" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "c:\\Users\\Aniket\\miniconda3\\Lib\\site-packages\\langchain_core\\_api\\deprecation.py:139: LangChainDeprecationWarning: The class `HuggingFaceEmbeddings` was deprecated in LangChain 0.2.2 and will be removed in 0.3.0. An updated version of the class exists in the langchain-huggingface package and should be used instead. To use it run `pip install -U langchain-huggingface` and import as `from langchain_huggingface import HuggingFaceEmbeddings`.\n", " warn_deprecated(\n" ] } ], "source": [ "embeddings = download_hugging_face_embeddings()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "\n", "chunk_embeddings = []\n", "for i in range(0, len(chunks)):\n", " chunk_embeddings.append(embeddings.embed_query(chunks[i].page_content))\n", "\n" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Document(page_content='TheGALE\\nENCYCLOPEDIA\\nofMEDICINE\\nSECOND EDITION', metadata={'source': 'data\\\\Medical_book.pdf', 'page': 1})" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(chunk_embeddings)\n", "chunks[0]" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "chunk_content = [chunks[i].page_content for i in range(0 , len(chunks))]" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "\n", "chunk_ids = [str(i+1) for i in range(0, len(chunks))]\n" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1\n", "[0.0017460489179939032, -0.033502884209156036, -0.03290388733148575, 0.007168094161897898, -0.01460327859967947, 0.010261928662657738, -0.01151528861373663, 0.22930213809013367, -0.023232396692037582, 0.004120402969419956, -0.036560822278261185, 0.08592110127210617, 0.012972140684723854, 0.05221788212656975, -0.10232618451118469, -0.003139043692499399, -0.012686969712376595, 0.000471863109851256, -0.02848585695028305, -0.050259195268154144, 0.01155101228505373, 0.0778065174818039, 0.09282823652029037, -0.0137972766533494, -0.016935130581259727, -0.025955867022275925, -0.04956510663032532, -0.046131301671266556, 0.00729052210226655, -0.013553328812122345, 0.038439445197582245, 0.06280472129583359, 0.018353812396526337, 0.008242843672633171, 0.0017155527602881193, -0.039861857891082764, -0.011638614349067211, 0.016446180641651154, 0.025595590472221375, 0.09104609489440918, 0.029672738164663315, -0.05416030064225197, -0.04576560854911804, -0.013853926211595535, 0.02577359229326248, 0.010323088616132736, -0.05363088846206665, 0.021221553906798363, 0.01702778786420822, 0.11612221598625183, -0.06963177025318146, -0.0957275778055191, -0.03983991965651512, 0.052369263023138046, 0.02526022121310234, -0.031274523586034775, -0.07005152851343155, -0.05956605449318886, -0.09544864296913147, -0.05412125587463379, -0.0002130465436493978, 0.0002552172227296978, 0.012184805236756802, 0.036847226321697235, -0.09168802946805954, -0.016031526029109955, 0.05677250772714615, -0.061103083193302155, 0.05796805024147034, -0.03652450442314148, -0.021421410143375397, -0.047219835221767426, 0.03454083576798439, 0.12064339220523834, -0.013788564130663872, -0.06848538666963577, 0.012004958465695381, -0.059728946536779404, -0.05643806606531143, -0.101061150431633, 0.05889026075601578, -0.02077680453658104, 0.09746569395065308, 0.07813995331525803, -0.035233210772275925, -0.014866253361105919, 0.040357090532779694, 0.07460875064134598, -0.013026430271565914, -0.02844160981476307, 0.10370924323797226, 0.019506312906742096, 0.029694247990846634, 0.007636230438947678, 0.005692535545676947, -0.0007858198950998485, -0.0431533046066761, 0.007750411983579397, -0.01790708862245083, 0.06111792102456093, -0.02531685307621956, -0.10494077950716019, -0.053424857556819916, 0.00989371445029974, 0.01446546521037817, -0.06589170545339584, 0.009222910739481449, -0.13625681400299072, 0.021162521094083786, -0.01161885168403387, 0.034509241580963135, 0.06049511581659317, 0.01565300114452839, -0.012806619517505169, -0.007194896228611469, 0.055682480335235596, 0.07992612570524216, 0.05983540043234825, 0.09587424248456955, 0.018204279243946075, 0.023596445098519325, -0.08910958468914032, -0.007128870114684105, -0.09076816588640213, 0.047233086079359055, 0.004134080838412046, 0.00339791108854115, -2.0226589867935018e-33, 0.015145692974328995, -0.0040165213868021965, 0.046035610139369965, 0.06628156453371048, 0.08750341087579727, 0.03237520158290863, -0.013098624534904957, -0.06530202180147171, 0.0794229656457901, -0.1063862293958664, -0.07034210115671158, 0.03889153152704239, 0.014388104900717735, 0.05448484793305397, -0.1063254252076149, 0.0015779227251186967, -0.07627759128808975, 0.02941804938018322, -0.020254988223314285, -0.010341203771531582, 0.0077161630615592, 0.015365850180387497, -0.030866345390677452, 0.03806672990322113, -0.08467362076044083, 0.061196524649858475, -0.006315906532108784, 0.02108607068657875, 0.0948043167591095, -0.02897580713033676, -0.02469867840409279, -0.02624206617474556, 0.010229134000837803, -0.04816938936710358, -0.05085073038935661, 0.06431731581687927, -0.06250055879354477, -0.017944322898983955, -0.0032577342353761196, -0.002177971415221691, 0.0040510413236916065, 0.05613983795046806, 0.02165955677628517, -0.028459111228585243, 0.06688956916332245, -0.018423588946461678, -0.1386348307132721, -6.533067062264308e-05, 0.09188717603683472, -0.05782890319824219, 0.051801200956106186, -0.0333990603685379, 0.0635690987110138, -0.03353029489517212, 0.003560281591489911, 0.05478646978735924, -0.0592782236635685, 0.02796691469848156, 0.01981506496667862, 0.03064817003905773, 0.09453834593296051, 0.07339683920145035, 0.0220828615128994, -0.034395359456539154, -0.008398751728236675, 0.016077488660812378, -0.1019863560795784, -0.10767035186290741, -0.04845403507351875, -0.04698591306805611, -0.12305164337158203, 0.01794605329632759, 0.02716820128262043, 0.04150307923555374, -0.012137887999415398, 0.016134945675730705, 0.007807416841387749, -0.030696384608745575, -0.03695548698306084, -0.07249006628990173, -0.06594054400920868, -0.052304696291685104, 0.022644832730293274, 0.10688149929046631, -0.03437632694840431, -0.007189977914094925, 0.004040680360049009, -0.012765718623995781, -0.035733193159103394, -0.009521384723484516, -0.037608999758958817, 0.0257247406989336, -0.07330161333084106, 0.018399816006422043, -0.013152354396879673, -2.4391491249173157e-33, -0.036498475819826126, -0.03898495435714722, -0.02466081827878952, 0.05277742072939873, 0.057153843343257904, 0.08485236018896103, -0.047778353095054626, 0.04885183647274971, 0.0936603769659996, 0.024455295875668526, 0.10703857988119125, -0.044878069311380386, 0.013439536094665527, 0.010380103252828121, -0.03215843066573143, 0.03756462037563324, -0.024516936391592026, -0.015364289283752441, -0.08388064056634903, 0.07285160571336746, -0.08349563926458359, 0.05313438922166824, -0.0502142570912838, 0.021773921325802803, 0.09023481607437134, 0.014781351201236248, 0.039709243923425674, -0.01778602786362171, -0.01812116988003254, -0.05736514553427696, -0.014358995482325554, -0.019216574728488922, -0.07116208970546722, -0.03887920826673508, -0.06919412314891815, 0.015993399545550346, 0.03935178369283676, -0.06669557094573975, -0.02723979577422142, -0.041191909462213516, 0.08880122750997543, -0.03152484819293022, 0.020989220589399338, 0.007525313645601273, 0.006864176597446203, -0.00016016913286875933, -0.03395673632621765, -0.01977171003818512, -0.013901437632739544, -0.02347422204911709, 0.05024150386452675, 0.004815816413611174, 0.0030840442050248384, 0.013711294159293175, 0.04268831014633179, -0.010319440625607967, -0.01561632938683033, -0.13140302896499634, 0.03932563588023186, 0.0034621721133589745, 0.004625402390956879, -0.022172948345541954, -0.07998225092887878, 0.07068227231502533, 0.013251686468720436, 0.013216551393270493, -0.031048668548464775, 0.01629941165447235, 0.07485268265008926, -0.020337823778390884, -0.03757909685373306, 0.0027204568032175303, -0.07825025916099548, -0.07273514568805695, -0.0015075246337801218, -0.0015040460275486112, -0.04806789010763168, -0.02269686758518219, 0.00234704976901412, -0.10254422575235367, -0.01062675192952156, -0.05898876115679741, -0.028257478028535843, 0.018367942422628403, -0.02433406002819538, 0.006143547594547272, 0.058125562965869904, 0.006210033316165209, -0.004125489853322506, 0.004883211106061935, -0.051727477461099625, -0.002652254654094577, -0.08568170666694641, 0.04144113138318062, -0.007155539933592081, -2.368167706379154e-08, 0.05613090097904205, -0.014770290814340115, -0.0016940254718065262, -0.0011319591430947185, -0.004112910944968462, -0.07847736030817032, -0.03272825479507446, 0.08840586245059967, -0.009142806753516197, 0.08083547651767731, -0.06821239739656448, 0.08852605521678925, 0.018825415521860123, -0.004044109955430031, 0.01947229728102684, 0.05469341203570366, -0.004570759367197752, 0.050132788717746735, -0.020886147394776344, -0.0043810331262648106, 0.012420092709362507, -0.040249839425086975, -0.017629530280828476, -0.07264230400323868, 0.027827059850096703, 0.026226632297039032, 0.009220139123499393, 0.02573232352733612, -0.009361241944134235, -0.09865128993988037, -0.03130580857396126, 0.051363445818424225, 0.011322762817144394, -0.0496138371527195, -0.04716363921761513, -0.0054739126935601234, 0.06890673190355301, -0.040705155581235886, 0.05423299968242645, -0.019241807982325554, 0.01434497069567442, -0.04443489387631416, -0.02567482925951481, 0.031354621052742004, 0.06115218624472618, -0.026838727295398712, 0.04913513734936714, 0.05644834786653519, 0.027938084676861763, -0.08463852107524872, -0.005661568138748407, 0.007060948293656111, 0.10265117883682251, -0.019236600026488304, -0.08145822584629059, 0.07125057280063629, 0.01803840510547161, -0.017572075128555298, -0.00043016800191253424, -0.10405920445919037, 0.0028415117412805557, -0.00555642182007432, 0.10660912096500397, 0.050997212529182434]\n" ] }, { "data": { "text/plain": [ "'TheGALE\\nENCYCLOPEDIA\\nofMEDICINE\\nSECOND EDITION'" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print(chunk_ids[0])\n", "print(chunk_embeddings[0])\n", "chunk_content[0]" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "index_name = \"medical-chatbot\"\n", "index=pinecone.Index(api_key=PINECONE_API_KEY, host=\"https://medical-chatbot-pv4ded8.svc.aped-4627-b74a.pinecone.io\")\n", "# from http import client\n", "pc = Pinecone(embedding=embeddings, text_key=chunk_content, index=index)\n", "# Replace with your index name\n" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "upsert_vectors = [\n", " {\n", " \"id\": chunk_id,\n", " \"values\": embedding,\n", " \"metadata\": {\"text\": content} # Replace with actual metadata if available\n", " }\n", " for chunk_id, embedding, content in zip(chunk_ids, chunk_embeddings, chunk_content)\n", "]\n" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': '1',\n", " 'values': [0.0017460489179939032,\n", " -0.033502884209156036,\n", " -0.03290388733148575,\n", " 0.007168094161897898,\n", " -0.01460327859967947,\n", " 0.010261928662657738,\n", " -0.01151528861373663,\n", " 0.22930213809013367,\n", " -0.023232396692037582,\n", " 0.004120402969419956,\n", " -0.036560822278261185,\n", " 0.08592110127210617,\n", " 0.012972140684723854,\n", " 0.05221788212656975,\n", " -0.10232618451118469,\n", " -0.003139043692499399,\n", " -0.012686969712376595,\n", " 0.000471863109851256,\n", " -0.02848585695028305,\n", " -0.050259195268154144,\n", " 0.01155101228505373,\n", " 0.0778065174818039,\n", " 0.09282823652029037,\n", " -0.0137972766533494,\n", " -0.016935130581259727,\n", " -0.025955867022275925,\n", " -0.04956510663032532,\n", " -0.046131301671266556,\n", " 0.00729052210226655,\n", " -0.013553328812122345,\n", " 0.038439445197582245,\n", " 0.06280472129583359,\n", " 0.018353812396526337,\n", " 0.008242843672633171,\n", " 0.0017155527602881193,\n", " -0.039861857891082764,\n", " -0.011638614349067211,\n", " 0.016446180641651154,\n", " 0.025595590472221375,\n", " 0.09104609489440918,\n", " 0.029672738164663315,\n", " -0.05416030064225197,\n", " -0.04576560854911804,\n", " -0.013853926211595535,\n", " 0.02577359229326248,\n", " 0.010323088616132736,\n", " -0.05363088846206665,\n", " 0.021221553906798363,\n", " 0.01702778786420822,\n", " 0.11612221598625183,\n", " -0.06963177025318146,\n", " -0.0957275778055191,\n", " -0.03983991965651512,\n", " 0.052369263023138046,\n", " 0.02526022121310234,\n", " -0.031274523586034775,\n", " -0.07005152851343155,\n", " -0.05956605449318886,\n", " -0.09544864296913147,\n", " -0.05412125587463379,\n", " -0.0002130465436493978,\n", " 0.0002552172227296978,\n", " 0.012184805236756802,\n", " 0.036847226321697235,\n", " -0.09168802946805954,\n", " -0.016031526029109955,\n", " 0.05677250772714615,\n", " -0.061103083193302155,\n", " 0.05796805024147034,\n", " -0.03652450442314148,\n", " -0.021421410143375397,\n", " -0.047219835221767426,\n", " 0.03454083576798439,\n", " 0.12064339220523834,\n", " -0.013788564130663872,\n", " -0.06848538666963577,\n", " 0.012004958465695381,\n", " -0.059728946536779404,\n", " -0.05643806606531143,\n", " -0.101061150431633,\n", " 0.05889026075601578,\n", " -0.02077680453658104,\n", " 0.09746569395065308,\n", " 0.07813995331525803,\n", " -0.035233210772275925,\n", " -0.014866253361105919,\n", " 0.040357090532779694,\n", " 0.07460875064134598,\n", " -0.013026430271565914,\n", " -0.02844160981476307,\n", " 0.10370924323797226,\n", " 0.019506312906742096,\n", " 0.029694247990846634,\n", " 0.007636230438947678,\n", " 0.005692535545676947,\n", " -0.0007858198950998485,\n", " -0.0431533046066761,\n", " 0.007750411983579397,\n", " -0.01790708862245083,\n", " 0.06111792102456093,\n", " -0.02531685307621956,\n", " -0.10494077950716019,\n", " -0.053424857556819916,\n", " 0.00989371445029974,\n", " 0.01446546521037817,\n", " -0.06589170545339584,\n", " 0.009222910739481449,\n", " -0.13625681400299072,\n", " 0.021162521094083786,\n", " -0.01161885168403387,\n", " 0.034509241580963135,\n", " 0.06049511581659317,\n", " 0.01565300114452839,\n", " -0.012806619517505169,\n", " -0.007194896228611469,\n", " 0.055682480335235596,\n", " 0.07992612570524216,\n", " 0.05983540043234825,\n", " 0.09587424248456955,\n", " 0.018204279243946075,\n", " 0.023596445098519325,\n", " -0.08910958468914032,\n", " -0.007128870114684105,\n", " -0.09076816588640213,\n", " 0.047233086079359055,\n", " 0.004134080838412046,\n", " 0.00339791108854115,\n", " -2.0226589867935018e-33,\n", " 0.015145692974328995,\n", " -0.0040165213868021965,\n", " 0.046035610139369965,\n", " 0.06628156453371048,\n", " 0.08750341087579727,\n", " 0.03237520158290863,\n", " -0.013098624534904957,\n", " -0.06530202180147171,\n", " 0.0794229656457901,\n", " -0.1063862293958664,\n", " -0.07034210115671158,\n", " 0.03889153152704239,\n", " 0.014388104900717735,\n", " 0.05448484793305397,\n", " -0.1063254252076149,\n", " 0.0015779227251186967,\n", " -0.07627759128808975,\n", " 0.02941804938018322,\n", " -0.020254988223314285,\n", " -0.010341203771531582,\n", " 0.0077161630615592,\n", " 0.015365850180387497,\n", " -0.030866345390677452,\n", " 0.03806672990322113,\n", " -0.08467362076044083,\n", " 0.061196524649858475,\n", " -0.006315906532108784,\n", " 0.02108607068657875,\n", " 0.0948043167591095,\n", " -0.02897580713033676,\n", " -0.02469867840409279,\n", " -0.02624206617474556,\n", " 0.010229134000837803,\n", " -0.04816938936710358,\n", " -0.05085073038935661,\n", " 0.06431731581687927,\n", " -0.06250055879354477,\n", " -0.017944322898983955,\n", " -0.0032577342353761196,\n", " -0.002177971415221691,\n", " 0.0040510413236916065,\n", " 0.05613983795046806,\n", " 0.02165955677628517,\n", " -0.028459111228585243,\n", " 0.06688956916332245,\n", " -0.018423588946461678,\n", " -0.1386348307132721,\n", " -6.533067062264308e-05,\n", " 0.09188717603683472,\n", " -0.05782890319824219,\n", " 0.051801200956106186,\n", " -0.0333990603685379,\n", " 0.0635690987110138,\n", " -0.03353029489517212,\n", " 0.003560281591489911,\n", " 0.05478646978735924,\n", " -0.0592782236635685,\n", " 0.02796691469848156,\n", " 0.01981506496667862,\n", " 0.03064817003905773,\n", " 0.09453834593296051,\n", " 0.07339683920145035,\n", " 0.0220828615128994,\n", " -0.034395359456539154,\n", " -0.008398751728236675,\n", " 0.016077488660812378,\n", " -0.1019863560795784,\n", " -0.10767035186290741,\n", " -0.04845403507351875,\n", " -0.04698591306805611,\n", " -0.12305164337158203,\n", " 0.01794605329632759,\n", " 0.02716820128262043,\n", " 0.04150307923555374,\n", " -0.012137887999415398,\n", " 0.016134945675730705,\n", " 0.007807416841387749,\n", " -0.030696384608745575,\n", " -0.03695548698306084,\n", " -0.07249006628990173,\n", " -0.06594054400920868,\n", " -0.052304696291685104,\n", " 0.022644832730293274,\n", " 0.10688149929046631,\n", " -0.03437632694840431,\n", " -0.007189977914094925,\n", " 0.004040680360049009,\n", " -0.012765718623995781,\n", " -0.035733193159103394,\n", " -0.009521384723484516,\n", " -0.037608999758958817,\n", " 0.0257247406989336,\n", " -0.07330161333084106,\n", " 0.018399816006422043,\n", " -0.013152354396879673,\n", " -2.4391491249173157e-33,\n", " -0.036498475819826126,\n", " -0.03898495435714722,\n", " -0.02466081827878952,\n", " 0.05277742072939873,\n", " 0.057153843343257904,\n", " 0.08485236018896103,\n", " -0.047778353095054626,\n", " 0.04885183647274971,\n", " 0.0936603769659996,\n", " 0.024455295875668526,\n", " 0.10703857988119125,\n", " -0.044878069311380386,\n", " 0.013439536094665527,\n", " 0.010380103252828121,\n", " -0.03215843066573143,\n", " 0.03756462037563324,\n", " -0.024516936391592026,\n", " -0.015364289283752441,\n", " -0.08388064056634903,\n", " 0.07285160571336746,\n", " -0.08349563926458359,\n", " 0.05313438922166824,\n", " -0.0502142570912838,\n", " 0.021773921325802803,\n", " 0.09023481607437134,\n", " 0.014781351201236248,\n", " 0.039709243923425674,\n", " -0.01778602786362171,\n", " -0.01812116988003254,\n", " -0.05736514553427696,\n", " -0.014358995482325554,\n", " -0.019216574728488922,\n", " -0.07116208970546722,\n", " -0.03887920826673508,\n", " -0.06919412314891815,\n", " 0.015993399545550346,\n", " 0.03935178369283676,\n", " -0.06669557094573975,\n", " -0.02723979577422142,\n", " -0.041191909462213516,\n", " 0.08880122750997543,\n", " -0.03152484819293022,\n", " 0.020989220589399338,\n", " 0.007525313645601273,\n", " 0.006864176597446203,\n", " -0.00016016913286875933,\n", " -0.03395673632621765,\n", " -0.01977171003818512,\n", " -0.013901437632739544,\n", " -0.02347422204911709,\n", " 0.05024150386452675,\n", " 0.004815816413611174,\n", " 0.0030840442050248384,\n", " 0.013711294159293175,\n", " 0.04268831014633179,\n", " -0.010319440625607967,\n", " -0.01561632938683033,\n", " -0.13140302896499634,\n", " 0.03932563588023186,\n", " 0.0034621721133589745,\n", " 0.004625402390956879,\n", " -0.022172948345541954,\n", " -0.07998225092887878,\n", " 0.07068227231502533,\n", " 0.013251686468720436,\n", " 0.013216551393270493,\n", " -0.031048668548464775,\n", " 0.01629941165447235,\n", " 0.07485268265008926,\n", " -0.020337823778390884,\n", " -0.03757909685373306,\n", " 0.0027204568032175303,\n", " -0.07825025916099548,\n", " -0.07273514568805695,\n", " -0.0015075246337801218,\n", " -0.0015040460275486112,\n", " -0.04806789010763168,\n", " -0.02269686758518219,\n", " 0.00234704976901412,\n", " -0.10254422575235367,\n", " -0.01062675192952156,\n", " -0.05898876115679741,\n", " -0.028257478028535843,\n", " 0.018367942422628403,\n", " -0.02433406002819538,\n", " 0.006143547594547272,\n", " 0.058125562965869904,\n", " 0.006210033316165209,\n", " -0.004125489853322506,\n", " 0.004883211106061935,\n", " -0.051727477461099625,\n", " -0.002652254654094577,\n", " -0.08568170666694641,\n", " 0.04144113138318062,\n", " -0.007155539933592081,\n", " -2.368167706379154e-08,\n", " 0.05613090097904205,\n", " -0.014770290814340115,\n", " -0.0016940254718065262,\n", " -0.0011319591430947185,\n", " -0.004112910944968462,\n", " -0.07847736030817032,\n", " -0.03272825479507446,\n", " 0.08840586245059967,\n", " -0.009142806753516197,\n", " 0.08083547651767731,\n", " -0.06821239739656448,\n", " 0.08852605521678925,\n", " 0.018825415521860123,\n", " -0.004044109955430031,\n", " 0.01947229728102684,\n", " 0.05469341203570366,\n", " -0.004570759367197752,\n", " 0.050132788717746735,\n", " -0.020886147394776344,\n", " -0.0043810331262648106,\n", " 0.012420092709362507,\n", " -0.040249839425086975,\n", " -0.017629530280828476,\n", " -0.07264230400323868,\n", " 0.027827059850096703,\n", " 0.026226632297039032,\n", " 0.009220139123499393,\n", " 0.02573232352733612,\n", " -0.009361241944134235,\n", " -0.09865128993988037,\n", " -0.03130580857396126,\n", " 0.051363445818424225,\n", " 0.011322762817144394,\n", " -0.0496138371527195,\n", " -0.04716363921761513,\n", " -0.0054739126935601234,\n", " 0.06890673190355301,\n", " -0.040705155581235886,\n", " 0.05423299968242645,\n", " -0.019241807982325554,\n", " 0.01434497069567442,\n", " -0.04443489387631416,\n", " -0.02567482925951481,\n", " 0.031354621052742004,\n", " 0.06115218624472618,\n", " -0.026838727295398712,\n", " 0.04913513734936714,\n", " 0.05644834786653519,\n", " 0.027938084676861763,\n", " -0.08463852107524872,\n", " -0.005661568138748407,\n", " 0.007060948293656111,\n", " 0.10265117883682251,\n", " -0.019236600026488304,\n", " -0.08145822584629059,\n", " 0.07125057280063629,\n", " 0.01803840510547161,\n", " -0.017572075128555298,\n", " -0.00043016800191253424,\n", " -0.10405920445919037,\n", " 0.0028415117412805557,\n", " -0.00555642182007432,\n", " 0.10660912096500397,\n", " 0.050997212529182434],\n", " 'metadata': {'text': 'TheGALE\\nENCYCLOPEDIA\\nofMEDICINE\\nSECOND EDITION'}}" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "upsert_vectors[0]" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "# docsearch = Pinecone.from_texts([t.page_content for t in chunks], embeddings, index_name)\n", "\n", "# Upsert the chunks into Pinecone\n", "# index.upsert(vectors=upsert_vectors)\n", "batch_size = 500 # Adjust as necessary based on your data size and Pinecone limits\n", "for i in range(0, len(upsert_vectors), batch_size):\n", " batch_vectors = upsert_vectors[i:i + batch_size]\n", " index.upsert(vectors=batch_vectors)\n", "\n" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "384\n" ] } ], "source": [ "query_embedding = embeddings.embed_query(\"What are allergies\")\n", "print(len(query_embedding))\n", "# Perform query to retrieve similar vectors\n", "results = index.query(vector=[query_embedding], top_k=3, include_values=False, include_metadata=True)\n" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'matches': [{'id': '1373',\n", " 'metadata': {'text': 'GALE ENCYCLOPEDIA OF MEDICINE 2 '\n", " '117Allergies\\n'\n", " 'Allergic rhinitis is commonly triggered '\n", " 'by\\n'\n", " 'exposure to household dust, animal fur,or '\n", " 'pollen. The foreign substance thattriggers '\n", " 'an allergic reaction is calledan '\n", " 'allergen.\\n'\n", " 'The presence of an allergen causes the\\n'\n", " \"body's lymphocytes to begin producingIgE \"\n", " 'antibodies. The lymphocytes of an allergy '\n", " 'sufferer produce an unusuallylarge amount '\n", " 'of IgE.\\n'\n", " 'IgE molecules attach to mast\\n'\n", " 'cells, which contain '\n", " 'histamine.HistaminePollen grains\\n'\n", " 'Lymphocyte\\n'\n", " 'FIRST EXPOSURE'},\n", " 'score': 0.682266653,\n", " 'values': []},\n", " {'id': '1356',\n", " 'metadata': {'text': 'allergens are the following:\\n'\n", " '• plant pollens\\n'\n", " '• animal fur and dander\\n'\n", " '• body parts from house mites (microscopic '\n", " 'creatures\\n'\n", " 'found in all houses)\\n'\n", " '• house dust• mold spores• cigarette '\n", " 'smoke• solvents• cleaners\\n'\n", " 'Common food allergens include the '\n", " 'following:\\n'\n", " '• nuts, especially peanuts, walnuts, and '\n", " 'brazil nuts\\n'\n", " '• fish, mollusks, and shellfish• eggs• '\n", " 'wheat• milk• food additives and '\n", " 'preservatives\\n'\n", " 'The following types of drugs commonly '\n", " 'cause aller-\\n'\n", " 'gic reactions:\\n'\n", " '• penicillin or other antibiotics'},\n", " 'score': 0.678239942,\n", " 'values': []},\n", " {'id': '1306',\n", " 'metadata': {'text': 'itchy, scratchy nose, eyes, and throat '\n", " 'common in aller-gic rhinitis.\\n'\n", " 'The number of possible airborne allergens '\n", " 'is enor-'},\n", " 'score': 0.676807582,\n", " 'values': []}],\n", " 'namespace': '',\n", " 'usage': {'read_units': 6}}\n" ] } ], "source": [ "print(results)" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['1373', '1356', '1306']\n" ] } ], "source": [ "matched_ids = [match['id'] for match in results['matches']]\n", "print(matched_ids)" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'mous. Seasonal AR is most commonly caused by grassand tree pollens, since their pollen is produced in largeamounts and is dispersed by the wind. Showy flowers,like roses or lilacs, that attract insects produce a stickypollen which is less likely to become airborne. Differentplants release their pollen at different times of the year,so seasonal AR sufferers may be most affected in spring,summer, or fall, depending on which plants provoke aresponse. The amount of pollen in the air is reflected'" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "chunks[1306].page_content\n", "# Now, based on these top results, I will send it to the llm and it will return the appropriate answer" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [], "source": [ "prompt_template = \"\"\"\n", "Use the following pieces of information to answer the user's question.\n", "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n", "\n", "Context: {context}\n", "Question: {question}\n", "\n", "Only return the helpful answer below nothing else.\n", "Helpful Answer: \n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [], "source": [ "PROMPT = PromptTemplate(template = prompt_template, input_variables=[\"context\", \"question\"])\n", "chain_type_kwargs = {\"prompt\":PROMPT}" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [], "source": [ "llm = CTransformers(model=\"model/llama-2-7b-chat.ggmlv3.q4_0.bin\", model_type=\"llama\", config={'max_new_tokens': 512, 'temperature': 1})" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [], "source": [ "# Create Pinecone retriever\n", "vector_store = Pinecone(index, embeddings, text_key=\"text\")\n" ] }, { "cell_type": "code", "execution_count": 76, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[Document(page_content='tively or by altering the skin of the scalp. One exampleis thyroid disorders. Hyperthyroidism (too much thy-\\nroid hormone) causes hair to become thin and fine.\\nGALE ENCYCLOPEDIA OF MEDICINE 2 125Alopecia\\nTop of balding male’s head. (Photograph by Kelly A. Quin.\\nReproduced by permission.)GEM - 0001 to 0432 - A 10/22/03 1:42 PM Page 125'), Document(page_content='plugs of skin, each containing one to several hairs,from the back side of the scalp. The bald sections arethen implanted with the plugs. Research completed in2000 looked at the new technique of hair grafting, andfound that micrografts (one to two hairs transplantedper follicle) resulted in fewer complications and thebest results\\nAnother surgical procedure used to treat androgenic'), Document(page_content='multitude of hair replacement methods performed byboth physicians and non-physicians. They range fromsimply weaving someone else’s hair in with the remainsof your own to surgically transplanting thousands of hairfollicles one at a time.\\nHair transplantation is completed by taking tiny')]\n" ] } ], "source": [ "\n", "# Example query\n", "query = \"How to Strengthen Hairs?\"\n", "answer = vector_store.similarity_search(query, k=3)\n", "print(answer)" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [], "source": [ "\n", "qa = RetrievalQA.from_chain_type(llm, chain_type=\"stuff\",retriever = vector_store.as_retriever(search_kwargs={\"k\": 2}), chain_type_kwargs=chain_type_kwargs)" ] }, { "cell_type": "code", "execution_count": 77, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'query': 'How to cure AIDS?', 'result': 'Unfortunately, there is no known cure for HIV or AIDS at this time. While advances have been made in treating the symptoms and slowing the progression of the disease, a cure has not yet been discovered. Research continues to be conducted on new treatments and potential cures, but as of now, there is no known way to completely eliminate the virus from the body or to restore the immune system to its full function.'}\n" ] } ], "source": [ "# Example query\n", "query = \"How to cure AIDS?\"\n", "answer = qa.invoke({\"query\":query})\n", "print(answer)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 2 }