Pinecone / utils.py
Ganesh43's picture
Create utils.py
5e06873 verified
from langchain.text_splitter import RecursiveCharacterTextSplitter
#The below import has been replaced by the later one
#from langchain.vectorstores import Pinecone
from langchain_community.vectorstores import Pinecone
from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
#Pinecone as made some changes recently and we have to import it in the below way from now on :)
from pinecone import Pinecone as PineconeClient
import asyncio
from langchain.document_loaders.sitemap import SitemapLoader
#Function to fetch data from website
#https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/sitemap
sitemap_url="https://www.accenture.com/in-en/careers/jobsearch?jk=Data&sb=0&vw=1&is_rj=0&pg=1&jt=entry-level%20job"
def get_website_data(sitemap_url):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loader = SitemapLoader(sitemap_url)
docs = loader.load()
return docs
d=get_website_data(sitemap_url)
print(len(d))
#Function to split data into smaller chunks
def split_data(docs):
text_splitter = RecursiveCharacterTextSplitter(
chunk_size = 1000,
chunk_overlap = 200,
length_function = len,
)
docs_chunks = text_splitter.split_documents(docs)
return docs_chunks
#Function to create embeddings instance
def create_embeddings():
embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
return embeddings
#Function to push data to Pinecone
def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs):
PineconeClient(
api_key=pinecone_apikey,
environment=pinecone_environment
)
index_name = pinecone_index_name
#PineconeStore is an alias name of Pinecone class, please look at the imports section at the top :)
index = Pinecone.from_documents(docs, embeddings, index_name=index_name)
return index
#Function to pull index data from Pinecone
def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings):
PineconeClient(
api_key=pinecone_apikey,
environment=pinecone_environment
)
index_name = pinecone_index_name
#PineconeStore is an alias name of Pinecone class, please look at the imports section at the top :)
index = Pinecone.from_existing_index(index_name, embeddings)
return index
#This function will help us in fetching the top relevent documents from our vector store - Pinecone Index
def get_similar_docs(index,query,k=2):
similar_docs = index.similarity_search(query, k=k)
return similar_docs