mery22 commited on
Commit
b956157
1 Parent(s): 1ebe7bf

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +12 -2
app.py CHANGED
@@ -64,12 +64,22 @@ bnb_config = BitsAndBytesConfig(
64
  model = AutoModelForCausalLM.from_pretrained(
65
  "mistralai/Mistral-7B-Instruct-v0.1",quantization_config=bnb_config,
66
  )
67
- new_db = FAISS.load_local("faiss_index", HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L12-v2'),allow_dangerous_deserialization=True)
 
 
 
 
 
 
 
 
68
  # Connect query to FAISS index using a retriever
69
- retriever = new_db.as_retriever(
70
  search_type="mmr",
71
  search_kwargs={'k': 1}
72
  )
 
 
73
  from langchain.llms import HuggingFacePipeline
74
  from langchain.prompts import PromptTemplate
75
  from langchain.embeddings.huggingface import HuggingFaceEmbeddings
 
64
  model = AutoModelForCausalLM.from_pretrained(
65
  "mistralai/Mistral-7B-Instruct-v0.1",quantization_config=bnb_config,
66
  )
67
+ dataset= load_dataset("mery22/testub")
68
+ loader = PyPDFLoader(dataset)
69
+ data = loader.load()
70
+ text_splitter1 = CharacterTextSplitter(chunk_size=512, chunk_overlap=0,separator="\n\n")
71
+ texts = text_splitter1.split_documents(data)
72
+ db = FAISS.from_documents(texts,
73
+ HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L12-v2'))
74
+
75
+
76
  # Connect query to FAISS index using a retriever
77
+ retriever = db.as_retriever(
78
  search_type="mmr",
79
  search_kwargs={'k': 1}
80
  )
81
+
82
+
83
  from langchain.llms import HuggingFacePipeline
84
  from langchain.prompts import PromptTemplate
85
  from langchain.embeddings.huggingface import HuggingFaceEmbeddings