Koshti10 commited on
Commit
0232f80
·
verified ·
1 Parent(s): 56dc125

Delete dnd_database.py

Browse files
Files changed (1) hide show
  1. dnd_database.py +0 -26
dnd_database.py DELETED
@@ -1,26 +0,0 @@
1
- from driveapi.drive import process_pdf
2
-
3
- from langchain.embeddings.openai import OpenAIEmbeddings
4
- from langchain.text_splitter import CharacterTextSplitter
5
- from langchain.vectorstores import FAISS
6
-
7
- def create_dnd_database(file_list):
8
- raw_text = ''
9
- if file_list is None:
10
- return None
11
- for pdf in file_list:
12
- raw_text += process_pdf(pdf)
13
-
14
- embedding = OpenAIEmbeddings()
15
-
16
- text_splitter = CharacterTextSplitter(
17
- separator = "\n",
18
- chunk_size = 1000,
19
- chunk_overlap = 200,
20
- length_function = len,
21
- )
22
- texts = text_splitter.split_text(raw_text)
23
- print('Length of text: ' + str(len(raw_text)))
24
- db = FAISS.from_texts(texts, embedding)
25
-
26
- return db