Spaces:
Runtime error
Runtime error
Commit
·
dfc2f0a
1
Parent(s):
1757d87
update chunk_size
Browse files
app.py
CHANGED
@@ -355,7 +355,7 @@ def process_documents_3(ignored_files: List[str] = []) -> List[Document]:
|
|
355 |
print("No new documents to load")
|
356 |
exit(0)
|
357 |
print(f"Loaded {len(documents)} new documents from {source_directory}")
|
358 |
-
text_splitter = RecursiveCharacterTextSplitter(chunk_size=
|
359 |
texts = text_splitter.split_documents(documents)
|
360 |
print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
|
361 |
return texts
|
|
|
355 |
print("No new documents to load")
|
356 |
exit(0)
|
357 |
print(f"Loaded {len(documents)} new documents from {source_directory}")
|
358 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=8000, chunk_overlap=1000)
|
359 |
texts = text_splitter.split_documents(documents)
|
360 |
print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
|
361 |
return texts
|