OuroborosM commited on
Commit
dfc2f0a
·
1 Parent(s): 1757d87

update chunk_size

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -355,7 +355,7 @@ def process_documents_3(ignored_files: List[str] = []) -> List[Document]:
355
  print("No new documents to load")
356
  exit(0)
357
  print(f"Loaded {len(documents)} new documents from {source_directory}")
358
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=500)
359
  texts = text_splitter.split_documents(documents)
360
  print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
361
  return texts
 
355
  print("No new documents to load")
356
  exit(0)
357
  print(f"Loaded {len(documents)} new documents from {source_directory}")
358
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=8000, chunk_overlap=1000)
359
  texts = text_splitter.split_documents(documents)
360
  print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
361
  return texts