Spaces:

GIZ
/

audit_assistant

Running on CPU Upgrade

ppsingh commited on Jul 23

Commit

1f81843

•

1 Parent(s): 9ca031b

Update auditqa/doc_process.py

Files changed (1) hide show

auditqa/doc_process.py CHANGED Viewed

@@ -27,7 +27,7 @@ def process_pdf():
     # text splitter based on the tokenizer of a model of your choosing
     # to make texts fit exactly a transformer's context window size
     # langchain text splitters: https://python.langchain.com/docs/modules/data_connection/document_transformers/
-    chunk_size = 512
     text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
             AutoTokenizer.from_pretrained("BAAI/bge-small-en-v1.5"),
             chunk_size=chunk_size,

     # text splitter based on the tokenizer of a model of your choosing
     # to make texts fit exactly a transformer's context window size
     # langchain text splitters: https://python.langchain.com/docs/modules/data_connection/document_transformers/
+    chunk_size = 256
     text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
             AutoTokenizer.from_pretrained("BAAI/bge-small-en-v1.5"),
             chunk_size=chunk_size,