OuroborosM commited on
Commit
0ded0b0
·
1 Parent(s): 73f1d5f

Update app.py

Browse files

Update doc processing func

Files changed (1) hide show
  1. app.py +7 -6
app.py CHANGED
@@ -131,12 +131,13 @@ def load_documents_2(all_files: List[str] = [], ignored_files: List[str] = []) -
131
  # )
132
  filtered_files = [file_path for file_path in all_files if file_path not in ignored_files]
133
 
134
- with Pool(processes=os.cpu_count()) as pool:
135
- results = []
136
- with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
137
- for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):
138
- results.extend(docs)
139
- pbar.update()
 
140
 
141
  return results
142
 
 
131
  # )
132
  filtered_files = [file_path for file_path in all_files if file_path not in ignored_files]
133
 
134
+
135
+ results = []
136
+ with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
137
+ for file in filtered_files:
138
+ docs = load_single_document(file)
139
+ results.extend(docs)
140
+ pbar.update()
141
 
142
  return results
143