neon_arch
commited on
Commit
·
44216e4
1
Parent(s):
9c71c9f
✨ feat: optimise search results filtering code (#163)
Browse files
src/results/aggregator.rs
CHANGED
@@ -1,7 +1,11 @@
|
|
1 |
//! This module provides the functionality to scrape and gathers all the results from the upstream
|
2 |
//! search engines and then removes duplicate results.
|
3 |
|
4 |
-
use std::{
|
|
|
|
|
|
|
|
|
5 |
|
6 |
use super::{
|
7 |
aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
|
@@ -176,10 +180,10 @@ fn filter_with_lists(
|
|
176 |
resultant_map: &mut HashMap<String, SearchResult>,
|
177 |
file_path: &str,
|
178 |
) -> Result<(), Box<dyn std::error::Error>> {
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
if re.is_match(&url.to_lowercase())
|
184 |
|| re.is_match(&search_result.title.to_lowercase())
|
185 |
|| re.is_match(&search_result.description.to_lowercase())
|
|
|
1 |
//! This module provides the functionality to scrape and gathers all the results from the upstream
|
2 |
//! search engines and then removes duplicate results.
|
3 |
|
4 |
+
use std::{
|
5 |
+
collections::HashMap,
|
6 |
+
io::{BufReader, Read},
|
7 |
+
time::Duration,
|
8 |
+
};
|
9 |
|
10 |
use super::{
|
11 |
aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
|
|
|
180 |
resultant_map: &mut HashMap<String, SearchResult>,
|
181 |
file_path: &str,
|
182 |
) -> Result<(), Box<dyn std::error::Error>> {
|
183 |
+
let mut reader = BufReader::new(File::open(file_path)?);
|
184 |
+
for line in reader.by_ref().lines() {
|
185 |
+
let re = Regex::new(&line?)?;
|
186 |
+
for (url, search_result) in map_to_be_filtered.clone().into_iter() {
|
187 |
if re.is_match(&url.to_lowercase())
|
188 |
|| re.is_match(&search_result.title.to_lowercase())
|
189 |
|| re.is_match(&search_result.description.to_lowercase())
|