Spaces:
Runtime error
Runtime error
neon_arch
Sabrina Jewson
commited on
Commit
•
2f01651
1
Parent(s):
0781385
✨ feat: fix bugs and add code to handle engine selections
Browse filesCo-authored-by: Sabrina Jewson <58880148+SabrinaJewson@users.noreply.github.com>
src/search_results_handler/aggregator.rs
CHANGED
@@ -59,26 +59,35 @@ pub async fn aggregate(
|
|
59 |
}
|
60 |
|
61 |
// fetch results from upstream search engines simultaneously/concurrently.
|
62 |
-
let search_engines: Vec<Box<dyn SearchEngine>> = upstream_search_engines
|
63 |
.iter()
|
64 |
.map(|engine| match engine.to_lowercase().as_str() {
|
65 |
-
"duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine>,
|
66 |
-
"searx
|
|
|
67 |
})
|
68 |
.collect();
|
69 |
|
|
|
|
|
70 |
let tasks: Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>> =
|
71 |
search_engines
|
72 |
-
.
|
73 |
.map(|search_engine| {
|
74 |
-
|
|
|
|
|
|
|
|
|
75 |
})
|
76 |
.collect();
|
77 |
|
78 |
-
let mut outputs = Vec::with_capacity(
|
79 |
|
80 |
for task in tasks {
|
81 |
-
|
|
|
|
|
82 |
}
|
83 |
|
84 |
let mut initial: bool = true;
|
@@ -87,8 +96,7 @@ pub async fn aggregate(
|
|
87 |
if initial {
|
88 |
match results {
|
89 |
Some(result) => {
|
90 |
-
|
91 |
-
result_map.extend(new_result.as_ref().unwrap().clone());
|
92 |
counter += 1;
|
93 |
initial = false
|
94 |
}
|
@@ -105,27 +113,21 @@ pub async fn aggregate(
|
|
105 |
} else {
|
106 |
match results {
|
107 |
Some(result) => {
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
|
122 |
-
|
123 |
-
value.visiting_url.clone(),
|
124 |
-
value.description.clone(),
|
125 |
-
value.engine.clone(),
|
126 |
-
)
|
127 |
-
});
|
128 |
-
});
|
129 |
counter += 1
|
130 |
}
|
131 |
None => {
|
|
|
59 |
}
|
60 |
|
61 |
// fetch results from upstream search engines simultaneously/concurrently.
|
62 |
+
let search_engines: Vec<Box<dyn SearchEngine + Send + Sync>> = upstream_search_engines
|
63 |
.iter()
|
64 |
.map(|engine| match engine.to_lowercase().as_str() {
|
65 |
+
"duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine + Send + Sync>,
|
66 |
+
"searx" => Box::new(searx::Searx) as Box<dyn SearchEngine + Send + Sync>,
|
67 |
+
&_ => panic!("Config Error: Incorrect config file option provided"),
|
68 |
})
|
69 |
.collect();
|
70 |
|
71 |
+
let task_capacity: usize = search_engines.len();
|
72 |
+
|
73 |
let tasks: Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>> =
|
74 |
search_engines
|
75 |
+
.into_iter()
|
76 |
.map(|search_engine| {
|
77 |
+
let query: String = query.clone();
|
78 |
+
let user_agent: String = user_agent.clone();
|
79 |
+
tokio::spawn(
|
80 |
+
async move { search_engine.results(query, page, user_agent.clone()).await },
|
81 |
+
)
|
82 |
})
|
83 |
.collect();
|
84 |
|
85 |
+
let mut outputs = Vec::with_capacity(task_capacity);
|
86 |
|
87 |
for task in tasks {
|
88 |
+
if let Ok(result) = task.await {
|
89 |
+
outputs.push(result.ok())
|
90 |
+
}
|
91 |
}
|
92 |
|
93 |
let mut initial: bool = true;
|
|
|
96 |
if initial {
|
97 |
match results {
|
98 |
Some(result) => {
|
99 |
+
result_map.extend(result.clone());
|
|
|
100 |
counter += 1;
|
101 |
initial = false
|
102 |
}
|
|
|
113 |
} else {
|
114 |
match results {
|
115 |
Some(result) => {
|
116 |
+
result.clone().into_iter().for_each(|(key, value)| {
|
117 |
+
result_map
|
118 |
+
.entry(key)
|
119 |
+
.and_modify(|result| {
|
120 |
+
result.add_engines(value.clone().engine());
|
121 |
+
})
|
122 |
+
.or_insert_with(|| -> RawSearchResult {
|
123 |
+
RawSearchResult::new(
|
124 |
+
value.title.clone(),
|
125 |
+
value.visiting_url.clone(),
|
126 |
+
value.description.clone(),
|
127 |
+
value.engine.clone(),
|
128 |
+
)
|
129 |
+
});
|
130 |
+
});
|
|
|
|
|
|
|
|
|
|
|
|
|
131 |
counter += 1
|
132 |
}
|
133 |
None => {
|