Commit
•
e63c1d7
1
Parent(s):
f09c5f3
Fixes "invalid url" error in websearch (#675)
Browse files* Fixed #656
* lint
---------
Co-authored-by: muhammadbaqir1327 <muhammad.baqir@arbisoft.com>
Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>
src/lib/server/websearch/runWebSearch.ts
CHANGED
@@ -48,11 +48,17 @@ export async function runWebSearch(
|
|
48 |
webSearch.results =
|
49 |
(results.organic_results &&
|
50 |
results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
|
51 |
-
|
52 |
-
|
53 |
-
|
|
|
|
|
|
|
|
|
|
|
54 |
})) ??
|
55 |
[];
|
|
|
56 |
webSearch.results = webSearch.results
|
57 |
.filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
|
58 |
.slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only
|
|
|
48 |
webSearch.results =
|
49 |
(results.organic_results &&
|
50 |
results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
|
51 |
+
try {
|
52 |
+
const { title, link, text } = el;
|
53 |
+
const { hostname } = new URL(link);
|
54 |
+
return { title, link, hostname, text };
|
55 |
+
} catch (e) {
|
56 |
+
// Ignore Errors
|
57 |
+
return null;
|
58 |
+
}
|
59 |
})) ??
|
60 |
[];
|
61 |
+
webSearch.results = webSearch.results.filter((value) => value !== null);
|
62 |
webSearch.results = webSearch.results
|
63 |
.filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
|
64 |
.slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only
|