nsarrazin's picture
nsarrazin HF staff
Backend refactor for streaming endpoints (#444)
e943a05 unverified
import { JSDOM, VirtualConsole } from "jsdom";
export async function parseWeb(url: string) {
const abortController = new AbortController();
setTimeout(() => abortController.abort(), 10000);
const htmlString = await fetch(url, { signal: abortController.signal })
.then((response) => response.text())
.catch();
const virtualConsole = new VirtualConsole();
virtualConsole.on("error", () => {
// No-op to skip console errors.
});
// put the html string into a DOM
const dom = new JSDOM(htmlString ?? "", {
virtualConsole,
});
const { document } = dom.window;
const textElTags = "p";
const paragraphs = document.querySelectorAll(textElTags);
if (!paragraphs.length) {
throw new Error(`webpage doesn't have any "${textElTags}" element`);
}
const paragraphTexts = Array.from(paragraphs).map((p) => p.textContent);
// combine text contents from paragraphs and then remove newlines and multiple spaces
const text = paragraphTexts.join(" ").replace(/ {2}|\r\n|\n|\r/gm, "");
return text;
}