2023-06-18 21:20:52 +00:00
|
|
|
import express from "express";
|
2023-06-19 10:05:59 +00:00
|
|
|
import { Readability, isProbablyReaderable } from "@mozilla/readability";
|
2023-06-18 21:20:52 +00:00
|
|
|
import got from "got";
|
2023-06-18 22:44:17 +00:00
|
|
|
import path from "path";
|
|
|
|
|
import { fileURLToPath } from "url";
|
2023-06-19 12:11:22 +00:00
|
|
|
import "dotenv/config";
|
2023-06-21 00:26:24 +00:00
|
|
|
import { parseHTML, parseJSON } from "linkedom";
|
2023-06-20 10:19:17 +00:00
|
|
|
// @ts-ignore
|
|
|
|
|
import XHR2 from "xhr2";
|
|
|
|
|
const XMLHttpRequest = XHR2.XMLHttpRequest;
|
2023-06-20 11:51:05 +00:00
|
|
|
import { minify } from "html-minifier";
|
2023-06-22 14:00:46 +00:00
|
|
|
import {
|
|
|
|
|
blazeFunctionality,
|
|
|
|
|
blazeUrl,
|
|
|
|
|
injectBlazeToPageLinks,
|
|
|
|
|
} from "./utils.js";
|
2023-06-18 21:20:52 +00:00
|
|
|
|
|
|
|
|
const app = express();
|
2023-06-19 10:05:59 +00:00
|
|
|
const port = 8888;
|
2023-06-18 21:20:52 +00:00
|
|
|
|
2023-06-20 11:51:05 +00:00
|
|
|
const minifierOptions = {
|
|
|
|
|
collapseWhitespace: true,
|
|
|
|
|
removeComments: true,
|
|
|
|
|
removeOptionalTags: true,
|
|
|
|
|
removeRedundantAttributes: true,
|
|
|
|
|
removeScriptTypeAttributes: true,
|
|
|
|
|
removeTagWhitespace: true,
|
|
|
|
|
useShortDoctype: true,
|
|
|
|
|
minifyCSS: true,
|
|
|
|
|
};
|
|
|
|
|
|
2023-06-18 22:46:43 +00:00
|
|
|
// @ts-ignore
|
2023-06-18 22:44:17 +00:00
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
|
|
|
|
|
|
|
|
const __dirname = path.dirname(__filename);
|
|
|
|
|
|
2023-06-19 11:58:21 +00:00
|
|
|
app.get("/", async (req, res) => {
|
|
|
|
|
const searchEngine = "https://api.search.brave.com/res/v1/web/search";
|
2023-06-20 10:19:17 +00:00
|
|
|
const query = req.query.q as string;
|
2023-06-18 21:20:52 +00:00
|
|
|
|
2023-06-19 10:05:59 +00:00
|
|
|
if (!query) {
|
2023-06-20 10:19:17 +00:00
|
|
|
return res.sendFile(path.join(__dirname, "/dist/index.html"));
|
2023-06-18 21:20:52 +00:00
|
|
|
}
|
|
|
|
|
|
2023-06-20 07:32:07 +00:00
|
|
|
const key = process.env.CYCLIC_BRAVE_KEY;
|
2023-06-19 12:01:17 +00:00
|
|
|
|
|
|
|
|
if (!key) {
|
|
|
|
|
throw new Error("No brave key found");
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-20 10:19:17 +00:00
|
|
|
try {
|
|
|
|
|
const xhr = new XMLHttpRequest();
|
2023-06-20 23:00:31 +00:00
|
|
|
xhr.open("GET", `${searchEngine}?q=${query}&safesearch=moderate`, true);
|
2023-06-20 10:19:17 +00:00
|
|
|
xhr.setRequestHeader("Accept", "*/*");
|
|
|
|
|
xhr.setRequestHeader("X-Subscription-Token", key);
|
|
|
|
|
|
|
|
|
|
xhr.onreadystatechange = () => {
|
|
|
|
|
if (xhr.readyState !== 4) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (xhr.status !== 200) {
|
|
|
|
|
console.error("XHR request failed:", xhr.status, xhr.statusText);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const data = JSON.parse(xhr.responseText);
|
|
|
|
|
|
|
|
|
|
// @ts-ignore
|
|
|
|
|
const results = data.web.results.map(
|
|
|
|
|
(result: any) => `
|
2023-06-20 07:45:18 +00:00
|
|
|
<article>
|
2023-06-20 23:00:31 +00:00
|
|
|
<a href="${blazeUrl}/blazed?url=${result.url}">
|
2023-06-20 10:19:17 +00:00
|
|
|
<h2>${result.title}</h2>
|
|
|
|
|
</a>
|
2023-06-19 13:27:14 +00:00
|
|
|
<span>${result.meta_url.hostname}</span>
|
2023-06-19 11:58:21 +00:00
|
|
|
<p>${result.description}</p>
|
2023-06-20 07:45:18 +00:00
|
|
|
</article>
|
2023-06-19 12:11:22 +00:00
|
|
|
<hr />
|
2023-06-20 10:19:17 +00:00
|
|
|
`
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const html = `
|
|
|
|
|
<html>
|
|
|
|
|
<head>
|
|
|
|
|
<meta charset="UTF-8">
|
|
|
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
|
|
|
<title>Blaze - ${query}</title>
|
|
|
|
|
<style>
|
|
|
|
|
body {font-family:sans-serif}
|
|
|
|
|
h2 {margin-bottom:0}
|
|
|
|
|
span {font-size:.9rem}
|
|
|
|
|
</style>
|
|
|
|
|
</head>
|
|
|
|
|
<body>
|
2023-06-22 14:00:46 +00:00
|
|
|
<header>
|
|
|
|
|
<label>
|
|
|
|
|
<a href="/"><strong>BLAZE</strong></a>
|
|
|
|
|
<input type="search" value="${query}" />
|
|
|
|
|
<button>Blaze it</button>
|
|
|
|
|
</label>
|
|
|
|
|
</header>
|
|
|
|
|
<hr/>
|
2023-06-20 10:19:17 +00:00
|
|
|
${results.join("")}
|
2023-06-22 14:00:46 +00:00
|
|
|
<script>
|
|
|
|
|
${blazeFunctionality}
|
|
|
|
|
blazeFunctionality('${blazeUrl}')
|
|
|
|
|
</script>
|
2023-06-20 10:19:17 +00:00
|
|
|
</body>
|
|
|
|
|
</html>
|
|
|
|
|
`;
|
|
|
|
|
|
2023-06-20 11:51:05 +00:00
|
|
|
const minifiedSerp = minify(html, minifierOptions);
|
|
|
|
|
|
|
|
|
|
res.send(minifiedSerp);
|
2023-06-20 10:19:17 +00:00
|
|
|
};
|
|
|
|
|
xhr.send();
|
|
|
|
|
} catch (err) {
|
|
|
|
|
console.error(err);
|
|
|
|
|
}
|
2023-06-18 21:20:52 +00:00
|
|
|
});
|
2023-06-18 22:52:20 +00:00
|
|
|
|
2023-06-20 09:48:11 +00:00
|
|
|
app.get("/blazed", async (req, res) => {
|
2023-06-20 07:32:07 +00:00
|
|
|
const pageToBlaze = req.query.url as string;
|
|
|
|
|
|
2023-06-20 09:48:11 +00:00
|
|
|
try {
|
2023-06-22 14:00:46 +00:00
|
|
|
const xhr = new XMLHttpRequest();
|
|
|
|
|
xhr.open("GET", pageToBlaze, true);
|
|
|
|
|
xhr.setRequestHeader("Accept", "text/html");
|
|
|
|
|
|
|
|
|
|
xhr.onreadystatechange = async () => {
|
|
|
|
|
if (xhr.readyState !== 4) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (xhr.status === 404) {
|
|
|
|
|
res.sendFile(path.join(__dirname, "/dist/404.html"));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (xhr.status !== 200) {
|
|
|
|
|
console.error("XHR request failed:", xhr.status, xhr.statusText);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const response = xhr.responseText;
|
|
|
|
|
const { document } = parseHTML(response);
|
|
|
|
|
|
|
|
|
|
if (!isProbablyReaderable(document)) {
|
|
|
|
|
// TODO: still a lot of bugs, must be refined to handle some cases, like
|
|
|
|
|
// cookie banners, etc.
|
|
|
|
|
document.querySelectorAll("link").forEach((l) => {
|
|
|
|
|
l.remove();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
document.querySelectorAll("style").forEach((s) => {
|
|
|
|
|
s.remove;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
document.querySelectorAll("script").forEach((s) => {
|
|
|
|
|
s.remove();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const blazeDisclaimer = document.createElement("div");
|
|
|
|
|
blazeDisclaimer.style.width = "100dvw";
|
|
|
|
|
blazeDisclaimer.style.border = "1px solid red";
|
|
|
|
|
blazeDisclaimer.style.padding = "1rem";
|
|
|
|
|
blazeDisclaimer.innerHTML = `
|
|
|
|
|
<h2 style="text-align: center">BLAZE INFO</h2>
|
|
|
|
|
<p>
|
|
|
|
|
The page you are seeing <strong>could not be correctly blazed</strong> due to these webpage characteristics.
|
|
|
|
|
<strong>Blaze served anyway</strong> a lightweight version of the page.
|
|
|
|
|
Keep in mind that this kind of pages <strong>can be hard or even impossible to use, read or understand</strong>.
|
|
|
|
|
</p>
|
|
|
|
|
`;
|
|
|
|
|
|
|
|
|
|
const referenceElement = document.body.firstChild;
|
|
|
|
|
document.body.insertBefore(blazeDisclaimer, referenceElement);
|
|
|
|
|
|
|
|
|
|
return res.send(document.toString());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//TODO: find if there are more performant ways to remove images or evaluate if is the case to remove images
|
|
|
|
|
document.querySelectorAll("img").forEach((img) => img.remove());
|
|
|
|
|
|
|
|
|
|
const reader = new Readability(document);
|
|
|
|
|
const article = reader.parse();
|
|
|
|
|
|
|
|
|
|
if (!article) {
|
|
|
|
|
return res.send("Something went wrong");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const blazedPage = `<html><head>
|
|
|
|
|
<meta charset="UTF-8" />
|
|
|
|
|
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
|
|
|
|
<style>body {font-family: sans-serif}</style>
|
|
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
${article.content}
|
|
|
|
|
<script>
|
|
|
|
|
${injectBlazeToPageLinks}
|
|
|
|
|
const url = "${blazeUrl}"
|
|
|
|
|
const currentUrl = "${req.query.url}"
|
|
|
|
|
injectBlazeToPageLinks(url, currentUrl)
|
|
|
|
|
</script>
|
|
|
|
|
</body></html>
|
|
|
|
|
`;
|
|
|
|
|
|
|
|
|
|
const minifiedBlazedPage = minify(blazedPage, minifierOptions);
|
|
|
|
|
|
|
|
|
|
res.send(minifiedBlazedPage);
|
|
|
|
|
};
|
|
|
|
|
xhr.send();
|
2023-06-20 09:48:11 +00:00
|
|
|
} catch (err) {
|
|
|
|
|
console.log(err);
|
|
|
|
|
}
|
2023-06-20 07:32:07 +00:00
|
|
|
});
|
2023-06-19 10:05:59 +00:00
|
|
|
|
2023-06-19 12:55:28 +00:00
|
|
|
app.get("/info", (_, res) => {
|
|
|
|
|
res.sendFile(path.join(__dirname + "/dist/info.html"));
|
2023-06-20 07:32:07 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
app.get("/ooops", (_, res) => {
|
|
|
|
|
res.sendFile(path.join(__dirname + "/dist/info_not_blazed.html"));
|
|
|
|
|
});
|
2023-06-19 12:55:28 +00:00
|
|
|
|
2023-06-20 07:41:34 +00:00
|
|
|
app.get("/favicon.svg", (_, res) => {
|
|
|
|
|
res.sendFile(path.join(__dirname + "/favicon.svg"));
|
|
|
|
|
});
|
|
|
|
|
|
2023-06-18 22:52:20 +00:00
|
|
|
app.listen(port, () => {
|
|
|
|
|
console.log(`Got request`);
|
|
|
|
|
});
|
