blaze/index.ts

import express from "express";
import { Readability, isProbablyReaderable } from "@mozilla/readability";
import got from "got";
import path from "path";
import { fileURLToPath } from "url";
import fetch from "node-fetch";
import "dotenv/config";
import { parseHTML } from "linkedom";

const app = express();
const port = 8888;

// @ts-ignore
const __filename = fileURLToPath(import.meta.url);

const __dirname = path.dirname(__filename);

app.get("/", async (req, res) => {
  const searchEngine = "https://api.search.brave.com/res/v1/web/search";
  let query = req.query.q as string;

  if (!query) {
    res.sendFile(path.join(__dirname + "/dist/index.html"));
    return;
  }

  const key = process.env.CYCLIC_BRAVE_KEY;

  if (!key) {
    throw new Error("No brave key found");
  }

  fetch(`${searchEngine}?q=${query}`, {
    headers: {
      Accept: "*/*",
      "Accept-Encoding": "gzip, deflate, br",
      "X-Subscription-Token": key,
    },
  })
    .then((page) => {
      page.json().then((response) => {
        const results: any[] = [];
        const url = process.env.DEV_MODE
          ? "http://localhost:8888/blazed"
          : "https://blaze.cyclic.app/blazed";
        // @ts-ignore
        response.web.results.forEach((result) => {
          results.push(`
            <article>
            <a href="${url}?url=${result.url}">
              <h2>${result.title}</h2>
              </a>              
              <span>${result.meta_url.hostname}</span>
              <p>${result.description}</p>
            </article>
            <hr />
          `);
        });
        res.send(`<html>
          <head>
            <meta charset="UTF-8">
            <meta name="viewport" content="width=device-width,initial-scale=1">
            <title>Blaze - ${query}</title>
            <style>
              body {font-family:sans-serif}
              h2 {margin-bottom:0}
              span {font-size:.9rem}
            </style>
          </head>
          <body>
            ${results.join("")}
          </body>
        </html>`);
      });
    })
    .catch((err) => {
      console.log(err);
    });
});

app.get("/blazed", async (req, res) => {
  const pageToBlaze = req.query.url as string;
  console.time("blaze");

  try {
    const response = await got(pageToBlaze);
    const { document } = parseHTML(response.body);

    if (!isProbablyReaderable(document)) {
      return res.sendFile(path.join(__dirname, "/dist/not_blazed.html"));
    }

    const reader = new Readability(document);
    const article = reader.parse();

    if (!article) {
      return res.send("Something went wrong");
    }

    res.send(article.content);
  } catch (err) {
    console.log(err);
  } finally {
    console.timeEnd("blaze");
  }
});

app.get("/info", (_, res) => {
  res.sendFile(path.join(__dirname + "/dist/info.html"));
});

app.get("/ooops", (_, res) => {
  res.sendFile(path.join(__dirname + "/dist/info_not_blazed.html"));
});

app.get("/favicon.svg", (_, res) => {
  res.sendFile(path.join(__dirname + "/favicon.svg"));
});

app.listen(port, () => {
  console.log(`Got request`);
});
First commit 2023-06-18 21:20:52 +00:00			`import express from "express";`
Change everything to become a search engine 2023-06-19 10:05:59 +00:00			`import { Readability, isProbablyReaderable } from "@mozilla/readability";`
First commit 2023-06-18 21:20:52 +00:00			`import got from "got";`
Make things pretty 2023-06-18 22:44:17 +00:00			`import path from "path";`
			`import { fileURLToPath } from "url";`
Add some content 2023-06-20 07:32:07 +00:00			`import fetch from "node-fetch";`
Fix serp html 2023-06-19 12:11:22 +00:00			`import "dotenv/config";`
Improve parsing performance 2023-06-20 09:48:11 +00:00			`import { parseHTML } from "linkedom";`
First commit 2023-06-18 21:20:52 +00:00
			`const app = express();`
Change everything to become a search engine 2023-06-19 10:05:59 +00:00			`const port = 8888;`
First commit 2023-06-18 21:20:52 +00:00
Forgive me, it's late 2023-06-18 22:46:43 +00:00			`// @ts-ignore`
Make things pretty 2023-06-18 22:44:17 +00:00			`const __filename = fileURLToPath(import.meta.url);`

			`const __dirname = path.dirname(__filename);`

Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			`app.get("/", async (req, res) => {`
			`const searchEngine = "https://api.search.brave.com/res/v1/web/search";`
Change everything to become a search engine 2023-06-19 10:05:59 +00:00			`let query = req.query.q as string;`
First commit 2023-06-18 21:20:52 +00:00
Change everything to become a search engine 2023-06-19 10:05:59 +00:00			`if (!query) {`
Make things pretty 2023-06-18 22:44:17 +00:00			`res.sendFile(path.join(__dirname + "/dist/index.html"));`
Add some content 2023-06-20 07:32:07 +00:00			`return;`
First commit 2023-06-18 21:20:52 +00:00			`}`

Add some content 2023-06-20 07:32:07 +00:00			`const key = process.env.CYCLIC_BRAVE_KEY;`
Ignore ts for poc 2023-06-19 12:01:17 +00:00
			`if (!key) {`
			`throw new Error("No brave key found");`
			`}`

Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			fetch(`${searchEngine}?q=${query}`, {
Try to add user agent 2023-06-19 10:40:07 +00:00			`headers: {`
Add some content 2023-06-20 07:32:07 +00:00			`Accept: "/",`
Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			`"Accept-Encoding": "gzip, deflate, br",`
Ignore ts for poc 2023-06-19 12:01:17 +00:00			`"X-Subscription-Token": key,`
Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			`},`
			`})`
			`.then((page) => {`
Add some content 2023-06-20 07:32:07 +00:00			`page.json().then((response) => {`
			`const results: any[] = [];`
			`const url = process.env.DEV_MODE`
			`? "http://localhost:8888/blazed"`
			`: "https://blaze.cyclic.app/blazed";`
Ignore ts for poc 2023-06-19 12:01:17 +00:00			`// @ts-ignore`
Add some content 2023-06-20 07:32:07 +00:00			`response.web.results.forEach((result) => {`
Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			results.push(`
change div with article 2023-06-20 07:45:18 +00:00			`<article>`
Add some content 2023-06-20 07:32:07 +00:00			`<a href="${url}?url=${result.url}">`
Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			`<h2>${result.title}</h2>`
Add some content 2023-06-20 07:32:07 +00:00			`</a>`
Add hostname to serp 2023-06-19 13:27:14 +00:00			`<span>${result.meta_url.hostname}</span>`
Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			`<p>${result.description}</p>`
change div with article 2023-06-20 07:45:18 +00:00			`</article>`
Fix serp html 2023-06-19 12:11:22 +00:00			`<hr />`
Move to brave search engine + use fetch 2023-06-19 11:58:21 +00:00			`);
Add some content 2023-06-20 07:32:07 +00:00			`});`
Fix serp html 2023-06-19 12:11:22 +00:00			res.send(`<html>
			`<head>`
			`<meta charset="UTF-8">`
			`<meta name="viewport" content="width=device-width,initial-scale=1">`
Add some content 2023-06-20 07:32:07 +00:00			`<title>Blaze - ${query}</title>`
Add hostname to serp 2023-06-19 13:27:14 +00:00			`<style>`
Add some content 2023-06-20 07:32:07 +00:00			`body {font-family:sans-serif}`
Add hostname to serp 2023-06-19 13:27:14 +00:00			`h2 {margin-bottom:0}`
Add some content 2023-06-20 07:32:07 +00:00			`span {font-size:.9rem}`
Add hostname to serp 2023-06-19 13:27:14 +00:00			`</style>`
Fix serp html 2023-06-19 12:11:22 +00:00			`</head>`
			`<body>`
			`${results.join("")}`
			`</body>`
			</html>`);
Add some content 2023-06-20 07:32:07 +00:00			`});`
First commit 2023-06-18 21:20:52 +00:00			`})`
			`.catch((err) => {`
			`console.log(err);`
			`});`
			`});`
It's really late 2023-06-18 22:52:20 +00:00
Improve parsing performance 2023-06-20 09:48:11 +00:00			`app.get("/blazed", async (req, res) => {`
Add some content 2023-06-20 07:32:07 +00:00			`const pageToBlaze = req.query.url as string;`
Improve parsing performance 2023-06-20 09:48:11 +00:00			`console.time("blaze");`
Add some content 2023-06-20 07:32:07 +00:00
Improve parsing performance 2023-06-20 09:48:11 +00:00			`try {`
			`const response = await got(pageToBlaze);`
			`const { document } = parseHTML(response.body);`
Add some content 2023-06-20 07:32:07 +00:00
Improve parsing performance 2023-06-20 09:48:11 +00:00			`if (!isProbablyReaderable(document)) {`
			`return res.sendFile(path.join(__dirname, "/dist/not_blazed.html"));`
			`}`
Add some content 2023-06-20 07:32:07 +00:00
Improve parsing performance 2023-06-20 09:48:11 +00:00			`const reader = new Readability(document);`
			`const article = reader.parse();`
Add some content 2023-06-20 07:32:07 +00:00
Improve parsing performance 2023-06-20 09:48:11 +00:00			`if (!article) {`
			`return res.send("Something went wrong");`
			`}`
Add some content 2023-06-20 07:32:07 +00:00
Improve parsing performance 2023-06-20 09:48:11 +00:00			`res.send(article.content);`
			`} catch (err) {`
			`console.log(err);`
			`} finally {`
			`console.timeEnd("blaze");`
			`}`
Add some content 2023-06-20 07:32:07 +00:00			`});`
Change everything to become a search engine 2023-06-19 10:05:59 +00:00
Fix frontend + info page 2023-06-19 12:55:28 +00:00			`app.get("/info", (_, res) => {`
			`res.sendFile(path.join(__dirname + "/dist/info.html"));`
Add some content 2023-06-20 07:32:07 +00:00			`});`

			`app.get("/ooops", (_, res) => {`
			`res.sendFile(path.join(__dirname + "/dist/info_not_blazed.html"));`
			`});`
Fix frontend + info page 2023-06-19 12:55:28 +00:00
Fix again favicon 2023-06-20 07:41:34 +00:00			`app.get("/favicon.svg", (_, res) => {`
			`res.sendFile(path.join(__dirname + "/favicon.svg"));`
			`});`

It's really late 2023-06-18 22:52:20 +00:00			`app.listen(port, () => {`
			console.log(`Got request`);
			`});`