Change everything to become a search engine

2024-12-22 08:16:26 +00:00 · 2023-06-19 12:05:59 +02:00 · 2023-06-19 12:05:59 +02:00 · d930272730
commit d930272730
parent 9a14cc9df3
4 changed files with 366 additions and 69 deletions
--- a/dist/index.html
+++ b/dist/index.html
@ -1,5 +1,58 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Blaze The Page</title></head><body><h1>Welcome to Blaze the Page</h1><label for="u">URL to blaze</label><input name="u" id="u" type="text" placeholder="Write an URL..."><button>BLAZE IT!</button><hr><h2>What?</h2><p>What is "Blaze the page"? It is a small service that provides a minimal version of a web page. Simply paste a URL and click the button to obtain an extremely lightweight version of the page, concentrating solely on the content.</p><h2>Why?</h2><p>One day, I exceeded my monthly data limit for high-speed browsing. Consequently, my internet speed was reduced significantly. As a result, web pages took an extremely long time to load, and I encountered timeout errors 90% of the time. It was at that moment I contemplated a solution to this issue and conceived the idea of Blaze this page.</p><h3>How?</h3><p>It's actually very straightforward. When you click on "BLAZE IT!", the URL changes to https://ill-red-skunk-wig.cyclic.app?at= followed by the URL you pasted. This address is where the backend is hosted. When the site is accessed with the "at" parameter, a simple Node.js application retrieves the content and generates an extremely lightweight version of the page, retaining only the essential elements: the content. This minimal page is the result of the request and will be displayed in your browser.</p><h3>Does it work?</h3><p>Yes. The purpose of keeping this page as minimal as possible is to make it significantly lighter than a typical webpage. As a result, even with a poor internet connection, you can load this page relatively quickly. When you click on "BLAZE IT!", you receive a webpage that is only a few kilobytes in size instead of several megabytes. This makes it feasible to load the page even under challenging connection conditions, preventing timeout errors from occurring.</p><script>const b = document.querySelector("button"),
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title>Blaze The Page</title>
+  </head>
+  <body>
+    <h1>Welcome to Blaze the Page</h1>
+    <label for="u">URL to blaze</label
+    ><input name="u" id="u" type="text" placeholder="Write an URL..." /><button>
+      BLAZE IT!
+    </button>
+    <hr />
+    <h2>What?</h2>
+    <p>
+      What is "Blaze the page"? It is a small service that provides a minimal
+      version of a web page. Simply paste a URL and click the button to obtain
+      an extremely lightweight version of the page, concentrating solely on the
+      content.
+    </p>
+    <h2>Why?</h2>
+    <p>
+      One day, I exceeded my monthly data limit for high-speed browsing.
+      Consequently, my internet speed was reduced significantly. As a result,
+      web pages took an extremely long time to load, and I encountered timeout
+      errors 90% of the time. It was at that moment I contemplated a solution to
+      this issue and conceived the idea of Blaze this page.
+    </p>
+    <h3>How?</h3>
+    <p>
+      It's actually very straightforward. When you click on "BLAZE IT!", the URL
+      changes to https://ill-red-skunk-wig.cyclic.app?at= followed by the URL
+      you pasted. This address is where the backend is hosted. When the site is
+      accessed with the "at" parameter, a simple Node.js application retrieves
+      the content and generates an extremely lightweight version of the page,
+      retaining only the essential elements: the content. This minimal page is
+      the result of the request and will be displayed in your browser.
+    </p>
+    <h3>Does it work?</h3>
+    <p>
+      Yes. The purpose of keeping this page as minimal as possible is to make it
+      significantly lighter than a typical webpage. As a result, even with a
+      poor internet connection, you can load this page relatively quickly. When
+      you click on "BLAZE IT!", you receive a webpage that is only a few
+      kilobytes in size instead of several megabytes. This makes it feasible to
+      load the page even under challenging connection conditions, preventing
+      timeout errors from occurring.
+    </p>
+    <script>
+      const b = document.querySelector("button"),
        i = document.querySelector("input");
      b.addEventListener("click", () => {
-        location.href = `https://ill-red-skunk-wig.cyclic.app?at=${i.value}`;
-      });</script></body></html>
+        location.href = `https://ill-red-skunk-wig.cyclic.app?q=${encodeURI(i.value)}`;
+      });
+    </script>
+  </body>
+</html>
--- a/index.ts
+++ b/index.ts
@ -1,12 +1,12 @@
 import express from "express";
-import { Readability } from "@mozilla/readability";
+import { Readability, isProbablyReaderable } from "@mozilla/readability";
 import { JSDOM } from "jsdom";
 import got from "got";
 import path from "path";
 import { fileURLToPath } from "url";

 const app = express();
-const port = 5763;
+const port = 8888;

 // @ts-ignore
 const __filename = fileURLToPath(import.meta.url);
@ -14,37 +14,86 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);

 app.get("/", (req, res) => {
-  let url = req.query.at as string;
+  const searchEngine = "https://duckduckgo.com/html/";
+  let query = req.query.q as string;

-  if (!url) {
+  if (!query) {
    res.sendFile(path.join(__dirname + "/dist/index.html"));
+    return
  }

-  var re = new RegExp("^(http|https)://", "i");
-  var match = re.test(url);
-
-  if (!match) {
-    url = `https://${url}`
-  }
-
-  got(url)
+  got(`${searchEngine}?q=${query}`)
    .then((response) => {
      const dom = new JSDOM(response.body);
-      let reader = new Readability(dom.window.document);
-      let article = reader.parse();
+      const links = dom.window.document.querySelectorAll('link')
+      const results = dom.window.document.querySelectorAll('.result')

-      if (!article) {
-        res.send("Something went wrong");
-        return;
-      }
+      // add custom style
+      const style = dom.window.document.createElement('style')
+      dom.window.document.querySelector('head')?.appendChild(style)
+      style.innerHTML = `
+        .result {
+          padding: 1rem;
+          border: 1px solid black;
+          margin-bottom: 10px:
+        }

-      res.send(article.content);
+        .result__snippet, .result__url {
+          text-decoration: none;
+          color: black
+        }
+      `
+      // clean up dom
+      links.forEach(linkTag => linkTag.remove())
+      dom.window.document.querySelector('form')?.remove()
+      dom.window.document.querySelector('#header')?.remove()
+
+      // modify urls
+      results.forEach(result => {
+        const hrefArray = result.querySelector('.result__snippet')?.getAttribute('href')?.split('')
+        hrefArray?.splice(0,25)
+        const cleanHref = hrefArray?.join('')
+        const newHref = `http://localhost:8888/blazed?url=${cleanHref}`
+        result.querySelector('.result__snippet')?.setAttribute('href', newHref)
+        result.querySelector('.result__url')?.setAttribute('href', newHref)
+        result.querySelector('.result__a')?.setAttribute('href', newHref)
+      })
+
+      res.send(dom.serialize())
+      return;
    })
    .catch((err) => {
      console.log(err);
    });
 });

+app.get("/blazed", (req, res) => {
+const pageToBlaze = req.query.url as string
+ 
+got(pageToBlaze)
+   .then((response) => {
+     const dom = new JSDOM(response.body);
+
+     if (!isProbablyReaderable(dom.window.document)) {
+      res.send("This page can not be blazed... Sorry!")
+      return;
+     }
+
+     let reader = new Readability(dom.window.document);
+     let article = reader.parse();
+
+     if (!article) {
+       res.send("Something went wrong");
+       return;
+     }
+
+     res.send(article.content);
+   })
+   .catch((err) => {
+     console.log(err);
+   }); 
+})
+
 app.listen(port, () => {
  console.log(`Got request`);
 });
--- a/package-lock.json
+++ b/package-lock.json
@ -11,9 +11,10 @@
      "dependencies": {
        "@mozilla/readability": "^0.4.4",
        "express": "^4.18.2",
+        "fetch": "^1.1.0",
        "got": "^13.0.0",
        "jsdom": "^22.1.0",
-        "node-ts": "^6.0.1"
+        "node-html-parser": "^6.1.5"
      },
      "devDependencies": {
        "@types/express": "^4.17.17",
@ -44,19 +45,6 @@
        "node": ">=14.0.0"
      }
    },
-    "node_modules/@rauschma/stringio": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/@rauschma/stringio/-/stringio-1.4.0.tgz",
-      "integrity": "sha512-3uor2f/MXZkmX5RJf8r+OC3WvZVzpSme0yyL0rQDPEnatE02qRcqwEwnsgpgriEck0S/n4vWtUd6tTtrJwk45Q==",
-      "dependencies": {
-        "@types/node": "^10.0.3"
-      }
-    },
-    "node_modules/@rauschma/stringio/node_modules/@types/node": {
-      "version": "10.17.60",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",
-      "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw=="
-    },
    "node_modules/@sindresorhus/is": {
      "version": "5.4.1",
      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.4.1.tgz",
@ -155,7 +143,8 @@
    "node_modules/@types/node": {
      "version": "20.3.1",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.1.tgz",
-      "integrity": "sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg=="
+      "integrity": "sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==",
+      "dev": true
    },
    "node_modules/@types/qs": {
      "version": "6.9.7",
@ -312,6 +301,17 @@
        "node": ">=8"
      }
    },
+    "node_modules/biskviit": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/biskviit/-/biskviit-1.0.1.tgz",
+      "integrity": "sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w==",
+      "dependencies": {
+        "psl": "^1.1.7"
+      },
+      "engines": {
+        "node": ">=1.0.0"
+      }
+    },
    "node_modules/body-parser": {
      "version": "1.20.1",
      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@ -335,6 +335,11 @@
        "npm": "1.2.8000 || >= 1.4.16"
      }
    },
+    "node_modules/boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
+    },
    "node_modules/brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -565,6 +570,32 @@
      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
      "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
    },
+    "node_modules/css-select": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
+      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
+      "dependencies": {
+        "boolbase": "^1.0.0",
+        "css-what": "^6.1.0",
+        "domhandler": "^5.0.2",
+        "domutils": "^3.0.1",
+        "nth-check": "^2.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
+    "node_modules/css-what": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
+      "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
+      "engines": {
+        "node": ">= 6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
    "node_modules/cssstyle": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz",
@ -676,6 +707,30 @@
        "npm": "1.2.8000 || >= 1.4.16"
      }
    },
+    "node_modules/dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+      "dependencies": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
+      },
+      "funding": {
+        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+      }
+    },
+    "node_modules/domelementtype": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ]
+    },
    "node_modules/domexception": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",
@ -687,6 +742,33 @@
        "node": ">=12"
      }
    },
+    "node_modules/domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+      "dependencies": {
+        "domelementtype": "^2.3.0"
+      },
+      "engines": {
+        "node": ">= 4"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domhandler?sponsor=1"
+      }
+    },
+    "node_modules/domutils": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
+      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
+      "dependencies": {
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domutils?sponsor=1"
+      }
+    },
    "node_modules/ee-first": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@ -706,6 +788,14 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/encoding": {
+      "version": "0.1.12",
+      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
+      "integrity": "sha512-bl1LAgiQc4ZWr++pNYUdRe/alecaHFeHxIJ/pNciqGdKXghaTCOwKkbKp6ye7pKZGu/GcaSXFk8PBVhgs+dJdA==",
+      "dependencies": {
+        "iconv-lite": "~0.4.13"
+      }
+    },
    "node_modules/entities": {
      "version": "4.5.0",
      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
@ -780,6 +870,15 @@
        "node": ">= 0.10.0"
      }
    },
+    "node_modules/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-5O8TwrGzoNblBG/jtK4NFuZwNCkZX6s5GfRNOaGtm+QGJEuNakSC/i2RW0R93KX6E0jVjNXm6O3CRN4Ql3K+yA==",
+      "dependencies": {
+        "biskviit": "1.0.1",
+        "encoding": "0.1.12"
+      }
+    },
    "node_modules/fill-range": {
      "version": "7.0.1",
      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@ -977,6 +1076,14 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
+      "bin": {
+        "he": "bin/he"
+      }
+    },
    "node_modules/html-encoding-sniffer": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
@ -1331,16 +1438,13 @@
        "node": ">= 0.6"
      }
    },
-    "node_modules/node-ts": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/node-ts/-/node-ts-6.0.1.tgz",
-      "integrity": "sha512-nxFz4tJYv9Yu6u4e3B1rmfh45Zmge7rD/MlTA5V98+Q5eFsqzCWTWDYvbrNEPGtnzhkqNLk4E2r3C6VSfNoZNQ==",
+    "node_modules/node-html-parser": {
+      "version": "6.1.5",
+      "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz",
+      "integrity": "sha512-fAaM511feX++/Chnhe475a0NHD8M7AxDInsqQpz6x63GRF7xYNdS8Vo5dKsIVPgsOvG7eioRRTZQnWBrhDHBSg==",
      "dependencies": {
-        "@rauschma/stringio": "^1.4.0",
-        "@types/node": "^20.2.4"
-      },
-      "engines": {
-        "node": ">=18.0.0"
+        "css-select": "^5.1.0",
+        "he": "1.2.0"
      }
    },
    "node_modules/nodemon": {
@ -1442,6 +1546,17 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/nth-check": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+      "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+      "dependencies": {
+        "boolbase": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/nth-check?sponsor=1"
+      }
+    },
    "node_modules/nwsapi": {
      "version": "2.2.5",
      "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.5.tgz",
@ -2159,21 +2274,6 @@
      "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.4.4.tgz",
      "integrity": "sha512-MCgZyANpJ6msfvVMi6+A0UAsvZj//4OHREYUB9f2087uXHVoU+H+SWhuihvb1beKpM323bReQPRio0WNk2+V6g=="
    },
-    "@rauschma/stringio": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/@rauschma/stringio/-/stringio-1.4.0.tgz",
-      "integrity": "sha512-3uor2f/MXZkmX5RJf8r+OC3WvZVzpSme0yyL0rQDPEnatE02qRcqwEwnsgpgriEck0S/n4vWtUd6tTtrJwk45Q==",
-      "requires": {
-        "@types/node": "^10.0.3"
-      },
-      "dependencies": {
-        "@types/node": {
-          "version": "10.17.60",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",
-          "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw=="
-        }
-      }
-    },
    "@sindresorhus/is": {
      "version": "5.4.1",
      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.4.1.tgz",
@ -2260,7 +2360,8 @@
    "@types/node": {
      "version": "20.3.1",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.1.tgz",
-      "integrity": "sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg=="
+      "integrity": "sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==",
+      "dev": true
    },
    "@types/qs": {
      "version": "6.9.7",
@ -2390,6 +2491,14 @@
      "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
      "dev": true
    },
+    "biskviit": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/biskviit/-/biskviit-1.0.1.tgz",
+      "integrity": "sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w==",
+      "requires": {
+        "psl": "^1.1.7"
+      }
+    },
    "body-parser": {
      "version": "1.20.1",
      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@ -2409,6 +2518,11 @@
        "unpipe": "1.0.0"
      }
    },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
+    },
    "brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -2578,6 +2692,23 @@
      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
      "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
    },
+    "css-select": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
+      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
+      "requires": {
+        "boolbase": "^1.0.0",
+        "css-what": "^6.1.0",
+        "domhandler": "^5.0.2",
+        "domutils": "^3.0.1",
+        "nth-check": "^2.0.1"
+      }
+    },
+    "css-what": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
+      "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw=="
+    },
    "cssstyle": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz",
@ -2653,6 +2784,21 @@
      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
      "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="
    },
+    "dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+      "requires": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
+      }
+    },
+    "domelementtype": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="
+    },
    "domexception": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",
@ -2661,6 +2807,24 @@
        "webidl-conversions": "^7.0.0"
      }
    },
+    "domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+      "requires": {
+        "domelementtype": "^2.3.0"
+      }
+    },
+    "domutils": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
+      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
+      "requires": {
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
+      }
+    },
    "ee-first": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@ -2677,6 +2841,14 @@
      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
      "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="
    },
+    "encoding": {
+      "version": "0.1.12",
+      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
+      "integrity": "sha512-bl1LAgiQc4ZWr++pNYUdRe/alecaHFeHxIJ/pNciqGdKXghaTCOwKkbKp6ye7pKZGu/GcaSXFk8PBVhgs+dJdA==",
+      "requires": {
+        "iconv-lite": "~0.4.13"
+      }
+    },
    "entities": {
      "version": "4.5.0",
      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
@ -2736,6 +2908,15 @@
        "vary": "~1.1.2"
      }
    },
+    "fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-5O8TwrGzoNblBG/jtK4NFuZwNCkZX6s5GfRNOaGtm+QGJEuNakSC/i2RW0R93KX6E0jVjNXm6O3CRN4Ql3K+yA==",
+      "requires": {
+        "biskviit": "1.0.1",
+        "encoding": "0.1.12"
+      }
+    },
    "fill-range": {
      "version": "7.0.1",
      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@ -2869,6 +3050,11 @@
      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
    },
+    "he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="
+    },
    "html-encoding-sniffer": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
@ -3128,13 +3314,13 @@
      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
      "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
    },
-    "node-ts": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/node-ts/-/node-ts-6.0.1.tgz",
-      "integrity": "sha512-nxFz4tJYv9Yu6u4e3B1rmfh45Zmge7rD/MlTA5V98+Q5eFsqzCWTWDYvbrNEPGtnzhkqNLk4E2r3C6VSfNoZNQ==",
+    "node-html-parser": {
+      "version": "6.1.5",
+      "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz",
+      "integrity": "sha512-fAaM511feX++/Chnhe475a0NHD8M7AxDInsqQpz6x63GRF7xYNdS8Vo5dKsIVPgsOvG7eioRRTZQnWBrhDHBSg==",
      "requires": {
-        "@rauschma/stringio": "^1.4.0",
-        "@types/node": "^20.2.4"
+        "css-select": "^5.1.0",
+        "he": "1.2.0"
      }
    },
    "nodemon": {
@ -3207,6 +3393,14 @@
      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.0.tgz",
      "integrity": "sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw=="
    },
+    "nth-check": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+      "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+      "requires": {
+        "boolbase": "^1.0.0"
+      }
+    },
    "nwsapi": {
      "version": "2.2.5",
      "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.5.tgz",
--- a/package.json
+++ b/package.json
@ -14,9 +14,10 @@
  "dependencies": {
    "@mozilla/readability": "^0.4.4",
    "express": "^4.18.2",
+    "fetch": "^1.1.0",
    "got": "^13.0.0",
    "jsdom": "^22.1.0",
-    "node-ts": "^6.0.1"
+    "node-html-parser": "^6.1.5"
  },
  "devDependencies": {
    "@types/express": "^4.17.17",