Move to brave search engine + use fetch

index.ts

@@ -4,6 +4,7 @@ import { JSDOM } from "jsdom";
 import got from "got";
 import path from "path";
 import { fileURLToPath } from "url";
+import fetch from 'node-fetch'
 
 const app = express();
 const port = 8888;
@@ -13,8 +14,8 @@ const __filename = fileURLToPath(import.meta.url);
 
 const __dirname = path.dirname(__filename);
 
-app.get("/", (req, res) => {
+app.get("/", async (req, res) => {
-  const searchEngine = "http://html.duckduckgo.com/html/";
+  const searchEngine = "https://api.search.brave.com/res/v1/web/search";
   let query = req.query.q as string;
 
   if (!query) {
@@ -22,58 +23,29 @@ app.get("/", (req, res) => {
     return
   }
 
-  const options = {
+  fetch(`${searchEngine}?q=${query}`, {
     headers: {
-      'user-agent': req.headers["user-agent"]
+      "Accept": "*/*",
-    }
+      "Accept-Encoding": "gzip, deflate, br",
-  }
+      "X-Subscription-Token": process.env.CYCLIC_BRAVE_KEY,
-
+    },
-  got(`${searchEngine}?q=${query}`, options)
+  })
-    .then((response) => {
+    .then((page) => {
-
+      page.json().then(response => {
-      if (response.statusCode !== 200) {
+        const results = []
-        console.log("Error code:", response.statusCode)
+        response.web.results.forEach(result => {
-        res.send(`Error code ${response.statusCode}`)
+          results.push(`
-        return;
+            <div>
-      }
+              <h2>${result.title}</h2>
-
+              <a href="https://ill-red-skunk-wig.cyclic.app/blazed?url=${result.url}">${result.url}</a>
-      const dom = new JSDOM(response.body);
+             <a href="${result.url}">
-      const links = dom.window.document.querySelectorAll('link')
+              <p>${result.description}</p>
-      const results = dom.window.document.querySelectorAll('.result')
+             </a>              
-
+            </div>
-      // add custom style
+          `);
-      const style = dom.window.document.createElement('style')
+        })
-      dom.window.document.querySelector('head')?.appendChild(style)
+        res.send(results.join(''))
-      style.innerHTML = `
-        .result {
-          padding: 1rem;
-          border: 1px solid black;
-          margin-bottom: 10px:
-        }
-
-        .result__snippet, .result__url {
-          text-decoration: none;
-          color: black
-        }
-      `
-      // clean up dom
-      links.forEach(linkTag => linkTag.remove())
-      dom.window.document.querySelector('form')?.remove()
-      dom.window.document.querySelector('#header')?.remove()
-
-      // modify urls
-      results.forEach(result => {
-        const hrefArray = result.querySelector('.result__snippet')?.getAttribute('href')?.split('')
-        hrefArray?.splice(0,25)
-        const cleanHref = hrefArray?.join('')
-        const newHref = `https://ill-red-skunk-wig.cyclic.app/blazed?url=${cleanHref}`;
-        result.querySelector('.result__snippet')?.setAttribute('href', newHref)
-        result.querySelector('.result__url')?.setAttribute('href', newHref)
-        result.querySelector('.result__a')?.setAttribute('href', newHref)
       })
-
-      res.send(dom.serialize())
-      return;
     })
     .catch((err) => {
       console.log(err);

package-lock.json

@@ -14,6 +14,7 @@
         "fetch": "^1.1.0",
         "got": "^13.0.0",
         "jsdom": "^22.1.0",
+        "node-fetch": "^3.3.1",
         "node-html-parser": "^6.1.5"
       },
       "devDependencies": {
@@ -607,6 +608,14 @@
         "node": ">=14"
       }
     },
+    "node_modules/data-uri-to-buffer": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
+      "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
+      "engines": {
+        "node": ">= 12"
+      }
+    },
     "node_modules/data-urls": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz",
@@ -879,6 +888,28 @@
         "encoding": "0.1.12"
       }
     },
+    "node_modules/fetch-blob": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
+      "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "paypal",
+          "url": "https://paypal.me/jimmywarting"
+        }
+      ],
+      "dependencies": {
+        "node-domexception": "^1.0.0",
+        "web-streams-polyfill": "^3.0.3"
+      },
+      "engines": {
+        "node": "^12.20 || >= 14.13"
+      }
+    },
     "node_modules/fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -929,6 +960,17 @@
         "node": ">= 14.17"
       }
     },
+    "node_modules/formdata-polyfill": {
+      "version": "4.0.10",
+      "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
+      "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
+      "dependencies": {
+        "fetch-blob": "^3.1.2"
+      },
+      "engines": {
+        "node": ">=12.20.0"
+      }
+    },
     "node_modules/forwarded": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -1438,6 +1480,41 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/node-domexception": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "github",
+          "url": "https://paypal.me/jimmywarting"
+        }
+      ],
+      "engines": {
+        "node": ">=10.5.0"
+      }
+    },
+    "node_modules/node-fetch": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz",
+      "integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==",
+      "dependencies": {
+        "data-uri-to-buffer": "^4.0.0",
+        "fetch-blob": "^3.1.4",
+        "formdata-polyfill": "^4.0.10"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/node-fetch"
+      }
+    },
     "node_modules/node-html-parser": {
       "version": "6.1.5",
       "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz",
@@ -2122,6 +2199,14 @@
         "node": ">=14"
       }
     },
+    "node_modules/web-streams-polyfill": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
+      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/webidl-conversions": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
@@ -2717,6 +2802,11 @@
         "rrweb-cssom": "^0.6.0"
       }
     },
+    "data-uri-to-buffer": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
+      "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="
+    },
     "data-urls": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz",
@@ -2917,6 +3007,15 @@
         "encoding": "0.1.12"
       }
     },
+    "fetch-blob": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
+      "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
+      "requires": {
+        "node-domexception": "^1.0.0",
+        "web-streams-polyfill": "^3.0.3"
+      }
+    },
     "fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -2955,6 +3054,14 @@
       "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz",
       "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw=="
     },
+    "formdata-polyfill": {
+      "version": "4.0.10",
+      "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
+      "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
+      "requires": {
+        "fetch-blob": "^3.1.2"
+      }
+    },
     "forwarded": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -3314,6 +3421,21 @@
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
       "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
     },
+    "node-domexception": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="
+    },
+    "node-fetch": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz",
+      "integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==",
+      "requires": {
+        "data-uri-to-buffer": "^4.0.0",
+        "fetch-blob": "^3.1.4",
+        "formdata-polyfill": "^4.0.10"
+      }
+    },
     "node-html-parser": {
       "version": "6.1.5",
       "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz",
@@ -3817,6 +3939,11 @@
         "xml-name-validator": "^4.0.0"
       }
     },
+    "web-streams-polyfill": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
+      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
+    },
     "webidl-conversions": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",

package.json

@@ -17,6 +17,7 @@
     "fetch": "^1.1.0",
     "got": "^13.0.0",
     "jsdom": "^22.1.0",
+    "node-fetch": "^3.3.1",
     "node-html-parser": "^6.1.5"
   },
   "devDependencies": {