From afa1804214ab1e4c9649637a0e342d159d5e6c3e Mon Sep 17 00:00:00 2001 From: Danny Spina Date: Mon, 19 Jun 2023 13:58:21 +0200 Subject: [PATCH] Move to brave search engine + use fetch --- index.ts | 76 +++++++++------------------ package-lock.json | 127 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 152 insertions(+), 52 deletions(-) diff --git a/index.ts b/index.ts index 5e54ea7..a45fde9 100644 --- a/index.ts +++ b/index.ts @@ -4,6 +4,7 @@ import { JSDOM } from "jsdom"; import got from "got"; import path from "path"; import { fileURLToPath } from "url"; +import fetch from 'node-fetch' const app = express(); const port = 8888; @@ -13,8 +14,8 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); -app.get("/", (req, res) => { - const searchEngine = "http://html.duckduckgo.com/html/"; +app.get("/", async (req, res) => { + const searchEngine = "https://api.search.brave.com/res/v1/web/search"; let query = req.query.q as string; if (!query) { @@ -22,58 +23,29 @@ app.get("/", (req, res) => { return } - const options = { + fetch(`${searchEngine}?q=${query}`, { headers: { - 'user-agent': req.headers["user-agent"] - } - } - - got(`${searchEngine}?q=${query}`, options) - .then((response) => { - - if (response.statusCode !== 200) { - console.log("Error code:", response.statusCode) - res.send(`Error code ${response.statusCode}`) - return; - } - - const dom = new JSDOM(response.body); - const links = dom.window.document.querySelectorAll('link') - const results = dom.window.document.querySelectorAll('.result') - - // add custom style - const style = dom.window.document.createElement('style') - dom.window.document.querySelector('head')?.appendChild(style) - style.innerHTML = ` - .result { - padding: 1rem; - border: 1px solid black; - margin-bottom: 10px: - } - - .result__snippet, .result__url { - text-decoration: none; - color: black - } - ` - // clean up dom - links.forEach(linkTag => linkTag.remove()) - dom.window.document.querySelector('form')?.remove() - dom.window.document.querySelector('#header')?.remove() - - // modify urls - results.forEach(result => { - const hrefArray = result.querySelector('.result__snippet')?.getAttribute('href')?.split('') - hrefArray?.splice(0,25) - const cleanHref = hrefArray?.join('') - const newHref = `https://ill-red-skunk-wig.cyclic.app/blazed?url=${cleanHref}`; - result.querySelector('.result__snippet')?.setAttribute('href', newHref) - result.querySelector('.result__url')?.setAttribute('href', newHref) - result.querySelector('.result__a')?.setAttribute('href', newHref) + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate, br", + "X-Subscription-Token": process.env.CYCLIC_BRAVE_KEY, + }, + }) + .then((page) => { + page.json().then(response => { + const results = [] + response.web.results.forEach(result => { + results.push(` +
+

${result.title}

+ ${result.url} + +

${result.description}

+ +
+ `); + }) + res.send(results.join('')) }) - - res.send(dom.serialize()) - return; }) .catch((err) => { console.log(err); diff --git a/package-lock.json b/package-lock.json index 297d8a5..1aeb848 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "fetch": "^1.1.0", "got": "^13.0.0", "jsdom": "^22.1.0", + "node-fetch": "^3.3.1", "node-html-parser": "^6.1.5" }, "devDependencies": { @@ -607,6 +608,14 @@ "node": ">=14" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", @@ -879,6 +888,28 @@ "encoding": "0.1.12" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -929,6 +960,17 @@ "node": ">= 14.17" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1438,6 +1480,41 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz", + "integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/node-html-parser": { "version": "6.1.5", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz", @@ -2122,6 +2199,14 @@ "node": ">=14" } }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -2717,6 +2802,11 @@ "rrweb-cssom": "^0.6.0" } }, + "data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" + }, "data-urls": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", @@ -2917,6 +3007,15 @@ "encoding": "0.1.12" } }, + "fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "requires": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -2955,6 +3054,14 @@ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==" }, + "formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "requires": { + "fetch-blob": "^3.1.2" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -3314,6 +3421,21 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" + }, + "node-fetch": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz", + "integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==", + "requires": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + } + }, "node-html-parser": { "version": "6.1.5", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz", @@ -3817,6 +3939,11 @@ "xml-name-validator": "^4.0.0" } }, + "web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==" + }, "webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", diff --git a/package.json b/package.json index b0e012d..4d6d875 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "fetch": "^1.1.0", "got": "^13.0.0", "jsdom": "^22.1.0", + "node-fetch": "^3.3.1", "node-html-parser": "^6.1.5" }, "devDependencies": {