mirror of
https://github.com/cupcakearmy/blaze.git
synced 2024-12-22 16:26:26 +00:00
Move to brave search engine + use fetch
This commit is contained in:
Danny Spina
parent
d082022007
commit
afa1804214
76
index.ts
76
index.ts
@ -4,6 +4,7 @@ import { JSDOM } from "jsdom";
|
||||
import got from "got";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import fetch from 'node-fetch'
|
||||
|
||||
const app = express();
|
||||
const port = 8888;
|
||||
@ -13,8 +14,8 @@ const __filename = fileURLToPath(import.meta.url);
|
||||
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
app.get("/", (req, res) => {
|
||||
const searchEngine = "http://html.duckduckgo.com/html/";
|
||||
app.get("/", async (req, res) => {
|
||||
const searchEngine = "https://api.search.brave.com/res/v1/web/search";
|
||||
let query = req.query.q as string;
|
||||
|
||||
if (!query) {
|
||||
@ -22,58 +23,29 @@ app.get("/", (req, res) => {
|
||||
return
|
||||
}
|
||||
|
||||
const options = {
|
||||
fetch(`${searchEngine}?q=${query}`, {
|
||||
headers: {
|
||||
'user-agent': req.headers["user-agent"]
|
||||
}
|
||||
}
|
||||
|
||||
got(`${searchEngine}?q=${query}`, options)
|
||||
.then((response) => {
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
console.log("Error code:", response.statusCode)
|
||||
res.send(`Error code ${response.statusCode}`)
|
||||
return;
|
||||
}
|
||||
|
||||
const dom = new JSDOM(response.body);
|
||||
const links = dom.window.document.querySelectorAll('link')
|
||||
const results = dom.window.document.querySelectorAll('.result')
|
||||
|
||||
// add custom style
|
||||
const style = dom.window.document.createElement('style')
|
||||
dom.window.document.querySelector('head')?.appendChild(style)
|
||||
style.innerHTML = `
|
||||
.result {
|
||||
padding: 1rem;
|
||||
border: 1px solid black;
|
||||
margin-bottom: 10px:
|
||||
}
|
||||
|
||||
.result__snippet, .result__url {
|
||||
text-decoration: none;
|
||||
color: black
|
||||
}
|
||||
`
|
||||
// clean up dom
|
||||
links.forEach(linkTag => linkTag.remove())
|
||||
dom.window.document.querySelector('form')?.remove()
|
||||
dom.window.document.querySelector('#header')?.remove()
|
||||
|
||||
// modify urls
|
||||
results.forEach(result => {
|
||||
const hrefArray = result.querySelector('.result__snippet')?.getAttribute('href')?.split('')
|
||||
hrefArray?.splice(0,25)
|
||||
const cleanHref = hrefArray?.join('')
|
||||
const newHref = `https://ill-red-skunk-wig.cyclic.app/blazed?url=${cleanHref}`;
|
||||
result.querySelector('.result__snippet')?.setAttribute('href', newHref)
|
||||
result.querySelector('.result__url')?.setAttribute('href', newHref)
|
||||
result.querySelector('.result__a')?.setAttribute('href', newHref)
|
||||
"Accept": "*/*",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"X-Subscription-Token": process.env.CYCLIC_BRAVE_KEY,
|
||||
},
|
||||
})
|
||||
.then((page) => {
|
||||
page.json().then(response => {
|
||||
const results = []
|
||||
response.web.results.forEach(result => {
|
||||
results.push(`
|
||||
<div>
|
||||
<h2>${result.title}</h2>
|
||||
<a href="https://ill-red-skunk-wig.cyclic.app/blazed?url=${result.url}">${result.url}</a>
|
||||
<a href="${result.url}">
|
||||
<p>${result.description}</p>
|
||||
</a>
|
||||
</div>
|
||||
`);
|
||||
})
|
||||
res.send(results.join(''))
|
||||
})
|
||||
|
||||
res.send(dom.serialize())
|
||||
return;
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log(err);
|
||||
|
||||
127
package-lock.json
generated
127
package-lock.json
generated
@ -14,6 +14,7 @@
|
||||
"fetch": "^1.1.0",
|
||||
"got": "^13.0.0",
|
||||
"jsdom": "^22.1.0",
|
||||
"node-fetch": "^3.3.1",
|
||||
"node-html-parser": "^6.1.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
@ -607,6 +608,14 @@
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/data-uri-to-buffer": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
|
||||
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
}
|
||||
},
|
||||
"node_modules/data-urls": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz",
|
||||
@ -879,6 +888,28 @@
|
||||
"encoding": "0.1.12"
|
||||
}
|
||||
},
|
||||
"node_modules/fetch-blob": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
|
||||
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "paypal",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"node-domexception": "^1.0.0",
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20 || >= 14.13"
|
||||
}
|
||||
},
|
||||
"node_modules/fill-range": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
|
||||
@ -929,6 +960,17 @@
|
||||
"node": ">= 14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/formdata-polyfill": {
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
|
||||
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
|
||||
"dependencies": {
|
||||
"fetch-blob": "^3.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/forwarded": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
|
||||
@ -1438,6 +1480,41 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz",
|
||||
"integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==",
|
||||
"dependencies": {
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/node-fetch"
|
||||
}
|
||||
},
|
||||
"node_modules/node-html-parser": {
|
||||
"version": "6.1.5",
|
||||
"resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz",
|
||||
@ -2122,6 +2199,14 @@
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
|
||||
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
@ -2717,6 +2802,11 @@
|
||||
"rrweb-cssom": "^0.6.0"
|
||||
}
|
||||
},
|
||||
"data-uri-to-buffer": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
|
||||
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="
|
||||
},
|
||||
"data-urls": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz",
|
||||
@ -2917,6 +3007,15 @@
|
||||
"encoding": "0.1.12"
|
||||
}
|
||||
},
|
||||
"fetch-blob": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
|
||||
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
|
||||
"requires": {
|
||||
"node-domexception": "^1.0.0",
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"fill-range": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
|
||||
@ -2955,6 +3054,14 @@
|
||||
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz",
|
||||
"integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw=="
|
||||
},
|
||||
"formdata-polyfill": {
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
|
||||
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
|
||||
"requires": {
|
||||
"fetch-blob": "^3.1.2"
|
||||
}
|
||||
},
|
||||
"forwarded": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
|
||||
@ -3314,6 +3421,21 @@
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
|
||||
"integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
|
||||
},
|
||||
"node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz",
|
||||
"integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==",
|
||||
"requires": {
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
}
|
||||
},
|
||||
"node-html-parser": {
|
||||
"version": "6.1.5",
|
||||
"resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz",
|
||||
@ -3817,6 +3939,11 @@
|
||||
"xml-name-validator": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"web-streams-polyfill": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
|
||||
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
|
||||
},
|
||||
"webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
"fetch": "^1.1.0",
|
||||
"got": "^13.0.0",
|
||||
"jsdom": "^22.1.0",
|
||||
"node-fetch": "^3.3.1",
|
||||
"node-html-parser": "^6.1.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user