This commit is contained in:
cupcakearmy 2021-11-16 11:20:33 +01:00
commit 8770b1fa2a
No known key found for this signature in database
GPG Key ID: 3235314B4D31232F
14 changed files with 2174 additions and 0 deletions

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
node_modules
dist
assets
*.tsbuildinfo

3
TODO.md Normal file
View File

@ -0,0 +1,3 @@
- storage drivers
- auto-format
- retention

2
morphus.yaml Normal file
View File

@ -0,0 +1,2 @@
allowedDomains:
- https://images.unsplash.com

35
package.json Normal file
View File

@ -0,0 +1,35 @@
{
"name": "morphus",
"version": "0.1.0",
"description": "",
"author": "Niccolo Borgioli",
"license": "MIT",
"scripts": {
"build": "tsc",
"dev": "tsnd src"
},
"devDependencies": {
"@types/convict": "^6.1.1",
"@types/flat": "^5.0.2",
"@types/ms": "^0.7.31",
"@types/node": "^16.11.7",
"@types/sharp": "^0.29.3",
"ts-node-dev": "^1.1.8",
"typescript": "^4.4.4"
},
"dependencies": {
"caniuse-db": "^1.0.30001280",
"class-validator": "^0.13.1",
"convict": "^6.2.1",
"device-detector-js": "^3.0.0",
"fastify": "^3.23.1",
"fastify-caching": "^6.1.0",
"fastify-compress": "^3.6.1",
"fastify-cors": "^6.0.2",
"flat": "^5.0.2",
"ms": "^2.1.3",
"sharp": "^0.29.3",
"under-pressure": "^5.8.0",
"yaml": "^1.10.2"
}
}

1417
pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

66
src/config.ts Normal file
View File

@ -0,0 +1,66 @@
import convict from 'convict'
import yaml from 'yaml'
export enum StorageType {
Local = 'local',
// S3 = 's3',
// GCS = 'gcs',
// Azure = 'azure',
}
export enum URLClean {
Off = 'off',
Fragment = 'fragment',
Query = 'query',
}
convict.addParser({ extension: ['yml', 'yaml'], parse: yaml.parse })
const config = convict({
// Security
allowedDomains: {
doc: 'The domains that are allowed to be used as image sources',
format: Array,
default: [] as string[],
env: 'ALLOWED_DOMAINS',
},
cleanUrls: {
doc: 'Whether to clean URLs',
format: Object.values(URLClean),
default: URLClean.Fragment,
env: 'CLEAN_URLS',
},
// Caching
maxAge: {
doc: 'The maximum age of a cached image',
format: String,
default: '1d',
env: 'MAX_AGE',
},
storage: {
doc: 'The storage engine to use',
format: Object.values(StorageType),
default: StorageType.Local,
env: 'STORAGE',
},
// Local storage
assets: {
doc: 'The path to the assets folder',
format: String,
default: './assets',
env: 'ASSETS',
},
})
for (const file of ['morphus.yaml', 'morphus.yaml', 'morphus.json']) {
try {
config.loadFile(file)
break
} catch {}
}
export const Config = config.get()
console.debug(Config)

209
src/controllers/index.ts Normal file
View File

@ -0,0 +1,209 @@
import {
IsDefined,
IsInt,
IsOptional,
IsPositive,
IsString,
IsUrl,
IsIn,
IsObject,
ValidateNested,
} from 'class-validator'
import { RouteHandlerMethod } from 'fastify'
import sharp, { FitEnum, FormatEnum } from 'sharp'
import { flatten, unflatten } from 'flat'
import ms from 'ms'
import DeviceDetector from 'device-detector-js'
import Avif from 'caniuse-db/features-json/avif.json'
import WebP from 'caniuse-db/features-json/webp.json'
import { storage } from '../storage'
import { transform } from '../transform'
import { sha3, sortObjectByKeys, validateSyncOrFail } from '../utils/utils'
import { Config, URLClean } from '../config'
const detector = new DeviceDetector()
export class ComplexParameter<N = string, T extends object = {}> {
@IsString()
name: N
@IsObject()
options: T
constructor(parameter: string) {
const [name, optionsRaw] = parameter.split('|')
if (!name) throw new Error('Invalid parameter')
this.name = name as any
this.options = {} as any
if (optionsRaw) {
for (const option of optionsRaw.split(',')) {
const [key, value] = option.split(':')
if (!key || !value) continue
// @ts-ignore
this.options[key] = ComplexParameter.ParseValue(value)
}
}
this.options = unflatten(this.options)
}
static ParseValue(value: string) {
if (value === 'true') return true
if (value === 'false') return false
const asNumber = Number(value)
if (!isNaN(asNumber)) return asNumber
return value
}
}
export enum ImageOperations {
resize,
flip,
flop,
affine,
sharpen,
median,
blur,
flatten,
gamma,
negate,
normalise,
normalize,
clahe,
convolve,
threshold,
boolean,
linear,
recomb,
modulate,
}
export enum ImageFormat {
jpeg,
png,
webp,
gif,
jp2,
tiff,
avif,
heif,
raw,
}
export class TransformQueryBase {
@IsString()
@IsUrl()
@IsDefined()
url!: string
@IsOptional()
@ValidateNested()
format?: ComplexParameter<keyof FormatEnum>
@IsOptional()
@IsIn(Object.values(sharp.fit))
resize?: keyof FitEnum
@IsOptional()
@IsInt()
@IsPositive()
width?: number
@IsOptional()
@IsInt()
@IsPositive()
height?: number
@ValidateNested()
op: ComplexParameter[] = []
hash: string
constructor(data: any, options: { ua?: string }) {
Object.assign(this, data)
if (this.width) this.width = parseInt(this.width as any)
if (this.height) this.height = parseInt(this.height as any)
this.op = Array.isArray(this.op) ? this.op : [this.op]
this.op = this.op.map((op) => new ComplexParameter(op as any))
if (this.format) this.format = new ComplexParameter(this.format as any)
if ((this.format?.name as string) === 'auto') {
if (!options.ua) throw new Error('cannot use auto format without user agent')
this.autoFormat(options.ua)
}
switch (Config.cleanUrls) {
case URLClean.Query: {
this.url = this.url.split('#')[0]!
this.url = this.url.split('?')[0]!
break
}
case URLClean.Fragment: {
this.url = this.url.split('#')[0]!
break
}
}
validateSyncOrFail(this)
if (this.resize) {
if (!this.width && !this.height) {
throw new Error('width or height is required when resizing')
}
}
this.hash = sha3(this.toString())
}
toString(): string {
const data = flatten(this) as Record<string, any>
return new URLSearchParams(sortObjectByKeys(data)).toString()
}
isAllowed(prefixes: string[]): boolean {
for (const prefix of prefixes) {
if (this.url.startsWith(prefix)) return true
}
return false
}
autoFormat(ua: string) {
const parsed = detector.parse(ua)
//https://caniuse.com/avif
console.log(parsed)
console.log(WebP)
// https://caniuse.com/webp
}
}
export const handler: RouteHandlerMethod = async (request, reply) => {
try {
const q = new TransformQueryBase(request.query, { ua: request.headers['user-agent'] })
if (!q.isAllowed(Config.allowedDomains)) {
reply.code(403).send({ error: 'Forbidden' })
return
}
// @ts-ignore
reply.etag(q.hash)
// @ts-ignore
reply.expires(new Date(Date.now() + ms(Config.maxAge)))
let stream: NodeJS.ReadableStream = (await storage.exists(q.hash))
? await storage.readStream(q.hash)
: await transform(q)
reply.code(200).headers({
'Content-Type': `image/${q.format?.name}`,
})
return stream
// .send(stream)
} catch (err) {
reply.code(400).send(err)
return
}
}

32
src/index.ts Normal file
View File

@ -0,0 +1,32 @@
// Require the framework and instantiate it
import fastify from 'fastify'
import compress from 'fastify-compress'
import cors from 'fastify-cors'
// @ts-ignore
import cache from 'fastify-caching'
import ms from 'ms'
import underPressure from 'under-pressure'
import { Config } from './config'
import { handler } from './controllers'
import { init } from './storage'
init()
const app = fastify({ logger: true })
app.register(underPressure)
app.register(cache, { expiresIn: ms(Config.maxAge) / 1000 })
app.register(compress, { global: true })
app.register(cors, { origin: true })
app.get('/api/image', handler)
async function start() {
try {
await app.listen(3000)
} catch (err) {
app.log.error(err)
process.exit(1)
}
}
start()

27
src/storage/index.ts Normal file
View File

@ -0,0 +1,27 @@
import { Config, StorageType } from '../config'
import { Local } from './local'
export interface Storage {
read(path: string): Promise<Buffer>
write(path: string, data: Buffer): Promise<void>
exists(path: string): Promise<boolean>
delete(path: string): Promise<void>
readStream(path: string): Promise<NodeJS.ReadableStream>
writeStream(path: string): Promise<NodeJS.WritableStream>
// list(path: string): Promise<string[]>
}
export let storage: Storage
export function init() {
if (!storage) {
switch (Config.storage) {
case StorageType.Local:
storage = new Local(Config.assets)
break
default:
throw new Error(`Unknown storage type: ${Config.storage}`)
}
}
}

76
src/storage/local.ts Normal file
View File

@ -0,0 +1,76 @@
import { resolve, join } from 'path'
import fs from 'fs'
import { Storage } from './'
export class Local implements Storage {
constructor(private readonly root: string) {
this.root = resolve(root)
}
read(path: string): Promise<Buffer> {
const file = join(this.root, path)
return new Promise((resolve, reject) => {
fs.readFile(file, (err, data) => {
if (err) {
return reject(err)
}
resolve(data)
})
})
}
write(path: string, data: Buffer): Promise<void> {
const file = join(this.root, path)
return new Promise((resolve, reject) => {
fs.writeFile(file, data, (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
exists(path: string): Promise<boolean> {
const file = join(this.root, path)
return new Promise((resolve, reject) => {
fs.access(file, fs.constants.F_OK, (err) => {
if (err) {
return resolve(false)
}
resolve(true)
})
})
}
delete(path: string): Promise<void> {
const file = join(this.root, path)
return new Promise((resolve, reject) => {
fs.unlink(file, (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
readStream(path: string): Promise<NodeJS.ReadableStream> {
const file = join(this.root, path)
return new Promise((resolve, reject) => {
const stream = fs.createReadStream(file)
stream.on('error', reject)
resolve(stream)
})
}
writeStream(path: string): Promise<NodeJS.WritableStream> {
const file = join(this.root, path)
return new Promise((resolve, reject) => {
const stream = fs.createWriteStream(file)
stream.on('error', reject)
resolve(stream)
})
}
}

114
src/transform/index.ts Normal file
View File

@ -0,0 +1,114 @@
import { get } from 'https'
import sharp from 'sharp'
import { PassThrough } from 'stream'
import { ComplexParameter, TransformQueryBase } from '../controllers'
import { storage } from '../storage'
import { sha3, splitter } from '../utils/utils'
async function downloadImage(url: string): Promise<NodeJS.ReadableStream> {
const disk = await storage.writeStream(sha3(url))
return new Promise((resolve) => {
get(url, (res) => {
const out = new PassThrough()
splitter(res, out, disk)
resolve(out)
})
})
}
export async function getImage(url: string): Promise<NodeJS.ReadableStream> {
const id = sha3(url)
if (!(await storage.exists(id))) {
return await downloadImage(url)
}
return await storage.readStream(id)
}
function applyOperation(pipeline: sharp.Sharp, { name, options }: ComplexParameter<string, any>): sharp.Sharp {
switch (name) {
case 'negate':
case 'clahe':
case 'convolve':
case 'modulate':
return pipeline[name](options)
case 'flip':
case 'flop':
case 'normalise':
case 'normalize':
case 'greyscale':
case 'grayscale':
case 'removeAlpha':
return pipeline[name]()
case 'rotate': {
const { angle, ...rest } = options
return pipeline.rotate(angle, rest)
}
case 'threshold': {
const { threshold, ...rest } = options
return pipeline.threshold(threshold, rest)
}
case 'boolean': {
const { operator, operand, ...rest } = options
return pipeline.boolean(operand, operator, rest)
}
case 'linear':
return pipeline.linear(options.a, options.b)
case 'sharpen':
return pipeline.sharpen(options.sigma, options.flat, options.jagged)
case 'media':
return pipeline.median(options.size)
case 'blur':
return pipeline.blur(options.sigma)
case 'flatten':
return pipeline.flatten(options.background)
case 'gamma':
return pipeline.gamma(options.gamma)
case 'tint':
return pipeline.tint(options.rgb)
case 'pipelineColorspace':
case 'pipelineColourspace':
return pipeline.pipelineColorspace(options.colorspace || options.colourspace)
case 'toColorspace':
case 'toColourspace':
return pipeline.toColorspace(options.colorspace || options.colourspace)
case 'ensureAlpha':
return pipeline.ensureAlpha(options.alpha)
case 'extractChannel':
return pipeline.extractChannel(options.channel)
default:
throw new Error(`Unsupported operation ${name}`)
}
}
export function buildPipeline(options: TransformQueryBase) {
let pipeline = sharp()
if (options.resize) {
pipeline = pipeline.resize({
fit: options.resize,
width: options.width,
height: options.height,
})
}
if (options.format) {
pipeline = pipeline.toFormat(options.format.name, options.format.options)
}
for (const op of options.op) {
try {
pipeline = applyOperation(pipeline, op)
} catch (e) {
throw new Error(`${op.name} is not a valid operation: ${e}`)
}
}
return pipeline
}
export async function transform(options: TransformQueryBase): Promise<NodeJS.ReadableStream> {
const source = await getImage(options.url)
const pipeline = buildPipeline(options)
const writer = await storage.writeStream(options.hash)
const out = new PassThrough()
splitter(source.pipe(pipeline), writer, out)
return out
}

50
src/utils/caniuse.ts Normal file
View File

@ -0,0 +1,50 @@
import DeviceDetector from 'device-detector-js'
import Avif from 'caniuse-db/features-json/avif.json'
import WebP from 'caniuse-db/features-json/webp.json'
const detector = new DeviceDetector()
function findLowestCompatibleVersion(stat: Record<string, string>): string {
const entries = Object.entries(stat).sort((a, b) => parseInt(a[0]) - parseInt(b[0]))
for (const [version, support] of entries) {
if (support.startsWith('y') || support.startsWith('a')) {
return version
}
}
}
const mapping = {
'Internet Explorer': 'ie',
'Microsoft Edge': 'edge',
Firefox: 'firefox',
Chrome: 'chrome',
Safari: 'safari',
Opera: 'opera',
'Mobile Safari': 'ios_saf',
'Opera Mini': 'op_mini',
'Android Browser': 'android',
'Chrome Mobile': 'and_chr',
'Firefox Mobile': 'and_ff',
'UC Browser': 'and_uc',
'Samsung Browser': 'samsung',
'QQ Browser': 'and_qq',
}
function matchBrowserToStat(browser: DeviceDetector.DeviceDetectorResult): string {
if (!browser.os || !browser.client) throw new Error('Invalid browser')
if (browser.os.name === 'iOS') {
return 'ios_saf'
}
if (browser.os.name in mapping) {
return mapping[browser.os.name as keyof typeof mapping]
}
throw new Error('Could not determine mapping for browser')
}
function match(feature: typeof Avif | typeof WebP, ua: string): boolean {
const browser = detector.parse(ua)
const stats = feature.stats[matchBrowserToStat(browser) as keyof typeof feature.stats]
console.debug(stats)
console.debug(findLowestCompatibleVersion(stats))
return false
}

39
src/utils/utils.ts Normal file
View File

@ -0,0 +1,39 @@
import { createHash } from 'crypto'
import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator'
import { PassThrough, Readable } from 'stream'
export class ValidationError extends Error {
override message: string
constructor(errors: VE[]) {
super()
this.message = errors
.map((e) => Object.values(e.constraints!))
.flat()
.join(', ')
}
}
export function validateSyncOrFail(data: object, options: ValidatorOptions = {}) {
options = Object.assign({ whitelist: true, forbidUnknownValues: true, skipMissingProperties: false }, options)
const errors = validateSync(data, options)
if (errors.length > 0) {
throw new ValidationError(errors)
}
}
export function sha3(url: string) {
return createHash('sha3-256').update(url).digest('hex')
}
export function sortObjectByKeys<T extends object>(obj: T): T {
return Object.fromEntries(Object.entries(obj).sort((a, b) => a[0].localeCompare(b[0]))) as T
}
export function splitter(from: NodeJS.ReadableStream, ...streams: NodeJS.WritableStream[]) {
const splitter = new PassThrough()
for (const stream of streams) {
splitter.pipe(stream)
}
from.pipe(splitter)
}

100
tsconfig.json Normal file
View File

@ -0,0 +1,100 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Projects */
"incremental": true /* Enable incremental compilation */,
"composite": true /* Enable constraints that allow a TypeScript project to be used with project references. */,
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
"experimentalDecorators": true /* Enable experimental support for TC39 stage 2 draft decorators. */,
"emitDecoratorMetadata": true /* Emit design-type metadata for decorated declarations in source files. */,
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
/* Modules */
"module": "commonjs" /* Specify what module code is generated. */,
"rootDir": "./src" /* Specify the root folder within your source files. */,
// "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
"resolveJsonModule": true /* Enable importing .json files */,
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
/* Emit */
"declaration": true /* Generate .d.ts files from TypeScript and JavaScript files in your project. */,
"declarationMap": true /* Create sourcemaps for d.ts files. */,
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
"sourceMap": true /* Create source map files for emitted JavaScript files. */,
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
"outDir": "./dist" /* Specify an output folder for all emitted files. */,
// "removeComments": true, /* Disable emitting comments. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
/* Interop Constraints */
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": true /* Enable all strict type-checking options. */,
"noImplicitAny": true /* Enable error reporting for expressions and declarations with an implied `any` type.. */,
"strictNullChecks": true /* When type checking, take into account `null` and `undefined`. */,
"strictFunctionTypes": true /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */,
"strictBindCallApply": true /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */,
"strictPropertyInitialization": true /* Check for class properties that are declared but not set in the constructor. */,
"noImplicitThis": true /* Enable error reporting when `this` is given the type `any`. */,
"useUnknownInCatchVariables": true /* Type catch clause variables as 'unknown' instead of 'any'. */,
"alwaysStrict": true /* Ensure 'use strict' is always emitted. */,
// "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */
// "exactOptionalPropertyTypes": true /* Interpret optional property types as written, rather than adding 'undefined'. */,
"noImplicitReturns": true /* Enable error reporting for codepaths that do not explicitly return in a function. */,
"noFallthroughCasesInSwitch": true /* Enable error reporting for fallthrough cases in switch statements. */,
"noUncheckedIndexedAccess": true /* Include 'undefined' in index signature results */,
"noImplicitOverride": true /* Ensure overriding members in derived classes are marked with an override modifier. */,
"noPropertyAccessFromIndexSignature": true /* Enforces using indexed accessors for keys declared using an indexed type */,
"allowUnusedLabels": true /* Disable error reporting for unused labels. */,
"allowUnreachableCode": true /* Disable error reporting for unreachable code. */,
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */
}
}