From bc71889ae26484b744e53f6d9d080ca5688a2b40 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Tue, 16 Nov 2021 16:16:14 +0100 Subject: [PATCH] progress --- README.md | 10 ++ TODO.md | 1 - morphus.yaml | 5 +- package.json | 5 +- pnpm-lock.yaml | 26 +++-- src/config.ts | 46 ++++++++- src/controllers/image.ts | 204 ++++++++++++++++++++++++++++++++++++ src/controllers/index.ts | 205 +------------------------------------ src/controllers/version.ts | 7 ++ src/index.ts | 18 ++-- src/transform/index.ts | 2 +- src/utils/errors.ts | 5 + src/utils/utils.ts | 10 ++ tsconfig.json | 5 +- 14 files changed, 321 insertions(+), 228 deletions(-) create mode 100644 README.md create mode 100644 src/controllers/image.ts create mode 100644 src/controllers/version.ts create mode 100644 src/utils/errors.ts diff --git a/README.md b/README.md new file mode 100644 index 0000000..bcb19af --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# morphus 🖼 + +## 🌈 Features + +- Config driven +- Domain protection +- Host verification +- Multiple storage adapters (Local, S3, GCS) +- Caniuse based automatic formatting +- Highly performant diff --git a/TODO.md b/TODO.md index 37bda08..6c970fd 100644 --- a/TODO.md +++ b/TODO.md @@ -1,3 +1,2 @@ - storage drivers -- auto-format - retention diff --git a/morphus.yaml b/morphus.yaml index 6b9ba6b..443a8a6 100644 --- a/morphus.yaml +++ b/morphus.yaml @@ -1,2 +1,5 @@ allowedDomains: - - https://images.unsplash.com + - !regexp ^https?:\/\/images.unsplash.com +allowedHosts: + - foo.local:5000 +cleanUrls: query diff --git a/package.json b/package.json index 96f9e87..d023c9e 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "devDependencies": { "@types/convict": "^6.1.1", "@types/flat": "^5.0.2", + "@types/js-yaml": "^4.0.4", "@types/ms": "^0.7.31", "@types/node": "^16.11.7", "@types/sharp": "^0.29.3", @@ -27,9 +28,9 @@ "fastify-compress": "^3.6.1", "fastify-cors": "^6.0.2", "flat": "^5.0.2", + "js-yaml": "^4.1.0", "ms": "^2.1.3", "sharp": "^0.29.3", - "under-pressure": "^5.8.0", - "yaml": "^1.10.2" + "under-pressure": "^5.8.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1968ea9..1663d90 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3,6 +3,7 @@ lockfileVersion: 5.3 specifiers: '@types/convict': ^6.1.1 '@types/flat': ^5.0.2 + '@types/js-yaml': ^4.0.4 '@types/ms': ^0.7.31 '@types/node': ^16.11.7 '@types/sharp': ^0.29.3 @@ -15,12 +16,12 @@ specifiers: fastify-compress: ^3.6.1 fastify-cors: ^6.0.2 flat: ^5.0.2 + js-yaml: ^4.1.0 ms: ^2.1.3 sharp: ^0.29.3 ts-node-dev: ^1.1.8 typescript: ^4.4.4 under-pressure: ^5.8.0 - yaml: ^1.10.2 dependencies: caniuse-db: 1.0.30001280 @@ -32,14 +33,15 @@ dependencies: fastify-compress: 3.6.1 fastify-cors: 6.0.2 flat: 5.0.2 + js-yaml: 4.1.0 ms: 2.1.3 sharp: 0.29.3 under-pressure: 5.8.0 - yaml: 1.10.2 devDependencies: '@types/convict': 6.1.1 '@types/flat': 5.0.2 + '@types/js-yaml': 4.0.4 '@types/ms': 0.7.31 '@types/node': 16.11.7 '@types/sharp': 0.29.3 @@ -64,6 +66,10 @@ packages: resolution: {integrity: sha512-3zsplnP2djeps5P9OyarTxwRpMLoe5Ash8aL9iprw0JxB+FAHjY+ifn4yZUuW4/9hqtnmor6uvjSRzJhiVbrEQ==} dev: true + /@types/js-yaml/4.0.4: + resolution: {integrity: sha512-AuHubXUmg0AzkXH0Mx6sIxeY/1C110mm/EkE/gB1sTRz3h2dao2W/63q42SlVST+lICxz5Oki2hzYA6+KnnieQ==} + dev: true + /@types/ms/0.7.31: resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} dev: true @@ -152,6 +158,10 @@ packages: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} dev: true + /argparse/2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: false + /atomic-sleep/1.0.0: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} @@ -753,6 +763,13 @@ packages: resolution: {integrity: sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=} dev: false + /js-yaml/4.1.0: + resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} + hasBin: true + dependencies: + argparse: 2.0.1 + dev: false + /json-schema-traverse/0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} dev: false @@ -1401,11 +1418,6 @@ packages: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} dev: false - /yaml/1.10.2: - resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} - engines: {node: '>= 6'} - dev: false - /yargs-parser/20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} diff --git a/src/config.ts b/src/config.ts index ec28944..a40ab33 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,5 @@ import convict from 'convict' -import yaml from 'yaml' +import yaml from 'js-yaml' export enum StorageType { Local = 'local', @@ -14,15 +14,43 @@ export enum URLClean { Query = 'query', } -convict.addParser({ extension: ['yml', 'yaml'], parse: yaml.parse }) +const RegExpTag = new yaml.Type('!regexp', { + kind: 'scalar', + construct: (value: string) => new RegExp(value), + instanceOf: RegExp, +}) + +const Schema = yaml.DEFAULT_SCHEMA.extend([RegExpTag]) + +export type NullableStringOrRegexpArray = (string | RegExp)[] | null + +function formatNullableStringOrRegexpArray(values: any) { + if (!Array.isArray(values)) throw new Error('must be an array') + if (values.length === 0) throw new Error('must be an array with at least one element') + for (const value of values) { + if (typeof value === 'string') continue + if (value instanceof RegExp) continue + throw new Error('must be an array of strings or regexps') + } +} + +convict.addParser({ extension: ['yml', 'yaml'], parse: (s) => yaml.load(s, { schema: Schema }) }) const config = convict({ // Security allowedDomains: { doc: 'The domains that are allowed to be used as image sources', - format: Array, - default: [] as string[], + format: formatNullableStringOrRegexpArray, + default: null as NullableStringOrRegexpArray, + nullable: true, env: 'ALLOWED_DOMAINS', }, + allowedHosts: { + doc: 'The hosts that are allowed to access the images', + format: formatNullableStringOrRegexpArray, + default: null as NullableStringOrRegexpArray, + nullable: true, + env: 'ALLOWED_HOSTS', + }, cleanUrls: { doc: 'Whether to clean URLs', format: Object.values(URLClean), @@ -61,6 +89,16 @@ for (const file of ['morphus.yaml', 'morphus.yaml', 'morphus.json']) { } catch {} } +try { + config.validate({ allowed: 'strict' }) +} catch (e) { + if (e instanceof Error) { + console.error(e.message) + } else { + console.error(e) + } + process.exit(1) +} export const Config = config.get() console.debug(Config) diff --git a/src/controllers/image.ts b/src/controllers/image.ts new file mode 100644 index 0000000..e52e8df --- /dev/null +++ b/src/controllers/image.ts @@ -0,0 +1,204 @@ +import { + IsDefined, + IsInt, + IsOptional, + IsPositive, + IsString, + IsUrl, + IsIn, + IsObject, + ValidateNested, +} from 'class-validator' +import { RouteHandlerMethod } from 'fastify' +import sharp, { FitEnum, FormatEnum } from 'sharp' +import { flatten, unflatten } from 'flat' +import ms from 'ms' + +import { storage } from '../storage' +import { transform } from '../transform' +import { sha3, sortObjectByKeys, testForPrefixOrRegexp, validateSyncOrFail } from '../utils/utils' +import { Config, NullableStringOrRegexpArray, URLClean } from '../config' +import { supportsAvif, supportsWebP } from '../utils/caniuse' +import { ForbiddenError } from '../utils/errors' + +export enum ImageOperations { + resize, + flip, + flop, + affine, + sharpen, + median, + blur, + flatten, + gamma, + negate, + normalise, + normalize, + clahe, + convolve, + threshold, + boolean, + linear, + recomb, + modulate, +} + +export enum ImageFormat { + jpeg, + png, + webp, + gif, + jp2, + tiff, + avif, + heif, + raw, +} +export class ComplexParameter { + @IsString() + name: N + + @IsObject() + options: T + + constructor(parameter: string) { + const [name, optionsRaw] = parameter.split('|') + if (!name) throw new Error('Invalid parameter') + this.name = name as any + this.options = {} as any + if (optionsRaw) { + for (const option of optionsRaw.split(',')) { + const [key, value] = option.split(':') + if (!key || !value) continue + // @ts-ignore + this.options[key] = ComplexParameter.ParseValue(value) + } + } + this.options = unflatten(this.options) + } + + static ParseValue(value: string) { + if (value === 'true') return true + if (value === 'false') return false + + const asNumber = Number(value) + if (!isNaN(asNumber)) return asNumber + + return value + } +} + +export class TransformQueryBase { + @IsString() + @IsUrl() + @IsDefined() + url!: string + + @IsOptional() + @ValidateNested() + format: ComplexParameter + + @IsOptional() + @IsIn(Object.values(sharp.fit)) + resize?: keyof FitEnum + + @IsOptional() + @IsInt() + @IsPositive() + width?: number + + @IsOptional() + @IsInt() + @IsPositive() + height?: number + + @ValidateNested() + op: ComplexParameter[] = [] + + constructor(data: any, options: { ua?: string }) { + Object.assign(this, data) + + if (this.width) this.width = parseInt(this.width as any) + if (this.height) this.height = parseInt(this.height as any) + + this.op = Array.isArray(this.op) ? this.op : [this.op] + this.op = this.op.map((op) => new ComplexParameter(op as any)) + + // @ts-ignore + this.format = new ComplexParameter((this.format as any) || 'auto') + if ((this.format.name as string) === 'auto') { + if (!options.ua) throw new Error('cannot use auto format without user agent') + this.autoFormat(options.ua) + } + + validateSyncOrFail(this) + if (this.resize) { + if (!this.width && !this.height) { + throw new Error('width or height is required when resizing') + } + } + + switch (Config.cleanUrls) { + case URLClean.Query: { + this.url = this.url.split('#')[0]! + this.url = this.url.split('?')[0]! + break + } + case URLClean.Fragment: { + this.url = this.url.split('#')[0]! + break + } + } + } + + toString(): string { + const data = flatten(this) as Record + return new URLSearchParams(sortObjectByKeys(data)).toString() + } + + autoFormat(ua: string) { + if (supportsAvif(ua)) this.format!.name = 'avif' + else if (supportsWebP(ua)) this.format!.name = 'webp' + else this.format!.name = 'jpeg' + } + + get hash(): string { + return sha3(this.toString()) + } +} + +export const image: RouteHandlerMethod = async (request, reply) => { + try { + const q = new TransformQueryBase(request.query, { ua: request.headers['user-agent'] }) + + if (Config.allowedDomains) { + if (!testForPrefixOrRegexp(q.url, Config.allowedDomains)) + return ForbiddenError(reply, 'source domain not allowed') + } + + if (Config.allowedHosts) { + const host = request.headers.host + console.debug('Testing host', host, Config.allowedHosts) + if (!host || !testForPrefixOrRegexp(host, Config.allowedHosts)) return ForbiddenError(reply, 'host not allowed') + } + + // @ts-ignore + reply.etag(q.hash) + // @ts-ignore + reply.expires(new Date(Date.now() + ms(Config.maxAge))) + + let stream: NodeJS.ReadableStream = (await storage.exists(q.hash)) + ? await storage.readStream(q.hash) + : await transform(q) + + reply.code(200).headers({ + 'Content-Type': `image/${q.format?.name}`, + }) + + return stream + // .send(stream) + } catch (err) { + reply.code(400).send(err) + return + } +} diff --git a/src/controllers/index.ts b/src/controllers/index.ts index a6d8a4b..b5ef511 100644 --- a/src/controllers/index.ts +++ b/src/controllers/index.ts @@ -1,203 +1,2 @@ -import { - IsDefined, - IsInt, - IsOptional, - IsPositive, - IsString, - IsUrl, - IsIn, - IsObject, - ValidateNested, -} from 'class-validator' -import { RouteHandlerMethod } from 'fastify' -import sharp, { FitEnum, FormatEnum } from 'sharp' -import { flatten, unflatten } from 'flat' -import ms from 'ms' - -import { storage } from '../storage' -import { transform } from '../transform' -import { sha3, sortObjectByKeys, validateSyncOrFail } from '../utils/utils' -import { Config, URLClean } from '../config' -import { supportsAvif, supportsWebP } from '../utils/caniuse' - -export class ComplexParameter { - @IsString() - name: N - - @IsObject() - options: T - - constructor(parameter: string) { - const [name, optionsRaw] = parameter.split('|') - if (!name) throw new Error('Invalid parameter') - this.name = name as any - this.options = {} as any - if (optionsRaw) { - for (const option of optionsRaw.split(',')) { - const [key, value] = option.split(':') - if (!key || !value) continue - // @ts-ignore - this.options[key] = ComplexParameter.ParseValue(value) - } - } - this.options = unflatten(this.options) - } - - static ParseValue(value: string) { - if (value === 'true') return true - if (value === 'false') return false - - const asNumber = Number(value) - if (!isNaN(asNumber)) return asNumber - - return value - } -} - -export enum ImageOperations { - resize, - flip, - flop, - affine, - sharpen, - median, - blur, - flatten, - gamma, - negate, - normalise, - normalize, - clahe, - convolve, - threshold, - boolean, - linear, - recomb, - modulate, -} - -export enum ImageFormat { - jpeg, - png, - webp, - gif, - jp2, - tiff, - avif, - heif, - raw, -} - -export class TransformQueryBase { - @IsString() - @IsUrl() - @IsDefined() - url!: string - - @IsOptional() - @ValidateNested() - format?: ComplexParameter - - @IsOptional() - @IsIn(Object.values(sharp.fit)) - resize?: keyof FitEnum - - @IsOptional() - @IsInt() - @IsPositive() - width?: number - - @IsOptional() - @IsInt() - @IsPositive() - height?: number - - @ValidateNested() - op: ComplexParameter[] = [] - - hash: string - - constructor(data: any, options: { ua?: string }) { - Object.assign(this, data) - - if (this.width) this.width = parseInt(this.width as any) - if (this.height) this.height = parseInt(this.height as any) - - this.op = Array.isArray(this.op) ? this.op : [this.op] - this.op = this.op.map((op) => new ComplexParameter(op as any)) - if (this.format) this.format = new ComplexParameter(this.format as any) - if ((this.format?.name as string) === 'auto') { - if (!options.ua) throw new Error('cannot use auto format without user agent') - this.autoFormat(options.ua) - } - - switch (Config.cleanUrls) { - case URLClean.Query: { - this.url = this.url.split('#')[0]! - this.url = this.url.split('?')[0]! - break - } - case URLClean.Fragment: { - this.url = this.url.split('#')[0]! - break - } - } - - validateSyncOrFail(this) - if (this.resize) { - if (!this.width && !this.height) { - throw new Error('width or height is required when resizing') - } - } - - this.hash = sha3(this.toString()) - } - - toString(): string { - const data = flatten(this) as Record - return new URLSearchParams(sortObjectByKeys(data)).toString() - } - - isAllowed(prefixes: string[]): boolean { - for (const prefix of prefixes) { - if (this.url.startsWith(prefix)) return true - } - return false - } - - autoFormat(ua: string) { - if (supportsAvif(ua)) this.format!.name = 'avif' - else if (supportsWebP(ua)) this.format!.name = 'webp' - else this.format!.name = 'jpeg' - } -} - -export const handler: RouteHandlerMethod = async (request, reply) => { - try { - const q = new TransformQueryBase(request.query, { ua: request.headers['user-agent'] }) - - if (!q.isAllowed(Config.allowedDomains)) { - reply.code(403).send({ error: 'Forbidden' }) - return - } - - // @ts-ignore - reply.etag(q.hash) - // @ts-ignore - reply.expires(new Date(Date.now() + ms(Config.maxAge))) - - let stream: NodeJS.ReadableStream = (await storage.exists(q.hash)) - ? await storage.readStream(q.hash) - : await transform(q) - - reply.code(200).headers({ - 'Content-Type': `image/${q.format?.name}`, - }) - - return stream - // .send(stream) - } catch (err) { - reply.code(400).send(err) - return - } -} +export * from './image' +export * from './version' diff --git a/src/controllers/version.ts b/src/controllers/version.ts new file mode 100644 index 0000000..dfc2e5c --- /dev/null +++ b/src/controllers/version.ts @@ -0,0 +1,7 @@ +import { RouteHandlerMethod } from 'fastify' + +import { version as v } from '../../package.json' + +export const version: RouteHandlerMethod = async (request, reply) => { + return { version: v } +} diff --git a/src/index.ts b/src/index.ts index 05634b8..3bbd236 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,24 +2,28 @@ import fastify from 'fastify' import compress from 'fastify-compress' import cors from 'fastify-cors' -// @ts-ignore -import cache from 'fastify-caching' -import ms from 'ms' import underPressure from 'under-pressure' -import { Config } from './config' -import { handler } from './controllers' +import './config' +import { version } from './controllers' +import { image } from './controllers/image' import { init } from './storage' init() const app = fastify({ logger: true }) app.register(underPressure) -app.register(cache, { expiresIn: ms(Config.maxAge) / 1000 }) +app.register(require('fastify-caching')) app.register(compress, { global: true }) app.register(cors, { origin: true }) -app.get('/api/image', handler) +app.addHook('preHandler', (request, reply, done) => { + reply.header('Server', 'morphus') + done() +}) + +app.get('/api/image', image) +app.get('/version', version) async function start() { try { diff --git a/src/transform/index.ts b/src/transform/index.ts index 26c7717..7ad0ff3 100644 --- a/src/transform/index.ts +++ b/src/transform/index.ts @@ -1,7 +1,7 @@ import { get } from 'https' import sharp from 'sharp' import { PassThrough } from 'stream' -import { ComplexParameter, TransformQueryBase } from '../controllers' +import { ComplexParameter, TransformQueryBase } from '../controllers/image' import { storage } from '../storage' import { sha3, splitter } from '../utils/utils' diff --git a/src/utils/errors.ts b/src/utils/errors.ts new file mode 100644 index 0000000..66c97b0 --- /dev/null +++ b/src/utils/errors.ts @@ -0,0 +1,5 @@ +import { FastifyReply } from 'fastify' + +export function ForbiddenError(reply: FastifyReply, message?: string) { + reply.code(403).send({ error: 'Forbidden', message }) +} diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 2dbfa26..c6eb5e7 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,6 +1,7 @@ import { createHash } from 'crypto' import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator' import { PassThrough, Readable } from 'stream' +import { NullableStringOrRegexpArray } from '../config' export class ValidationError extends Error { override message: string @@ -37,3 +38,12 @@ export function splitter(from: NodeJS.ReadableStream, ...streams: NodeJS.Writabl } from.pipe(splitter) } + +export function testForPrefixOrRegexp(str: string, values: (string | RegExp)[]): boolean { + for (const value of values) { + if (typeof value === 'string') { + if (str.startsWith(value)) return true + } else if (value.test(str)) return true + } + return false +} diff --git a/tsconfig.json b/tsconfig.json index 9bea134..b6fe9ca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,7 +25,7 @@ /* Modules */ "module": "commonjs" /* Specify what module code is generated. */, - "rootDir": "./src" /* Specify the root folder within your source files. */, + // "rootDir": "./src" /* Specify the root folder within your source files. */, // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ @@ -96,5 +96,6 @@ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ - } + }, + "include": ["./src", "./package.json"] }