progress

README.md

@@ -0,0 +1,10 @@
+# morphus 🖼
+
+## 🌈 Features
+
+- Config driven
+- Domain protection
+- Host verification
+- Multiple storage adapters (Local, S3, GCS)
+- Caniuse based automatic formatting
+- Highly performant

TODO.md

@@ -1,3 +1,2 @@
 - storage drivers
-- auto-format
 - retention

morphus.yaml

@@ -1,2 +1,5 @@
 allowedDomains:
-  - https://images.unsplash.com
+  - !regexp ^https?:\/\/images.unsplash.com
+allowedHosts:
+  - foo.local:5000
+cleanUrls: query

package.json

@@ -11,6 +11,7 @@
   "devDependencies": {
     "@types/convict": "^6.1.1",
     "@types/flat": "^5.0.2",
+    "@types/js-yaml": "^4.0.4",
     "@types/ms": "^0.7.31",
     "@types/node": "^16.11.7",
     "@types/sharp": "^0.29.3",
@@ -27,9 +28,9 @@
     "fastify-compress": "^3.6.1",
     "fastify-cors": "^6.0.2",
     "flat": "^5.0.2",
+    "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
     "sharp": "^0.29.3",
-    "under-pressure": "^5.8.0",
+    "under-pressure": "^5.8.0"
-    "yaml": "^1.10.2"
   }
 }

pnpm-lock.yaml

@@ -3,6 +3,7 @@ lockfileVersion: 5.3
 specifiers:
   '@types/convict': ^6.1.1
   '@types/flat': ^5.0.2
+  '@types/js-yaml': ^4.0.4
   '@types/ms': ^0.7.31
   '@types/node': ^16.11.7
   '@types/sharp': ^0.29.3
@@ -15,12 +16,12 @@ specifiers:
   fastify-compress: ^3.6.1
   fastify-cors: ^6.0.2
   flat: ^5.0.2
+  js-yaml: ^4.1.0
   ms: ^2.1.3
   sharp: ^0.29.3
   ts-node-dev: ^1.1.8
   typescript: ^4.4.4
   under-pressure: ^5.8.0
-  yaml: ^1.10.2
 
 dependencies:
   caniuse-db: 1.0.30001280
@@ -32,14 +33,15 @@ dependencies:
   fastify-compress: 3.6.1
   fastify-cors: 6.0.2
   flat: 5.0.2
+  js-yaml: 4.1.0
   ms: 2.1.3
   sharp: 0.29.3
   under-pressure: 5.8.0
-  yaml: 1.10.2
 
 devDependencies:
   '@types/convict': 6.1.1
   '@types/flat': 5.0.2
+  '@types/js-yaml': 4.0.4
   '@types/ms': 0.7.31
   '@types/node': 16.11.7
   '@types/sharp': 0.29.3
@@ -64,6 +66,10 @@ packages:
     resolution: {integrity: sha512-3zsplnP2djeps5P9OyarTxwRpMLoe5Ash8aL9iprw0JxB+FAHjY+ifn4yZUuW4/9hqtnmor6uvjSRzJhiVbrEQ==}
     dev: true
 
+  /@types/js-yaml/4.0.4:
+    resolution: {integrity: sha512-AuHubXUmg0AzkXH0Mx6sIxeY/1C110mm/EkE/gB1sTRz3h2dao2W/63q42SlVST+lICxz5Oki2hzYA6+KnnieQ==}
+    dev: true
+
   /@types/ms/0.7.31:
     resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
     dev: true
@@ -152,6 +158,10 @@ packages:
     resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
     dev: true
 
+  /argparse/2.0.1:
+    resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
+    dev: false
+
   /atomic-sleep/1.0.0:
     resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
     engines: {node: '>=8.0.0'}
@@ -753,6 +763,13 @@ packages:
     resolution: {integrity: sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=}
     dev: false
 
+  /js-yaml/4.1.0:
+    resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==}
+    hasBin: true
+    dependencies:
+      argparse: 2.0.1
+    dev: false
+
   /json-schema-traverse/0.4.1:
     resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
     dev: false
@@ -1401,11 +1418,6 @@ packages:
     resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
     dev: false
 
-  /yaml/1.10.2:
-    resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
-    engines: {node: '>= 6'}
-    dev: false
-
   /yargs-parser/20.2.9:
     resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==}
     engines: {node: '>=10'}

src/config.ts

@@ -1,5 +1,5 @@
 import convict from 'convict'
-import yaml from 'yaml'
+import yaml from 'js-yaml'
 
 export enum StorageType {
   Local = 'local',
@@ -14,15 +14,43 @@ export enum URLClean {
   Query = 'query',
 }
 
-convict.addParser({ extension: ['yml', 'yaml'], parse: yaml.parse })
+const RegExpTag = new yaml.Type('!regexp', {
+  kind: 'scalar',
+  construct: (value: string) => new RegExp(value),
+  instanceOf: RegExp,
+})
+
+const Schema = yaml.DEFAULT_SCHEMA.extend([RegExpTag])
+
+export type NullableStringOrRegexpArray = (string | RegExp)[] | null
+
+function formatNullableStringOrRegexpArray(values: any) {
+  if (!Array.isArray(values)) throw new Error('must be an array')
+  if (values.length === 0) throw new Error('must be an array with at least one element')
+  for (const value of values) {
+    if (typeof value === 'string') continue
+    if (value instanceof RegExp) continue
+    throw new Error('must be an array of strings or regexps')
+  }
+}
+
+convict.addParser({ extension: ['yml', 'yaml'], parse: (s) => yaml.load(s, { schema: Schema }) })
 const config = convict({
   // Security
   allowedDomains: {
     doc: 'The domains that are allowed to be used as image sources',
-    format: Array,
+    format: formatNullableStringOrRegexpArray,
-    default: [] as string[],
+    default: null as NullableStringOrRegexpArray,
+    nullable: true,
     env: 'ALLOWED_DOMAINS',
   },
+  allowedHosts: {
+    doc: 'The hosts that are allowed to access the images',
+    format: formatNullableStringOrRegexpArray,
+    default: null as NullableStringOrRegexpArray,
+    nullable: true,
+    env: 'ALLOWED_HOSTS',
+  },
   cleanUrls: {
     doc: 'Whether to clean URLs',
     format: Object.values(URLClean),
@@ -61,6 +89,16 @@ for (const file of ['morphus.yaml', 'morphus.yaml', 'morphus.json']) {
   } catch {}
 }
 
+try {
+  config.validate({ allowed: 'strict' })
+} catch (e) {
+  if (e instanceof Error) {
+    console.error(e.message)
+  } else {
+    console.error(e)
+  }
+  process.exit(1)
+}
 export const Config = config.get()
 
 console.debug(Config)

src/controllers/image.ts

@@ -0,0 +1,204 @@
+import {
+  IsDefined,
+  IsInt,
+  IsOptional,
+  IsPositive,
+  IsString,
+  IsUrl,
+  IsIn,
+  IsObject,
+  ValidateNested,
+} from 'class-validator'
+import { RouteHandlerMethod } from 'fastify'
+import sharp, { FitEnum, FormatEnum } from 'sharp'
+import { flatten, unflatten } from 'flat'
+import ms from 'ms'
+
+import { storage } from '../storage'
+import { transform } from '../transform'
+import { sha3, sortObjectByKeys, testForPrefixOrRegexp, validateSyncOrFail } from '../utils/utils'
+import { Config, NullableStringOrRegexpArray, URLClean } from '../config'
+import { supportsAvif, supportsWebP } from '../utils/caniuse'
+import { ForbiddenError } from '../utils/errors'
+
+export enum ImageOperations {
+  resize,
+  flip,
+  flop,
+  affine,
+  sharpen,
+  median,
+  blur,
+  flatten,
+  gamma,
+  negate,
+  normalise,
+  normalize,
+  clahe,
+  convolve,
+  threshold,
+  boolean,
+  linear,
+  recomb,
+  modulate,
+}
+
+export enum ImageFormat {
+  jpeg,
+  png,
+  webp,
+  gif,
+  jp2,
+  tiff,
+  avif,
+  heif,
+  raw,
+}
+export class ComplexParameter<N = string, T extends object = {}> {
+  @IsString()
+  name: N
+
+  @IsObject()
+  options: T
+
+  constructor(parameter: string) {
+    const [name, optionsRaw] = parameter.split('|')
+    if (!name) throw new Error('Invalid parameter')
+    this.name = name as any
+    this.options = {} as any
+    if (optionsRaw) {
+      for (const option of optionsRaw.split(',')) {
+        const [key, value] = option.split(':')
+        if (!key || !value) continue
+        // @ts-ignore
+        this.options[key] = ComplexParameter.ParseValue(value)
+      }
+    }
+    this.options = unflatten(this.options)
+  }
+
+  static ParseValue(value: string) {
+    if (value === 'true') return true
+    if (value === 'false') return false
+
+    const asNumber = Number(value)
+    if (!isNaN(asNumber)) return asNumber
+
+    return value
+  }
+}
+
+export class TransformQueryBase {
+  @IsString()
+  @IsUrl()
+  @IsDefined()
+  url!: string
+
+  @IsOptional()
+  @ValidateNested()
+  format: ComplexParameter<keyof FormatEnum>
+
+  @IsOptional()
+  @IsIn(Object.values(sharp.fit))
+  resize?: keyof FitEnum
+
+  @IsOptional()
+  @IsInt()
+  @IsPositive()
+  width?: number
+
+  @IsOptional()
+  @IsInt()
+  @IsPositive()
+  height?: number
+
+  @ValidateNested()
+  op: ComplexParameter[] = []
+
+  constructor(data: any, options: { ua?: string }) {
+    Object.assign(this, data)
+
+    if (this.width) this.width = parseInt(this.width as any)
+    if (this.height) this.height = parseInt(this.height as any)
+
+    this.op = Array.isArray(this.op) ? this.op : [this.op]
+    this.op = this.op.map((op) => new ComplexParameter(op as any))
+
+    // @ts-ignore
+    this.format = new ComplexParameter((this.format as any) || 'auto')
+    if ((this.format.name as string) === 'auto') {
+      if (!options.ua) throw new Error('cannot use auto format without user agent')
+      this.autoFormat(options.ua)
+    }
+
+    validateSyncOrFail(this)
+    if (this.resize) {
+      if (!this.width && !this.height) {
+        throw new Error('width or height is required when resizing')
+      }
+    }
+
+    switch (Config.cleanUrls) {
+      case URLClean.Query: {
+        this.url = this.url.split('#')[0]!
+        this.url = this.url.split('?')[0]!
+        break
+      }
+      case URLClean.Fragment: {
+        this.url = this.url.split('#')[0]!
+        break
+      }
+    }
+  }
+
+  toString(): string {
+    const data = flatten(this) as Record<string, any>
+    return new URLSearchParams(sortObjectByKeys(data)).toString()
+  }
+
+  autoFormat(ua: string) {
+    if (supportsAvif(ua)) this.format!.name = 'avif'
+    else if (supportsWebP(ua)) this.format!.name = 'webp'
+    else this.format!.name = 'jpeg'
+  }
+
+  get hash(): string {
+    return sha3(this.toString())
+  }
+}
+
+export const image: RouteHandlerMethod = async (request, reply) => {
+  try {
+    const q = new TransformQueryBase(request.query, { ua: request.headers['user-agent'] })
+
+    if (Config.allowedDomains) {
+      if (!testForPrefixOrRegexp(q.url, Config.allowedDomains))
+        return ForbiddenError(reply, 'source domain not allowed')
+    }
+
+    if (Config.allowedHosts) {
+      const host = request.headers.host
+      console.debug('Testing host', host, Config.allowedHosts)
+      if (!host || !testForPrefixOrRegexp(host, Config.allowedHosts)) return ForbiddenError(reply, 'host not allowed')
+    }
+
+    // @ts-ignore
+    reply.etag(q.hash)
+    // @ts-ignore
+    reply.expires(new Date(Date.now() + ms(Config.maxAge)))
+
+    let stream: NodeJS.ReadableStream = (await storage.exists(q.hash))
+      ? await storage.readStream(q.hash)
+      : await transform(q)
+
+    reply.code(200).headers({
+      'Content-Type': `image/${q.format?.name}`,
+    })
+
+    return stream
+    // .send(stream)
+  } catch (err) {
+    reply.code(400).send(err)
+    return
+  }
+}

src/controllers/index.ts

@@ -1,203 +1,2 @@
-import {
+export * from './image'
-  IsDefined,
+export * from './version'
-  IsInt,
-  IsOptional,
-  IsPositive,
-  IsString,
-  IsUrl,
-  IsIn,
-  IsObject,
-  ValidateNested,
-} from 'class-validator'
-import { RouteHandlerMethod } from 'fastify'
-import sharp, { FitEnum, FormatEnum } from 'sharp'
-import { flatten, unflatten } from 'flat'
-import ms from 'ms'
-
-import { storage } from '../storage'
-import { transform } from '../transform'
-import { sha3, sortObjectByKeys, validateSyncOrFail } from '../utils/utils'
-import { Config, URLClean } from '../config'
-import { supportsAvif, supportsWebP } from '../utils/caniuse'
-
-export class ComplexParameter<N = string, T extends object = {}> {
-  @IsString()
-  name: N
-
-  @IsObject()
-  options: T
-
-  constructor(parameter: string) {
-    const [name, optionsRaw] = parameter.split('|')
-    if (!name) throw new Error('Invalid parameter')
-    this.name = name as any
-    this.options = {} as any
-    if (optionsRaw) {
-      for (const option of optionsRaw.split(',')) {
-        const [key, value] = option.split(':')
-        if (!key || !value) continue
-        // @ts-ignore
-        this.options[key] = ComplexParameter.ParseValue(value)
-      }
-    }
-    this.options = unflatten(this.options)
-  }
-
-  static ParseValue(value: string) {
-    if (value === 'true') return true
-    if (value === 'false') return false
-
-    const asNumber = Number(value)
-    if (!isNaN(asNumber)) return asNumber
-
-    return value
-  }
-}
-
-export enum ImageOperations {
-  resize,
-  flip,
-  flop,
-  affine,
-  sharpen,
-  median,
-  blur,
-  flatten,
-  gamma,
-  negate,
-  normalise,
-  normalize,
-  clahe,
-  convolve,
-  threshold,
-  boolean,
-  linear,
-  recomb,
-  modulate,
-}
-
-export enum ImageFormat {
-  jpeg,
-  png,
-  webp,
-  gif,
-  jp2,
-  tiff,
-  avif,
-  heif,
-  raw,
-}
-
-export class TransformQueryBase {
-  @IsString()
-  @IsUrl()
-  @IsDefined()
-  url!: string
-
-  @IsOptional()
-  @ValidateNested()
-  format?: ComplexParameter<keyof FormatEnum>
-
-  @IsOptional()
-  @IsIn(Object.values(sharp.fit))
-  resize?: keyof FitEnum
-
-  @IsOptional()
-  @IsInt()
-  @IsPositive()
-  width?: number
-
-  @IsOptional()
-  @IsInt()
-  @IsPositive()
-  height?: number
-
-  @ValidateNested()
-  op: ComplexParameter[] = []
-
-  hash: string
-
-  constructor(data: any, options: { ua?: string }) {
-    Object.assign(this, data)
-
-    if (this.width) this.width = parseInt(this.width as any)
-    if (this.height) this.height = parseInt(this.height as any)
-
-    this.op = Array.isArray(this.op) ? this.op : [this.op]
-    this.op = this.op.map((op) => new ComplexParameter(op as any))
-    if (this.format) this.format = new ComplexParameter(this.format as any)
-    if ((this.format?.name as string) === 'auto') {
-      if (!options.ua) throw new Error('cannot use auto format without user agent')
-      this.autoFormat(options.ua)
-    }
-
-    switch (Config.cleanUrls) {
-      case URLClean.Query: {
-        this.url = this.url.split('#')[0]!
-        this.url = this.url.split('?')[0]!
-        break
-      }
-      case URLClean.Fragment: {
-        this.url = this.url.split('#')[0]!
-        break
-      }
-    }
-
-    validateSyncOrFail(this)
-    if (this.resize) {
-      if (!this.width && !this.height) {
-        throw new Error('width or height is required when resizing')
-      }
-    }
-
-    this.hash = sha3(this.toString())
-  }
-
-  toString(): string {
-    const data = flatten(this) as Record<string, any>
-    return new URLSearchParams(sortObjectByKeys(data)).toString()
-  }
-
-  isAllowed(prefixes: string[]): boolean {
-    for (const prefix of prefixes) {
-      if (this.url.startsWith(prefix)) return true
-    }
-    return false
-  }
-
-  autoFormat(ua: string) {
-    if (supportsAvif(ua)) this.format!.name = 'avif'
-    else if (supportsWebP(ua)) this.format!.name = 'webp'
-    else this.format!.name = 'jpeg'
-  }
-}
-
-export const handler: RouteHandlerMethod = async (request, reply) => {
-  try {
-    const q = new TransformQueryBase(request.query, { ua: request.headers['user-agent'] })
-
-    if (!q.isAllowed(Config.allowedDomains)) {
-      reply.code(403).send({ error: 'Forbidden' })
-      return
-    }
-
-    // @ts-ignore
-    reply.etag(q.hash)
-    // @ts-ignore
-    reply.expires(new Date(Date.now() + ms(Config.maxAge)))
-
-    let stream: NodeJS.ReadableStream = (await storage.exists(q.hash))
-      ? await storage.readStream(q.hash)
-      : await transform(q)
-
-    reply.code(200).headers({
-      'Content-Type': `image/${q.format?.name}`,
-    })
-
-    return stream
-    // .send(stream)
-  } catch (err) {
-    reply.code(400).send(err)
-    return
-  }
-}

src/controllers/version.ts

@@ -0,0 +1,7 @@
+import { RouteHandlerMethod } from 'fastify'
+
+import { version as v } from '../../package.json'
+
+export const version: RouteHandlerMethod = async (request, reply) => {
+  return { version: v }
+}

src/index.ts

@@ -2,24 +2,28 @@
 import fastify from 'fastify'
 import compress from 'fastify-compress'
 import cors from 'fastify-cors'
-// @ts-ignore
-import cache from 'fastify-caching'
-import ms from 'ms'
 import underPressure from 'under-pressure'
 
-import { Config } from './config'
+import './config'
-import { handler } from './controllers'
+import { version } from './controllers'
+import { image } from './controllers/image'
 import { init } from './storage'
 
 init()
 
 const app = fastify({ logger: true })
 app.register(underPressure)
-app.register(cache, { expiresIn: ms(Config.maxAge) / 1000 })
+app.register(require('fastify-caching'))
 app.register(compress, { global: true })
 app.register(cors, { origin: true })
 
-app.get('/api/image', handler)
+app.addHook('preHandler', (request, reply, done) => {
+  reply.header('Server', 'morphus')
+  done()
+})
+
+app.get('/api/image', image)
+app.get('/version', version)
 
 async function start() {
   try {

src/transform/index.ts

@@ -1,7 +1,7 @@
 import { get } from 'https'
 import sharp from 'sharp'
 import { PassThrough } from 'stream'
-import { ComplexParameter, TransformQueryBase } from '../controllers'
+import { ComplexParameter, TransformQueryBase } from '../controllers/image'
 
 import { storage } from '../storage'
 import { sha3, splitter } from '../utils/utils'

src/utils/errors.ts

@@ -0,0 +1,5 @@
+import { FastifyReply } from 'fastify'
+
+export function ForbiddenError(reply: FastifyReply, message?: string) {
+  reply.code(403).send({ error: 'Forbidden', message })
+}

src/utils/utils.ts

@@ -1,6 +1,7 @@
 import { createHash } from 'crypto'
 import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator'
 import { PassThrough, Readable } from 'stream'
+import { NullableStringOrRegexpArray } from '../config'
 
 export class ValidationError extends Error {
   override message: string
@@ -37,3 +38,12 @@ export function splitter(from: NodeJS.ReadableStream, ...streams: NodeJS.Writabl
   }
   from.pipe(splitter)
 }
+
+export function testForPrefixOrRegexp(str: string, values: (string | RegExp)[]): boolean {
+  for (const value of values) {
+    if (typeof value === 'string') {
+      if (str.startsWith(value)) return true
+    } else if (value.test(str)) return true
+  }
+  return false
+}

tsconfig.json

@@ -25,7 +25,7 @@
 
     /* Modules */
     "module": "commonjs" /* Specify what module code is generated. */,
-    "rootDir": "./src" /* Specify the root folder within your source files. */,
+    // "rootDir": "./src" /* Specify the root folder within your source files. */,
     // "moduleResolution": "node",                       /* Specify how TypeScript looks up a file from a given module specifier. */
     // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
     // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
@@ -96,5 +96,6 @@
     /* Completeness */
     // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
     "skipLibCheck": true /* Skip type checking all .d.ts files. */
-  }
+  },
+  "include": ["./src", "./package.json"]
 }