initial

src/config.ts

@@ -0,0 +1,66 @@
+import convict from 'convict'
+import yaml from 'yaml'
+
+export enum StorageType {
+  Local = 'local',
+  // S3 = 's3',
+  // GCS = 'gcs',
+  // Azure = 'azure',
+}
+
+export enum URLClean {
+  Off = 'off',
+  Fragment = 'fragment',
+  Query = 'query',
+}
+
+convict.addParser({ extension: ['yml', 'yaml'], parse: yaml.parse })
+const config = convict({
+  // Security
+  allowedDomains: {
+    doc: 'The domains that are allowed to be used as image sources',
+    format: Array,
+    default: [] as string[],
+    env: 'ALLOWED_DOMAINS',
+  },
+  cleanUrls: {
+    doc: 'Whether to clean URLs',
+    format: Object.values(URLClean),
+    default: URLClean.Fragment,
+    env: 'CLEAN_URLS',
+  },
+
+  // Caching
+  maxAge: {
+    doc: 'The maximum age of a cached image',
+    format: String,
+    default: '1d',
+    env: 'MAX_AGE',
+  },
+
+  storage: {
+    doc: 'The storage engine to use',
+    format: Object.values(StorageType),
+    default: StorageType.Local,
+    env: 'STORAGE',
+  },
+
+  // Local storage
+  assets: {
+    doc: 'The path to the assets folder',
+    format: String,
+    default: './assets',
+    env: 'ASSETS',
+  },
+})
+
+for (const file of ['morphus.yaml', 'morphus.yaml', 'morphus.json']) {
+  try {
+    config.loadFile(file)
+    break
+  } catch {}
+}
+
+export const Config = config.get()
+
+console.debug(Config)

src/controllers/index.ts

@@ -0,0 +1,209 @@
+import {
+  IsDefined,
+  IsInt,
+  IsOptional,
+  IsPositive,
+  IsString,
+  IsUrl,
+  IsIn,
+  IsObject,
+  ValidateNested,
+} from 'class-validator'
+import { RouteHandlerMethod } from 'fastify'
+import sharp, { FitEnum, FormatEnum } from 'sharp'
+import { flatten, unflatten } from 'flat'
+import ms from 'ms'
+import DeviceDetector from 'device-detector-js'
+import Avif from 'caniuse-db/features-json/avif.json'
+import WebP from 'caniuse-db/features-json/webp.json'
+
+import { storage } from '../storage'
+import { transform } from '../transform'
+import { sha3, sortObjectByKeys, validateSyncOrFail } from '../utils/utils'
+import { Config, URLClean } from '../config'
+
+const detector = new DeviceDetector()
+
+export class ComplexParameter<N = string, T extends object = {}> {
+  @IsString()
+  name: N
+
+  @IsObject()
+  options: T
+
+  constructor(parameter: string) {
+    const [name, optionsRaw] = parameter.split('|')
+    if (!name) throw new Error('Invalid parameter')
+    this.name = name as any
+    this.options = {} as any
+    if (optionsRaw) {
+      for (const option of optionsRaw.split(',')) {
+        const [key, value] = option.split(':')
+        if (!key || !value) continue
+        // @ts-ignore
+        this.options[key] = ComplexParameter.ParseValue(value)
+      }
+    }
+    this.options = unflatten(this.options)
+  }
+
+  static ParseValue(value: string) {
+    if (value === 'true') return true
+    if (value === 'false') return false
+
+    const asNumber = Number(value)
+    if (!isNaN(asNumber)) return asNumber
+
+    return value
+  }
+}
+
+export enum ImageOperations {
+  resize,
+  flip,
+  flop,
+  affine,
+  sharpen,
+  median,
+  blur,
+  flatten,
+  gamma,
+  negate,
+  normalise,
+  normalize,
+  clahe,
+  convolve,
+  threshold,
+  boolean,
+  linear,
+  recomb,
+  modulate,
+}
+
+export enum ImageFormat {
+  jpeg,
+  png,
+  webp,
+  gif,
+  jp2,
+  tiff,
+  avif,
+  heif,
+  raw,
+}
+
+export class TransformQueryBase {
+  @IsString()
+  @IsUrl()
+  @IsDefined()
+  url!: string
+
+  @IsOptional()
+  @ValidateNested()
+  format?: ComplexParameter<keyof FormatEnum>
+
+  @IsOptional()
+  @IsIn(Object.values(sharp.fit))
+  resize?: keyof FitEnum
+
+  @IsOptional()
+  @IsInt()
+  @IsPositive()
+  width?: number
+
+  @IsOptional()
+  @IsInt()
+  @IsPositive()
+  height?: number
+
+  @ValidateNested()
+  op: ComplexParameter[] = []
+
+  hash: string
+
+  constructor(data: any, options: { ua?: string }) {
+    Object.assign(this, data)
+
+    if (this.width) this.width = parseInt(this.width as any)
+    if (this.height) this.height = parseInt(this.height as any)
+
+    this.op = Array.isArray(this.op) ? this.op : [this.op]
+    this.op = this.op.map((op) => new ComplexParameter(op as any))
+    if (this.format) this.format = new ComplexParameter(this.format as any)
+    if ((this.format?.name as string) === 'auto') {
+      if (!options.ua) throw new Error('cannot use auto format without user agent')
+      this.autoFormat(options.ua)
+    }
+
+    switch (Config.cleanUrls) {
+      case URLClean.Query: {
+        this.url = this.url.split('#')[0]!
+        this.url = this.url.split('?')[0]!
+        break
+      }
+      case URLClean.Fragment: {
+        this.url = this.url.split('#')[0]!
+        break
+      }
+    }
+
+    validateSyncOrFail(this)
+    if (this.resize) {
+      if (!this.width && !this.height) {
+        throw new Error('width or height is required when resizing')
+      }
+    }
+
+    this.hash = sha3(this.toString())
+  }
+
+  toString(): string {
+    const data = flatten(this) as Record<string, any>
+    return new URLSearchParams(sortObjectByKeys(data)).toString()
+  }
+
+  isAllowed(prefixes: string[]): boolean {
+    for (const prefix of prefixes) {
+      if (this.url.startsWith(prefix)) return true
+    }
+    return false
+  }
+
+  autoFormat(ua: string) {
+    const parsed = detector.parse(ua)
+    //https://caniuse.com/avif
+    console.log(parsed)
+    console.log(WebP)
+    // https://caniuse.com/webp
+  }
+}
+
+export const handler: RouteHandlerMethod = async (request, reply) => {
+  try {
+    const q = new TransformQueryBase(request.query, { ua: request.headers['user-agent'] })
+
+    if (!q.isAllowed(Config.allowedDomains)) {
+      reply.code(403).send({ error: 'Forbidden' })
+      return
+    }
+
+    // @ts-ignore
+    reply.etag(q.hash)
+    // @ts-ignore
+    reply.expires(new Date(Date.now() + ms(Config.maxAge)))
+
+    let stream: NodeJS.ReadableStream = (await storage.exists(q.hash))
+      ? await storage.readStream(q.hash)
+      : await transform(q)
+
+    reply.code(200).headers({
+      'Content-Type': `image/${q.format?.name}`,
+    })
+
+    return stream
+    // .send(stream)
+  } catch (err) {
+    reply.code(400).send(err)
+    return
+  }
+}

src/index.ts

@@ -0,0 +1,32 @@
+// Require the framework and instantiate it
+import fastify from 'fastify'
+import compress from 'fastify-compress'
+import cors from 'fastify-cors'
+// @ts-ignore
+import cache from 'fastify-caching'
+import ms from 'ms'
+import underPressure from 'under-pressure'
+
+import { Config } from './config'
+import { handler } from './controllers'
+import { init } from './storage'
+
+init()
+
+const app = fastify({ logger: true })
+app.register(underPressure)
+app.register(cache, { expiresIn: ms(Config.maxAge) / 1000 })
+app.register(compress, { global: true })
+app.register(cors, { origin: true })
+
+app.get('/api/image', handler)
+
+async function start() {
+  try {
+    await app.listen(3000)
+  } catch (err) {
+    app.log.error(err)
+    process.exit(1)
+  }
+}
+start()

src/storage/index.ts

@@ -0,0 +1,27 @@
+import { Config, StorageType } from '../config'
+import { Local } from './local'
+
+export interface Storage {
+  read(path: string): Promise<Buffer>
+  write(path: string, data: Buffer): Promise<void>
+  exists(path: string): Promise<boolean>
+  delete(path: string): Promise<void>
+
+  readStream(path: string): Promise<NodeJS.ReadableStream>
+  writeStream(path: string): Promise<NodeJS.WritableStream>
+  // list(path: string): Promise<string[]>
+}
+
+export let storage: Storage
+
+export function init() {
+  if (!storage) {
+    switch (Config.storage) {
+      case StorageType.Local:
+        storage = new Local(Config.assets)
+        break
+      default:
+        throw new Error(`Unknown storage type: ${Config.storage}`)
+    }
+  }
+}

src/storage/local.ts

@@ -0,0 +1,76 @@
+import { resolve, join } from 'path'
+import fs from 'fs'
+
+import { Storage } from './'
+
+export class Local implements Storage {
+  constructor(private readonly root: string) {
+    this.root = resolve(root)
+  }
+
+  read(path: string): Promise<Buffer> {
+    const file = join(this.root, path)
+    return new Promise((resolve, reject) => {
+      fs.readFile(file, (err, data) => {
+        if (err) {
+          return reject(err)
+        }
+        resolve(data)
+      })
+    })
+  }
+
+  write(path: string, data: Buffer): Promise<void> {
+    const file = join(this.root, path)
+    return new Promise((resolve, reject) => {
+      fs.writeFile(file, data, (err) => {
+        if (err) {
+          return reject(err)
+        }
+        resolve()
+      })
+    })
+  }
+
+  exists(path: string): Promise<boolean> {
+    const file = join(this.root, path)
+    return new Promise((resolve, reject) => {
+      fs.access(file, fs.constants.F_OK, (err) => {
+        if (err) {
+          return resolve(false)
+        }
+        resolve(true)
+      })
+    })
+  }
+
+  delete(path: string): Promise<void> {
+    const file = join(this.root, path)
+    return new Promise((resolve, reject) => {
+      fs.unlink(file, (err) => {
+        if (err) {
+          return reject(err)
+        }
+        resolve()
+      })
+    })
+  }
+
+  readStream(path: string): Promise<NodeJS.ReadableStream> {
+    const file = join(this.root, path)
+    return new Promise((resolve, reject) => {
+      const stream = fs.createReadStream(file)
+      stream.on('error', reject)
+      resolve(stream)
+    })
+  }
+
+  writeStream(path: string): Promise<NodeJS.WritableStream> {
+    const file = join(this.root, path)
+    return new Promise((resolve, reject) => {
+      const stream = fs.createWriteStream(file)
+      stream.on('error', reject)
+      resolve(stream)
+    })
+  }
+}

src/transform/index.ts

@@ -0,0 +1,114 @@
+import { get } from 'https'
+import sharp from 'sharp'
+import { PassThrough } from 'stream'
+import { ComplexParameter, TransformQueryBase } from '../controllers'
+
+import { storage } from '../storage'
+import { sha3, splitter } from '../utils/utils'
+
+async function downloadImage(url: string): Promise<NodeJS.ReadableStream> {
+  const disk = await storage.writeStream(sha3(url))
+  return new Promise((resolve) => {
+    get(url, (res) => {
+      const out = new PassThrough()
+      splitter(res, out, disk)
+      resolve(out)
+    })
+  })
+}
+
+export async function getImage(url: string): Promise<NodeJS.ReadableStream> {
+  const id = sha3(url)
+  if (!(await storage.exists(id))) {
+    return await downloadImage(url)
+  }
+  return await storage.readStream(id)
+}
+
+function applyOperation(pipeline: sharp.Sharp, { name, options }: ComplexParameter<string, any>): sharp.Sharp {
+  switch (name) {
+    case 'negate':
+    case 'clahe':
+    case 'convolve':
+    case 'modulate':
+      return pipeline[name](options)
+    case 'flip':
+    case 'flop':
+    case 'normalise':
+    case 'normalize':
+    case 'greyscale':
+    case 'grayscale':
+    case 'removeAlpha':
+      return pipeline[name]()
+    case 'rotate': {
+      const { angle, ...rest } = options
+      return pipeline.rotate(angle, rest)
+    }
+    case 'threshold': {
+      const { threshold, ...rest } = options
+      return pipeline.threshold(threshold, rest)
+    }
+    case 'boolean': {
+      const { operator, operand, ...rest } = options
+      return pipeline.boolean(operand, operator, rest)
+    }
+    case 'linear':
+      return pipeline.linear(options.a, options.b)
+    case 'sharpen':
+      return pipeline.sharpen(options.sigma, options.flat, options.jagged)
+    case 'media':
+      return pipeline.median(options.size)
+    case 'blur':
+      return pipeline.blur(options.sigma)
+    case 'flatten':
+      return pipeline.flatten(options.background)
+    case 'gamma':
+      return pipeline.gamma(options.gamma)
+    case 'tint':
+      return pipeline.tint(options.rgb)
+    case 'pipelineColorspace':
+    case 'pipelineColourspace':
+      return pipeline.pipelineColorspace(options.colorspace || options.colourspace)
+    case 'toColorspace':
+    case 'toColourspace':
+      return pipeline.toColorspace(options.colorspace || options.colourspace)
+    case 'ensureAlpha':
+      return pipeline.ensureAlpha(options.alpha)
+    case 'extractChannel':
+      return pipeline.extractChannel(options.channel)
+    default:
+      throw new Error(`Unsupported operation ${name}`)
+  }
+}
+
+export function buildPipeline(options: TransformQueryBase) {
+  let pipeline = sharp()
+  if (options.resize) {
+    pipeline = pipeline.resize({
+      fit: options.resize,
+      width: options.width,
+      height: options.height,
+    })
+  }
+  if (options.format) {
+    pipeline = pipeline.toFormat(options.format.name, options.format.options)
+  }
+
+  for (const op of options.op) {
+    try {
+      pipeline = applyOperation(pipeline, op)
+    } catch (e) {
+      throw new Error(`${op.name} is not a valid operation: ${e}`)
+    }
+  }
+  return pipeline
+}
+
+export async function transform(options: TransformQueryBase): Promise<NodeJS.ReadableStream> {
+  const source = await getImage(options.url)
+  const pipeline = buildPipeline(options)
+  const writer = await storage.writeStream(options.hash)
+  const out = new PassThrough()
+  splitter(source.pipe(pipeline), writer, out)
+  return out
+}

src/utils/caniuse.ts

@@ -0,0 +1,50 @@
+import DeviceDetector from 'device-detector-js'
+import Avif from 'caniuse-db/features-json/avif.json'
+import WebP from 'caniuse-db/features-json/webp.json'
+
+const detector = new DeviceDetector()
+
+function findLowestCompatibleVersion(stat: Record<string, string>): string {
+  const entries = Object.entries(stat).sort((a, b) => parseInt(a[0]) - parseInt(b[0]))
+  for (const [version, support] of entries) {
+    if (support.startsWith('y') || support.startsWith('a')) {
+      return version
+    }
+  }
+}
+
+const mapping = {
+  'Internet Explorer': 'ie',
+  'Microsoft Edge': 'edge',
+  Firefox: 'firefox',
+  Chrome: 'chrome',
+  Safari: 'safari',
+  Opera: 'opera',
+  'Mobile Safari': 'ios_saf',
+  'Opera Mini': 'op_mini',
+  'Android Browser': 'android',
+  'Chrome Mobile': 'and_chr',
+  'Firefox Mobile': 'and_ff',
+  'UC Browser': 'and_uc',
+  'Samsung Browser': 'samsung',
+  'QQ Browser': 'and_qq',
+}
+
+function matchBrowserToStat(browser: DeviceDetector.DeviceDetectorResult): string {
+  if (!browser.os || !browser.client) throw new Error('Invalid browser')
+  if (browser.os.name === 'iOS') {
+    return 'ios_saf'
+  }
+  if (browser.os.name in mapping) {
+    return mapping[browser.os.name as keyof typeof mapping]
+  }
+  throw new Error('Could not determine mapping for browser')
+}
+
+function match(feature: typeof Avif | typeof WebP, ua: string): boolean {
+  const browser = detector.parse(ua)
+  const stats = feature.stats[matchBrowserToStat(browser) as keyof typeof feature.stats]
+  console.debug(stats)
+  console.debug(findLowestCompatibleVersion(stats))
+  return false
+}

src/utils/utils.ts

@@ -0,0 +1,39 @@
+import { createHash } from 'crypto'
+import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator'
+import { PassThrough, Readable } from 'stream'
+
+export class ValidationError extends Error {
+  override message: string
+
+  constructor(errors: VE[]) {
+    super()
+    this.message = errors
+      .map((e) => Object.values(e.constraints!))
+      .flat()
+      .join(', ')
+  }
+}
+
+export function validateSyncOrFail(data: object, options: ValidatorOptions = {}) {
+  options = Object.assign({ whitelist: true, forbidUnknownValues: true, skipMissingProperties: false }, options)
+  const errors = validateSync(data, options)
+  if (errors.length > 0) {
+    throw new ValidationError(errors)
+  }
+}
+
+export function sha3(url: string) {
+  return createHash('sha3-256').update(url).digest('hex')
+}
+
+export function sortObjectByKeys<T extends object>(obj: T): T {
+  return Object.fromEntries(Object.entries(obj).sort((a, b) => a[0].localeCompare(b[0]))) as T
+}
+
+export function splitter(from: NodeJS.ReadableStream, ...streams: NodeJS.WritableStream[]) {
+  const splitter = new PassThrough()
+  for (const stream of streams) {
+    splitter.pipe(stream)
+  }
+  from.pipe(splitter)
+}