google cloud storage

src/config.ts

@@ -1,5 +1,5 @@
-import type { FastifyInstance } from 'fastify'
 import convict from 'convict'
+import type { FastifyInstance } from 'fastify'
 import yaml from 'js-yaml'
 
 convict.addFormat(require('convict-format-with-validator').ipaddress)
@@ -8,7 +8,7 @@ export enum StorageType {
   Local = 'local',
   Minio = 'minio',
   S3 = 's3',
-  // GCS = 'gcs',
+  GCS = 'gcs',
   // Azure = 'azure',
   // B2 = 'b2',
 }
@@ -173,9 +173,25 @@ const config = convict({
       sensitive: true,
     },
   },
+
+  // GCS storage
+  gcs: {
+    bucket: {
+      doc: 'The GCS bucket to use',
+      format: String,
+      default: '',
+      env: 'GCS_BUCKET',
+    },
+    keyFilename: {
+      doc: 'The GCS key file to use',
+      format: String,
+      default: '',
+      env: 'GCS_KEY_FILENAME',
+    },
+  },
 })
 
-for (const file of ['morphus.yaml', 'morphus.yaml']) {
+for (const file of ['morphus.yaml', 'morphus.yml']) {
   try {
     config.loadFile(file)
     break

src/controllers/image.ts

@@ -1,25 +1,26 @@
 import {
   IsDefined,
+  IsIn,
   IsInt,
+  IsObject,
   IsOptional,
   IsPositive,
   IsString,
   IsUrl,
-  IsIn,
-  IsObject,
   ValidateNested,
 } from 'class-validator'
 import { RouteHandlerMethod } from 'fastify'
-import sharp, { FitEnum, FormatEnum } from 'sharp'
 import { flatten, unflatten } from 'flat'
 import ms from 'ms'
+import sharp, { FitEnum, FormatEnum } from 'sharp'
 
+import { App } from '..'
+import { Config, URLClean } from '../config'
 import { storage } from '../storage'
 import { transform } from '../transform'
-import { sha3, sortObjectByKeys, testForPrefixOrRegexp, validateSyncOrFail } from '../utils/utils'
-import { Config, URLClean } from '../config'
 import { supportsAvif, supportsWebP } from '../utils/caniuse'
 import { ForbiddenError } from '../utils/errors'
+import { sha3, sortObjectByKeys, testForPrefixOrRegexp, validateSyncOrFail } from '../utils/utils'
 
 export enum ImageOperations {
   resize,
@@ -190,7 +191,8 @@ export const image: RouteHandlerMethod = async (request, reply) => {
     let stream: NodeJS.ReadableStream
     try {
       stream = await storage.readStream(q.hash)
-    } catch (err) {
+    } catch {
+      App.log.debug(`Transforming`)
       stream = await transform(q)
     }
 
@@ -199,7 +201,6 @@ export const image: RouteHandlerMethod = async (request, reply) => {
     })
 
     return stream
-    // .send(stream)
   } catch (err) {
     reply.code(400).send(err)
     return

src/index.ts

@@ -2,9 +2,9 @@ import fastify from 'fastify'
 
 import { Config, init as initConfig } from './config'
 import { init as initRoutes } from './controllers'
-import { init as initStorage } from './storage'
-import { init as initMiddleware } from './fastify/middleware'
 import { init as initHooks } from './fastify/hooks'
+import { init as initMiddleware } from './fastify/middleware'
+import { init as initStorage } from './storage'
 
 export const App = fastify({ logger: { prettyPrint: true, level: Config.logLevel } })
 

src/storage/gcs.ts

@@ -0,0 +1,40 @@
+import { Bucket, Storage as GCStorage } from '@google-cloud/storage'
+
+import { Storage } from '.'
+
+export type GCSConfig = {
+  bucket: string
+  keyFilename: string
+}
+
+export class GCS implements Storage {
+  client: GCStorage
+  bucket: Bucket
+
+  constructor(private options: GCSConfig) {
+    this.client = new GCStorage(options)
+    this.bucket = this.client.bucket(options.bucket)
+  }
+
+  async init(): Promise<void> {
+    await this.client.bucket(this.options.bucket).getFiles({ maxResults: 1 })
+  }
+
+  async readStream(path: string): Promise<NodeJS.ReadableStream> {
+    if (!(await this.exists(path))) throw new Error(`File ${path} does not exist`)
+    return this.bucket.file(path).createReadStream()
+  }
+
+  async writeStream(path: string): Promise<NodeJS.WritableStream> {
+    return this.bucket.file(path).createWriteStream()
+  }
+
+  async exists(path: string): Promise<boolean> {
+    const [exists] = await this.bucket.file(path).exists()
+    return exists
+  }
+
+  async delete(path: string): Promise<void> {
+    await this.bucket.file(path).delete()
+  }
+}

src/storage/index.ts

@@ -1,14 +1,13 @@
 import { FastifyInstance } from 'fastify'
+
 import { Config, StorageType } from '../config'
+import { GCS } from './gcs'
 import { Local } from './local'
 import { Minio } from './minio'
 
 export abstract class Storage {
   abstract init(): Promise<void>
 
-  abstract read(path: string): Promise<Buffer>
-  abstract write(path: string, data: Buffer): Promise<void>
-
   abstract readStream(path: string): Promise<NodeJS.ReadableStream>
   abstract writeStream(path: string): Promise<NodeJS.WritableStream>
 
@@ -26,12 +25,6 @@ export async function init(App: FastifyInstance) {
         storage = new Local(Config.localAssets)
         break
       case StorageType.S3:
-        // storage = new S3({
-        //   accessKeyId: Config.s3.accessKey,
-        //   secretAccessKey: Config.s3.secretKey,
-        //   bucket: Config.s3.bucket,
-        //   region: Config.s3.region,
-        // })
         storage = new Minio({
           accessKey: Config.s3.accessKey,
           secretKey: Config.s3.secretKey,
@@ -49,6 +42,12 @@ export async function init(App: FastifyInstance) {
           bucket: Config.minio.bucket,
         })
         break
+      case StorageType.GCS:
+        storage = new GCS({
+          bucket: Config.gcs.bucket,
+          keyFilename: Config.gcs.keyFilename,
+        })
+        break
       default:
         throw new Error(`Unknown storage type: ${Config.storage}`)
     }
@@ -56,6 +55,7 @@ export async function init(App: FastifyInstance) {
       await storage.init()
       App.log.debug(`Storage initialized: ${Config.storage}`)
     } catch (e) {
+      App.log.error((e as Error).message)
       App.log.error(`Storage initialization failed: ${Config.storage}`)
       process.exit(1)
     }

src/storage/local.ts

@@ -1,5 +1,5 @@
-import { resolve, join } from 'path'
 import fs from 'fs'
+import { join, resolve } from 'path'
 import { promisify } from 'util'
 
 import { Storage } from './'
@@ -13,30 +13,6 @@ export class Local implements Storage {
     await promisify(fs.mkdir)(this.root, { recursive: true })
   }
 
-  read(path: string): Promise<Buffer> {
-    const file = join(this.root, path)
-    return new Promise((resolve, reject) => {
-      fs.readFile(file, (err, data) => {
-        if (err) {
-          return reject(err)
-        }
-        resolve(data)
-      })
-    })
-  }
-
-  write(path: string, data: Buffer): Promise<void> {
-    const file = join(this.root, path)
-    return new Promise((resolve, reject) => {
-      fs.writeFile(file, data, (err) => {
-        if (err) {
-          return reject(err)
-        }
-        resolve()
-      })
-    })
-  }
-
   exists(path: string): Promise<boolean> {
     const file = join(this.root, path)
     return new Promise((resolve, reject) => {

src/storage/minio.ts

@@ -2,7 +2,6 @@ import { Client } from 'minio'
 import { PassThrough } from 'stream'
 
 import { Storage } from '.'
-import { StreamUtils } from '../utils/utils'
 
 export type MinioConfig = {
   accessKey: string
@@ -30,15 +29,6 @@ export class Minio implements Storage {
     await this.client.bucketExists(this.options.bucket)
   }
 
-  async read(path: string): Promise<Buffer> {
-    const stream = await this.client.getObject(this.options.bucket, path)
-    return StreamUtils.toBuffer(stream)
-  }
-  async write(path: string, data: Buffer): Promise<void> {
-    const stream = await StreamUtils.fromBuffer(data)
-    await this.client.putObject(this.options.bucket, path, stream)
-  }
-
   async readStream(path: string): Promise<NodeJS.ReadableStream> {
     const stream = await this.client.getObject(this.options.bucket, path)
     return stream

src/transform/index.ts

@@ -1,8 +1,8 @@
 import { get } from 'https'
 import sharp from 'sharp'
 import { PassThrough } from 'stream'
-import { ComplexParameter, TransformQueryBase } from '../controllers/image'
 
+import { ComplexParameter, TransformQueryBase } from '../controllers/image'
 import { storage } from '../storage'
 import { sha3, splitter } from '../utils/utils'
 

src/utils/caniuse.ts

@@ -1,6 +1,6 @@
-import DeviceDetector from 'device-detector-js'
 import Avif from 'caniuse-db/features-json/avif.json'
 import WebP from 'caniuse-db/features-json/webp.json'
+import DeviceDetector from 'device-detector-js'
 
 const detector = new DeviceDetector()
 

src/utils/utils.ts

@@ -1,5 +1,5 @@
+import { validateSync, ValidationError as VE, ValidatorOptions } from 'class-validator'
 import { createHash } from 'crypto'
-import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator'
 import { PassThrough, Readable } from 'stream'
 
 export class ValidationError extends Error {