support for s3 and minio

README.md

@@ -8,7 +8,7 @@ The heavy lifting is done by [`libvips`](https://github.com/libvips/libvips) and
 - Config driven
 - Domain protection
 - Host verification
-- Multiple storage adapters (Local, S3, GCS)
+- Multiple storage adapters (Local, Minio, S3)
 - Caniuse based automatic formatting
 - ETag caching
 

package.json

@@ -13,6 +13,7 @@
     "@types/convict": "^6.1.1",
     "@types/flat": "^5.0.2",
     "@types/js-yaml": "^4.0.4",
+    "@types/minio": "^7.0.11",
     "@types/ms": "^0.7.31",
     "@types/node": "^16.11.7",
     "@types/sharp": "^0.29.3",
@@ -31,6 +32,7 @@
     "fastify-cors": "^6.0.2",
     "flat": "^5.0.2",
     "js-yaml": "^4.1.0",
+    "minio": "^7.0.19",
     "ms": "^2.1.3",
     "pino-pretty": "^7.2.0",
     "sharp": "^0.29.3",

pnpm-lock.yaml

@@ -4,6 +4,7 @@ specifiers:
   '@types/convict': ^6.1.1
   '@types/flat': ^5.0.2
   '@types/js-yaml': ^4.0.4
+  '@types/minio': ^7.0.11
   '@types/ms': ^0.7.31
   '@types/node': ^16.11.7
   '@types/sharp': ^0.29.3
@@ -18,6 +19,7 @@ specifiers:
   fastify-cors: ^6.0.2
   flat: ^5.0.2
   js-yaml: ^4.1.0
+  minio: ^7.0.19
   ms: ^2.1.3
   pino-pretty: ^7.2.0
   sharp: ^0.29.3
@@ -37,6 +39,7 @@ dependencies:
   fastify-cors: 6.0.2
   flat: 5.0.2
   js-yaml: 4.1.0
+  minio: 7.0.19
   ms: 2.1.3
   pino-pretty: 7.2.0
   sharp: 0.29.3
@@ -46,6 +49,7 @@ devDependencies:
   '@types/convict': 6.1.1
   '@types/flat': 5.0.2
   '@types/js-yaml': 4.0.4
+  '@types/minio': 7.0.11
   '@types/ms': 0.7.31
   '@types/node': 16.11.7
   '@types/sharp': 0.29.3
@@ -74,6 +78,12 @@ packages:
     resolution: {integrity: sha512-AuHubXUmg0AzkXH0Mx6sIxeY/1C110mm/EkE/gB1sTRz3h2dao2W/63q42SlVST+lICxz5Oki2hzYA6+KnnieQ==}
     dev: true
 
+  /@types/minio/7.0.11:
+    resolution: {integrity: sha512-ltn30nGhtxytil4jFU1Tt6lvD+JnUyCYHfNBKsRjZ76ueSkrQdIByghcnhFvr15at3cnvj+tVn+euCqTX/5ejQ==}
+    dependencies:
+      '@types/node': 16.11.7
+    dev: true
+
   /@types/ms/0.7.31:
     resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
     dev: true
@@ -100,6 +110,12 @@ packages:
     resolution: {integrity: sha512-+qogUELb4gMhrMjSh/seKmGVvN+uQLfyqJAqYRWqVHsvBsUO2xDBCL8CJ/ZSukbd8vXaoYbpIssAmfLEzzBHEw==}
     dev: false
 
+  /@zxing/text-encoding/0.9.0:
+    resolution: {integrity: sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==}
+    requiresBuild: true
+    dev: false
+    optional: true
+
   /abstract-cache/1.0.1:
     resolution: {integrity: sha512-EfUeMhRUbG5bVVbrSY/ogLlFXoyfMAPxMlSP7wrEqH53d+59r2foVy9a5KjmprLKFLOfPQCNKEfpBN/nQ76chw==}
     dependencies:
@@ -183,11 +199,20 @@ packages:
       mri: 1.1.4
     dev: false
 
+  /async/3.2.2:
+    resolution: {integrity: sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g==}
+    dev: false
+
   /atomic-sleep/1.0.0:
     resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
     engines: {node: '>=8.0.0'}
     dev: false
 
+  /available-typed-arrays/1.0.5:
+    resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==}
+    engines: {node: '>= 0.4'}
+    dev: false
+
   /avvio/7.2.2:
     resolution: {integrity: sha512-XW2CMCmZaCmCCsIaJaLKxAzPwF37fXi1KGxNOvedOpeisLdmxZnblGc3hpHWYnlP+KOUxZsazh43WXNHgXpbqw==}
     dependencies:
@@ -231,6 +256,12 @@ packages:
       readable-stream: 3.6.0
     dev: false
 
+  /block-stream2/2.1.0:
+    resolution: {integrity: sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==}
+    dependencies:
+      readable-stream: 3.6.0
+    dev: false
+
   /bluebird/3.4.7:
     resolution: {integrity: sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=}
     dev: false
@@ -268,6 +299,13 @@ packages:
     engines: {node: '>=0.2.0'}
     dev: false
 
+  /call-bind/1.0.2:
+    resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==}
+    dependencies:
+      function-bind: 1.1.1
+      get-intrinsic: 1.1.1
+    dev: false
+
   /camelcase/5.0.0:
     resolution: {integrity: sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==}
     engines: {node: '>=6'}
@@ -436,6 +474,13 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /define-properties/1.1.3:
+    resolution: {integrity: sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      object-keys: 1.1.1
+    dev: false
+
   /delegates/1.0.0:
     resolution: {integrity: sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=}
     dev: false
@@ -497,6 +542,45 @@ packages:
       once: 1.4.0
     dev: false
 
+  /es-abstract/1.19.1:
+    resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      es-to-primitive: 1.2.1
+      function-bind: 1.1.1
+      get-intrinsic: 1.1.1
+      get-symbol-description: 1.0.0
+      has: 1.0.3
+      has-symbols: 1.0.2
+      internal-slot: 1.0.3
+      is-callable: 1.2.4
+      is-negative-zero: 2.0.1
+      is-regex: 1.1.4
+      is-shared-array-buffer: 1.0.1
+      is-string: 1.0.7
+      is-weakref: 1.0.1
+      object-inspect: 1.11.0
+      object-keys: 1.1.1
+      object.assign: 4.1.2
+      string.prototype.trimend: 1.0.4
+      string.prototype.trimstart: 1.0.4
+      unbox-primitive: 1.0.1
+    dev: false
+
+  /es-to-primitive/1.2.1:
+    resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      is-callable: 1.2.4
+      is-date-object: 1.0.5
+      is-symbol: 1.0.4
+    dev: false
+
+  /es6-error/4.1.1:
+    resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==}
+    dev: false
+
   /escape-string-regexp/1.0.5:
     resolution: {integrity: sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=}
     engines: {node: '>=0.8.0'}
@@ -538,6 +622,11 @@ packages:
     resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}
     dev: false
 
+  /fast-xml-parser/3.19.0:
+    resolution: {integrity: sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg==}
+    hasBin: true
+    dev: false
+
   /fastify-caching/6.1.0:
     resolution: {integrity: sha512-xhzjpI21qpHqoOKlUXlqpPSWm7UBTTgWrXxjfFpmxCbNBtN+JiQtzuXuSF8dUj5iig2ztRaD9+sD9uyPmNkHOw==}
     dependencies:
@@ -639,6 +728,10 @@ packages:
     resolution: {integrity: sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==}
     dev: false
 
+  /foreach/2.0.5:
+    resolution: {integrity: sha1-C+4AUBiusmDQo6865ljdATbsG5k=}
+    dev: false
+
   /forwarded/0.2.0:
     resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
     engines: {node: '>= 0.6'}
@@ -678,7 +771,6 @@ packages:
 
   /function-bind/1.1.1:
     resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==}
-    dev: true
 
   /gauge/2.7.4:
     resolution: {integrity: sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=}
@@ -693,6 +785,22 @@ packages:
       wide-align: 1.1.5
     dev: false
 
+  /get-intrinsic/1.1.1:
+    resolution: {integrity: sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==}
+    dependencies:
+      function-bind: 1.1.1
+      has: 1.0.3
+      has-symbols: 1.0.2
+    dev: false
+
+  /get-symbol-description/1.0.0:
+    resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      get-intrinsic: 1.1.1
+    dev: false
+
   /github-from-package/0.0.0:
     resolution: {integrity: sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=}
     dev: false
@@ -718,11 +826,27 @@ packages:
     resolution: {integrity: sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==}
     dev: false
 
+  /has-bigints/1.0.1:
+    resolution: {integrity: sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==}
+    dev: false
+
   /has-flag/3.0.0:
     resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
     engines: {node: '>=4'}
     dev: false
 
+  /has-symbols/1.0.2:
+    resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==}
+    engines: {node: '>= 0.4'}
+    dev: false
+
+  /has-tostringtag/1.0.0:
+    resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      has-symbols: 1.0.2
+    dev: false
+
   /has-unicode/2.0.1:
     resolution: {integrity: sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=}
     dev: false
@@ -732,7 +856,6 @@ packages:
     engines: {node: '>= 0.4.0'}
     dependencies:
       function-bind: 1.1.1
-    dev: true
 
   /ieee754/1.2.1:
     resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
@@ -751,6 +874,15 @@ packages:
     resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
     dev: false
 
+  /internal-slot/1.0.3:
+    resolution: {integrity: sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      get-intrinsic: 1.1.1
+      has: 1.0.3
+      side-channel: 1.0.4
+    dev: false
+
   /into-stream/6.0.0:
     resolution: {integrity: sha512-XHbaOAvP+uFKUFsOgoNPRjLkwB+I22JFPFe5OjTkQ0nwgj6+pSjb4NmB6VMxaPshLiOf+zcpOCBQuLwC1KHhZA==}
     engines: {node: '>=10'}
@@ -764,10 +896,24 @@ packages:
     engines: {node: '>= 0.10'}
     dev: false
 
+  /is-arguments/1.1.1:
+    resolution: {integrity: sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      has-tostringtag: 1.0.0
+    dev: false
+
   /is-arrayish/0.3.2:
     resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
     dev: false
 
+  /is-bigint/1.0.4:
+    resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==}
+    dependencies:
+      has-bigints: 1.0.1
+    dev: false
+
   /is-binary-path/2.1.0:
     resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
     engines: {node: '>=8'}
@@ -775,12 +921,32 @@ packages:
       binary-extensions: 2.2.0
     dev: true
 
+  /is-boolean-object/1.1.2:
+    resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      has-tostringtag: 1.0.0
+    dev: false
+
+  /is-callable/1.2.4:
+    resolution: {integrity: sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==}
+    engines: {node: '>= 0.4'}
+    dev: false
+
   /is-core-module/2.8.0:
     resolution: {integrity: sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==}
     dependencies:
       has: 1.0.3
     dev: true
 
+  /is-date-object/1.0.5:
+    resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      has-tostringtag: 1.0.0
+    dev: false
+
   /is-deflate/1.0.0:
     resolution: {integrity: sha1-yGKQHDwWH7CdrHzcfnhPgOmPLxQ=}
     dev: false
@@ -797,6 +963,13 @@ packages:
       number-is-nan: 1.0.1
     dev: false
 
+  /is-generator-function/1.0.10:
+    resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      has-tostringtag: 1.0.0
+    dev: false
+
   /is-glob/4.0.3:
     resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
     engines: {node: '>=0.10.0'}
@@ -809,6 +982,18 @@ packages:
     engines: {node: '>=4'}
     dev: false
 
+  /is-negative-zero/2.0.1:
+    resolution: {integrity: sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==}
+    engines: {node: '>= 0.4'}
+    dev: false
+
+  /is-number-object/1.0.6:
+    resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      has-tostringtag: 1.0.0
+    dev: false
+
   /is-number/7.0.0:
     resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
     engines: {node: '>=0.12.0'}
@@ -819,11 +1004,54 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /is-regex/1.1.4:
+    resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      has-tostringtag: 1.0.0
+    dev: false
+
+  /is-shared-array-buffer/1.0.1:
+    resolution: {integrity: sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==}
+    dev: false
+
   /is-stream/2.0.1:
     resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
     engines: {node: '>=8'}
     dev: false
 
+  /is-string/1.0.7:
+    resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      has-tostringtag: 1.0.0
+    dev: false
+
+  /is-symbol/1.0.4:
+    resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      has-symbols: 1.0.2
+    dev: false
+
+  /is-typed-array/1.1.8:
+    resolution: {integrity: sha512-HqH41TNZq2fgtGT8WHVFVJhBVGuY3AnP3Q36K8JKXUxSxRgk/d+7NjmwG2vo2mYmXK8UYZKu0qH8bVP5gEisjA==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      available-typed-arrays: 1.0.5
+      call-bind: 1.0.2
+      es-abstract: 1.19.1
+      foreach: 2.0.5
+      has-tostringtag: 1.0.0
+    dev: false
+
+  /is-weakref/1.0.1:
+    resolution: {integrity: sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==}
+    dependencies:
+      call-bind: 1.0.2
+    dev: false
+
   /is-zip/1.0.0:
     resolution: {integrity: sha1-R7Co/004p2QxzP2ZqOFaTIa6IyU=}
     engines: {node: '>=0.10.0'}
@@ -853,6 +1081,10 @@ packages:
     resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
     dev: false
 
+  /json-stream/1.0.0:
+    resolution: {integrity: sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg=}
+    dev: false
+
   /leven/2.1.0:
     resolution: {integrity: sha1-wuep93IJTe6dNCAq6KzORoeHVYA=}
     engines: {node: '>=0.10.0'}
@@ -879,6 +1111,10 @@ packages:
     resolution: {integrity: sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=}
     dev: false
 
+  /lodash/4.17.21:
+    resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
+    dev: false
+
   /lru-cache/6.0.0:
     resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
     engines: {node: '>=10'}
@@ -906,6 +1142,13 @@ packages:
     engines: {node: '>= 0.6'}
     dev: false
 
+  /mime-types/2.1.34:
+    resolution: {integrity: sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==}
+    engines: {node: '>= 0.6'}
+    dependencies:
+      mime-db: 1.51.0
+    dev: false
+
   /mimic-response/3.1.0:
     resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
     engines: {node: '>=10'}
@@ -919,6 +1162,25 @@ packages:
   /minimist/1.2.5:
     resolution: {integrity: sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==}
 
+  /minio/7.0.19:
+    resolution: {integrity: sha512-DOGKauWLdmj0/y2QKXdnrhqyzRFEnUteHi6q382uujg9TjSDrA84BiQVppS2Ew6V8Rcg+2IaRkF4GR34zw9sIA==}
+    engines: {node: '>= 4'}
+    dependencies:
+      async: 3.2.2
+      block-stream2: 2.1.0
+      es6-error: 4.1.1
+      fast-xml-parser: 3.19.0
+      json-stream: 1.0.0
+      lodash: 4.17.21
+      mime-types: 2.1.34
+      mkdirp: 0.5.5
+      querystring: 0.2.0
+      through2: 3.0.2
+      web-encoding: 1.1.5
+      xml: 1.0.1
+      xml2js: 0.4.23
+    dev: false
+
   /minipass/3.1.5:
     resolution: {integrity: sha512-+8NzxD82XQoNKNrl1d/FSi+X8wAEWR+sbYAfIvub4Nz0d22plFG72CEVVaufV8PNf4qSslFTD8VMOxNVhHCjTw==}
     engines: {node: '>=8'}
@@ -995,6 +1257,25 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /object-inspect/1.11.0:
+    resolution: {integrity: sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==}
+    dev: false
+
+  /object-keys/1.1.1:
+    resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==}
+    engines: {node: '>= 0.4'}
+    dev: false
+
+  /object.assign/4.1.2:
+    resolution: {integrity: sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+      has-symbols: 1.0.2
+      object-keys: 1.1.1
+    dev: false
+
   /once/1.4.0:
     resolution: {integrity: sha1-WDsap3WWHUsROsF9nFC6753Xa9E=}
     dependencies:
@@ -1120,6 +1401,12 @@ packages:
     engines: {node: '>=6'}
     dev: false
 
+  /querystring/0.2.0:
+    resolution: {integrity: sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=}
+    engines: {node: '>=0.4.x'}
+    deprecated: The querystring API is considered Legacy. new code should use the URLSearchParams API instead.
+    dev: false
+
   /queue-microtask/1.2.3:
     resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
     dev: false
@@ -1217,6 +1504,10 @@ packages:
       ret: 0.2.2
     dev: false
 
+  /sax/1.2.4:
+    resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==}
+    dev: false
+
   /secure-json-parse/2.4.0:
     resolution: {integrity: sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==}
     dev: false
@@ -1260,6 +1551,14 @@ packages:
       tunnel-agent: 0.6.0
     dev: false
 
+  /side-channel/1.0.4:
+    resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==}
+    dependencies:
+      call-bind: 1.0.2
+      get-intrinsic: 1.1.1
+      object-inspect: 1.11.0
+    dev: false
+
   /signal-exit/3.0.5:
     resolution: {integrity: sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ==}
     dev: false
@@ -1335,6 +1634,20 @@ packages:
       strip-ansi: 3.0.1
     dev: false
 
+  /string.prototype.trimend/1.0.4:
+    resolution: {integrity: sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+    dev: false
+
+  /string.prototype.trimstart/1.0.4:
+    resolution: {integrity: sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+    dev: false
+
   /string_decoder/1.1.1:
     resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==}
     dependencies:
@@ -1402,6 +1715,13 @@ packages:
       xtend: 4.0.2
     dev: false
 
+  /through2/3.0.2:
+    resolution: {integrity: sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==}
+    dependencies:
+      inherits: 2.0.4
+      readable-stream: 3.6.0
+    dev: false
+
   /tiny-lru/7.0.6:
     resolution: {integrity: sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow==}
     engines: {node: '>=6'}
@@ -1491,6 +1811,15 @@ packages:
       random-bytes: 1.0.0
     dev: false
 
+  /unbox-primitive/1.0.1:
+    resolution: {integrity: sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==}
+    dependencies:
+      function-bind: 1.1.1
+      has-bigints: 1.0.1
+      has-symbols: 1.0.2
+      which-boxed-primitive: 1.0.2
+    dev: false
+
   /under-pressure/5.8.0:
     resolution: {integrity: sha512-8ADLZkFEGDAsKof1FEICH/OLyGWjDZy6KR/Exq3MTv7W81zC2W23VekY05Fo350lip1ywYNH9wP7itiVnk4wHg==}
     engines: {node: '>=10'}
@@ -1524,6 +1853,17 @@ packages:
     resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=}
     dev: false
 
+  /util/0.12.4:
+    resolution: {integrity: sha512-bxZ9qtSlGUWSOy9Qa9Xgk11kSslpuZwaxCg4sNIDj6FLucDab2JxnHwyNTCpHMtK1MjoQiWQ6DiUMZYbSrO+Sw==}
+    dependencies:
+      inherits: 2.0.4
+      is-arguments: 1.1.1
+      is-generator-function: 1.0.10
+      is-typed-array: 1.1.8
+      safe-buffer: 5.2.1
+      which-typed-array: 1.1.7
+    dev: false
+
   /validator/13.7.0:
     resolution: {integrity: sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==}
     engines: {node: '>= 0.10'}
@@ -1534,6 +1874,36 @@ packages:
     engines: {node: '>= 0.8'}
     dev: false
 
+  /web-encoding/1.1.5:
+    resolution: {integrity: sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==}
+    dependencies:
+      util: 0.12.4
+    optionalDependencies:
+      '@zxing/text-encoding': 0.9.0
+    dev: false
+
+  /which-boxed-primitive/1.0.2:
+    resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==}
+    dependencies:
+      is-bigint: 1.0.4
+      is-boolean-object: 1.1.2
+      is-number-object: 1.0.6
+      is-string: 1.0.7
+      is-symbol: 1.0.4
+    dev: false
+
+  /which-typed-array/1.1.7:
+    resolution: {integrity: sha512-vjxaB4nfDqwKI0ws7wZpxIlde1XrLX5uB0ZjpfshgmapJMD7jJWhZI+yToJTqaFByF0eNBcYxbjmCzoRP7CfEw==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      available-typed-arrays: 1.0.5
+      call-bind: 1.0.2
+      es-abstract: 1.19.1
+      foreach: 2.0.5
+      has-tostringtag: 1.0.0
+      is-typed-array: 1.1.8
+    dev: false
+
   /wide-align/1.1.5:
     resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==}
     dependencies:
@@ -1543,6 +1913,23 @@ packages:
   /wrappy/1.0.2:
     resolution: {integrity: sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=}
 
+  /xml/1.0.1:
+    resolution: {integrity: sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=}
+    dev: false
+
+  /xml2js/0.4.23:
+    resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==}
+    engines: {node: '>=4.0.0'}
+    dependencies:
+      sax: 1.2.4
+      xmlbuilder: 11.0.1
+    dev: false
+
+  /xmlbuilder/11.0.1:
+    resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==}
+    engines: {node: '>=4.0'}
+    dev: false
+
   /xtend/4.0.2:
     resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
     engines: {node: '>=0.4'}

src/config.ts

@@ -6,9 +6,11 @@ convict.addFormat(require('convict-format-with-validator').ipaddress)
 
 export enum StorageType {
   Local = 'local',
-  // S3 = 's3',
+  Minio = 'minio',
+  S3 = 's3',
   // GCS = 'gcs',
   // Azure = 'azure',
+  // B2 = 'b2',
 }
 
 export enum URLClean {
@@ -53,6 +55,14 @@ const config = convict({
     env: 'ADDRESS',
   },
 
+  // Logging
+  logLevel: {
+    doc: 'The level of logging to use.',
+    format: ['trace', 'debug', 'info', 'warn', 'error', 'fatal'],
+    default: 'info',
+    env: 'LOG_LEVEL',
+  },
+
   // Security
   allowedDomains: {
     doc: 'The domains that are allowed to be used as image sources',
@@ -97,6 +107,72 @@ const config = convict({
     default: './assets',
     env: 'LOCAL_ASSETS',
   },
+
+  // Minio storage
+  minio: {
+    accessKey: {
+      doc: 'The access key for Minio',
+      format: String,
+      default: '',
+      env: 'MINIO_ACCESS_KEY',
+      sensitive: true,
+    },
+    secretKey: {
+      doc: 'The secret key for Minio',
+      format: String,
+      default: '',
+      env: 'MINIO_SECRET_KEY',
+      sensitive: true,
+    },
+    endpoint: {
+      doc: 'The endpoint for Minio',
+      format: String,
+      default: '',
+      env: 'MINIO_ENDPOINT',
+    },
+    bucket: {
+      doc: 'The bucket to use for Minio',
+      format: String,
+      default: '',
+      env: 'MINIO_BUCKET',
+    },
+    region: {
+      doc: 'The region for Minio',
+      format: String,
+      default: '',
+      env: 'MINIO_REGION',
+    },
+  },
+
+  // S3 storage
+  s3: {
+    bucket: {
+      doc: 'The S3 bucket to use',
+      format: String,
+      default: '',
+      env: 'S3_BUCKET',
+    },
+    region: {
+      doc: 'The S3 region to use',
+      format: String,
+      default: '',
+      env: 'S3_REGION',
+    },
+    accessKey: {
+      doc: 'The S3 access key id to use',
+      format: String,
+      default: '',
+      env: 'S3_ACCESS_KEY_ID',
+      sensitive: true,
+    },
+    secretKey: {
+      doc: 'The S3 secret access key to use',
+      format: String,
+      default: '',
+      env: 'S3_SECRET_ACCESS_KEY',
+      sensitive: true,
+    },
+  },
 })
 
 for (const file of ['morphus.yaml', 'morphus.yaml']) {

src/controllers/image.ts

@@ -187,9 +187,12 @@ export const image: RouteHandlerMethod = async (request, reply) => {
     // @ts-ignore
     reply.expires(new Date(Date.now() + ms(Config.maxAge)))
 
-    let stream: NodeJS.ReadableStream = (await storage.exists(q.hash))
+    let stream: NodeJS.ReadableStream
-      ? await storage.readStream(q.hash)
+    try {
-      : await transform(q)
+      stream = await storage.readStream(q.hash)
+    } catch (err) {
+      stream = await transform(q)
+    }
 
     reply.code(200).headers({
       'Content-Type': `image/${q.format?.name}`,

src/index.ts

@@ -6,7 +6,7 @@ import { init as initStorage } from './storage'
 import { init as initMiddleware } from './fastify/middleware'
 import { init as initHooks } from './fastify/hooks'
 
-export const App = fastify({ logger: { prettyPrint: true } })
+export const App = fastify({ logger: { prettyPrint: true, level: Config.logLevel } })
 
 process.on('SIGINT', async function () {
   App.log.info('Stopping server')
@@ -19,7 +19,7 @@ async function main() {
   try {
     // Internal
     initConfig(App)
-    initStorage()
+    await initStorage(App)
 
     // Fastify
     initMiddleware(App)

src/storage/index.ts

@@ -1,30 +1,63 @@
+import { FastifyInstance } from 'fastify'
 import { Config, StorageType } from '../config'
 import { Local } from './local'
+import { Minio } from './minio'
 
 export abstract class Storage {
+  abstract init(): Promise<void>
+
   abstract read(path: string): Promise<Buffer>
   abstract write(path: string, data: Buffer): Promise<void>
-  abstract exists(path: string): Promise<boolean>
-  abstract delete(path: string): Promise<void>
 
   abstract readStream(path: string): Promise<NodeJS.ReadableStream>
   abstract writeStream(path: string): Promise<NodeJS.WritableStream>
-  // list(path: string): Promise<string[]>
 
-  abstract init(): Promise<void>
+  // list(path: string): Promise<string[]>
+  abstract exists(path: string): Promise<boolean>
+  abstract delete(path: string): Promise<void>
 }
 
 export let storage: Storage
 
-export async function init() {
+export async function init(App: FastifyInstance) {
   if (!storage) {
     switch (Config.storage) {
       case StorageType.Local:
         storage = new Local(Config.localAssets)
         break
+      case StorageType.S3:
+        // storage = new S3({
+        //   accessKeyId: Config.s3.accessKey,
+        //   secretAccessKey: Config.s3.secretKey,
+        //   bucket: Config.s3.bucket,
+        //   region: Config.s3.region,
+        // })
+        storage = new Minio({
+          accessKey: Config.s3.accessKey,
+          secretKey: Config.s3.secretKey,
+          bucket: Config.s3.bucket,
+          region: Config.s3.region,
+          endpoint: 'https://s3.amazonaws.com',
+        })
+        break
+      case StorageType.Minio:
+        storage = new Minio({
+          accessKey: Config.minio.accessKey,
+          secretKey: Config.minio.secretKey,
+          endpoint: Config.minio.endpoint,
+          region: Config.minio.region,
+          bucket: Config.minio.bucket,
+        })
+        break
       default:
         throw new Error(`Unknown storage type: ${Config.storage}`)
     }
+    try {
       await storage.init()
+      App.log.debug(`Storage initialized: ${Config.storage}`)
+    } catch (e) {
+      App.log.error(`Storage initialization failed: ${Config.storage}`)
+      process.exit(1)
+    }
   }
 }

src/storage/minio.ts

@@ -0,0 +1,64 @@
+import { Client } from 'minio'
+import { PassThrough } from 'stream'
+
+import { Storage } from '.'
+import { StreamUtils } from '../utils/utils'
+
+export type MinioConfig = {
+  accessKey: string
+  secretKey: string
+  endpoint: string
+  region?: string
+  bucket: string
+}
+
+export class Minio implements Storage {
+  client: Client
+
+  constructor(private options: MinioConfig) {
+    const url = new URL(this.options.endpoint)
+    this.client = new Client({
+      accessKey: options.accessKey,
+      secretKey: options.secretKey,
+      endPoint: url.hostname,
+      port: parseInt(url.port),
+      useSSL: url.protocol === 'https:',
+    })
+  }
+
+  async init(): Promise<void> {
+    await this.client.bucketExists(this.options.bucket)
+  }
+
+  async read(path: string): Promise<Buffer> {
+    const stream = await this.client.getObject(this.options.bucket, path)
+    return StreamUtils.toBuffer(stream)
+  }
+  async write(path: string, data: Buffer): Promise<void> {
+    const stream = await StreamUtils.fromBuffer(data)
+    await this.client.putObject(this.options.bucket, path, stream)
+  }
+
+  async readStream(path: string): Promise<NodeJS.ReadableStream> {
+    const stream = await this.client.getObject(this.options.bucket, path)
+    return stream
+  }
+  async writeStream(path: string): Promise<NodeJS.WritableStream> {
+    const stream = new PassThrough()
+    this.client.putObject(this.options.bucket, path, stream)
+    return stream
+  }
+
+  async exists(path: string): Promise<boolean> {
+    try {
+      await this.client.statObject(this.options.bucket, path)
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  delete(path: string): Promise<void> {
+    throw new Error('Method not implemented. Delete')
+  }
+}

src/transform/index.ts

@@ -6,8 +6,8 @@ import { ComplexParameter, TransformQueryBase } from '../controllers/image'
 import { storage } from '../storage'
 import { sha3, splitter } from '../utils/utils'
 
-async function downloadImage(url: string): Promise<NodeJS.ReadableStream> {
+async function downloadAndSaveImage(url: string, path: string): Promise<NodeJS.ReadableStream> {
-  const disk = await storage.writeStream(sha3(url))
+  const disk = await storage.writeStream(path)
   return new Promise((resolve) => {
     get(url, (res) => {
       const out = new PassThrough()
@@ -19,10 +19,11 @@ async function downloadImage(url: string): Promise<NodeJS.ReadableStream> {
 
 export async function getImage(url: string): Promise<NodeJS.ReadableStream> {
   const id = sha3(url)
-  if (!(await storage.exists(id))) {
+  try {
-    return await downloadImage(url)
-  }
     return await storage.readStream(id)
+  } catch {
+    return await downloadAndSaveImage(url, id)
+  }
 }
 
 function applyOperation(pipeline: sharp.Sharp, { name, options }: ComplexParameter<string, any>): sharp.Sharp {

src/utils/utils.ts

@@ -1,7 +1,6 @@
 import { createHash } from 'crypto'
 import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator'
 import { PassThrough, Readable } from 'stream'
-import { NullableStringOrRegexpArray } from '../config'
 
 export class ValidationError extends Error {
   override message: string
@@ -47,3 +46,21 @@ export function testForPrefixOrRegexp(str: string, values: (string | RegExp)[]):
   }
   return false
 }
+
+export class StreamUtils {
+  static fromBuffer(buffer: Buffer) {
+    const stream = new Readable()
+    stream.push(buffer)
+    stream.push(null)
+    return stream
+  }
+
+  static toBuffer(stream: NodeJS.ReadableStream) {
+    return new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = []
+      stream.on('data', (chunk) => chunks.push(chunk))
+      stream.on('error', reject)
+      stream.on('end', () => resolve(Buffer.concat(chunks)))
+    })
+  }
+}