support for s3 and minio

This commit is contained in:
cupcakearmy 2021-11-18 19:16:44 +01:00
parent 63e801ad05
commit c05ee888eb
No known key found for this signature in database
GPG Key ID: 3235314B4D31232F
10 changed files with 604 additions and 21 deletions

View File

@ -8,7 +8,7 @@ The heavy lifting is done by [`libvips`](https://github.com/libvips/libvips) and
- Config driven
- Domain protection
- Host verification
- Multiple storage adapters (Local, S3, GCS)
- Multiple storage adapters (Local, Minio, S3)
- Caniuse based automatic formatting
- ETag caching

View File

@ -13,6 +13,7 @@
"@types/convict": "^6.1.1",
"@types/flat": "^5.0.2",
"@types/js-yaml": "^4.0.4",
"@types/minio": "^7.0.11",
"@types/ms": "^0.7.31",
"@types/node": "^16.11.7",
"@types/sharp": "^0.29.3",
@ -31,6 +32,7 @@
"fastify-cors": "^6.0.2",
"flat": "^5.0.2",
"js-yaml": "^4.1.0",
"minio": "^7.0.19",
"ms": "^2.1.3",
"pino-pretty": "^7.2.0",
"sharp": "^0.29.3",

391
pnpm-lock.yaml generated
View File

@ -4,6 +4,7 @@ specifiers:
'@types/convict': ^6.1.1
'@types/flat': ^5.0.2
'@types/js-yaml': ^4.0.4
'@types/minio': ^7.0.11
'@types/ms': ^0.7.31
'@types/node': ^16.11.7
'@types/sharp': ^0.29.3
@ -18,6 +19,7 @@ specifiers:
fastify-cors: ^6.0.2
flat: ^5.0.2
js-yaml: ^4.1.0
minio: ^7.0.19
ms: ^2.1.3
pino-pretty: ^7.2.0
sharp: ^0.29.3
@ -37,6 +39,7 @@ dependencies:
fastify-cors: 6.0.2
flat: 5.0.2
js-yaml: 4.1.0
minio: 7.0.19
ms: 2.1.3
pino-pretty: 7.2.0
sharp: 0.29.3
@ -46,6 +49,7 @@ devDependencies:
'@types/convict': 6.1.1
'@types/flat': 5.0.2
'@types/js-yaml': 4.0.4
'@types/minio': 7.0.11
'@types/ms': 0.7.31
'@types/node': 16.11.7
'@types/sharp': 0.29.3
@ -74,6 +78,12 @@ packages:
resolution: {integrity: sha512-AuHubXUmg0AzkXH0Mx6sIxeY/1C110mm/EkE/gB1sTRz3h2dao2W/63q42SlVST+lICxz5Oki2hzYA6+KnnieQ==}
dev: true
/@types/minio/7.0.11:
resolution: {integrity: sha512-ltn30nGhtxytil4jFU1Tt6lvD+JnUyCYHfNBKsRjZ76ueSkrQdIByghcnhFvr15at3cnvj+tVn+euCqTX/5ejQ==}
dependencies:
'@types/node': 16.11.7
dev: true
/@types/ms/0.7.31:
resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
dev: true
@ -100,6 +110,12 @@ packages:
resolution: {integrity: sha512-+qogUELb4gMhrMjSh/seKmGVvN+uQLfyqJAqYRWqVHsvBsUO2xDBCL8CJ/ZSukbd8vXaoYbpIssAmfLEzzBHEw==}
dev: false
/@zxing/text-encoding/0.9.0:
resolution: {integrity: sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==}
requiresBuild: true
dev: false
optional: true
/abstract-cache/1.0.1:
resolution: {integrity: sha512-EfUeMhRUbG5bVVbrSY/ogLlFXoyfMAPxMlSP7wrEqH53d+59r2foVy9a5KjmprLKFLOfPQCNKEfpBN/nQ76chw==}
dependencies:
@ -183,11 +199,20 @@ packages:
mri: 1.1.4
dev: false
/async/3.2.2:
resolution: {integrity: sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g==}
dev: false
/atomic-sleep/1.0.0:
resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
engines: {node: '>=8.0.0'}
dev: false
/available-typed-arrays/1.0.5:
resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==}
engines: {node: '>= 0.4'}
dev: false
/avvio/7.2.2:
resolution: {integrity: sha512-XW2CMCmZaCmCCsIaJaLKxAzPwF37fXi1KGxNOvedOpeisLdmxZnblGc3hpHWYnlP+KOUxZsazh43WXNHgXpbqw==}
dependencies:
@ -231,6 +256,12 @@ packages:
readable-stream: 3.6.0
dev: false
/block-stream2/2.1.0:
resolution: {integrity: sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==}
dependencies:
readable-stream: 3.6.0
dev: false
/bluebird/3.4.7:
resolution: {integrity: sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=}
dev: false
@ -268,6 +299,13 @@ packages:
engines: {node: '>=0.2.0'}
dev: false
/call-bind/1.0.2:
resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==}
dependencies:
function-bind: 1.1.1
get-intrinsic: 1.1.1
dev: false
/camelcase/5.0.0:
resolution: {integrity: sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==}
engines: {node: '>=6'}
@ -436,6 +474,13 @@ packages:
engines: {node: '>=0.10.0'}
dev: false
/define-properties/1.1.3:
resolution: {integrity: sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==}
engines: {node: '>= 0.4'}
dependencies:
object-keys: 1.1.1
dev: false
/delegates/1.0.0:
resolution: {integrity: sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=}
dev: false
@ -497,6 +542,45 @@ packages:
once: 1.4.0
dev: false
/es-abstract/1.19.1:
resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==}
engines: {node: '>= 0.4'}
dependencies:
call-bind: 1.0.2
es-to-primitive: 1.2.1
function-bind: 1.1.1
get-intrinsic: 1.1.1
get-symbol-description: 1.0.0
has: 1.0.3
has-symbols: 1.0.2
internal-slot: 1.0.3
is-callable: 1.2.4
is-negative-zero: 2.0.1
is-regex: 1.1.4
is-shared-array-buffer: 1.0.1
is-string: 1.0.7
is-weakref: 1.0.1
object-inspect: 1.11.0
object-keys: 1.1.1
object.assign: 4.1.2
string.prototype.trimend: 1.0.4
string.prototype.trimstart: 1.0.4
unbox-primitive: 1.0.1
dev: false
/es-to-primitive/1.2.1:
resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==}
engines: {node: '>= 0.4'}
dependencies:
is-callable: 1.2.4
is-date-object: 1.0.5
is-symbol: 1.0.4
dev: false
/es6-error/4.1.1:
resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==}
dev: false
/escape-string-regexp/1.0.5:
resolution: {integrity: sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=}
engines: {node: '>=0.8.0'}
@ -538,6 +622,11 @@ packages:
resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}
dev: false
/fast-xml-parser/3.19.0:
resolution: {integrity: sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg==}
hasBin: true
dev: false
/fastify-caching/6.1.0:
resolution: {integrity: sha512-xhzjpI21qpHqoOKlUXlqpPSWm7UBTTgWrXxjfFpmxCbNBtN+JiQtzuXuSF8dUj5iig2ztRaD9+sD9uyPmNkHOw==}
dependencies:
@ -639,6 +728,10 @@ packages:
resolution: {integrity: sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==}
dev: false
/foreach/2.0.5:
resolution: {integrity: sha1-C+4AUBiusmDQo6865ljdATbsG5k=}
dev: false
/forwarded/0.2.0:
resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
engines: {node: '>= 0.6'}
@ -678,7 +771,6 @@ packages:
/function-bind/1.1.1:
resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==}
dev: true
/gauge/2.7.4:
resolution: {integrity: sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=}
@ -693,6 +785,22 @@ packages:
wide-align: 1.1.5
dev: false
/get-intrinsic/1.1.1:
resolution: {integrity: sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==}
dependencies:
function-bind: 1.1.1
has: 1.0.3
has-symbols: 1.0.2
dev: false
/get-symbol-description/1.0.0:
resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==}
engines: {node: '>= 0.4'}
dependencies:
call-bind: 1.0.2
get-intrinsic: 1.1.1
dev: false
/github-from-package/0.0.0:
resolution: {integrity: sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=}
dev: false
@ -718,11 +826,27 @@ packages:
resolution: {integrity: sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==}
dev: false
/has-bigints/1.0.1:
resolution: {integrity: sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==}
dev: false
/has-flag/3.0.0:
resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
engines: {node: '>=4'}
dev: false
/has-symbols/1.0.2:
resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==}
engines: {node: '>= 0.4'}
dev: false
/has-tostringtag/1.0.0:
resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==}
engines: {node: '>= 0.4'}
dependencies:
has-symbols: 1.0.2
dev: false
/has-unicode/2.0.1:
resolution: {integrity: sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=}
dev: false
@ -732,7 +856,6 @@ packages:
engines: {node: '>= 0.4.0'}
dependencies:
function-bind: 1.1.1
dev: true
/ieee754/1.2.1:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
@ -751,6 +874,15 @@ packages:
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
dev: false
/internal-slot/1.0.3:
resolution: {integrity: sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==}
engines: {node: '>= 0.4'}
dependencies:
get-intrinsic: 1.1.1
has: 1.0.3
side-channel: 1.0.4
dev: false
/into-stream/6.0.0:
resolution: {integrity: sha512-XHbaOAvP+uFKUFsOgoNPRjLkwB+I22JFPFe5OjTkQ0nwgj6+pSjb4NmB6VMxaPshLiOf+zcpOCBQuLwC1KHhZA==}
engines: {node: '>=10'}
@ -764,10 +896,24 @@ packages:
engines: {node: '>= 0.10'}
dev: false
/is-arguments/1.1.1:
resolution: {integrity: sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==}
engines: {node: '>= 0.4'}
dependencies:
call-bind: 1.0.2
has-tostringtag: 1.0.0
dev: false
/is-arrayish/0.3.2:
resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
dev: false
/is-bigint/1.0.4:
resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==}
dependencies:
has-bigints: 1.0.1
dev: false
/is-binary-path/2.1.0:
resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
engines: {node: '>=8'}
@ -775,12 +921,32 @@ packages:
binary-extensions: 2.2.0
dev: true
/is-boolean-object/1.1.2:
resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==}
engines: {node: '>= 0.4'}
dependencies:
call-bind: 1.0.2
has-tostringtag: 1.0.0
dev: false
/is-callable/1.2.4:
resolution: {integrity: sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==}
engines: {node: '>= 0.4'}
dev: false
/is-core-module/2.8.0:
resolution: {integrity: sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==}
dependencies:
has: 1.0.3
dev: true
/is-date-object/1.0.5:
resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==}
engines: {node: '>= 0.4'}
dependencies:
has-tostringtag: 1.0.0
dev: false
/is-deflate/1.0.0:
resolution: {integrity: sha1-yGKQHDwWH7CdrHzcfnhPgOmPLxQ=}
dev: false
@ -797,6 +963,13 @@ packages:
number-is-nan: 1.0.1
dev: false
/is-generator-function/1.0.10:
resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==}
engines: {node: '>= 0.4'}
dependencies:
has-tostringtag: 1.0.0
dev: false
/is-glob/4.0.3:
resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
engines: {node: '>=0.10.0'}
@ -809,6 +982,18 @@ packages:
engines: {node: '>=4'}
dev: false
/is-negative-zero/2.0.1:
resolution: {integrity: sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==}
engines: {node: '>= 0.4'}
dev: false
/is-number-object/1.0.6:
resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==}
engines: {node: '>= 0.4'}
dependencies:
has-tostringtag: 1.0.0
dev: false
/is-number/7.0.0:
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
engines: {node: '>=0.12.0'}
@ -819,11 +1004,54 @@ packages:
engines: {node: '>=0.10.0'}
dev: false
/is-regex/1.1.4:
resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==}
engines: {node: '>= 0.4'}
dependencies:
call-bind: 1.0.2
has-tostringtag: 1.0.0
dev: false
/is-shared-array-buffer/1.0.1:
resolution: {integrity: sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==}
dev: false
/is-stream/2.0.1:
resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
engines: {node: '>=8'}
dev: false
/is-string/1.0.7:
resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==}
engines: {node: '>= 0.4'}
dependencies:
has-tostringtag: 1.0.0
dev: false
/is-symbol/1.0.4:
resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==}
engines: {node: '>= 0.4'}
dependencies:
has-symbols: 1.0.2
dev: false
/is-typed-array/1.1.8:
resolution: {integrity: sha512-HqH41TNZq2fgtGT8WHVFVJhBVGuY3AnP3Q36K8JKXUxSxRgk/d+7NjmwG2vo2mYmXK8UYZKu0qH8bVP5gEisjA==}
engines: {node: '>= 0.4'}
dependencies:
available-typed-arrays: 1.0.5
call-bind: 1.0.2
es-abstract: 1.19.1
foreach: 2.0.5
has-tostringtag: 1.0.0
dev: false
/is-weakref/1.0.1:
resolution: {integrity: sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==}
dependencies:
call-bind: 1.0.2
dev: false
/is-zip/1.0.0:
resolution: {integrity: sha1-R7Co/004p2QxzP2ZqOFaTIa6IyU=}
engines: {node: '>=0.10.0'}
@ -853,6 +1081,10 @@ packages:
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
dev: false
/json-stream/1.0.0:
resolution: {integrity: sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg=}
dev: false
/leven/2.1.0:
resolution: {integrity: sha1-wuep93IJTe6dNCAq6KzORoeHVYA=}
engines: {node: '>=0.10.0'}
@ -879,6 +1111,10 @@ packages:
resolution: {integrity: sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=}
dev: false
/lodash/4.17.21:
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
dev: false
/lru-cache/6.0.0:
resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
engines: {node: '>=10'}
@ -906,6 +1142,13 @@ packages:
engines: {node: '>= 0.6'}
dev: false
/mime-types/2.1.34:
resolution: {integrity: sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==}
engines: {node: '>= 0.6'}
dependencies:
mime-db: 1.51.0
dev: false
/mimic-response/3.1.0:
resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
engines: {node: '>=10'}
@ -919,6 +1162,25 @@ packages:
/minimist/1.2.5:
resolution: {integrity: sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==}
/minio/7.0.19:
resolution: {integrity: sha512-DOGKauWLdmj0/y2QKXdnrhqyzRFEnUteHi6q382uujg9TjSDrA84BiQVppS2Ew6V8Rcg+2IaRkF4GR34zw9sIA==}
engines: {node: '>= 4'}
dependencies:
async: 3.2.2
block-stream2: 2.1.0
es6-error: 4.1.1
fast-xml-parser: 3.19.0
json-stream: 1.0.0
lodash: 4.17.21
mime-types: 2.1.34
mkdirp: 0.5.5
querystring: 0.2.0
through2: 3.0.2
web-encoding: 1.1.5
xml: 1.0.1
xml2js: 0.4.23
dev: false
/minipass/3.1.5:
resolution: {integrity: sha512-+8NzxD82XQoNKNrl1d/FSi+X8wAEWR+sbYAfIvub4Nz0d22plFG72CEVVaufV8PNf4qSslFTD8VMOxNVhHCjTw==}
engines: {node: '>=8'}
@ -995,6 +1257,25 @@ packages:
engines: {node: '>=0.10.0'}
dev: false
/object-inspect/1.11.0:
resolution: {integrity: sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==}
dev: false
/object-keys/1.1.1:
resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==}
engines: {node: '>= 0.4'}
dev: false
/object.assign/4.1.2:
resolution: {integrity: sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==}
engines: {node: '>= 0.4'}
dependencies:
call-bind: 1.0.2
define-properties: 1.1.3
has-symbols: 1.0.2
object-keys: 1.1.1
dev: false
/once/1.4.0:
resolution: {integrity: sha1-WDsap3WWHUsROsF9nFC6753Xa9E=}
dependencies:
@ -1120,6 +1401,12 @@ packages:
engines: {node: '>=6'}
dev: false
/querystring/0.2.0:
resolution: {integrity: sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=}
engines: {node: '>=0.4.x'}
deprecated: The querystring API is considered Legacy. new code should use the URLSearchParams API instead.
dev: false
/queue-microtask/1.2.3:
resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
dev: false
@ -1217,6 +1504,10 @@ packages:
ret: 0.2.2
dev: false
/sax/1.2.4:
resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==}
dev: false
/secure-json-parse/2.4.0:
resolution: {integrity: sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==}
dev: false
@ -1260,6 +1551,14 @@ packages:
tunnel-agent: 0.6.0
dev: false
/side-channel/1.0.4:
resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==}
dependencies:
call-bind: 1.0.2
get-intrinsic: 1.1.1
object-inspect: 1.11.0
dev: false
/signal-exit/3.0.5:
resolution: {integrity: sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ==}
dev: false
@ -1335,6 +1634,20 @@ packages:
strip-ansi: 3.0.1
dev: false
/string.prototype.trimend/1.0.4:
resolution: {integrity: sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==}
dependencies:
call-bind: 1.0.2
define-properties: 1.1.3
dev: false
/string.prototype.trimstart/1.0.4:
resolution: {integrity: sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==}
dependencies:
call-bind: 1.0.2
define-properties: 1.1.3
dev: false
/string_decoder/1.1.1:
resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==}
dependencies:
@ -1402,6 +1715,13 @@ packages:
xtend: 4.0.2
dev: false
/through2/3.0.2:
resolution: {integrity: sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==}
dependencies:
inherits: 2.0.4
readable-stream: 3.6.0
dev: false
/tiny-lru/7.0.6:
resolution: {integrity: sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow==}
engines: {node: '>=6'}
@ -1491,6 +1811,15 @@ packages:
random-bytes: 1.0.0
dev: false
/unbox-primitive/1.0.1:
resolution: {integrity: sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==}
dependencies:
function-bind: 1.1.1
has-bigints: 1.0.1
has-symbols: 1.0.2
which-boxed-primitive: 1.0.2
dev: false
/under-pressure/5.8.0:
resolution: {integrity: sha512-8ADLZkFEGDAsKof1FEICH/OLyGWjDZy6KR/Exq3MTv7W81zC2W23VekY05Fo350lip1ywYNH9wP7itiVnk4wHg==}
engines: {node: '>=10'}
@ -1524,6 +1853,17 @@ packages:
resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=}
dev: false
/util/0.12.4:
resolution: {integrity: sha512-bxZ9qtSlGUWSOy9Qa9Xgk11kSslpuZwaxCg4sNIDj6FLucDab2JxnHwyNTCpHMtK1MjoQiWQ6DiUMZYbSrO+Sw==}
dependencies:
inherits: 2.0.4
is-arguments: 1.1.1
is-generator-function: 1.0.10
is-typed-array: 1.1.8
safe-buffer: 5.2.1
which-typed-array: 1.1.7
dev: false
/validator/13.7.0:
resolution: {integrity: sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==}
engines: {node: '>= 0.10'}
@ -1534,6 +1874,36 @@ packages:
engines: {node: '>= 0.8'}
dev: false
/web-encoding/1.1.5:
resolution: {integrity: sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==}
dependencies:
util: 0.12.4
optionalDependencies:
'@zxing/text-encoding': 0.9.0
dev: false
/which-boxed-primitive/1.0.2:
resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==}
dependencies:
is-bigint: 1.0.4
is-boolean-object: 1.1.2
is-number-object: 1.0.6
is-string: 1.0.7
is-symbol: 1.0.4
dev: false
/which-typed-array/1.1.7:
resolution: {integrity: sha512-vjxaB4nfDqwKI0ws7wZpxIlde1XrLX5uB0ZjpfshgmapJMD7jJWhZI+yToJTqaFByF0eNBcYxbjmCzoRP7CfEw==}
engines: {node: '>= 0.4'}
dependencies:
available-typed-arrays: 1.0.5
call-bind: 1.0.2
es-abstract: 1.19.1
foreach: 2.0.5
has-tostringtag: 1.0.0
is-typed-array: 1.1.8
dev: false
/wide-align/1.1.5:
resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==}
dependencies:
@ -1543,6 +1913,23 @@ packages:
/wrappy/1.0.2:
resolution: {integrity: sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=}
/xml/1.0.1:
resolution: {integrity: sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=}
dev: false
/xml2js/0.4.23:
resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==}
engines: {node: '>=4.0.0'}
dependencies:
sax: 1.2.4
xmlbuilder: 11.0.1
dev: false
/xmlbuilder/11.0.1:
resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==}
engines: {node: '>=4.0'}
dev: false
/xtend/4.0.2:
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
engines: {node: '>=0.4'}

View File

@ -6,9 +6,11 @@ convict.addFormat(require('convict-format-with-validator').ipaddress)
export enum StorageType {
Local = 'local',
// S3 = 's3',
Minio = 'minio',
S3 = 's3',
// GCS = 'gcs',
// Azure = 'azure',
// B2 = 'b2',
}
export enum URLClean {
@ -53,6 +55,14 @@ const config = convict({
env: 'ADDRESS',
},
// Logging
logLevel: {
doc: 'The level of logging to use.',
format: ['trace', 'debug', 'info', 'warn', 'error', 'fatal'],
default: 'info',
env: 'LOG_LEVEL',
},
// Security
allowedDomains: {
doc: 'The domains that are allowed to be used as image sources',
@ -97,6 +107,72 @@ const config = convict({
default: './assets',
env: 'LOCAL_ASSETS',
},
// Minio storage
minio: {
accessKey: {
doc: 'The access key for Minio',
format: String,
default: '',
env: 'MINIO_ACCESS_KEY',
sensitive: true,
},
secretKey: {
doc: 'The secret key for Minio',
format: String,
default: '',
env: 'MINIO_SECRET_KEY',
sensitive: true,
},
endpoint: {
doc: 'The endpoint for Minio',
format: String,
default: '',
env: 'MINIO_ENDPOINT',
},
bucket: {
doc: 'The bucket to use for Minio',
format: String,
default: '',
env: 'MINIO_BUCKET',
},
region: {
doc: 'The region for Minio',
format: String,
default: '',
env: 'MINIO_REGION',
},
},
// S3 storage
s3: {
bucket: {
doc: 'The S3 bucket to use',
format: String,
default: '',
env: 'S3_BUCKET',
},
region: {
doc: 'The S3 region to use',
format: String,
default: '',
env: 'S3_REGION',
},
accessKey: {
doc: 'The S3 access key id to use',
format: String,
default: '',
env: 'S3_ACCESS_KEY_ID',
sensitive: true,
},
secretKey: {
doc: 'The S3 secret access key to use',
format: String,
default: '',
env: 'S3_SECRET_ACCESS_KEY',
sensitive: true,
},
},
})
for (const file of ['morphus.yaml', 'morphus.yaml']) {

View File

@ -187,9 +187,12 @@ export const image: RouteHandlerMethod = async (request, reply) => {
// @ts-ignore
reply.expires(new Date(Date.now() + ms(Config.maxAge)))
let stream: NodeJS.ReadableStream = (await storage.exists(q.hash))
? await storage.readStream(q.hash)
: await transform(q)
let stream: NodeJS.ReadableStream
try {
stream = await storage.readStream(q.hash)
} catch (err) {
stream = await transform(q)
}
reply.code(200).headers({
'Content-Type': `image/${q.format?.name}`,

View File

@ -6,7 +6,7 @@ import { init as initStorage } from './storage'
import { init as initMiddleware } from './fastify/middleware'
import { init as initHooks } from './fastify/hooks'
export const App = fastify({ logger: { prettyPrint: true } })
export const App = fastify({ logger: { prettyPrint: true, level: Config.logLevel } })
process.on('SIGINT', async function () {
App.log.info('Stopping server')
@ -19,7 +19,7 @@ async function main() {
try {
// Internal
initConfig(App)
initStorage()
await initStorage(App)
// Fastify
initMiddleware(App)

View File

@ -1,30 +1,63 @@
import { FastifyInstance } from 'fastify'
import { Config, StorageType } from '../config'
import { Local } from './local'
import { Minio } from './minio'
export abstract class Storage {
abstract init(): Promise<void>
abstract read(path: string): Promise<Buffer>
abstract write(path: string, data: Buffer): Promise<void>
abstract exists(path: string): Promise<boolean>
abstract delete(path: string): Promise<void>
abstract readStream(path: string): Promise<NodeJS.ReadableStream>
abstract writeStream(path: string): Promise<NodeJS.WritableStream>
// list(path: string): Promise<string[]>
abstract init(): Promise<void>
// list(path: string): Promise<string[]>
abstract exists(path: string): Promise<boolean>
abstract delete(path: string): Promise<void>
}
export let storage: Storage
export async function init() {
export async function init(App: FastifyInstance) {
if (!storage) {
switch (Config.storage) {
case StorageType.Local:
storage = new Local(Config.localAssets)
break
case StorageType.S3:
// storage = new S3({
// accessKeyId: Config.s3.accessKey,
// secretAccessKey: Config.s3.secretKey,
// bucket: Config.s3.bucket,
// region: Config.s3.region,
// })
storage = new Minio({
accessKey: Config.s3.accessKey,
secretKey: Config.s3.secretKey,
bucket: Config.s3.bucket,
region: Config.s3.region,
endpoint: 'https://s3.amazonaws.com',
})
break
case StorageType.Minio:
storage = new Minio({
accessKey: Config.minio.accessKey,
secretKey: Config.minio.secretKey,
endpoint: Config.minio.endpoint,
region: Config.minio.region,
bucket: Config.minio.bucket,
})
break
default:
throw new Error(`Unknown storage type: ${Config.storage}`)
}
await storage.init()
try {
await storage.init()
App.log.debug(`Storage initialized: ${Config.storage}`)
} catch (e) {
App.log.error(`Storage initialization failed: ${Config.storage}`)
process.exit(1)
}
}
}

64
src/storage/minio.ts Normal file
View File

@ -0,0 +1,64 @@
import { Client } from 'minio'
import { PassThrough } from 'stream'
import { Storage } from '.'
import { StreamUtils } from '../utils/utils'
export type MinioConfig = {
accessKey: string
secretKey: string
endpoint: string
region?: string
bucket: string
}
export class Minio implements Storage {
client: Client
constructor(private options: MinioConfig) {
const url = new URL(this.options.endpoint)
this.client = new Client({
accessKey: options.accessKey,
secretKey: options.secretKey,
endPoint: url.hostname,
port: parseInt(url.port),
useSSL: url.protocol === 'https:',
})
}
async init(): Promise<void> {
await this.client.bucketExists(this.options.bucket)
}
async read(path: string): Promise<Buffer> {
const stream = await this.client.getObject(this.options.bucket, path)
return StreamUtils.toBuffer(stream)
}
async write(path: string, data: Buffer): Promise<void> {
const stream = await StreamUtils.fromBuffer(data)
await this.client.putObject(this.options.bucket, path, stream)
}
async readStream(path: string): Promise<NodeJS.ReadableStream> {
const stream = await this.client.getObject(this.options.bucket, path)
return stream
}
async writeStream(path: string): Promise<NodeJS.WritableStream> {
const stream = new PassThrough()
this.client.putObject(this.options.bucket, path, stream)
return stream
}
async exists(path: string): Promise<boolean> {
try {
await this.client.statObject(this.options.bucket, path)
return true
} catch {
return false
}
}
delete(path: string): Promise<void> {
throw new Error('Method not implemented. Delete')
}
}

View File

@ -6,8 +6,8 @@ import { ComplexParameter, TransformQueryBase } from '../controllers/image'
import { storage } from '../storage'
import { sha3, splitter } from '../utils/utils'
async function downloadImage(url: string): Promise<NodeJS.ReadableStream> {
const disk = await storage.writeStream(sha3(url))
async function downloadAndSaveImage(url: string, path: string): Promise<NodeJS.ReadableStream> {
const disk = await storage.writeStream(path)
return new Promise((resolve) => {
get(url, (res) => {
const out = new PassThrough()
@ -19,10 +19,11 @@ async function downloadImage(url: string): Promise<NodeJS.ReadableStream> {
export async function getImage(url: string): Promise<NodeJS.ReadableStream> {
const id = sha3(url)
if (!(await storage.exists(id))) {
return await downloadImage(url)
try {
return await storage.readStream(id)
} catch {
return await downloadAndSaveImage(url, id)
}
return await storage.readStream(id)
}
function applyOperation(pipeline: sharp.Sharp, { name, options }: ComplexParameter<string, any>): sharp.Sharp {

View File

@ -1,7 +1,6 @@
import { createHash } from 'crypto'
import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator'
import { PassThrough, Readable } from 'stream'
import { NullableStringOrRegexpArray } from '../config'
export class ValidationError extends Error {
override message: string
@ -47,3 +46,21 @@ export function testForPrefixOrRegexp(str: string, values: (string | RegExp)[]):
}
return false
}
export class StreamUtils {
static fromBuffer(buffer: Buffer) {
const stream = new Readable()
stream.push(buffer)
stream.push(null)
return stream
}
static toBuffer(stream: NodeJS.ReadableStream) {
return new Promise<Buffer>((resolve, reject) => {
const chunks: Buffer[] = []
stream.on('data', (chunk) => chunks.push(chunk))
stream.on('error', reject)
stream.on('end', () => resolve(Buffer.concat(chunks)))
})
}
}