diff --git a/README.md b/README.md index 3563a33..6c0866d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The heavy lifting is done by [`libvips`](https://github.com/libvips/libvips) and - Config driven - Domain protection - Host verification -- Multiple storage adapters (Local, S3, GCS) +- Multiple storage adapters (Local, Minio, S3) - Caniuse based automatic formatting - ETag caching diff --git a/package.json b/package.json index 6999c02..67de363 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "@types/convict": "^6.1.1", "@types/flat": "^5.0.2", "@types/js-yaml": "^4.0.4", + "@types/minio": "^7.0.11", "@types/ms": "^0.7.31", "@types/node": "^16.11.7", "@types/sharp": "^0.29.3", @@ -31,6 +32,7 @@ "fastify-cors": "^6.0.2", "flat": "^5.0.2", "js-yaml": "^4.1.0", + "minio": "^7.0.19", "ms": "^2.1.3", "pino-pretty": "^7.2.0", "sharp": "^0.29.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 11ff39d..1fb92f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,7 @@ specifiers: '@types/convict': ^6.1.1 '@types/flat': ^5.0.2 '@types/js-yaml': ^4.0.4 + '@types/minio': ^7.0.11 '@types/ms': ^0.7.31 '@types/node': ^16.11.7 '@types/sharp': ^0.29.3 @@ -18,6 +19,7 @@ specifiers: fastify-cors: ^6.0.2 flat: ^5.0.2 js-yaml: ^4.1.0 + minio: ^7.0.19 ms: ^2.1.3 pino-pretty: ^7.2.0 sharp: ^0.29.3 @@ -37,6 +39,7 @@ dependencies: fastify-cors: 6.0.2 flat: 5.0.2 js-yaml: 4.1.0 + minio: 7.0.19 ms: 2.1.3 pino-pretty: 7.2.0 sharp: 0.29.3 @@ -46,6 +49,7 @@ devDependencies: '@types/convict': 6.1.1 '@types/flat': 5.0.2 '@types/js-yaml': 4.0.4 + '@types/minio': 7.0.11 '@types/ms': 0.7.31 '@types/node': 16.11.7 '@types/sharp': 0.29.3 @@ -74,6 +78,12 @@ packages: resolution: {integrity: sha512-AuHubXUmg0AzkXH0Mx6sIxeY/1C110mm/EkE/gB1sTRz3h2dao2W/63q42SlVST+lICxz5Oki2hzYA6+KnnieQ==} dev: true + /@types/minio/7.0.11: + resolution: {integrity: sha512-ltn30nGhtxytil4jFU1Tt6lvD+JnUyCYHfNBKsRjZ76ueSkrQdIByghcnhFvr15at3cnvj+tVn+euCqTX/5ejQ==} + dependencies: + '@types/node': 16.11.7 + dev: true + /@types/ms/0.7.31: resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} dev: true @@ -100,6 +110,12 @@ packages: resolution: {integrity: sha512-+qogUELb4gMhrMjSh/seKmGVvN+uQLfyqJAqYRWqVHsvBsUO2xDBCL8CJ/ZSukbd8vXaoYbpIssAmfLEzzBHEw==} dev: false + /@zxing/text-encoding/0.9.0: + resolution: {integrity: sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==} + requiresBuild: true + dev: false + optional: true + /abstract-cache/1.0.1: resolution: {integrity: sha512-EfUeMhRUbG5bVVbrSY/ogLlFXoyfMAPxMlSP7wrEqH53d+59r2foVy9a5KjmprLKFLOfPQCNKEfpBN/nQ76chw==} dependencies: @@ -183,11 +199,20 @@ packages: mri: 1.1.4 dev: false + /async/3.2.2: + resolution: {integrity: sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g==} + dev: false + /atomic-sleep/1.0.0: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} dev: false + /available-typed-arrays/1.0.5: + resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==} + engines: {node: '>= 0.4'} + dev: false + /avvio/7.2.2: resolution: {integrity: sha512-XW2CMCmZaCmCCsIaJaLKxAzPwF37fXi1KGxNOvedOpeisLdmxZnblGc3hpHWYnlP+KOUxZsazh43WXNHgXpbqw==} dependencies: @@ -231,6 +256,12 @@ packages: readable-stream: 3.6.0 dev: false + /block-stream2/2.1.0: + resolution: {integrity: sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==} + dependencies: + readable-stream: 3.6.0 + dev: false + /bluebird/3.4.7: resolution: {integrity: sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=} dev: false @@ -268,6 +299,13 @@ packages: engines: {node: '>=0.2.0'} dev: false + /call-bind/1.0.2: + resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==} + dependencies: + function-bind: 1.1.1 + get-intrinsic: 1.1.1 + dev: false + /camelcase/5.0.0: resolution: {integrity: sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==} engines: {node: '>=6'} @@ -436,6 +474,13 @@ packages: engines: {node: '>=0.10.0'} dev: false + /define-properties/1.1.3: + resolution: {integrity: sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==} + engines: {node: '>= 0.4'} + dependencies: + object-keys: 1.1.1 + dev: false + /delegates/1.0.0: resolution: {integrity: sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=} dev: false @@ -497,6 +542,45 @@ packages: once: 1.4.0 dev: false + /es-abstract/1.19.1: + resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + es-to-primitive: 1.2.1 + function-bind: 1.1.1 + get-intrinsic: 1.1.1 + get-symbol-description: 1.0.0 + has: 1.0.3 + has-symbols: 1.0.2 + internal-slot: 1.0.3 + is-callable: 1.2.4 + is-negative-zero: 2.0.1 + is-regex: 1.1.4 + is-shared-array-buffer: 1.0.1 + is-string: 1.0.7 + is-weakref: 1.0.1 + object-inspect: 1.11.0 + object-keys: 1.1.1 + object.assign: 4.1.2 + string.prototype.trimend: 1.0.4 + string.prototype.trimstart: 1.0.4 + unbox-primitive: 1.0.1 + dev: false + + /es-to-primitive/1.2.1: + resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} + engines: {node: '>= 0.4'} + dependencies: + is-callable: 1.2.4 + is-date-object: 1.0.5 + is-symbol: 1.0.4 + dev: false + + /es6-error/4.1.1: + resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} + dev: false + /escape-string-regexp/1.0.5: resolution: {integrity: sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=} engines: {node: '>=0.8.0'} @@ -538,6 +622,11 @@ packages: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} dev: false + /fast-xml-parser/3.19.0: + resolution: {integrity: sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg==} + hasBin: true + dev: false + /fastify-caching/6.1.0: resolution: {integrity: sha512-xhzjpI21qpHqoOKlUXlqpPSWm7UBTTgWrXxjfFpmxCbNBtN+JiQtzuXuSF8dUj5iig2ztRaD9+sD9uyPmNkHOw==} dependencies: @@ -639,6 +728,10 @@ packages: resolution: {integrity: sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==} dev: false + /foreach/2.0.5: + resolution: {integrity: sha1-C+4AUBiusmDQo6865ljdATbsG5k=} + dev: false + /forwarded/0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -678,7 +771,6 @@ packages: /function-bind/1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - dev: true /gauge/2.7.4: resolution: {integrity: sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=} @@ -693,6 +785,22 @@ packages: wide-align: 1.1.5 dev: false + /get-intrinsic/1.1.1: + resolution: {integrity: sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==} + dependencies: + function-bind: 1.1.1 + has: 1.0.3 + has-symbols: 1.0.2 + dev: false + + /get-symbol-description/1.0.0: + resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + get-intrinsic: 1.1.1 + dev: false + /github-from-package/0.0.0: resolution: {integrity: sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=} dev: false @@ -718,11 +826,27 @@ packages: resolution: {integrity: sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==} dev: false + /has-bigints/1.0.1: + resolution: {integrity: sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==} + dev: false + /has-flag/3.0.0: resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=} engines: {node: '>=4'} dev: false + /has-symbols/1.0.2: + resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==} + engines: {node: '>= 0.4'} + dev: false + + /has-tostringtag/1.0.0: + resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} + engines: {node: '>= 0.4'} + dependencies: + has-symbols: 1.0.2 + dev: false + /has-unicode/2.0.1: resolution: {integrity: sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=} dev: false @@ -732,7 +856,6 @@ packages: engines: {node: '>= 0.4.0'} dependencies: function-bind: 1.1.1 - dev: true /ieee754/1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -751,6 +874,15 @@ packages: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} dev: false + /internal-slot/1.0.3: + resolution: {integrity: sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==} + engines: {node: '>= 0.4'} + dependencies: + get-intrinsic: 1.1.1 + has: 1.0.3 + side-channel: 1.0.4 + dev: false + /into-stream/6.0.0: resolution: {integrity: sha512-XHbaOAvP+uFKUFsOgoNPRjLkwB+I22JFPFe5OjTkQ0nwgj6+pSjb4NmB6VMxaPshLiOf+zcpOCBQuLwC1KHhZA==} engines: {node: '>=10'} @@ -764,10 +896,24 @@ packages: engines: {node: '>= 0.10'} dev: false + /is-arguments/1.1.1: + resolution: {integrity: sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + has-tostringtag: 1.0.0 + dev: false + /is-arrayish/0.3.2: resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} dev: false + /is-bigint/1.0.4: + resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==} + dependencies: + has-bigints: 1.0.1 + dev: false + /is-binary-path/2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} @@ -775,12 +921,32 @@ packages: binary-extensions: 2.2.0 dev: true + /is-boolean-object/1.1.2: + resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + has-tostringtag: 1.0.0 + dev: false + + /is-callable/1.2.4: + resolution: {integrity: sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==} + engines: {node: '>= 0.4'} + dev: false + /is-core-module/2.8.0: resolution: {integrity: sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==} dependencies: has: 1.0.3 dev: true + /is-date-object/1.0.5: + resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==} + engines: {node: '>= 0.4'} + dependencies: + has-tostringtag: 1.0.0 + dev: false + /is-deflate/1.0.0: resolution: {integrity: sha1-yGKQHDwWH7CdrHzcfnhPgOmPLxQ=} dev: false @@ -797,6 +963,13 @@ packages: number-is-nan: 1.0.1 dev: false + /is-generator-function/1.0.10: + resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==} + engines: {node: '>= 0.4'} + dependencies: + has-tostringtag: 1.0.0 + dev: false + /is-glob/4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -809,6 +982,18 @@ packages: engines: {node: '>=4'} dev: false + /is-negative-zero/2.0.1: + resolution: {integrity: sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==} + engines: {node: '>= 0.4'} + dev: false + + /is-number-object/1.0.6: + resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==} + engines: {node: '>= 0.4'} + dependencies: + has-tostringtag: 1.0.0 + dev: false + /is-number/7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} @@ -819,11 +1004,54 @@ packages: engines: {node: '>=0.10.0'} dev: false + /is-regex/1.1.4: + resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + has-tostringtag: 1.0.0 + dev: false + + /is-shared-array-buffer/1.0.1: + resolution: {integrity: sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==} + dev: false + /is-stream/2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} dev: false + /is-string/1.0.7: + resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} + engines: {node: '>= 0.4'} + dependencies: + has-tostringtag: 1.0.0 + dev: false + + /is-symbol/1.0.4: + resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==} + engines: {node: '>= 0.4'} + dependencies: + has-symbols: 1.0.2 + dev: false + + /is-typed-array/1.1.8: + resolution: {integrity: sha512-HqH41TNZq2fgtGT8WHVFVJhBVGuY3AnP3Q36K8JKXUxSxRgk/d+7NjmwG2vo2mYmXK8UYZKu0qH8bVP5gEisjA==} + engines: {node: '>= 0.4'} + dependencies: + available-typed-arrays: 1.0.5 + call-bind: 1.0.2 + es-abstract: 1.19.1 + foreach: 2.0.5 + has-tostringtag: 1.0.0 + dev: false + + /is-weakref/1.0.1: + resolution: {integrity: sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==} + dependencies: + call-bind: 1.0.2 + dev: false + /is-zip/1.0.0: resolution: {integrity: sha1-R7Co/004p2QxzP2ZqOFaTIa6IyU=} engines: {node: '>=0.10.0'} @@ -853,6 +1081,10 @@ packages: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} dev: false + /json-stream/1.0.0: + resolution: {integrity: sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg=} + dev: false + /leven/2.1.0: resolution: {integrity: sha1-wuep93IJTe6dNCAq6KzORoeHVYA=} engines: {node: '>=0.10.0'} @@ -879,6 +1111,10 @@ packages: resolution: {integrity: sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=} dev: false + /lodash/4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + /lru-cache/6.0.0: resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} engines: {node: '>=10'} @@ -906,6 +1142,13 @@ packages: engines: {node: '>= 0.6'} dev: false + /mime-types/2.1.34: + resolution: {integrity: sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.51.0 + dev: false + /mimic-response/3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} @@ -919,6 +1162,25 @@ packages: /minimist/1.2.5: resolution: {integrity: sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==} + /minio/7.0.19: + resolution: {integrity: sha512-DOGKauWLdmj0/y2QKXdnrhqyzRFEnUteHi6q382uujg9TjSDrA84BiQVppS2Ew6V8Rcg+2IaRkF4GR34zw9sIA==} + engines: {node: '>= 4'} + dependencies: + async: 3.2.2 + block-stream2: 2.1.0 + es6-error: 4.1.1 + fast-xml-parser: 3.19.0 + json-stream: 1.0.0 + lodash: 4.17.21 + mime-types: 2.1.34 + mkdirp: 0.5.5 + querystring: 0.2.0 + through2: 3.0.2 + web-encoding: 1.1.5 + xml: 1.0.1 + xml2js: 0.4.23 + dev: false + /minipass/3.1.5: resolution: {integrity: sha512-+8NzxD82XQoNKNrl1d/FSi+X8wAEWR+sbYAfIvub4Nz0d22plFG72CEVVaufV8PNf4qSslFTD8VMOxNVhHCjTw==} engines: {node: '>=8'} @@ -995,6 +1257,25 @@ packages: engines: {node: '>=0.10.0'} dev: false + /object-inspect/1.11.0: + resolution: {integrity: sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==} + dev: false + + /object-keys/1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + dev: false + + /object.assign/4.1.2: + resolution: {integrity: sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + has-symbols: 1.0.2 + object-keys: 1.1.1 + dev: false + /once/1.4.0: resolution: {integrity: sha1-WDsap3WWHUsROsF9nFC6753Xa9E=} dependencies: @@ -1120,6 +1401,12 @@ packages: engines: {node: '>=6'} dev: false + /querystring/0.2.0: + resolution: {integrity: sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=} + engines: {node: '>=0.4.x'} + deprecated: The querystring API is considered Legacy. new code should use the URLSearchParams API instead. + dev: false + /queue-microtask/1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} dev: false @@ -1217,6 +1504,10 @@ packages: ret: 0.2.2 dev: false + /sax/1.2.4: + resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} + dev: false + /secure-json-parse/2.4.0: resolution: {integrity: sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==} dev: false @@ -1260,6 +1551,14 @@ packages: tunnel-agent: 0.6.0 dev: false + /side-channel/1.0.4: + resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} + dependencies: + call-bind: 1.0.2 + get-intrinsic: 1.1.1 + object-inspect: 1.11.0 + dev: false + /signal-exit/3.0.5: resolution: {integrity: sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ==} dev: false @@ -1335,6 +1634,20 @@ packages: strip-ansi: 3.0.1 dev: false + /string.prototype.trimend/1.0.4: + resolution: {integrity: sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + dev: false + + /string.prototype.trimstart/1.0.4: + resolution: {integrity: sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + dev: false + /string_decoder/1.1.1: resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} dependencies: @@ -1402,6 +1715,13 @@ packages: xtend: 4.0.2 dev: false + /through2/3.0.2: + resolution: {integrity: sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==} + dependencies: + inherits: 2.0.4 + readable-stream: 3.6.0 + dev: false + /tiny-lru/7.0.6: resolution: {integrity: sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow==} engines: {node: '>=6'} @@ -1491,6 +1811,15 @@ packages: random-bytes: 1.0.0 dev: false + /unbox-primitive/1.0.1: + resolution: {integrity: sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==} + dependencies: + function-bind: 1.1.1 + has-bigints: 1.0.1 + has-symbols: 1.0.2 + which-boxed-primitive: 1.0.2 + dev: false + /under-pressure/5.8.0: resolution: {integrity: sha512-8ADLZkFEGDAsKof1FEICH/OLyGWjDZy6KR/Exq3MTv7W81zC2W23VekY05Fo350lip1ywYNH9wP7itiVnk4wHg==} engines: {node: '>=10'} @@ -1524,6 +1853,17 @@ packages: resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=} dev: false + /util/0.12.4: + resolution: {integrity: sha512-bxZ9qtSlGUWSOy9Qa9Xgk11kSslpuZwaxCg4sNIDj6FLucDab2JxnHwyNTCpHMtK1MjoQiWQ6DiUMZYbSrO+Sw==} + dependencies: + inherits: 2.0.4 + is-arguments: 1.1.1 + is-generator-function: 1.0.10 + is-typed-array: 1.1.8 + safe-buffer: 5.2.1 + which-typed-array: 1.1.7 + dev: false + /validator/13.7.0: resolution: {integrity: sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==} engines: {node: '>= 0.10'} @@ -1534,6 +1874,36 @@ packages: engines: {node: '>= 0.8'} dev: false + /web-encoding/1.1.5: + resolution: {integrity: sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==} + dependencies: + util: 0.12.4 + optionalDependencies: + '@zxing/text-encoding': 0.9.0 + dev: false + + /which-boxed-primitive/1.0.2: + resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} + dependencies: + is-bigint: 1.0.4 + is-boolean-object: 1.1.2 + is-number-object: 1.0.6 + is-string: 1.0.7 + is-symbol: 1.0.4 + dev: false + + /which-typed-array/1.1.7: + resolution: {integrity: sha512-vjxaB4nfDqwKI0ws7wZpxIlde1XrLX5uB0ZjpfshgmapJMD7jJWhZI+yToJTqaFByF0eNBcYxbjmCzoRP7CfEw==} + engines: {node: '>= 0.4'} + dependencies: + available-typed-arrays: 1.0.5 + call-bind: 1.0.2 + es-abstract: 1.19.1 + foreach: 2.0.5 + has-tostringtag: 1.0.0 + is-typed-array: 1.1.8 + dev: false + /wide-align/1.1.5: resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} dependencies: @@ -1543,6 +1913,23 @@ packages: /wrappy/1.0.2: resolution: {integrity: sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=} + /xml/1.0.1: + resolution: {integrity: sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=} + dev: false + + /xml2js/0.4.23: + resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==} + engines: {node: '>=4.0.0'} + dependencies: + sax: 1.2.4 + xmlbuilder: 11.0.1 + dev: false + + /xmlbuilder/11.0.1: + resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==} + engines: {node: '>=4.0'} + dev: false + /xtend/4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} diff --git a/src/config.ts b/src/config.ts index 3e46031..328476b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,9 +6,11 @@ convict.addFormat(require('convict-format-with-validator').ipaddress) export enum StorageType { Local = 'local', - // S3 = 's3', + Minio = 'minio', + S3 = 's3', // GCS = 'gcs', // Azure = 'azure', + // B2 = 'b2', } export enum URLClean { @@ -53,6 +55,14 @@ const config = convict({ env: 'ADDRESS', }, + // Logging + logLevel: { + doc: 'The level of logging to use.', + format: ['trace', 'debug', 'info', 'warn', 'error', 'fatal'], + default: 'info', + env: 'LOG_LEVEL', + }, + // Security allowedDomains: { doc: 'The domains that are allowed to be used as image sources', @@ -97,6 +107,72 @@ const config = convict({ default: './assets', env: 'LOCAL_ASSETS', }, + + // Minio storage + minio: { + accessKey: { + doc: 'The access key for Minio', + format: String, + default: '', + env: 'MINIO_ACCESS_KEY', + sensitive: true, + }, + secretKey: { + doc: 'The secret key for Minio', + format: String, + default: '', + env: 'MINIO_SECRET_KEY', + sensitive: true, + }, + endpoint: { + doc: 'The endpoint for Minio', + format: String, + default: '', + env: 'MINIO_ENDPOINT', + }, + bucket: { + doc: 'The bucket to use for Minio', + format: String, + default: '', + env: 'MINIO_BUCKET', + }, + region: { + doc: 'The region for Minio', + format: String, + default: '', + env: 'MINIO_REGION', + }, + }, + + // S3 storage + s3: { + bucket: { + doc: 'The S3 bucket to use', + format: String, + default: '', + env: 'S3_BUCKET', + }, + region: { + doc: 'The S3 region to use', + format: String, + default: '', + env: 'S3_REGION', + }, + accessKey: { + doc: 'The S3 access key id to use', + format: String, + default: '', + env: 'S3_ACCESS_KEY_ID', + sensitive: true, + }, + secretKey: { + doc: 'The S3 secret access key to use', + format: String, + default: '', + env: 'S3_SECRET_ACCESS_KEY', + sensitive: true, + }, + }, }) for (const file of ['morphus.yaml', 'morphus.yaml']) { diff --git a/src/controllers/image.ts b/src/controllers/image.ts index 662a019..5ccfcc9 100644 --- a/src/controllers/image.ts +++ b/src/controllers/image.ts @@ -187,9 +187,12 @@ export const image: RouteHandlerMethod = async (request, reply) => { // @ts-ignore reply.expires(new Date(Date.now() + ms(Config.maxAge))) - let stream: NodeJS.ReadableStream = (await storage.exists(q.hash)) - ? await storage.readStream(q.hash) - : await transform(q) + let stream: NodeJS.ReadableStream + try { + stream = await storage.readStream(q.hash) + } catch (err) { + stream = await transform(q) + } reply.code(200).headers({ 'Content-Type': `image/${q.format?.name}`, diff --git a/src/index.ts b/src/index.ts index 5d7c991..86731e0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ import { init as initStorage } from './storage' import { init as initMiddleware } from './fastify/middleware' import { init as initHooks } from './fastify/hooks' -export const App = fastify({ logger: { prettyPrint: true } }) +export const App = fastify({ logger: { prettyPrint: true, level: Config.logLevel } }) process.on('SIGINT', async function () { App.log.info('Stopping server') @@ -19,7 +19,7 @@ async function main() { try { // Internal initConfig(App) - initStorage() + await initStorage(App) // Fastify initMiddleware(App) diff --git a/src/storage/index.ts b/src/storage/index.ts index df0a8e1..128f4e5 100644 --- a/src/storage/index.ts +++ b/src/storage/index.ts @@ -1,30 +1,63 @@ +import { FastifyInstance } from 'fastify' import { Config, StorageType } from '../config' import { Local } from './local' +import { Minio } from './minio' export abstract class Storage { + abstract init(): Promise + abstract read(path: string): Promise abstract write(path: string, data: Buffer): Promise - abstract exists(path: string): Promise - abstract delete(path: string): Promise abstract readStream(path: string): Promise abstract writeStream(path: string): Promise - // list(path: string): Promise - abstract init(): Promise + // list(path: string): Promise + abstract exists(path: string): Promise + abstract delete(path: string): Promise } export let storage: Storage -export async function init() { +export async function init(App: FastifyInstance) { if (!storage) { switch (Config.storage) { case StorageType.Local: storage = new Local(Config.localAssets) break + case StorageType.S3: + // storage = new S3({ + // accessKeyId: Config.s3.accessKey, + // secretAccessKey: Config.s3.secretKey, + // bucket: Config.s3.bucket, + // region: Config.s3.region, + // }) + storage = new Minio({ + accessKey: Config.s3.accessKey, + secretKey: Config.s3.secretKey, + bucket: Config.s3.bucket, + region: Config.s3.region, + endpoint: 'https://s3.amazonaws.com', + }) + break + case StorageType.Minio: + storage = new Minio({ + accessKey: Config.minio.accessKey, + secretKey: Config.minio.secretKey, + endpoint: Config.minio.endpoint, + region: Config.minio.region, + bucket: Config.minio.bucket, + }) + break default: throw new Error(`Unknown storage type: ${Config.storage}`) } - await storage.init() + try { + await storage.init() + App.log.debug(`Storage initialized: ${Config.storage}`) + } catch (e) { + App.log.error(`Storage initialization failed: ${Config.storage}`) + process.exit(1) + } } } diff --git a/src/storage/minio.ts b/src/storage/minio.ts new file mode 100644 index 0000000..df9330b --- /dev/null +++ b/src/storage/minio.ts @@ -0,0 +1,64 @@ +import { Client } from 'minio' +import { PassThrough } from 'stream' + +import { Storage } from '.' +import { StreamUtils } from '../utils/utils' + +export type MinioConfig = { + accessKey: string + secretKey: string + endpoint: string + region?: string + bucket: string +} + +export class Minio implements Storage { + client: Client + + constructor(private options: MinioConfig) { + const url = new URL(this.options.endpoint) + this.client = new Client({ + accessKey: options.accessKey, + secretKey: options.secretKey, + endPoint: url.hostname, + port: parseInt(url.port), + useSSL: url.protocol === 'https:', + }) + } + + async init(): Promise { + await this.client.bucketExists(this.options.bucket) + } + + async read(path: string): Promise { + const stream = await this.client.getObject(this.options.bucket, path) + return StreamUtils.toBuffer(stream) + } + async write(path: string, data: Buffer): Promise { + const stream = await StreamUtils.fromBuffer(data) + await this.client.putObject(this.options.bucket, path, stream) + } + + async readStream(path: string): Promise { + const stream = await this.client.getObject(this.options.bucket, path) + return stream + } + async writeStream(path: string): Promise { + const stream = new PassThrough() + this.client.putObject(this.options.bucket, path, stream) + return stream + } + + async exists(path: string): Promise { + try { + await this.client.statObject(this.options.bucket, path) + return true + } catch { + return false + } + } + + delete(path: string): Promise { + throw new Error('Method not implemented. Delete') + } +} diff --git a/src/transform/index.ts b/src/transform/index.ts index 7ad0ff3..a8b08f8 100644 --- a/src/transform/index.ts +++ b/src/transform/index.ts @@ -6,8 +6,8 @@ import { ComplexParameter, TransformQueryBase } from '../controllers/image' import { storage } from '../storage' import { sha3, splitter } from '../utils/utils' -async function downloadImage(url: string): Promise { - const disk = await storage.writeStream(sha3(url)) +async function downloadAndSaveImage(url: string, path: string): Promise { + const disk = await storage.writeStream(path) return new Promise((resolve) => { get(url, (res) => { const out = new PassThrough() @@ -19,10 +19,11 @@ async function downloadImage(url: string): Promise { export async function getImage(url: string): Promise { const id = sha3(url) - if (!(await storage.exists(id))) { - return await downloadImage(url) + try { + return await storage.readStream(id) + } catch { + return await downloadAndSaveImage(url, id) } - return await storage.readStream(id) } function applyOperation(pipeline: sharp.Sharp, { name, options }: ComplexParameter): sharp.Sharp { diff --git a/src/utils/utils.ts b/src/utils/utils.ts index c6eb5e7..253e3d0 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,7 +1,6 @@ import { createHash } from 'crypto' import { validateSync, ValidatorOptions, ValidationError as VE } from 'class-validator' import { PassThrough, Readable } from 'stream' -import { NullableStringOrRegexpArray } from '../config' export class ValidationError extends Error { override message: string @@ -47,3 +46,21 @@ export function testForPrefixOrRegexp(str: string, values: (string | RegExp)[]): } return false } + +export class StreamUtils { + static fromBuffer(buffer: Buffer) { + const stream = new Readable() + stream.push(buffer) + stream.push(null) + return stream + } + + static toBuffer(stream: NodeJS.ReadableStream) { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + stream.on('data', (chunk) => chunks.push(chunk)) + stream.on('error', reject) + stream.on('end', () => resolve(Buffer.concat(chunks))) + }) + } +}