From 1d6907e4d2ed069d30c87a0da02aad002f16d299 Mon Sep 17 00:00:00 2001 From: seaznCode Date: Mon, 29 Jun 2026 23:15:16 +0200 Subject: [PATCH] feature: download exoscale storage --- controller/dev/DevManagementController.js | 22 ++ package-lock.json | 314 +++++++++++++++++++++- package.json | 1 + routes/getRoutes.js | 1 + services/dev/DevManagementService.js | 80 +++++- 5 files changed, 410 insertions(+), 8 deletions(-) diff --git a/controller/dev/DevManagementController.js b/controller/dev/DevManagementController.js index 96810eb..716670d 100644 --- a/controller/dev/DevManagementController.js +++ b/controller/dev/DevManagementController.js @@ -84,3 +84,25 @@ exports.listGhostDirectories = async (_req, res) => { return res.status(500).json({ success: false, error: e?.message || 'Failed to list ghost directories' }); } }; + +exports.downloadStorageArchive = async (_req, res) => { + const now = new Date(); + const pad = (value) => String(value).padStart(2, '0'); + const fileName = `exoscale-storage-${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}.zip`; + + try { + res.setHeader('Content-Type', 'application/zip'); + res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`); + + const meta = await DevManagementService.streamStorageArchive(res); + logger.info('[DevManagementController.downloadStorageArchive] success', meta); + } catch (e) { + logger.error('[DevManagementController.downloadStorageArchive] error', { msg: e?.message }); + if (!res.headersSent) { + return res.status(500).json({ success: false, error: e?.message || 'Failed to export Exoscale storage' }); + } + if (!res.writableEnded) { + res.destroy(e); + } + } +}; diff --git a/package-lock.json b/package-lock.json index ceb6f91..406732b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@aws-sdk/client-s3": "^3.992.0", "@aws-sdk/s3-request-presigner": "^3.992.0", "@getbrevo/brevo": "^3.0.1", + "archiver": "^8.0.0", "argon2": "^0.44.0", "cookie-parser": "^1.4.7", "cors": "^2.8.6", @@ -1894,6 +1895,18 @@ "@types/node": "*" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -1978,6 +1991,51 @@ "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", "license": "MIT" }, + "node_modules/archiver": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-8.0.0.tgz", + "integrity": "sha512-fV1orZfsnPn9BaSByR/qE67rJCLJEy2Ox5bq7nJh+jquWaNh6Sfec75kJ2T6PtdGUbPQlrVoSVCEOa5SdiTQ1g==", + "license": "MIT", + "dependencies": { + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "is-stream": "^4.0.0", + "lazystream": "^1.0.0", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^3.0.0", + "tar-stream": "^3.0.0", + "zip-stream": "^7.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/archiver/node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/argon2": { "version": "0.44.0", "resolved": "https://registry.npmjs.org/argon2/-/argon2-0.44.0.tgz", @@ -2045,7 +2103,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.3.tgz", "integrity": "sha512-1pHv8LX9CpKut1Zp4EXey7Z8OfH11ONNH6Dhi2WDUt31VVZFXZzKwXcysBgqSumFCmR+0dqjMK5v5JiFHzi0+g==", - "dev": true, "license": "MIT", "engines": { "node": "20 || >=22" @@ -2214,7 +2271,6 @@ "version": "5.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz", "integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -2245,6 +2301,30 @@ "base64-js": "^1.1.2" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -2431,6 +2511,38 @@ "node": ">=18" } }, + "node_modules/compress-commons": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-7.0.1.tgz", + "integrity": "sha512-g0S8KAD8qf4+V//pr3BfB1aBnARLXNz2Gx+jmHU0LEriUuoQUOPOulVquHKTJ8+EAIIO7fhseNDr9wK5Q9FKBQ==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^7.0.1", + "is-stream": "^4.0.0", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/concat-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", @@ -2496,6 +2608,12 @@ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", "license": "MIT" }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/cors": { "version": "2.8.6", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", @@ -2539,6 +2657,47 @@ } } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-7.0.1.tgz", + "integrity": "sha512-IBWsY8xznyQrcHn8h4bC8/4ErNke5elzgG8GcqF4RFPw6aHkWWRc7Tgw6upjaTX/CT/yQgqYENkxYsTYN+hW2g==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/cross-env": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-10.1.0.tgz", @@ -2644,8 +2803,7 @@ "version": "0.0.1581282", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1581282.tgz", "integrity": "sha512-nv7iKtNZQshSW2hKzYNr46nM/Cfh5SEvE2oV0/SEGgc9XupIY5ggf84Cz8eJIkBce7S3bmTAauFD6aysMpnqsQ==", - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/dfa": { "version": "1.2.0", @@ -2848,6 +3006,24 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/events-universal": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", @@ -3311,6 +3487,26 @@ "url": "https://opencollective.com/express" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore-by-default": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", @@ -3443,6 +3639,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -3529,6 +3731,48 @@ "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", "license": "MIT" }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/linebreak": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/linebreak/-/linebreak-1.1.0.tgz", @@ -3702,7 +3946,6 @@ "version": "10.2.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.2" @@ -3917,7 +4160,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4149,6 +4391,21 @@ "resolved": "https://registry.npmjs.org/png-js/-/png-js-1.0.0.tgz", "integrity": "sha512-k+YsbhpA9e+EFfKjTCH3VW6aoKlyNYI6NYdTfDL4CIvFnvsuO84ttonmZE7rc+v23SLTH8XX+5w/Ak9v0xGY4g==" }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -4305,6 +4562,21 @@ "node": ">= 6" } }, + "node_modules/readdir-glob": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-3.0.0.tgz", + "integrity": "sha512-AhNB2KgKeVJr16nK9LLZbJNWnYoT23ZrumNKFDebHBdkC8KHSqWo871JAUhoWC/RtjEVdqNMFpM6qrwRbaUqpw==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^10.2.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/yqnn" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -5102,6 +5374,36 @@ "node": ">=12" } }, + "node_modules/zip-stream": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-7.0.5.tgz", + "integrity": "sha512-dSvYKdvLsAHCDqPOhIwk/q5CvuWtTB3Dgpoe0uVEFjTzIOAmsQpprX25InCvrvJsirEbu1OHyy67n/kAj1Sw/w==", + "license": "MIT", + "dependencies": { + "compress-commons": "^7.0.0", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index f12c600..59de1f6 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@aws-sdk/client-s3": "^3.992.0", "@aws-sdk/s3-request-presigner": "^3.992.0", "@getbrevo/brevo": "^3.0.1", + "archiver": "^8.0.0", "argon2": "^0.44.0", "cookie-parser": "^1.4.7", "cors": "^2.8.6", diff --git a/routes/getRoutes.js b/routes/getRoutes.js index 8812d5d..e479161 100644 --- a/routes/getRoutes.js +++ b/routes/getRoutes.js @@ -78,6 +78,7 @@ router.get('/admin/server-status', authMiddleware, adminOnly, ServerStatusContro router.get('/admin/dev/exoscale/folder-structure-issues', authMiddleware, adminOnly, DevManagementController.listFolderStructureIssues); router.get('/admin/dev/exoscale/loose-files', authMiddleware, adminOnly, DevManagementController.listLooseFiles); router.get('/admin/dev/exoscale/ghost-directories', authMiddleware, adminOnly, DevManagementController.listGhostDirectories); +router.get('/admin/dev/exoscale/download-archive', authMiddleware, adminOnly, DevManagementController.downloadStorageArchive); // Admin: dashboard platforms router.get('/admin/dashboard-platforms', authMiddleware, adminOnly, DashboardPlatformsController.list); diff --git a/services/dev/DevManagementService.js b/services/dev/DevManagementService.js index 67c8a57..ce9b32b 100644 --- a/services/dev/DevManagementService.js +++ b/services/dev/DevManagementService.js @@ -1,9 +1,11 @@ const db = require('../../database/database'); const AdminRepository = require('../../repositories/admin/AdminRepository'); const { s3 } = require('../../utils/exoscaleUploader'); -const { ListObjectsV2Command, CopyObjectCommand, DeleteObjectCommand, PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); +const { ListObjectsV2Command, CopyObjectCommand, DeleteObjectCommand, PutObjectCommand, HeadObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const { logger } = require('../../middleware/logger'); const path = require('path'); +const { Readable } = require('stream'); +const { ZipArchive } = require('archiver'); async function executeDump(sql) { const conn = await db.getMultiStatementConnection(); @@ -52,6 +54,15 @@ async function listTopLevelFolders(prefix) { return folders; } +function toReadable(body) { + if (!body) return null; + if (typeof body.pipe === 'function') return body; + if (Buffer.isBuffer(body) || body instanceof Uint8Array) { + return Readable.from(body); + } + return null; +} + function getTimestamp() { const d = new Date(); const pad = (n) => String(n).padStart(2, '0'); @@ -338,11 +349,76 @@ async function listGhostDirectories() { }; } +async function streamStorageArchive(outputStream) { + const archive = new ZipArchive({ zlib: { level: 9 } }); + + archive.on('warning', (error) => { + logger.warn('DevManagementService.streamStorageArchive:warning', { error: error?.message || String(error) }); + }); + + archive.on('error', (error) => { + logger.error('DevManagementService.streamStorageArchive:archive_error', { error: error?.message || String(error) }); + outputStream.destroy(error); + }); + + archive.pipe(outputStream); + + let totalFiles = 0; + let totalBytes = 0; + let token = undefined; + + do { + const page = await s3.send(new ListObjectsV2Command({ + Bucket: process.env.EXOSCALE_BUCKET, + ContinuationToken: token + })); + + const objects = (page && page.Contents ? page.Contents : []) + .filter(item => item && item.Key) + .filter(item => !item.Key.endsWith('/')); + + for (const object of objects) { + const key = object.Key; + const response = await s3.send(new GetObjectCommand({ + Bucket: process.env.EXOSCALE_BUCKET, + Key: key + })); + const bodyStream = toReadable(response && response.Body); + + if (!bodyStream) { + throw new Error(`Object body for ${key} is not readable`); + } + + await new Promise((resolve, reject) => { + bodyStream.once('error', reject); + bodyStream.once('end', resolve); + archive.append(bodyStream, { + name: key, + date: object.LastModified || new Date() + }); + }); + + totalFiles += 1; + totalBytes += Number(object.Size || 0); + } + + token = page && page.IsTruncated ? page.NextContinuationToken : undefined; + } while (token); + + await archive.finalize(); + + return { + totalFiles, + totalBytes + }; +} + module.exports = { executeDump, listFolderStructureIssues, createFolderStructure, listLooseFiles, moveLooseFilesToContract, - listGhostDirectories + listGhostDirectories, + streamStorageArchive }; -- 2.39.5