From b4f9d291293210fc2331bbf09fc40bced41ad8db Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 14 Nov 2022 10:40:28 +0100 Subject: [PATCH] fix: application persistent storage things --- .../migration.sql | 45 ++++++++++++ apps/api/prisma/schema.prisma | 44 ++++++------ apps/api/src/index.ts | 5 +- apps/api/src/jobs/deployApplication.ts | 6 +- apps/api/src/lib.ts | 32 ++++++++- apps/api/src/lib/buildPacks/compose.ts | 26 ++++--- apps/ui/src/lib/locales/en.json | 2 +- .../routes/applications/[id]/_Storage.svelte | 69 ++++++++++--------- .../routes/applications/[id]/storages.svelte | 54 ++++++++++++--- .../src/routes/services/[id]/storages.svelte | 2 +- 10 files changed, 203 insertions(+), 82 deletions(-) create mode 100644 apps/api/prisma/migrations/20221114093217_application_storage_path_migration/migration.sql diff --git a/apps/api/prisma/migrations/20221114093217_application_storage_path_migration/migration.sql b/apps/api/prisma/migrations/20221114093217_application_storage_path_migration/migration.sql new file mode 100644 index 000000000..6913c8b00 --- /dev/null +++ b/apps/api/prisma/migrations/20221114093217_application_storage_path_migration/migration.sql @@ -0,0 +1,45 @@ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_Setting" ( + "id" TEXT NOT NULL PRIMARY KEY, + "fqdn" TEXT, + "isAPIDebuggingEnabled" BOOLEAN DEFAULT false, + "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT false, + "dualCerts" BOOLEAN NOT NULL DEFAULT false, + "minPort" INTEGER NOT NULL DEFAULT 9000, + "maxPort" INTEGER NOT NULL DEFAULT 9100, + "proxyPassword" TEXT NOT NULL, + "proxyUser" TEXT NOT NULL, + "proxyHash" TEXT, + "proxyDefaultRedirect" TEXT, + "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false, + "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true, + "DNSServers" TEXT, + "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + "ipv4" TEXT, + "ipv6" TEXT, + "arch" TEXT, + "concurrentBuilds" INTEGER NOT NULL DEFAULT 1, + "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false +); +INSERT INTO "new_Setting" ("DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt") SELECT "DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt" FROM "Setting"; +DROP TABLE "Setting"; +ALTER TABLE "new_Setting" RENAME TO "Setting"; +CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn"); +CREATE TABLE "new_ApplicationPersistentStorage" ( + "id" TEXT NOT NULL PRIMARY KEY, + "applicationId" TEXT NOT NULL, + "path" TEXT NOT NULL, + "oldPath" BOOLEAN NOT NULL DEFAULT false, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ApplicationPersistentStorage_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "Application" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_ApplicationPersistentStorage" ("applicationId", "createdAt", "id", "path", "updatedAt") SELECT "applicationId", "createdAt", "id", "path", "updatedAt" FROM "ApplicationPersistentStorage"; +DROP TABLE "ApplicationPersistentStorage"; +ALTER TABLE "new_ApplicationPersistentStorage" RENAME TO "ApplicationPersistentStorage"; +CREATE UNIQUE INDEX "ApplicationPersistentStorage_applicationId_path_key" ON "ApplicationPersistentStorage"("applicationId", "path"); +PRAGMA foreign_key_check; +PRAGMA foreign_keys=ON; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index b2de9d23d..49d40694d 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -19,27 +19,28 @@ model Certificate { } model Setting { - id String @id @default(cuid()) - fqdn String? @unique - isAPIDebuggingEnabled Boolean? @default(false) - isRegistrationEnabled Boolean @default(false) - dualCerts Boolean @default(false) - minPort Int @default(9000) - maxPort Int @default(9100) - proxyPassword String - proxyUser String - proxyHash String? - proxyDefaultRedirect String? - isAutoUpdateEnabled Boolean @default(false) - isDNSCheckEnabled Boolean @default(true) - DNSServers String? - isTraefikUsed Boolean @default(true) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - ipv4 String? - ipv6 String? - arch String? - concurrentBuilds Int @default(1) + id String @id @default(cuid()) + fqdn String? @unique + isAPIDebuggingEnabled Boolean? @default(false) + isRegistrationEnabled Boolean @default(false) + dualCerts Boolean @default(false) + minPort Int @default(9000) + maxPort Int @default(9100) + proxyPassword String + proxyUser String + proxyHash String? + proxyDefaultRedirect String? + isAutoUpdateEnabled Boolean @default(false) + isDNSCheckEnabled Boolean @default(true) + DNSServers String? + isTraefikUsed Boolean @default(true) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + ipv4 String? + ipv6 String? + arch String? + concurrentBuilds Int @default(1) + applicationStoragePathMigrationFinished Boolean @default(false) } model User { @@ -186,6 +187,7 @@ model ApplicationPersistentStorage { id String @id @default(cuid()) applicationId String path String + oldPath Boolean @default(false) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt application Application @relation(fields: [applicationId], references: [id]) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index b9d0fb884..ab87297e3 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -17,7 +17,7 @@ import yaml from 'js-yaml' import fs from 'fs/promises'; import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers'; import { checkContainer } from './lib/docker'; -import { migrateServicesToNewTemplate } from './lib'; +import { migrateApplicationPersistentStorage, migrateServicesToNewTemplate } from './lib'; import { refreshTags, refreshTemplates } from './routes/api/v1/handlers'; declare module 'fastify' { @@ -142,7 +142,8 @@ const host = '0.0.0.0'; await socketIOServer(fastify) console.log(`Coolify's API is listening on ${host}:${port}`); - migrateServicesToNewTemplate() + migrateServicesToNewTemplate(); + await migrateApplicationPersistentStorage(); await initServer(); const graceful = new Graceful({ brees: [scheduler] }); diff --git a/apps/api/src/jobs/deployApplication.ts b/apps/api/src/jobs/deployApplication.ts index 42e1bcef8..edc5c4864 100644 --- a/apps/api/src/jobs/deployApplication.ts +++ b/apps/api/src/jobs/deployApplication.ts @@ -117,8 +117,10 @@ import * as buildpacks from '../lib/buildPacks'; let domain = getDomain(fqdn); const volumes = persistentStorage?.map((storage) => { - return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : '' - }${storage.path}`; + if (storage.oldPath) { + return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app','')}:${storage.path}`; + } + return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`; }) || []; // Previews, we need to get the source branch and set subdomain if (pullmergeRequestId) { diff --git a/apps/api/src/lib.ts b/apps/api/src/lib.ts index 639de27e5..d8fc9ced7 100644 --- a/apps/api/src/lib.ts +++ b/apps/api/src/lib.ts @@ -2,6 +2,32 @@ import cuid from "cuid"; import { decrypt, encrypt, fixType, generatePassword, prisma } from "./lib/common"; import { getTemplates } from "./lib/services"; +export async function migrateApplicationPersistentStorage() { + const settings = await prisma.setting.findFirst() + if (settings) { + const { id: settingsId, applicationStoragePathMigrationFinished } = settings + try { + if (!applicationStoragePathMigrationFinished) { + const applications = await prisma.application.findMany({ include: { persistentStorage: true } }); + for (const application of applications) { + if (application.persistentStorage && application.persistentStorage.length > 0 && application?.buildPack !== 'docker') { + for (const storage of application.persistentStorage) { + let { id, path } = storage + if (!path.startsWith('/app')) { + path = `/app${path}` + await prisma.applicationPersistentStorage.update({ where: { id }, data: { path, oldPath: true } }) + } + } + } + } + } + } catch (error) { + console.log(error) + } finally { + await prisma.setting.update({ where: { id: settingsId }, data: { applicationStoragePathMigrationFinished: true } }) + } + } +} export async function migrateServicesToNewTemplate() { // This function migrates old hardcoded services to the new template based services try { @@ -456,9 +482,9 @@ async function migrateSettings(settings: any[], service: any, template: any) { variableName = `$$config_${name.toLowerCase()}` } // console.log('Migrating setting', name, value, 'for service', service.id, ', service name:', service.name, 'variableName: ', variableName) - + await prisma.serviceSetting.findFirst({ where: { name: minio, serviceId: service.id } }) || await prisma.serviceSetting.create({ data: { name: minio, value, variableName, service: { connect: { id: service.id } } } }) - } catch(error) { + } catch (error) { console.log(error) } } @@ -473,7 +499,7 @@ async function migrateSecrets(secrets: any[], service: any) { } // console.log('Migrating secret', name, value, 'for service', service.id, ', service name:', service.name) await prisma.serviceSecret.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSecret.create({ data: { name, value, service: { connect: { id: service.id } } } }) - } catch(error) { + } catch (error) { console.log(error) } } diff --git a/apps/api/src/lib/buildPacks/compose.ts b/apps/api/src/lib/buildPacks/compose.ts index acd19205d..a87161849 100644 --- a/apps/api/src/lib/buildPacks/compose.ts +++ b/apps/api/src/lib/buildPacks/compose.ts @@ -64,13 +64,14 @@ export default async function (data) { } catch (error) { // } - const composeVolumes = volumes.map((volume) => { - return { - [`${volume.split(':')[0]}`]: { - name: volume.split(':')[0] - } - }; - }); + const composeVolumes = []; + for (const volume of volumes) { + let [v, _] = volume.split(':'); + composeVolumes[v] = { + name: v, + } + + } let networks = {} for (let [key, value] of Object.entries(dockerComposeYaml.services)) { value['container_name'] = `${applicationId}-${key}` @@ -79,16 +80,19 @@ export default async function (data) { // TODO: If we support separated volume for each service, we need to add it here if (value['volumes'].length > 0) { value['volumes'] = value['volumes'].map((volume) => { - let [v, path] = volume.split(':'); + let [v, path, permission] = volume.split(':'); v = `${applicationId}-${v}` composeVolumes[v] = { name: v } - return `${v}:${path}` + return `${v}:${path}${permission ? ':' + permission : ''}` }) } + if (volumes.length > 0) { - value['volumes'] = [...value['volumes'] || '', volumes] + for (const volume of volumes) { + value['volumes'].push(volume) + } } if (dockerComposeConfiguration[key].port) { value['expose'] = [dockerComposeConfiguration[key].port] @@ -104,7 +108,7 @@ export default async function (data) { dockerComposeYaml.services[key] = { ...dockerComposeYaml.services[key], restart: defaultComposeConfiguration(network).restart, deploy: defaultComposeConfiguration(network).deploy } } if (Object.keys(composeVolumes).length > 0) { - dockerComposeYaml['volumes'] = {...composeVolumes} + dockerComposeYaml['volumes'] = { ...composeVolumes } } dockerComposeYaml['networks'] = Object.assign({ ...networks }, { [network]: { external: true } }) await fs.writeFile(`${workdir}/docker-compose.${isYml ? 'yml' : 'yaml'}`, yaml.dump(dockerComposeYaml)); diff --git a/apps/ui/src/lib/locales/en.json b/apps/ui/src/lib/locales/en.json index 709ba4227..f04ae6d8e 100644 --- a/apps/ui/src/lib/locales/en.json +++ b/apps/ui/src/lib/locales/en.json @@ -159,7 +159,7 @@ "storage_saved": "Storage saved.", "storage_updated": "Storage updated.", "storage_deleted": "Storage deleted.", - "persistent_storage_explainer": "You can specify any folder that you want to be persistent across deployments.
/example means it will preserve /app/example in the container as /app is the root directory for your application.

This is useful for storing data such as a database (SQLite) or a cache." + "persistent_storage_explainer": "You can specify any folder that you want to be persistent across deployments.

/example means it will preserve /example between deployments.

Your application's data is copied to /app inside the container, you can preserve data under it as well, like /app/db.

This is useful for storing data such as a database (SQLite) or a cache." }, "deployment_queued": "Deployment queued.", "confirm_to_delete": "Are you sure you would like to delete '{{name}}'?", diff --git a/apps/ui/src/routes/applications/[id]/_Storage.svelte b/apps/ui/src/routes/applications/[id]/_Storage.svelte index cf54e16df..6e91c7d8f 100644 --- a/apps/ui/src/routes/applications/[id]/_Storage.svelte +++ b/apps/ui/src/routes/applications/[id]/_Storage.svelte @@ -60,49 +60,54 @@
-
- {#if storage.id} -
- - -
- {/if} -
- + {#if storage.predefined} +
+ + +
+ {:else} +
+ {#if storage.applicationId} + {#if storage.oldPath} + + {:else} + + {/if} + {/if} -
-
- {#if isNew} -
- -
- {:else} -
-
- + {#if isNew} +
+
+ {:else}
-
- {/if} + {/if} +
-
+ {/if}
diff --git a/apps/ui/src/routes/applications/[id]/storages.svelte b/apps/ui/src/routes/applications/[id]/storages.svelte index 06e3c87d2..54e81b0c6 100644 --- a/apps/ui/src/routes/applications/[id]/storages.svelte +++ b/apps/ui/src/routes/applications/[id]/storages.svelte @@ -2,9 +2,11 @@ import type { Load } from '@sveltejs/kit'; export const load: Load = async ({ params, stuff, url }) => { try { + const { application } = stuff; const response = await get(`/applications/${params.id}/storages`); return { props: { + application, ...response } }; @@ -19,12 +21,26 @@