fix: application persistent storage things

apps/api/prisma/migrations/20221114093217_application_storage_path_migration/migration.sql

@@ -0,0 +1,45 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Setting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fqdn" TEXT,
+    "isAPIDebuggingEnabled" BOOLEAN DEFAULT false,
+    "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "minPort" INTEGER NOT NULL DEFAULT 9000,
+    "maxPort" INTEGER NOT NULL DEFAULT 9100,
+    "proxyPassword" TEXT NOT NULL,
+    "proxyUser" TEXT NOT NULL,
+    "proxyHash" TEXT,
+    "proxyDefaultRedirect" TEXT,
+    "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "DNSServers" TEXT,
+    "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "ipv4" TEXT,
+    "ipv6" TEXT,
+    "arch" TEXT,
+    "concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
+    "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false
+);
+INSERT INTO "new_Setting" ("DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt") SELECT "DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt" FROM "Setting";
+DROP TABLE "Setting";
+ALTER TABLE "new_Setting" RENAME TO "Setting";
+CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
+CREATE TABLE "new_ApplicationPersistentStorage" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "applicationId" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "oldPath" BOOLEAN NOT NULL DEFAULT false,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "ApplicationPersistentStorage_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "Application" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_ApplicationPersistentStorage" ("applicationId", "createdAt", "id", "path", "updatedAt") SELECT "applicationId", "createdAt", "id", "path", "updatedAt" FROM "ApplicationPersistentStorage";
+DROP TABLE "ApplicationPersistentStorage";
+ALTER TABLE "new_ApplicationPersistentStorage" RENAME TO "ApplicationPersistentStorage";
+CREATE UNIQUE INDEX "ApplicationPersistentStorage_applicationId_path_key" ON "ApplicationPersistentStorage"("applicationId", "path");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/schema.prisma

@@ -40,6 +40,7 @@ model Setting {
   ipv6                                    String?
   arch                                    String?
   concurrentBuilds                        Int      @default(1)
+  applicationStoragePathMigrationFinished Boolean  @default(false)
 }
 
 model User {
@@ -186,6 +187,7 @@ model ApplicationPersistentStorage {
   id            String      @id @default(cuid())
   applicationId String
   path          String
+  oldPath       Boolean     @default(false)
   createdAt     DateTime    @default(now())
   updatedAt     DateTime    @updatedAt
   application   Application @relation(fields: [applicationId], references: [id])

apps/api/src/index.ts

@@ -17,7 +17,7 @@ import yaml from 'js-yaml'
 import fs from 'fs/promises';
 import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers';
 import { checkContainer } from './lib/docker';
-import { migrateServicesToNewTemplate } from './lib';
+import { migrateApplicationPersistentStorage, migrateServicesToNewTemplate } from './lib';
 import { refreshTags, refreshTemplates } from './routes/api/v1/handlers';
 
 declare module 'fastify' {
@@ -142,7 +142,8 @@ const host = '0.0.0.0';
 		await socketIOServer(fastify)
 		console.log(`Coolify's API is listening on ${host}:${port}`);
 
-		migrateServicesToNewTemplate()
+		migrateServicesToNewTemplate();
+		await migrateApplicationPersistentStorage();
 		await initServer();
 
 		const graceful = new Graceful({ brees: [scheduler] });

apps/api/src/jobs/deployApplication.ts

@@ -117,8 +117,10 @@ import * as buildpacks from '../lib/buildPacks';
 								let domain = getDomain(fqdn);
 								const volumes =
 									persistentStorage?.map((storage) => {
-										return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : ''
+										if (storage.oldPath) {
-											}${storage.path}`;
+											return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app','')}:${storage.path}`;
+										}
+										return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
 									}) || [];
 								// Previews, we need to get the source branch and set subdomain
 								if (pullmergeRequestId) {

apps/api/src/lib.ts

@@ -2,6 +2,32 @@ import cuid from "cuid";
 import { decrypt, encrypt, fixType, generatePassword, prisma } from "./lib/common";
 import { getTemplates } from "./lib/services";
 
+export async function migrateApplicationPersistentStorage() {
+    const settings = await prisma.setting.findFirst()
+    if (settings) {
+        const { id: settingsId, applicationStoragePathMigrationFinished } = settings
+        try {
+            if (!applicationStoragePathMigrationFinished) {
+                const applications = await prisma.application.findMany({ include: { persistentStorage: true } });
+                for (const application of applications) {
+                    if (application.persistentStorage && application.persistentStorage.length > 0 && application?.buildPack !== 'docker') {
+                        for (const storage of application.persistentStorage) {
+                            let { id, path } = storage
+                            if (!path.startsWith('/app')) {
+                                path = `/app${path}`
+                                await prisma.applicationPersistentStorage.update({ where: { id }, data: { path, oldPath: true } })
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (error) {
+            console.log(error)
+        } finally {
+            await prisma.setting.update({ where: { id: settingsId }, data: { applicationStoragePathMigrationFinished: true } })
+        }
+    }
+}
 export async function migrateServicesToNewTemplate() {
     // This function migrates old hardcoded services to the new template based services
     try {

apps/api/src/lib/buildPacks/compose.ts

@@ -64,13 +64,14 @@ export default async function (data) {
     } catch (error) {
         //
     }
-    const composeVolumes = volumes.map((volume) => {
+    const composeVolumes = [];
-        return {
+    for (const volume of volumes) {
-            [`${volume.split(':')[0]}`]: {
+        let [v, _] = volume.split(':');
-                name: volume.split(':')[0]
+        composeVolumes[v] = {
+            name: v,
+        }
+     
     }
-        };
-    });
     let networks = {}
     for (let [key, value] of Object.entries(dockerComposeYaml.services)) {
         value['container_name'] = `${applicationId}-${key}`
@@ -79,16 +80,19 @@ export default async function (data) {
         // TODO: If we support separated volume for each service, we need to add it here
         if (value['volumes'].length > 0) {
             value['volumes'] = value['volumes'].map((volume) => {
-                let [v, path] = volume.split(':');
+                let [v, path, permission] = volume.split(':');
                 v = `${applicationId}-${v}`
                 composeVolumes[v] = {
                     name: v
                 }
-                return `${v}:${path}`
+                return `${v}:${path}${permission ? ':' + permission : ''}`
             })
         }
+
         if (volumes.length > 0) {
-            value['volumes'] = [...value['volumes'] || '', volumes]
+            for (const volume of volumes) {
+                value['volumes'].push(volume)
+            }
         }
         if (dockerComposeConfiguration[key].port) {
             value['expose'] = [dockerComposeConfiguration[key].port]

apps/ui/src/lib/locales/en.json

@@ -159,7 +159,7 @@
 			"storage_saved": "Storage saved.",
 			"storage_updated": "Storage updated.",
 			"storage_deleted": "Storage deleted.",
-			"persistent_storage_explainer": "You can specify any folder that you want to be persistent across deployments.<br><span class='text-settings '>/example</span> means it will preserve <span class='text-settings '>/app/example</span> in the container as <span class='text-settings '>/app</span> is <span class='text-settings '>the root directory</span> for your application.<br><br>This is useful for storing data such as a <span class='text-settings '>database (SQLite)</span> or a <span class='text-settings '>cache</span>."
+			"persistent_storage_explainer": "You can specify any folder that you want to be persistent across deployments.<br><br><span class='text-settings '>/example</span> means it will preserve <span class='text-settings '>/example</span> between deployments.<br><br>Your application's data is copied to <span class='text-settings '>/app</span> inside the container, you can preserve data under it as well, like <span class='text-settings '>/app/db</span>.<br><br>This is useful for storing data such as a <span class='text-settings '>database (SQLite)</span> or a <span class='text-settings '>cache</span>."
 		},
 		"deployment_queued": "Deployment queued.",
 		"confirm_to_delete": "Are you sure you would like to delete '{{name}}'?",

apps/ui/src/routes/applications/[id]/_Storage.svelte

@@ -60,49 +60,54 @@
 </script>
 
 <div class="w-full lg:px-0 px-4">
-	<div class="grid grid-col-1 lg:grid-cols-3 lg:space-x-4" class:pt-8={isNew}>
+	{#if storage.predefined}
-		{#if storage.id}
+		<div class="flex flex-col lg:flex-row gap-4 pb-2">
-			<div class="flex flex-col">
+			<input disabled readonly class="w-full" value={storage.id} />
-				<label for="name" class="pb-2 uppercase font-bold">Volume name</label>
+			<input disabled readonly class="w-full" bind:value={storage.path} />
+		</div>
+	{:else}
+		<div class="flex gap-4 pb-2" class:pt-8={isNew}>
+			{#if storage.applicationId}
+				{#if storage.oldPath}
 					<input
 						disabled
 						readonly
-					class="w-full lg:w-64"
+						class="w-full"
-					value="{storage.id}{storage.path.replace(/\//gi, '-')}"
+						value="{storage.applicationId}{storage.path.replace(/\//gi, '-').replace('-app', '')}"
 					/>
-			</div>
+				{:else}
-		{/if}
-		<div class="flex flex-col">
-			<label for="name" class="pb-2 uppercase font-bold">{isNew ? 'New Path' : 'Path'}</label>
 					<input
-				class="w-full lg:w-64"
+						disabled
+						readonly
+						class="w-full"
+						value="{storage.applicationId}{storage.path.replace(/\//gi, '-')}"
+					/>
+				{/if}
+			{/if}
+			<input
+				disabled={!isNew}
+				readonly={!isNew}
+				class="w-full"
 				bind:value={storage.path}
 				required
-				placeholder="eg: /sqlite.db"
+				placeholder="eg: /data"
 			/>
-		</div>
 
-		<div class="pt-8">
+			<div class="flex items-center justify-center">
 				{#if isNew}
-				<div class="flex items-center justify-center w-full lg:w-64">
+					<div class="w-full lg:w-64">
 						<button class="btn btn-sm btn-primary w-full" on:click={() => saveStorage(true)}
 							>{$t('forms.add')}</button
 						>
 					</div>
 				{:else}
-				<div class="flex flex-row items-center justify-center space-x-2 w-full lg:w-64">
-					<div class="flex items-center justify-center">
-						<button class="btn btn-sm btn-primary" on:click={() => saveStorage(false)}
-							>{$t('forms.set')}</button
-						>
-					</div>
 					<div class="flex justify-center">
 						<button class="btn btn-sm btn-error" on:click={removeStorage}
 							>{$t('forms.remove')}</button
 						>
 					</div>
-				</div>
 				{/if}
 			</div>
 		</div>
+	{/if}
 </div>

apps/ui/src/routes/applications/[id]/storages.svelte

@@ -2,9 +2,11 @@
 	import type { Load } from '@sveltejs/kit';
 	export const load: Load = async ({ params, stuff, url }) => {
 		try {
+			const { application } = stuff;
 			const response = await get(`/applications/${params.id}/storages`);
 			return {
 				props: {
+					application,
 					...response
 				}
 			};
@@ -19,12 +21,26 @@
 
 <script lang="ts">
 	export let persistentStorages: any;
+	export let application: any;
 	import { page } from '$app/stores';
 	import Storage from './_Storage.svelte';
 	import { get } from '$lib/api';
 	import { t } from '$lib/translations';
 	import Explainer from '$lib/components/Explainer.svelte';
 	
+	let composeJson = JSON.parse(application?.dockerComposeFile || '{}');
+	let predefinedVolumes: any[] = [];
+	if (composeJson?.services) {
+		for (const [_, service] of Object.entries(composeJson.services)) {
+			if (service?.volumes) {
+				for (const [_, volumeName] of Object.entries(service.volumes)) {
+					let [volume, target] = volumeName.split(':');
+					volume = `${application.id}-${volume}`;
+					predefinedVolumes.push({ id: volume, path: target, predefined: true });
+				}
+			}
+		}
+	}
 	const { id } = $page.params;
 	async function refreshStorage() {
 		const data = await get(`/applications/${id}/storages`);
@@ -35,19 +51,39 @@
 <div class="w-full">
 	<div class="mx-auto w-full">
 		<div class="flex flex-row border-b border-coolgray-500 mb-6 space-x-2">
-			<div class="title font-bold pb-3">
+			<div class="title font-bold pb-3">Persistent Volumes</div>
-				Persistent Volumes <Explainer
+		</div>
-					position="dropdown-bottom"
+		{#if predefinedVolumes.length > 0}
-					explanation={$t('application.storage.persistent_storage_explainer')}
+			<div class="title">Predefined Volumes</div>
-				/>
+			<div class="w-full lg:px-0 px-4">
+				<div class="grid grid-col-1 lg:grid-cols-2 py-2 gap-2">
+					<div class="font-bold uppercase">Volume Id</div>
+					<div class="font-bold uppercase">Mount Dir</div>
 				</div>
 			</div>
 
+			<div class="gap-4">
+				{#each predefinedVolumes as storage}
+					{#key storage.id}
+						<Storage on:refresh={refreshStorage} {storage} />
+					{/key}
+				{/each}
+			</div>
+		{/if}
+		{#if persistentStorages.length > 0}
+			<div class="title pt-10">Custom Volumes</div>
+		{/if}
 		{#each persistentStorages as storage}
 			{#key storage.id}
 				<Storage on:refresh={refreshStorage} {storage} />
 			{/key}
 		{/each}
+		<div class="title pt-10">
+			Add New Volume <Explainer
+				position="dropdown-bottom"
+				explanation={$t('application.storage.persistent_storage_explainer')}
+			/>
+		</div>
 		<Storage on:refresh={refreshStorage} isNew />
 	</div>
 </div>