fix: cleanupstorage

This commit is contained in:
Andras Bacsai 2022-12-26 21:17:53 +01:00
parent d25a9d7515
commit 4680b63911
4 changed files with 201 additions and 145 deletions

View File

@ -6,14 +6,27 @@ import cookie from '@fastify/cookie';
import multipart from '@fastify/multipart';
import path, { join } from 'path';
import autoLoad from '@fastify/autoload';
import socketIO from 'fastify-socket.io'
import socketIOServer from './realtime'
import socketIO from 'fastify-socket.io';
import socketIOServer from './realtime';
import { cleanupDockerStorage, createRemoteEngineConfiguration, decrypt, executeCommand, generateDatabaseConfiguration, isDev, listSettings, prisma, sentryDSN, startTraefikProxy, startTraefikTCPProxy, version } from './lib/common';
import {
cleanupDockerStorage,
createRemoteEngineConfiguration,
decrypt,
executeCommand,
generateDatabaseConfiguration,
isDev,
listSettings,
prisma,
sentryDSN,
startTraefikProxy,
startTraefikTCPProxy,
version
} from './lib/common';
import { scheduler } from './lib/scheduler';
import { compareVersions } from 'compare-versions';
import Graceful from '@ladjs/graceful'
import yaml from 'js-yaml'
import Graceful from '@ladjs/graceful';
import yaml from 'js-yaml';
import fs from 'fs/promises';
import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers';
import { checkContainer } from './lib/docker';
@ -23,13 +36,13 @@ import * as Sentry from '@sentry/node';
declare module 'fastify' {
interface FastifyInstance {
config: {
COOLIFY_APP_ID: string,
COOLIFY_SECRET_KEY: string,
COOLIFY_DATABASE_URL: string,
COOLIFY_IS_ON: string,
COOLIFY_WHITE_LABELED: string,
COOLIFY_WHITE_LABELED_ICON: string | null,
COOLIFY_AUTO_UPDATE: string,
COOLIFY_APP_ID: string;
COOLIFY_SECRET_KEY: string;
COOLIFY_DATABASE_URL: string;
COOLIFY_IS_ON: string;
COOLIFY_WHITE_LABELED: string;
COOLIFY_WHITE_LABELED_ICON: string | null;
COOLIFY_AUTO_UPDATE: string;
};
}
}
@ -38,7 +51,7 @@ const port = isDev ? 3001 : 3000;
const host = '0.0.0.0';
(async () => {
const settings = await prisma.setting.findFirst()
const settings = await prisma.setting.findFirst();
const fastify = Fastify({
logger: settings?.isAPIDebuggingEnabled || false,
trustProxy: true
@ -49,10 +62,10 @@ const host = '0.0.0.0';
required: ['COOLIFY_SECRET_KEY', 'COOLIFY_DATABASE_URL', 'COOLIFY_IS_ON'],
properties: {
COOLIFY_APP_ID: {
type: 'string',
type: 'string'
},
COOLIFY_SECRET_KEY: {
type: 'string',
type: 'string'
},
COOLIFY_DATABASE_URL: {
type: 'string',
@ -73,8 +86,7 @@ const host = '0.0.0.0';
COOLIFY_AUTO_UPDATE: {
type: 'string',
default: 'false'
},
}
}
};
const options = {
@ -103,13 +115,13 @@ const host = '0.0.0.0';
fastify.register(autoLoad, {
dir: join(__dirname, 'routes')
});
fastify.register(cookie)
fastify.register(cookie);
fastify.register(cors);
fastify.register(socketIO, {
cors: {
origin: isDev ? "*" : ''
origin: isDev ? '*' : ''
}
})
});
// To detect allowed origins
// fastify.addHook('onRequest', async (request, reply) => {
// console.log(request.headers.host)
@ -131,10 +143,9 @@ const host = '0.0.0.0';
// }
// })
try {
await fastify.listen({ port, host })
await socketIOServer(fastify)
await fastify.listen({ port, host });
await socketIOServer(fastify);
console.log(`Coolify's API is listening on ${host}:${port}`);
migrateServicesToNewTemplate();
@ -148,105 +159,100 @@ const host = '0.0.0.0';
if (!scheduler.workers.has('deployApplication')) {
scheduler.run('deployApplication');
}
}, 2000)
}, 2000);
// autoUpdater
setInterval(async () => {
await autoUpdater()
}, 60000 * 15)
await autoUpdater();
}, 60000 * 15);
// cleanupStorage
setInterval(async () => {
await cleanupStorage()
}, 60000 * 10)
await cleanupStorage();
}, 2000);
// checkProxies, checkFluentBit & refresh templates
setInterval(async () => {
await checkProxies();
await checkFluentBit();
}, 60000)
}, 60000);
// Refresh and check templates
setInterval(async () => {
await refreshTemplates()
}, 60000)
await refreshTemplates();
}, 60000);
setInterval(async () => {
await refreshTags()
}, 60000)
await refreshTags();
}, 60000);
setInterval(async () => {
await migrateServicesToNewTemplate()
}, isDev ? 10000 : 60000)
setInterval(
async () => {
await migrateServicesToNewTemplate();
},
isDev ? 10000 : 60000
);
setInterval(async () => {
await copySSLCertificates();
}, 10000)
await Promise.all([
getTagsTemplates(),
getArch(),
getIPAddress(),
configureRemoteDockers(),
])
}, 10000);
await Promise.all([getTagsTemplates(), getArch(), getIPAddress(), configureRemoteDockers()]);
} catch (error) {
console.error(error);
process.exit(1);
}
})();
async function getIPAddress() {
const { publicIpv4, publicIpv6 } = await import('public-ip')
const { publicIpv4, publicIpv6 } = await import('public-ip');
try {
const settings = await listSettings();
if (!settings.ipv4) {
const ipv4 = await publicIpv4({ timeout: 2000 })
const ipv4 = await publicIpv4({ timeout: 2000 });
console.log(`Getting public IPv4 address...`);
await prisma.setting.update({ where: { id: settings.id }, data: { ipv4 } })
await prisma.setting.update({ where: { id: settings.id }, data: { ipv4 } });
}
if (!settings.ipv6) {
const ipv6 = await publicIpv6({ timeout: 2000 })
const ipv6 = await publicIpv6({ timeout: 2000 });
console.log(`Getting public IPv6 address...`);
await prisma.setting.update({ where: { id: settings.id }, data: { ipv6 } })
await prisma.setting.update({ where: { id: settings.id }, data: { ipv6 } });
}
} catch (error) {}
}
async function getTagsTemplates() {
const { default: got } = await import('got')
const { default: got } = await import('got');
try {
if (isDev) {
const templates = await fs.readFile('./devTemplates.yaml', 'utf8')
const tags = await fs.readFile('./devTags.json', 'utf8')
await fs.writeFile('./templates.json', JSON.stringify(yaml.load(templates)))
await fs.writeFile('./tags.json', tags)
console.log('[004] Tags and templates loaded in dev mode...')
const templates = await fs.readFile('./devTemplates.yaml', 'utf8');
const tags = await fs.readFile('./devTags.json', 'utf8');
await fs.writeFile('./templates.json', JSON.stringify(yaml.load(templates)));
await fs.writeFile('./tags.json', tags);
console.log('[004] Tags and templates loaded in dev mode...');
} else {
const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text()
const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text()
await fs.writeFile('/app/templates.json', JSON.stringify(yaml.load(response)))
await fs.writeFile('/app/tags.json', tags)
console.log('[004] Tags and templates loaded...')
const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text();
const response = await got
.get('https://get.coollabs.io/coolify/service-templates.yaml')
.text();
await fs.writeFile('/app/templates.json', JSON.stringify(yaml.load(response)));
await fs.writeFile('/app/tags.json', tags);
console.log('[004] Tags and templates loaded...');
}
} catch (error) {
console.log("Couldn't get latest templates.")
console.log(error)
console.log("Couldn't get latest templates.");
console.log(error);
}
}
async function initServer() {
const appId = process.env['COOLIFY_APP_ID'];
const settings = await prisma.setting.findUnique({ where: { id: '0' } })
const settings = await prisma.setting.findUnique({ where: { id: '0' } });
try {
if (settings.doNotTrack === true) {
console.log('[000] Telemetry disabled...')
console.log('[000] Telemetry disabled...');
} else {
if (settings.sentryDSN !== sentryDSN) {
await prisma.setting.update({ where: { id: '0' }, data: { sentryDSN } })
await prisma.setting.update({ where: { id: '0' }, data: { sentryDSN } });
}
// Initialize Sentry
// Sentry.init({
@ -257,7 +263,7 @@ async function initServer() {
// console.log('[000] Sentry initialized...')
}
} catch (error) {
console.error(error)
console.error(error);
}
try {
console.log(`[001] Initializing server...`);
@ -267,23 +273,26 @@ async function initServer() {
console.log(`[002] Cleanup stucked builds...`);
const isOlder = compareVersions('3.8.1', version);
if (isOlder === 1) {
await prisma.build.updateMany({ where: { status: { in: ['running', 'queued'] } }, data: { status: 'failed' } });
await prisma.build.updateMany({
where: { status: { in: ['running', 'queued'] } },
data: { status: 'failed' }
});
}
} catch (error) {}
try {
console.log('[003] Cleaning up old build sources under /tmp/build-sources/...');
await fs.rm('/tmp/build-sources', { recursive: true, force: true })
await fs.rm('/tmp/build-sources', { recursive: true, force: true });
} catch (error) {
console.log(error)
console.log(error);
}
}
async function getArch() {
try {
const settings = await prisma.setting.findFirst({})
const settings = await prisma.setting.findFirst({});
if (settings && !settings.arch) {
console.log(`Getting architecture...`);
await prisma.setting.update({ where: { id: settings.id }, data: { arch: process.arch } })
await prisma.setting.update({ where: { id: settings.id }, data: { arch: process.arch } });
}
} catch (error) {}
}
@ -296,37 +305,44 @@ async function configureRemoteDockers() {
if (remoteDocker.length > 0) {
console.log(`Verifying Remote Docker Engines...`);
for (const docker of remoteDocker) {
console.log('Verifying:', docker.remoteIpAddress)
console.log('Verifying:', docker.remoteIpAddress);
await verifyRemoteDockerEngineFn(docker.id);
}
}
} catch (error) {
console.log(error)
console.log(error);
}
}
async function autoUpdater() {
try {
const { default: got } = await import('got')
const { default: got } = await import('got');
const currentVersion = version;
const { coolify } = await got.get('https://get.coollabs.io/versions.json', {
const { coolify } = await got
.get('https://get.coollabs.io/versions.json', {
searchParams: {
appId: process.env['COOLIFY_APP_ID'] || undefined,
version: currentVersion
}
}).json()
})
.json();
const latestVersion = coolify.main.version;
const isUpdateAvailable = compareVersions(latestVersion, currentVersion);
if (isUpdateAvailable === 1) {
const activeCount = 0
const activeCount = 0;
if (activeCount === 0) {
if (!isDev) {
const { isAutoUpdateEnabled } = await prisma.setting.findFirst();
if (isAutoUpdateEnabled) {
await executeCommand({ command: `docker pull coollabsio/coolify:${latestVersion}` })
await executeCommand({ shell: true, command: `env | grep '^COOLIFY' > .env` })
await executeCommand({ command: `sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env` })
await executeCommand({ shell: true, command: `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"` })
await executeCommand({ command: `docker pull coollabsio/coolify:${latestVersion}` });
await executeCommand({ shell: true, command: `env | grep '^COOLIFY' > .env` });
await executeCommand({
command: `sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env`
});
await executeCommand({
shell: true,
command: `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"`
});
}
} else {
console.log('Updating (not really in dev mode).');
@ -334,7 +350,7 @@ async function autoUpdater() {
}
}
} catch (error) {
console.log(error)
console.log(error);
}
}
@ -345,14 +361,18 @@ async function checkFluentBit() {
const { id } = await prisma.destinationDocker.findFirst({
where: { engine, network: 'coolify' }
});
const { found } = await checkContainer({ dockerId: id, container: 'coolify-fluentbit', remove: true });
const { found } = await checkContainer({
dockerId: id,
container: 'coolify-fluentbit',
remove: true
});
if (!found) {
await executeCommand({ shell: true, command: `env | grep '^COOLIFY' > .env` });
await executeCommand({ command: `docker compose up -d fluent-bit` });
}
}
} catch (error) {
console.log(error)
console.log(error);
}
}
async function checkProxies() {
@ -368,7 +388,7 @@ async function checkProxies() {
where: { engine, network: 'coolify', isCoolifyProxyUsed: true }
});
if (localDocker) {
portReachable = await isReachable(80, { host: ipv4 || ipv6 })
portReachable = await isReachable(80, { host: ipv4 || ipv6 });
if (!portReachable) {
await startTraefikProxy(localDocker.id);
}
@ -380,13 +400,13 @@ async function checkProxies() {
if (remoteDocker.length > 0) {
for (const docker of remoteDocker) {
if (docker.isCoolifyProxyUsed) {
portReachable = await isReachable(80, { host: docker.remoteIpAddress })
portReachable = await isReachable(80, { host: docker.remoteIpAddress });
if (!portReachable) {
await startTraefikProxy(docker.id);
}
}
try {
await createRemoteEngineConfiguration(docker.id)
await createRemoteEngineConfiguration(docker.id);
} catch (error) {}
}
}
@ -426,80 +446,105 @@ async function checkProxies() {
// await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000);
// }
// }
} catch (error) {
}
} catch (error) {}
}
async function copySSLCertificates() {
try {
const pAll = await import('p-all');
const actions = []
const certificates = await prisma.certificate.findMany({ include: { team: true } })
const teamIds = certificates.map(c => c.teamId)
const destinations = await prisma.destinationDocker.findMany({ where: { isCoolifyProxyUsed: true, teams: { some: { id: { in: [...teamIds] } } } } })
const actions = [];
const certificates = await prisma.certificate.findMany({ include: { team: true } });
const teamIds = certificates.map((c) => c.teamId);
const destinations = await prisma.destinationDocker.findMany({
where: { isCoolifyProxyUsed: true, teams: { some: { id: { in: [...teamIds] } } } }
});
for (const certificate of certificates) {
const { id, key, cert } = certificate
const decryptedKey = decrypt(key)
await fs.writeFile(`/tmp/${id}-key.pem`, decryptedKey)
await fs.writeFile(`/tmp/${id}-cert.pem`, cert)
const { id, key, cert } = certificate;
const decryptedKey = decrypt(key);
await fs.writeFile(`/tmp/${id}-key.pem`, decryptedKey);
await fs.writeFile(`/tmp/${id}-cert.pem`, cert);
for (const destination of destinations) {
if (destination.remoteEngine) {
if (destination.remoteVerified) {
const { id: dockerId, remoteIpAddress } = destination
actions.push(async () => copyRemoteCertificates(id, dockerId, remoteIpAddress))
const { id: dockerId, remoteIpAddress } = destination;
actions.push(async () => copyRemoteCertificates(id, dockerId, remoteIpAddress));
}
} else {
actions.push(async () => copyLocalCertificates(id))
actions.push(async () => copyLocalCertificates(id));
}
}
}
await pAll.default(actions, { concurrency: 1 })
await pAll.default(actions, { concurrency: 1 });
} catch (error) {
console.log(error)
console.log(error);
} finally {
await executeCommand({ command: `find /tmp/ -maxdepth 1 -type f -name '*-*.pem' -delete` })
await executeCommand({ command: `find /tmp/ -maxdepth 1 -type f -name '*-*.pem' -delete` });
}
}
async function copyRemoteCertificates(id: string, dockerId: string, remoteIpAddress: string) {
try {
await executeCommand({ command: `scp /tmp/${id}-cert.pem /tmp/${id}-key.pem ${remoteIpAddress}:/tmp/` })
await executeCommand({ sshCommand: true, shell: true, dockerId, command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'` })
await executeCommand({ sshCommand: true, dockerId, command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/` })
await executeCommand({ sshCommand: true, dockerId, command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/` })
await executeCommand({
command: `scp /tmp/${id}-cert.pem /tmp/${id}-key.pem ${remoteIpAddress}:/tmp/`
});
await executeCommand({
sshCommand: true,
shell: true,
dockerId,
command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'`
});
await executeCommand({
sshCommand: true,
dockerId,
command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/`
});
await executeCommand({
sshCommand: true,
dockerId,
command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/`
});
} catch (error) {
console.log({ error })
console.log({ error });
}
}
async function copyLocalCertificates(id: string) {
try {
await executeCommand({ command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'`, shell: true })
await executeCommand({ command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/` })
await executeCommand({ command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/` })
await executeCommand({
command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'`,
shell: true
});
await executeCommand({
command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/`
});
await executeCommand({
command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/`
});
} catch (error) {
console.log({ error })
console.log({ error });
}
}
async function cleanupStorage() {
const destinationDockers = await prisma.destinationDocker.findMany();
let enginesDone = new Set()
let enginesDone = new Set();
for (const destination of destinationDockers) {
if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return
if (destination.engine) enginesDone.add(destination.engine)
if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress)
if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return;
if (destination.engine) enginesDone.add(destination.engine);
if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress);
let force = false;
let lowDiskSpace = false;
try {
let stdout = null
let stdout = null;
if (!isDev) {
const output = await executeCommand({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'`, shell: true })
const output = await executeCommand({
dockerId: destination.id,
command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'`,
shell: true
});
stdout = output.stdout;
} else {
const output = await executeCommand({
command:
`df -kPT /`
command: `df -kPT /`
});
stdout = output.stdout;
}
@ -529,9 +574,10 @@ async function cleanupStorage() {
const { capacity } = data[0];
if (capacity > 0.8) {
lowDiskSpace = true;
force = true;
}
}
} catch (error) {}
await cleanupDockerStorage(destination.id, lowDiskSpace, false)
await cleanupDockerStorage(destination.id, lowDiskSpace, force);
}
}

View File

@ -19,7 +19,7 @@ import { saveBuildLog, saveDockerRegistryCredentials } from './buildPacks/common
import { scheduler } from './scheduler';
import type { ExecaChildProcess } from 'execa';
export const version = '3.12.4';
export const version = '3.12.5';
export const isDev = process.env.NODE_ENV === 'development';
export const sentryDSN =
'https://409f09bcb7af47928d3e0f46b78987f3@o1082494.ingest.sentry.io/4504236622217216';
@ -584,7 +584,7 @@ export async function executeCommand({
}
if (sshCommand) {
if (shell) {
return execaCommand(`ssh ${remoteIpAddress}-remote ${command}`);
return execaCommand(`ssh ${remoteIpAddress}-remote ${command}`, { shell: true });
}
return await execa('ssh', [`${remoteIpAddress}-remote`, dockerCommand, ...dockerArgs]);
}
@ -651,11 +651,13 @@ export async function executeCommand({
} else {
if (shell) {
return await execaCommand(command, {
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine },
shell: true
});
} else {
return await execa(dockerCommand, dockerArgs, {
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine },
shell: false
});
}
}
@ -1751,6 +1753,10 @@ export async function cleanupDockerStorage(dockerId, lowDiskSpace, force) {
let keepImage = [];
for (const image2 of imagesArray) {
if (image2.startsWith(image)) {
if (force) {
deleteImage.push(image2);
continue;
}
if (keepImage.length >= numberOfDockerImagesKeptLocally) {
deleteImage.push(image2);
} else {
@ -1760,7 +1766,11 @@ export async function cleanupDockerStorage(dockerId, lowDiskSpace, force) {
}
}
for (const image of deleteImage) {
try {
await executeCommand({ dockerId, command: `docker image rm -f ${image}` });
} catch (error) {
console.log(error);
}
}
// Prune coolify managed containers

File diff suppressed because one or more lines are too long

View File

@ -1,7 +1,7 @@
{
"name": "coolify",
"description": "An open-source & self-hostable Heroku / Netlify alternative.",
"version": "3.12.4",
"version": "3.12.5",
"license": "Apache-2.0",
"repository": "github:coollabsio/coolify",
"scripts": {