diff --git a/Dockerfile b/Dockerfile index 5d0a8912b..9494ff702 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ WORKDIR /app LABEL coolify.managed true -RUN apk add --no-cache git openssh-client curl jq cmake sqlite +RUN apk add --no-cache git git-lfs openssh-client curl jq cmake sqlite RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm@6 RUN pnpm add -g pnpm diff --git a/src/lib/common.ts b/src/lib/common.ts index 49a637065..d6b36b209 100644 --- a/src/lib/common.ts +++ b/src/lib/common.ts @@ -46,11 +46,30 @@ const customConfig: Config = { export const version = currentVersion; export const asyncExecShell = util.promisify(child.exec); export const asyncSleep = (delay) => new Promise((resolve) => setTimeout(resolve, delay)); +export const asyncUntil = (condition, delay, maxIterations) => { + return new Promise(async (resolve, reject) => { + let iterations = 0; + while (!condition()) { + if (maxIterations && iterations >= maxIterations) { + reject(new Error('Max iterations reached.')); + return; + } + await asyncSleep(delay); + iterations++; + } + resolve(); + }); +}; + export const sentry = Sentry; export const uniqueName = () => uniqueNamesGenerator(customConfig); export const saveBuildLog = async ({ line, buildId, applicationId }) => { + if (line.includes('ghs_')) { + const regex = /ghs_.*@/g; + line = line.replace(regex, '@'); + } const addTimestamp = `${generateTimestamp()} ${line}`; return await buildLogQueue.add(buildId, { buildId, line: addTimestamp, applicationId }); }; diff --git a/src/lib/importers/github.ts b/src/lib/importers/github.ts index 1ab3ca90a..53754205e 100644 --- a/src/lib/importers/github.ts +++ b/src/lib/importers/github.ts @@ -15,40 +15,35 @@ export default async function ({ branch, buildId }): Promise { - try { - const url = htmlUrl.replace('https://', '').replace('http://', ''); - await saveBuildLog({ line: 'GitHub importer started.', buildId, applicationId }); - const { privateKey, appId, installationId } = await db.getUniqueGithubApp({ githubAppId }); - const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, ''); + const url = htmlUrl.replace('https://', '').replace('http://', ''); + await saveBuildLog({ line: 'GitHub importer started.', buildId, applicationId }); + const { privateKey, appId, installationId } = await db.getUniqueGithubApp({ githubAppId }); + const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, ''); - const payload = { - iat: Math.round(new Date().getTime() / 1000), - exp: Math.round(new Date().getTime() / 1000 + 60), - iss: appId - }; - const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, { - algorithm: 'RS256' - }); - const { token } = await got - .post(`${apiUrl}/app/installations/${installationId}/access_tokens`, { - headers: { - Authorization: `Bearer ${jwtToken}`, - Accept: 'application/vnd.github.machine-man-preview+json' - } - }) - .json(); - await saveBuildLog({ - line: `Cloning ${repository}:${branch} branch.`, - buildId, - applicationId - }); - await asyncExecShell( - `git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && cd ..` - ); - const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); - return commit.replace('\n', ''); - } catch (error) { - console.log({ error }); - return ErrorHandler(error); - } + const payload = { + iat: Math.round(new Date().getTime() / 1000), + exp: Math.round(new Date().getTime() / 1000 + 60), + iss: appId + }; + const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, { + algorithm: 'RS256' + }); + const { token } = await got + .post(`${apiUrl}/app/installations/${installationId}/access_tokens`, { + headers: { + Authorization: `Bearer ${jwtToken}`, + Accept: 'application/vnd.github.machine-man-preview+json' + } + }) + .json(); + await saveBuildLog({ + line: `Cloning ${repository}:${branch} branch.`, + buildId, + applicationId + }); + await asyncExecShell( + `git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. ` + ); + const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); + return commit.replace('\n', ''); } diff --git a/src/lib/importers/gitlab.ts b/src/lib/importers/gitlab.ts index 07bbf7954..81ef54c10 100644 --- a/src/lib/importers/gitlab.ts +++ b/src/lib/importers/gitlab.ts @@ -22,7 +22,7 @@ export default async function ({ }); await asyncExecShell( - `git clone -q -b ${branch} git@${url}:${repository}.git --config core.sshCommand="ssh -q -i ${repodir}id.rsa -o StrictHostKeyChecking=no" ${workdir}/ && cd ${workdir}/ && git submodule update --init --recursive && cd ..` + `git clone -q -b ${branch} git@${url}:${repository}.git --config core.sshCommand="ssh -q -i ${repodir}id.rsa -o StrictHostKeyChecking=no" ${workdir}/ && cd ${workdir}/ && git submodule update --init --recursive && git lfs pull && cd .. ` ); const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); return commit.replace('\n', ''); diff --git a/src/lib/queues/index.ts b/src/lib/queues/index.ts index 0f7dd5815..58b9e583d 100644 --- a/src/lib/queues/index.ts +++ b/src/lib/queues/index.ts @@ -11,7 +11,7 @@ import proxy from './proxy'; import ssl from './ssl'; import sslrenewal from './sslrenewal'; -import { asyncExecShell, saveBuildLog } from '$lib/common'; +import { asyncExecShell, asyncUntil, saveBuildLog } from '$lib/common'; let { Queue, Worker } = Bullmq; let redisHost = 'localhost'; @@ -128,7 +128,22 @@ buildWorker.on('completed', async (job: Bullmq.Job) => { buildWorker.on('failed', async (job: Bullmq.Job, failedReason) => { try { - await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'failed' } }); + await asyncUntil( + async () => { + const found = await prisma.build.findFirst({ + where: { id: job.data.build_id, status: 'failed' } + }); + if (!found) { + return await prisma.build.update({ + where: { id: job.data.build_id }, + data: { status: 'failed' } + }); + } + return true; + }, + 200, + 5 + ); } catch (error) { console.log(error); } finally {