cupcakearmy/coolify
1
0
Fork 0
mirror of https://github.com/cupcakearmy/coolify.git synced 2024-10-22 08:04:19 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: delete preview deployments + cleanup stucked

Browse Source 
fix: parser
This commit is contained in:
Andras Bacsai 2024-08-28 22:05:49 +02:00
parent be8a8bf2ee
commit 5525c02c7f
7 changed files with 108 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/Console/Commands/CleanupStuckedResources.php
View File

@ -3,6 +3,7 @@
namespace App\Console\Commands;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\ScheduledTask;
use App\Models\Service;
use App\Models\ServiceApplication;
@ -42,6 +43,17 @@ private function cleanup_stucked_resources()
} catch (\Throwable $e) {
echo "Error in cleaning stuck application: {$e->getMessage()}\n";
}
try {
$applicationsPreviews = ApplicationPreview::get();
foreach ($applicationsPreviews as $applicationPreview) {
if (! data_get($applicationPreview, 'application')) {
echo "Deleting stuck application preview: {$applicationPreview->uuid}\n";
$applicationPreview->delete();
}
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck application: {$e->getMessage()}\n";
}
try {
$postgresqls = StandalonePostgresql::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($postgresqls as $postgresql) {

72
app/Jobs/ApplicationDeploymentJob.php
View File

@ -192,8 +192,8 @@ public function __construct(int $application_deployment_queue_id)
$this->preserveRepository = $this->application->settings->is_preserve_repository_enabled;
$this->basedir = $this->application->generateBaseDir($this->deployment_uuid);
$this->workdir = "{$this->basedir}" . rtrim($this->application->base_directory, '/');
$this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}";
$this->workdir = "{$this->basedir}".rtrim($this->application->base_directory, '/');
$this->configuration_dir = application_configuration_dir()."/{$this->application->uuid}";
$this->is_debug_enabled = $this->application->settings->is_debug_enabled;
$this->container_name = generateApplicationContainerName($this->application, $this->pull_request_id);
@ -402,13 +402,13 @@ private function deploy_docker_compose_buildpack()
if (data_get($this->application, 'docker_compose_custom_start_command')) {
$this->docker_compose_custom_start_command = $this->application->docker_compose_custom_start_command;
if (! str($this->docker_compose_custom_start_command)->contains('--project-directory')) {
$this->docker_compose_custom_start_command = str($this->docker_compose_custom_start_command)->replaceFirst('compose', 'compose --project-directory ' . $this->workdir)->value();
$this->docker_compose_custom_start_command = str($this->docker_compose_custom_start_command)->replaceFirst('compose', 'compose --project-directory '.$this->workdir)->value();
}
}
if (data_get($this->application, 'docker_compose_custom_build_command')) {
$this->docker_compose_custom_build_command = $this->application->docker_compose_custom_build_command;
if (! str($this->docker_compose_custom_build_command)->contains('--project-directory')) {
$this->docker_compose_custom_build_command = str($this->docker_compose_custom_build_command)->replaceFirst('compose', 'compose --project-directory ' . $this->workdir)->value();
$this->docker_compose_custom_build_command = str($this->docker_compose_custom_build_command)->replaceFirst('compose', 'compose --project-directory '.$this->workdir)->value();
}
}
if ($this->pull_request_id === 0) {
@ -422,7 +422,7 @@ private function deploy_docker_compose_buildpack()
if ($this->preserveRepository) {
foreach ($this->application->fileStorages as $fileStorage) {
$path = $fileStorage->fs_path;
$saveName = 'file_stat_' . $fileStorage->id;
$saveName = 'file_stat_'.$fileStorage->id;
$realPathInGit = str($path)->replace($this->application->workdir(), $this->workdir)->value();
// check if the file is a directory or a file inside the repository
$this->execute_remote_command(
@ -740,7 +740,7 @@ private function push_to_docker_registry()
return;
}
ray('push_to_docker_registry noww: ' . $this->production_image_name);
ray('push_to_docker_registry noww: '.$this->production_image_name);
try {
instant_remote_process(["docker images --format '{{json .}}' {$this->production_image_name}"], $this->server);
$this->application_deployment_queue->addLogEntry('----------------------------------------');
@ -927,12 +927,12 @@ private function save_environment_variables()
$real_value = $env->real_value;
} else {
if ($env->is_literal || $env->is_multiline) {
$real_value = '\'' . $real_value . '\'';
$real_value = '\''.$real_value.'\'';
} else {
$real_value = escapeEnvVariables($env->real_value);
}
}
$envs->push($env->key . '=' . $real_value);
$envs->push($env->key.'='.$real_value);
}
// Add PORT if not exists, use the first port as default
if ($this->build_pack !== 'dockercompose') {
@ -977,12 +977,12 @@ private function save_environment_variables()
$real_value = $env->real_value;
} else {
if ($env->is_literal || $env->is_multiline) {
$real_value = '\'' . $real_value . '\'';
$real_value = '\''.$real_value.'\'';
} else {
$real_value = escapeEnvVariables($env->real_value);
}
}
$envs->push($env->key . '=' . $real_value);
$envs->push($env->key.'='.$real_value);
}
// Add PORT if not exists, use the first port as default
if ($this->build_pack !== 'dockercompose') {
@ -1349,7 +1349,7 @@ private function deploy_to_additional_destinations()
destination: $destination,
no_questions_asked: true,
);
$this->application_deployment_queue->addLogEntry("Deployment to {$server->name}. Logs: " . route('project.application.deployment.show', [
$this->application_deployment_queue->addLogEntry("Deployment to {$server->name}. Logs: ".route('project.application.deployment.show', [
'project_uuid' => data_get($this->application, 'environment.project.uuid'),
'application_uuid' => data_get($this->application, 'uuid'),
'deployment_uuid' => $deployment_uuid,
@ -1686,27 +1686,27 @@ private function generate_compose_file()
'CMD-SHELL',
$this->generate_healthcheck_commands(),
],
'interval' => $this->application->health_check_interval . 's',
'timeout' => $this->application->health_check_timeout . 's',
'interval' => $this->application->health_check_interval.'s',
'timeout' => $this->application->health_check_timeout.'s',
'retries' => $this->application->health_check_retries,
'start_period' => $this->application->health_check_start_period . 's',
'start_period' => $this->application->health_check_start_period.'s',
];
if (! is_null($this->application->limits_cpuset)) {
data_set($docker_compose, 'services.' . $this->container_name . '.cpuset', $this->application->limits_cpuset);
data_set($docker_compose, 'services.'.$this->container_name.'.cpuset', $this->application->limits_cpuset);
}
if ($this->server->isSwarm()) {
data_forget($docker_compose, 'services.' . $this->container_name . '.container_name');
data_forget($docker_compose, 'services.' . $this->container_name . '.expose');
data_forget($docker_compose, 'services.' . $this->container_name . '.restart');
data_forget($docker_compose, 'services.'.$this->container_name.'.container_name');
data_forget($docker_compose, 'services.'.$this->container_name.'.expose');
data_forget($docker_compose, 'services.'.$this->container_name.'.restart');
data_forget($docker_compose, 'services.' . $this->container_name . '.mem_limit');
data_forget($docker_compose, 'services.' . $this->container_name . '.memswap_limit');
data_forget($docker_compose, 'services.' . $this->container_name . '.mem_swappiness');
data_forget($docker_compose, 'services.' . $this->container_name . '.mem_reservation');
data_forget($docker_compose, 'services.' . $this->container_name . '.cpus');
data_forget($docker_compose, 'services.' . $this->container_name . '.cpuset');
data_forget($docker_compose, 'services.' . $this->container_name . '.cpu_shares');
data_forget($docker_compose, 'services.'.$this->container_name.'.mem_limit');
data_forget($docker_compose, 'services.'.$this->container_name.'.memswap_limit');
data_forget($docker_compose, 'services.'.$this->container_name.'.mem_swappiness');
data_forget($docker_compose, 'services.'.$this->container_name.'.mem_reservation');
data_forget($docker_compose, 'services.'.$this->container_name.'.cpus');
data_forget($docker_compose, 'services.'.$this->container_name.'.cpuset');
data_forget($docker_compose, 'services.'.$this->container_name.'.cpu_shares');
$docker_compose['services'][$this->container_name]['deploy'] = [
'mode' => 'replicated',
@ -1775,20 +1775,20 @@ private function generate_compose_file()
}
}
if ($this->application->isHealthcheckDisabled()) {
data_forget($docker_compose, 'services.' . $this->container_name . '.healthcheck');
data_forget($docker_compose, 'services.'.$this->container_name.'.healthcheck');
}
if (count($this->application->ports_mappings_array) > 0 && $this->pull_request_id === 0) {
$docker_compose['services'][$this->container_name]['ports'] = $this->application->ports_mappings_array;
}
if (count($persistent_storages) > 0) {
if (! data_get($docker_compose, 'services.' . $this->container_name . '.volumes')) {
if (! data_get($docker_compose, 'services.'.$this->container_name.'.volumes')) {
$docker_compose['services'][$this->container_name]['volumes'] = [];
}
$docker_compose['services'][$this->container_name]['volumes'] = array_merge($docker_compose['services'][$this->container_name]['volumes'], $persistent_storages);
}
if (count($persistent_file_volumes) > 0) {
if (! data_get($docker_compose, 'services.' . $this->container_name . '.volumes')) {
if (! data_get($docker_compose, 'services.'.$this->container_name.'.volumes')) {
$docker_compose['services'][$this->container_name]['volumes'] = [];
}
$docker_compose['services'][$this->container_name]['volumes'] = array_merge($docker_compose['services'][$this->container_name]['volumes'], $persistent_file_volumes->map(function ($item) {
@ -1856,9 +1856,9 @@ private function generate_local_persistent_volumes()
$volume_name = $persistentStorage->name;
}
if ($this->pull_request_id !== 0) {
$volume_name = $volume_name . '-pr-' . $this->pull_request_id;
$volume_name = $volume_name.'-pr-'.$this->pull_request_id;
}
$local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path;
$local_persistent_volumes[] = $volume_name.':'.$persistentStorage->mount_path;
}
return $local_persistent_volumes;
@ -1874,7 +1874,7 @@ private function generate_local_persistent_volumes_only_volume_names()
$name = $persistentStorage->name;
if ($this->pull_request_id !== 0) {
$name = $name . '-pr-' . $this->pull_request_id;
$name = $name.'-pr-'.$this->pull_request_id;
}
$local_persistent_volumes_names[$name] = [
@ -2152,7 +2152,7 @@ private function stop_running_container(bool $force = false)
$containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id);
if ($this->pull_request_id === 0) {
$containers = $containers->filter(function ($container) {
return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== $this->container_name . '-pr-' . $this->pull_request_id;
return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== $this->container_name.'-pr-'.$this->pull_request_id;
});
}
$containers->each(function ($container) {
@ -2259,8 +2259,8 @@ private function run_pre_deployment_command()
foreach ($containers as $container) {
$containerName = data_get($container, 'Names');
if ($containers->count() == 1 || str_starts_with($containerName, $this->application->pre_deployment_command_container . '-' . $this->application->uuid)) {
$cmd = "sh -c '" . str_replace("'", "'\''", $this->application->pre_deployment_command) . "'";
if ($containers->count() == 1 || str_starts_with($containerName, $this->application->pre_deployment_command_container.'-'.$this->application->uuid)) {
$cmd = "sh -c '".str_replace("'", "'\''", $this->application->pre_deployment_command)."'";
$exec = "docker exec {$containerName} {$cmd}";
$this->execute_remote_command(
[
@ -2286,8 +2286,8 @@ private function run_post_deployment_command()
$containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id);
foreach ($containers as $container) {
$containerName = data_get($container, 'Names');
if ($containers->count() == 1 || str_starts_with($containerName, $this->application->post_deployment_command_container . '-' . $this->application->uuid)) {
$cmd = "sh -c '" . str_replace("'", "'\''", $this->application->post_deployment_command) . "'";
if ($containers->count() == 1 || str_starts_with($containerName, $this->application->post_deployment_command_container.'-'.$this->application->uuid)) {
$cmd = "sh -c '".str_replace("'", "'\''", $this->application->post_deployment_command)."'";
$exec = "docker exec {$containerName} {$cmd}";
try {
$this->execute_remote_command(

1
app/Livewire/Project/Application/General.php
View File

@ -4,7 +4,6 @@
use App\Models\Application;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Livewire\Component;
use Visus\Cuid2\Cuid2;

1
app/Models/Application.php
View File

@ -142,6 +142,7 @@ protected static function booted()
$task->delete();
}
$application->tags()->detach();
$application->previews()->delete();
});
}

4
app/Models/ApplicationPreview.php
View File

@ -12,9 +12,9 @@ class ApplicationPreview extends BaseModel
protected static function booted()
{
static::deleting(function ($preview) {
if ($preview->application->build_pack === 'dockercompose') {
if (data_get($preview, 'application.build_pack') === 'dockercompose') {
$server = $preview->application->destination->server;
$composeFile = $preview->application->oldParser(pull_request_id: $preview->pull_request_id);
$composeFile = newParser($preview->application, pull_request_id: $preview->pull_request_id);
$volumes = data_get($composeFile, 'volumes');
$networks = data_get($composeFile, 'networks');
$networkKeys = collect($networks)->keys();

56
bootstrap/helpers/shared.php
View File

@ -3046,6 +3046,9 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
} else {
$mainDirectory = str(base_configuration_dir().'/applications/'.$uuid);
$source = replaceLocalSource($source, $mainDirectory);
if ($isApplication && $isPullRequest) {
$source = $source."-pr-$pullRequestId";
}
LocalFileVolume::updateOrCreate(
[
@ -3076,6 +3079,10 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
}
$slugWithoutUuid = Str::slug($source, '-');
$name = "{$uuid}_{$slugWithoutUuid}";
if ($isApplication && $isPullRequest) {
$name = "{$name}-pr-$pullRequestId";
}
if (is_string($volume)) {
$source = str($volume)->before(':');
$target = str($volume)->after(':')->beforeLast(':');
@ -3106,6 +3113,23 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
$volumesParsed->put($index, $volume);
}
}
if ($depends_on?->count() > 0) {
if ($isApplication && $isPullRequest) {
$newDependsOn = collect([]);
$depends_on->each(function ($dependency, $condition) use ($pullRequestId, $newDependsOn) {
if (is_numeric($condition)) {
$dependency = "$dependency-pr-$pullRequestId";
$newDependsOn->put($condition, $dependency);
} else {
$condition = "$condition-pr-$pullRequestId";
$newDependsOn->put($condition, $dependency);
}
});
$depends_on = $newDependsOn;
}
}
if ($topLevel->get('networks')?->count() > 0) {
foreach ($topLevel->get('networks') as $networkName => $network) {
if ($networkName === 'default') {
@ -3196,7 +3220,6 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
// remove $environment from $allEnvironments
$coolifyDefinedEnvironments = $allEnvironments->diffKeys($environment);
// filter magic environments
$magicEnvironments = $environment->filter(function ($value, $key) {
$value = str(replaceVariables(str($value)));
@ -3340,12 +3363,11 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
}
}
if ($isApplication) {
$branch = $originalResource->git_branch;
$branch = $originalResource->git_branch;
if ($pullRequestId !== 0) {
$branch = "pull/{$pullRequestId}/head";
}
if ($originalResource->environment_variables->where('key', 'COOLIFY_BRANCH')->isEmpty()) {
ray($branch);
$environment->put('COOLIFY_BRANCH', $branch);
}
}
@ -3371,10 +3393,10 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
$fqdns = collect([]);
}
} else {
$fqdns = $fqdns->map(function ($fqdn) use ($pullRequestId) {
$preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->id, $pullRequestId);
$fqdns = $fqdns->map(function ($fqdn) use ($pullRequestId, $resource) {
$preview = ApplicationPreview::findPreviewByApplicationAndPullId($resource->id, $pullRequestId);
$url = Url::fromString($fqdn);
$template = $this->preview_url_template;
$template = $resource->preview_url_template;
$host = $url->getHost();
$schema = $url->getScheme();
$random = new Cuid2;
@ -3420,14 +3442,24 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
if (! $isDatabase && $fqdns?->count() > 0) {
if ($isApplication) {
$shouldGenerateLabelsExactly = $resource->destination->server->settings->generate_exact_labels;
$uuid = $resource->uuid;
$network = $resource->destination->network;
if ($isPullRequest) {
$uuid = "{$resource->uuid}-{$pullRequestId}";
}
if ($isPullRequest) {
$network = "{$resource->destination->network}-{$pullRequestId}";
}
} else {
$shouldGenerateLabelsExactly = $resource->server->settings->generate_exact_labels;
$uuid = $resource->uuid;
$network = $resource->server->destination->network;
}
if ($shouldGenerateLabelsExactly) {
switch ($server->proxyType()) {
case ProxyTypes::TRAEFIK->value:
$serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik(
uuid: $resource->uuid,
uuid: $uuid,
domains: $fqdns,
is_force_https_enabled: true,
serviceLabels: $serviceLabels,
@ -3439,8 +3471,8 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
break;
case ProxyTypes::CADDY->value:
$serviceLabels = $serviceLabels->merge(fqdnLabelsForCaddy(
network: $resource->destination->network,
uuid: $resource->uuid,
network: $network,
uuid: $uuid,
domains: $fqdns,
is_force_https_enabled: true,
serviceLabels: $serviceLabels,
@ -3454,7 +3486,7 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
}
} else {
$serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik(
uuid: $resource->uuid,
uuid: $uuid,
domains: $fqdns,
is_force_https_enabled: true,
serviceLabels: $serviceLabels,
@ -3464,8 +3496,8 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
image: $image
));
$serviceLabels = $serviceLabels->merge(fqdnLabelsForCaddy(
network: $resource->destination->network,
uuid: $resource->uuid,
network: $network,
uuid: $uuid,
domains: $fqdns,
is_force_https_enabled: true,
serviceLabels: $serviceLabels,

18
tests/Feature/DockerComposeParseTest.php
View File

@ -1,6 +1,7 @@
<?php
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\GithubApp;
use App\Models\Server;
use App\Models\Service;
@ -9,7 +10,6 @@
use Symfony\Component\Yaml\Yaml;
use Visus\Cuid2\Cuid2;
ray()->clearAll();
beforeEach(function () {
$this->applicationYaml = '
version: "3.8"
@ -62,7 +62,8 @@
'domain' => 'http://bcoowoookw0co4cok4sgc4k8.127.0.0.1.sslip.io',
],
]),
'uuid' => 'bcoowoookw0co4cok4sgc4k8',
'preview_url_template' => '{{pr_id}}.{{domain}}',
'uuid' => 'bcoowoookw0co4cok4sgc4k8s',
'repository_project_id' => 603035348,
'git_repository' => 'coollabsio/coolify-examples',
'git_branch' => 'main',
@ -77,6 +78,13 @@
'source_id' => 1,
'source_type' => GithubApp::class,
]);
$this->application->environment_variables_preview()->where('key', 'APP_DEBUG')->update(['value' => 'true']);
$this->applicationPreview = ApplicationPreview::create([
'git_type' => 'github',
'application_id' => $this->application->id,
'pull_request_id' => 1,
'pull_request_html_url' => 'https://github.com/coollabsio/coolify-examples/pull/1',
]);
$this->serviceYaml = '
version: "3.8"
services:
@ -156,6 +164,7 @@
});
afterEach(function () {
// $this->applicationPreview->forceDelete();
$this->application->forceDelete();
$this->service->forceDelete();
});
@ -324,11 +333,10 @@
// });
test('ServiceComposeParseNew', function () {
ray()->clearAll();
$output = newParser($this->service);
$output = newParser($this->application, pull_request_id: 1, preview_id: $this->applicationPreview->id);
// ray('New parser');
// ray($output->toArray());
ray($this->service->environment_variables->pluck('value', 'key')->toArray());
ray($this->service->environment_variables_preview->pluck('value', 'key')->toArray());
expect($output)->toBeInstanceOf(Collection::class);
});