From 5525c02c7f46aa41e965e648ed26f0ee4e7cfaf2 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 28 Aug 2024 22:05:49 +0200 Subject: [PATCH] fix: delete preview deployments + cleanup stucked fix: parser --- .../Commands/CleanupStuckedResources.php | 12 ++++ app/Jobs/ApplicationDeploymentJob.php | 72 +++++++++---------- app/Livewire/Project/Application/General.php | 1 - app/Models/Application.php | 1 + app/Models/ApplicationPreview.php | 4 +- bootstrap/helpers/shared.php | 56 +++++++++++---- tests/Feature/DockerComposeParseTest.php | 18 +++-- 7 files changed, 108 insertions(+), 56 deletions(-) diff --git a/app/Console/Commands/CleanupStuckedResources.php b/app/Console/Commands/CleanupStuckedResources.php index fbbf2c820..0a544d2ff 100644 --- a/app/Console/Commands/CleanupStuckedResources.php +++ b/app/Console/Commands/CleanupStuckedResources.php @@ -3,6 +3,7 @@ namespace App\Console\Commands; use App\Models\Application; +use App\Models\ApplicationPreview; use App\Models\ScheduledTask; use App\Models\Service; use App\Models\ServiceApplication; @@ -42,6 +43,17 @@ private function cleanup_stucked_resources() } catch (\Throwable $e) { echo "Error in cleaning stuck application: {$e->getMessage()}\n"; } + try { + $applicationsPreviews = ApplicationPreview::get(); + foreach ($applicationsPreviews as $applicationPreview) { + if (! data_get($applicationPreview, 'application')) { + echo "Deleting stuck application preview: {$applicationPreview->uuid}\n"; + $applicationPreview->delete(); + } + } + } catch (\Throwable $e) { + echo "Error in cleaning stuck application: {$e->getMessage()}\n"; + } try { $postgresqls = StandalonePostgresql::withTrashed()->whereNotNull('deleted_at')->get(); foreach ($postgresqls as $postgresql) { diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index b0c513a33..72cb55a5a 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -192,8 +192,8 @@ public function __construct(int $application_deployment_queue_id) $this->preserveRepository = $this->application->settings->is_preserve_repository_enabled; $this->basedir = $this->application->generateBaseDir($this->deployment_uuid); - $this->workdir = "{$this->basedir}" . rtrim($this->application->base_directory, '/'); - $this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}"; + $this->workdir = "{$this->basedir}".rtrim($this->application->base_directory, '/'); + $this->configuration_dir = application_configuration_dir()."/{$this->application->uuid}"; $this->is_debug_enabled = $this->application->settings->is_debug_enabled; $this->container_name = generateApplicationContainerName($this->application, $this->pull_request_id); @@ -402,13 +402,13 @@ private function deploy_docker_compose_buildpack() if (data_get($this->application, 'docker_compose_custom_start_command')) { $this->docker_compose_custom_start_command = $this->application->docker_compose_custom_start_command; if (! str($this->docker_compose_custom_start_command)->contains('--project-directory')) { - $this->docker_compose_custom_start_command = str($this->docker_compose_custom_start_command)->replaceFirst('compose', 'compose --project-directory ' . $this->workdir)->value(); + $this->docker_compose_custom_start_command = str($this->docker_compose_custom_start_command)->replaceFirst('compose', 'compose --project-directory '.$this->workdir)->value(); } } if (data_get($this->application, 'docker_compose_custom_build_command')) { $this->docker_compose_custom_build_command = $this->application->docker_compose_custom_build_command; if (! str($this->docker_compose_custom_build_command)->contains('--project-directory')) { - $this->docker_compose_custom_build_command = str($this->docker_compose_custom_build_command)->replaceFirst('compose', 'compose --project-directory ' . $this->workdir)->value(); + $this->docker_compose_custom_build_command = str($this->docker_compose_custom_build_command)->replaceFirst('compose', 'compose --project-directory '.$this->workdir)->value(); } } if ($this->pull_request_id === 0) { @@ -422,7 +422,7 @@ private function deploy_docker_compose_buildpack() if ($this->preserveRepository) { foreach ($this->application->fileStorages as $fileStorage) { $path = $fileStorage->fs_path; - $saveName = 'file_stat_' . $fileStorage->id; + $saveName = 'file_stat_'.$fileStorage->id; $realPathInGit = str($path)->replace($this->application->workdir(), $this->workdir)->value(); // check if the file is a directory or a file inside the repository $this->execute_remote_command( @@ -740,7 +740,7 @@ private function push_to_docker_registry() return; } - ray('push_to_docker_registry noww: ' . $this->production_image_name); + ray('push_to_docker_registry noww: '.$this->production_image_name); try { instant_remote_process(["docker images --format '{{json .}}' {$this->production_image_name}"], $this->server); $this->application_deployment_queue->addLogEntry('----------------------------------------'); @@ -927,12 +927,12 @@ private function save_environment_variables() $real_value = $env->real_value; } else { if ($env->is_literal || $env->is_multiline) { - $real_value = '\'' . $real_value . '\''; + $real_value = '\''.$real_value.'\''; } else { $real_value = escapeEnvVariables($env->real_value); } } - $envs->push($env->key . '=' . $real_value); + $envs->push($env->key.'='.$real_value); } // Add PORT if not exists, use the first port as default if ($this->build_pack !== 'dockercompose') { @@ -977,12 +977,12 @@ private function save_environment_variables() $real_value = $env->real_value; } else { if ($env->is_literal || $env->is_multiline) { - $real_value = '\'' . $real_value . '\''; + $real_value = '\''.$real_value.'\''; } else { $real_value = escapeEnvVariables($env->real_value); } } - $envs->push($env->key . '=' . $real_value); + $envs->push($env->key.'='.$real_value); } // Add PORT if not exists, use the first port as default if ($this->build_pack !== 'dockercompose') { @@ -1349,7 +1349,7 @@ private function deploy_to_additional_destinations() destination: $destination, no_questions_asked: true, ); - $this->application_deployment_queue->addLogEntry("Deployment to {$server->name}. Logs: " . route('project.application.deployment.show', [ + $this->application_deployment_queue->addLogEntry("Deployment to {$server->name}. Logs: ".route('project.application.deployment.show', [ 'project_uuid' => data_get($this->application, 'environment.project.uuid'), 'application_uuid' => data_get($this->application, 'uuid'), 'deployment_uuid' => $deployment_uuid, @@ -1686,27 +1686,27 @@ private function generate_compose_file() 'CMD-SHELL', $this->generate_healthcheck_commands(), ], - 'interval' => $this->application->health_check_interval . 's', - 'timeout' => $this->application->health_check_timeout . 's', + 'interval' => $this->application->health_check_interval.'s', + 'timeout' => $this->application->health_check_timeout.'s', 'retries' => $this->application->health_check_retries, - 'start_period' => $this->application->health_check_start_period . 's', + 'start_period' => $this->application->health_check_start_period.'s', ]; if (! is_null($this->application->limits_cpuset)) { - data_set($docker_compose, 'services.' . $this->container_name . '.cpuset', $this->application->limits_cpuset); + data_set($docker_compose, 'services.'.$this->container_name.'.cpuset', $this->application->limits_cpuset); } if ($this->server->isSwarm()) { - data_forget($docker_compose, 'services.' . $this->container_name . '.container_name'); - data_forget($docker_compose, 'services.' . $this->container_name . '.expose'); - data_forget($docker_compose, 'services.' . $this->container_name . '.restart'); + data_forget($docker_compose, 'services.'.$this->container_name.'.container_name'); + data_forget($docker_compose, 'services.'.$this->container_name.'.expose'); + data_forget($docker_compose, 'services.'.$this->container_name.'.restart'); - data_forget($docker_compose, 'services.' . $this->container_name . '.mem_limit'); - data_forget($docker_compose, 'services.' . $this->container_name . '.memswap_limit'); - data_forget($docker_compose, 'services.' . $this->container_name . '.mem_swappiness'); - data_forget($docker_compose, 'services.' . $this->container_name . '.mem_reservation'); - data_forget($docker_compose, 'services.' . $this->container_name . '.cpus'); - data_forget($docker_compose, 'services.' . $this->container_name . '.cpuset'); - data_forget($docker_compose, 'services.' . $this->container_name . '.cpu_shares'); + data_forget($docker_compose, 'services.'.$this->container_name.'.mem_limit'); + data_forget($docker_compose, 'services.'.$this->container_name.'.memswap_limit'); + data_forget($docker_compose, 'services.'.$this->container_name.'.mem_swappiness'); + data_forget($docker_compose, 'services.'.$this->container_name.'.mem_reservation'); + data_forget($docker_compose, 'services.'.$this->container_name.'.cpus'); + data_forget($docker_compose, 'services.'.$this->container_name.'.cpuset'); + data_forget($docker_compose, 'services.'.$this->container_name.'.cpu_shares'); $docker_compose['services'][$this->container_name]['deploy'] = [ 'mode' => 'replicated', @@ -1775,20 +1775,20 @@ private function generate_compose_file() } } if ($this->application->isHealthcheckDisabled()) { - data_forget($docker_compose, 'services.' . $this->container_name . '.healthcheck'); + data_forget($docker_compose, 'services.'.$this->container_name.'.healthcheck'); } if (count($this->application->ports_mappings_array) > 0 && $this->pull_request_id === 0) { $docker_compose['services'][$this->container_name]['ports'] = $this->application->ports_mappings_array; } if (count($persistent_storages) > 0) { - if (! data_get($docker_compose, 'services.' . $this->container_name . '.volumes')) { + if (! data_get($docker_compose, 'services.'.$this->container_name.'.volumes')) { $docker_compose['services'][$this->container_name]['volumes'] = []; } $docker_compose['services'][$this->container_name]['volumes'] = array_merge($docker_compose['services'][$this->container_name]['volumes'], $persistent_storages); } if (count($persistent_file_volumes) > 0) { - if (! data_get($docker_compose, 'services.' . $this->container_name . '.volumes')) { + if (! data_get($docker_compose, 'services.'.$this->container_name.'.volumes')) { $docker_compose['services'][$this->container_name]['volumes'] = []; } $docker_compose['services'][$this->container_name]['volumes'] = array_merge($docker_compose['services'][$this->container_name]['volumes'], $persistent_file_volumes->map(function ($item) { @@ -1856,9 +1856,9 @@ private function generate_local_persistent_volumes() $volume_name = $persistentStorage->name; } if ($this->pull_request_id !== 0) { - $volume_name = $volume_name . '-pr-' . $this->pull_request_id; + $volume_name = $volume_name.'-pr-'.$this->pull_request_id; } - $local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path; + $local_persistent_volumes[] = $volume_name.':'.$persistentStorage->mount_path; } return $local_persistent_volumes; @@ -1874,7 +1874,7 @@ private function generate_local_persistent_volumes_only_volume_names() $name = $persistentStorage->name; if ($this->pull_request_id !== 0) { - $name = $name . '-pr-' . $this->pull_request_id; + $name = $name.'-pr-'.$this->pull_request_id; } $local_persistent_volumes_names[$name] = [ @@ -2152,7 +2152,7 @@ private function stop_running_container(bool $force = false) $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id); if ($this->pull_request_id === 0) { $containers = $containers->filter(function ($container) { - return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== $this->container_name . '-pr-' . $this->pull_request_id; + return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== $this->container_name.'-pr-'.$this->pull_request_id; }); } $containers->each(function ($container) { @@ -2259,8 +2259,8 @@ private function run_pre_deployment_command() foreach ($containers as $container) { $containerName = data_get($container, 'Names'); - if ($containers->count() == 1 || str_starts_with($containerName, $this->application->pre_deployment_command_container . '-' . $this->application->uuid)) { - $cmd = "sh -c '" . str_replace("'", "'\''", $this->application->pre_deployment_command) . "'"; + if ($containers->count() == 1 || str_starts_with($containerName, $this->application->pre_deployment_command_container.'-'.$this->application->uuid)) { + $cmd = "sh -c '".str_replace("'", "'\''", $this->application->pre_deployment_command)."'"; $exec = "docker exec {$containerName} {$cmd}"; $this->execute_remote_command( [ @@ -2286,8 +2286,8 @@ private function run_post_deployment_command() $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id); foreach ($containers as $container) { $containerName = data_get($container, 'Names'); - if ($containers->count() == 1 || str_starts_with($containerName, $this->application->post_deployment_command_container . '-' . $this->application->uuid)) { - $cmd = "sh -c '" . str_replace("'", "'\''", $this->application->post_deployment_command) . "'"; + if ($containers->count() == 1 || str_starts_with($containerName, $this->application->post_deployment_command_container.'-'.$this->application->uuid)) { + $cmd = "sh -c '".str_replace("'", "'\''", $this->application->post_deployment_command)."'"; $exec = "docker exec {$containerName} {$cmd}"; try { $this->execute_remote_command( diff --git a/app/Livewire/Project/Application/General.php b/app/Livewire/Project/Application/General.php index fedbdaabd..3aceb449b 100644 --- a/app/Livewire/Project/Application/General.php +++ b/app/Livewire/Project/Application/General.php @@ -4,7 +4,6 @@ use App\Models\Application; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Log; use Livewire\Component; use Visus\Cuid2\Cuid2; diff --git a/app/Models/Application.php b/app/Models/Application.php index db2708274..abdd339e9 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -142,6 +142,7 @@ protected static function booted() $task->delete(); } $application->tags()->detach(); + $application->previews()->delete(); }); } diff --git a/app/Models/ApplicationPreview.php b/app/Models/ApplicationPreview.php index ff96ef092..abf402377 100644 --- a/app/Models/ApplicationPreview.php +++ b/app/Models/ApplicationPreview.php @@ -12,9 +12,9 @@ class ApplicationPreview extends BaseModel protected static function booted() { static::deleting(function ($preview) { - if ($preview->application->build_pack === 'dockercompose') { + if (data_get($preview, 'application.build_pack') === 'dockercompose') { $server = $preview->application->destination->server; - $composeFile = $preview->application->oldParser(pull_request_id: $preview->pull_request_id); + $composeFile = newParser($preview->application, pull_request_id: $preview->pull_request_id); $volumes = data_get($composeFile, 'volumes'); $networks = data_get($composeFile, 'networks'); $networkKeys = collect($networks)->keys(); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index df64bc41f..968124aba 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -3046,6 +3046,9 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int } else { $mainDirectory = str(base_configuration_dir().'/applications/'.$uuid); $source = replaceLocalSource($source, $mainDirectory); + if ($isApplication && $isPullRequest) { + $source = $source."-pr-$pullRequestId"; + } LocalFileVolume::updateOrCreate( [ @@ -3076,6 +3079,10 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int } $slugWithoutUuid = Str::slug($source, '-'); $name = "{$uuid}_{$slugWithoutUuid}"; + + if ($isApplication && $isPullRequest) { + $name = "{$name}-pr-$pullRequestId"; + } if (is_string($volume)) { $source = str($volume)->before(':'); $target = str($volume)->after(':')->beforeLast(':'); @@ -3106,6 +3113,23 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int $volumesParsed->put($index, $volume); } } + + if ($depends_on?->count() > 0) { + if ($isApplication && $isPullRequest) { + $newDependsOn = collect([]); + $depends_on->each(function ($dependency, $condition) use ($pullRequestId, $newDependsOn) { + if (is_numeric($condition)) { + $dependency = "$dependency-pr-$pullRequestId"; + + $newDependsOn->put($condition, $dependency); + } else { + $condition = "$condition-pr-$pullRequestId"; + $newDependsOn->put($condition, $dependency); + } + }); + $depends_on = $newDependsOn; + } + } if ($topLevel->get('networks')?->count() > 0) { foreach ($topLevel->get('networks') as $networkName => $network) { if ($networkName === 'default') { @@ -3196,7 +3220,6 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int // remove $environment from $allEnvironments $coolifyDefinedEnvironments = $allEnvironments->diffKeys($environment); - // filter magic environments $magicEnvironments = $environment->filter(function ($value, $key) { $value = str(replaceVariables(str($value))); @@ -3340,12 +3363,11 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int } } if ($isApplication) { - $branch = $originalResource->git_branch; + $branch = $originalResource->git_branch; if ($pullRequestId !== 0) { $branch = "pull/{$pullRequestId}/head"; } if ($originalResource->environment_variables->where('key', 'COOLIFY_BRANCH')->isEmpty()) { - ray($branch); $environment->put('COOLIFY_BRANCH', $branch); } } @@ -3371,10 +3393,10 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int $fqdns = collect([]); } } else { - $fqdns = $fqdns->map(function ($fqdn) use ($pullRequestId) { - $preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->id, $pullRequestId); + $fqdns = $fqdns->map(function ($fqdn) use ($pullRequestId, $resource) { + $preview = ApplicationPreview::findPreviewByApplicationAndPullId($resource->id, $pullRequestId); $url = Url::fromString($fqdn); - $template = $this->preview_url_template; + $template = $resource->preview_url_template; $host = $url->getHost(); $schema = $url->getScheme(); $random = new Cuid2; @@ -3420,14 +3442,24 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int if (! $isDatabase && $fqdns?->count() > 0) { if ($isApplication) { $shouldGenerateLabelsExactly = $resource->destination->server->settings->generate_exact_labels; + $uuid = $resource->uuid; + $network = $resource->destination->network; + if ($isPullRequest) { + $uuid = "{$resource->uuid}-{$pullRequestId}"; + } + if ($isPullRequest) { + $network = "{$resource->destination->network}-{$pullRequestId}"; + } } else { $shouldGenerateLabelsExactly = $resource->server->settings->generate_exact_labels; + $uuid = $resource->uuid; + $network = $resource->server->destination->network; } if ($shouldGenerateLabelsExactly) { switch ($server->proxyType()) { case ProxyTypes::TRAEFIK->value: $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik( - uuid: $resource->uuid, + uuid: $uuid, domains: $fqdns, is_force_https_enabled: true, serviceLabels: $serviceLabels, @@ -3439,8 +3471,8 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int break; case ProxyTypes::CADDY->value: $serviceLabels = $serviceLabels->merge(fqdnLabelsForCaddy( - network: $resource->destination->network, - uuid: $resource->uuid, + network: $network, + uuid: $uuid, domains: $fqdns, is_force_https_enabled: true, serviceLabels: $serviceLabels, @@ -3454,7 +3486,7 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int } } else { $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik( - uuid: $resource->uuid, + uuid: $uuid, domains: $fqdns, is_force_https_enabled: true, serviceLabels: $serviceLabels, @@ -3464,8 +3496,8 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int image: $image )); $serviceLabels = $serviceLabels->merge(fqdnLabelsForCaddy( - network: $resource->destination->network, - uuid: $resource->uuid, + network: $network, + uuid: $uuid, domains: $fqdns, is_force_https_enabled: true, serviceLabels: $serviceLabels, diff --git a/tests/Feature/DockerComposeParseTest.php b/tests/Feature/DockerComposeParseTest.php index d4c0f3977..870690aa9 100644 --- a/tests/Feature/DockerComposeParseTest.php +++ b/tests/Feature/DockerComposeParseTest.php @@ -1,6 +1,7 @@ clearAll(); beforeEach(function () { $this->applicationYaml = ' version: "3.8" @@ -62,7 +62,8 @@ 'domain' => 'http://bcoowoookw0co4cok4sgc4k8.127.0.0.1.sslip.io', ], ]), - 'uuid' => 'bcoowoookw0co4cok4sgc4k8', + 'preview_url_template' => '{{pr_id}}.{{domain}}', + 'uuid' => 'bcoowoookw0co4cok4sgc4k8s', 'repository_project_id' => 603035348, 'git_repository' => 'coollabsio/coolify-examples', 'git_branch' => 'main', @@ -77,6 +78,13 @@ 'source_id' => 1, 'source_type' => GithubApp::class, ]); + $this->application->environment_variables_preview()->where('key', 'APP_DEBUG')->update(['value' => 'true']); + $this->applicationPreview = ApplicationPreview::create([ + 'git_type' => 'github', + 'application_id' => $this->application->id, + 'pull_request_id' => 1, + 'pull_request_html_url' => 'https://github.com/coollabsio/coolify-examples/pull/1', + ]); $this->serviceYaml = ' version: "3.8" services: @@ -156,6 +164,7 @@ }); afterEach(function () { + // $this->applicationPreview->forceDelete(); $this->application->forceDelete(); $this->service->forceDelete(); }); @@ -324,11 +333,10 @@ // }); test('ServiceComposeParseNew', function () { - ray()->clearAll(); - $output = newParser($this->service); + $output = newParser($this->application, pull_request_id: 1, preview_id: $this->applicationPreview->id); // ray('New parser'); // ray($output->toArray()); - ray($this->service->environment_variables->pluck('value', 'key')->toArray()); + ray($this->service->environment_variables_preview->pluck('value', 'key')->toArray()); expect($output)->toBeInstanceOf(Collection::class); });