From 1e24ab9146af0c354c71de4a7161b92d90535b0e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 22 Aug 2024 15:05:04 +0200 Subject: [PATCH] fix: parser parser parser --- app/Jobs/ApplicationDeploymentJob.php | 3 +- app/Livewire/Project/Application/General.php | 2 +- app/Livewire/Project/Resource/Create.php | 2 +- app/Models/Application.php | 4 + bootstrap/helpers/shared.php | 213 ++++++++++++++++-- .../project/application/general.blade.php | 3 + tests/Feature/DockerComposeParseTest.php | 45 +++- 7 files changed, 244 insertions(+), 28 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 512b77346..860a7d55c 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -472,7 +472,7 @@ private function deploy_docker_compose_buildpack() return; } - $yaml = Yaml::dump($composeFile->toArray(), 10); + $yaml = Yaml::dump(convertToArray($composeFile), 10); } $this->docker_compose_base64 = base64_encode($yaml); $this->execute_remote_command([ @@ -559,6 +559,7 @@ private function deploy_docker_compose_buildpack() $this->execute_remote_command( [executeInDocker($this->deployment_uuid, $command), 'hidden' => true], ); + $this->write_deployment_configurations(); } } diff --git a/app/Livewire/Project/Application/General.php b/app/Livewire/Project/Application/General.php index cf5f0979a..5fad9caaf 100644 --- a/app/Livewire/Project/Application/General.php +++ b/app/Livewire/Project/Application/General.php @@ -204,7 +204,7 @@ public function loadComposeFile($isInit = false) return; } - $compose = $this->application->parseCompose(); + $this->application->parseCompose(); $this->dispatch('success', 'Docker compose file loaded.'); $this->dispatch('compose_loaded'); $this->dispatch('refreshStorages'); diff --git a/app/Livewire/Project/Resource/Create.php b/app/Livewire/Project/Resource/Create.php index ba33e2f08..4aeddce9f 100644 --- a/app/Livewire/Project/Resource/Create.php +++ b/app/Livewire/Project/Resource/Create.php @@ -19,7 +19,7 @@ public function mount() $destination_uuid = request()->query('destination'); $server_id = request()->query('server_id'); $database_image = request()->query('database_image'); - ray($database_image); + $project = currentTeam()->load(['projects'])->projects->where('uuid', request()->route('project_uuid'))->first(); if (! $project) { return redirect()->route('dashboard'); diff --git a/app/Models/Application.php b/app/Models/Application.php index 8383169a5..e328607d8 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -1106,6 +1106,10 @@ public function parseCompose(int $pull_request_id = 0, ?int $preview_id = null) if (! $this->docker_compose_raw) { return collect([]); } + + // $compose = dockerComposeParserForApplications($this); + + // return $compose; $isNew = false; $isSameDockerComposeFile = false; diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 6a4d47a38..427d3f14d 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -5,6 +5,7 @@ use App\Jobs\ServerFilesFromServerJob; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; +use App\Models\ApplicationPreview; use App\Models\EnvironmentVariable; use App\Models\InstanceSettings; use App\Models\LocalFileVolume; @@ -780,15 +781,16 @@ function replaceLocalSource(Stringable $source, Stringable $replacedWith) return $source; } -function dockerComposeParserForApplications(Application $application, Collection $compose): Collection +function dockerComposeParserForApplications(Application $application): Collection { - $isPullRequest = data_get($application, 'pull_request_id', 0) === 0 ? false : true; + $pullRequestId = data_get($application, 'pull_request_id', 0); + $isPullRequest = $pullRequestId === 0 ? false : true; $uuid = data_get($application, 'uuid'); - $pullRequestId = data_get($application, 'pull_request_id'); $server = data_get($application, 'destination.server'); - - $services = data_get($compose, 'services', collect([])); + $compose = data_get($application, 'docker_compose_raw'); + $yaml = Yaml::parse($compose); + $services = data_get($yaml, 'services', collect([])); $topLevel = collect([ 'volumes' => collect(data_get($compose, 'volumes', [])), 'networks' => collect(data_get($compose, 'networks', [])), @@ -817,11 +819,26 @@ function dockerComposeParserForApplications(Application $application, Collection // Let's loop through the services foreach ($services as $serviceName => $service) { $isDatabase = isDatabaseImage(data_get_str($service, 'image')); + $image = data_get_str($service, 'image'); + $restart = data_get_str($service, 'restart', RESTART_MODE); + $logging = data_get($service, 'logging'); + $healthcheck = data_get($service, 'healthcheck'); + + if ($server->isLogDrainEnabled() && $application->isLogDrainEnabled()) { + $logging = [ + 'driver' => 'fluentd', + 'options' => [ + 'fluentd-address' => 'tcp://127.0.0.1:24224', + 'fluentd-async' => 'true', + 'fluentd-sub-second-precision' => 'true', + ], + ]; + } $volumes = collect(data_get($service, 'volumes', [])); $ports = collect(data_get($service, 'ports', [])); $networks = collect(data_get($service, 'networks', [])); - $dependencies = collect(data_get($service, 'depends_on', [])); + $depends_on = collect(data_get($service, 'depends_on', [])); $labels = collect(data_get($service, 'labels', [])); $environment = collect(data_get($service, 'environment', [])); $buildArgs = collect(data_get($service, 'build.args', [])); @@ -895,7 +912,7 @@ function dockerComposeParserForApplications(Application $application, Collection $source = $source."-pr-$pullRequestId"; } if ( - ! $application->settings->is_preserve_repository_enabled || $foundConfig->is_based_on_git + ! $application?->settings?->is_preserve_repository_enabled || $foundConfig?->is_based_on_git ) { // ray([ // 'fs_path' => $source->value(), @@ -969,12 +986,11 @@ function dockerComposeParserForApplications(Application $application, Collection $volumesParsed->put($index, $volume); } } - if ($topLevel->get('dependencies')?->count() > 0) { + if ($depends_on?->count() > 0) { if ($isPullRequest) { - $topLevel->get('dependencies')->transform(function ($dependency) use ($pullRequestId) { + $depends_on->transform(function ($dependency) use ($pullRequestId) { return "$dependency-pr-$pullRequestId"; }); - data_set($service, 'depends_on', $topLevel->get('dependencies')->toArray()); } } @@ -1087,11 +1103,11 @@ function dockerComposeParserForApplications(Application $application, Collection $fqdn = "$fqdn$path"; } - ray([ - 'key' => $key, - 'value' => $fqdn, - ]); - ray($application->environment_variables()->where('key', $key)->where('application_id', $application->id)->first()); + // ray([ + // 'key' => $key, + // 'value' => $fqdn, + // ]); + // ray($application->environment_variables()->where('key', $key)->where('application_id', $application->id)->first()); $application->environment_variables()->where('key', $key)->where('application_id', $application->id)->firstOrCreate([ 'key' => $key, 'application_id' => $application->id, @@ -1154,20 +1170,179 @@ function dockerComposeParserForApplications(Application $application, Collection $environment = $application->environment_variables()->where('application_id', $application->id)->get()->mapWithKeys(function ($item) { return [$item['key'] => $item['value']]; }); - $parsedServices->put($serviceName, [ + + // Labels + $fqdns = collect([]); + if ($application?->serviceType()) { + $fqdns = generateServiceSpecificFqdns($application); + } else { + $domains = collect(json_decode($application->docker_compose_domains)) ?? collect([]); + if ($domains->count() !== 0) { + $fqdns = data_get($domains, "$serviceName.domain"); + if (! $fqdns) { + $fqdns = collect([]); + } else { + $fqdns = str($fqdns)->explode(','); + if ($isPullRequest) { + $preview = $application->previews()->find($pullRequestId); + $docker_compose_domains = collect(json_decode(data_get($preview, 'docker_compose_domains'))); + if ($docker_compose_domains->count() > 0) { + $found_fqdn = data_get($docker_compose_domains, "$serviceName.domain"); + if ($found_fqdn) { + $fqdns = collect($found_fqdn); + } else { + $fqdns = collect([]); + } + } else { + $fqdns = $fqdns->map(function ($fqdn) use ($pullRequestId) { + $preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->id, $pullRequestId); + $url = Url::fromString($fqdn); + $template = $this->preview_url_template; + $host = $url->getHost(); + $schema = $url->getScheme(); + $random = new Cuid2; + $preview_fqdn = str_replace('{{random}}', $random, $template); + $preview_fqdn = str_replace('{{domain}}', $host, $preview_fqdn); + $preview_fqdn = str_replace('{{pr_id}}', $pullRequestId, $preview_fqdn); + $preview_fqdn = "$schema://$preview_fqdn"; + $preview->fqdn = $preview_fqdn; + $preview->save(); + + return $preview_fqdn; + }); + } + } + $shouldGenerateLabelsExactly = $server->settings->generate_exact_labels; + if ($shouldGenerateLabelsExactly) { + switch ($server->proxyType()) { + case ProxyTypes::TRAEFIK->value: + $labels = $labels->merge( + fqdnLabelsForTraefik( + uuid: $application->uuid, + domains: $fqdns, + serviceLabels: $labels, + generate_unique_uuid: $application->build_pack === 'dockercompose', + image: $image, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled(), + ) + ); + break; + case ProxyTypes::CADDY->value: + $labels = $labels->merge( + fqdnLabelsForCaddy( + network: $application->destination->network, + uuid: $application->uuid, + domains: $fqdns, + serviceLabels: $labels, + image: $image, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled(), + ) + ); + break; + } + } else { + $labels = $labels->merge( + fqdnLabelsForTraefik( + uuid: $application->uuid, + domains: $fqdns, + serviceLabels: $labels, + generate_unique_uuid: $application->build_pack === 'dockercompose', + image: $image, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled(), + ) + ); + $labels = $labels->merge( + fqdnLabelsForCaddy( + network: $application->destination->network, + uuid: $application->uuid, + domains: $fqdns, + serviceLabels: $labels, + image: $image, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled(), + ) + ); + } + } + } + } + + $defaultLabels = defaultLabels( + id: $application->id, + name: $containerName, + pull_request_id: $pullRequestId, + type: 'application'); + $labels = $labels->merge($defaultLabels); + + if ($labels->count() > 0 && $application->settings->is_container_label_escape_enabled) { + $labels = $labels->map(function ($value, $key) { + return escapeDollarSign($value); + }); + } + $payload = [ + 'image' => $image, + 'restart' => $restart, 'container_name' => $containerName, 'volumes' => $volumesParsed, - 'ports' => $ports, 'networks' => $networks_temp, - 'dependencies' => $dependencies, 'labels' => $labels, 'environment' => $environment, - ]); + + ]; + if ($ports->count() > 0) { + $payload['ports'] = $ports; + } + if ($logging) { + $payload['logging'] = $logging; + } + if ($depends_on->count() > 0) { + $payload['depends_on'] = $depends_on; + } + if ($healthcheck) { + $payload['healthcheck'] = $healthcheck; + } + + $parsedServices->put($serviceName, $payload); + } + $topLevel->put('services', $parsedServices); + $customOrder = ['services', 'volumes', 'networks', 'configs', 'secrets']; + + $topLevel = $topLevel->sortBy(function ($value, $key) use ($customOrder) { + return array_search($key, $customOrder); + }); + $application->docker_compose = Yaml::dump(convertToArray($topLevel), 10, 2); + data_forget($application, 'environment_variables'); + data_forget($application, 'environment_variables_preview'); + $application->save(); return $topLevel; } + +function convertToArray($collection) +{ + if ($collection instanceof Collection) { + return $collection->map(function ($item) { + return convertToArray($item); + })->toArray(); + } elseif ($collection instanceof Stringable) { + return (string) $collection; + } elseif (is_array($collection)) { + return array_map(function ($item) { + return convertToArray($item); + }, $collection); + } + + return $collection; +} function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id = 0, ?int $preview_id = null) { if ($resource->getMorphClass() === 'App\Models\Service') { diff --git a/resources/views/livewire/project/application/general.blade.php b/resources/views/livewire/project/application/general.blade.php index 389c9b000..4d242fc6b 100644 --- a/resources/views/livewire/project/application/general.blade.php +++ b/resources/views/livewire/project/application/general.blade.php @@ -254,6 +254,9 @@ class="underline" href="https://coolify.io/docs/knowledge-base/docker/registry" helper="You need to modify the docker compose file." monacoEditorLanguage="yaml" useMonacoEditor /> @else + {{-- --}} diff --git a/tests/Feature/DockerComposeParseTest.php b/tests/Feature/DockerComposeParseTest.php index 9ed9abbdf..c9bae520f 100644 --- a/tests/Feature/DockerComposeParseTest.php +++ b/tests/Feature/DockerComposeParseTest.php @@ -24,7 +24,30 @@ './:/var/www/html', './nginx:/etc/nginx', ], + 'depends_on' => [ + 'db' => [ + 'condition' => 'service_healthy', + ], + ], ], + 'db' => [ + 'image' => 'postgres', + 'environment' => [ + 'POSTGRES_USER' => 'postgres', + 'POSTGRES_PASSWORD' => 'postgres', + ], + 'volumes' => [ + 'dbdata:/var/lib/postgresql/data', + ], + 'healthcheck' => [ + 'test' => ['CMD', 'pg_isready', '-U', 'postgres'], + 'interval' => '2s', + 'timeout' => '10s', + 'retries' => 10, + ], + + ], + ], 'networks' => [ 'default' => [ @@ -32,12 +55,11 @@ ], ], ]; - $this->composeFileString = Yaml::dump($this->composeFile, 4, 2); + $this->composeFileString = Yaml::dump($this->composeFile, 10, 2); $this->jsonComposeFile = json_encode($this->composeFile, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR); $this->application = Application::create([ 'name' => 'Application for tests', - 'fqdn' => 'http://test.com', 'repository_project_id' => 603035348, 'git_repository' => 'coollabsio/coolify-examples', 'git_branch' => 'main', @@ -59,15 +81,26 @@ }); test('ComposeParse', function () { + // expect($this->jsonComposeFile)->toBeJson()->ray(); - expect($this->jsonComposeFile)->toBeJson()->ray(); - - $yaml = Yaml::parse($this->jsonComposeFile); $output = dockerComposeParserForApplications( application: $this->application, - compose: collect($yaml), ); + $outputOld = $this->application->parseCompose(); expect($output)->toBeInstanceOf(Collection::class)->ray(); + expect($outputOld)->toBeInstanceOf(Collection::class)->ray(); + + // Test if image is parsed correctly + $image = data_get_str($output, 'services.app.image'); + expect($image->value())->toBe('nginx'); + + $imageOld = data_get_str($outputOld, 'services.app.image'); + expect($image->value())->toBe($imageOld->value()); + + // Test environment variables are parsed correctly + $environment = data_get_str($output, 'services.app.environment'); + $service_fqdn_app = data_get_str($environment, 'SERVICE_FQDN_APP'); + }); test('DockerBinaryAvailableOnLocalhost', function () {