diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 0decaf0bc..ad663de8b 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -11,9 +11,30 @@ class CleanupDocker public function handle(Server $server) { - // cleanup docker images, containers, and builder caches - instant_remote_process(['docker image prune -af'], $server, false); - instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $server, false); - instant_remote_process(['docker builder prune -af'], $server, false); + + $commands = $this->getCommands($force); + + foreach ($commands as $command) { + instant_remote_process([$command], $server, false); + } + } + + private function getCommands(bool $force): array + { + $commonCommands = [ + 'docker container prune -f --filter "label=coolify.managed=true"', + 'docker image prune -f', + 'docker builder prune -f', + ]; + + if ($force) { + return array_merge([ + 'docker container prune -f --filter "label=coolify.managed=true"', + 'docker image prune -af', + 'docker builder prune -af', + ], $commonCommands); + } + + return $commonCommands; } } diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index e322488a5..12b514c6a 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -35,7 +35,6 @@ protected function schedule(Schedule $schedule): void // Instance Jobs $schedule->command('horizon:snapshot')->everyMinute(); $schedule->job(new CleanupInstanceStuffsJob)->everyMinute()->onOneServer(); - // Server Jobs $this->check_scheduled_backups($schedule); $this->check_resources($schedule); @@ -44,7 +43,7 @@ protected function schedule(Schedule $schedule): void } else { // Instance Jobs $schedule->command('horizon:snapshot')->everyFiveMinutes(); - $schedule->command('cleanup:unreachable-servers')->daily(); + $schedule->command('cleanup:unreachable-servers')->daily()->onOneServer(); $schedule->job(new PullCoolifyImageJob)->cron($settings->update_check_frequency)->onOneServer(); $schedule->job(new PullTemplatesFromCDN)->cron($settings->update_check_frequency)->onOneServer(); $schedule->job(new CleanupInstanceStuffsJob)->everyTwoMinutes()->onOneServer(); @@ -67,7 +66,14 @@ private function pull_images($schedule) $servers = $this->all_servers->where('settings.is_usable', true)->where('settings.is_reachable', true)->where('ip', '!=', '1.2.3.4'); foreach ($servers as $server) { if ($server->isSentinelEnabled()) { - $schedule->job(new PullSentinelImageJob($server))->cron($settings->update_check_frequency)->onOneServer(); + $schedule->job(function () use ($server) { + $sentinel_found = instant_remote_process(['docker inspect coolify-sentinel'], $server, false); + $sentinel_found = json_decode($sentinel_found, true); + $status = data_get($sentinel_found, '0.State.Status', 'exited'); + if ($status !== 'running') { + PullSentinelImageJob::dispatch($server); + } + })->cron($settings->update_check_frequency)->onOneServer(); } $schedule->job(new PullHelperImageJob($server))->cron($settings->update_check_frequency)->onOneServer(); } @@ -97,7 +103,11 @@ private function check_resources($schedule) } foreach ($servers as $server) { $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); - $schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer(); + //The lines below need to be added as soon as timzone is merged!! + //$serverTimezone = $server->settings->server_timezone; + //$schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer(); + $schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->onOneServer(); + } } @@ -108,7 +118,7 @@ private function check_scheduled_backups($schedule) return; } foreach ($scheduled_backups as $scheduled_backup) { - if (! $scheduled_backup->enabled) { + if (!$scheduled_backup->enabled) { continue; } if (is_null(data_get($scheduled_backup, 'database'))) { @@ -140,7 +150,7 @@ private function check_scheduled_tasks($schedule) $service = $scheduled_task->service; $application = $scheduled_task->application; - if (! $application && ! $service) { + if (!$application && !$service) { ray('application/service attached to scheduled task does not exist'); $scheduled_task->delete(); @@ -167,7 +177,7 @@ private function check_scheduled_tasks($schedule) protected function commands(): void { - $this->load(__DIR__.'/Commands'); + $this->load(__DIR__ . '/Commands'); require base_path('routes/console.php'); } diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 163b579eb..e53b54f5b 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -27,20 +27,25 @@ public function __construct(public Server $server) {} public function handle(): void { - if (! $this->server->isFunctional()) { - return; - } - if ($this->server->settings->is_force_cleanup_enabled) { - Log::info('DockerCleanupJob force cleanup on '.$this->server->name); - CleanupDocker::run(server: $this->server); + try { + if (! $this->server->isFunctional()) { + return; + } + if ($this->server->settings->force_docker_cleanup) { + Log::info('DockerCleanupJob force cleanup on ' . $this->server->name); + CleanupDocker::run(server: $this->server, force: true); + + return; + } + return; } try { $this->usageBefore = $this->server->getDiskUsage(); - if (str($this->usageBefore)->trim()->isEmpty()) { - Log::info('DockerCleanupJob force cleanup on '.$this->server->name); - CleanupDocker::run(server: $this->server); + if (str($this->usageBefore)->isEmpty() || $this->usageBefore === null || $this->usageBefore === 0) { + Log::info('DockerCleanupJob force cleanup on ' . $this->server->name); + CleanupDocker::run(server: $this->server, force: true); return; } @@ -48,13 +53,13 @@ public function handle(): void CleanupDocker::run(server: $this->server); $usageAfter = $this->server->getDiskUsage(); if ($usageAfter < $this->usageBefore) { - $this->server->team?->notify(new DockerCleanup($this->server, 'Saved '.($this->usageBefore - $usageAfter).'% disk space.')); - Log::info('DockerCleanupJob done: Saved '.($this->usageBefore - $usageAfter).'% disk space on '.$this->server->name); + $this->server->team?->notify(new DockerCleanup($this->server, 'Saved ' . ($this->usageBefore - $usageAfter) . '% disk space.')); + Log::info('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); } else { - Log::info('DockerCleanupJob failed to save disk space on '.$this->server->name); + Log::info('DockerCleanupJob failed to save disk space on ' . $this->server->name); } } else { - Log::info('No need to clean up '.$this->server->name); + Log::info('No need to clean up ' . $this->server->name); } } catch (\Throwable $e) { CleanupDocker::run(server: $this->server); @@ -62,4 +67,4 @@ public function handle(): void throw $e; } } -} +} \ No newline at end of file diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index 1db15cfd4..703b199c5 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -79,7 +79,6 @@ public function handle() } GetContainersStatus::run($this->server, $this->containers, $containerReplicates); $this->checkLogDrainContainer(); - $this->checkSentinel(); } } catch (\Throwable $e) { @@ -90,21 +89,6 @@ public function handle() } - private function checkSentinel() - { - if ($this->server->isSentinelEnabled()) { - $sentinelContainerFound = $this->containers->filter(function ($value, $key) { - return data_get($value, 'Name') === '/coolify-sentinel'; - })->first(); - if ($sentinelContainerFound) { - $status = data_get($sentinelContainerFound, 'State.Status'); - if ($status !== 'running') { - PullSentinelImageJob::dispatch($this); - } - } - } - } - private function serverStatus() { ['uptime' => $uptime] = $this->server->validateConnection(); diff --git a/app/Livewire/Server/Form.php b/app/Livewire/Server/Form.php index 9934ea345..7c176c952 100644 --- a/app/Livewire/Server/Form.php +++ b/app/Livewire/Server/Form.php @@ -37,7 +37,6 @@ class Form extends Component 'server.settings.is_swarm_manager' => 'required|boolean', 'server.settings.is_swarm_worker' => 'required|boolean', 'server.settings.is_build_server' => 'required|boolean', - 'server.settings.is_force_cleanup_enabled' => 'required|boolean', 'server.settings.concurrent_builds' => 'required|integer|min:1', 'server.settings.dynamic_timeout' => 'required|integer|min:1', 'server.settings.is_metrics_enabled' => 'required|boolean', @@ -46,6 +45,9 @@ class Form extends Component 'server.settings.metrics_history_days' => 'required|integer|min:1', 'wildcard_domain' => 'nullable|url', 'server.settings.is_server_api_enabled' => 'required|boolean', + 'server.settings.force_docker_cleanup' => 'required|boolean', + 'server.settings.docker_cleanup_frequency' => 'required_if:server.settings.force_docker_cleanup,true|string', + 'server.settings.docker_cleanup_threshold' => 'required_if:server.settings.force_docker_cleanup,false|integer|min:1|max:100', ]; protected $validationAttributes = [ @@ -71,7 +73,19 @@ class Form extends Component public function mount() { $this->wildcard_domain = $this->server->settings->wildcard_domain; - $this->cleanup_after_percentage = $this->server->settings->cleanup_after_percentage; + $this->server->settings->docker_cleanup_threshold = $this->server->settings->docker_cleanup_threshold; + $this->server->settings->docker_cleanup_frequency = $this->server->settings->docker_cleanup_frequency; + } + + public function updated($field) + { + if ($field === 'server.settings.docker_cleanup_frequency') { + $frequency = $this->server->settings->docker_cleanup_frequency; + if (empty($frequency) || !validate_cron_expression($frequency)) { + $this->dispatch('error', 'Invalid Cron / Human expression for Docker Cleanup Frequency. Resetting to default 10 minutes.'); + $this->server->settings->docker_cleanup_frequency = '*/10 * * * *'; + } + } } public function serverInstalled() @@ -116,7 +130,6 @@ public function instantSave() } if ($this->server->settings->isDirty('is_server_api_enabled') && $this->server->settings->is_server_api_enabled === true) { ray('Starting sentinel'); - } } else { ray('Sentinel is not enabled'); @@ -156,7 +169,7 @@ public function checkLocalhostConnection() $this->server->settings->save(); $this->dispatch('proxyStatusUpdated'); } else { - $this->dispatch('error', 'Server is not reachable.', 'Please validate your configuration and connection.

Check this documentation for further help.

Error: '.$error); + $this->dispatch('error', 'Server is not reachable.', 'Please validate your configuration and connection.

Check this documentation for further help.

Error: ' . $error); return; } @@ -172,7 +185,7 @@ public function validateServer($install = true) public function submit() { - if (isCloud() && ! isDev()) { + if (isCloud() && !isDev()) { $this->validate(); $this->validate([ 'server.ip' => 'required', @@ -190,9 +203,13 @@ public function submit() } refresh_server_connection($this->server->privateKey); $this->server->settings->wildcard_domain = $this->wildcard_domain; - $this->server->settings->cleanup_after_percentage = $this->cleanup_after_percentage; + if ($this->server->settings->force_docker_cleanup) { + $this->server->settings->docker_cleanup_frequency = $this->server->settings->docker_cleanup_frequency; + } else { + $this->server->settings->docker_cleanup_threshold = $this->server->settings->docker_cleanup_threshold; + } $this->server->settings->save(); $this->server->save(); $this->dispatch('success', 'Server updated.'); } -} +} \ No newline at end of file diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index c39982b91..6ef7f46ee 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -10,10 +10,10 @@ type: 'object', properties: [ 'id' => ['type' => 'integer'], - 'cleanup_after_percentage' => ['type' => 'integer'], 'concurrent_builds' => ['type' => 'integer'], 'dynamic_timeout' => ['type' => 'integer'], 'force_disabled' => ['type' => 'boolean'], + 'force_server_cleanup' => ['type' => 'boolean'], 'is_build_server' => ['type' => 'boolean'], 'is_cloudflare_tunnel' => ['type' => 'boolean'], 'is_jump_server' => ['type' => 'boolean'], @@ -37,6 +37,8 @@ 'metrics_history_days' => ['type' => 'integer'], 'metrics_refresh_rate_seconds' => ['type' => 'integer'], 'metrics_token' => ['type' => 'string'], + 'docker_cleanup_frequency' => ['type' => 'string'], + 'docker_cleanup_threshold' => ['type' => 'integer'], 'server_id' => ['type' => 'integer'], 'wildcard_domain' => ['type' => 'string'], 'created_at' => ['type' => 'string'], @@ -47,8 +49,13 @@ class ServerSetting extends Model { protected $guarded = []; + protected $casts = [ + 'force_docker_cleanup' => 'boolean', + 'docker_cleanup_threshold' => 'integer', + ]; + public function server() { return $this->belongsTo(Server::class); } -} +} \ No newline at end of file diff --git a/database/migrations/2024_08_09_215659_add_server_cleanup_fields_to_server_settings_table.php b/database/migrations/2024_08_09_215659_add_server_cleanup_fields_to_server_settings_table.php new file mode 100644 index 000000000..4e1bfb357 --- /dev/null +++ b/database/migrations/2024_08_09_215659_add_server_cleanup_fields_to_server_settings_table.php @@ -0,0 +1,46 @@ +boolean('force_docker_cleanup')->default(false); + $table->string('docker_cleanup_frequency')->default('*/10 * * * *'); + $table->integer('docker_cleanup_threshold')->default(80); + + + // Remove old columns + $table->dropColumn('cleanup_after_percentage'); + $table->dropColumn('is_force_cleanup_enabled'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('server_settings', function (Blueprint $table) { + $table->dropColumn('force_docker_cleanup'); + $table->dropColumn('docker_cleanup_frequency'); + $table->dropColumn('docker_cleanup_threshold'); + + + // Add back old columns + $table->integer('cleanup_after_percentage')->default(80); + $table->boolean('force_server_cleanup')->default(false); + }); + } +} diff --git a/resources/views/livewire/server/form.blade.php b/resources/views/livewire/server/form.blade.php index 3c137df91..ee8855321 100644 --- a/resources/views/livewire/server/form.blade.php +++ b/resources/views/livewire/server/form.blade.php @@ -3,73 +3,68 @@

General

@if ($server->id === 0) - - You could lose a lot of functionalities if you change the server details of the server where Coolify - is - running on.
Please think again. -
+ + You could lose a lot of functionalities if you change the server details of the server where Coolify + is + running on.
Please think again. +
@else - Save - @if ($server->isFunctional()) - - Validate & configure - - - - - Revalidate server - - - @endif - @endif -
- @if ($server->isFunctional()) - Server is reachable and validated. - @else - You can't use this server until it is validated. - @endif - @if ((!$server->settings->is_reachable || !$server->settings->is_usable) && $server->id !== 0) + Save + @if ($server->isFunctional()) Validate & configure - + - - Validate Server & Install Docker Engine + + Revalidate server - @if ($server->validation_logs) -

Previous Validation Logs

-
- {!! $server->validation_logs !!} -
@endif + @endif + + @if ($server->isFunctional()) + Server is reachable and validated. + @else + You can't use this server until it is validated. + @endif + @if ((!$server->settings->is_reachable || !$server->settings->is_usable) && $server->id !== 0) + + Validate & configure + + + + + Validate Server & Install Docker Engine + + + @if ($server->validation_logs) +

Previous Validation Logs

+
+ {!! $server->validation_logs !!} +
+ @endif @endif @if ((!$server->settings->is_reachable || !$server->settings->is_usable) && $server->id === 0) - - Validate Server - + + Validate Server + @endif @if ($server->isForceDisabled() && isCloud()) -
The system has disabled the server because you have exceeded the - number of servers for which you have paid.
+
The system has disabled the server because you have exceeded the + number of servers for which you have paid.
@endif
@if (!$server->settings->is_swarm_worker && !$server->settings->is_build_server) - + @endif
- +
@@ -77,106 +72,80 @@ class="w-full mt-8 mb-4 font-bold box-without-bg bg-coollabs hover:bg-coollabs-1
@if ($server->isFunctional()) - @if (!$server->isLocalhost()) - -
-

Cloudflare Tunnels -

- -
- @if ($server->settings->is_cloudflare_tunnel) - - @else - - - - @endif - @if (!$server->isBuildServer()) -

Swarm (experimental)

-
Read the docs here. -
- @if ($server->settings->is_swarm_worker) - - @else - - @endif - - @if ($server->settings->is_swarm_manager) - - @else - - @endif - @endif - @endif + @if (!$server->isLocalhost()) + +
+

Cloudflare Tunnels +

+ +
+ @if ($server->settings->is_cloudflare_tunnel) + @else -
-

Cloudflare Tunnels -

- -
- @if ($server->settings->is_cloudflare_tunnel) - - @else - - - - @endif + + + + @endif + @if (!$server->isBuildServer()) +

Swarm (experimental)

+
Read the docs here. +
+ @if ($server->settings->is_swarm_worker) + + @else + + @endif + + @if ($server->settings->is_swarm_manager) + + @else + + @endif + @endif + @endif + @else +
+

Cloudflare Tunnels +

+ +
+ @if ($server->settings->is_cloudflare_tunnel) + + @else + + + + @endif @endif
@if ($server->isFunctional()) -

Settings

-
-
- @if ($server->settings->is_force_cleanup_enabled) -
- -
- @else - -
- -
- @endif -
-
- - -
+

Settings

+
+
+
+
+ @if ($server->settings->force_docker_cleanup) + + @else + + @endif
-
-

Sentinel

- {{-- @if ($server->isSentinelEnabled()) --}} - {{-- Restart --}} - {{-- @endif --}} +
+ +
-
Metrics are disabled until a few bugs are fixed.
- {{--
+
+
+

Sentinel

+ {{-- @if ($server->isSentinelEnabled()) --}} + {{-- Restart --}} + {{-- @endif --}} +
+
Metrics are disabled until a few bugs are fixed.
+ {{--
@@ -192,4 +161,4 @@ class="w-full mt-8 mb-4 font-bold box-without-bg bg-coollabs hover:bg-coollabs-1
--}} @endif -
+
\ No newline at end of file