fix: proxy check, reduce jobs, etc

This commit is contained in:
Andras Bacsai 2023-09-11 22:29:34 +02:00
parent 42daae10c6
commit 8a39a4469a
19 changed files with 146 additions and 214 deletions

View File

@ -2,8 +2,6 @@
namespace App\Actions\Proxy;
use App\Enums\ProxyStatus;
use App\Enums\ProxyTypes;
use App\Models\Server;
use Illuminate\Support\Str;
use Spatie\Activitylog\Models\Activity;
@ -36,11 +34,14 @@ public function __invoke(Server $server): Activity
"echo '####### Creating Docker Compose file...'",
"echo '####### Pulling docker image...'",
'docker compose pull',
"echo '####### Stopping existing proxy...'",
"echo '####### Stopping existing coolify-proxy...'",
'docker compose down -v --remove-orphans',
"lsof -nt -i:80 | xargs -r kill -9",
"lsof -nt -i:443 | xargs -r kill -9",
"echo '####### Starting proxy...'",
"systemctl disable nginx > /dev/null 2>&1 || true",
"systemctl disable apache2 > /dev/null 2>&1 || true",
"systemctl disable apache > /dev/null 2>&1 || true",
"echo '####### Starting coolify-proxy...'",
'docker compose up -d --remove-orphans',
"echo '####### Proxy installed successfully...'"
], $server);

View File

@ -21,7 +21,6 @@
use Illuminate\Console\Command;
use Illuminate\Mail\Message;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Support\Facades\Process;
use Mail;
use Str;

View File

@ -2,6 +2,7 @@
namespace App\Console;
use App\Enums\ProxyTypes;
use App\Jobs\ApplicationContainerStatusJob;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CleanupInstanceStuffsJob;
@ -9,11 +10,11 @@
use App\Jobs\DatabaseContainerStatusJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceAutoUpdateJob;
use App\Jobs\ProxyCheckJob;
use App\Jobs\ResourceStatusJob;
use App\Jobs\ProxyContainerStatusJob;
use App\Models\Application;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@ -24,22 +25,26 @@ protected function schedule(Schedule $schedule): void
{
if (isDev()) {
$schedule->command('horizon:snapshot')->everyMinute();
// $schedule->job(new ResourceStatusJob)->everyMinute();
$schedule->job(new ProxyCheckJob)->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
// $schedule->job(new CheckResaleLicenseJob)->hourly();
$schedule->job(new DockerCleanupJob)->everyOddHour();
} else {
$schedule->command('horizon:snapshot')->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyTenMinutes()->onOneServer();
// $schedule->job(new ResourceStatusJob)->everyMinute()->onOneServer();
$schedule->job(new CheckResaleLicenseJob)->hourly()->onOneServer();
$schedule->job(new ProxyCheckJob)->everyFiveMinutes()->onOneServer();
$schedule->job(new DockerCleanupJob)->everyTenMinutes()->onOneServer();
}
$this->instance_auto_update($schedule);
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
$this->check_proxies($schedule);
}
private function check_proxies($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true)->whereNotNull('proxy.type')->where('proxy.type', '!=', ProxyTypes::NONE->value);
foreach ($servers as $server) {
$schedule->job(new ProxyContainerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function check_resources($schedule)
{
@ -53,7 +58,8 @@ private function check_resources($schedule)
$schedule->job(new DatabaseContainerStatusJob($postgresql))->everyMinute()->onOneServer();
}
}
private function instance_auto_update($schedule){
private function instance_auto_update($schedule)
{
if (isDev()) {
return;
}
@ -74,7 +80,7 @@ private function check_scheduled_backups($schedule)
if (!$scheduled_backup->enabled) {
continue;
}
if (is_null(data_get($scheduled_backup,'database'))) {
if (is_null(data_get($scheduled_backup, 'database'))) {
ray('database not found');
$scheduled_backup->delete();
continue;

View File

@ -14,14 +14,14 @@ class Proxy extends Component
public ?string $selectedProxy = null;
public $proxy_settings = null;
public string|null $redirect_url = null;
public ?string $redirect_url = null;
protected $listeners = ['proxyStatusUpdated', 'saveConfiguration' => 'submit'];
public function mount()
{
$this->selectedProxy = $this->server->proxy->type;
$this->redirect_url = $this->server->proxy->redirect_url;
$this->selectedProxy = data_get($this->server, 'proxy.type');
$this->redirect_url = data_get($this->server, 'proxy.redirect_url');
}
public function proxyStatusUpdated()
@ -69,9 +69,10 @@ public function reset_proxy_configuration()
}
}
public function load_proxy_configuration()
public function loadProxyConfiguration()
{
try {
ray('loadProxyConfiguration');
$this->proxy_settings = resolve(CheckConfigurationSync::class)($this->server);
} catch (\Throwable $e) {
return general_error_handler(err: $e);

View File

@ -10,15 +10,20 @@ class Deploy extends Component
{
public Server $server;
public $proxy_settings = null;
protected $listeners = ['proxyStatusUpdated'];
public function start_proxy()
public function proxyStatusUpdated() {
$this->server->refresh();
}
public function startProxy()
{
if (
$this->server->proxy->last_applied_settings &&
$this->server->proxy->last_saved_settings !== $this->server->proxy->last_applied_settings
) {
$this->emit('saveConfiguration', $this->server);
resolve(SaveConfigurationSync::class)($this->server, $this->proxy_settings);
}
$activity = resolve(StartProxy::class)($this->server);
$this->emit('newMonitorActivity', $activity->id);
}

View File

@ -2,7 +2,6 @@
namespace App\Http\Livewire\Server\Proxy;
use App\Jobs\ProxyContainerStatusJob;
use App\Models\Server;
use Livewire\Component;
@ -10,14 +9,26 @@ class Status extends Component
{
public Server $server;
public function get_status()
protected $listeners = ['proxyStatusUpdated'];
public function proxyStatusUpdated() {
$this->server->refresh();
}
public function getProxyStatus()
{
if (data_get($this->server,'settings.is_usable')) {
dispatch_sync(new ProxyContainerStatusJob(
server: $this->server
));
$this->server->refresh();
$this->emit('proxyStatusUpdated');
try {
if (data_get($this->server, 'settings.is_usable') && data_get($this->server, 'settings.is_reachable')) {
$container = getContainerStatus(server: $this->server, container_id: 'coolify-proxy');
$this->server->proxy->status = $container;
$this->server->save();
$this->emit('proxyStatusUpdated');
}
} catch (\Throwable $e) {
return general_error_handler(err: $e);
}
}
public function getProxyStatusWithNoti() {
$this->emit('success', 'Refreshing proxy status.');
$this->getProxyStatus();
}
}

View File

@ -2,7 +2,7 @@
namespace App\Http\Livewire\Settings;
use App\Jobs\ProxyStartJob;
use App\Jobs\ProxyContainerStatusJob;
use App\Models\InstanceSettings as ModelsInstanceSettings;
use App\Models\Server;
use Livewire\Component;
@ -124,7 +124,7 @@ private function setup_instance_fqdn()
];
}
$this->save_configuration_to_disk($traefik_dynamic_conf, $file);
dispatch(new ProxyStartJob($this->server));
dispatch(new ProxyContainerStatusJob($this->server));
}
}

View File

@ -135,7 +135,7 @@ public function handle(): void
$this->deploy();
}
}
if ($this->application->fqdn) dispatch(new ProxyStartJob($this->server));
if ($this->application->fqdn) dispatch(new ProxyContainerStatusJob($this->server));
$this->next(ApplicationDeploymentStatus::FINISHED->value);
} catch (Exception $e) {
ray($e);

View File

@ -1,46 +0,0 @@
<?php
namespace App\Jobs;
use App\Actions\Proxy\StartProxy;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ProxyCheckJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct()
{
}
public function handle()
{
try {
$container_name = 'coolify-proxy';
$servers = Server::all();
foreach ($servers as $server) {
if (
$server->settings->is_reachable === false || $server->settings->is_usable === false
) {
continue;
}
$status = getContainerStatus(server: $server, container_id: $container_name);
if ($status === 'running') {
continue;
}
if (data_get($server, 'proxy.type')) {
resolve(StartProxy::class)($server);
}
}
} catch (\Throwable $e) {
ray($e->getMessage());
send_internal_notification('ProxyCheckJob failed with: ' . $e->getMessage());
throw $e;
}
}
}

View File

@ -2,6 +2,8 @@
namespace App\Jobs;
use App\Actions\Proxy\StartProxy;
use App\Enums\ProxyStatus;
use App\Enums\ProxyTypes;
use App\Models\Server;
use Illuminate\Bus\Queueable;
@ -11,7 +13,6 @@
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Str;
class ProxyContainerStatusJob implements ShouldQueue, ShouldBeUnique
{
@ -28,29 +29,49 @@ public function __construct(Server $server)
public function middleware(): array
{
return [new WithoutOverlapping($this->server->id)];
return [new WithoutOverlapping($this->server->uuid)];
}
public function uniqueId(): int
public function uniqueId(): string
{
return $this->server->id;
ray($this->server->uuid);
return $this->server->uuid;
}
public function handle(): void
{
try {
$container = getContainerStatus(server: $this->server, all_data: true, container_id: 'coolify-proxy', throwError: false);
$status = data_get($container, 'State.Status');
if ($status && data_get($this->server, 'proxy.status') !== $status) {
$this->server->proxy->status = $status;
if ($this->server->proxy->status === 'running') {
$traefik = $container['Config']['Labels']['org.opencontainers.image.title'];
$version = $container['Config']['Labels']['org.opencontainers.image.version'];
if (isset($version) && isset($traefik) && $traefik === 'Traefik' && Str::of($version)->startsWith('v2')) {
$this->server->proxy->type = ProxyTypes::TRAEFIK_V2->value;
}
$proxyType = data_get($this->server, 'proxy.type');
if ($proxyType === ProxyTypes::NONE->value) {
return;
}
if (is_null($proxyType)) {
if ($this->server->isProxyShouldRun()) {
$this->server->proxy->type = ProxyTypes::TRAEFIK_V2->value;
$this->server->proxy->status = ProxyStatus::EXITED->value;
$this->server->save();
resolve(StartProxy::class)($this->server);
return;
}
}
$container = getContainerStatus(server: $this->server, all_data: true, container_id: 'coolify-proxy', throwError: false);
$containerStatus = data_get($container, 'State.Status');
$databaseContainerStatus = data_get($this->server, 'proxy.status', 'exited');
if ($proxyType !== ProxyTypes::NONE->value) {
if ($containerStatus === 'running') {
$this->server->proxy->status = $containerStatus;
$this->server->save();
return;
}
if ((is_null($containerStatus) ||$containerStatus !== 'running' || $databaseContainerStatus !== 'running' || ($containerStatus && $databaseContainerStatus !== $containerStatus)) && $this->server->isProxyShouldRun()) {
$this->server->proxy->status = $containerStatus;
$this->server->save();
resolve(StartProxy::class)($this->server);
return;
}
$this->server->save();
}
} catch (\Throwable $e) {
if ($e->getCode() === 1) {

View File

@ -1,44 +0,0 @@
<?php
namespace App\Jobs;
use App\Actions\Proxy\StartProxy;
use App\Enums\ProxyStatus;
use App\Enums\ProxyTypes;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ProxyStartJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct(protected Server $server)
{
}
public function handle()
{
try {
$container_name = 'coolify-proxy';
ray('Starting proxy for server: ' . $this->server->name);
$status = getContainerStatus(server: $this->server, container_id: $container_name);
if ($status === 'running') {
return;
}
if (is_null(data_get($this->server, 'proxy.type'))) {
$this->server->proxy->type = ProxyTypes::TRAEFIK_V2->value;
$this->server->proxy->status = ProxyStatus::EXITED->value;
$this->server->save();
}
resolve(StartProxy::class)($this->server);
} catch (\Throwable $e) {
send_internal_notification('ProxyStartJob failed with: ' . $e->getMessage());
ray($e->getMessage());
throw $e;
}
}
}

View File

@ -1,46 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\Application;
use App\Models\StandalonePostgresql;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ResourceStatusJob implements ShouldQueue, ShouldBeUnique
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $applications;
public $postgresqls;
public function __construct()
{
$this->applications = Application::all();
$this->postgresqls = StandalonePostgresql::all();
}
public function handle(): void
{
try {
foreach ($this->applications as $application) {
dispatch(new ApplicationContainerStatusJob(
application: $application,
));
}
foreach ($this->postgresqls as $postgresql) {
dispatch(new DatabaseContainerStatusJob(
database: $postgresql,
));
}
} catch (\Throwable $e) {
send_internal_notification('ResourceStatusJob failed with: ' . $e->getMessage());
ray($e);
throw $e;
}
}
}

View File

@ -30,7 +30,6 @@ protected static function booted()
'server_id' => $server->id,
]);
}
});
static::deleting(function ($server) {
$server->destinations()->each(function ($destination) {
@ -72,7 +71,6 @@ static public function destinationsByServer(string $server_id)
$swarmDocker = collect($server->swarmDockers->all());
return $standaloneDocker->concat($swarmDocker);
}
public function settings()
{
return $this->hasOne(ServerSetting::class);
@ -93,7 +91,8 @@ public function isEmpty()
return false;
}
public function databases() {
public function databases()
{
return $this->destinations()->map(function ($standaloneDocker) {
$postgresqls = $standaloneDocker->postgresqls;
return $postgresqls?->concat([]) ?? collect([]);
@ -137,4 +136,21 @@ public function team()
{
return $this->belongsTo(Team::class);
}
public function isProxyShouldRun()
{
$shouldRun = false;
foreach ($this->applications() as $application) {
if (data_get($application, 'fqdn')) {
$shouldRun = true;
break;
}
}
if ($this->id === 0) {
$settings = InstanceSettings::get();
if (data_get($settings, 'fqdn')) {
$shouldRun = true;
}
}
return $shouldRun;
}
}

View File

@ -13,9 +13,10 @@ class Modal extends Component
*/
public function __construct(
public string $modalId,
public string|null $modalTitle = null,
public string|null $modalBody = null,
public string|null $modalSubmit = null,
public ?string $submitWireAction = null,
public ?string $modalTitle = null,
public ?string $modalBody = null,
public ?string $modalSubmit = null,
public bool $noSubmit = false,
public bool $yesOrNo = false,
public string $action = 'delete'

View File

@ -34,7 +34,8 @@
</form>
@else
<form method="dialog" class="flex flex-col w-11/12 max-w-5xl gap-2 rounded modal-box"
@if(!$noSubmit) wire:submit.prevent='submit' @endif>
@if($submitWireAction) wire:submit.prevent={{$submitWireAction}} @endif
@if(!$noSubmit && !$submitWireAction) wire:submit.prevent='submit' @endif>
@isset($modalTitle)
<h3 class="text-lg font-bold">{{ $modalTitle }}</h3>
@endisset

View File

@ -52,16 +52,15 @@
</x-forms.button>
@endif
@if ($server->settings->is_reachable && !$server->settings->is_usable && $server->id !== 0)
<x-forms.button wire:poll.2000ms='validateServer' class="mt-8 mb-4 box" onclick="installDocker.showModal()" wire:click.prevent='installDocker' isHighlighted>
Install Docker Engine 24.0
<x-forms.button wire:poll.2000ms='validateServer' class="mt-8 mb-4 box" onclick="installDocker.showModal()"
wire:click.prevent='installDocker' isHighlighted>
Install Docker Engine 24.0
</x-forms.button>
@endif
@if ($server->settings->is_usable)
<h3 class="py-4">Settings</h3>
<div class="flex items-center w-64 gap-2">
<x-forms.input id="cleanup_after_percentage" label="Disk Cleanup threshold (%)" required
helper="Disk cleanup job will be executed if disk usage is more than this number." />
</div>
@endif
</form>
<h2 class="pt-4">Danger Zone</h2>

View File

@ -1,7 +1,17 @@
<div>
@if ($server->settings->is_usable)
@if ($server->proxy->type)
<div x-init="$wire.load_proxy_configuration">
@if (data_get($server,'settings.is_usable'))
@if (data_get($server,'proxy.type'))
<x-modal submitWireAction="proxyStatusUpdated" modalId="startProxy">
<x-slot:modalBody>
<livewire:activity-monitor header="Proxy Startup Logs" />
</x-slot:modalBody>
<x-slot:modalSubmit>
<x-forms.button onclick="startProxy.close()" type="submit">
Close
</x-forms.button>
</x-slot:modalSubmit>
</x-modal>
<div x-init="$wire.loadProxyConfiguration">
@if ($selectedProxy === 'TRAEFIK_V2')
<form wire:submit.prevent='submit'>
<div class="flex items-center gap-2">
@ -19,15 +29,12 @@
configurations.
</div>
@endif
<div class="container w-full pb-4 mx-auto">
<livewire:activity-monitor header="Logs" />
</div>
<x-forms.input placeholder="https://coolify.io" id="redirect_url" label="Default Redirect 404"
helper="All urls that has no service available will be redirected to this domain.<span class='text-helper'>You can set to your main marketing page or your social media link.</span>" />
<div wire:loading wire:target="load_proxy_configuration" class="pt-4">
<div wire:loading wire:target="loadProxyConfiguration" class="pt-4">
<x-loading text="Loading proxy configuration..." />
</div>
<div wire:loading.remove wire:target="load_proxy_configuration">
<div wire:loading.remove wire:target="loadProxyConfiguration">
@if ($proxy_settings)
<div class="flex flex-col gap-2 pt-2">
<x-forms.textarea label="Configuration file: traefik.conf" name="proxy_settings"

View File

@ -7,16 +7,7 @@
</p>
</x-slot:modalBody>
</x-modal>
<x-modal yesOrNo modalId="startProxy" modalTitle="Start Proxy" action="start_proxy">
<x-slot:modalBody>
<p>This will start the proxy on this server and
<x-highlighted text="stop any running process that is using port 80 and 443" />.
<br>Please think
again.
</p>
</x-slot:modalBody>
</x-modal>
@if (data_get($server, 'proxy.type'))
@if (is_null(data_get($server, 'proxy.type')) || data_get($server, 'proxy.type') !== 'NONE')
@if (data_get($server, 'proxy.status') === 'running')
<div class="flex gap-4">
<button>
@ -37,8 +28,8 @@ class="flex items-center gap-2 cursor-pointer hover:text-white text-neutral-400"
</x-forms.button>
</div>
@else
<x-forms.button isModal noStyle modalId="startProxy"
class="flex items-center gap-2 cursor-pointer hover:text-white">
<button wire:click='startProxy' onclick="startProxy.showModal()"
class="flex items-center gap-2 cursor-pointer hover:text-white text-neutral-400">
<svg xmlns="http://www.w3.org/2000/svg" class="w-5 h-5 text-warning" viewBox="0 0 24 24"
stroke-width="1.5" stroke="currentColor" fill="none" stroke-linecap="round"
stroke-linejoin="round">
@ -46,7 +37,7 @@ class="flex items-center gap-2 cursor-pointer hover:text-white">
<path d="M7 4v16l13 -8z" />
</svg>
Start Proxy
</x-forms.button>
</button>
@endif
@endif
</div>

View File

@ -1,4 +1,4 @@
<div wire:poll.10000ms="get_status" x-init="$wire.get_status">
<div class="flex gap-2" x-init="$wire.getProxyStatus">
@if ($server->proxy->status === 'running')
<x-status.running text="Proxy Running" />
@elseif ($server->proxy->status === 'restarting')
@ -6,4 +6,13 @@
@else
<x-status.stopped text="Proxy Stopped" />
@endif
<button wire:click.prevent='getProxyStatusWithNoti'><svg class="icon" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<g fill="#FCD44F">
<path
d="M12.079 3v-.75V3Zm-8.4 8.333h-.75h.75Zm0 1.667l-.527.532a.75.75 0 0 0 1.056 0L3.68 13Zm2.209-1.134A.75.75 0 1 0 4.83 10.8l1.057 1.065ZM2.528 10.8a.75.75 0 0 0-1.056 1.065L2.528 10.8Zm16.088-3.408a.75.75 0 1 0 1.277-.786l-1.277.786ZM12.079 2.25c-5.047 0-9.15 4.061-9.15 9.083h1.5c0-4.182 3.42-7.583 7.65-7.583v-1.5Zm-9.15 9.083V13h1.5v-1.667h-1.5Zm1.28 2.2l1.679-1.667L4.83 10.8l-1.68 1.667l1.057 1.064Zm0-1.065L2.528 10.8l-1.057 1.065l1.68 1.666l1.056-1.064Zm15.684-5.86A9.158 9.158 0 0 0 12.08 2.25v1.5a7.658 7.658 0 0 1 6.537 3.643l1.277-.786Z" />
<path fill="#fff"
d="M11.883 21v.75V21Zm8.43-8.333h.75h-.75Zm0-1.667l.528-.533a.75.75 0 0 0-1.055 0l.528.533ZM18.1 12.133a.75.75 0 1 0 1.055 1.067L18.1 12.133Zm3.373 1.067a.75.75 0 1 0 1.054-1.067L21.473 13.2ZM5.318 16.606a.75.75 0 1 0-1.277.788l1.277-.788Zm6.565 5.144c5.062 0 9.18-4.058 9.18-9.083h-1.5c0 4.18-3.43 7.583-7.68 7.583v1.5Zm9.18-9.083V11h-1.5v1.667h1.5Zm-1.277-2.2L18.1 12.133l1.055 1.067l1.686-1.667l-1.055-1.066Zm0 1.066l1.687 1.667l1.054-1.067l-1.686-1.666l-1.055 1.066Zm-15.745 5.86a9.197 9.197 0 0 0 7.841 4.357v-1.5a7.697 7.697 0 0 1-6.564-3.644l-1.277.788Z"
opacity=".5" />
</g>
</svg></button>
</div>