fix: better server vlaidation

fix: remove unnecessary coolify.yaml from cloud hosted servers
This commit is contained in:
Andras Bacsai 2024-05-03 10:22:28 +02:00
parent 140ec1f90f
commit f4cb7cea21
3 changed files with 60 additions and 58 deletions

View File

@ -72,13 +72,13 @@ private function check_resources($schedule)
$containerServers = $servers->where('settings.is_swarm_worker', false)->where('settings.is_build_server', false); $containerServers = $servers->where('settings.is_swarm_worker', false)->where('settings.is_build_server', false);
} }
foreach ($containerServers as $server) { foreach ($containerServers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyTwoMinutes()->onOneServer(); $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
if ($server->isLogDrainEnabled()) { if ($server->isLogDrainEnabled()) {
$schedule->job(new CheckLogDrainContainerJob($server))->everyTwoMinutes()->onOneServer(); $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
} }
} }
foreach ($servers as $server) { foreach ($servers as $server) {
$schedule->job(new ServerStatusJob($server))->everyTwoMinutes()->onOneServer(); $schedule->job(new ServerStatusJob($server))->everyMinute()->onOneServer();
} }
} }
private function instance_auto_update($schedule) private function instance_auto_update($schedule)

View File

@ -38,11 +38,12 @@ public function uniqueId(): int
public function handle() public function handle()
{ {
if (!$this->server->isServerReady($this->tries)) { if (!$this->server->isServerReady($this->tries)) {
throw new \RuntimeException('Server is not ready.'); return "Server is not ready yet.";
}; };
try { try {
if ($this->server->isFunctional()) { if ($this->server->isFunctional()) {
$this->cleanup(notify: false); $this->cleanup(notify: false);
$this->removeCoolifyYaml();
} }
} catch (\Throwable $e) { } catch (\Throwable $e) {
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
@ -50,6 +51,16 @@ public function handle()
return handleError($e); return handleError($e);
} }
} }
private function removeCoolifyYaml()
{
// This will remote the coolify.yaml file from the server as it is not needed on cloud servers
if (isCloud()) {
$file = $this->server->proxyPath() . "/dynamic/coolify.yaml";
return instant_remote_process([
"rm -f $file",
], $this->server, false);
}
}
public function cleanup(bool $notify = false): void public function cleanup(bool $notify = false): void
{ {
$this->disk_usage = $this->server->getDiskUsage(); $this->disk_usage = $this->server->getDiskUsage();

View File

@ -10,6 +10,7 @@
use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Sleep;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str; use Illuminate\Support\Str;
@ -239,7 +240,7 @@ public function setupDynamicProxyConfiguration()
$dynamic_config_path = $this->proxyPath() . "/dynamic"; $dynamic_config_path = $this->proxyPath() . "/dynamic";
if ($this->proxyType() === 'TRAEFIK_V2') { if ($this->proxyType() === 'TRAEFIK_V2') {
$file = "$dynamic_config_path/coolify.yaml"; $file = "$dynamic_config_path/coolify.yaml";
if (empty($settings->fqdn)) { if (empty($settings->fqdn) || isCloud()) {
instant_remote_process([ instant_remote_process([
"rm -f $file", "rm -f $file",
], $this); ], $this);
@ -467,63 +468,53 @@ public function isServerReady(int $tries = 3)
if ($this->skipServer()) { if ($this->skipServer()) {
return false; return false;
} }
$serverUptimeCheckNumber = $this->unreachable_count; $checkIteration = 1;
if ($this->unreachable_count < $tries) { $isServerReady = false;
$serverUptimeCheckNumber = $this->unreachable_count + 1; while ($checkIteration < $tries) {
} ['uptime' => $uptime] = $this->validateConnection();
if ($this->unreachable_count > $tries) { if ($uptime) {
$serverUptimeCheckNumber = $tries; if ($this->unreachable_notification_sent === true) {
} $this->update(['unreachable_notification_sent' => false]);
$serverUptimeCheckNumberMax = $tries;
// ray('server: ' . $this->name);
// ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
// ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax);
['uptime' => $uptime] = $this->validateConnection();
if ($uptime) {
if ($this->unreachable_notification_sent === true) {
$this->update(['unreachable_notification_sent' => false]);
}
return true;
} else {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
// Reached max number of retries
if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...');
$this->team?->notify(new Unreachable($this));
$this->update(['unreachable_notification_sent' => true]);
} }
if ($this->settings->is_reachable === true) { $this->settings()->update([
$this->settings()->update([ 'is_reachable' => true,
'is_reachable' => false,
]);
}
foreach ($this->applications() as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->databases() as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->services()->get() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
} else {
$this->update([
'unreachable_count' => $this->unreachable_count + 1,
]); ]);
$isServerReady = true;
break;
} else {
ray('Server is not ready yet.');
$checkIteration++;
Sleep::for(10)->seconds();
} }
return false;
} }
if ($isServerReady) {
return $isServerReady;
}
if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...');
$this->team?->notify(new Unreachable($this));
$this->update(['unreachable_notification_sent' => true]);
}
$this->settings()->update([
'is_reachable' => false,
]);
foreach ($this->applications() as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->databases() as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->services()->get() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
return false;
} }
public function getDiskUsage() public function getDiskUsage()
{ {