diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 7f8c86b94..fa195aafd 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -11,6 +11,7 @@ use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; +use App\Models\Team; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; @@ -52,6 +53,8 @@ private function check_resources($schedule) { if (isCloud()) { $servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false)->where('ip', '!=', '1.2.3.4'); + $own = Team::find(0)->servers; + $servers = $servers->merge($own); } else { $servers = Server::all()->where('ip', '!=', '1.2.3.4'); } diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 2762b8b54..1c66ef53f 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -39,7 +39,9 @@ public function handle(): void { ray("checking container statuses for {$this->server->id}"); try { - $this->server->checkServerRediness(); + if (!$this->server->isServerReady()) { + return; + }; $containers = instant_remote_process(["docker container ls -q"], $this->server); if (!$containers) { return; @@ -266,7 +268,7 @@ public function handle(): void $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); } } catch (\Throwable $e) { - send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage()); + send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); ray($e->getMessage()); handleError($e); } diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index 7d591bc83..cfe61598f 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -34,8 +34,9 @@ public function handle(): void { ray("checking server status for {$this->server->id}"); try { - $this->server->checkServerRediness(); - $this->cleanup(notify: false); + if ($this->server->isServerReady()) { + $this->cleanup(notify: false); + } } catch (\Throwable $e) { send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); ray($e->getMessage()); diff --git a/app/Models/Server.php b/app/Models/Server.php index 77b80e680..f00f772a8 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -128,15 +128,17 @@ public function skipServer() } return false; } - public function checkServerRediness() + public function isServerReady() { $serverUptimeCheckNumber = $this->unreachable_count; $serverUptimeCheckNumberMax = 3; $currentTime = now()->timestamp; - $runtime5Minutes = 1 * 60; - // Run for 1 minutes max and check every 5 seconds for 3 times - while ($currentTime + $runtime5Minutes > now()->timestamp) { + $runtime = 30; + + $isReady = false; + // Run for 30 seconds max and check every 5 seconds for 3 times + while ($currentTime + $runtime > now()->timestamp) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { if ($this->unreachable_notification_sent === false) { ray('Server unreachable, sending notification...'); @@ -165,7 +167,8 @@ public function checkServerRediness() $db->update(['status' => 'exited']); } } - throw new \Exception('Server is not reachable.'); + $isReady = false; + break; } $result = $this->validateConnection(); ray('validateConnection: ' . $result); @@ -177,9 +180,10 @@ public function checkServerRediness() Sleep::for(5)->seconds(); return; } + $isReady = true; break; } - return true; + return $isReady; } public function getDiskUsage() { diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php index ae100b804..07f801fe8 100644 --- a/app/Notifications/Server/Unreachable.php +++ b/app/Notifications/Server/Unreachable.php @@ -43,7 +43,7 @@ public function via(object $notifiable): array public function toMail(): MailMessage { $mail = new MailMessage(); - $mail->subject("Coolify: Server ({$this->server->name}) is unreachable after trying to connect to it 5 times"); + $mail->subject("Coolify: Server ({$this->server->name}) is unreachable after trying to connect to it 3 times"); $mail->view('emails.server-lost-connection', [ 'name' => $this->server->name, ]); @@ -52,13 +52,13 @@ public function toMail(): MailMessage public function toDiscord(): string { - $message = "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."; + $message = "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 3 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."; return $message; } public function toTelegram(): array { return [ - "message" => "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations." + "message" => "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 3 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations." ]; } } diff --git a/config/sentry.php b/config/sentry.php index 5655b866b..d1111bbd4 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.142', + 'release' => '4.0.0-beta.143', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index 8b7d1633c..318484b79 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@