Fix server readiness check in ContainerStatusJob and ServerStatusJob

This commit is contained in:
Andras Bacsai 2023-11-17 14:46:04 +01:00
parent 60171093c5
commit 467471f54a
3 changed files with 13 additions and 6 deletions

View File

@ -39,7 +39,9 @@ public function handle(): void
{
ray("checking container statuses for {$this->server->id}");
try {
$this->server->checkServerRediness();
if (!$this->server->isServerReady()) {
throw new \Exception('Server is not ready.');
};
$containers = instant_remote_process(["docker container ls -q"], $this->server);
if (!$containers) {
return;

View File

@ -34,8 +34,9 @@ public function handle(): void
{
ray("checking server status for {$this->server->id}");
try {
$this->server->checkServerRediness();
$this->cleanup(notify: false);
if ($this->server->isServerReady()) {
$this->cleanup(notify: false);
}
} catch (\Throwable $e) {
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage());

View File

@ -128,13 +128,15 @@ public function skipServer()
}
return false;
}
public function checkServerRediness()
public function isServerReady()
{
$serverUptimeCheckNumber = $this->unreachable_count;
$serverUptimeCheckNumberMax = 3;
$currentTime = now()->timestamp;
$runtime5Minutes = 1 * 60;
$isReady = false;
// Run for 1 minutes max and check every 5 seconds for 3 times
while ($currentTime + $runtime5Minutes > now()->timestamp) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
@ -165,7 +167,8 @@ public function checkServerRediness()
$db->update(['status' => 'exited']);
}
}
throw new \Exception('Server is not reachable.');
$isReady = false;
break;
}
$result = $this->validateConnection();
ray('validateConnection: ' . $result);
@ -177,9 +180,10 @@ public function checkServerRediness()
Sleep::for(5)->seconds();
return;
}
$isReady = true;
break;
}
return true;
return $isReady;
}
public function getDiskUsage()
{