cupcakearmy/coolify
1
0
Fork 0
mirror of https://github.com/cupcakearmy/coolify.git synced 2024-10-22 08:04:19 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: new server checking job

Browse Source 
feat: show if the server  has problems on ui
This commit is contained in:
Andras Bacsai 2024-08-05 15:48:15 +02:00
parent bc48b42ff1
commit e4b92bb660
19 changed files with 539 additions and 383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

588
app/Actions/Docker/GetContainersStatus.php
View File

@ -12,6 +12,7 @@
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Container\ContainerStopped;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Lorisleiva\Actions\Concerns\AsAction;
class GetContainersStatus
@ -20,13 +21,16 @@ class GetContainersStatus
public $applications;
public ?Collection $containers;
public ?Collection $containerReplicates;
public $server;
public function handle(Server $server)
public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null)
{
// if (isDev()) {
// $server = Server::find(0);
// }
$this->containers = $containers;
$this->containerReplicates = $containerReplicates;
$this->server = $server;
if (! $this->server->isFunctional()) {
return 'Server is not ready.';
@ -66,322 +70,312 @@ public function handle(Server $server)
// }
}
private function sentinel()
{
try {
$containers = $this->server->getContainers();
if ($containers->count() === 0) {
return;
}
$databases = $this->server->databases();
$services = $this->server->services()->get();
$previews = $this->server->previews();
$foundApplications = [];
$foundApplicationPreviews = [];
$foundDatabases = [];
$foundServices = [];
// private function sentinel()
// {
// try {
// $this->containers = $this->server->getContainersWithSentinel();
// if ($this->containers->count() === 0) {
// return;
// }
// $databases = $this->server->databases();
// $services = $this->server->services()->get();
// $previews = $this->server->previews();
// $foundApplications = [];
// $foundApplicationPreviews = [];
// $foundDatabases = [];
// $foundServices = [];
foreach ($containers as $container) {
$labels = Arr::undot(data_get($container, 'labels'));
$containerStatus = data_get($container, 'state');
$containerHealth = data_get($container, 'health_status', 'unhealthy');
$containerStatus = "$containerStatus ($containerHealth)";
$applicationId = data_get($labels, 'coolify.applicationId');
if ($applicationId) {
$pullRequestId = data_get($labels, 'coolify.pullRequestId');
if ($pullRequestId) {
if (str($applicationId)->contains('-')) {
$applicationId = str($applicationId)->before('-');
}
$preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first();
if ($preview) {
$foundApplicationPreviews[] = $preview->id;
$statusFromDb = $preview->status;
if ($statusFromDb !== $containerStatus) {
$preview->update(['status' => $containerStatus]);
}
} else {
//Notify user that this container should not be there.
}
} else {
$application = $this->applications->where('id', $applicationId)->first();
if ($application) {
$foundApplications[] = $application->id;
$statusFromDb = $application->status;
if ($statusFromDb !== $containerStatus) {
$application->update(['status' => $containerStatus]);
}
} else {
//Notify user that this container should not be there.
}
}
} else {
$uuid = data_get($labels, 'com.docker.compose.service');
$type = data_get($labels, 'coolify.type');
if ($uuid) {
if ($type === 'service') {
$database_id = data_get($labels, 'coolify.service.subId');
if ($database_id) {
$service_db = ServiceDatabase::where('id', $database_id)->first();
if ($service_db) {
$uuid = $service_db->service->uuid;
$isPublic = data_get($service_db, 'is_public');
if ($isPublic) {
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
if ($this->server->isSwarm()) {
// TODO: fix this with sentinel
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
} else {
return data_get($value, 'name') === "$uuid-proxy";
}
})->first();
if (! $foundTcpProxy) {
StartDatabaseProxy::run($service_db);
// $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server));
}
}
}
}
} else {
$database = $databases->where('uuid', $uuid)->first();
if ($database) {
$isPublic = data_get($database, 'is_public');
$foundDatabases[] = $database->id;
$statusFromDb = $database->status;
if ($statusFromDb !== $containerStatus) {
$database->update(['status' => $containerStatus]);
}
if ($isPublic) {
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
if ($this->server->isSwarm()) {
// TODO: fix this with sentinel
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
} else {
return data_get($value, 'name') === "$uuid-proxy";
}
})->first();
if (! $foundTcpProxy) {
StartDatabaseProxy::run($database);
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
}
}
} else {
// Notify user that this container should not be there.
}
}
}
if (data_get($container, 'name') === 'coolify-db') {
$foundDatabases[] = 0;
}
}
$serviceLabelId = data_get($labels, 'coolify.serviceId');
if ($serviceLabelId) {
$subType = data_get($labels, 'coolify.service.subType');
$subId = data_get($labels, 'coolify.service.subId');
$service = $services->where('id', $serviceLabelId)->first();
if (! $service) {
continue;
}
if ($subType === 'application') {
$service = $service->applications()->where('id', $subId)->first();
} else {
$service = $service->databases()->where('id', $subId)->first();
}
if ($service) {
$foundServices[] = "$service->id-$service->name";
$statusFromDb = $service->status;
if ($statusFromDb !== $containerStatus) {
// ray('Updating status: ' . $containerStatus);
$service->update(['status' => $containerStatus]);
}
}
}
}
$exitedServices = collect([]);
foreach ($services as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
if (in_array("$app->id-$app->name", $foundServices)) {
continue;
} else {
$exitedServices->push($app);
}
}
foreach ($dbs as $db) {
if (in_array("$db->id-$db->name", $foundServices)) {
continue;
} else {
$exitedServices->push($db);
}
}
}
$exitedServices = $exitedServices->unique('id');
foreach ($exitedServices as $exitedService) {
if (str($exitedService->status)->startsWith('exited')) {
continue;
}
$name = data_get($exitedService, 'name');
$fqdn = data_get($exitedService, 'fqdn');
if ($name) {
if ($fqdn) {
$containerName = "$name, available at $fqdn";
} else {
$containerName = $name;
}
} else {
if ($fqdn) {
$containerName = $fqdn;
} else {
$containerName = null;
}
}
$projectUuid = data_get($service, 'environment.project.uuid');
$serviceUuid = data_get($service, 'uuid');
$environmentName = data_get($service, 'environment.name');
// foreach ($this->containers as $container) {
// $labels = Arr::undot(data_get($container, 'labels'));
// $containerStatus = data_get($container, 'state');
// $containerHealth = data_get($container, 'health_status', 'unhealthy');
// $containerStatus = "$containerStatus ($containerHealth)";
// $applicationId = data_get($labels, 'coolify.applicationId');
// if ($applicationId) {
// $pullRequestId = data_get($labels, 'coolify.pullRequestId');
// if ($pullRequestId) {
// if (str($applicationId)->contains('-')) {
// $applicationId = str($applicationId)->before('-');
// }
// $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first();
// if ($preview) {
// $foundApplicationPreviews[] = $preview->id;
// $statusFromDb = $preview->status;
// if ($statusFromDb !== $containerStatus) {
// $preview->update(['status' => $containerStatus]);
// }
// } else {
// //Notify user that this container should not be there.
// }
// } else {
// $application = $this->applications->where('id', $applicationId)->first();
// if ($application) {
// $foundApplications[] = $application->id;
// $statusFromDb = $application->status;
// if ($statusFromDb !== $containerStatus) {
// $application->update(['status' => $containerStatus]);
// }
// } else {
// //Notify user that this container should not be there.
// }
// }
// } else {
// $uuid = data_get($labels, 'com.docker.compose.service');
// $type = data_get($labels, 'coolify.type');
// if ($uuid) {
// if ($type === 'service') {
// $database_id = data_get($labels, 'coolify.service.subId');
// if ($database_id) {
// $service_db = ServiceDatabase::where('id', $database_id)->first();
// if ($service_db) {
// $uuid = $service_db->service->uuid;
// $isPublic = data_get($service_db, 'is_public');
// if ($isPublic) {
// $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
// if ($this->server->isSwarm()) {
// // TODO: fix this with sentinel
// return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
// } else {
// return data_get($value, 'name') === "$uuid-proxy";
// }
// })->first();
// if (! $foundTcpProxy) {
// StartDatabaseProxy::run($service_db);
// // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server));
// }
// }
// }
// }
// } else {
// $database = $databases->where('uuid', $uuid)->first();
// if ($database) {
// $isPublic = data_get($database, 'is_public');
// $foundDatabases[] = $database->id;
// $statusFromDb = $database->status;
// if ($statusFromDb !== $containerStatus) {
// $database->update(['status' => $containerStatus]);
// }
// if ($isPublic) {
// $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
// if ($this->server->isSwarm()) {
// // TODO: fix this with sentinel
// return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
// } else {
// return data_get($value, 'name') === "$uuid-proxy";
// }
// })->first();
// if (! $foundTcpProxy) {
// StartDatabaseProxy::run($database);
// $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
// }
// }
// } else {
// // Notify user that this container should not be there.
// }
// }
// }
// if (data_get($container, 'name') === 'coolify-db') {
// $foundDatabases[] = 0;
// }
// }
// $serviceLabelId = data_get($labels, 'coolify.serviceId');
// if ($serviceLabelId) {
// $subType = data_get($labels, 'coolify.service.subType');
// $subId = data_get($labels, 'coolify.service.subId');
// $service = $services->where('id', $serviceLabelId)->first();
// if (! $service) {
// continue;
// }
// if ($subType === 'application') {
// $service = $service->applications()->where('id', $subId)->first();
// } else {
// $service = $service->databases()->where('id', $subId)->first();
// }
// if ($service) {
// $foundServices[] = "$service->id-$service->name";
// $statusFromDb = $service->status;
// if ($statusFromDb !== $containerStatus) {
// // ray('Updating status: ' . $containerStatus);
// $service->update(['status' => $containerStatus]);
// }
// }
// }
// }
// $exitedServices = collect([]);
// foreach ($services as $service) {
// $apps = $service->applications()->get();
// $dbs = $service->databases()->get();
// foreach ($apps as $app) {
// if (in_array("$app->id-$app->name", $foundServices)) {
// continue;
// } else {
// $exitedServices->push($app);
// }
// }
// foreach ($dbs as $db) {
// if (in_array("$db->id-$db->name", $foundServices)) {
// continue;
// } else {
// $exitedServices->push($db);
// }
// }
// }
// $exitedServices = $exitedServices->unique('id');
// foreach ($exitedServices as $exitedService) {
// if (str($exitedService->status)->startsWith('exited')) {
// continue;
// }
// $name = data_get($exitedService, 'name');
// $fqdn = data_get($exitedService, 'fqdn');
// if ($name) {
// if ($fqdn) {
// $containerName = "$name, available at $fqdn";
// } else {
// $containerName = $name;
// }
// } else {
// if ($fqdn) {
// $containerName = $fqdn;
// } else {
// $containerName = null;
// }
// }
// $projectUuid = data_get($service, 'environment.project.uuid');
// $serviceUuid = data_get($service, 'uuid');
// $environmentName = data_get($service, 'environment.name');
if ($projectUuid && $serviceUuid && $environmentName) {
$url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid;
} else {
$url = null;
}
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
$exitedService->update(['status' => 'exited']);
}
// if ($projectUuid && $serviceUuid && $environmentName) {
// $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid;
// } else {
// $url = null;
// }
// // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
// $exitedService->update(['status' => 'exited']);
// }
$notRunningApplications = $this->applications->pluck('id')->diff($foundApplications);
foreach ($notRunningApplications as $applicationId) {
$application = $this->applications->where('id', $applicationId)->first();
if (str($application->status)->startsWith('exited')) {
continue;
}
$application->update(['status' => 'exited']);
// $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications);
// foreach ($notRunningApplications as $applicationId) {
// $application = $this->applications->where('id', $applicationId)->first();
// if (str($application->status)->startsWith('exited')) {
// continue;
// }
// $application->update(['status' => 'exited']);
$name = data_get($application, 'name');
$fqdn = data_get($application, 'fqdn');
// $name = data_get($application, 'name');
// $fqdn = data_get($application, 'fqdn');
$containerName = $name ? "$name ($fqdn)" : $fqdn;
// $containerName = $name ? "$name ($fqdn)" : $fqdn;
$projectUuid = data_get($application, 'environment.project.uuid');
$applicationUuid = data_get($application, 'uuid');
$environment = data_get($application, 'environment.name');
// $projectUuid = data_get($application, 'environment.project.uuid');
// $applicationUuid = data_get($application, 'uuid');
// $environment = data_get($application, 'environment.name');
if ($projectUuid && $applicationUuid && $environment) {
$url = base_url().'/project/'.$projectUuid.'/'.$environment.'/application/'.$applicationUuid;
} else {
$url = null;
}
// if ($projectUuid && $applicationUuid && $environment) {
// $url = base_url().'/project/'.$projectUuid.'/'.$environment.'/application/'.$applicationUuid;
// } else {
// $url = null;
// }
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
}
$notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
foreach ($notRunningApplicationPreviews as $previewId) {
$preview = $previews->where('id', $previewId)->first();
if (str($preview->status)->startsWith('exited')) {
continue;
}
$preview->update(['status' => 'exited']);
// // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
// }
// $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
// foreach ($notRunningApplicationPreviews as $previewId) {
// $preview = $previews->where('id', $previewId)->first();
// if (str($preview->status)->startsWith('exited')) {
// continue;
// }
// $preview->update(['status' => 'exited']);
$name = data_get($preview, 'name');
$fqdn = data_get($preview, 'fqdn');
// $name = data_get($preview, 'name');
// $fqdn = data_get($preview, 'fqdn');
$containerName = $name ? "$name ($fqdn)" : $fqdn;
// $containerName = $name ? "$name ($fqdn)" : $fqdn;
$projectUuid = data_get($preview, 'application.environment.project.uuid');
$environmentName = data_get($preview, 'application.environment.name');
$applicationUuid = data_get($preview, 'application.uuid');
// $projectUuid = data_get($preview, 'application.environment.project.uuid');
// $environmentName = data_get($preview, 'application.environment.name');
// $applicationUuid = data_get($preview, 'application.uuid');
if ($projectUuid && $applicationUuid && $environmentName) {
$url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/application/'.$applicationUuid;
} else {
$url = null;
}
// if ($projectUuid && $applicationUuid && $environmentName) {
// $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/application/'.$applicationUuid;
// } else {
// $url = null;
// }
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
}
$notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
foreach ($notRunningDatabases as $database) {
$database = $databases->where('id', $database)->first();
if (str($database->status)->startsWith('exited')) {
continue;
}
$database->update(['status' => 'exited']);
// // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
// }
// $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
// foreach ($notRunningDatabases as $database) {
// $database = $databases->where('id', $database)->first();
// if (str($database->status)->startsWith('exited')) {
// continue;
// }
// $database->update(['status' => 'exited']);
$name = data_get($database, 'name');
$fqdn = data_get($database, 'fqdn');
// $name = data_get($database, 'name');
// $fqdn = data_get($database, 'fqdn');
$containerName = $name;
// $containerName = $name;
$projectUuid = data_get($database, 'environment.project.uuid');
$environmentName = data_get($database, 'environment.name');
$databaseUuid = data_get($database, 'uuid');
// $projectUuid = data_get($database, 'environment.project.uuid');
// $environmentName = data_get($database, 'environment.name');
// $databaseUuid = data_get($database, 'uuid');
if ($projectUuid && $databaseUuid && $environmentName) {
$url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid;
} else {
$url = null;
}
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
}
// if ($projectUuid && $databaseUuid && $environmentName) {
// $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid;
// } else {
// $url = null;
// }
// // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
// }
// Check if proxy is running
$this->server->proxyType();
$foundProxyContainer = $containers->filter(function ($value, $key) {
if ($this->server->isSwarm()) {
// TODO: fix this with sentinel
return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
} else {
return data_get($value, 'name') === 'coolify-proxy';
}
})->first();
if (! $foundProxyContainer) {
try {
$shouldStart = CheckProxy::run($this->server);
if ($shouldStart) {
StartProxy::run($this->server, false);
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
}
} catch (\Throwable $e) {
ray($e);
}
} else {
$this->server->proxy->status = data_get($foundProxyContainer, 'state');
$this->server->save();
$connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
}
} catch (\Exception $e) {
// send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
ray($e->getMessage());
// // Check if proxy is running
// $this->server->proxyType();
// $foundProxyContainer = $this->containers->filter(function ($value, $key) {
// if ($this->server->isSwarm()) {
// // TODO: fix this with sentinel
// return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
// } else {
// return data_get($value, 'name') === 'coolify-proxy';
// }
// })->first();
// if (! $foundProxyContainer) {
// try {
// $shouldStart = CheckProxy::run($this->server);
// if ($shouldStart) {
// StartProxy::run($this->server, false);
// $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
// }
// } catch (\Throwable $e) {
// ray($e);
// }
// } else {
// $this->server->proxy->status = data_get($foundProxyContainer, 'state');
// $this->server->save();
// $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
// instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
// }
// } catch (\Exception $e) {
// // send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
// ray($e->getMessage());
return handleError($e);
}
}
// return handleError($e);
// }
// }
private function old_way()
{
if ($this->server->isSwarm()) {
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
} else {
// Precheck for containers
$containers = instant_remote_process(['docker container ls -q'], $this->server, false);
if (! $containers) {
return;
}
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false);
$containerReplicates = null;
if ($this->containers === null) {
['containers' => $this->containers,'containerReplicates' => $this->containerReplicates] = $this->server->getContainers();
}
if (is_null($containers)) {
if (is_null($this->containers)) {
return;
}
$containers = format_docker_command_output_to_json($containers);
if ($containerReplicates) {
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
foreach ($containerReplicates as $containerReplica) {
if ($this->containerReplicates) {
foreach ($this->containerReplicates as $containerReplica) {
$name = data_get($containerReplica, 'Name');
$containers = $containers->map(function ($container) use ($name, $containerReplica) {
$this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) {
if (data_get($container, 'Spec.Name') === $name) {
$replicas = data_get($containerReplica, 'Replicas');
$running = str($replicas)->explode('/')[0];
@ -407,7 +401,7 @@ private function old_way()
$foundDatabases = [];
$foundServices = [];
foreach ($containers as $container) {
foreach ($this->containers as $container) {
if ($this->server->isSwarm()) {
$labels = data_get($container, 'Spec.Labels');
$uuid = data_get($labels, 'coolify.name');
@ -461,7 +455,7 @@ private function old_way()
if ($uuid) {
$isPublic = data_get($service_db, 'is_public');
if ($isPublic) {
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
$foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
if ($this->server->isSwarm()) {
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
} else {
@ -486,7 +480,7 @@ private function old_way()
$database->update(['status' => $containerStatus]);
}
if ($isPublic) {
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
$foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
if ($this->server->isSwarm()) {
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
} else {
@ -659,7 +653,7 @@ private function old_way()
// Check if proxy is running
$this->server->proxyType();
$foundProxyContainer = $containers->filter(function ($value, $key) {
$foundProxyContainer = $this->containers->filter(function ($value, $key) {
if ($this->server->isSwarm()) {
return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
} else {

22
app/Console/Kernel.php
View File

@ -12,6 +12,7 @@
use App\Jobs\PullSentinelImageJob;
use App\Jobs\PullTemplatesFromCDN;
use App\Jobs\ScheduledTaskJob;
use App\Jobs\ServerCheckJob;
use App\Jobs\ServerStatusJob;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledTask;
@ -34,7 +35,8 @@ protected function schedule(Schedule $schedule): void
$schedule->job(new PullTemplatesFromCDN)->everyTwoHours()->onOneServer();
// Server Jobs
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
$this->checkResourcesNew($schedule);
// $this->check_resources($schedule);
$this->check_scheduled_backups($schedule);
$this->check_scheduled_tasks($schedule);
$schedule->command('uploads:clear')->everyTwoMinutes();
@ -49,7 +51,8 @@ protected function schedule(Schedule $schedule): void
// Server Jobs
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
$this->checkResourcesNew($schedule);
// $this->check_resources($schedule);
$this->pull_images($schedule);
$this->check_scheduled_tasks($schedule);
@ -69,6 +72,21 @@ private function pull_images($schedule)
}
}
private function checkResourcesNew($schedule)
{
if (isCloud()) {
$servers = $this->all_servers->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false)->where('ip', '!=', '1.2.3.4');
$own = Team::find(0)->servers;
$servers = $servers->merge($own);
} else {
$servers = $this->all_servers->where('ip', '!=', '1.2.3.4');
}
foreach ($servers as $server) {
$schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer();
$schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer();
}
}
private function check_resources($schedule)
{
if (isCloud()) {

82
app/Jobs/ServerCheckJob.php
View File

@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Docker\GetContainersStatus;
use App\Actions\Proxy\CheckProxy;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\InstallLogDrain;
@ -15,7 +16,6 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
@ -42,20 +42,24 @@ public function backoff(): int
public function __construct(public Server $server) {}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->uuid))];
}
// public function middleware(): array
// {
// return [(new WithoutOverlapping($this->server->uuid))];
// }
public function uniqueId(): int
{
return $this->server->uuid;
}
// public function uniqueId(): int
// {
// return $this->server->uuid;
// }
public function handle()
{
try {
$this->applications = $this->server->applications();
$this->databases = $this->server->databases();
$this->services = $this->server->services()->get();
$this->previews = $this->server->previews();
$up = $this->serverStatus();
if (! $up) {
ray('Server is not reachable.');
@ -67,14 +71,17 @@ public function handle()
return 'Server is not ready.';
}
$this->checkSentinel();
$this->getContainers();
if (is_null($this->containers)) {
return 'No containers found.';
if (! $this->server->isSwarmWorker() && ! $this->server->isBuildServer()) {
ray('Server is not a worker or build server.');
['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers();
if (is_null($this->containers)) {
return 'No containers found.';
}
GetContainersStatus::run($this->server, $this->containers, $containerReplicates);
// $this->containerStatus();
$this->checkLogDrainContainer();
$this->checkSentinel();
}
$this->checkLogDrainContainer();
$this->containerStatus();
} catch (\Throwable $e) {
ray($e->getMessage());
@ -108,6 +115,7 @@ private function serverStatus()
$this->server->update(['unreachable_notification_sent' => false]);
}
} else {
// $this->server->team?->notify(new Unreachable($this->server));
foreach ($this->applications as $application) {
$application->update(['status' => 'exited']);
}
@ -159,49 +167,9 @@ private function checkLogDrainContainer()
}
}
private function getContainers()
{
if ($this->server->isSwarm()) {
$this->containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
$this->containers = format_docker_command_output_to_json($this->containers);
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
if ($containerReplicates) {
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
foreach ($containerReplicates as $containerReplica) {
$name = data_get($containerReplica, 'Name');
$this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) {
if (data_get($container, 'Spec.Name') === $name) {
$replicas = data_get($containerReplica, 'Replicas');
$running = str($replicas)->explode('/')[0];
$total = str($replicas)->explode('/')[1];
if ($running === $total) {
data_set($container, 'State.Status', 'running');
data_set($container, 'State.Health.Status', 'healthy');
} else {
data_set($container, 'State.Status', 'starting');
data_set($container, 'State.Health.Status', 'unhealthy');
}
}
return $container;
});
}
}
} else {
$this->containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false);
$this->containers = format_docker_command_output_to_json($this->containers);
}
}
private function containerStatus()
{
$this->applications = $this->server->applications();
$this->databases = $this->server->databases();
$this->services = $this->server->services()->get();
$this->previews = $this->server->previews();
$foundApplications = [];
$foundApplicationPreviews = [];
$foundDatabases = [];

24
app/Models/Application.php
View File

@ -104,6 +104,8 @@ class Application extends BaseModel
protected $guarded = [];
protected $appends = ['server_status'];
protected static function booted()
{
static::saving(function ($application) {
@ -466,6 +468,28 @@ public function realStatus()
return $this->getRawOriginal('status');
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
if ($this->additional_servers->count() === 0) {
return $this->destination->server->isFunctional();
} else {
$additional_servers_status = $this->additional_servers->pluck('pivot.status');
$main_server_status = $this->destination->server->isFunctional();
foreach ($additional_servers_status as $status) {
$server_status = str($status)->before(':')->value();
if ($main_server_status !== $server_status) {
return false;
}
}
return true;
}
}
);
}
public function status(): Attribute
{
return Attribute::make(

60
app/Models/Server.php
View File

@ -5,6 +5,7 @@
use App\Actions\Server\InstallDocker;
use App\Enums\ProxyTypes;
use App\Jobs\PullSentinelImageJob;
use App\Notifications\Server\Revived;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Collection;
@ -677,7 +678,49 @@ public function startUnmanaged($id)
return instant_remote_process(["docker start $id"], $this);
}
public function getContainers(): Collection
public function getContainers()
{
$containers = collect([]);
$containerReplicates = collect([]);
if ($this->isSwarm()) {
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false);
$containers = format_docker_command_output_to_json($containers);
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this, false);
if ($containerReplicates) {
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
foreach ($containerReplicates as $containerReplica) {
$name = data_get($containerReplica, 'Name');
$containers = $containers->map(function ($container) use ($name, $containerReplica) {
if (data_get($container, 'Spec.Name') === $name) {
$replicas = data_get($containerReplica, 'Replicas');
$running = str($replicas)->explode('/')[0];
$total = str($replicas)->explode('/')[1];
if ($running === $total) {
data_set($container, 'State.Status', 'running');
data_set($container, 'State.Health.Status', 'healthy');
} else {
data_set($container, 'State.Status', 'starting');
data_set($container, 'State.Health.Status', 'unhealthy');
}
}
return $container;
});
}
}
} else {
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this, false);
$containers = format_docker_command_output_to_json($containers);
$containerReplicates = collect([]);
}
return [
'containers' => $containers ?? collect([]),
'containerReplicates' => $containerReplicates ?? collect([]),
];
}
public function getContainersWithSentinel(): Collection
{
$sentinel_found = instant_remote_process(['docker inspect coolify-sentinel'], $this, false);
$sentinel_found = json_decode($sentinel_found, true);
@ -690,21 +733,6 @@ public function getContainers(): Collection
$containers = data_get(json_decode($containers, true), 'containers', []);
return collect($containers);
} else {
if ($this->isSwarm()) {
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false);
} else {
$containers = instant_remote_process(['docker container ls -q'], $this, false);
if (! $containers) {
return collect([]);
}
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this, false);
}
if (is_null($containers)) {
return collect([]);
}
return format_docker_command_output_to_json($containers);
}
}

12
app/Models/Service.php
View File

@ -2,6 +2,7 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\SoftDeletes;
@ -39,6 +40,8 @@ class Service extends BaseModel
protected $guarded = [];
protected $appends = ['server_status'];
public function isConfigurationChanged(bool $save = false)
{
$domains = $this->applications()->get()->pluck('fqdn')->sort()->toArray();
@ -77,6 +80,15 @@ public function isConfigurationChanged(bool $save = false)
}
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->server->isFunctional();
}
);
}
public function isRunning()
{
return (bool) str($this->status())->contains('running');

11
app/Models/StandaloneClickhouse.php
View File

@ -14,7 +14,7 @@ class StandaloneClickhouse extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type'];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status'];
protected $casts = [
'clickhouse_password' => 'encrypted',
@ -40,6 +40,15 @@ protected static function booted()
});
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function isConfigurationChanged(bool $save = false)
{
$newConfigHash = $this->image.$this->ports_mappings;

11
app/Models/StandaloneDragonfly.php
View File

@ -14,7 +14,7 @@ class StandaloneDragonfly extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type'];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status'];
protected $casts = [
'dragonfly_password' => 'encrypted',
@ -40,6 +40,15 @@ protected static function booted()
});
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function isConfigurationChanged(bool $save = false)
{
$newConfigHash = $this->image.$this->ports_mappings;

11
app/Models/StandaloneKeydb.php
View File

@ -14,7 +14,7 @@ class StandaloneKeydb extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url'];
protected $appends = ['internal_db_url', 'external_db_url', 'server_status'];
protected $casts = [
'keydb_password' => 'encrypted',
@ -40,6 +40,15 @@ protected static function booted()
});
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function isConfigurationChanged(bool $save = false)
{
$newConfigHash = $this->image.$this->ports_mappings.$this->keydb_conf;

11
app/Models/StandaloneMariadb.php
View File

@ -14,7 +14,7 @@ class StandaloneMariadb extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type'];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status'];
protected $casts = [
'mariadb_password' => 'encrypted',
@ -40,6 +40,15 @@ protected static function booted()
});
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function isConfigurationChanged(bool $save = false)
{
$newConfigHash = $this->image.$this->ports_mappings.$this->mariadb_conf;

11
app/Models/StandaloneMongodb.php
View File

@ -14,7 +14,7 @@ class StandaloneMongodb extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type'];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status'];
protected static function booted()
{
@ -44,6 +44,15 @@ protected static function booted()
});
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function isConfigurationChanged(bool $save = false)
{
$newConfigHash = $this->image.$this->ports_mappings.$this->mongo_conf;

11
app/Models/StandaloneMysql.php
View File

@ -14,7 +14,7 @@ class StandaloneMysql extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type'];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status'];
protected $casts = [
'mysql_password' => 'encrypted',
@ -41,6 +41,15 @@ protected static function booted()
});
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function isConfigurationChanged(bool $save = false)
{
$newConfigHash = $this->image.$this->ports_mappings.$this->mysql_conf;

11
app/Models/StandalonePostgresql.php
View File

@ -14,7 +14,7 @@ class StandalonePostgresql extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type'];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status'];
protected $casts = [
'init_scripts' => 'array',
@ -46,6 +46,15 @@ public function workdir()
return database_configuration_dir()."/{$this->uuid}";
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function delete_configurations()
{
$server = data_get($this, 'destination.server');

11
app/Models/StandaloneRedis.php
View File

@ -14,7 +14,7 @@ class StandaloneRedis extends BaseModel
protected $guarded = [];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type'];
protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status'];
protected static function booted()
{
@ -36,6 +36,15 @@ protected static function booted()
});
}
protected function serverStatus(): Attribute
{
return Attribute::make(
get: function () {
return $this->destination->server->isFunctional();
}
);
}
public function isConfigurationChanged(bool $save = false)
{
$newConfigHash = $this->image.$this->ports_mappings.$this->redis_conf;

15
app/Notifications/Server/Revived.php
View File

@ -12,6 +12,7 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Support\Facades\RateLimiter;
class Revived extends Notification implements ShouldQueue
{
@ -44,8 +45,20 @@ public function via(object $notifiable): array
if ($isTelegramEnabled) {
$channels[] = TelegramChannel::class;
}
$executed = RateLimiter::attempt(
'notification-server-revived-'.$this->server->uuid,
1,
function () use ($channels) {
return $channels;
},
7200,
);
return $channels;
if (! $executed) {
return [];
}
return $executed;
}
public function toMail(): MailMessage

15
app/Notifications/Server/Unreachable.php
View File

@ -10,6 +10,7 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Support\Facades\RateLimiter;
class Unreachable extends Notification implements ShouldQueue
{
@ -35,8 +36,20 @@ public function via(object $notifiable): array
if ($isTelegramEnabled) {
$channels[] = TelegramChannel::class;
}
$executed = RateLimiter::attempt(
'notification-server-unreachable-'.$this->server->uuid,
1,
function () use ($channels) {
return $channels;
},
7200,
);
return $channels;
if (! $executed) {
return [];
}
return $executed;
}
public function toMail(): MailMessage

8
resources/views/livewire/project/application/configuration.blade.php
View File

@ -41,6 +41,14 @@
</svg>
</span>
@endif
@if ($application->server_status == false)
<span title="The underlying server(s) has problems.">
<svg class="w-4 h-4 text-error" viewBox="0 0 256 256" xmlns="http://www.w3.org/2000/svg">
<path fill="currentColor"
d="M240.26 186.1L152.81 34.23a28.74 28.74 0 0 0-49.62 0L15.74 186.1a27.45 27.45 0 0 0 0 27.71A28.31 28.31 0 0 0 40.55 228h174.9a28.31 28.31 0 0 0 24.79-14.19a27.45 27.45 0 0 0 .02-27.71m-20.8 15.7a4.46 4.46 0 0 1-4 2.2H40.55a4.46 4.46 0 0 1-4-2.2a3.56 3.56 0 0 1 0-3.73L124 46.2a4.77 4.77 0 0 1 8 0l87.44 151.87a3.56 3.56 0 0 1 .02 3.73M116 136v-32a12 12 0 0 1 24 0v32a12 12 0 0 1-24 0m28 40a16 16 0 1 1-16-16a16 16 0 0 1 16 16" />
</svg>
</span>
@endif
</a>
<a class="menu-item" :class="activeTab === 'scheduled-tasks' && 'menu-item-active'"
@click.prevent="activeTab = 'scheduled-tasks'; window.location.hash = 'scheduled-tasks'"

4
resources/views/livewire/project/resource/index.blade.php
View File

@ -72,6 +72,10 @@ class="items-center justify-center box">+ Add New Resource</a>
</div>
<div class="max-w-full px-4 truncate box-description" x-text="item.description"></div>
<div class="max-w-full px-4 truncate box-description" x-text="item.fqdn"></div>
<template x-if="item.server_status == false">
<div class="px-4 text-xs font-bold text-error">The underlying server has problems
</div>
</template>
</div>
</a>
<div

4
resources/views/livewire/project/shared/destination.blade.php
View File

@ -19,6 +19,9 @@ class="relative flex flex-col bg-white border cursor-default dark:text-white box
<div class="box-description">
Network: {{ data_get($resource, 'destination.network') }}
</div>
@if ($resource->server_status == false)
<div class="text-xs font-bold text-error"> This server has connection problems. </div>
@endif
</div>
@if ($resource?->additional_networks?->count() > 0)
<div class="flex gap-2">
@ -76,7 +79,6 @@ class="absolute bg-error -top-1 -left-1 badge "></div>
@if ($resource->getMorphClass() === 'App\Models\Application' && data_get($resource, 'build_pack') !== 'dockercompose')
@if (count($networks) > 0)
<h4>Choose another server</h4>
<div class="pb-4 description">(experimental) </div>
<div class="grid grid-cols-1 gap-4">
@foreach ($networks as $network)
<div wire:click="addServer('{{ $network->id }}','{{ data_get($network, 'server.id') }}')"