coolify/app/Models/Server.php

<?php

namespace App\Models;

use App\Enums\ApplicationDeploymentStatus;
use App\Enums\ProxyStatus;
use App\Enums\ProxyTypes;
use App\Notifications\Server\Revived;
use App\Notifications\Server\Unreachable;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str;
use Illuminate\Support\Stringable;

class Server extends BaseModel
{
    use SchemalessAttributesTrait;
    public static $batch_counter = 0;

    protected static function booted()
    {
        static::saving(function ($server) {
            $payload = [];
            if ($server->user) {
                $payload['user'] = Str::of($server->user)->trim();
            }
            if ($server->ip) {
                $payload['ip'] = Str::of($server->ip)->trim();
            }
            $server->forceFill($payload);
        });
        static::created(function ($server) {
            ServerSetting::create([
                'server_id' => $server->id,
            ]);
        });
        static::deleting(function ($server) {
            $server->destinations()->each(function ($destination) {
                $destination->delete();
            });
            $server->settings()->delete();
        });
    }

    public $casts = [
        'proxy' => SchemalessAttributes::class,
        'logdrain_axiom_api_key' => 'encrypted',
        'logdrain_newrelic_license_key' => 'encrypted',
    ];
    protected $schemalessAttributes = [
        'proxy',
    ];
    protected $guarded = [];

    static public function isReachable()
    {
        return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true);
    }

    static public function ownedByCurrentTeam(array $select = ['*'])
    {
        $teamId = currentTeam()->id;
        $selectArray = collect($select)->concat(['id']);
        return Server::whereTeamId($teamId)->with('settings', 'swarmDockers', 'standaloneDockers')->select($selectArray->all())->orderBy('name');
    }

    static public function isUsable()
    {
        return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true)->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_swarm_worker', false)->whereRelation('settings', 'is_build_server', false);
    }

    static public function destinationsByServer(string $server_id)
    {
        $server = Server::ownedByCurrentTeam()->get()->where('id', $server_id)->firstOrFail();
        $standaloneDocker = collect($server->standaloneDockers->all());
        $swarmDocker = collect($server->swarmDockers->all());
        return $standaloneDocker->concat($swarmDocker);
    }
    public function settings()
    {
        return $this->hasOne(ServerSetting::class);
    }
    public function addInitialNetwork()
    {
        if ($this->id === 0) {
            if ($this->isSwarm()) {
                SwarmDocker::create([
                    'id' => 0,
                    'name' => 'coolify',
                    'network' => 'coolify-overlay',
                    'server_id' => $this->id,
                ]);
            } else {
                StandaloneDocker::create([
                    'id' => 0,
                    'name' => 'coolify',
                    'network' => 'coolify',
                    'server_id' => $this->id,
                ]);
            }
        } else {
            if ($this->isSwarm()) {
                SwarmDocker::create([
                    'name' => 'coolify-overlay',
                    'network' => 'coolify-overlay',
                    'server_id' => $this->id,
                ]);
            } else {
                StandaloneDocker::create([
                    'name' => 'coolify',
                    'network' => 'coolify',
                    'server_id' => $this->id,
                ]);
            }
        }
    }
    public function proxyType()
    {
        $proxyType = $this->proxy->get('type');
        if ($proxyType === ProxyTypes::NONE->value) {
            return $proxyType;
        }
        if (is_null($proxyType)) {
            $this->proxy->type = ProxyTypes::TRAEFIK_V2->value;
            $this->proxy->status = ProxyStatus::EXITED->value;
            $this->save();
        }
        return $this->proxy->get('type');
    }
    public function scopeWithProxy(): Builder
    {
        return $this->proxy->modelScope();
    }

    public function isLocalhost()
    {
        return $this->ip === 'host.docker.internal' || $this->id === 0;
    }
    static public function buildServers($teamId)
    {
        return Server::whereTeamId($teamId)->whereRelation('settings', 'is_reachable', true)->whereRelation('settings', 'is_build_server', true);
    }
    public function skipServer()
    {
        if ($this->ip === '1.2.3.4') {
            ray('skipping 1.2.3.4');
            return true;
        }
        return false;
    }
    public function isServerReady(int $tries = 3)
    {
        if ($this->skipServer()) {
            return false;
        }
        $serverUptimeCheckNumber = $this->unreachable_count;
        if ($this->unreachable_count < $tries) {
            $serverUptimeCheckNumber = $this->unreachable_count + 1;
        }
        if ($this->unreachable_count > $tries) {
            $serverUptimeCheckNumber = $tries;
        }

        $serverUptimeCheckNumberMax = $tries;

        // ray('server: ' . $this->name);
        // ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
        // ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax);

        $result = $this->validateConnection();
        if ($result) {
            if ($this->unreachable_notification_sent === true) {
                $this->update(['unreachable_notification_sent' => false]);
            }
            return true;
        } else {
            if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
                // Reached max number of retries
                if ($this->unreachable_notification_sent === false) {
                    ray('Server unreachable, sending notification...');
                    $this->team?->notify(new Unreachable($this));
                    $this->update(['unreachable_notification_sent' => true]);
                }
                if ($this->settings->is_reachable === true) {
                    $this->settings()->update([
                        'is_reachable' => false,
                    ]);
                }

                foreach ($this->applications() as $application) {
                    $application->update(['status' => 'exited']);
                }
                foreach ($this->databases() as $database) {
                    $database->update(['status' => 'exited']);
                }
                foreach ($this->services()->get() as $service) {
                    $apps = $service->applications()->get();
                    $dbs = $service->databases()->get();
                    foreach ($apps as $app) {
                        $app->update(['status' => 'exited']);
                    }
                    foreach ($dbs as $db) {
                        $db->update(['status' => 'exited']);
                    }
                }
            } else {
                $this->update([
                    'unreachable_count' => $this->unreachable_count + 1,
                ]);
            }
            return false;
        }
    }
    public function getDiskUsage()
    {
        return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
    }
    public function definedResources()
    {
        $applications = $this->applications();
        $databases = $this->databases();
        $services = $this->services();
        return $applications->concat($databases)->concat($services->get());
    }
    public function hasDefinedResources()
    {
        $applications = $this->applications()->count() > 0;
        $databases = $this->databases()->count() > 0;
        $services = $this->services()->count() > 0;
        if ($applications || $databases || $services) {
            return true;
        }
        return false;
    }

    public function databases()
    {
        return $this->destinations()->map(function ($standaloneDocker) {
            $postgresqls = data_get($standaloneDocker, 'postgresqls', collect([]));
            $redis = data_get($standaloneDocker, 'redis', collect([]));
            $mongodbs = data_get($standaloneDocker, 'mongodbs', collect([]));
            $mysqls = data_get($standaloneDocker, 'mysqls', collect([]));
            $mariadbs = data_get($standaloneDocker, 'mariadbs', collect([]));
            return $postgresqls->concat($redis)->concat($mongodbs)->concat($mysqls)->concat($mariadbs);
        })->flatten();
    }
    public function applications()
    {
        return $this->destinations()->map(function ($standaloneDocker) {
            return $standaloneDocker->applications;
        })->flatten();
    }
    public function dockerComposeBasedApplications()
    {
        return $this->applications()->filter(function ($application) {
            return data_get($application, 'build_pack') === 'dockercompose';
        });
    }
    public function dockerComposeBasedPreviewDeployments()
    {
        return $this->previews()->filter(function ($preview) {
            $applicationId = data_get($preview, 'application_id');
            $application = Application::find($applicationId);
            if (!$application) {
                return false;
            }
            return data_get($application, 'build_pack') === 'dockercompose';
        });
    }
    public function services()
    {
        return $this->hasMany(Service::class);
    }
    public function getIp(): Attribute
    {
        return Attribute::make(
            get: function () {
                if (isDev()) {
                    return '127.0.0.1';
                }
                if ($this->isLocalhost()) {
                    return base_ip();
                }
                return $this->ip;
            }
        );
    }
    public function previews()
    {
        return $this->destinations()->map(function ($standaloneDocker) {
            return $standaloneDocker->applications->map(function ($application) {
                return $application->previews;
            })->flatten();
        })->flatten();
    }

    public function destinations()
    {
        $standalone_docker = $this->hasMany(StandaloneDocker::class)->get();
        $swarm_docker = $this->hasMany(SwarmDocker::class)->get();
        return $standalone_docker->concat($swarm_docker);
    }

    public function standaloneDockers()
    {
        return $this->hasMany(StandaloneDocker::class);
    }

    public function swarmDockers()
    {
        return $this->hasMany(SwarmDocker::class);
    }

    public function privateKey()
    {
        return $this->belongsTo(PrivateKey::class);
    }

    public function muxFilename()
    {
        return "{$this->ip}_{$this->port}_{$this->user}";
    }

    public function team()
    {
        return $this->belongsTo(Team::class);
    }
    public function isProxyShouldRun()
    {
        if ($this->proxyType() === ProxyTypes::NONE->value || $this->settings->is_build_server) {
            return false;
        }
        // foreach ($this->applications() as $application) {
        //     if (data_get($application, 'fqdn')) {
        //         $shouldRun = true;
        //         break;
        //     }
        // }
        // ray($this->services()->get());

        // if ($this->id === 0) {
        //     $settings = InstanceSettings::get();
        //     if (data_get($settings, 'fqdn')) {
        //         $shouldRun = true;
        //     }
        // }
        return true;
    }
    public function isFunctional()
    {
        return $this->settings->is_reachable && $this->settings->is_usable;
    }
    public function isLogDrainEnabled()
    {
        return $this->settings->is_logdrain_newrelic_enabled || $this->settings->is_logdrain_highlight_enabled || $this->settings->is_logdrain_axiom_enabled || $this->settings->is_logdrain_custom_enabled;
    }
    public function validateOS(): bool | Stringable
    {
        $os_release = instant_remote_process(['cat /etc/os-release'], $this);
        $releaseLines = collect(explode("\n", $os_release));
        $collectedData = collect([]);
        foreach ($releaseLines as $line) {
            $item = Str::of($line)->trim();
            $collectedData->put($item->before('=')->value(), $item->after('=')->lower()->replace('"', '')->value());
        }
        $ID = data_get($collectedData, 'ID');
        // $ID_LIKE = data_get($collectedData, 'ID_LIKE');
        // $VERSION_ID = data_get($collectedData, 'VERSION_ID');
        $supported = collect(SUPPORTED_OS)->filter(function ($supportedOs) use ($ID) {
            if (str($supportedOs)->contains($ID)) {
                return str($ID);
            }
        });
        if ($supported->count() === 1) {
            ray('supported');
            return str($supported->first());
        } else {
            ray('not supported');
            return false;
        }
    }
    public function isSwarm()
    {
        return data_get($this, 'settings.is_swarm_manager') || data_get($this, 'settings.is_swarm_worker');
    }
    public function isSwarmManager()
    {
        return data_get($this, 'settings.is_swarm_manager');
    }
    public function isSwarmWorker()
    {
        return data_get($this, 'settings.is_swarm_worker');
    }
    public function validateConnection()
    {
        $server = Server::find($this->id);
        if (!$server) {
            return false;
        }
        if ($server->skipServer()) {
            return false;
        }
        $uptime = instant_remote_process(['uptime'], $server, false);
        if (!$uptime) {
            $server->settings()->update([
                'is_reachable' => false,
            ]);
            return false;
        } else {
            $server->settings()->update([
                'is_reachable' => true,
            ]);
            $server->update([
                'unreachable_count' => 0,
            ]);
        }

        if (data_get($server, 'unreachable_notification_sent') === true) {
            $server->team?->notify(new Revived($server));
            $server->update(['unreachable_notification_sent' => false]);
        }

        return true;
    }
    public function validateDockerEngine($throwError = false)
    {
        $dockerBinary = instant_remote_process(["command -v docker"], $this, false);
        if (is_null($dockerBinary)) {
            $this->settings->is_usable = false;
            $this->settings->save();
            if ($throwError) {
                throw new \Exception('Server is not usable. Docker Engine is not installed.');
            }
            return false;
        }
        $this->settings->is_usable = true;
        $this->settings->save();
        $this->validateCoolifyNetwork(isSwarm: false, isBuildServer: $this->settings->is_build_server);
        return true;
    }
    public function validateDockerSwarm()
    {
        $swarmStatus = instant_remote_process(["docker info|grep -i swarm"], $this, false);
        $swarmStatus = str($swarmStatus)->trim()->after(':')->trim();
        if ($swarmStatus === 'inactive') {
            throw new \Exception('Docker Swarm is not initiated. Please join the server to a swarm before continuing.');
            return false;
        }
        $this->settings->is_usable = true;
        $this->settings->save();
        $this->validateCoolifyNetwork(isSwarm: true);
        return true;
    }
    public function validateDockerEngineVersion()
    {
        $dockerVersion = instant_remote_process(["docker version|head -2|grep -i version| awk '{print $2}'"], $this, false);
        $dockerVersion = checkMinimumDockerEngineVersion($dockerVersion);
        if (is_null($dockerVersion)) {
            $this->settings->is_usable = false;
            $this->settings->save();
            return false;
        }
        $this->settings->is_reachable = true;
        $this->settings->is_usable = true;
        $this->settings->save();
        return true;
    }
    public function validateCoolifyNetwork($isSwarm = false, $isBuildServer = false)
    {
        if ($isBuildServer) {
            return;
        }
        if ($isSwarm) {
            return instant_remote_process(["docker network create --attachable --driver overlay coolify-overlay >/dev/null 2>&1 || true"], $this, false);
        } else {
            return instant_remote_process(["docker network create coolify --attachable >/dev/null 2>&1 || true"], $this, false);
        }
    }
    public function executeRemoteCommand(Collection $commands, ?ApplicationDeploymentQueue $loggingModel = null)
    {
        static::$batch_counter++;
        foreach ($commands as $command) {
            $realCommand = data_get($command, 'command');
            if (is_null($realCommand)) {
                throw new \RuntimeException('Command is not set');
            }
            $hidden = data_get($command, 'hidden', false);
            $ignoreErrors = data_get($command, 'ignoreErrors', false);
            $customOutputType = data_get($command, 'customOutputType');
            $name = data_get($command, 'name');
            $remoteCommand = generateSshCommand($this, $realCommand);

            $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($realCommand, $hidden, $customOutputType, $loggingModel, $name) {
                $output = str($output)->trim();
                if ($output->startsWith('╔')) {
                    $output = "\n" . $output;
                }
                $newLogEntry = [
                    'command' => remove_iip($realCommand),
                    'output' => remove_iip($output),
                    'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout',
                    'timestamp' => Carbon::now('UTC'),
                    'hidden' => $hidden,
                    'batch' => static::$batch_counter,
                ];
                if ($loggingModel) {
                    if (!$loggingModel->logs) {
                        $newLogEntry['order'] = 1;
                    } else {
                        $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
                        $newLogEntry['order'] = count($previousLogs) + 1;
                    }
                    if ($name) {
                        $newLogEntry['name'] = $name;
                    }

                    $previousLogs[] = $newLogEntry;
                    $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR);
                    $loggingModel->save();
                }
            });
            if ($loggingModel) {
                $loggingModel->update([
                    'current_process_id' => $process->id(),
                ]);
            }
            $processResult = $process->wait();
            if ($processResult->exitCode() !== 0) {
                if (!$ignoreErrors) {
                    if ($loggingModel) {
                        $status = ApplicationDeploymentStatus::FAILED->value;
                        $loggingModel->status = $status;
                        $loggingModel->save();
                    }
                    throw new \RuntimeException($processResult->errorOutput());
                }
            }
        }
    }
    public function stopApplicationRelatedRunningContainers(string $applicationId, string $containerName)
    {
        $containers = getCurrentApplicationContainerStatus($this, $applicationId, 0);
        $containers = $containers->filter(function ($container) use ($containerName) {
            return data_get($container, 'Names') !== $containerName;
        });
        $containers->each(function ($container) {
            $removableContainer = data_get($container, 'Names');
            $this->server->executeRemoteCommand(
                commands: collect([
                    'command' => "docker rm -f $removableContainer >/dev/null 2>&1",
                    'hidden' => true,
                    'ignoreErrors' => true
                ]),
                loggingModel: $this->deploymentQueueEntry
            );
        });
    }
    public function getHostIPMappings($network)
    {
        $addHosts = null;
        $allContainers = instant_remote_process(["docker network inspect {$network} -f '{{json .Containers}}' "], $this);
        if (!is_null($allContainers)) {
            $allContainers = format_docker_command_output_to_json($allContainers);
            $ips = collect([]);
            if (count($allContainers) > 0) {
                $allContainers = $allContainers[0];
                foreach ($allContainers as $container) {
                    $containerName = data_get($container, 'Name');
                    if ($containerName === 'coolify-proxy') {
                        continue;
                    }
                    $containerIp = data_get($container, 'IPv4Address');
                    if ($containerName && $containerIp) {
                        $containerIp = str($containerIp)->before('/');
                        $ips->put($containerName, $containerIp->value());
                    }
                }
            }
            $addHosts = $ips->map(function ($ip, $name) {
                return "--add-host $name:$ip";
            })->implode(' ');
        }
        return $addHosts;
    }
    public function checkIfDockerImageExists(string $imageName, ApplicationDeploymentQueue $deployment)
    {
        $this->executeRemoteCommand(
            commands: collect([
                [
                    "name" => "local_image_found",
                    "command" => "docker images -q {$imageName} 2>/dev/null",
                    "hidden" => true,
                ]
            ]),
            loggingModel: $deployment
        );
        if (str($deployment->getOutput('local_image_found'))->isEmpty()) {
            $this->executeRemoteCommand(
                commands: collect([
                    [
                        "command" => "docker pull {$imageName} 2>/dev/null",
                        "ignoreErrors" => true,
                        "hidden" => true
                    ],
                    [
                        "name" => "local_image_found",
                        "command" => "docker images -q {$imageName} 2>/dev/null",
                        "hidden" => true,
                    ]
                ]),
                loggingModel: $deployment
            );
        }
    }
    public function createWorkDirForDeployment(string $workdir, ApplicationDeploymentQueue $deployment)
    {
        $this->executeRemoteCommand(
            commands: collect([
                [
                    "command" => executeInDocker($deployment->deployment_uuid, "mkdir -p {$workdir}"),
                    "ignoreErrors" => true,
                    "hidden" => true
                ],
            ]),
            loggingModel: $deployment
        );
    }
}