From 6c7e091e1bf7472eca757994a2b15a1ecc9b31b9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 17 Nov 2023 00:37:09 +0100 Subject: [PATCH] feat: log drain (wip) --- app/Actions/Server/InstallLogDrain.php | 130 ++++++++++++++++++ app/Http/Livewire/Server/LogDrains.php | 74 ++++++++++ app/Jobs/ApplicationDeploymentJob.php | 18 +-- app/Models/Server.php | 6 + .../2023_11_16_220647_add_log_drains.php | 38 +++++ examples/fluent-bit/fluent-bit.conf | 16 --- examples/fluent-bit/fluent-bit.yaml | 9 -- examples/newrelic.yaml | 21 --- examples/otl/config.yaml | 34 ----- .../views/components/server/navbar.blade.php | 6 + .../livewire/server/log-drains.blade.php | 49 +++++++ routes/web.php | 2 + 12 files changed, 315 insertions(+), 88 deletions(-) create mode 100644 app/Actions/Server/InstallLogDrain.php create mode 100644 app/Http/Livewire/Server/LogDrains.php create mode 100644 database/migrations/2023_11_16_220647_add_log_drains.php delete mode 100644 examples/fluent-bit/fluent-bit.conf delete mode 100644 examples/fluent-bit/fluent-bit.yaml delete mode 100644 examples/newrelic.yaml delete mode 100644 examples/otl/config.yaml create mode 100644 resources/views/livewire/server/log-drains.blade.php diff --git a/app/Actions/Server/InstallLogDrain.php b/app/Actions/Server/InstallLogDrain.php new file mode 100644 index 000000000..79acda8e7 --- /dev/null +++ b/app/Actions/Server/InstallLogDrain.php @@ -0,0 +1,130 @@ +settings->is_logdrain_newrelic_enabled) { + throw new \Exception('New Relic log drain is not enabled.'); + } + $config = base64_encode(" +[SERVICE] + Flush 5 + Daemon off + Tag container_logs +[INPUT] + Name forward + Buffer_Chunk_Size 1M + Buffer_Max_Size 6M +[FILTER] + Name grep + Match * + Exclude log 127.0.0.1 +[FILTER] + Name modify + Match * + Set server_name {$server->name} + +[OUTPUT] + Name nrlogs + Match * + license_key \${LICENSE_KEY} + # https://log-api.eu.newrelic.com/log/v1 - EU + # https://log-api.newrelic.com/log/v1 - US + base_uri \${BASE_URI} +"); + } else if ($type === 'highlight') { + if (!$server->settings->is_logdrain_highlight_enabled) { + throw new \Exception('Highlight log drain is not enabled.'); + } + $config = base64_encode(' +[SERVICE] + Flush 5 + Daemon off + Tag container_logs +[INPUT] + Name forward + tag ${HIGHLIGHT_PROJECT_ID} + Buffer_Chunk_Size 1M + Buffer_Max_Size 6M +[FILTER] + Name grep + Match * + Exclude log 127.0.0.1 +[FILTER] + Name modify + Match * + Set server_name {$server->name} +[OUTPUT] + Name forward + Match * + Host otel.highlight.io + Port 24224 +'); + } + + $compose = base64_encode(" +services: + coolify-log-drain: + image: cr.fluentbit.io/fluent/fluent-bit:2.0 + container_name: coolify-log-drain + command: -c /fluent-bit.conf + env_file: + - .env + volumes: + - ./fluent-bit.conf:/fluent-bit.conf + ports: + - 127.0.0.1:24224:24224 +"); + $readme = base64_encode('# New Relic Log Drain +This log drain is based on [Fluent Bit](https://fluentbit.io/) and New Relic Log Forwarder. + +Files: +- `fluent-bit.conf` - configuration file for Fluent Bit +- `docker-compose.yml` - docker-compose file to run Fluent Bit +- `.env` - environment variables for Fluent Bit +'); + $license_key = $server->settings->logdrain_newrelic_license_key; + $base_uri = $server->settings->logdrain_newrelic_base_uri; + $base_path = config('coolify.base_config_path'); + + $config_path = $base_path . '/log-drains'; + $fluent_bit_config = $config_path . '/fluent-bit.conf'; + $compose_path = $config_path . '/docker-compose.yml'; + $readme_path = $config_path . '/README.md'; + $command = [ + "echo 'Saving configuration'", + "mkdir -p $config_path", + "echo '{$config}' | base64 -d > $fluent_bit_config", + "echo '{$compose}' | base64 -d > $compose_path", + "echo '{$readme}' | base64 -d > $readme_path", + "rm $config_path/.env || true", + + ]; + if ($type === 'newrelic') { + $add_envs_command = [ + "echo LICENSE_KEY=$license_key >> $config_path/.env", + "echo BASE_URI=$base_uri >> $config_path/.env", + ]; + } else if ($type === 'highlight') { + $add_envs_command = [ + "echo HIGHLIGHT_PROJECT_ID={$server->settings->logdrain_highlight_project_id} >> $config_path/.env", + ]; + } + $restart_command = [ + "echo 'Stopping old Fluent Bit'", + "cd $config_path && docker rm -f coolify-log-drain || true", + "echo 'Starting Fluent Bit'", + "cd $config_path && docker compose up -d --remove-orphans", + ]; + $command = array_merge($command, $add_envs_command, $restart_command); + return instant_remote_process($command, $server); + } +} diff --git a/app/Http/Livewire/Server/LogDrains.php b/app/Http/Livewire/Server/LogDrains.php new file mode 100644 index 000000000..823886b3f --- /dev/null +++ b/app/Http/Livewire/Server/LogDrains.php @@ -0,0 +1,74 @@ + 'required|boolean', + 'server.settings.logdrain_newrelic_license_key' => 'required|string', + 'server.settings.logdrain_newrelic_base_uri' => 'required|string', + 'server.settings.is_logdrain_highlight_enabled' => 'required|boolean', + 'server.settings.logdrain_highlight_project_id' => 'required|string', + ]; + + public function mount() { + $this->parameters = get_route_parameters(); + try { + $this->server = Server::ownedByCurrentTeam(['name', 'description', 'ip', 'port', 'user', 'proxy'])->whereUuid(request()->server_uuid)->first(); + if (is_null($this->server)) { + return redirect()->route('server.all'); + } + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + public function configureLogDrain(string $type) { + try { + $this->server->logDrain($type); + $this->emit('serverRefresh'); + $this->emit('success', 'Log drain configured successfully.'); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + public function instantSave(string $type) { + $this->submit($type); + } + public function submit(string $type) { + try { + $this->resetErrorBag(); + if ($type === 'newrelic') { + $this->validate([ + 'server.settings.is_logdrain_newrelic_enabled' => 'required|boolean', + 'server.settings.logdrain_newrelic_license_key' => 'required|string', + 'server.settings.logdrain_newrelic_base_uri' => 'required|string', + ]); + $this->server->settings->update([ + 'is_logdrain_highlight_enabled' => false, + ]); + } else if ($type === 'highlight') { + $this->validate([ + 'server.settings.is_logdrain_highlight_enabled' => 'required|boolean', + 'server.settings.logdrain_highlight_project_id' => 'required|string', + ]); + $this->server->settings->update([ + 'is_logdrain_newrelic_enabled' => false, + ]); + } + $this->server->settings->save(); + $this->emit('success', 'Settings saved successfully.'); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + public function render() + { + return view('livewire.server.log-drains'); + } +} diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 83674e815..c74c973f9 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -498,7 +498,7 @@ private function health_check() if ($this->full_healthcheck_url) { $this->execute_remote_command( [ - "echo 'Healthcheck URL inside your container: {$this->full_healthcheck_url}'" + "echo 'Healthcheck URL (inside the container): {$this->full_healthcheck_url}'" ] ); } @@ -837,13 +837,15 @@ private function generate_compose_file() 'networks' => [ $this->destination->network, ], - // 'logging' => [ - // 'driver' => 'fluentd', - // 'options' => [ - // 'fluentd-async' => 'true', - // 'tag' => $this->application->name . '-' . $this->application->uuid - // ] - // ], + 'logging' => [ + 'driver' => 'fluentd', + 'options' => [ + 'fluentd-address' => "tcp://127.0.0.1:24224", + 'fluentd-async' => "true", + 'fluentd-sub-second-precision' => "true", + + ] + ], 'healthcheck' => [ 'test' => [ 'CMD-SHELL', diff --git a/app/Models/Server.php b/app/Models/Server.php index d1b11a080..6fa5bda7b 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -2,6 +2,8 @@ namespace App\Models; +use App\Actions\Server\InstallLogDrain; +use App\Actions\Server\InstallNewRelic; use App\Enums\ProxyStatus; use App\Enums\ProxyTypes; use App\Notifications\Server\Revived; @@ -296,6 +298,10 @@ public function isProxyShouldRun() // } return true; } + public function logDrain($type) + { + InstallLogDrain::run($this, $type); + } public function isFunctional() { return $this->settings->is_reachable && $this->settings->is_usable; diff --git a/database/migrations/2023_11_16_220647_add_log_drains.php b/database/migrations/2023_11_16_220647_add_log_drains.php new file mode 100644 index 000000000..82274a999 --- /dev/null +++ b/database/migrations/2023_11_16_220647_add_log_drains.php @@ -0,0 +1,38 @@ +boolean('is_logdrain_newrelic_enabled')->default(false); + $table->string('logdrain_newrelic_license_key')->nullable(); + $table->string('logdrain_newrelic_base_uri')->nullable(); + + $table->boolean('is_logdrain_highlight_enabled')->default(false); + $table->string('logdrain_highlight_project_id')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('server_settings', function (Blueprint $table) { + $table->dropColumn('is_logdrain_newrelic_enabled'); + $table->dropColumn('logdrain_newrelic_license_key'); + $table->dropColumn('logdrain_newrelic_base_uri'); + + $table->dropColumn('is_logdrain_highlight_enabled'); + $table->dropColumn('logdrain_highlight_project_id'); + }); + } +}; diff --git a/examples/fluent-bit/fluent-bit.conf b/examples/fluent-bit/fluent-bit.conf deleted file mode 100644 index 5d10f559a..000000000 --- a/examples/fluent-bit/fluent-bit.conf +++ /dev/null @@ -1,16 +0,0 @@ -[SERVICE] - Flush 1 - Daemon off -[INPUT] - Name forward - Buffer_Chunk_Size 1M - Buffer_Max_Size 6M -# [OUTPUT] -# Name nrlogs -# Match * -# license_key ${LICENSE_KEY} -# base_uri https://log-api.eu.newrelic.com/log/v1 - -[OUTPUT] - Name stdout - Match * diff --git a/examples/fluent-bit/fluent-bit.yaml b/examples/fluent-bit/fluent-bit.yaml deleted file mode 100644 index ca573635e..000000000 --- a/examples/fluent-bit/fluent-bit.yaml +++ /dev/null @@ -1,9 +0,0 @@ -version: '3' -services: - coolify-fluent-bit: - image: cr.fluentbit.io/fluent/fluent-bit:2.0 - command: -c /fluent-bit.conf - volumes: - - ./fluent-bit.conf:/fluent-bit.conf - ports: - - 24224:24224 diff --git a/examples/newrelic.yaml b/examples/newrelic.yaml deleted file mode 100644 index 40bd5b0f2..000000000 --- a/examples/newrelic.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: '3' -services: - newrelic-infra: - container_name: newrelic-infra - image: newrelic/infrastructure:latest - networks: - - coolify - cap_add: - - SYS_PTRACE - privileged: true - pid: host - volumes: - - "/:/host:ro" - - "/var/run/docker.sock:/var/run/docker.sock" - - "newrelic-infra:/etc/newrelic-infra" - environment: - - NRIA_LICENSE_KEY=${NRIA_LICENSE_KEY} - - NRIA_DISPLAY_NAME=${HOSTNAME} - -networks: - coolify: diff --git a/examples/otl/config.yaml b/examples/otl/config.yaml deleted file mode 100644 index a1b8b7ec4..000000000 --- a/examples/otl/config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -receivers: - hostmetrics: - collection_interval: 5s - scrapers: - cpu: - metrics: - system.cpu.utilization: - enabled: true -processors: - resourcedetection: - detectors: [env, system] - system: - hostname_sources: ["os"] - resource_attributes: - host.id: - enabled: true - batch: - memory_limiter: - check_interval: 1s - limit_mib: 1000 - spike_limit_mib: 200 -exporters: - debug: - verbosity: detailed - otlp: - endpoint: ${OTLP_ENDPOINT} - headers: - api-key: ${OTLP_API_KEY} -service: - pipelines: - metrics: - receivers: [hostmetrics] - processors: [memory_limiter, resourcedetection, batch] - exporters: [debug, otlp] diff --git a/resources/views/components/server/navbar.blade.php b/resources/views/components/server/navbar.blade.php index fbbdc545b..56b11ef78 100644 --- a/resources/views/components/server/navbar.blade.php +++ b/resources/views/components/server/navbar.blade.php @@ -32,6 +32,12 @@ ]) }}"> + + +
diff --git a/resources/views/livewire/server/log-drains.blade.php b/resources/views/livewire/server/log-drains.blade.php new file mode 100644 index 000000000..38c5b0cc4 --- /dev/null +++ b/resources/views/livewire/server/log-drains.blade.php @@ -0,0 +1,49 @@ +
+ +

Log Drains

+
Sends resource logs to external services.
+
+
+

New Relic

+
+ +
+
+
+
+ + +
+ +
+
+ + Save + + + Configure On Server + +
+
+

Highlight.io

+
+ +
+
+
+
+ +
+
+
+ + Save + + + Configure On Server + +
+
+
+
+
diff --git a/routes/web.php b/routes/web.php index 54226ca89..6ac0d578c 100644 --- a/routes/web.php +++ b/routes/web.php @@ -16,6 +16,7 @@ use App\Http\Livewire\Server\All; use App\Http\Livewire\Server\Create; use App\Http\Livewire\Server\Destination\Show as DestinationShow; +use App\Http\Livewire\Server\LogDrains; use App\Http\Livewire\Server\PrivateKey\Show as PrivateKeyShow; use App\Http\Livewire\Server\Proxy\Show as ProxyShow; use App\Http\Livewire\Server\Proxy\Logs as ProxyLogs; @@ -130,6 +131,7 @@ Route::get('/server/{server_uuid}/proxy/logs', ProxyLogs::class)->name('server.proxy.logs'); Route::get('/server/{server_uuid}/private-key', PrivateKeyShow::class)->name('server.private-key'); Route::get('/server/{server_uuid}/destinations', DestinationShow::class)->name('server.destinations'); + Route::get('/server/{server_uuid}/log-drains', LogDrains::class)->name('server.log-drains'); });