Refactor + package updates + improve local backups

This commit is contained in:
Andras Bacsai 2023-08-10 15:52:54 +02:00
parent d2a4dbf283
commit e17f1935d2
30 changed files with 757 additions and 366 deletions

View File

@ -2,9 +2,9 @@
namespace App\Console;
use App\Jobs\BackupDatabaseJob;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CheckResaleLicenseKeys;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceApplicationsStatusJob;
use App\Jobs\InstanceAutoUpdateJob;
@ -50,7 +50,7 @@ private function check_scheduled_backups($schedule)
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
}
$schedule->job(new BackupDatabaseJob(
$schedule->job(new DatabaseBackupJob(
backup: $scheduled_backup
))->cron($scheduled_backup->frequency);
}

View File

@ -26,6 +26,29 @@ public function configuration()
return view('project.database.configuration', ['database' => $database]);
}
public function backup_logs()
{
$backup_uuid = request()->route('backup_uuid');
$project = session('currentTeam')->load(['projects'])->projects->where('uuid', request()->route('project_uuid'))->first();
if (!$project) {
return redirect()->route('dashboard');
}
$environment = $project->load(['environments'])->environments->where('name', request()->route('environment_name'))->first()->load(['applications']);
if (!$environment) {
return redirect()->route('dashboard');
}
$database = $environment->databases->where('uuid', request()->route('database_uuid'))->first();
if (!$database) {
return redirect()->route('dashboard');
}
$backup = $database->scheduledBackups->where('uuid', $backup_uuid)->first();
if (!$backup) {
return redirect()->route('dashboard');
}
$backup_executions = collect($backup->executions)->sortByDesc('created_at');
return view('project.database.backups.logs', ['database' => $database, 'backup' => $backup, 'backup_executions' => $backup_executions]);
}
public function backups()
{
$project = session('currentTeam')->load(['projects'])->projects->where('uuid', request()->route('project_uuid'))->first();
@ -40,6 +63,6 @@ public function backups()
if (!$database) {
return redirect()->route('dashboard');
}
return view('project.database.backups', ['database' => $database]);
return view('project.database.backups.all', ['database' => $database]);
}
}

View File

@ -0,0 +1,41 @@
<?php
namespace App\Http\Livewire\Project\Database;
use Livewire\Component;
class BackupEdit extends Component
{
public $backup;
protected $rules = [
'backup.enabled' => 'required|boolean',
'backup.frequency' => 'required|string',
'backup.number_of_backups_locally' => 'required|integer|min:1',
];
protected $validationAttributes = [
'backup.enabled' => 'Enabled',
'backup.frequency' => 'Frequency',
'backup.number_of_backups_locally' => 'Number of Backups Locally',
];
public function instantSave()
{
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
}
public function submit()
{
$isValid = validate_cron_expression($this->backup->frequency);
if (!$isValid) {
$this->emit('error', 'Invalid Cron / Human expression');
return;
}
$this->validate();
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
}
}

View File

@ -0,0 +1,24 @@
<?php
namespace App\Http\Livewire\Project\Database;
use App\Models\ScheduledDatabaseBackupExecution;
use Livewire\Component;
class BackupExecution extends Component
{
public ScheduledDatabaseBackupExecution $execution;
public function download()
{
}
public function delete(): void
{
delete_backup_locally($this->execution->filename, $this->execution->scheduledDatabaseBackup->database->destination->server);
$this->execution->delete();
$this->emit('success', 'Backup execution deleted successfully.');
$this->emit('refreshBackupExecutions');
}
}

View File

@ -0,0 +1,17 @@
<?php
namespace App\Http\Livewire\Project\Database;
use Livewire\Component;
class BackupExecutions extends Component
{
public $backup;
public $executions;
protected $listeners = ['refreshBackupExecutions'];
public function refreshBackupExecutions(): void
{
$this->executions = collect($this->backup->executions)->sortByDesc('created_at');
}
}

View File

@ -4,24 +4,20 @@
use App\Models\ScheduledDatabaseBackup;
use Livewire\Component;
use Poliander\Cron\CronExpression;
class CreateScheduledBackup extends Component
{
public $database;
public $frequency;
public bool $enabled = true;
public bool $keep_locally = true;
public bool $save_s3 = true;
protected $rules = [
'frequency' => 'required|string',
'keep_locally' => 'required|boolean',
'save_s3' => 'required|boolean',
];
protected $validationAttributes = [
'frequency' => 'Backup Frequency',
'keep_locally' => 'Keep Locally',
'save_s3' => 'Save to S3',
];
@ -29,13 +25,7 @@ public function submit(): void
{
try {
$this->validate();
$expression = new CronExpression($this->frequency);
$isValid = $expression->isValid();
if (isset(VALID_CRON_STRINGS[$this->frequency])) {
$isValid = true;
}
$isValid = validate_cron_expression($this->frequency);
if (!$isValid) {
$this->emit('error', 'Invalid Cron / Human expression');
return;
@ -43,7 +33,6 @@ public function submit(): void
ScheduledDatabaseBackup::create([
'enabled' => true,
'frequency' => $this->frequency,
'keep_locally' => $this->keep_locally,
'save_s3' => $this->save_s3,
'database_id' => $this->database->id,
'database_type' => $this->database->getMorphClass(),
@ -54,7 +43,6 @@ public function submit(): void
general_error_handler($e, $this);
} finally {
$this->frequency = '';
$this->keep_locally = true;
$this->save_s3 = true;
}
}

View File

@ -7,9 +7,22 @@
class ScheduledBackups extends Component
{
public $database;
public $parameters;
protected $listeners = ['refreshScheduledBackups'];
public function refreshScheduledBackups()
public function mount(): void
{
$this->parameters = get_route_parameters();
}
public function delete($scheduled_backup_id): void
{
$this->database->scheduledBackups->find($scheduled_backup_id)->delete();
$this->emit('success', 'Scheduled backup deleted successfully.');
$this->refreshScheduledBackups();
}
public function refreshScheduledBackups(): void
{
ray('refreshScheduledBackups');
$this->database->refresh();

View File

@ -1,78 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Throwable;
class BackupDatabaseJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public Team|null $team = null;
public Server $server;
public ScheduledDatabaseBackup|null $backup;
public string $database_type;
public StandalonePostgresql $database;
public string $status;
public function __construct($backup)
{
$this->backup = $backup;
$this->team = Team::find($backup->team_id);
$this->database = $this->backup->database->first();
$this->database_type = $this->database->type();
$this->server = $this->database->destination->server;
$this->status = $this->database->status;
}
public function middleware(): array
{
return [new WithoutOverlapping($this->backup->id)];
}
public function uniqueId(): int
{
return $this->backup->id;
}
public function handle()
{
if ($this->status !== 'running') {
ray('database not running');
return;
}
if ($this->database_type === 'standalone-postgresql') {
$this->backup_standalone_postgresql();
}
}
private function backup_standalone_postgresql()
{
try {
$backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql";
$commands[] = "mkdir -p " . backup_dir();
$commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}";
$commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $backup_filename";
instant_remote_process($commands, $this->server);
ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $backup_filename);
if (!$this->backup->keep_locally) {
$commands[] = "rm -rf $backup_filename";
instant_remote_process($commands, $this->server);
}
} catch (Throwable $th) {
ray($th);
//throw $th;
}
}
}

View File

@ -0,0 +1,142 @@
<?php
namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Throwable;
class DatabaseBackupJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public Team|null $team = null;
public Server $server;
public ScheduledDatabaseBackup|null $backup;
public string $database_type;
public StandalonePostgresql $database;
public string $database_status;
public ScheduledDatabaseBackupExecution|null $backup_log = null;
public string $backup_status;
public string|null $backup_filename = null;
public int $size = 0;
public string|null $backup_output = null;
public function __construct($backup)
{
$this->backup = $backup;
$this->team = Team::find($backup->team_id);
$this->database = $this->backup->database->first();
$this->database_type = $this->database->type();
$this->server = $this->database->destination->server;
$this->database_status = $this->database->status;
}
public function middleware(): array
{
return [new WithoutOverlapping($this->backup->id)];
}
public function uniqueId(): int
{
return $this->backup->id;
}
public function handle()
{
if ($this->database_status !== 'running') {
ray('database not running');
return;
}
$this->backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql";
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'filename' => $this->backup_filename,
'scheduled_database_backup_id' => $this->backup->id,
]);
if ($this->database_type === 'standalone-postgresql') {
$this->backup_standalone_postgresql();
}
$this->calculate_size();
$this->remove_old_backups();
$this->save_backup_logs();
}
private function backup_standalone_postgresql()
{
try {
$commands[] = "mkdir -p " . backup_dir();
$commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}";
$commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $this->backup_filename";
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
}
ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $this->backup_filename);
$this->backup_status = 'success';
} catch (Throwable $th) {
$this->backup_status = 'failed';
$this->add_to_backup_output($th->getMessage());
ray('Backup failed for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $this->backup_filename . '\n\nError:' . $th->getMessage());
} finally {
$this->backup_log->update([
'status' => $this->backup_status,
]);
}
}
private function add_to_backup_output($output)
{
if ($this->backup_output) {
$this->backup_output = $this->backup_output . "\n" . $output;
} else {
$this->backup_output = $output;
}
}
private function calculate_size()
{
$this->size = instant_remote_process(["du -b $this->backup_filename | cut -f1"], $this->server);
}
private function remove_old_backups()
{
if ($this->backup->number_of_backups_locally === 0) {
$deletable = $this->backup->executions()->where('status', 'success');
} else {
$deletable = $this->backup->executions()->where('status', 'success')->orderByDesc('created_at')->skip($this->backup->number_of_backups_locally);
}
ray($deletable->get());
foreach ($deletable->get() as $execution) {
delete_backup_locally($execution->filename, $this->server);
$execution->delete();
}
}
private function save_backup_logs()
{
$this->backup_log->update([
'status' => $this->backup_status,
'message' => $this->backup_output,
'size' => $this->size,
]);
}
}

View File

@ -3,12 +3,26 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Database\Eloquent\Relations\MorphTo;
class ScheduledDatabaseBackup extends BaseModel
{
protected $guarded = [];
public function database()
public function database(): MorphTo
{
return $this->morphTo();
}
public function latest_log(): HasOne
{
return $this->hasOne(ScheduledDatabaseBackupExecution::class)->latest();
}
public function executions(): HasMany
{
return $this->hasMany(ScheduledDatabaseBackupExecution::class);
}
}

View File

@ -0,0 +1,15 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
class ScheduledDatabaseBackupExecution extends BaseModel
{
protected $guarded = [];
public function scheduledDatabaseBackup(): BelongsTo
{
return $this->belongsTo(ScheduledDatabaseBackup::class);
}
}

View File

@ -64,11 +64,6 @@ public function destination()
return $this->morphTo();
}
public function scheduled_database_backups()
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function environment_variables(): HasMany
{
return $this->hasMany(EnvironmentVariable::class);

View File

@ -2,6 +2,7 @@
const DATABASE_TYPES = ['postgresql'];
const VALID_CRON_STRINGS = [
'every_minute' => '* * * * *',
'hourly' => '0 * * * *',
'daily' => '0 0 * * *',
'weekly' => '0 0 * * 0',

View File

@ -1,5 +1,6 @@
<?php
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use Visus\Cuid2\Cuid2;
@ -24,5 +25,18 @@ function create_standalone_postgresql($environment_id, $destination_uuid): Stand
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
]);
}
/**
* Delete file locally on the filesystem.
* @param string $filename
* @param Server $server
* @return void
*/
function delete_backup_locally(string|null $filename, Server $server): void
{
if (empty($filename)) {
return;
}
instant_remote_process(["rm -f \"{$filename}\""], $server, throwError: false);
}

View File

@ -6,6 +6,7 @@
use Illuminate\Support\Facades\Route;
use Illuminate\Support\Str;
use Nubs\RandomNameGenerator\All;
use Poliander\Cron\CronExpression;
use Visus\Cuid2\Cuid2;
function application_configuration_dir(): string
@ -166,3 +167,16 @@ function is_cloud(): bool
return !config('coolify.self_hosted');
}
function validate_cron_expression($expression_to_validate): bool
{
$isValid = false;
$expression = new CronExpression($expression_to_validate);
$isValid = $expression->isValid();
if (isset(VALID_CRON_STRINGS[$expression_to_validate])) {
$isValid = true;
}
return $isValid;
}

554
composer.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -11,9 +11,9 @@ public function up(): void
$table->id();
$table->string('uuid')->unique();
$table->boolean('enabled')->default(true);
$table->boolean('keep_locally')->default(false);
$table->string('save_s3')->default(true);
$table->string('frequency');
$table->integer('number_of_backups_locally')->default(7);
$table->morphs('database');
$table->foreignId('team_id');
$table->timestamps();

View File

@ -0,0 +1,26 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration {
public function up(): void
{
Schema::create('scheduled_database_backup_executions', function (Blueprint $table) {
$table->id();
$table->string('uuid')->unique();
$table->enum('status', ['success', 'failed', 'running'])->default('running');
$table->longText('message')->nullable();
$table->text('size')->nullable();
$table->text('filename')->nullable();
$table->foreignId('scheduled_database_backup_id');
$table->timestamps();
});
}
public function down(): void
{
Schema::dropIfExists('scheduled_database_backup_executions');
}
};

View File

@ -32,7 +32,8 @@ public function run(): void
LocalPersistentVolumeSeeder::class,
S3StorageSeeder::class,
StandalonePostgresqlSeeder::class,
ScheduledDatabaseBackupSeeder::class
ScheduledDatabaseBackupSeeder::class,
ScheduledDatabaseBackupExecutionSeeder::class,
]);
}
}

View File

@ -0,0 +1,28 @@
<?php
namespace Database\Seeders;
use App\Models\ScheduledDatabaseBackupExecution;
use Illuminate\Database\Seeder;
class ScheduledDatabaseBackupExecutionSeeder extends Seeder
{
/**
* Run the database seeds.
*/
public function run(): void
{
ScheduledDatabaseBackupExecution::create([
'status' => 'success',
'message' => 'Backup created successfully.',
'size' => '10243467789556',
'scheduled_database_backup_id' => 1,
]);
ScheduledDatabaseBackupExecution::create([
'status' => 'failed',
'message' => 'Backup failed.',
'size' => '10243456',
'scheduled_database_backup_id' => 1,
]);
}
}

View File

@ -15,7 +15,7 @@ public function run(): void
ScheduledDatabaseBackup::create([
'enabled' => true,
'frequency' => '* * * * *',
'keep_locally' => true,
'number_of_backups_locally' => 2,
'database_id' => 1,
'database_type' => 'App\Models\StandalonePostgresql',
'team_id' => 0,

View File

@ -3,8 +3,8 @@
href="{{ route('project.database.configuration', $parameters) }}">
<button>Configuration</button>
</a>
<a class="{{ request()->routeIs('project.database.backups') ? 'text-white' : '' }}"
href="{{ route('project.database.backups', $parameters) }}">
<a class="{{ request()->routeIs('project.database.backups.all') ? 'text-white' : '' }}"
href="{{ route('project.database.backups.all', $parameters) }}">
<button>Backups</button>
</a>
{{-- <x-applications.links :application="$application" /> --}}

View File

@ -0,0 +1,16 @@
<form wire:submit.prevent="submit">
<div class="flex gap-2 pb-2">
<h2>Scheduled Backup</h2>
<x-forms.button type="submit">
Save
</x-forms.button>
</div>
<div class="flex py-2 gap-10">
<x-forms.checkbox instantSave label="Enabled" id="backup.enabled"/>
<x-forms.checkbox instantSave label="Save to S3" id="backup.save_s3"/>
</div>
<div class="flex gap-2">
<x-forms.input label="Frequency" id="backup.frequency"/>
<x-forms.input label="Number of backups to keep (locally)" id="backup.number_of_backups_locally"/>
</div>
</form>

View File

@ -0,0 +1,7 @@
<div class="flex gap-2">
<div class="flex-1"></div>
{{-- @if(data_get($execution,'status') !== 'failed')--}}
{{-- <x-forms.button class="bg-coollabs-100 hover:bg-coollabs" wire:click="download">Download</x-forms.button>--}}
{{-- @endif--}}
<x-forms.button isError wire:click="delete">Delete</x-forms.button>
</div>

View File

@ -0,0 +1,22 @@
<div class="flex flex-col gap-2">
@forelse($executions as $execution)
<form class="border-1 bg-coolgray-300 p-2 border-dotted flex flex-col"
@class([
'border-green-500' => data_get($execution,'status') === 'success',
'border-red-500' => data_get($execution,'status') === 'failed',
])>
<div>Status: {{data_get($execution,'status')}}</div>
@if(data_get($execution,'message'))
<div>Message: {{data_get($execution,'message')}}</div>
@endif
<div>Size: {{data_get($execution,'size')}} B / {{round((int)data_get($execution,'size') / 1024,2)}}
kB / {{round((int)data_get($execution,'size')/1024/1024,2)}} MB
</div>
<div>Location: {{data_get($execution,'filename')}}</div>
<livewire:project.database.backup-execution :execution="$execution" :wire:key="$execution->id"/>
</form>
@empty
<div>No logs found.</div>
@endforelse
</div>

View File

@ -1,7 +1,6 @@
<dialog id="createScheduledBackup" class="modal">
<form method="dialog" class="flex flex-col gap-2 rounded modal-box" wire:submit.prevent='submit'>
<x-forms.input placeholder="1 * * * *" id="frequency" label="Frequency" required/>
<x-forms.checkbox id="keep_locally" label="Keep Backups Locally"/>
<x-forms.checkbox id="save_s3" label="Save to preconfigured S3"/>
<x-forms.button onclick="createScheduledBackup.close()" type="submit">
Save

View File

@ -1,10 +1,11 @@
<div class="flex flex-wrap gap-2">
@forelse($database->scheduledBackups as $backup)
<div class="box flex flex-col">
<a class="box flex flex-col"
href="{{ route('project.database.backups.logs', [...$parameters,'backup_uuid'=> $backup->uuid]) }}">
<div>Frequency: {{$backup->frequency}}</div>
<div>Keep locally: {{$backup->keep_locally}}</div>
<div>Sync to S3: {{$backup->save_s3}}</div>
</div>
<div>Last backup: {{data_get($backup->latest_log, 'status','No backup yet')}}</div>
<div>Number of backups to keep (locally): {{$backup->number_of_backups_locally}}</div>
</a>
@empty
<div>No scheduled backups configured.</div>
@endforelse

View File

@ -0,0 +1,19 @@
<x-layout>
<h1>Backups</h1>
<livewire:project.database.heading :database="$database"/>
<x-modal modalId="startDatabase">
<x-slot:modalBody>
<livewire:activity-monitor header="Startup Logs"/>
</x-slot:modalBody>
<x-slot:modalSubmit>
<x-forms.button onclick="startDatabase.close()" type="submit">
Close
</x-forms.button>
</x-slot:modalSubmit>
</x-modal>
<div class="pt-6">
<livewire:project.database.backup-edit :backup="$backup"/>
<h3 class="py-4">Executions</h3>
<livewire:project.database.backup-executions :backup="$backup" :executions="$backup_executions"/>
</div>
</x-layout>

View File

@ -63,7 +63,8 @@
// Databases
Route::get('/project/{project_uuid}/{environment_name}/database/{database_uuid}', [DatabaseController::class, 'configuration'])->name('project.database.configuration');
Route::get('/project/{project_uuid}/{environment_name}/database/{database_uuid}/backups', [DatabaseController::class, 'backups'])->name('project.database.backups');
Route::get('/project/{project_uuid}/{environment_name}/database/{database_uuid}/backups', [DatabaseController::class, 'backups'])->name('project.database.backups.all');
Route::get('/project/{project_uuid}/{environment_name}/database/{database_uuid}/backups/{backup_uuid}', [DatabaseController::class, 'backup_logs'])->name('project.database.backups.logs');
});
Route::middleware(['auth'])->group(function () {