From d635e5dbaef2a670ebf64e92bc43ef82b340008b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 13 Oct 2023 15:45:24 +0200 Subject: [PATCH] fix: backup database one-by-one. --- .../Livewire/Project/Database/BackupEdit.php | 8 +- .../Livewire/Project/Database/BackupNow.php | 2 +- app/Http/Livewire/Settings/Backup.php | 4 +- app/Jobs/DatabaseBackupJob.php | 91 +++++++++++-------- ...2432_add_database_selection_to_backups.php | 34 +++++++ .../project/database/backup-edit.blade.php | 1 + .../database/backup-executions.blade.php | 9 +- 7 files changed, 104 insertions(+), 45 deletions(-) create mode 100644 database/migrations/2023_10_12_132432_add_database_selection_to_backups.php diff --git a/app/Http/Livewire/Project/Database/BackupEdit.php b/app/Http/Livewire/Project/Database/BackupEdit.php index ea217f526..951f95468 100644 --- a/app/Http/Livewire/Project/Database/BackupEdit.php +++ b/app/Http/Livewire/Project/Database/BackupEdit.php @@ -17,6 +17,7 @@ class BackupEdit extends Component 'backup.number_of_backups_locally' => 'required|integer|min:1', 'backup.save_s3' => 'required|boolean', 'backup.s3_storage_id' => 'nullable|integer', + 'backup.databases_to_backup' => 'nullable', ]; protected $validationAttributes = [ 'backup.enabled' => 'Enabled', @@ -24,6 +25,7 @@ class BackupEdit extends Component 'backup.number_of_backups_locally' => 'Number of Backups Locally', 'backup.save_s3' => 'Save to S3', 'backup.s3_storage_id' => 'S3 Storage', + 'backup.databases_to_backup' => 'Databases to Backup', ]; protected $messages = [ 'backup.s3_storage_id' => 'Select a S3 Storage', @@ -37,7 +39,6 @@ public function mount() } } - public function delete() { // TODO: Delete backup from server and add a confirmation modal @@ -49,6 +50,7 @@ public function instantSave() { try { $this->custom_validate(); + $this->backup->save(); $this->backup->refresh(); $this->emit('success', 'Backup updated successfully'); @@ -71,9 +73,11 @@ private function custom_validate() public function submit() { - ray($this->backup->s3_storage_id); try { $this->custom_validate(); + if ($this->backup->databases_to_backup == '' || $this->backup->databases_to_backup === null) { + $this->backup->databases_to_backup = null; + } $this->backup->save(); $this->backup->refresh(); $this->emit('success', 'Backup updated successfully'); diff --git a/app/Http/Livewire/Project/Database/BackupNow.php b/app/Http/Livewire/Project/Database/BackupNow.php index ea25c4744..35bd8318d 100644 --- a/app/Http/Livewire/Project/Database/BackupNow.php +++ b/app/Http/Livewire/Project/Database/BackupNow.php @@ -13,6 +13,6 @@ public function backup_now() dispatch(new DatabaseBackupJob( backup: $this->backup )); - $this->emit('success', 'Backup queued. It will be available in a few minutes'); + $this->emit('success', 'Backup queued. It will be available in a few minutes.'); } } diff --git a/app/Http/Livewire/Settings/Backup.php b/app/Http/Livewire/Settings/Backup.php index 263db027d..916344ddc 100644 --- a/app/Http/Livewire/Settings/Backup.php +++ b/app/Http/Livewire/Settings/Backup.php @@ -78,10 +78,10 @@ public function backup_now() dispatch(new DatabaseBackupJob( backup: $this->backup )); - $this->emit('success', 'Backup queued. It will be available in a few minutes'); + $this->emit('success', 'Backup queued. It will be available in a few minutes.'); } public function submit() { - $this->emit('success', 'Backup updated successfully'); + $this->emit('success', 'Backup updated successfully.'); } } diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index cc65cea9b..742ed9cee 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -66,50 +66,77 @@ public function handle(): void ray('database not running'); return; } + $databaseType = $this->database->type(); + $databasesToBackup = data_get($this->backup, 'databases_to_backup'); + + if (is_null($databasesToBackup)) { + if ($databaseType === 'standalone-postgresql') { + $databasesToBackup = [$this->database->postgres_db]; + } else { + return; + } + } else { + $databasesToBackup = explode(',', $databasesToBackup); + $databasesToBackup = array_map('trim', $databasesToBackup); + } $this->container_name = $this->database->uuid; $this->backup_dir = backup_dir() . "/databases/" . Str::of($this->team->name)->slug() . '-' . $this->team->id . '/' . $this->container_name; if ($this->database->name === 'coolify-db') { + $databasesToBackup = ['coolify']; $this->container_name = "coolify-db"; $ip = Str::slug($this->server->ip); $this->backup_dir = backup_dir() . "/coolify" . "/coolify-db-$ip"; } - $this->backup_file = "/pg-backup-customformat-" . Carbon::now()->timestamp . ".backup"; - $this->backup_location = $this->backup_dir . $this->backup_file; - - $this->backup_log = ScheduledDatabaseBackupExecution::create([ - 'filename' => $this->backup_location, - 'scheduled_database_backup_id' => $this->backup->id, - ]); - if ($this->database->type() === 'standalone-postgresql') { - $this->backup_standalone_postgresql(); + foreach ($databasesToBackup as $database) { + $size = 0; + ray('Backing up ' . $database); + try { + $this->backup_file = "/pg-dump-$database-" . Carbon::now()->timestamp . ".dmp"; + $this->backup_location = $this->backup_dir . $this->backup_file; + $this->backup_log = ScheduledDatabaseBackupExecution::create([ + 'database_name' => $database, + 'filename' => $this->backup_location, + 'scheduled_database_backup_id' => $this->backup->id, + ]); + if ($databaseType === 'standalone-postgresql') { + $this->backup_standalone_postgresql($database); + } + $size = $this->calculate_size(); + $this->remove_old_backups(); + if ($this->backup->save_s3) { + $this->upload_to_s3(); + } + $this->team->notify(new BackupSuccess($this->backup, $this->database)); + $this->backup_log->update([ + 'status' => 'success', + 'message' => $this->backup_output, + 'size' => $size, + ]); + } catch (\Throwable $e) { + $this->backup_log->update([ + 'status' => 'failed', + 'message' => $this->backup_output, + 'size' => $size, + 'filename' => null + ]); + send_internal_notification('DatabaseBackupJob failed with: ' . $e->getMessage()); + $this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output)); + throw $e; + } } - $this->calculate_size(); - $this->remove_old_backups(); - if ($this->backup->save_s3) { - $this->upload_to_s3(); - } - $this->save_backup_logs(); - $this->team->notify(new BackupSuccess($this->backup, $this->database)); - $this->backup_status = 'success'; } catch (\Throwable $e) { - $this->backup_status = 'failed'; send_internal_notification('DatabaseBackupJob failed with: ' . $e->getMessage()); - $this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output)); throw $e; - } finally { - $this->backup_log->update([ - 'status' => $this->backup_status, - ]); } } - private function backup_standalone_postgresql(): void + private function backup_standalone_postgresql(string $database): void { try { ray($this->backup_dir); $commands[] = "mkdir -p " . $this->backup_dir; - $commands[] = "docker exec $this->container_name pg_dump -Fc -U {$this->database->postgres_user} > $this->backup_location"; + $commands[] = "docker exec $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $database > $this->backup_location"; $this->backup_output = instant_remote_process($commands, $this->server); $this->backup_output = trim($this->backup_output); if ($this->backup_output === '') { @@ -119,6 +146,7 @@ private function backup_standalone_postgresql(): void } catch (\Throwable $e) { $this->add_to_backup_output($e->getMessage()); ray('Backup failed for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location . '\n\nError:' . $e->getMessage()); + throw $e; } } @@ -131,9 +159,9 @@ private function add_to_backup_output($output): void } } - private function calculate_size(): void + private function calculate_size() { - $this->size = instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server); + return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false); } private function remove_old_backups(): void @@ -180,13 +208,4 @@ private function upload_to_s3(): void instant_remote_process([$command], $this->server); } } - - private function save_backup_logs(): void - { - $this->backup_log->update([ - 'status' => $this->backup_status, - 'message' => $this->backup_output, - 'size' => $this->size, - ]); - } } diff --git a/database/migrations/2023_10_12_132432_add_database_selection_to_backups.php b/database/migrations/2023_10_12_132432_add_database_selection_to_backups.php new file mode 100644 index 000000000..40ade57e8 --- /dev/null +++ b/database/migrations/2023_10_12_132432_add_database_selection_to_backups.php @@ -0,0 +1,34 @@ +text('databases_to_backup')->nullable(); + }); + Schema::table('scheduled_database_backup_executions', function (Blueprint $table) { + $table->string('database_name')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('scheduled_database_backups', function (Blueprint $table) { + $table->dropColumn('databases_to_backup'); + }); + Schema::table('scheduled_database_backup_executions', function (Blueprint $table) { + $table->dropColumn('database_name'); + }); + } +}; diff --git a/resources/views/livewire/project/database/backup-edit.blade.php b/resources/views/livewire/project/database/backup-edit.blade.php index 412595958..5a0559a2d 100644 --- a/resources/views/livewire/project/database/backup-edit.blade.php +++ b/resources/views/livewire/project/database/backup-edit.blade.php @@ -26,6 +26,7 @@ @endif
+
diff --git a/resources/views/livewire/project/database/backup-executions.blade.php b/resources/views/livewire/project/database/backup-executions.blade.php index 83ae8f1d1..30af9e93e 100644 --- a/resources/views/livewire/project/database/backup-executions.blade.php +++ b/resources/views/livewire/project/database/backup-executions.blade.php @@ -1,18 +1,19 @@ -
+
@forelse($executions as $execution)
data_get($execution, 'status') === 'success', 'border-red-500' => data_get($execution, 'status') === 'failed', ])> -
Started At: {{ data_get($execution, 'created_at') }}
+
Database: {{ data_get($execution, 'database_name', 'N/A') }}
Status: {{ data_get($execution, 'status') }}
+
Started At: {{ data_get($execution, 'created_at') }}
@if (data_get($execution, 'message'))
Message: {{ data_get($execution, 'message') }}
@endif
Size: {{ data_get($execution, 'size') }} B / {{ round((int) data_get($execution, 'size') / 1024, 2) }} - kB / {{ round((int) data_get($execution, 'size') / 1024 / 1024, 2) }} MB + kB / {{ round((int) data_get($execution, 'size') / 1024 / 1024, 3) }} MB
-
Location: {{ data_get($execution, 'filename') }}
+
Location: {{ data_get($execution, 'filename', 'N/A') }}
@empty