mirror of
https://github.com/cupcakearmy/coolify.git
synced 2024-10-22 08:04:19 +02:00
feat: upload large backups
This commit is contained in:
Andras Bacsai
parent
f35b7ab6f4
commit
9c7f40e4fe
83
app/Http/Controllers/UploadController.php
Normal file
83
app/Http/Controllers/UploadController.php
Normal file
@ -0,0 +1,83 @@
|
||||
<?php
|
||||
|
||||
namespace App\Http\Controllers;
|
||||
|
||||
use Illuminate\Routing\Controller as BaseController;
|
||||
use Illuminate\Http\JsonResponse;
|
||||
use Pion\Laravel\ChunkUpload\Exceptions\UploadFailedException;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Http\UploadedFile;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Pion\Laravel\ChunkUpload\Exceptions\UploadMissingFileException;
|
||||
use Pion\Laravel\ChunkUpload\Handler\AbstractHandler;
|
||||
use Pion\Laravel\ChunkUpload\Handler\HandlerFactory;
|
||||
use Pion\Laravel\ChunkUpload\Receiver\FileReceiver;
|
||||
|
||||
class UploadController extends BaseController
|
||||
{
|
||||
public function upload(Request $request)
|
||||
{
|
||||
$resource = getResourceByUuid(request()->route('databaseUuid'), data_get(auth()->user()->currentTeam(), 'id'));
|
||||
if (is_null($resource)) {
|
||||
return response()->json(['error' => 'You do not have permission for this database'], 500);
|
||||
}
|
||||
$receiver = new FileReceiver("file", $request, HandlerFactory::classFromRequest($request));
|
||||
|
||||
if ($receiver->isUploaded() === false) {
|
||||
throw new UploadMissingFileException();
|
||||
}
|
||||
|
||||
$save = $receiver->receive();
|
||||
|
||||
if ($save->isFinished()) {
|
||||
return $this->saveFile($save->getFile(), $resource);
|
||||
}
|
||||
|
||||
$handler = $save->handler();
|
||||
return response()->json([
|
||||
"done" => $handler->getPercentageDone(),
|
||||
'status' => true
|
||||
]);
|
||||
}
|
||||
// protected function saveFileToS3($file)
|
||||
// {
|
||||
// $fileName = $this->createFilename($file);
|
||||
|
||||
// $disk = Storage::disk('s3');
|
||||
// // It's better to use streaming Streaming (laravel 5.4+)
|
||||
// $disk->putFileAs('photos', $file, $fileName);
|
||||
|
||||
// // for older laravel
|
||||
// // $disk->put($fileName, file_get_contents($file), 'public');
|
||||
// $mime = str_replace('/', '-', $file->getMimeType());
|
||||
|
||||
// // We need to delete the file when uploaded to s3
|
||||
// unlink($file->getPathname());
|
||||
|
||||
// return response()->json([
|
||||
// 'path' => $disk->url($fileName),
|
||||
// 'name' => $fileName,
|
||||
// 'mime_type' => $mime
|
||||
// ]);
|
||||
// }
|
||||
protected function saveFile(UploadedFile $file, $resource)
|
||||
{
|
||||
$mime = str_replace('/', '-', $file->getMimeType());
|
||||
$filePath = "upload/{$resource->uuid}";
|
||||
$finalPath = storage_path("app/" . $filePath);
|
||||
$file->move($finalPath, 'restore');
|
||||
|
||||
return response()->json([
|
||||
'mime_type' => $mime
|
||||
]);
|
||||
}
|
||||
protected function createFilename(UploadedFile $file)
|
||||
{
|
||||
$extension = $file->getClientOriginalExtension();
|
||||
$filename = str_replace("." . $extension, "", $file->getClientOriginalName()); // Filename without extension
|
||||
|
||||
$filename .= "_" . md5(time()) . "." . $extension;
|
||||
|
||||
return $filename;
|
||||
}
|
||||
}
|
||||
@ -3,22 +3,24 @@
|
||||
namespace App\Livewire\Project\Database;
|
||||
|
||||
use Livewire\Component;
|
||||
use Livewire\WithFileUploads;
|
||||
use App\Models\Server;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
class Import extends Component
|
||||
{
|
||||
use WithFileUploads;
|
||||
|
||||
public $file;
|
||||
public $resource;
|
||||
public $parameters;
|
||||
public $containers;
|
||||
public bool $validated = true;
|
||||
public bool $scpInProgress = false;
|
||||
public bool $importRunning = false;
|
||||
public string $validationMsg = '';
|
||||
|
||||
public ?string $filename = null;
|
||||
public ?string $filesize = null;
|
||||
public bool $isUploading = false;
|
||||
public int $progress = 0;
|
||||
public bool $error = false;
|
||||
|
||||
public Server $server;
|
||||
public string $container;
|
||||
public array $importCommands = [];
|
||||
@ -45,7 +47,7 @@ public function getContainers()
|
||||
if (!data_get($this->parameters, 'database_uuid')) {
|
||||
abort(404);
|
||||
}
|
||||
$resource = getResourceByUuid($this->parameters['database_uuid'], data_get(auth()->user()->currentTeam(),'id'));
|
||||
$resource = getResourceByUuid($this->parameters['database_uuid'], data_get(auth()->user()->currentTeam(), 'id'));
|
||||
if (is_null($resource)) {
|
||||
abort(404);
|
||||
}
|
||||
@ -56,11 +58,6 @@ public function getContainers()
|
||||
$this->containers->push($this->container);
|
||||
}
|
||||
|
||||
if ($this->containers->count() > 1) {
|
||||
$this->validated = false;
|
||||
$this->validationMsg = 'The database service has more than one container running. Cannot import.';
|
||||
}
|
||||
|
||||
if (
|
||||
$this->resource->getMorphClass() == 'App\Models\StandaloneRedis' ||
|
||||
$this->resource->getMorphClass() == 'App\Models\StandaloneKeydb' ||
|
||||
@ -68,29 +65,27 @@ public function getContainers()
|
||||
$this->resource->getMorphClass() == 'App\Models\StandaloneClickhouse' ||
|
||||
$this->resource->getMorphClass() == 'App\Models\StandaloneMongodb'
|
||||
) {
|
||||
$this->validated = false;
|
||||
$this->validationMsg = 'This database type is not currently supported.';
|
||||
$this->dispatch('error', 'Import is not supported for this resource.');
|
||||
}
|
||||
}
|
||||
|
||||
public function runImport()
|
||||
{
|
||||
$this->validate([
|
||||
'file' => 'required|file|max:102400'
|
||||
]);
|
||||
|
||||
$this->importRunning = true;
|
||||
$this->scpInProgress = true;
|
||||
|
||||
if ($this->filename == '') {
|
||||
$this->dispatch('error', 'Please select a file to import.');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
$uploadedFilename = $this->file->store('backup-import');
|
||||
$uploadedFilename = "upload/{$this->resource->uuid}/restore";
|
||||
$path = Storage::path($uploadedFilename);
|
||||
if (!Storage::exists($uploadedFilename)) {
|
||||
$this->dispatch('error', 'The file does not exist or has been deleted.');
|
||||
return;
|
||||
}
|
||||
$tmpPath = '/tmp/' . basename($uploadedFilename);
|
||||
|
||||
// SCP the backup file to the server.
|
||||
instant_scp($path, $tmpPath, $this->server);
|
||||
$this->scpInProgress = false;
|
||||
|
||||
Storage::delete($uploadedFilename);
|
||||
$this->importCommands[] = "docker cp {$tmpPath} {$this->container}:{$tmpPath}";
|
||||
|
||||
switch ($this->resource->getMorphClass()) {
|
||||
@ -116,8 +111,7 @@ public function runImport()
|
||||
$this->dispatch('activityMonitor', $activity->id);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
$this->validated = false;
|
||||
$this->validationMsg = $e->getMessage();
|
||||
return handleError($e, $this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -27,6 +27,7 @@
|
||||
"lorisleiva/laravel-actions": "^2.7",
|
||||
"nubs/random-name-generator": "^2.2",
|
||||
"phpseclib/phpseclib": "~3.0",
|
||||
"pion/laravel-chunk-upload": "^1.5",
|
||||
"poliander/cron": "^3.0",
|
||||
"purplepixie/phpdns": "^2.1",
|
||||
"pusher/pusher-php-server": "^7.2",
|
||||
|
||||
68
composer.lock
generated
68
composer.lock
generated
@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "e095b8a9eb22df2943cbc3e9649ff9e8",
|
||||
"content-hash": "e6fd1d5c5183226a78df717b52343393",
|
||||
"packages": [
|
||||
{
|
||||
"name": "amphp/amp",
|
||||
@ -6370,6 +6370,72 @@
|
||||
},
|
||||
"time": "2021-10-28T11:13:42+00:00"
|
||||
},
|
||||
{
|
||||
"name": "pion/laravel-chunk-upload",
|
||||
"version": "v1.5.4",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/pionl/laravel-chunk-upload.git",
|
||||
"reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/pionl/laravel-chunk-upload/zipball/cfbc4292ddcace51308a4f2f446d310aa04e6133",
|
||||
"reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"illuminate/console": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
|
||||
"illuminate/filesystem": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
|
||||
"illuminate/http": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
|
||||
"illuminate/support": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^2.16.0 | ^3.52.0",
|
||||
"mockery/mockery": "^1.1.0 | ^1.3.0 | ^1.6.0",
|
||||
"overtrue/phplint": "^1.1 | ^2.0 | ^9.1",
|
||||
"phpunit/phpunit": "5.7 | 6.0 | 7.0 | 7.5 | 8.4 | ^8.5 | ^9.3 | ^10.0 | ^11.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"laravel": {
|
||||
"providers": [
|
||||
"Pion\\Laravel\\ChunkUpload\\Providers\\ChunkUploadServiceProvider"
|
||||
]
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Pion\\Laravel\\ChunkUpload\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Martin Kluska",
|
||||
"email": "martin@kluska.cz"
|
||||
}
|
||||
],
|
||||
"description": "Service for chunked upload with several js providers",
|
||||
"support": {
|
||||
"issues": "https://github.com/pionl/laravel-chunk-upload/issues",
|
||||
"source": "https://github.com/pionl/laravel-chunk-upload/tree/v1.5.4"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://revolut.me/martinpv7n",
|
||||
"type": "custom"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/pionl",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2024-03-25T15:50:07+00:00"
|
||||
},
|
||||
{
|
||||
"name": "poliander/cron",
|
||||
"version": "3.1.0",
|
||||
|
||||
44
config/chunk-upload.php
Normal file
44
config/chunk-upload.php
Normal file
@ -0,0 +1,44 @@
|
||||
<?php
|
||||
/**
|
||||
* @see https://github.com/pionl/laravel-chunk-upload
|
||||
*/
|
||||
|
||||
return [
|
||||
/*
|
||||
* The storage config
|
||||
*/
|
||||
'storage' => [
|
||||
/*
|
||||
* Returns the folder name of the chunks. The location is in storage/app/{folder_name}
|
||||
*/
|
||||
'chunks' => 'chunks',
|
||||
'disk' => 'local',
|
||||
],
|
||||
'clear' => [
|
||||
/*
|
||||
* How old chunks we should delete
|
||||
*/
|
||||
'timestamp' => '-1 HOURS',
|
||||
'schedule' => [
|
||||
'enabled' => true,
|
||||
'cron' => '25 * * * *', // run every hour on the 25th minute
|
||||
],
|
||||
],
|
||||
'chunk' => [
|
||||
// setup for the chunk naming setup to ensure same name upload at same time
|
||||
'name' => [
|
||||
'use' => [
|
||||
'session' => true, // should the chunk name use the session id? The uploader must send cookie!,
|
||||
'browser' => false, // instead of session we can use the ip and browser?
|
||||
],
|
||||
],
|
||||
],
|
||||
'handlers' => [
|
||||
// A list of handlers/providers that will be appended to existing list of handlers
|
||||
'custom' => [],
|
||||
// Overrides the list of handlers - use only what you really want
|
||||
'override' => [
|
||||
// \Pion\Laravel\ChunkUpload\Handler\DropZoneUploadHandler::class
|
||||
],
|
||||
],
|
||||
];
|
||||
@ -289,3 +289,7 @@ .fullscreen {
|
||||
.toast {
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
.dz-button {
|
||||
@apply w-full p-4 py-10 my-4 font-bold bg-white border dark:border-coolgray-400 dark:text-white dark:bg-transparent hover:dark:bg-coolgray-400;
|
||||
}
|
||||
|
||||
@ -34,12 +34,13 @@ class="relative flex flex-col p-4 bg-white box-without-bg dark:bg-coolgray-100"
|
||||
</x-modal-confirmation>
|
||||
</div>
|
||||
</form>
|
||||
<script>
|
||||
function download_file(executionId) {
|
||||
window.open('/download/backup/' + executionId, '_blank');
|
||||
}
|
||||
</script>
|
||||
|
||||
@empty
|
||||
<div>No executions found.</div>
|
||||
@endforelse
|
||||
<script>
|
||||
function download_file(executionId) {
|
||||
window.open('/download/backup/' + executionId, '_blank');
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
|
||||
@ -1,4 +1,41 @@
|
||||
<div>
|
||||
<div x-data="{ error: $wire.entangle('error'), filesize: $wire.entangle('filesize'), filename: $wire.entangle('filename'), isUploading: $wire.entangle('isUploading'), progress: $wire.entangle('progress') }">
|
||||
<script src="https://unpkg.com/dropzone@5/dist/min/dropzone.min.js"></script>
|
||||
@script
|
||||
<script>
|
||||
Dropzone.options.myDropzone = {
|
||||
chunking: true,
|
||||
method: "POST",
|
||||
maxFilesize: 1000000000,
|
||||
chunkSize: 10000000,
|
||||
createImageThumbnails: false,
|
||||
disablePreviews: true,
|
||||
parallelChunkUploads: false,
|
||||
init: function() {
|
||||
let button = this.element.querySelector('button');
|
||||
button.innerText = 'Select or drop files here...'
|
||||
this.on('sending', function(file, xhr, formData) {
|
||||
const token = document.querySelector('meta[name="csrf-token"]').getAttribute('content');
|
||||
formData.append("_token", token);
|
||||
});
|
||||
this.on("addedfile", file => {
|
||||
$wire.isUploading = true;
|
||||
});
|
||||
this.on('uploadprogress', function(file, progress, bytesSent) {
|
||||
$wire.progress = progress;
|
||||
});
|
||||
this.on('complete', function(file) {
|
||||
$wire.filename = file.name;
|
||||
$wire.filesize = Number(file.size / 1024 / 1024).toFixed(2) + ' MB';
|
||||
$wire.isUploading = false;
|
||||
});
|
||||
this.on('error', function(file, message) {
|
||||
$wire.error = true;
|
||||
$wire.$dispatch('error', message.error)
|
||||
});
|
||||
}
|
||||
};
|
||||
</script>
|
||||
@endscript
|
||||
<h2>Import Backup</h2>
|
||||
<div class="mt-2 mb-4 rounded alert-error">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="w-6 h-6 stroke-current shrink-0" fill="none" viewBox="0 0 24 24">
|
||||
@ -9,52 +46,32 @@
|
||||
</div>
|
||||
|
||||
@if (str(data_get($resource, 'status'))->startsWith('running'))
|
||||
@if (!$validated)
|
||||
<div>{{ $validationMsg }}</div>
|
||||
@else
|
||||
<form disabled wire:submit.prevent="runImport" x-data="{ isFinished: false, isUploading: false, progress: 0 }">
|
||||
@if ($resource->type() === 'standalone-postgresql')
|
||||
<x-forms.input class="mb-2" label="Custom Import Command"
|
||||
wire:model='postgresqlRestoreCommand'></x-forms.input>
|
||||
@elseif ($resource->type() === 'standalone-mysql')
|
||||
<x-forms.input class="mb-2" label="Custom Import Command"
|
||||
wire:model='mysqlRestoreCommand'></x-forms.input>
|
||||
@elseif ($resource->type() === 'standalone-mariadb')
|
||||
<x-forms.input class="mb-2" label="Custom Import Command"
|
||||
wire:model='mariadbRestoreCommand'></x-forms.input>
|
||||
@endif
|
||||
<div x-on:livewire-upload-start="isUploading = true; isFinished = false"
|
||||
x-on:livewire-upload-finish="isUploading = false; isFinished = true"
|
||||
x-on:livewire-upload-error="isUploading = false"
|
||||
x-on:livewire-upload-progress="progress = $event.detail.progress">
|
||||
<label class="block mb-2 text-sm font-medium text-gray-900 dark:text-white" for="file_input">Upload
|
||||
file</label>
|
||||
<input wire:model="file"
|
||||
class="block w-full text-sm rounded cursor-pointer text-whiteborder bg-coolgray-100 border-coolgray-400 focus:outline-none"
|
||||
aria-describedby="file_input_help" id="file_input" type="file">
|
||||
<p class="mt-1 text-sm text-neutral-500" id="file_input_help">Max file size: 256MB
|
||||
</p>
|
||||
|
||||
@error('file')
|
||||
<span class="error">{{ $message }}</span>
|
||||
@enderror
|
||||
<div x-show="isUploading">
|
||||
<progress max="100" x-bind:value="progress"
|
||||
class="progress progress-warning"></progress>
|
||||
</div>
|
||||
</div>
|
||||
<x-forms.button type="submit" class="w-full mt-4" x-show="isFinished">Import Backup</x-forms.button>
|
||||
</form>
|
||||
@if ($resource->type() === 'standalone-postgresql')
|
||||
<x-forms.input class="mb-2" label="Custom Import Command"
|
||||
wire:model='postgresqlRestoreCommand'></x-forms.input>
|
||||
@elseif ($resource->type() === 'standalone-mysql')
|
||||
<x-forms.input class="mb-2" label="Custom Import Command"
|
||||
wire:model='mysqlRestoreCommand'></x-forms.input>
|
||||
@elseif ($resource->type() === 'standalone-mariadb')
|
||||
<x-forms.input class="mb-2" label="Custom Import Command"
|
||||
wire:model='mariadbRestoreCommand'></x-forms.input>
|
||||
@endif
|
||||
|
||||
@if ($scpInProgress)
|
||||
<div>Database backup is being copied to server...</div>
|
||||
@endif
|
||||
<div x-show="isUploading">
|
||||
<progress max="100" x-bind:value="progress" class="progress progress-warning"></progress>
|
||||
</div>
|
||||
<div x-show="filename && !error">
|
||||
<div>File: <span x-text="filename ?? 'N/A'"></span> <span x-text="filesize">/ </span></div>
|
||||
<x-forms.button class="w-full my-4" wire:click='runImport'>Restore Backup</x-forms.button>
|
||||
</div>
|
||||
<form action="/upload/backup/{{ $resource->uuid }}" class="dropzone" id="my-dropzone">
|
||||
@csrf
|
||||
</form>
|
||||
|
||||
<div class="container w-full pt-4 mx-auto">
|
||||
<livewire:activity-monitor header="Database import output" />
|
||||
<div class="container w-full mx-auto">
|
||||
<livewire:activity-monitor header="Database restore output" />
|
||||
</div>
|
||||
@else
|
||||
<div>Database must be running to import a backup.</div>
|
||||
<div>Database must be running to restore a backup.</div>
|
||||
@endif
|
||||
</div>
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
use App\Http\Controllers\Controller;
|
||||
use App\Http\Controllers\MagicController;
|
||||
use App\Http\Controllers\OauthController;
|
||||
use App\Http\Controllers\UploadController;
|
||||
use App\Livewire\Admin\Index as AdminIndex;
|
||||
use App\Livewire\Dev\Compose as Compose;
|
||||
|
||||
@ -225,6 +226,7 @@
|
||||
});
|
||||
|
||||
Route::middleware(['auth'])->group(function () {
|
||||
Route::post('/upload/backup/{databaseUuid}', [UploadController::class, 'upload'])->name('upload.backup');
|
||||
Route::get('/download/backup/{executionId}', function () {
|
||||
try {
|
||||
$team = auth()->user()->currentTeam();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user