From 57ffa1e3faebbb8b5da725f204970abf35e8f454 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 2 Dec 2019 14:20:19 +0100 Subject: [PATCH] Fix #2: support pruning and forget via snapshot policies --- README.md | 57 ++++++++++++++++++++++++++++++++++++++++---- src/autorestic.ts | 3 ++- src/forget.ts | 60 +++++++++++++++++++++++++++++++++++++++++++++++ src/handlers.ts | 18 ++++++++++++++ src/types.ts | 12 ++++++++++ 5 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 src/forget.ts diff --git a/README.md b/README.md index 65ffb62..b690579 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Autorestic is a wrapper around the amazing [restic](https://restic.net/). While - Config files, no CLI - Predictable - Backup locations to multiple backends +- Snapshot policies and pruning - Simple interface - Fully encrypted @@ -32,7 +33,7 @@ locations: home: from: /home/me to: remote - + important: from: /path/to/important/stuff to: @@ -45,7 +46,7 @@ backends: path: 'myBucket:backup/home' B2_ACCOUNT_ID: account_id B2_ACCOUNT_KEY: account_key - + hdd: type: local path: /mnt/my_external_storage @@ -57,7 +58,7 @@ Then we check if everything is correct by running the `check` command. We will p autorestic check -a ``` -If we would check only one location we could run the following: `autorestic -l home check`. +If we would check only one location we could run the following: `autorestic -l home check`. ### Backup @@ -108,7 +109,7 @@ For each backend you need to specify the right variables as shown in the example ##### `path` The path on the remote server. -For object storages as +For object storages as ##### Example @@ -121,9 +122,55 @@ backends: B2_ACCOUNT_KEY: backblaze_account_key ``` +## Pruning and snapshot policies +Autorestic supports declaring snapshot policies for location to avoid keeping old snapshot around if you don't need them. + +This is based on [Restic's snapshots policies](https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy), and can be enabled for each location as shown below: + +```yaml +locations: + etc: + from: /etc + to: local + keep: + # options matches the --keep-* options used in the restic forget CLI + # cf https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy + last: 5 # always keep at least 5 snapshots + hourly: 3 # keep 3 last hourly shapshots + daily: 4 # keep 4 last daily shapshots + weekly: 1 # keep 1 last weekly shapshots + monthly: 12 # keep 12 last monthly shapshots + yearly: 7 # keep 7 last yearly shapshots + within: "2w" # keep snapshots from the last 2 weeks +``` + +Pruning can be triggered using `autorestic forget -a`, for all locations, or selectively with `autorestic forget -l `. **please note that contrary to the restic CLI, `restic forget` will call `restic prune` internally.** + + + +Run with the `--dry-run` flag to only print information about the process without actually pruning the snapshots. This is especially useful for debugging or testing policies: + +``` +$ autorestic forget -a --dry-run --verbose + +Configuring Backends +local : Done ✓ + +Removing old shapshots according to policy +etc ▶ local : Removing old spnapshots… ⏳ +etc ▶ local : Running in dry-run mode, not touching data +etc ▶ local : Forgeting old snapshots… ⏳Applying Policy: all snapshots within 2d of the newest +keep 3 snapshots: +ID Time Host Tags Reasons Paths +----------------------------------------------------------------------------- +531b692a 2019-12-02 12:07:28 computer within 2w /etc +51659674 2019-12-02 12:08:46 computer within 2w /etc +f8f8f976 2019-12-02 12:11:08 computer within 2w /etc +----------------------------------------------------------------------------- +3 snapshots +``` ##### Note Note that the data is automatically encrypted on the server. The key will be generated and added to your config file. Every backend will have a separate key. You should keep a copy of the keys somewhere in case your server dies. Otherwise DATA IS LOST! - diff --git a/src/autorestic.ts b/src/autorestic.ts index 5eb7d85..f90d59f 100644 --- a/src/autorestic.ts +++ b/src/autorestic.ts @@ -18,8 +18,9 @@ export const { _: commands, ...flags } = minimist(process.argv.slice(2), { a: 'all', l: 'location', b: 'backend', + d: 'dry-run', }, - boolean: ['a'], + boolean: ['a', 'd'], string: ['l', 'b'], }) diff --git a/src/forget.ts b/src/forget.ts new file mode 100644 index 0000000..0f6468a --- /dev/null +++ b/src/forget.ts @@ -0,0 +1,60 @@ +import { Writer } from 'clitastic' + +import { config, VERBOSE } from './autorestic' +import { getEnvFromBackend } from './backend' +import { Locations, Location, ForgetPolicy, Flags } from './types' +import { exec, ConfigError } from './utils' + +export const forgetSingle = (dryRun: boolean, name: string, from: string, to: string, policy: ForgetPolicy) => { + if (!config) throw ConfigError + const writer = new Writer(name + to.blue + ' : ' + 'Removing old spnapshots… ⏳') + const backend = config.backends[to] + const flags = [] as any[] + for (const [name, value] of Object.entries(policy)) { + flags.push(`--keep-${name}`) + flags.push(value) + } + if (dryRun) { + flags.push('--dry-run') + } + const env = getEnvFromBackend(backend) + writer.appendLn(name + to.blue + ' : ' + 'Forgeting old snapshots… ⏳') + const cmd = exec('restic', ['forget', '--path', from, '--prune', ...flags], {env}) + + if (VERBOSE) console.log(cmd.out, cmd.err) + writer.done(name + to.blue + ' : ' + 'Done ✓'.green) +} + +export const forgetLocation = (dryRun: boolean, name: string, backup: Location, policy?: ForgetPolicy) => { + const display = name.yellow + ' ▶ ' + if (!policy) { + console.log(display + 'skipping, no policy declared') + } + else { + if (Array.isArray(backup.to)) { + let first = true + for (const t of backup.to) { + const nameOrBlankSpaces: string = first + ? display + : new Array(name.length + 3).fill(' ').join('') + forgetSingle(dryRun, nameOrBlankSpaces, backup.from, t, policy) + if (first) first = false + } + } else forgetSingle(dryRun, display, backup.from, backup.to, policy) + } + } + +export const forgetAll = (dryRun: boolean, backups?: Locations) => { + if (!config) throw ConfigError + if (!backups) { + backups = config.locations + } + + console.log('\nRemoving old shapshots according to policy'.underline.grey) + if (dryRun) console.log('Running in dry-run mode, not touching data\n'.yellow) + + for (const [name, backup] of Object.entries(backups)) { + var policy = config.locations[name].keep + forgetLocation(dryRun, name, backup, policy) + } +} diff --git a/src/handlers.ts b/src/handlers.ts index 4016bdc..734f11d 100644 --- a/src/handlers.ts +++ b/src/handlers.ts @@ -7,6 +7,7 @@ import { join, resolve } from 'path' import { config, INSTALL_DIR, VERSION } from './autorestic' import { checkAndConfigureBackends, getEnvFromBackend } from './backend' import { backupAll } from './backup' +import { forgetAll } from './forget' import { Backends, Flags, Locations } from './types' import { checkIfCommandIsAvailable, @@ -102,6 +103,22 @@ const handlers: Handlers = { w.done(name.green + '\t\tDone 🎉') } }, + forget(args, flags) { + if (!config) throw ConfigError + checkIfResticIsAvailable() + const locations: Locations = parseLocations(flags) + + const backends = new Set() + for (const to of Object.values(locations).map(location => location.to)) + Array.isArray(to) ? to.forEach(t => backends.add(t)) : backends.add(to) + + checkAndConfigureBackends( + filterObjectByKey(config.backends, Array.from(backends)) + ) + forgetAll(flags['dry-run'], locations) + + console.log('\nFinished!'.underline + ' 🎉') + }, exec(args, flags) { checkIfResticIsAvailable() const backends = parseBackend(flags) @@ -219,6 +236,7 @@ export const help = () => { '\nCommands:'.yellow + '\n check [-b, --backend] [-a, --all] Check backends' + '\n backup [-l, --location] [-a, --all] Backup all or specified locations' + + '\n forget [-l, --location] [-a, --all] [--dry-run] Forget old snapshots according to declared policies' + '\n restore [-l, --location] [-- --target ] Restore all or specified locations' + '\n' + '\n exec [-b, --backend] [-a, --all] -- [native options] Execute native restic command' + diff --git a/src/types.ts b/src/types.ts index cbcf86d..13ef81c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -62,9 +62,21 @@ export type Backend = export type Backends = { [name: string]: Backend } +export type ForgetPolicy = { + last?: number, + hourly?: number, + daily?: number, + weekly?: number, + monthly?: number, + yearly?: number, + within?: string, + tags?: string[], +} + export type Location = { from: string to: string | string[] + keep?: ForgetPolicy } export type Locations = { [name: string]: Location }