Fix #2: support pruning and forget via snapshot policies

This commit is contained in:
Eliot Berriot 2019-12-02 14:20:19 +01:00
parent 652158d1ed
commit 57ffa1e3fa
No known key found for this signature in database
GPG Key ID: 6B501DFD73514E14
5 changed files with 144 additions and 6 deletions

View File

@ -10,6 +10,7 @@ Autorestic is a wrapper around the amazing [restic](https://restic.net/). While
- Config files, no CLI
- Predictable
- Backup locations to multiple backends
- Snapshot policies and pruning
- Simple interface
- Fully encrypted
@ -32,7 +33,7 @@ locations:
home:
from: /home/me
to: remote
important:
from: /path/to/important/stuff
to:
@ -45,7 +46,7 @@ backends:
path: 'myBucket:backup/home'
B2_ACCOUNT_ID: account_id
B2_ACCOUNT_KEY: account_key
hdd:
type: local
path: /mnt/my_external_storage
@ -57,7 +58,7 @@ Then we check if everything is correct by running the `check` command. We will p
autorestic check -a
```
If we would check only one location we could run the following: `autorestic -l home check`.
If we would check only one location we could run the following: `autorestic -l home check`.
### Backup
@ -108,7 +109,7 @@ For each backend you need to specify the right variables as shown in the example
##### `path`
The path on the remote server.
For object storages as
For object storages as
##### Example
@ -121,9 +122,55 @@ backends:
B2_ACCOUNT_KEY: backblaze_account_key
```
## Pruning and snapshot policies
Autorestic supports declaring snapshot policies for location to avoid keeping old snapshot around if you don't need them.
This is based on [Restic's snapshots policies](https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy), and can be enabled for each location as shown below:
```yaml
locations:
etc:
from: /etc
to: local
keep:
# options matches the --keep-* options used in the restic forget CLI
# cf https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy
last: 5 # always keep at least 5 snapshots
hourly: 3 # keep 3 last hourly shapshots
daily: 4 # keep 4 last daily shapshots
weekly: 1 # keep 1 last weekly shapshots
monthly: 12 # keep 12 last monthly shapshots
yearly: 7 # keep 7 last yearly shapshots
within: "2w" # keep snapshots from the last 2 weeks
```
Pruning can be triggered using `autorestic forget -a`, for all locations, or selectively with `autorestic forget -l <location>`. **please note that contrary to the restic CLI, `restic forget` will call `restic prune` internally.**
Run with the `--dry-run` flag to only print information about the process without actually pruning the snapshots. This is especially useful for debugging or testing policies:
```
$ autorestic forget -a --dry-run --verbose
Configuring Backends
local : Done ✓
Removing old shapshots according to policy
etc ▶ local : Removing old spnapshots… ⏳
etc ▶ local : Running in dry-run mode, not touching data
etc ▶ local : Forgeting old snapshots… ⏳Applying Policy: all snapshots within 2d of the newest
keep 3 snapshots:
ID Time Host Tags Reasons Paths
-----------------------------------------------------------------------------
531b692a 2019-12-02 12:07:28 computer within 2w /etc
51659674 2019-12-02 12:08:46 computer within 2w /etc
f8f8f976 2019-12-02 12:11:08 computer within 2w /etc
-----------------------------------------------------------------------------
3 snapshots
```
##### Note
Note that the data is automatically encrypted on the server. The key will be generated and added to your config file. Every backend will have a separate key. You should keep a copy of the keys somewhere in case your server dies. Otherwise DATA IS LOST!

View File

@ -18,8 +18,9 @@ export const { _: commands, ...flags } = minimist(process.argv.slice(2), {
a: 'all',
l: 'location',
b: 'backend',
d: 'dry-run',
},
boolean: ['a'],
boolean: ['a', 'd'],
string: ['l', 'b'],
})

60
src/forget.ts Normal file
View File

@ -0,0 +1,60 @@
import { Writer } from 'clitastic'
import { config, VERBOSE } from './autorestic'
import { getEnvFromBackend } from './backend'
import { Locations, Location, ForgetPolicy, Flags } from './types'
import { exec, ConfigError } from './utils'
export const forgetSingle = (dryRun: boolean, name: string, from: string, to: string, policy: ForgetPolicy) => {
if (!config) throw ConfigError
const writer = new Writer(name + to.blue + ' : ' + 'Removing old spnapshots… ⏳')
const backend = config.backends[to]
const flags = [] as any[]
for (const [name, value] of Object.entries(policy)) {
flags.push(`--keep-${name}`)
flags.push(value)
}
if (dryRun) {
flags.push('--dry-run')
}
const env = getEnvFromBackend(backend)
writer.appendLn(name + to.blue + ' : ' + 'Forgeting old snapshots… ⏳')
const cmd = exec('restic', ['forget', '--path', from, '--prune', ...flags], {env})
if (VERBOSE) console.log(cmd.out, cmd.err)
writer.done(name + to.blue + ' : ' + 'Done ✓'.green)
}
export const forgetLocation = (dryRun: boolean, name: string, backup: Location, policy?: ForgetPolicy) => {
const display = name.yellow + ' ▶ '
if (!policy) {
console.log(display + 'skipping, no policy declared')
}
else {
if (Array.isArray(backup.to)) {
let first = true
for (const t of backup.to) {
const nameOrBlankSpaces: string = first
? display
: new Array(name.length + 3).fill(' ').join('')
forgetSingle(dryRun, nameOrBlankSpaces, backup.from, t, policy)
if (first) first = false
}
} else forgetSingle(dryRun, display, backup.from, backup.to, policy)
}
}
export const forgetAll = (dryRun: boolean, backups?: Locations) => {
if (!config) throw ConfigError
if (!backups) {
backups = config.locations
}
console.log('\nRemoving old shapshots according to policy'.underline.grey)
if (dryRun) console.log('Running in dry-run mode, not touching data\n'.yellow)
for (const [name, backup] of Object.entries(backups)) {
var policy = config.locations[name].keep
forgetLocation(dryRun, name, backup, policy)
}
}

View File

@ -7,6 +7,7 @@ import { join, resolve } from 'path'
import { config, INSTALL_DIR, VERSION } from './autorestic'
import { checkAndConfigureBackends, getEnvFromBackend } from './backend'
import { backupAll } from './backup'
import { forgetAll } from './forget'
import { Backends, Flags, Locations } from './types'
import {
checkIfCommandIsAvailable,
@ -102,6 +103,22 @@ const handlers: Handlers = {
w.done(name.green + '\t\tDone 🎉')
}
},
forget(args, flags) {
if (!config) throw ConfigError
checkIfResticIsAvailable()
const locations: Locations = parseLocations(flags)
const backends = new Set<string>()
for (const to of Object.values(locations).map(location => location.to))
Array.isArray(to) ? to.forEach(t => backends.add(t)) : backends.add(to)
checkAndConfigureBackends(
filterObjectByKey(config.backends, Array.from(backends))
)
forgetAll(flags['dry-run'], locations)
console.log('\nFinished!'.underline + ' 🎉')
},
exec(args, flags) {
checkIfResticIsAvailable()
const backends = parseBackend(flags)
@ -219,6 +236,7 @@ export const help = () => {
'\nCommands:'.yellow +
'\n check [-b, --backend] [-a, --all] Check backends' +
'\n backup [-l, --location] [-a, --all] Backup all or specified locations' +
'\n forget [-l, --location] [-a, --all] [--dry-run] Forget old snapshots according to declared policies' +
'\n restore [-l, --location] [-- --target <out dir>] Restore all or specified locations' +
'\n' +
'\n exec [-b, --backend] [-a, --all] <command> -- [native options] Execute native restic command' +

View File

@ -62,9 +62,21 @@ export type Backend =
export type Backends = { [name: string]: Backend }
export type ForgetPolicy = {
last?: number,
hourly?: number,
daily?: number,
weekly?: number,
monthly?: number,
yearly?: number,
within?: string,
tags?: string[],
}
export type Location = {
from: string
to: string | string[]
keep?: ForgetPolicy
}
export type Locations = { [name: string]: Location }