Merge pull request #6 from EliotBerriot/2-forget

Fix #2: support pruning and forget via snapshot policies

README.md

@@ -10,6 +10,7 @@ Autorestic is a wrapper around the amazing [restic](https://restic.net/). While
 - Config files, no CLI
 - Predictable
 - Backup locations to multiple backends
+- Snapshot policies and pruning
 - Simple interface
 - Fully encrypted
 
@@ -32,7 +33,7 @@ locations:
   home:
     from: /home/me
     to: remote
-  
+
   important:
     from: /path/to/important/stuff
     to:
@@ -45,7 +46,7 @@ backends:
     path: 'myBucket:backup/home'
     B2_ACCOUNT_ID: account_id
     B2_ACCOUNT_KEY: account_key
-  
+
   hdd:
     type: local
     path: /mnt/my_external_storage
@@ -57,7 +58,7 @@ Then we check if everything is correct by running the `check` command. We will p
 autorestic check -a
 ```
 
-If we would check only one location we could run the following: `autorestic -l home check`. 
+If we would check only one location we could run the following: `autorestic -l home check`.
 
 ### Backup
 
@@ -108,7 +109,7 @@ For each backend you need to specify the right variables as shown in the example
 ##### `path`
 
 The path on the remote server.
-For object storages as 
+For object storages as
 
 ##### Example
 
@@ -121,7 +122,54 @@ backends:
     B2_ACCOUNT_KEY: backblaze_account_key
 ```
 
+## Pruning and snapshot policies
 
+Autorestic supports declaring snapshot policies for location to avoid keeping old snapshot around if you don't need them.
+
+This is based on [Restic's snapshots policies](https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy), and can be enabled for each location as shown below:
+
+```yaml
+locations:
+  etc:
+    from: /etc
+    to: local
+    keep:
+      # options matches the --keep-* options used in the restic forget CLI
+      # cf https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy
+      last: 5             # always keep at least 5 snapshots
+      hourly: 3           # keep 3 last hourly shapshots
+      daily: 4            # keep 4 last daily shapshots
+      weekly: 1           # keep 1 last weekly shapshots
+      monthly: 12         # keep 12 last monthly shapshots
+      yearly: 7           # keep 7 last yearly shapshots
+      within: "2w"        # keep snapshots from the last 2 weeks
+```
+
+Pruning can be triggered using `autorestic forget -a`, for all locations, or selectively with `autorestic forget -l <location>`. **please note that contrary to the restic CLI, `restic forget` will call `restic prune` internally.**
+
+
+
+Run with the `--dry-run` flag to only print information about the process without actually pruning the snapshots. This is especially useful for debugging or testing policies:
+
+```
+$ autorestic forget -a --dry-run --verbose
+
+Configuring Backends
+local : Done ✓
+
+Removing old shapshots according to policy
+etc ▶ local : Removing old spnapshots… ⏳
+etc ▶ local : Running in dry-run mode, not touching data
+etc ▶ local : Forgeting old snapshots… ⏳Applying Policy: all snapshots within 2d of the newest
+keep 3 snapshots:
+ID        Time                 Host        Tags        Reasons    Paths
+-----------------------------------------------------------------------------
+531b692a  2019-12-02 12:07:28  computer                within 2w  /etc
+51659674  2019-12-02 12:08:46  computer                within 2w  /etc
+f8f8f976  2019-12-02 12:11:08  computer                within 2w  /etc
+-----------------------------------------------------------------------------
+3 snapshots
+```
 
 ##### Note
 

src/autorestic.ts

@@ -18,8 +18,9 @@ export const { _: commands, ...flags } = minimist(process.argv.slice(2), {
     a: 'all',
     l: 'location',
     b: 'backend',
+    d: 'dry-run',
   },
-  boolean: ['a'],
+  boolean: ['a', 'd'],
   string: ['l', 'b'],
 })
 

src/forget.ts

@@ -0,0 +1,60 @@
+import { Writer } from 'clitastic'
+
+import { config, VERBOSE } from './autorestic'
+import { getEnvFromBackend } from './backend'
+import { Locations, Location, ForgetPolicy, Flags } from './types'
+import { exec, ConfigError } from './utils'
+
+export const forgetSingle = (dryRun: boolean, name: string, from: string, to: string, policy: ForgetPolicy) => {
+  if (!config) throw ConfigError
+  const writer = new Writer(name + to.blue + ' : ' + 'Removing old spnapshots… ⏳')
+  const backend = config.backends[to]
+  const flags = [] as any[]
+  for (const [name, value] of Object.entries(policy)) {
+    flags.push(`--keep-${name}`)
+    flags.push(value)
+  }
+  if (dryRun) {
+    flags.push('--dry-run')
+  }
+  const env = getEnvFromBackend(backend)
+  writer.appendLn(name + to.blue + ' : ' + 'Forgeting old snapshots… ⏳')
+  const cmd = exec('restic', ['forget', '--path', from, '--prune', ...flags], {env})
+
+  if (VERBOSE) console.log(cmd.out, cmd.err)
+  writer.done(name + to.blue + ' : ' + 'Done ✓'.green)
+}
+
+export const forgetLocation = (dryRun: boolean, name: string, backup: Location, policy?: ForgetPolicy) => {
+  const display = name.yellow + ' ▶ '
+  if (!policy) {
+    console.log(display + 'skipping, no policy declared')
+  }
+  else {
+    if (Array.isArray(backup.to)) {
+      let first = true
+      for (const t of backup.to) {
+        const nameOrBlankSpaces: string = first
+          ? display
+          : new Array(name.length + 3).fill(' ').join('')
+        forgetSingle(dryRun, nameOrBlankSpaces, backup.from, t, policy)
+        if (first) first = false
+      }
+    } else forgetSingle(dryRun, display, backup.from, backup.to, policy)
+  }
+  }
+
+export const forgetAll = (dryRun: boolean, backups?: Locations) => {
+  if (!config) throw ConfigError
+  if (!backups) {
+    backups = config.locations
+  }
+
+  console.log('\nRemoving old shapshots according to policy'.underline.grey)
+  if (dryRun) console.log('Running in dry-run mode, not touching data\n'.yellow)
+
+  for (const [name, backup] of Object.entries(backups)) {
+    var policy = config.locations[name].keep
+    forgetLocation(dryRun, name, backup, policy)
+  }
+}

src/handlers.ts

@@ -7,6 +7,7 @@ import { join, resolve } from 'path'
 import { config, INSTALL_DIR, VERSION } from './autorestic'
 import { checkAndConfigureBackends, getEnvFromBackend } from './backend'
 import { backupAll } from './backup'
+import { forgetAll } from './forget'
 import { Backends, Flags, Locations } from './types'
 import {
   checkIfCommandIsAvailable,
@@ -102,6 +103,22 @@ const handlers: Handlers = {
       w.done(name.green + '\t\tDone 🎉')
     }
   },
+  forget(args, flags) {
+    if (!config) throw ConfigError
+    checkIfResticIsAvailable()
+    const locations: Locations = parseLocations(flags)
+
+    const backends = new Set<string>()
+    for (const to of Object.values(locations).map(location => location.to))
+      Array.isArray(to) ? to.forEach(t => backends.add(t)) : backends.add(to)
+
+    checkAndConfigureBackends(
+      filterObjectByKey(config.backends, Array.from(backends))
+    )
+    forgetAll(flags['dry-run'], locations)
+
+    console.log('\nFinished!'.underline + ' 🎉')
+  },
   exec(args, flags) {
     checkIfResticIsAvailable()
     const backends = parseBackend(flags)
@@ -219,6 +236,7 @@ export const help = () => {
       '\nCommands:'.yellow +
       '\n  check    [-b, --backend]  [-a, --all]                                 Check backends' +
       '\n  backup   [-l, --location] [-a, --all]                                 Backup all or specified locations' +
+      '\n  forget   [-l, --location] [-a, --all] [--dry-run]                     Forget old snapshots according to declared policies' +
       '\n  restore  [-l, --location] [-- --target <out dir>]                     Restore all or specified locations' +
       '\n' +
       '\n  exec     [-b, --backend]  [-a, --all] <command> -- [native options]   Execute native restic command' +

src/types.ts

@@ -62,9 +62,21 @@ export type Backend =
 
 export type Backends = { [name: string]: Backend }
 
+export type ForgetPolicy = {
+  last?: number,
+  hourly?: number,
+  daily?: number,
+  weekly?: number,
+  monthly?: number,
+  yearly?: number,
+  within?: string,
+  tags?: string[],
+}
+
 export type Location = {
   from: string
   to: string | string[]
+  keep?: ForgetPolicy
 }
 
 export type Locations = { [name: string]: Location }