changelog

.gitignore

@@ -1,10 +1,19 @@
+# Editors
+.idea
+
+# Node
 node_modules/
 package-lock.json
-.idea
 yarn.lock
 
+# Build & Runtime
 bin
 lib
 data
+restore
+docker
+Dockerfile
 
-.autorestic.yml
+# Config
+.autorestic.yml
+.docker.yml

CHANGELOG.md

@@ -0,0 +1,53 @@
+# Changelog
+
+## 0.13
+
+- Restored files are now without the prefix path.
+- Support for making backups of docker volumes and restoring them (not incremental).
+- Show error to user during backup
+
+## 0.12
+
+- fix self update on linux (Fix #15)
+
+## 0.11
+
+- tilde in arguments (Fix #14)
+
+## 0.10
+
+- Show elapsed time (Fix #12)
+- Remove some code duplication
+- New info command to quickly show an overview of your config (Fix #11)
+
+## 0.9
+
+- Hooks
+- Cleanup
+
+## 0.8
+
+- Support for native flags in the backup and forget commands.
+- Forget cleanup
+
+## 0.7
+
+- Cleanup
+- Support for excluding files
+- Ability to prune keeping the last x snapshots according to restic policy rules
+
+## 0.6
+
+- support for absolute paths
+
+## 0.5
+
+- config optional if not required for current operation
+
+## 0.4
+
+- show version number
+
+## 0.3
+
+- test autoupdate function

README.md

@@ -22,6 +22,7 @@ Autorestic is a wrapper around the amazing [restic](https://restic.net/). While
   * [Hooks](#before--after-hooks)
 * [Backends](#-backends)
 * [Commands](#-commands)
+* [Examples](#-examples)
 
 ## 🛳 Installation
 
@@ -281,12 +282,6 @@ autorestic restore -l home --from hdd --to /path/where/to/restore
 
 This will restore the location `home` to the `/path/where/to/restore` folder and taking the data from the backend `hdd`
 
-```
-autorestic restore
-```
-
-Performes a backup of all locations if the `-a` flag is passed. To only backup some locations pass one or more `-l` or `--location` flags.
-
 ### Forget
 
 
@@ -323,6 +318,22 @@ Uninstall both restic and autorestic
 
 Upgrades both restic and autorestic automagically
 
+## 🐣 Examples
+
+### List all the snapshots for all the backends
+
+```
+autorestic -a exec snapshots
+```
+
+### Unlock a locked repository
+
+⚠️ Only do this if you know what you are doing. E.g. if you accidentally cancelled a running operation
+
+```
+autorestic -b my-backend exec unlock
+```
+
 ## Contributors
 
 This amazing people helped the project!

src/autorestic.ts

@@ -25,20 +25,22 @@ export const { _: commands, ...flags } = minimist(process.argv.slice(2), {
 	string: ['l', 'b'],
 })
 
-export const VERSION = '0.10'
+export const VERSION = '0.13'
 export const INSTALL_DIR = '/usr/local/bin'
 export const VERBOSE = flags.verbose
 
 export const config = init()
 
 
-function main() {
+async function main() {
 	if (commands.length < 1) return help()
 
 	const command: string = commands[0]
 	const args: string[] = commands.slice(1)
-	;(handlers[command] || error)(args, flags)
+
+	const fn = handlers[command] || error
+	await fn(args, flags)
 }
 
 
-main()
+main().catch((e: Error) => console.error(e.message))

src/backend.ts

@@ -2,7 +2,7 @@ import { Writer } from 'clitastic'
 
 import { config, VERBOSE } from './autorestic'
 import { Backend, Backends, Locations } from './types'
-import { exec, ConfigError } from './utils'
+import { exec, ConfigError, pathRelativeToConfigFile } from './utils'
 
 
 
@@ -11,7 +11,7 @@ const ALREADY_EXISTS = /(?=.*already)(?=.*config).*/
 export const getPathFromBackend = (backend: Backend): string => {
 	switch (backend.type) {
 		case 'local':
-			return backend.path
+			return pathRelativeToConfigFile(backend.path)
 		case 'b2':
 		case 'azure':
 		case 'gs':

src/backup.ts

@@ -1,8 +1,10 @@
 import { Writer } from 'clitastic'
+import { mkdirSync } from 'fs'
 
 import { config, VERBOSE } from './autorestic'
 import { getEnvFromBackend } from './backend'
-import { Locations, Location } from './types'
+import { LocationFromPrefixes } from './config'
+import { Locations, Location, Backend } from './types'
 import {
 	exec,
 	ConfigError,
@@ -10,27 +12,72 @@ import {
 	getFlagsFromLocation,
 	makeArrayIfIsNot,
 	execPlain,
-	MeasureDuration, fill,
+	MeasureDuration,
+	fill,
+	decodeLocationFromPrefix,
+	checkIfDockerVolumeExistsOrFail,
+	getPathFromVolume,
 } from './utils'
 
 
 
+export const backupFromFilesystem = (from: string, location: Location, backend: Backend, tags?: string[]) => {
+	const path = pathRelativeToConfigFile(from)
+
+	const { out, err, status } = exec(
+		'restic',
+		['backup', '.', ...getFlagsFromLocation(location, 'backup')],
+		{ env: getEnvFromBackend(backend), cwd: path },
+	)
+
+	if (VERBOSE) console.log(out, err)
+	if (status != 0 || err.length > 0)
+		throw new Error(err)
+}
+
+export const backupFromVolume = (volume: string, location: Location, backend: Backend) => {
+	const tmp = getPathFromVolume(volume)
+	try {
+		mkdirSync(tmp)
+		checkIfDockerVolumeExistsOrFail(volume)
+
+		// For incremental backups. Unfortunately due to how the docker mounts work the permissions get lost.
+		// execPlain(`docker run --rm -v ${volume}:/data -v ${tmp}:/backup alpine cp -aT /data /backup`)
+		execPlain(`docker run --rm -v ${volume}:/data -v ${tmp}:/backup alpine tar cf /backup/archive.tar -C /data .`)
+
+		backupFromFilesystem(tmp, location, backend)
+	} catch (e) {
+		throw e
+	} finally {
+		execPlain(`rm -rf ${tmp}`)
+	}
+}
+
 export const backupSingle = (name: string, to: string, location: Location) => {
 	if (!config) throw ConfigError
 	const delta = new MeasureDuration()
 	const writer = new Writer(name + to.blue + ' : ' + 'Backing up... ⏳')
 
-	const backend = config.backends[to]
-	const path = pathRelativeToConfigFile(location.from)
+	try {
+		const backend = config.backends[to]
+		const [type, value] = decodeLocationFromPrefix(location.from)
 
-	const cmd = exec(
-		'restic',
-		['backup', path, ...getFlagsFromLocation(location, 'backup')],
-		{ env: getEnvFromBackend(backend) },
-	)
+		switch (type) {
 
-	if (VERBOSE) console.log(cmd.out, cmd.err)
-	writer.done(`${name}${to.blue} : ${'Done ✓'.green} (${delta.finished(true)})`)
+			case LocationFromPrefixes.Filesystem:
+				backupFromFilesystem(value, location, backend)
+				break
+
+			case LocationFromPrefixes.DockerVolume:
+				backupFromVolume(value, location, backend)
+				break
+
+		}
+
+		writer.done(`${name}${to.blue} : ${'Done ✓'.green} (${delta.finished(true)})`)
+	} catch (e) {
+		writer.done(`${name}${to.blue} : ${'Failed!'.red} (${delta.finished(true)}) ${e.message}`)
+	}
 }
 
 export const backupLocation = (name: string, location: Location) => {
@@ -40,8 +87,8 @@ export const backupLocation = (name: string, location: Location) => {
 
 	if (location.hooks && location.hooks.before)
 		for (const command of makeArrayIfIsNot(location.hooks.before)) {
-			const cmd = execPlain(command)
-			if (cmd) console.log(cmd.out, cmd.err)
+			const cmd = execPlain(command, {})
+			console.log(cmd.out, cmd.err)
 		}
 
 	for (const t of makeArrayIfIsNot(location.to)) {
@@ -52,7 +99,7 @@ export const backupLocation = (name: string, location: Location) => {
 	if (location.hooks && location.hooks.after)
 		for (const command of makeArrayIfIsNot(location.hooks.after)) {
 			const cmd = execPlain(command)
-			if (cmd) console.log(cmd.out, cmd.err)
+			console.log(cmd.out, cmd.err)
 		}
 }
 

src/config.ts

@@ -1,13 +1,21 @@
 import { readFileSync, writeFileSync, statSync } from 'fs'
 import { resolve } from 'path'
+import { homedir } from 'os'
+
 import yaml from 'js-yaml'
+
 import { flags } from './autorestic'
 import { Backend, Config } from './types'
 import { makeArrayIfIsNot, makeObjectKeysLowercase, rand } from './utils'
-import { homedir } from 'os'
 
 
 
+export enum LocationFromPrefixes {
+	Filesystem,
+	DockerVolume
+}
+
+
 export const normalizeAndCheckBackends = (config: Config) => {
 	config.backends = makeObjectKeysLowercase(config.backends)
 

src/forget.ts

@@ -2,6 +2,7 @@ import { Writer } from 'clitastic'
 
 import { config, VERBOSE } from './autorestic'
 import { getEnvFromBackend } from './backend'
+import { LocationFromPrefixes } from './config'
 import { Locations, Location, Flags } from './types'
 import {
 	exec,
@@ -9,7 +10,7 @@ import {
 	pathRelativeToConfigFile,
 	getFlagsFromLocation,
 	makeArrayIfIsNot,
-	fill,
+	fill, decodeLocationFromPrefix, getPathFromVolume,
 } from './utils'
 
 
@@ -20,11 +21,21 @@ export const forgetSingle = (name: string, to: string, location: Location, dryRu
 	const writer = new Writer(base + 'Removing old snapshots… ⏳')
 
 	const backend = config.backends[to]
-	const path = pathRelativeToConfigFile(location.from)
 	const flags = getFlagsFromLocation(location, 'forget')
 
+	const [type, value] = decodeLocationFromPrefix(location.from)
+	let path: string
+	switch (type) {
+		case LocationFromPrefixes.Filesystem:
+			path = pathRelativeToConfigFile(value)
+			break
+		case LocationFromPrefixes.DockerVolume:
+			path = getPathFromVolume(value)
+			break
+	}
+
 	if (flags.length == 0) {
-		writer.done(base + 'skipping, no policy declared')
+		writer.done(base + 'Skipping, no policy declared')
 		return
 	}
 	if (dryRun) flags.push('--dry-run')

src/handlers.ts

@@ -1,14 +1,16 @@
-import axios from 'axios'
-import { Writer } from 'clitastic'
-import { unlinkSync } from 'fs'
+import { chmodSync, renameSync, unlinkSync } from 'fs'
 import { tmpdir } from 'os'
 import { join, resolve } from 'path'
 
+import axios from 'axios'
+import { Writer } from 'clitastic'
+
 import { config, INSTALL_DIR, VERSION } from './autorestic'
 import { checkAndConfigureBackends, getBackendsFromLocations, getEnvFromBackend } from './backend'
 import { backupAll } from './backup'
 import { forgetAll } from './forget'
 import showAll from './info'
+import { restoreSingle } from './restore'
 import { Backends, Flags, Locations } from './types'
 import {
 	checkIfCommandIsAvailable,
@@ -85,36 +87,12 @@ const handlers: Handlers = {
 		if (!config) throw ConfigError
 		checkIfResticIsAvailable()
 
-		if (!flags.to) {
-			console.log(`You need to specify the restore path with --to`.red)
-			return
-		}
-
 		const locations = parseLocations(flags)
-		for (const [name, location] of Object.entries(locations)) {
-			const baseText = name.green + '\t\t'
-			const w = new Writer(baseText + `Starting...`)
+		const keys = Object.keys(locations)
+		if (keys.length < 1) throw new Error(`You need to specify the location to restore with --location`.red)
+		if (keys.length > 2) throw new Error(`Only one location is supported at a time when restoring`.red)
 
-			let backend: string = Array.isArray(location.to) ? location.to[0] : location.to
-			if (flags.from) {
-				if (!location.to.includes(flags.from)) {
-					w.done(baseText + `Backend ${flags.from} is not a valid location for ${name}`.red)
-					continue
-				}
-				backend = flags.from
-				w.replaceLn(baseText + `Restoring from ${backend.blue}...`)
-			} else if (Array.isArray(location.to) && location.to.length > 1) {
-				w.replaceLn(baseText + `Restoring from ${backend.blue}...\tTo select a specific backend pass the ${'--from'.blue} flag`)
-			}
-			const env = getEnvFromBackend(config.backends[backend])
-
-			exec(
-				'restic',
-				['restore', 'latest', '--path', resolve(location.from), '--target', flags.to],
-				{ env },
-			)
-			w.done(name.green + '\t\tDone 🎉')
-		}
+		restoreSingle(keys[0], flags.from, flags.to)
 	},
 	forget(args, flags) {
 		if (!config) throw ConfigError
@@ -139,7 +117,7 @@ const handlers: Handlers = {
 			console.log(out, err)
 		}
 	},
-	async info() {
+	info() {
 		showAll()
 	},
 	async install() {
@@ -147,7 +125,7 @@ const handlers: Handlers = {
 			checkIfResticIsAvailable()
 			console.log('Restic is already installed')
 			return
-		} catch (e) {
+		} catch {
 		}
 
 		const w = new Writer('Checking latest version... ⏳')
@@ -164,9 +142,7 @@ const handlers: Handlers = {
 		}
 
 		w.replaceLn('Downloading binary... 🌎')
-		const name = `${json.name.replace(' ', '_')}_${process.platform}_${
-			archMap[process.arch]
-		}.bz2`
+		const name = `${json.name.replace(' ', '_')}_${process.platform}_${archMap[process.arch]}.bz2`
 		const dl = json.assets.find((asset: any) => asset.name === name)
 		if (!dl)
 			return console.log(
@@ -184,8 +160,8 @@ const handlers: Handlers = {
 		unlinkSync(tmp)
 
 		w.replaceLn(`Moving to ${INSTALL_DIR} 🚙`)
-		exec('chmod', ['+x', extracted])
-		exec('mv', [extracted, INSTALL_DIR + '/restic'])
+		chmodSync(extracted, 0o755)
+		renameSync(extracted, INSTALL_DIR + '/restic')
 
 		w.done(
 			`\nFinished! restic is installed under: ${INSTALL_DIR}`.underline + ' 🎉',
@@ -225,7 +201,7 @@ const handlers: Handlers = {
 			w.replaceLn('Downloading binary... 🌎')
 			await downloadFile(dl.browser_download_url, to)
 
-			exec('chmod', ['+x', to])
+			chmodSync(to, 0o755)
 		}
 
 		w.done('All up to date! 🚀')

src/restore.ts

@@ -0,0 +1,81 @@
+import { Writer } from 'clitastic'
+import { resolve } from 'path'
+
+import { config } from './autorestic'
+import { getEnvFromBackend } from './backend'
+import { LocationFromPrefixes } from './config'
+import { Backend } from './types'
+import {
+	checkIfDockerVolumeExistsOrFail,
+	ConfigError,
+	decodeLocationFromPrefix,
+	exec,
+	execPlain,
+	getPathFromVolume,
+} from './utils'
+
+
+
+export const restoreToFilesystem = (from: string, to: string, backend: Backend) => {
+	exec(
+		'restic',
+		['restore', 'latest', '--path', resolve(from), '--target', to],
+		{ env: getEnvFromBackend(backend) },
+	)
+}
+
+export const restoreToVolume = (volume: string, backend: Backend) => {
+	const tmp = getPathFromVolume(volume)
+	try {
+		restoreToFilesystem(tmp, tmp, backend)
+		try {
+			checkIfDockerVolumeExistsOrFail(volume)
+		} catch {
+			execPlain(`docker volume create ${volume}`)
+		}
+
+		// For incremental backups. Unfortunately due to how the docker mounts work the permissions get lost.
+		// execPlain(`docker run --rm -v ${volume}:/data -v ${tmp}:/backup alpine cp -aT /backup /data`)
+		execPlain(`docker run --rm -v ${volume}:/data -v ${tmp}:/backup alpine tar xf /backup/archive.tar -C /data`)
+	} finally {
+		execPlain(`rm -rf ${tmp}`)
+	}
+}
+
+export const restoreSingle = (locationName: string, from: string, to?: string) => {
+	if (!config) throw ConfigError
+
+	const location = config.locations[locationName]
+
+	const baseText = locationName.green + '\t\t'
+	const w = new Writer(baseText + `Restoring...`)
+
+	let backendName: string = Array.isArray(location.to) ? location.to[0] : location.to
+	if (from) {
+		if (!location.to.includes(from)) {
+			w.done(baseText + `Backend ${from} is not a valid location for ${locationName}`.red)
+			return
+		}
+		backendName = from
+		w.replaceLn(baseText + `Restoring from ${backendName.blue}...`)
+	} else if (Array.isArray(location.to) && location.to.length > 1) {
+		w.replaceLn(baseText + `Restoring from ${backendName.blue}...\tTo select a specific backend pass the ${'--from'.blue} flag`)
+	}
+	const backend = config.backends[backendName]
+
+	const [type, value] = decodeLocationFromPrefix(location.from)
+	switch (type) {
+
+		case LocationFromPrefixes.Filesystem:
+			if (!to) throw new Error(`You need to specify the restore path with --to`.red)
+			restoreToFilesystem(value, to, backend)
+			break
+
+		case LocationFromPrefixes.DockerVolume:
+			restoreToVolume(value, backend)
+			break
+
+	}
+	w.done(locationName.green + '\t\tDone 🎉')
+}
+

src/utils.ts

@@ -1,21 +1,18 @@
 import axios from 'axios'
 import { spawnSync, SpawnSyncOptions } from 'child_process'
-import { randomBytes } from 'crypto'
-import { createWriteStream } from 'fs'
-import { dirname, isAbsolute, resolve } from 'path'
+import { createHash, randomBytes } from 'crypto'
+import { createWriteStream, renameSync, unlinkSync } from 'fs'
+import { homedir, tmpdir } from 'os'
+import { dirname, isAbsolute, join, resolve } from 'path'
 import { Duration, Humanizer } from 'uhrwerk'
 
-import { CONFIG_FILE } from './config'
+import { CONFIG_FILE, LocationFromPrefixes } from './config'
 import { Location } from './types'
 
 
 
-export const exec = (
-	command: string,
-	args: string[],
-	{ env, ...rest }: SpawnSyncOptions = {},
-) => {
-	const cmd = spawnSync(command, args, {
+export const exec = (command: string, args: string[], { env, ...rest }: SpawnSyncOptions = {}) => {
+	const { stdout, stderr, status } = spawnSync(command, args, {
 		...rest,
 		env: {
 			...process.env,
@@ -23,18 +20,15 @@ export const exec = (
 		},
 	})
 
-	const out = cmd.stdout && cmd.stdout.toString().trim()
-	const err = cmd.stderr && cmd.stderr.toString().trim()
+	const out = stdout && stdout.toString().trim()
+	const err = stderr && stderr.toString().trim()
 
-	return { out, err }
+	return { out, err, status }
 }
 
 export const execPlain = (command: string, opt: SpawnSyncOptions = {}) => {
 	const split = command.split(' ')
-	if (split.length < 1) {
-		console.log(`The command ${command} is not valid`.red)
-		return
-	}
+	if (split.length < 1) throw new Error(`The command ${command} is not valid`.red)
 
 	return exec(split[0], split.slice(1), opt)
 }
@@ -42,13 +36,14 @@ export const execPlain = (command: string, opt: SpawnSyncOptions = {}) => {
 export const checkIfResticIsAvailable = () =>
 	checkIfCommandIsAvailable(
 		'restic',
-		'Restic is not installed'.red +
-		' https://restic.readthedocs.io/en/latest/020_installation.html#stable-releases',
+		'restic is not installed'.red +
+		'\nEither run ' + 'autorestic install'.green +
+		'\nOr go to https://restic.readthedocs.io/en/latest/020_installation.html#stable-releases',
 	)
 
 export const checkIfCommandIsAvailable = (cmd: string, errorMsg?: string) => {
-	if (require('child_process').spawnSync(cmd).error)
-		throw new Error(errorMsg ? errorMsg : `"${errorMsg}" is not installed`.red)
+	if (spawnSync(cmd).error)
+		throw new Error(errorMsg ? errorMsg : `"${cmd}" is not installed`.red)
 }
 
 export const makeObjectKeysLowercase = (object: Object): any =>
@@ -81,11 +76,19 @@ export const downloadFile = async (url: string, to: string) =>
 			responseType: 'stream',
 		})
 
-		const stream = createWriteStream(to)
+		const tmp = join(tmpdir(), rand(64))
+		const stream = createWriteStream(tmp)
 
 		const writer = file.pipe(stream)
 		writer.on('close', () => {
 			stream.close()
+			try {
+				// Delete file if already exists. Needed if the binary wants to replace itself.
+				// Unix does not allow to overwrite a file that is being executed, but you can remove it and save other one at its place
+				unlinkSync(to)
+			} catch {
+			}
+			renameSync(tmp, to)
 			res()
 		})
 	})
@@ -95,6 +98,11 @@ export const pathRelativeToConfigFile = (path: string): string => isAbsolute(pat
 	? path
 	: resolve(dirname(CONFIG_FILE), path)
 
+export const resolveTildePath = (path: string): string | null =>
+	(path.length === 0 || path[0] !== '~')
+		? null
+		: join(homedir(), path.slice(1))
+
 export const ConfigError = new Error('Config file not found')
 
 export const getFlagsFromLocation = (location: Location, command?: string): string[] => {
@@ -108,8 +116,11 @@ export const getFlagsFromLocation = (location: Location, command?: string): stri
 	let flags: string[] = []
 	// Map the flags to an array for the exec function.
 	for (let [flag, values] of Object.entries(all))
-		for (const value of makeArrayIfIsNot(values))
-			flags = [...flags, `--${String(flag)}`, String(value)]
+		for (const value of makeArrayIfIsNot(values)) {
+			const stringValue = String(value)
+			const resolvedTilde = resolveTildePath(stringValue)
+			flags = [...flags, `--${String(flag)}`, resolvedTilde === null ? stringValue : resolvedTilde]
+		}
 
 	return flags
 }
@@ -156,3 +167,34 @@ export class MeasureDuration {
 	}
 
 }
+
+
+export const decodeLocationFromPrefix = (from: string): [LocationFromPrefixes, string] => {
+	const firstDelimiter = from.indexOf(':')
+	if (firstDelimiter === -1) return [LocationFromPrefixes.Filesystem, from]
+
+	const type = from.substr(0, firstDelimiter)
+	const value = from.substr(firstDelimiter + 1)
+
+	switch (type.toLowerCase()) {
+		case 'volume':
+			return [LocationFromPrefixes.DockerVolume, value]
+		case 'path':
+			return [LocationFromPrefixes.Filesystem, value]
+		default:
+			throw new Error(`Could not decode the location from: ${from}`.red)
+	}
+}
+
+export const hash = (plain: string): string => createHash('sha1').update(plain).digest().toString('hex')
+
+export const getPathFromVolume = (volume: string) => pathRelativeToConfigFile(hash(volume))
+
+export const checkIfDockerVolumeExistsOrFail = (volume: string) => {
+	const cmd = exec('docker', [
+		'volume', 'inspect', volume,
+	])
+	if (cmd.err.length > 0)
+		throw new Error('Volume not found')
+}
+