Compare commits

...

57 Commits
0.5 ... 0.9

Author SHA1 Message Date
fd2fd91635 Update README.md 2019-12-05 00:31:41 +01:00
9c09ce1d79 Update README.md 2019-12-05 00:31:05 +01:00
c2f6f51789 Update README.md 2019-12-05 00:27:01 +01:00
cupcakearmy
f09cf90653 hooks for backups 2019-12-05 00:24:20 +01:00
cupcakearmy
d352aced37 version bump 2019-12-05 00:24:11 +01:00
cupcakearmy
563d4ffb96 remove duplicate code 2019-12-05 00:23:49 +01:00
cupcakearmy
1c6a061dd1 cleanup types 2019-12-05 00:23:15 +01:00
cupcakearmy
504ad639ab function to convert a variable to an array if its not already 2019-12-05 00:23:06 +01:00
f7a15c6d86 Update README.md 2019-12-05 00:22:01 +01:00
cupcakearmy
2f0092befe Merge remote-tracking branch 'origin/master' 2019-12-04 23:50:07 +01:00
cupcakearmy
1026e68b68 version bump 2019-12-04 23:49:39 +01:00
2389c59aa9 Update README.md 2019-12-04 23:48:22 +01:00
087aeaf578 Update README.md 2019-12-04 23:43:01 +01:00
3b7062f733 Update README.md 2019-12-04 23:39:19 +01:00
cupcakearmy
96b63c744b switch to yarn & update typescript 2019-12-04 23:36:49 +01:00
cupcakearmy
9669b70e20 refactor forget and make compatible with new options api 2019-12-04 23:36:27 +01:00
cupcakearmy
bcb081234c formatting 2019-12-04 23:36:04 +01:00
cupcakearmy
336f44e9dc simplify for 2019-12-04 23:35:41 +01:00
cupcakearmy
d0cda7f1d5 always convert to string 2019-12-04 23:35:26 +01:00
cupcakearmy
a8f4c23254 version bump 2019-12-04 20:53:06 +01:00
cupcakearmy
1c9f6d7d91 Merge remote-tracking branch 'origin/master' 2019-12-04 20:50:41 +01:00
cupcakearmy
18c3f4a06f use a simpler restore flag 2019-12-04 20:50:32 +01:00
632062a23f Update README.md 2019-12-04 20:50:12 +01:00
3d1d7ba256 Update README.md 2019-12-04 20:45:58 +01:00
cupcakearmy
417c54db4d cleanup 2019-12-04 20:38:59 +01:00
cupcakearmy
a9696bbc0c parse the flags in the config file to an array for the exec command 2019-12-04 20:38:48 +01:00
cupcakearmy
45f7506478 added options to the location type 2019-12-04 20:38:27 +01:00
cupcakearmy
d7cdeafe60 moved around params 2019-12-04 20:38:14 +01:00
cupcakearmy
cf09cdbb30 cleanup and support for exclusion 2019-12-04 20:38:04 +01:00
cupcakearmy
88059fe405 method to get all the backends from a list of locations 2019-12-04 20:37:50 +01:00
cupcakearmy
cdf18430b6 remove old todo 2019-12-03 23:38:49 +01:00
cupcakearmy
352754dad9 formatting & trailing commas 2019-12-03 23:37:55 +01:00
cupcakearmy
b68dc75053 removed unused import 2019-12-03 23:31:20 +01:00
cupcakearmy
6a055d3114 moved path resolver into utils 2019-12-03 23:31:13 +01:00
cupcakearmy
b5daff07eb replace indead of adding 2019-12-03 23:30:53 +01:00
b2d01d77d9 Update README.md 2019-12-03 09:52:11 +01:00
f41c042fce Merge pull request #6 from EliotBerriot/2-forget
Fix #2: support pruning and forget via snapshot policies
2019-12-03 09:43:38 +01:00
a81498ac42 Merge branch 'master' into 2-forget 2019-12-03 09:43:28 +01:00
1731ee30b3 Update README.md 2019-12-03 09:39:51 +01:00
1f4f1a1855 Merge pull request #5 from EliotBerriot/1-s3
Fix #1: fixed broken initialization check under S3
2019-12-03 09:39:25 +01:00
13cb764067 Update README.md 2019-12-03 09:19:26 +01:00
8058f37368 Merge pull request #3 from ChanceM/patch-1
Update README.md spelling correction.
2019-12-03 09:16:02 +01:00
Eliot Berriot
57ffa1e3fa Fix #2: support pruning and forget via snapshot policies 2019-12-02 14:57:10 +01:00
Eliot Berriot
671542cd30 Fix #1: fixed broken initialization check under S3 2019-12-02 11:14:02 +01:00
Gregory Moore
322df9f0bd Update README.md spelling correction.
Habe to have.
2019-11-30 09:53:41 -08:00
cupcakearmy
652158d1ed use bash 2019-11-27 19:30:01 +01:00
cupcakearmy
06ce8180fb support for absolute paths 2019-10-26 21:50:48 +02:00
81f513d77b Update README.md 2019-10-26 21:49:25 +02:00
e32521e6ea Update README.md 2019-10-26 21:49:06 +02:00
f5c5b39b30 Update README.md 2019-10-26 21:48:32 +02:00
e016c8defc Update README.md 2019-10-26 21:40:15 +02:00
a2e0a0c9cc Update README.md 2019-10-26 21:34:02 +02:00
cupcakearmy
f9b04ea342 remove sample 2019-10-26 21:31:33 +02:00
770c9dd7d4 Update README.md 2019-10-26 21:30:47 +02:00
cupcakearmy
851bbe5776 sketch 2019-10-26 21:21:56 +02:00
cupcakearmy
8fb6bdb3c6 version bump 2019-10-26 21:03:22 +02:00
cupcakearmy
47f5d91e89 version as normal command 2019-10-26 21:03:08 +02:00
14 changed files with 835 additions and 458 deletions

3
.gitignore vendored
View File

@@ -3,7 +3,8 @@ package-lock.json
.idea
yarn.lock
config.yml
bin
lib
data
.autorestic.yml

238
README.md
View File

@@ -1,9 +1,241 @@
# autorestic
High level CLI utility for restic
High backup level CLI utility for [restic](https://restic.net/).
Autorestic is a wrapper around the amazing [restic](https://restic.net/). While being amazing the restic cli can be a bit overwhelming and difficoult to manage if you have many different location that you want to backup to multiple locations. This utility is aimed at making this easier 🙂
## Installation
![Sketch](./docs/Sketch.png)
## 🌈 Features
- Config files, no CLI
- Predictable
- Backup locations to multiple backends
- Snapshot policies and pruning
- Simple interface
- Fully encrypted
###### 📒 Docs
- [Locations](#-locations)
- [Pruning & Deleting old files](#pruning-and-snapshot-policies)
- [Excluding files](#excluding-filesfolders)
- [Hooks](#before--after-hooks)
- [Backends](#-backends)
## 🛳 Installation
Linux & macOS. Windows is not supported.
```
curl -s https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/install.sh | sh
curl -s https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/install.sh | bash
```
## 🚀 Quickstart
### Setup
First we need to configure our locations and backends. Simply create a `.autorestic.yml` either in your home directory of in the folder from which you will execute `autorestic`.
Optionally you can specify the location of your config file by passing it as argument: `autorestic -c ../path/config.yml`
```yaml
locations:
home:
from: /home/me
to: remote
important:
from: /path/to/important/stuff
to:
- remote
- hdd
backends:
remote:
type: b2
path: 'myBucket:backup/home'
B2_ACCOUNT_ID: account_id
B2_ACCOUNT_KEY: account_key
hdd:
type: local
path: /mnt/my_external_storage
```
Then we check if everything is correct by running the `check` command. We will pass the `-a` (or `--all`) to tell autorestic to check all the locations.
If we would check only one location we could run the following: `autorestic check -l home`. Otherwise simpply check all locations with `autorestic check -a`
##### Note
Note that the data is automatically encrypted on the server. The key will be generated and added to your config file. Every backend will have a separate key. You should keep a copy of the keys somewhere in case your server dies. Otherwise DATA IS LOST!
### 📦 Backup
```
autorestic backup -a
```
### 📼 Restore
```
autorestic restore -a --to /path/where/to/restore
```
This will restore all the locations to the selected target. If for one location there are more than one backends specified autorestic will take the first one.
Lets see a more realistic example (from the config above)
```
autorestic restore -l home --from hdd --to /path/where/to/restore
```
This will restore the location `home` to the `/path/where/to/restore` folder and taking the data from the backend `hdd`
### 📲 Updates
Autorestic can update itself! Super handy right? Simply run `autorestic update` and we will check for you if there are updates for restic and autorestic and install them if necessary.
## 🗂 Locations
A location simply a folder on your machine that restic will backup. The paths can be relative from the config file. A location can have multiple backends, so that the data is secured across multiple servers.
```yaml
locations:
my-location-name:
from: path/to/backup
to:
- name-of-backend
- also-backup-to-this-backend
```
#### Pruning and snapshot policies
Autorestic supports declaring snapshot policies for location to avoid keeping old snapshot around if you don't need them.
This is based on [Restic's snapshots policies](https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy), and can be enabled for each location as shown below:
```yaml
locations:
etc:
from: /etc
to: local
options:
forget:
keep-last: 5 # always keep at least 5 snapshots
keep-hourly: 3 # keep 3 last hourly shapshots
keep-daily: 4 # keep 4 last daily shapshots
keep-weekly: 1 # keep 1 last weekly shapshots
keep-monthly: 12 # keep 12 last monthly shapshots
keep-yearly: 7 # keep 7 last yearly shapshots
keep-within: "2w" # keep snapshots from the last 2 weeks
```
Pruning can be triggered using `autorestic forget -a`, for all locations, or selectively with `autorestic forget -l <location>`. **please note that contrary to the restic CLI, `restic forget` will call `restic prune` internally.**
Run with the `--dry-run` flag to only print information about the process without actually pruning the snapshots. This is especially useful for debugging or testing policies:
```
$ autorestic forget -a --dry-run --verbose
Configuring Backends
local : Done ✓
Removing old shapshots according to policy
etc ▶ local : Removing old spnapshots… ⏳
etc ▶ local : Running in dry-run mode, not touching data
etc ▶ local : Forgeting old snapshots… ⏳Applying Policy: all snapshots within 2d of the newest
keep 3 snapshots:
ID Time Host Tags Reasons Paths
-----------------------------------------------------------------------------
531b692a 2019-12-02 12:07:28 computer within 2w /etc
51659674 2019-12-02 12:08:46 computer within 2w /etc
f8f8f976 2019-12-02 12:11:08 computer within 2w /etc
-----------------------------------------------------------------------------
3 snapshots
```
#### Excluding files/folders
If you want to exclude certain files or folders it done easily by specifiyng the right flags in the location you desire to filter. The flags are taken straight from the [restic cli exclude rules](https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files).
```yaml
locations:
my-location:
from: /data
to:
- local
- remote
options:
backup:
exclude:
- '*.nope'
- '*.abc'
exclude-file: .gitignore
backends:
local:
...
remote:
...
```
#### Before / After hooks
Sometimes you might want to stop an app/db before backing up data and start the service again after the backup has completed. This is what the hooks are made for. Simply add them to your location config. You can have as many commands as you wish.
```yaml
locations:
my-location:
from: /data
to:
- local
- remote
hooks:
before:
- echo "Hello"
- echo "Human"
after:
- echo "kthxbye"
```
## 💽 Backends
Backends are the place where you data will be saved. Backups are incremental and encrypted.
### Fields
##### `type`
Type of the backend see a list [here](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html)
Supported are:
- [Local](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#local)
- [Backblaze B2](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#backblaze-b2)
- [Amazon S3](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#amazon-s3)
- [Minio](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#minio-server)
- [Google Cloud Storage](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#google-cloud-storage)
- [Microsoft Azure Storage](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)
For each backend you need to specify the right variables as shown in the example below.
##### `path`
The path on the remote server.
For object storages as
##### Example
```yaml
backends:
name-of-backend:
type: b2
path: 'myAccount:myBucket/my/path'
B2_ACCOUNT_ID: backblaze_account_id
B2_ACCOUNT_KEY: backblaze_account_key
```
## Contributors
This amazing people helped the project!
- @ChanceM [Docs]
- @EliotBerriot [Docs, Pruning, S3]

View File

@@ -1,18 +0,0 @@
locations:
home:
from: /home/myUser
to: remote
important:
from: /path/to/important/stuff
to:
- remote
- hdd
backends:
remote:
type: b2
path: 'myBucket:backup/home'
B2_ACCOUNT_ID: account_id
B2_ACCOUNT_KEY: account_key
hdd:
type: local
path: /mnt/my_external_storage

BIN
docs/Sketch.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.9 MiB

View File

@@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
OUT_FILE=/usr/local/bin/autorestic

View File

@@ -4,16 +4,15 @@
"build": "tsc",
"build:watch": "tsc -w",
"dev": "tsnd --no-notify --respawn ./src/autorestic.ts",
"bin": "npm run build && pkg lib/autorestic.js --targets latest-macos-x64,latest-linux-x64 --out-path bin"
"bin": "yarn run build && pkg lib/autorestic.js --targets latest-macos-x64,latest-linux-x64 --out-path bin"
},
"devDependencies": {
"@types/decompress": "^4.2.3",
"@types/js-yaml": "^3.12.1",
"@types/minimist": "^1.2.0",
"@types/node": "^12.11.7",
"pkg": "^4.4.0",
"ts-node-dev": "^1.0.0-pre.40",
"typescript": "^3.5.1"
"typescript": "^3.7"
},
"dependencies": {
"axios": "^0.19.0",

View File

@@ -3,43 +3,42 @@ import minimist from 'minimist'
import { init } from './config'
import handlers, { error, help } from './handlers'
import { Config } from './types'
process.on('uncaughtException', err => {
console.log(err.message)
process.exit(1)
console.log(err.message)
process.exit(1)
})
export const { _: commands, ...flags } = minimist(process.argv.slice(2), {
alias: {
c: 'config',
v: 'version',
h: 'help',
a: 'all',
l: 'location',
b: 'backend',
},
boolean: ['a'],
string: ['l', 'b'],
alias: {
c: 'config',
v: 'version',
h: 'help',
a: 'all',
l: 'location',
b: 'backend',
d: 'dry-run',
},
boolean: ['a', 'd'],
string: ['l', 'b'],
})
export const VERSION = '0.5'
export const VERSION = '0.9'
export const INSTALL_DIR = '/usr/local/bin'
export const VERBOSE = flags.verbose
if (flags.version) {
console.log('version'.grey, VERSION)
process.exit(0)
}
export const config = init()
function main() {
if (commands.length < 1) return help()
const command: string = commands[0]
const args: string[] = commands.slice(1)
;(handlers[command] || error)(args, flags)
function main() {
if (commands.length < 1) return help()
const command: string = commands[0]
const args: string[] = commands.slice(1)
;(handlers[command] || error)(args, flags)
}
main()

View File

@@ -1,58 +1,67 @@
import { Writer } from 'clitastic'
import { config, VERBOSE } from './autorestic'
import { Backend, Backends } from './types'
import { Backend, Backends, Locations } from './types'
import { exec, ConfigError } from './utils'
const ALREADY_EXISTS = /(?=.*exists)(?=.*already)(?=.*config).*/
const ALREADY_EXISTS = /(?=.*already)(?=.*config).*/
export const getPathFromBackend = (backend: Backend): string => {
switch (backend.type) {
case 'local':
return backend.path
case 'b2':
case 'azure':
case 'gs':
case 's3':
return `${backend.type}:${backend.path}`
case 'sftp':
case 'rest':
throw new Error(`Unsupported backend type: "${backend.type}"`)
default:
throw new Error(`Unknown backend type.`)
}
switch (backend.type) {
case 'local':
return backend.path
case 'b2':
case 'azure':
case 'gs':
case 's3':
return `${backend.type}:${backend.path}`
case 'sftp':
case 'rest':
throw new Error(`Unsupported backend type: "${backend.type}"`)
default:
throw new Error(`Unknown backend type.`)
}
}
export const getEnvFromBackend = (backend: Backend) => {
const { type, path, key, ...rest } = backend
return {
RESTIC_PASSWORD: key,
RESTIC_REPOSITORY: getPathFromBackend(backend),
...rest,
}
const { type, path, key, ...rest } = backend
return {
RESTIC_PASSWORD: key,
RESTIC_REPOSITORY: getPathFromBackend(backend),
...rest,
}
}
export const getBackendsFromLocations = (locations: Locations): string[] => {
const backends = new Set<string>()
for (const to of Object.values(locations).map(location => location.to))
Array.isArray(to) ? to.forEach(t => backends.add(t)) : backends.add(to)
return Array.from(backends)
}
export const checkAndConfigureBackend = (name: string, backend: Backend) => {
const writer = new Writer(name.blue + ' : ' + 'Configuring... ⏳')
const env = getEnvFromBackend(backend)
const writer = new Writer(name.blue + ' : ' + 'Configuring... ⏳')
const env = getEnvFromBackend(backend)
const { out, err } = exec('restic', ['init'], { env })
const { out, err } = exec('restic', ['init'], { env })
if (err.length > 0 && !ALREADY_EXISTS.test(err))
throw new Error(`Could not load the backend "${name}": ${err}`)
if (err.length > 0 && !ALREADY_EXISTS.test(err))
throw new Error(`Could not load the backend "${name}": ${err}`)
if (VERBOSE && out.length > 0) console.log(out)
if (VERBOSE && out.length > 0) console.log(out)
writer.done(name.blue + ' : ' + 'Done ✓'.green)
writer.done(name.blue + ' : ' + 'Done ✓'.green)
}
export const checkAndConfigureBackends = (backends?: Backends) => {
if (!backends) {
if (!config) throw ConfigError
backends = config.backends
}
if (!backends) {
if (!config) throw ConfigError
backends = config.backends
}
console.log('\nConfiguring Backends'.grey.underline)
for (const [name, backend] of Object.entries(backends))
checkAndConfigureBackend(name, backend)
console.log('\nConfiguring Backends'.grey.underline)
for (const [name, backend] of Object.entries(backends))
checkAndConfigureBackend(name, backend)
}

View File

@@ -3,45 +3,57 @@ import { Writer } from 'clitastic'
import { config, VERBOSE } from './autorestic'
import { getEnvFromBackend } from './backend'
import { Locations, Location } from './types'
import { exec, ConfigError } from './utils'
import { CONFIG_FILE } from './config'
import { resolve, dirname } from 'path'
import { exec, ConfigError, pathRelativeToConfigFile, getFlagsFromLocation, makeArrayIfIsNot, execPlain } from './utils'
export const backupSingle = (name: string, from: string, to: string) => {
if (!config) throw ConfigError
const writer = new Writer(name + to.blue + ' : ' + 'Backing up... ⏳')
const backend = config.backends[to]
const pathRelativeToConfigFile = resolve(dirname(CONFIG_FILE), from)
const cmd = exec('restic', ['backup', pathRelativeToConfigFile], {
env: getEnvFromBackend(backend),
})
if (VERBOSE) console.log(cmd.out, cmd.err)
writer.done(name + to.blue + ' : ' + 'Done ✓'.green)
export const backupSingle = (name: string, to: string, location: Location) => {
if (!config) throw ConfigError
const writer = new Writer(name + to.blue + ' : ' + 'Backing up... ⏳')
const backend = config.backends[to]
const path = pathRelativeToConfigFile(location.from)
const cmd = exec(
'restic',
['backup', path, ...getFlagsFromLocation(location, 'backup')],
{ env: getEnvFromBackend(backend) },
)
if (VERBOSE) console.log(cmd.out, cmd.err)
writer.done(name + to.blue + ' : ' + 'Done ✓'.green)
}
export const backupLocation = (name: string, backup: Location) => {
const display = name.yellow + ' ▶ '
if (Array.isArray(backup.to)) {
let first = true
for (const t of backup.to) {
const nameOrBlankSpaces: string = first
? display
: new Array(name.length + 3).fill(' ').join('')
backupSingle(nameOrBlankSpaces, backup.from, t)
if (first) first = false
}
} else backupSingle(display, backup.from, backup.to)
export const backupLocation = (name: string, location: Location) => {
const display = name.yellow + ' ▶ '
const filler = new Array(name.length + 3).fill(' ').join('')
let first = true
if (location.hooks && location.hooks.before)
for (const command of makeArrayIfIsNot(location.hooks.before)) {
const cmd = execPlain(command)
if (cmd) console.log(cmd.out, cmd.err)
}
for (const t of makeArrayIfIsNot(location.to)) {
backupSingle(first ? display : filler, t, location)
if (first) first = false
}
if (location.hooks && location.hooks.after)
for (const command of makeArrayIfIsNot(location.hooks.after)) {
const cmd = execPlain(command)
if (cmd) console.log(cmd.out, cmd.err)
}
}
export const backupAll = (backups?: Locations) => {
if (!backups) {
if (!config) throw ConfigError
backups = config.locations
}
export const backupAll = (locations?: Locations) => {
if (!locations) {
if (!config) throw ConfigError
locations = config.locations
}
console.log('\nBacking Up'.underline.grey)
for (const [name, backup] of Object.entries(backups))
backupLocation(name, backup)
console.log('\nBacking Up'.underline.grey)
for (const [name, location] of Object.entries(locations))
backupLocation(name, location)
}

View File

@@ -3,87 +3,87 @@ import { resolve } from 'path'
import yaml from 'js-yaml'
import { flags } from './autorestic'
import { Backend, Config } from './types'
import { makeObjectKeysLowercase, rand } from './utils'
import { makeArrayIfIsNot, makeObjectKeysLowercase, rand } from './utils'
import { homedir } from 'os'
export const normalizeAndCheckBackends = (config: Config) => {
config.backends = makeObjectKeysLowercase(config.backends)
config.backends = makeObjectKeysLowercase(config.backends)
for (const [name, { type, path, key, ...rest }] of Object.entries(
config.backends
)) {
if (!type || !path)
throw new Error(
`The backend "${name}" is missing some required attributes`
)
for (const [name, { type, path, key, ...rest }] of Object.entries(
config.backends,
)) {
if (!type || !path)
throw new Error(
`The backend "${name}" is missing some required attributes`,
)
const tmp: any = {
type,
path,
key: key || rand(128),
}
for (const [key, value] of Object.entries(rest))
tmp[key.toUpperCase()] = value
const tmp: any = {
type,
path,
key: key || rand(128),
}
for (const [key, value] of Object.entries(rest))
tmp[key.toUpperCase()] = value
config.backends[name] = tmp as Backend
}
config.backends[name] = tmp as Backend
}
}
export const normalizeAndCheckBackups = (config: Config) => {
config.locations = makeObjectKeysLowercase(config.locations)
const backends = Object.keys(config.backends)
config.locations = makeObjectKeysLowercase(config.locations)
const backends = Object.keys(config.backends)
const checkDestination = (backend: string, backup: string) => {
if (!backends.includes(backend))
throw new Error(`Cannot find the backend "${backend}" for "${backup}"`)
}
const checkDestination = (backend: string, backup: string) => {
if (!backends.includes(backend))
throw new Error(`Cannot find the backend "${backend}" for "${backup}"`)
}
for (const [name, { from, to, ...rest }] of Object.entries(
config.locations
)) {
if (!from || !to)
throw new Error(
`The backup "${name}" is missing some required attributes`
)
for (const [name, { from, to, ...rest }] of Object.entries(
config.locations,
)) {
if (!from || !to)
throw new Error(
`The backup "${name}" is missing some required attributes`,
)
if (Array.isArray(to)) for (const t of to) checkDestination(t, name)
else checkDestination(to, name)
}
for (const t of makeArrayIfIsNot(to))
checkDestination(t, name)
}
}
const findConfigFile = (): string => {
const config = '.autorestic.yml'
const paths = [
resolve(flags.config || ''),
resolve('./' + config),
homedir() + '/' + config,
]
for (const path of paths) {
try {
const file = statSync(path)
if (file.isFile()) return path
} catch (e) {}
}
throw new Error('No Config file found')
const findConfigFile = (): string | undefined => {
const config = '.autorestic.yml'
const paths = [
resolve(flags.config || ''),
resolve('./' + config),
homedir() + '/' + config,
]
for (const path of paths) {
try {
const file = statSync(path)
if (file.isFile()) return path
} catch (e) {
}
}
}
export let CONFIG_FILE: string = ''
export const init = (): Config | undefined => {
try {
CONFIG_FILE = findConfigFile()
} catch (e) {
return
}
const file = findConfigFile()
if (file) CONFIG_FILE = file
else return
const raw: Config = makeObjectKeysLowercase(
yaml.safeLoad(readFileSync(CONFIG_FILE).toString())
)
const raw: Config = makeObjectKeysLowercase(
yaml.safeLoad(readFileSync(CONFIG_FILE).toString()),
)
normalizeAndCheckBackends(raw)
normalizeAndCheckBackups(raw)
normalizeAndCheckBackends(raw)
normalizeAndCheckBackups(raw)
writeFileSync(CONFIG_FILE, yaml.safeDump(raw))
writeFileSync(CONFIG_FILE, yaml.safeDump(raw))
return raw
return raw
}

60
src/forget.ts Normal file
View File

@@ -0,0 +1,60 @@
import { Writer } from 'clitastic'
import { config, VERBOSE } from './autorestic'
import { getEnvFromBackend } from './backend'
import { Locations, Location, Flags } from './types'
import { exec, ConfigError, pathRelativeToConfigFile, getFlagsFromLocation, makeArrayIfIsNot } from './utils'
export const forgetSingle = (name: string, to: string, location: Location, dryRun: boolean) => {
if (!config) throw ConfigError
const base = name + to.blue + ' : '
const writer = new Writer(base + 'Removing old snapshots… ⏳')
const backend = config.backends[to]
const path = pathRelativeToConfigFile(location.from)
const flags = getFlagsFromLocation(location, 'forget')
if (flags.length == 0) {
writer.done(base + 'skipping, no policy declared')
return
}
if (dryRun) flags.push('--dry-run')
writer.replaceLn(base + 'Forgetting old snapshots… ⏳')
const cmd = exec(
'restic',
['forget', '--path', path, '--prune', ...flags],
{ env: getEnvFromBackend(backend) },
)
if (VERBOSE) console.log(cmd.out, cmd.err)
writer.done(base + 'Done ✓'.green)
}
export const forgetLocation = (name: string, backup: Location, dryRun: boolean) => {
const display = name.yellow + ' ▶ '
const filler = new Array(name.length + 3).fill(' ').join('')
let first = true
for (const t of makeArrayIfIsNot(backup.to)) {
const nameOrBlankSpaces: string = first ? display : filler
forgetSingle(nameOrBlankSpaces, t, backup, dryRun)
if (first) first = false
}
}
export const forgetAll = (backups?: Locations, flags?: Flags) => {
if (!config) throw ConfigError
if (!backups) {
backups = config.locations
}
console.log('\nRemoving old snapshots according to policy'.underline.grey)
const dryRun = flags ? flags['dry-run'] : false
if (dryRun) console.log('Running in dry-run mode, not touching data\n'.yellow)
for (const [name, backup] of Object.entries(backups))
forgetLocation(name, backup, dryRun)
}

View File

@@ -5,237 +5,265 @@ import { tmpdir } from 'os'
import { join, resolve } from 'path'
import { config, INSTALL_DIR, VERSION } from './autorestic'
import { checkAndConfigureBackends, getEnvFromBackend } from './backend'
import { checkAndConfigureBackends, getBackendsFromLocations, getEnvFromBackend } from './backend'
import { backupAll } from './backup'
import { forgetAll } from './forget'
import { Backends, Flags, Locations } from './types'
import {
checkIfCommandIsAvailable,
checkIfResticIsAvailable,
downloadFile,
exec,
filterObjectByKey,
singleToArray,
ConfigError,
checkIfCommandIsAvailable,
checkIfResticIsAvailable,
downloadFile,
exec,
filterObjectByKey,
ConfigError, makeArrayIfIsNot,
} from './utils'
export type Handlers = {
[command: string]: (args: string[], flags: Flags) => void
[command: string]: (args: string[], flags: Flags) => void
}
const parseBackend = (flags: Flags): Backends => {
if (!config) throw ConfigError
if (!flags.all && !flags.backend)
throw new Error(
'No backends specified.'.red +
'\n--all [-a]\t\t\t\tCheck all.' +
'\n--backend [-b] myBackend\t\tSpecify one or more backend'
)
if (flags.all) return config.backends
else {
const backends = singleToArray<string>(flags.backend)
for (const backend of backends)
if (!config.backends[backend])
throw new Error('Invalid backend: '.red + backend)
return filterObjectByKey(config.backends, backends)
}
if (!config) throw ConfigError
if (!flags.all && !flags.backend)
throw new Error(
'No backends specified.'.red +
'\n--all [-a]\t\t\t\tCheck all.' +
'\n--backend [-b] myBackend\t\tSpecify one or more backend',
)
if (flags.all) return config.backends
else {
const backends = makeArrayIfIsNot<string>(flags.backend)
for (const backend of backends)
if (!config.backends[backend])
throw new Error('Invalid backend: '.red + backend)
return filterObjectByKey(config.backends, backends)
}
}
const parseLocations = (flags: Flags): Locations => {
if (!config) throw ConfigError
if (!flags.all && !flags.location)
throw new Error(
'No locations specified.'.red +
'\n--all [-a]\t\t\t\tBackup all.' +
'\n--location [-l] site1\t\t\tSpecify one or more locations'
)
if (!config) throw ConfigError
if (!flags.all && !flags.location)
throw new Error(
'No locations specified.'.red +
'\n--all [-a]\t\t\t\tBackup all.' +
'\n--location [-l] site1\t\t\tSpecify one or more locations',
)
if (flags.all) {
return config.locations
} else {
const locations = singleToArray<string>(flags.location)
for (const location of locations)
if (!config.locations[location])
throw new Error('Invalid location: '.red + location)
return filterObjectByKey(config.locations, locations)
}
if (flags.all) {
return config.locations
} else {
const locations = makeArrayIfIsNot<string>(flags.location)
for (const location of locations)
if (!config.locations[location])
throw new Error('Invalid location: '.red + location)
return filterObjectByKey(config.locations, locations)
}
}
const handlers: Handlers = {
check(args, flags) {
checkIfResticIsAvailable()
const backends = parseBackend(flags)
checkAndConfigureBackends(backends)
},
backup(args, flags) {
if (!config) throw ConfigError
checkIfResticIsAvailable()
const locations: Locations = parseLocations(flags)
check(args, flags) {
checkIfResticIsAvailable()
const backends = parseBackend(flags)
checkAndConfigureBackends(backends)
},
backup(args, flags) {
if (!config) throw ConfigError
checkIfResticIsAvailable()
const locations: Locations = parseLocations(flags)
const backends = new Set<string>()
for (const to of Object.values(locations).map(location => location.to))
Array.isArray(to) ? to.forEach(t => backends.add(t)) : backends.add(to)
checkAndConfigureBackends(
filterObjectByKey(config.backends, getBackendsFromLocations(locations)),
)
backupAll(locations)
checkAndConfigureBackends(
filterObjectByKey(config.backends, Array.from(backends))
)
backupAll(locations)
console.log('\nFinished!'.underline + ' 🎉')
},
restore(args, flags) {
if (!config) throw ConfigError
checkIfResticIsAvailable()
console.log('\nFinished!'.underline + ' 🎉')
},
restore(args, flags) {
if (!config) throw ConfigError
checkIfResticIsAvailable()
const locations = parseLocations(flags)
for (const [name, location] of Object.entries(locations)) {
const w = new Writer(name.green + `\t\tRestoring... ⏳`)
const env = getEnvFromBackend(
config.backends[
Array.isArray(location.to) ? location.to[0] : location.to
]
)
if (!flags.to) {
console.log(`You need to specify the restore path with --to`.red)
return
}
exec(
'restic',
['restore', 'latest', '--path', resolve(location.from), ...args],
{ env }
)
w.done(name.green + '\t\tDone 🎉')
}
},
exec(args, flags) {
checkIfResticIsAvailable()
const backends = parseBackend(flags)
for (const [name, backend] of Object.entries(backends)) {
console.log(`\n${name}:\n`.grey.underline)
const env = getEnvFromBackend(backend)
const locations = parseLocations(flags)
for (const [name, location] of Object.entries(locations)) {
const baseText = name.green + '\t\t'
const w = new Writer(baseText + `Starting...`)
const { out, err } = exec('restic', args, { env })
console.log(out, err)
}
},
async install() {
try {
checkIfResticIsAvailable()
console.log('Restic is already installed')
return
} catch (e) {}
let backend: string = Array.isArray(location.to) ? location.to[0] : location.to
if (flags.from) {
if (!location.to.includes(flags.from)) {
w.done(baseText + `Backend ${flags.from} is not a valid location for ${name}`.red)
continue
}
backend = flags.from
w.replaceLn(baseText + `Restoring from ${backend.blue}...`)
} else if (Array.isArray(location.to) && location.to.length > 1) {
w.replaceLn(baseText + `Restoring from ${backend.blue}...\tTo select a specific backend pass the ${'--from'.blue} flag`)
}
const env = getEnvFromBackend(config.backends[backend])
const w = new Writer('Checking latest version... ⏳')
checkIfCommandIsAvailable('bzip2')
const { data: json } = await axios({
method: 'get',
url: 'https://api.github.com/repos/restic/restic/releases/latest',
responseType: 'json',
})
exec(
'restic',
['restore', 'latest', '--path', resolve(location.from), '--target', flags.to],
{ env },
)
w.done(name.green + '\t\tDone 🎉')
}
},
forget(args, flags) {
if (!config) throw ConfigError
checkIfResticIsAvailable()
const locations: Locations = parseLocations(flags)
const archMap: { [a: string]: string } = {
x32: '386',
x64: 'amd64',
}
checkAndConfigureBackends(
filterObjectByKey(config.backends, getBackendsFromLocations(locations)),
)
forgetAll(locations, flags)
w.replaceLn('Downloading binary... 🌎')
const name = `${json.name.replace(' ', '_')}_${process.platform}_${
archMap[process.arch]
}.bz2`
const dl = json.assets.find((asset: any) => asset.name === name)
if (!dl)
return console.log(
'Cannot get the right binary.'.red,
'Please see https://bit.ly/2Y1Rzai'
)
console.log('\nFinished!'.underline + ' 🎉')
},
exec(args, flags) {
checkIfResticIsAvailable()
const backends = parseBackend(flags)
for (const [name, backend] of Object.entries(backends)) {
console.log(`\n${name}:\n`.grey.underline)
const env = getEnvFromBackend(backend)
const tmp = join(tmpdir(), name)
const extracted = tmp.slice(0, -4) //without the .bz2
const { out, err } = exec('restic', args, { env })
console.log(out, err)
}
},
async install() {
try {
checkIfResticIsAvailable()
console.log('Restic is already installed')
return
} catch (e) {
}
await downloadFile(dl.browser_download_url, tmp)
const w = new Writer('Checking latest version... ⏳')
checkIfCommandIsAvailable('bzip2')
const { data: json } = await axios({
method: 'get',
url: 'https://api.github.com/repos/restic/restic/releases/latest',
responseType: 'json',
})
// TODO: Native bz2
// Decompress
w.replaceLn('Decompressing binary... 📦')
exec('bzip2', ['-dk', tmp])
unlinkSync(tmp)
const archMap: { [a: string]: string } = {
x32: '386',
x64: 'amd64',
}
w.replaceLn(`Moving to ${INSTALL_DIR} 🚙`)
exec('chmod', ['+x', extracted])
exec('mv', [extracted, INSTALL_DIR + '/restic'])
w.replaceLn('Downloading binary... 🌎')
const name = `${json.name.replace(' ', '_')}_${process.platform}_${
archMap[process.arch]
}.bz2`
const dl = json.assets.find((asset: any) => asset.name === name)
if (!dl)
return console.log(
'Cannot get the right binary.'.red,
'Please see https://bit.ly/2Y1Rzai',
)
w.done(
`\nFinished! restic is installed under: ${INSTALL_DIR}`.underline + ' 🎉'
)
},
uninstall() {
for (const bin of ['restic', 'autorestic'])
try {
unlinkSync(INSTALL_DIR + '/' + bin)
console.log(`Finished! ${bin} was uninstalled`)
} catch (e) {
console.log(`${bin} is already uninstalled`.red)
}
},
async update() {
checkIfResticIsAvailable()
const w = new Writer('Checking for latest restic version... ⏳')
exec('restic', ['self-update'])
const tmp = join(tmpdir(), name)
const extracted = tmp.slice(0, -4) //without the .bz2
w.replaceLn('Checking for latest autorestic version... ⏳')
const { data: json } = await axios({
method: 'get',
url:
'https://api.github.com/repos/cupcakearmy/autorestic/releases/latest',
responseType: 'json',
})
await downloadFile(dl.browser_download_url, tmp)
if (json.tag_name != VERSION) {
const platformMap: { [key: string]: string } = {
darwin: 'macos',
}
w.replaceLn('Decompressing binary... 📦')
exec('bzip2', ['-dk', tmp])
unlinkSync(tmp)
const name = `autorestic_${platformMap[process.platform] ||
process.platform}_${process.arch}`
const dl = json.assets.find((asset: any) => asset.name === name)
w.replaceLn(`Moving to ${INSTALL_DIR} 🚙`)
exec('chmod', ['+x', extracted])
exec('mv', [extracted, INSTALL_DIR + '/restic'])
const to = INSTALL_DIR + '/autorestic'
w.replaceLn('Downloading binary... 🌎')
await downloadFile(dl.browser_download_url, to)
w.done(
`\nFinished! restic is installed under: ${INSTALL_DIR}`.underline + ' 🎉',
)
},
uninstall() {
for (const bin of ['restic', 'autorestic'])
try {
unlinkSync(INSTALL_DIR + '/' + bin)
console.log(`Finished! ${bin} was uninstalled`)
} catch (e) {
console.log(`${bin} is already uninstalled`.red)
}
},
async update() {
checkIfResticIsAvailable()
const w = new Writer('Checking for latest restic version... ⏳')
exec('restic', ['self-update'])
exec('chmod', ['+x', to])
}
w.replaceLn('Checking for latest autorestic version... ⏳')
const { data: json } = await axios({
method: 'get',
url:
'https://api.github.com/repos/cupcakearmy/autorestic/releases/latest',
responseType: 'json',
})
w.done('All up to date! 🚀')
},
if (json.tag_name != VERSION) {
const platformMap: { [key: string]: string } = {
darwin: 'macos',
}
const name = `autorestic_${platformMap[process.platform] || process.platform}_${process.arch}`
const dl = json.assets.find((asset: any) => asset.name === name)
const to = INSTALL_DIR + '/autorestic'
w.replaceLn('Downloading binary... 🌎')
await downloadFile(dl.browser_download_url, to)
exec('chmod', ['+x', to])
}
w.done('All up to date! 🚀')
},
version() {
console.log('version'.grey, VERSION)
},
}
export const help = () => {
console.log(
'\nAutorestic'.blue +
` - ${VERSION} - Easy Restic CLI Utility` +
'\n' +
'\nOptions:'.yellow +
`\n -c, --config Specify config file. Default: .autorestic.yml` +
'\n' +
'\nCommands:'.yellow +
'\n check [-b, --backend] [-a, --all] Check backends' +
'\n backup [-l, --location] [-a, --all] Backup all or specified locations' +
'\n restore [-l, --location] [-- --target <out dir>] Restore all or specified locations' +
'\n' +
'\n exec [-b, --backend] [-a, --all] <command> -- [native options] Execute native restic command' +
'\n' +
'\n install install restic' +
'\n uninstall uninstall restic' +
'\n update update restic' +
'\n help Show help' +
'\n' +
'\nExamples: '.yellow +
'https://git.io/fjVbg' +
'\n'
)
console.log(
'\nAutorestic'.blue +
` - ${VERSION} - Easy Restic CLI Utility` +
'\n' +
'\nOptions:'.yellow +
`\n -c, --config Specify config file. Default: .autorestic.yml` +
'\n' +
'\nCommands:'.yellow +
'\n check [-b, --backend] [-a, --all] Check backends' +
'\n backup [-l, --location] [-a, --all] Backup all or specified locations' +
'\n forget [-l, --location] [-a, --all] [--dry-run] Forget old snapshots according to declared policies' +
'\n restore [-l, --location] [--from backend] [--to <out dir>] Restore all or specified locations' +
'\n' +
'\n exec [-b, --backend] [-a, --all] <command> -- [native options] Execute native restic command' +
'\n' +
'\n install install restic' +
'\n uninstall uninstall restic' +
'\n update update restic' +
'\n help Show help' +
'\n' +
'\nExamples: '.yellow +
'https://git.io/fjVbg' +
'\n',
)
}
export const error = () => {
help()
console.log(
`Invalid Command:`.red.underline,
`${process.argv.slice(2).join(' ')}`
)
help()
console.log(
`Invalid Command:`.red.underline,
`${process.argv.slice(2).join(' ')}`,
)
}
export default handlers

View File

@@ -1,77 +1,94 @@
export type StringOrArray = string | string[]
// BACKENDS
type BackendLocal = {
type: 'local'
key: string
path: string
type: 'local'
key: string
path: string
}
type BackendSFTP = {
type: 'sftp'
key: string
path: string
password?: string
type: 'sftp'
key: string
path: string
password?: string
}
type BackendREST = {
type: 'rest'
key: string
path: string
user?: string
password?: string
type: 'rest'
key: string
path: string
user?: string
password?: string
}
type BackendS3 = {
type: 's3'
key: string
path: string
aws_access_key_id: string
aws_secret_access_key: string
type: 's3'
key: string
path: string
aws_access_key_id: string
aws_secret_access_key: string
}
type BackendB2 = {
type: 'b2'
key: string
path: string
b2_account_id: string
b2_account_key: string
type: 'b2'
key: string
path: string
b2_account_id: string
b2_account_key: string
}
type BackendAzure = {
type: 'azure'
key: string
path: string
azure_account_name: string
azure_account_key: string
type: 'azure'
key: string
path: string
azure_account_name: string
azure_account_key: string
}
type BackendGS = {
type: 'gs'
key: string
path: string
google_project_id: string
google_application_credentials: string
type: 'gs'
key: string
path: string
google_project_id: string
google_application_credentials: string
}
export type Backend =
| BackendAzure
| BackendB2
| BackendGS
| BackendLocal
| BackendREST
| BackendS3
| BackendSFTP
| BackendAzure
| BackendB2
| BackendGS
| BackendLocal
| BackendREST
| BackendS3
| BackendSFTP
export type Backends = { [name: string]: Backend }
// LOCATIONS
export type Location = {
from: string
to: string | string[]
from: string
to: StringOrArray
hooks?: {
before?: StringOrArray
after?: StringOrArray
}
options?: {
[key: string]: {
[key: string]: StringOrArray
}
}
}
export type Locations = { [name: string]: Location }
// OTHER
export type Config = {
locations: Locations
backends: Backends
locations: Locations
backends: Backends
}
export type Flags = { [arg: string]: any }

View File

@@ -2,76 +2,114 @@ import axios from 'axios'
import { spawnSync, SpawnSyncOptions } from 'child_process'
import { randomBytes } from 'crypto'
import { createWriteStream } from 'fs'
import { dirname, isAbsolute, resolve } from 'path'
import { CONFIG_FILE } from './config'
import { Location } from './types'
export const exec = (
command: string,
args: string[],
{ env, ...rest }: SpawnSyncOptions = {}
command: string,
args: string[],
{ env, ...rest }: SpawnSyncOptions = {},
) => {
const cmd = spawnSync(command, args, {
...rest,
env: {
...process.env,
...env,
},
})
const cmd = spawnSync(command, args, {
...rest,
env: {
...process.env,
...env,
},
})
const out = cmd.stdout && cmd.stdout.toString().trim()
const err = cmd.stderr && cmd.stderr.toString().trim()
const out = cmd.stdout && cmd.stdout.toString().trim()
const err = cmd.stderr && cmd.stderr.toString().trim()
return { out, err }
return { out, err }
}
export const execPlain = (command: string, opt: SpawnSyncOptions = {}) => {
const split = command.split(' ')
if (split.length < 1) {
console.log(`The command ${command} is not valid`.red)
return
}
return exec(split[0], split.slice(1), opt)
}
export const checkIfResticIsAvailable = () =>
checkIfCommandIsAvailable(
'restic',
'Restic is not installed'.red +
' https://restic.readthedocs.io/en/latest/020_installation.html#stable-releases'
)
checkIfCommandIsAvailable(
'restic',
'Restic is not installed'.red +
' https://restic.readthedocs.io/en/latest/020_installation.html#stable-releases',
)
export const checkIfCommandIsAvailable = (cmd: string, errorMsg?: string) => {
if (require('child_process').spawnSync(cmd).error)
throw new Error(errorMsg ? errorMsg : `"${errorMsg}" is not installed`.red)
if (require('child_process').spawnSync(cmd).error)
throw new Error(errorMsg ? errorMsg : `"${errorMsg}" is not installed`.red)
}
export const makeObjectKeysLowercase = (object: Object): any =>
Object.fromEntries(
Object.entries(object).map(([key, value]) => [key.toLowerCase(), value])
)
Object.fromEntries(
Object.entries(object).map(([key, value]) => [key.toLowerCase(), value]),
)
export function rand(length = 32): string {
return randomBytes(length / 2).toString('hex')
return randomBytes(length / 2).toString('hex')
}
export const singleToArray = <T>(singleOrArray: T | T[]): T[] =>
Array.isArray(singleOrArray) ? singleOrArray : [singleOrArray]
export const filterObject = <T>(
obj: { [key: string]: T },
filter: (item: [string, T]) => boolean
obj: { [key: string]: T },
filter: (item: [string, T]) => boolean,
): { [key: string]: T } =>
Object.fromEntries(Object.entries(obj).filter(filter))
Object.fromEntries(Object.entries(obj).filter(filter))
export const filterObjectByKey = <T>(
obj: { [key: string]: T },
keys: string[]
obj: { [key: string]: T },
keys: string[],
) => filterObject(obj, ([key]) => keys.includes(key))
export const downloadFile = async (url: string, to: string) =>
new Promise<void>(async res => {
const { data: file } = await axios({
method: 'get',
url: url,
responseType: 'stream',
})
new Promise<void>(async res => {
const { data: file } = await axios({
method: 'get',
url: url,
responseType: 'stream',
})
const stream = createWriteStream(to)
const stream = createWriteStream(to)
const writer = file.pipe(stream)
writer.on('close', () => {
stream.close()
res()
})
})
const writer = file.pipe(stream)
writer.on('close', () => {
stream.close()
res()
})
})
// Check if is an absolute path, otherwise get the path relative to the config file
export const pathRelativeToConfigFile = (path: string): string => isAbsolute(path)
? path
: resolve(dirname(CONFIG_FILE), path)
export const ConfigError = new Error('Config file not found')
export const getFlagsFromLocation = (location: Location, command?: string): string[] => {
if (!location.options) return []
const all = {
...location.options.global,
...(location.options[command || ''] || {}),
}
let flags: string[] = []
// Map the flags to an array for the exec function.
for (let [flag, values] of Object.entries(all))
for (const value of makeArrayIfIsNot(values))
flags = [...flags, `--${String(flag)}`, String(value)]
return flags
}
export const makeArrayIfIsNot = <T>(maybeArray: T | T[]): T[] => Array.isArray(maybeArray) ? maybeArray : [maybeArray]