Merge pull request #118 from cupcakearmy/1.4.0

1.4.0

CHANGELOG.md

@@ -5,6 +5,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0] - 2021-10-30
+
+### Added
+
+- Allow specify to specify a backend for location backup
+- Global restic flags
+- Generic ENV support for backends
+
+### Changed
+
+- Install now only requires `wget`
+- Env variable for the `KEY` has been renamed from `AUTORESTIC_[BACKEND NAME]_KEY` -> `AUTORESTIC_[BACKEND NAME]_RESTIC_PASSWORD`
+
+### Fixed
+
+- Error handling during upgrade & uninstall
+
+## [1.3.0] - 2021-10-26
+
+### Added
+
+- Pass restic backup metadata as ENV to hooks
+- Support for `XDG_CONFIG_HOME` and `${HOME}/.config` as default locations for `.autorestic.yaml` file.
+- Binary restic flags are now supported
+- Pass encryption keys from env variables or files.
+
 ## [1.2.0] - 2021-08-05
 
 ### Added

README.md

@@ -25,7 +25,7 @@
 
 ### 💭 Why / What?
 
-Autorestic is a wrapper around the amazing [restic](https://restic.net/). While being amazing the restic cli can be a bit overwhelming and difficult to manage if you have many different location that you want to backup to multiple locations. This utility is aimed at making this easier 🙂
+Autorestic is a wrapper around the amazing [restic](https://restic.net/). While being amazing the restic cli can be a bit overwhelming and difficult to manage if you have many different locations that you want to backup to multiple locations. This utility is aimed at making this easier 🙂.
 
 ### 🌈 Features
 

cmd/backup.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/cupcakearmy/autorestic/internal"
 	"github.com/cupcakearmy/autorestic/internal/colors"
@@ -22,8 +23,13 @@ var backupCmd = &cobra.Command{
 		CheckErr(err)
 		errors := 0
 		for _, name := range selected {
-			location, _ := internal.GetLocation(name)
-			errs := location.Backup(false)
+			var splitted = strings.Split(name, "@")
+			var specificBackend = ""
+			if len(splitted) > 1 {
+				specificBackend = splitted[1]
+			}
+			location, _ := internal.GetLocation(splitted[0])
+			errs := location.Backup(false, specificBackend)
 			for err := range errs {
 				colors.Error.Println(err)
 				errors++

cmd/root.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/cupcakearmy/autorestic/internal"
 	"github.com/cupcakearmy/autorestic/internal/colors"
@@ -50,11 +51,24 @@ func initConfig() {
 	if cfgFile != "" {
 		viper.SetConfigFile(cfgFile)
 	} else {
-		home, err := homedir.Dir()
-		CheckErr(err)
-
 		viper.AddConfigPath(".")
+
+		// Home
+		if home, err := homedir.Dir(); err != nil {
 			viper.AddConfigPath(home)
+		}
+
+		// XDG_CONFIG_HOME
+		{
+			prefix, found := os.LookupEnv("XDG_CONFIG_HOME")
+			if !found {
+				if home, err := homedir.Dir(); err != nil {
+					prefix = filepath.Join(home, ".config")
+				}
+			}
+			viper.AddConfigPath(filepath.Join(prefix, "autorestic"))
+		}
+
 		viper.SetConfigName(".autorestic")
 	}
 	viper.AutomaticEnv()

cmd/uninstall.go

@@ -9,12 +9,12 @@ var uninstallCmd = &cobra.Command{
 	Use:   "uninstall",
 	Short: "Uninstall restic and autorestic",
 	Run: func(cmd *cobra.Command, args []string) {
-		noRestic, _ := cmd.Flags().GetBool("no-restic")
-		bins.Uninstall(!noRestic)
+		restic, _ := cmd.Flags().GetBool("restic")
+		bins.Uninstall(restic)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(uninstallCmd)
-	uninstallCmd.Flags().Bool("no-restic", false, "do not uninstall restic.")
+	uninstallCmd.Flags().Bool("restic", false, "also uninstall restic.")
 }

cmd/upgrade.go

@@ -9,13 +9,13 @@ var upgradeCmd = &cobra.Command{
 	Use:   "upgrade",
 	Short: "Upgrade autorestic and restic",
 	Run: func(cmd *cobra.Command, args []string) {
-		noRestic, _ := cmd.Flags().GetBool("no-restic")
-		err := bins.Upgrade(!noRestic)
+		restic, _ := cmd.Flags().GetBool("restic")
+		err := bins.Upgrade(restic)
 		CheckErr(err)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(upgradeCmd)
-	upgradeCmd.Flags().Bool("no-restic", false, "also update restic")
+	upgradeCmd.Flags().Bool("restic", true, "also update restic")
 }

docs/markdown/_toc.md

@@ -23,6 +23,7 @@
 > [Overview](/backend/overview)
 > [Available Backends](/backend/available)
 > [Options](/backend/options)
+> [Environment](/backend/env)
 
 > :Collapse label=CLI
 >

docs/markdown/backend/available.md

@@ -4,6 +4,8 @@ In theory [all the restic backends](https://restic.readthedocs.io/en/stable/030_
 
 Those tested are the following:
 
+> ℹ️ You can also [specify the `env` variables in a config file](/backend/env) to separate them from the config file.
+
 ## Local
 
 ```yaml
@@ -19,9 +21,9 @@ backends:
 backends:
   name-of-backend:
     type: b2
-    path: 'backblaze_bucketID'
+    path: 'bucket_name'
     # Or With a path
-    # path: 'backblaze_bucketID:/some/path'
+    # path: 'bucket_name:/some/path'
     env:
       B2_ACCOUNT_ID: 'backblaze_keyID'
       B2_ACCOUNT_KEY: 'backblaze_applicationKey'
@@ -29,7 +31,7 @@ backends:
 
 #### API Keys gotcha
 
-If you use a _File name prefix_ when making the application key, do not include a leading slash. Make sure to include this prefix in the path (e.g. `path: 'backblaze_bucketID:my/path'`).
+If you use a _File name prefix_ when making the application key, do not include a leading slash. Make sure to include this prefix in the path (e.g. `path: 'bucket_name:my/path'`).
 
 ## S3 / Minio
 

docs/markdown/backend/env.md

@@ -0,0 +1,67 @@
+# Environment
+
+> ⚠ Available since version `v1.4.0`
+
+Sometimes it's favorable not having the encryption keys in the config files.
+For that `autorestic` allows passing the env variables to backend password as `ENV` variables, or through an env file.
+You can also pass whatever `env` variable to restic by prefixing it with `AUTORESTIC_[BACKEND NAME]_`.
+
+> ℹ️ Env variables and file overwrite the config file in the following order:
+>
+> Env Variables > Env File (`.autorestic.env`) > Config file (`.autorestic.yaml`)
+
+## Env file
+
+Alternatively `autorestic` can load an env file, located next to `.autorestic.yml` called `.autorestic.env`.
+
+```
+AUTORESTIC_FOO_RESTIC_PASSWORD=secret123
+```
+
+### Example with repository password
+
+The syntax for the `ENV` variables is as follows: `AUTORESTIC_[BACKEND NAME]_RESTIC_PASSWORD`.
+
+```yaml | autorestic.yaml
+backend:
+  foo:
+    type: ...
+    path: ...
+    key: secret123 # => AUTORESTIC_FOO_RESTIC_PASSWORD=secret123
+```
+
+This means we could remove `key: secret123` from `.autorestic.yaml` and execute as follows:
+
+```bash
+AUTORESTIC_FOO_RESTIC_PASSWORD=secret123 autorestic backup ...
+```
+
+### Example with Backblaze B2
+
+```yaml | autorestic.yaml
+backends:
+  bb:
+    type: b2
+    path: myBucket
+    key: myPassword
+    env:
+      B2_ACCOUNT_ID: 123
+      B2_ACCOUNT_KEY: 456
+```
+
+You could create an `.autorestic.env` or pass the following `ENV` variables to autorestic:
+
+```
+AUTORESTIC_BB_RESTIC_PASSWORD=myPassword
+AUTORESTIC_BB_B2_ACCOUNT_ID=123
+AUTORESTIC_BB_B2_ACCOUNT_KEY=456
+```
+
+```yaml | autorestic.yaml
+backends:
+  bb:
+    type: b2
+    path: myBucket
+```
+
+> :ToCPrevNext

docs/markdown/backend/options.md

@@ -1,8 +1,6 @@
 # Options
 
-For the `backup` and `forget` commands you can pass any native flags to `restic`.
-
-> It is also possible to set options for an [a specific location](/location/options).
+> ℹ️ For more detail see the [location docs](/location/options) for options, as they are the same.
 
 ```yaml
 backend:
@@ -18,6 +16,4 @@ backend:
 
 In this example, whenever `autorestic` runs `restic backup` it will append a `--tag abc --tag` to the native command.
 
-For more detail see the [location docs](/location/options) for options, as they are the same
-
 > :ToCPrevNext

docs/markdown/cli/backup.md

@@ -14,4 +14,12 @@ autorestic backup -a
 autorestic backup -l foo -l bar
 ```
 
+## Specific location
+
+`autorestic` also allows selecting specific backends for a location with the `location@backend` syntax.
+
+```bash
+autorestic backup -l location@backend
+```
+
 > :ToCPrevNext

docs/markdown/community.md

@@ -2,8 +2,10 @@
 
 A list of community driven projects. (No official affiliation)
 
+- SystemD Units: https://gitlab.com/py_crash/autorestic-systemd-units
 - Docker image: https://github.com/pascaliske/docker-autorestic
 - Ansible Role: https://github.com/adsanz/ansible-restic-role
 - Ansible Role: https://github.com/ItsNotGoodName/ansible-role-autorestic
+- Ansible Role: https://github.com/FuzzyMistborn/ansible-role-autorestic
 
 > :ToCPrevNext

docs/markdown/installation.md

@@ -2,10 +2,10 @@
 
 Linux & macOS. Windows is not supported. If you have problems installing please open an issue :)
 
-Autorestic requires `curl`, `wget` and `bzip2` to be installed. For most systems these should be already installed.
+Autorestic requires `bash`, `wget` and `bzip2` to be installed. For most systems these should be already installed.
 
 ```bash
-curl -s https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/install.sh | bash
+wget -qO - https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/install.sh | bash
 ```
 
 ## Alternatives

docs/markdown/location/cron.md

@@ -37,11 +37,17 @@ Then paste this at the bottom of the file and save it. Note that in this specifi
 PATH="/usr/local/bin:/usr/bin:/bin"
 
 # Example running every 5 minutes
-*/5 * * * * autorestic --ci cron
+*/5 * * * * autorestic -c /path/to/my/.autorestic.yml --ci cron
 ```
 
 > The `--ci` option is not required, but recommended
 
+To debug a cron job you can use
+
+```bash
+*/5 * * * * autorestic -c /path/to/my/.autorestic.yml --ci cron > /tmp/autorestic.log 2>&1
+```
+
 Now you can add as many `cron` attributes as you wish in the config file ⏱
 
 > Also note that manually triggered backups with `autorestic backup` will not influence the cron timeline, they are willingly not linked.

docs/markdown/location/hooks.md

@@ -35,8 +35,46 @@ locations:
 2. Run backup
 3. `after` hook
 4. - `success` hook if no errors were found
-   - `failure` hook if at least error was encountered
+   - `failure` hook if at least one error was encountered
 
 If the `before` hook encounters errors the backup and `after` hooks will be skipped and only the `failed` hooks will run.
 
+## Environment variables
+
+All hooks are exposed to the `AUTORESTIC_LOCATION` environment variable, which contains the location name.
+
+The `after` and `success` hooks have access to additional information with the following syntax:
+
+```bash
+AUTORESTIC_[TYPE]_[I]
+AUTORESTIC_[TYPE]_[BACKEND_NAME]
+```
+
+Every type of metadata is appended with both the name of the backend associated with and the number in which the backends where executed.
+
+### Available Metadata Types
+
+- `SNAPSHOT_ID`
+- `PARENT_SNAPSHOT_ID`
+- `FILES_ADDED`
+- `FILES_CHANGED`
+- `FILES_UNMODIFIED`
+- `DIRS_ADDED`
+- `DIRS_CHANGED`
+- `DIRS_UNMODIFIED`
+- `ADDED_SIZE`
+- `PROCESSED_FILES`
+- `PROCESSED_SIZE`
+- `PROCESSED_DURATION`
+
+#### Example
+
+Assuming you have a location `bar` that backs up to a single backend named `foo` you could expect the following env variables:
+
+```bash
+AUTORESTIC_LOCATION=bar
+AUTORESTIC_FILES_ADDED_0=42
+AUTORESTIC_FILES_ADDED_FOO=42
+```
+
 > :ToCPrevNext

docs/markdown/location/options.md

@@ -1,8 +1,32 @@
 # Options
 
-For the `backup` and `forget` commands you can pass any native flags to `restic`.
+For the `backup` and `forget` commands you can pass any native flags to `restic`. In addition you can specify flags for every command with `all`.
 
-> It is also possible to set options for an [entire backend](/backend/options).
+If flags don't start with `-` they will get prefixed with `--`.
+
+Flags without arguments can be set to `true`. They will be handled accordingly.
+
+> ℹ️ It is also possible to set options for an [entire backend](/backend/options) or globally (see below).
+
+```yaml
+locations:
+  foo:
+    # ...
+    options:
+      all:
+        some-flag: 123
+        # Equivalent to
+        --some-flag: 123
+      backup:
+        boolean-flag: true
+        tag:
+          - foo
+          - bar
+```
+
+## Example
+
+In this example, whenever `autorestic` runs `restic backup` it will append a `--tag abc --tag` to the native command.
 
 ```yaml
 locations:
@@ -16,6 +40,22 @@ locations:
           - bar
 ```
 
-In this example, whenever `autorestic` runs `restic backup` it will append a `--tag abc --tag` to the native command.
+## Global Options
+
+It is possible to specify global flags that will be run every time restic is invoked. To do so specify them under `global` in your config file.
+
+```yaml
+global:
+  all:
+    cache-dir: ~/restic
+  backup:
+    tag:
+      - foo
+
+backends:
+  # ...
+locations:
+  # ...
+```
 
 > :ToCPrevNext

docs/markdown/quick.md

@@ -3,7 +3,7 @@
 ## Installation
 
 ```bash
-curl -s https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/install.sh | bash
+wget -qO - https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/install.sh | bash
 ```
 
 See [installation](/installation) for alternative options.

docs/package-lock.json

@@ -214,9 +214,9 @@
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "node_modules/glob": {
-      "version": "7.1.7",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
-      "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
+      "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
       "dependencies": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -296,9 +296,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz",
+      "integrity": "sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==",
       "dependencies": {
         "has": "^1.0.3"
       },
@@ -315,9 +315,9 @@
       }
     },
     "node_modules/is-glob": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
-      "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
       "dependencies": {
         "is-extglob": "^2.1.1"
       },
@@ -475,9 +475,9 @@
       }
     },
     "node_modules/source-map-support": {
-      "version": "0.5.19",
-      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
-      "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
+      "version": "0.5.20",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.20.tgz",
+      "integrity": "sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw==",
       "dependencies": {
         "buffer-from": "^1.0.0",
         "source-map": "^0.6.0"
@@ -818,9 +818,9 @@
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "glob": {
-      "version": "7.1.7",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
-      "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
+      "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
       "requires": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -879,9 +879,9 @@
       }
     },
     "is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz",
+      "integrity": "sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==",
       "requires": {
         "has": "^1.0.3"
       }
@@ -892,9 +892,9 @@
       "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI="
     },
     "is-glob": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
-      "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
       "requires": {
         "is-extglob": "^2.1.1"
       }
@@ -1004,9 +1004,9 @@
       "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
     },
     "source-map-support": {
-      "version": "0.5.19",
-      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
-      "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
+      "version": "0.5.20",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.20.tgz",
+      "integrity": "sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw==",
       "requires": {
         "buffer-from": "^1.0.0",
         "source-map": "^0.6.0"

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/buger/goterm v1.0.0
 	github.com/fatih/color v1.10.0
+	github.com/joho/godotenv v1.4.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/robfig/cron v1.2.0
 	github.com/spf13/cobra v1.1.3

go.sum

@@ -101,6 +101,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=

install.sh

@@ -31,7 +31,7 @@ else
 fi
 echo $ARCH
 
-curl -s https://api.github.com/repos/cupcakearmy/autorestic/releases/latest \
+wget -qO - https://api.github.com/repos/cupcakearmy/autorestic/releases/latest \
 | grep "browser_download_url.*_${OS}_${ARCH}" \
 | cut -d : -f 2,3 \
 | tr -d \" \

internal/backend.go

@@ -58,12 +58,27 @@ func (b Backend) generateRepo() (string, error) {
 
 func (b Backend) getEnv() (map[string]string, error) {
 	env := make(map[string]string)
+	// Key
+	if b.Key != "" {
 		env["RESTIC_PASSWORD"] = b.Key
+	}
+
+	// From config file
 	repo, err := b.generateRepo()
 	env["RESTIC_REPOSITORY"] = repo
 	for key, value := range b.Env {
 		env[strings.ToUpper(key)] = value
 	}
+
+	// From Envfile and passed as env
+	var prefix = "AUTORESTIC_" + strings.ToUpper(b.name) + "_"
+	for _, variable := range os.Environ() {
+		var splitted = strings.SplitN(variable, "=", 2)
+		if strings.HasPrefix(splitted[0], prefix) {
+			env[strings.TrimPrefix(splitted[0], prefix)] = splitted[1]
+		}
+	}
+
 	return env, err
 }
 
@@ -85,6 +100,10 @@ func (b Backend) validate() error {
 		return fmt.Errorf(`Backend "%s" has no "path"`, b.name)
 	}
 	if b.Key == "" {
+		// Check if key is set in environment
+		env, _ := b.getEnv()
+		if _, found := env["RESTIC_PASSWORD"]; !found {
+			// No key set in config file or env => generate random key and save file
 			key := generateRandomKey()
 			b.Key = key
 			c := GetConfig()
@@ -95,6 +114,7 @@ func (b Backend) validate() error {
 				return err
 			}
 		}
+	}
 	env, err := b.getEnv()
 	if err != nil {
 		return err

internal/bins/bins.go

@@ -47,11 +47,11 @@ func dlJSON(url string) (GithubRelease, error) {
 
 func Uninstall(restic bool) error {
 	if err := os.Remove(path.Join(INSTALL_PATH, "autorestic")); err != nil {
-		colors.Error.Println(err)
+		return err
 	}
 	if restic {
 		if err := os.Remove(path.Join(INSTALL_PATH, "restic")); err != nil {
-			colors.Error.Println(err)
+			return err
 		}
 	}
 	return nil
@@ -79,11 +79,15 @@ func downloadAndInstallAsset(body GithubRelease, name string) error {
 				return err
 			}
 			defer tmp.Close()
-			tmp.Chmod(0755)
-			io.Copy(tmp, bz)
+			if err := tmp.Chmod(0755); err != nil {
+				return err
+			}
+			if _, err := io.Copy(tmp, bz); err != nil {
+				return err
+			}
 
 			to := path.Join(INSTALL_PATH, name)
-			os.Remove(to) // Delete if current, ignore error if file does not exits.
+			defer os.Remove(to) // Delete if current, ignore error if file does not exits.
 			if err := os.Rename(tmp.Name(), to); err != nil {
 				return nil
 			}
@@ -121,9 +125,11 @@ func Upgrade(restic bool) error {
 	// Upgrade restic
 	if restic {
 		if err := InstallRestic(); err != nil {
-			colors.Error.Println(err)
+			return err
+		}
+		if err := upgradeRestic(); err != nil {
+			return err
 		}
-		upgradeRestic()
 	}
 
 	// Upgrade self
@@ -140,7 +146,9 @@ func Upgrade(restic bool) error {
 		return err
 	}
 	if current.LT(latest) {
-		downloadAndInstallAsset(body, "autorestic")
+		if err := downloadAndInstallAsset(body, "autorestic"); err != nil {
+			return err
+		}
 		colors.Success.Println("Updated autorestic")
 	} else {
 		colors.Body.Println("Already up to date")

internal/config.go

@@ -10,21 +10,26 @@ import (
 
 	"github.com/cupcakearmy/autorestic/internal/colors"
 	"github.com/cupcakearmy/autorestic/internal/lock"
+	"github.com/joho/godotenv"
 	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
-const VERSION = "1.2.0"
+const VERSION = "1.4.0"
 
 var CI bool = false
 var VERBOSE bool = false
 var CRON_LEAN bool = false
 
+type OptionMap map[string][]interface{}
+type Options map[string]OptionMap
+
 type Config struct {
 	Extras    interface{}         `yaml:"extras"`
 	Locations map[string]Location `yaml:"locations"`
 	Backends  map[string]Backend  `yaml:"backends"`
+	Global    Options             `yaml:"global"`
 }
 
 var once sync.Once
@@ -36,7 +41,13 @@ func GetConfig() *Config {
 			if err := viper.ReadInConfig(); err == nil {
 				if !CRON_LEAN {
 					absConfig, _ := filepath.Abs(viper.ConfigFileUsed())
-					colors.Faint.Println("Using config file:", absConfig)
+					colors.Faint.Println("Using config: \t", absConfig)
+					// Load env file
+					envFile := filepath.Join(filepath.Dir(absConfig), ".autorestic.env")
+					err = godotenv.Load(envFile)
+					if err == nil {
+						colors.Faint.Println("Using env:\t", envFile)
+					}
 				}
 			} else {
 				return
@@ -178,20 +189,18 @@ func GetAllOrSelected(cmd *cobra.Command, backends bool) ([]string, error) {
 		selected, _ = cmd.Flags().GetStringSlice("location")
 	}
 	for _, s := range selected {
-		found := false
+		var splitted = strings.Split(s, "@")
 		for _, l := range list {
-			if l == s {
-				found = true
-				break
+			if l == splitted[0] {
+				goto found
 			}
 		}
-		if !found {
 		if backends {
 			return nil, fmt.Errorf("invalid backend \"%s\"", s)
 		} else {
 			return nil, fmt.Errorf("invalid location \"%s\"", s)
 		}
-		}
+	found:
 	}
 
 	if len(selected) == 0 {
@@ -228,12 +237,40 @@ func (c *Config) SaveConfig() error {
 	return viper.WriteConfig()
 }
 
+func optionToString(option string) string {
+	if !strings.HasPrefix(option, "-") {
+		return "--" + option
+	}
+	return option
+}
+
+func appendOptionsToSlice(str *[]string, options OptionMap) {
+	for key, values := range options {
+		for _, value := range values {
+			// Bool
+			asBool, ok := value.(bool)
+			if ok && asBool {
+				*str = append(*str, optionToString(key))
+				continue
+			}
+			// String
+			asString, ok := value.(string)
+			if ok {
+				*str = append(*str, optionToString(key), asString)
+				continue
+			}
+		}
+	}
+}
+
 func getOptions(options Options, key string) []string {
 	var selected []string
-	for k, values := range options[key] {
-		for _, value := range values {
-			selected = append(selected, fmt.Sprintf("--%s", k), value)
+	var keys = []string{"all"}
+	if key != "" {
+		keys = append(keys, key)
 	}
+	for _, key := range keys {
+		appendOptionsToSlice(&selected, options[key])
 	}
 	return selected
 }

internal/location.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/cupcakearmy/autorestic/internal/colors"
 	"github.com/cupcakearmy/autorestic/internal/lock"
+	"github.com/cupcakearmy/autorestic/internal/metadata"
 	"github.com/robfig/cron"
 )
 
@@ -30,8 +31,6 @@ type Hooks struct {
 	Failure HookArray `yaml:"failure,omitempty"`
 }
 
-type Options map[string]map[string][]string
-
 type Location struct {
 	name    string   `yaml:",omitempty"`
 	From    string   `yaml:"from,omitempty"`
@@ -124,12 +123,16 @@ func (l Location) getPath() (string, error) {
 	return "", fmt.Errorf("could not get path for location \"%s\"", l.name)
 }
 
-func (l Location) Backup(cron bool) []error {
+func (l Location) Backup(cron bool, specificBackend string) []error {
 	var errors []error
+	var backends []string
 	colors.PrimaryPrint("  Backing up location \"%s\"  ", l.name)
 	t := l.getType()
 	options := ExecuteOptions{
 		Command: "bash",
+		Envs: map[string]string{
+			"AUTORESTIC_LOCATION": l.name,
+		},
 	}
 
 	if err := l.validate(); err != nil {
@@ -151,7 +154,17 @@ func (l Location) Backup(cron bool) []error {
 	}
 
 	// Backup
-	for _, to := range l.To {
+	if specificBackend == "" {
+		backends = l.To
+	} else {
+		if l.hasBackend(specificBackend) {
+			backends = []string{specificBackend}
+		} else {
+			errors = append(errors, fmt.Errorf("backup location \"%s\" has no backend \"%s\"", l.name, specificBackend))
+			return errors
+		}
+	}
+	for i, to := range backends {
 		backend, _ := GetBackend(to)
 		colors.Secondary.Printf("Backend: %s\n", backend.name)
 		env, err := backend.getEnv()
@@ -187,6 +200,13 @@ func (l Location) Backup(cron bool) []error {
 			errors = append(errors, err)
 			continue
 		}
+
+		md := metadata.ExtractMetadataFromBackupLog(out)
+		mdEnv := metadata.MakeEnvFromMetadata(&md)
+		for k, v := range mdEnv {
+			options.Envs[k+"_"+fmt.Sprint(i)] = v
+			options.Envs[k+"_"+strings.ToUpper(backend.name)] = v
+		}
 		if VERBOSE {
 			colors.Faint.Println(out)
 		}
@@ -327,7 +347,7 @@ func (l Location) RunCron() error {
 	now := time.Now()
 	if now.After(next) {
 		lock.SetCron(l.name, now.Unix())
-		l.Backup(true)
+		l.Backup(true, "")
 	} else {
 		if !CRON_LEAN {
 			colors.Body.Printf("Skipping \"%s\", not due yet.\n", l.name)

internal/metadata/extractor_added.go

@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	"regexp"
+	"strings"
+)
+
+type addedExtractor struct {
+	re *regexp.Regexp
+}
+
+func (e addedExtractor) Matches(line string) bool {
+	return e.re.MatchString(line)
+}
+func (e addedExtractor) Extract(metadata *BackupLogMetadata, line string) {
+	// Sample line: "Added to the repo: 0 B"
+	metadata.AddedSize = strings.TrimSpace(e.re.ReplaceAllString(line, ""))
+}
+
+func NewAddedExtractor() MetadatExtractor {
+	return addedExtractor{regexp.MustCompile(`(?i)^Added to the repo:`)}
+}

internal/metadata/extractor_changeset.go

@@ -0,0 +1,57 @@
+package metadata
+
+import (
+	"regexp"
+	"strings"
+)
+
+type ChangeSetExtractor struct {
+	re      *regexp.Regexp
+	cleaner *regexp.Regexp
+	saver   changeSetSaver
+}
+
+func (e ChangeSetExtractor) Matches(line string) bool {
+	return e.re.MatchString(line)
+}
+func (e ChangeSetExtractor) Extract(metadata *BackupLogMetadata, line string) {
+	// Sample line: "Files:           0 new,     0 changed,     2 unmodified"
+	trimmed := strings.TrimSpace(e.re.ReplaceAllString(line, ""))
+	splitted := strings.Split(trimmed, ",")
+	var changeset BackupLogMetadataChangeset = BackupLogMetadataChangeset{}
+	changeset.Added = e.cleaner.ReplaceAllString(splitted[0], "")
+	changeset.Changed = e.cleaner.ReplaceAllString(splitted[1], "")
+	changeset.Unmodified = e.cleaner.ReplaceAllString(splitted[2], "")
+	e.saver.Save(metadata, changeset)
+}
+
+type changeSetSaver interface {
+	Save(metadata *BackupLogMetadata, changeset BackupLogMetadataChangeset)
+}
+
+type fileSaver struct{}
+
+func (f fileSaver) Save(metadata *BackupLogMetadata, changeset BackupLogMetadataChangeset) {
+	metadata.Files = changeset
+}
+
+type dirsSaver struct{}
+
+func (d dirsSaver) Save(metadata *BackupLogMetadata, changeset BackupLogMetadataChangeset) {
+	metadata.Dirs = changeset
+}
+
+func NewFilesExtractor() MetadatExtractor {
+	return ChangeSetExtractor{
+		re:      regexp.MustCompile(`(?i)^Files:`),
+		cleaner: regexp.MustCompile(`[^\d]`),
+		saver:   fileSaver{},
+	}
+}
+func NewDirsExtractor() MetadatExtractor {
+	return ChangeSetExtractor{
+		re:      regexp.MustCompile(`(?i)^Dirs:`),
+		cleaner: regexp.MustCompile(`[^\d]`),
+		saver:   dirsSaver{},
+	}
+}

internal/metadata/extractor_parent.go

@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	"regexp"
+	"strings"
+)
+
+type parentSnapshotIDExtractor struct {
+	re *regexp.Regexp
+}
+
+func (e parentSnapshotIDExtractor) Matches(line string) bool {
+	return e.re.MatchString(line)
+}
+func (e parentSnapshotIDExtractor) Extract(metadata *BackupLogMetadata, line string) {
+	// Sample line: "using parent snapshot c65d9310"
+	metadata.ParentSnapshotID = strings.TrimSpace(e.re.ReplaceAllString(line, ""))
+}
+
+func NewParentSnapshotIDExtractor() MetadatExtractor {
+	return parentSnapshotIDExtractor{regexp.MustCompile(`(?i)^using parent snapshot`)}
+}

internal/metadata/extractor_processed.go

@@ -0,0 +1,32 @@
+package metadata
+
+import (
+	"regexp"
+	"strings"
+)
+
+type processedExtractor struct {
+	re      *regexp.Regexp
+	cleaner *regexp.Regexp
+}
+
+func (e processedExtractor) Matches(line string) bool {
+	return e.re.MatchString(line)
+}
+func (e processedExtractor) Extract(metadata *BackupLogMetadata, line string) {
+	// Sample line: "processed 2 files, 24 B in 0:00"
+	var processed = BackupLogMetadataProcessed{}
+	split := strings.Split(line, "in")
+	processed.Duration = strings.TrimSpace(split[1])
+	split = strings.Split(split[0], ",")
+	processed.Files = e.cleaner.ReplaceAllString(split[0], "")
+	processed.Size = strings.TrimSpace(split[1])
+	metadata.Processed = processed
+}
+
+func NewProcessedExtractor() MetadatExtractor {
+	return processedExtractor{
+		regexp.MustCompile(`(?i)^processed \d* files`),
+		regexp.MustCompile(`(?i)[^\d]`),
+	}
+}

internal/metadata/extractor_snapshot.go

@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	"regexp"
+	"strings"
+)
+
+type snapshotExtractor struct {
+	re *regexp.Regexp
+}
+
+func (e snapshotExtractor) Matches(line string) bool {
+	return e.re.MatchString(line)
+}
+func (e snapshotExtractor) Extract(metadata *BackupLogMetadata, line string) {
+	// Sample line: "snapshot 917c7691 saved"
+	metadata.SnapshotID = strings.Split(line, " ")[1]
+}
+
+func NewSnapshotExtractor() MetadatExtractor {
+	return snapshotExtractor{regexp.MustCompile(`(?i)^snapshot \w+ saved`)}
+}

internal/metadata/metadata.go

@@ -0,0 +1,72 @@
+package metadata
+
+import (
+	"strings"
+)
+
+type BackupLogMetadataChangeset struct {
+	Added      string
+	Changed    string
+	Unmodified string
+}
+type BackupLogMetadataProcessed struct {
+	Files    string
+	Size     string
+	Duration string
+}
+type BackupLogMetadata struct {
+	ParentSnapshotID string
+	Files            BackupLogMetadataChangeset
+	Dirs             BackupLogMetadataChangeset
+	AddedSize        string
+	Processed        BackupLogMetadataProcessed
+	SnapshotID       string
+}
+
+type MetadatExtractor interface {
+	Matches(line string) bool
+	Extract(metadata *BackupLogMetadata, line string)
+}
+
+var extractors = []MetadatExtractor{
+	NewParentSnapshotIDExtractor(),
+	NewFilesExtractor(),
+	NewDirsExtractor(),
+	NewAddedExtractor(),
+	NewProcessedExtractor(),
+	NewSnapshotExtractor(),
+}
+
+func ExtractMetadataFromBackupLog(log string) BackupLogMetadata {
+	var md BackupLogMetadata
+	for _, line := range strings.Split(log, "\n") {
+		line = strings.TrimSpace(line)
+		for _, extractor := range extractors {
+			if extractor.Matches(line) {
+				extractor.Extract(&md, line)
+				continue
+			}
+		}
+	}
+	return md
+}
+
+func MakeEnvFromMetadata(metadata *BackupLogMetadata) map[string]string {
+	env := make(map[string]string)
+	var prefix = "AUTORESTIC_"
+
+	env[prefix+"SNAPSHOT_ID"] = metadata.SnapshotID
+	env[prefix+"PARENT_SNAPSHOT_ID"] = metadata.ParentSnapshotID
+	env[prefix+"FILES_ADDED"] = metadata.Files.Added
+	env[prefix+"FILES_CHANGED"] = metadata.Files.Changed
+	env[prefix+"FILES_UNMODIFIED"] = metadata.Files.Unmodified
+	env[prefix+"DIRS_ADDED"] = metadata.Dirs.Added
+	env[prefix+"DIRS_CHANGED"] = metadata.Dirs.Changed
+	env[prefix+"DIRS_UNMODIFIED"] = metadata.Dirs.Unmodified
+	env[prefix+"ADDED_SIZE"] = metadata.AddedSize
+	env[prefix+"PROCESSED_FILES"] = metadata.Processed.Files
+	env[prefix+"PROCESSED_SIZE"] = metadata.Processed.Size
+	env[prefix+"PROCESSED_DURATION"] = metadata.Processed.Duration
+
+	return env
+}

internal/utils.go

@@ -53,6 +53,9 @@ func ExecuteCommand(options ExecuteOptions, args ...string) (string, error) {
 
 func ExecuteResticCommand(options ExecuteOptions, args ...string) (string, error) {
 	options.Command = RESTIC_BIN
+	var c = GetConfig()
+	var optionsAsString = getOptions(c.Global, "")
+	args = append(optionsAsString, args...)
 	return ExecuteCommand(options, args...)
 }