Merge branch 'master' into lockfile-path-config

.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: CI
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '^1.21'
+      - run: go test -v ./...
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '^1.21'
+      - run: go build -v .

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22-alpine as builder
+FROM golang:1.23-alpine as builder
 
 WORKDIR /app
 COPY go.* .
@@ -6,7 +6,7 @@ RUN go mod download
 COPY . .
 RUN go build
 
-FROM restic/restic:0.16.4
+FROM restic/restic:0.17.2
 RUN apk add --no-cache rclone bash curl docker-cli
 COPY --from=builder /app/autorestic /usr/bin/autorestic
 ENTRYPOINT []

docs/.nvmrc

@@ -1 +1 @@
-v20.8.0
+v22.7.0

docs/package.json

@@ -1,14 +1,15 @@
 {
   "scripts": {
     "build": "NEXT_TELEMETRY_DISABLED=1 next build",
-    "dev": "NEXT_TELEMETRY_DISABLED=1 next"
+    "dev": "NEXT_TELEMETRY_DISABLED=1 next",
+    "start": "NEXT_TELEMETRY_DISABLED=1 next start"
   },
   "dependencies": {
-    "next": "^13.5.3",
+    "next": "^14.2.7",
-    "nextra": "^2.13.1",
+    "nextra": "^2.13.4",
-    "nextra-theme-docs": "^2.13.1",
+    "nextra-theme-docs": "^2.13.4",
-    "react": "^18.2.0",
+    "react": "^18.3.1",
-    "react-dom": "^18.2.0"
+    "react-dom": "^18.3.1"
   },
-  "packageManager": "pnpm@8.8.0"
+  "packageManager": "pnpm@9.9.0"
 }

docs/pages/backend/index.md

@@ -16,3 +16,25 @@ backends:
 ## Types
 
 We restic supports multiple types of backends. See the [full list](/backend/available) for details.
+
+## Avoid Generating Keys
+
+By default, `autorestic` will generate a key for every backend if none is defined. This is done by updating your config file with the key.
+
+In cases where you want to provide the key yourself, you can ensure that `autorestic` doesn't accidentally generate one for you by setting `requireKey: true`.
+
+Example:
+
+```yaml | .autorestic.yml
+version: 2
+
+backends:
+  foo:
+    type: local
+    path: /data/my/backups
+    # Alternatively, you can set the key through the `AUTORESTIC_FOO_RESTIC_PASSWORD` environment variable.
+    key: ... your key here ...
+    requireKey: true
+```
+
+With this setting, if a key is missing, `autorestic` will crash instead of generating a new key and updating your config file.

docs/pages/location/docker.md

@@ -18,7 +18,7 @@ services:
 
 ```yaml | .autorestic.yml
 locations:
-  foo:
+  hello:
     from: my-data
     type: volume
     # ...

go.mod

@@ -11,9 +11,11 @@ require (
 	github.com/robfig/cron v1.2.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
+	github.com/stretchr/testify v1.9.0
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -23,6 +25,7 @@ require (
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -32,5 +35,5 @@ require (
 	golang.org/x/text v0.3.8 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -182,8 +182,9 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -487,8 +488,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

install.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-
+set -e -o pipefail
 shopt -s nocaseglob
 
 OUT_FILE=/usr/local/bin/autorestic
@@ -10,6 +10,8 @@ if [[ $NATIVE_OS == *"linux"* ]]; then
     OS=linux
 elif [[ $NATIVE_OS == *"darwin"* ]]; then
     OS=darwin
+elif [[ $NATIVE_OS == *"freebsd"* ]]; then
+    OS=freebsd
 else
     echo "Could not determine OS automatically, please check the release page manually: https://github.com/cupcakearmy/autorestic/releases"
     exit 1
@@ -17,7 +19,7 @@ fi
 echo $OS
 
 NATIVE_ARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
-if [[ $NATIVE_ARCH == *"x86_64"* ]]; then
+if [[ $NATIVE_ARCH == *"x86_64"* || $NATIVE_ARCH == *"amd64"* ]]; then
     ARCH=amd64
 elif [[ $NATIVE_ARCH == *"arm64"* || $NATIVE_ARCH == *"aarch64"* ]]; then
     ARCH=arm64

internal/backend.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/cupcakearmy/autorestic/internal/colors"
@@ -13,18 +14,19 @@ import (
 )
 
 type BackendRest struct {
-	User     string `mapstructure:"user,omitempty"`
+	User     string `mapstructure:"user,omitempty" yaml:"user,omitempty"`
-	Password string `mapstructure:"password,omitempty"`
+	Password string `mapstructure:"password,omitempty" yaml:"password,omitempty"`
 }
 
 type Backend struct {
-	name    string
+	name       string
-	Type    string            `mapstructure:"type,omitempty"`
+	Type       string            `mapstructure:"type,omitempty" yaml:"type,omitempty"`
-	Path    string            `mapstructure:"path,omitempty"`
+	Path       string            `mapstructure:"path,omitempty" yaml:"path,omitempty"`
-	Key     string            `mapstructure:"key,omitempty"`
+	Key        string            `mapstructure:"key,omitempty" yaml:"key,omitempty"`
-	Env     map[string]string `mapstructure:"env,omitempty"`
+	RequireKey bool              `mapstructure:"requireKey,omitempty" yaml:"requireKey,omitempty"`
-	Rest    BackendRest       `mapstructure:"rest,omitempty"`
+	Env        map[string]string `mapstructure:"env,omitempty" yaml:"env,omitempty"`
-	Options Options           `mapstructure:"options,omitempty"`
+	Rest       BackendRest       `mapstructure:"rest,omitempty" yaml:"rest,omitempty"`
+	Options    Options           `mapstructure:"options,omitempty" yaml:"options,omitempty"`
 }
 
 func GetBackend(name string) (Backend, bool) {
@@ -57,6 +59,8 @@ func (b Backend) generateRepo() (string, error) {
 	}
 }
 
+var nonAlphaRegex = regexp.MustCompile("[^A-Za-z0-9]")
+
 func (b Backend) getEnv() (map[string]string, error) {
 	env := make(map[string]string)
 	// Key
@@ -72,7 +76,9 @@ func (b Backend) getEnv() (map[string]string, error) {
 	}
 
 	// From Envfile and passed as env
-	var prefix = "AUTORESTIC_" + strings.ToUpper(b.name) + "_"
+	nameForEnv := strings.ToUpper(b.name)
+	nameForEnv = nonAlphaRegex.ReplaceAllString(nameForEnv, "_")
+	var prefix = "AUTORESTIC_" + nameForEnv + "_"
 	for _, variable := range os.Environ() {
 		var splitted = strings.SplitN(variable, "=", 2)
 		if strings.HasPrefix(splitted[0], prefix) {
@@ -104,6 +110,9 @@ func (b Backend) validate() error {
 		// Check if key is set in environment
 		env, _ := b.getEnv()
 		if _, found := env["RESTIC_PASSWORD"]; !found {
+			if b.RequireKey {
+				return fmt.Errorf("backend %s requires a key but none was provided", b.name)
+			}
 			// No key set in config file or env => generate random key and save file
 			key := generateRandomKey()
 			b.Key = key

internal/backend_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGenerateRepo(t *testing.T) {
@@ -194,6 +195,33 @@ func TestGetEnv(t *testing.T) {
 		assertEqual(t, result["B2_ACCOUNT_ID"], "foo123")
 		assertEqual(t, result["B2_ACCOUNT_KEY"], "foo456")
 	})
+
+	for _, char := range "@-_:/" {
+		t.Run(fmt.Sprintf("env var with special char (%c)", char), func(t *testing.T) {
+			// generate env variables
+			// TODO better way to teardown
+			defer os.Unsetenv("AUTORESTIC_FOO_BAR_RESTIC_PASSWORD")
+			defer os.Unsetenv("AUTORESTIC_FOO_BAR_B2_ACCOUNT_ID")
+			defer os.Unsetenv("AUTORESTIC_FOO_BAR_B2_ACCOUNT_KEY")
+			os.Setenv("AUTORESTIC_FOO_BAR_RESTIC_PASSWORD", "secret123")
+			os.Setenv("AUTORESTIC_FOO_BAR_B2_ACCOUNT_ID", "foo123")
+			os.Setenv("AUTORESTIC_FOO_BAR_B2_ACCOUNT_KEY", "foo456")
+
+			b := Backend{
+				name: fmt.Sprintf("foo%cbar", char),
+				Type: "local",
+				Path: "/foo/bar",
+			}
+			result, err := b.getEnv()
+			if err != nil {
+				t.Errorf("unexpected error %v", err)
+			}
+			assertEqual(t, result["RESTIC_REPOSITORY"], "/foo/bar")
+			assertEqual(t, result["RESTIC_PASSWORD"], "secret123")
+			assertEqual(t, result["B2_ACCOUNT_ID"], "foo123")
+			assertEqual(t, result["B2_ACCOUNT_KEY"], "foo456")
+		})
+	}
 }
 
 func TestValidate(t *testing.T) {
@@ -222,4 +250,16 @@ func TestValidate(t *testing.T) {
 		}
 		assertEqual(t, err.Error(), "Backend \"foo\" has no \"path\"")
 	})
+
+	t.Run("require key with no key", func(t *testing.T) {
+		b := Backend{
+			name:       "foo",
+			Type:       "local",
+			Path:       "~/foo/bar",
+			RequireKey: true,
+		}
+		err := b.validate()
+		fmt.Printf("error: %v\n", err)
+		assert.EqualError(t, err, "backend foo requires a key but none was provided")
+	})
 }

internal/config.go

@@ -17,17 +17,17 @@ import (
 	"github.com/spf13/viper"
 )
 
-const VERSION = "1.8.2"
+const VERSION = "1.8.3"
 
 type OptionMap map[string][]interface{}
 type Options map[string]OptionMap
 
 type Config struct {
-	Version   string              `mapstructure:"version"`
+	Version   string              `mapstructure:"version" yaml:"version"`
-	Extras    interface{}         `mapstructure:"extras"`
+	Extras    interface{}         `mapstructure:"extras" yaml:"extras"`
-	Locations map[string]Location `mapstructure:"locations"`
+	Locations map[string]Location `mapstructure:"locations" yaml:"locations"`
-	Backends  map[string]Backend  `mapstructure:"backends"`
+	Backends  map[string]Backend  `mapstructure:"backends" yaml:"backends"`
-	Global    Options             `mapstructure:"global"`
+	Global    Options             `mapstructure:"global" yaml:"global"`
 }
 
 var once sync.Once

internal/config_test.go

@@ -1,10 +1,15 @@
 package internal
 
 import (
+	"path"
 	"reflect"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestOptionToString(t *testing.T) {
@@ -143,6 +148,48 @@ func TestGetOptionsMultipleKeys(t *testing.T) {
 	reflect.DeepEqual(result, expected)
 }
 
+func TestSaveConfigProducesReadableConfig(t *testing.T) {
+	workDir := t.TempDir()
+	viper.SetConfigFile(path.Join(workDir, ".autorestic.yml"))
+
+	// Required to appease the config reader
+	viper.Set("version", 2)
+
+	c := Config{
+		Version: "2",
+		Locations: map[string]Location{
+			"test": {
+				Type: "local",
+				name: "test",
+				From: []string{"in-dir"},
+				To:   []string{"test"},
+				// ForgetOption & ConfigOption have previously marshalled in a way that
+				// can't get read correctly
+				ForgetOption: "foo",
+				CopyOption:   map[string][]string{"foo": {"bar"}},
+			},
+		},
+		Backends: map[string]Backend{
+			"test": {
+				name: "test",
+				Type: "local",
+				Path: "backup-target",
+				Key:  "supersecret",
+			},
+		},
+	}
+
+	err := c.SaveConfig()
+	assert.NoError(t, err)
+
+	// Ensure we the config reading logic actually runs
+	config = nil
+	once = sync.Once{}
+	readConfig := GetConfig()
+	assert.NotNil(t, readConfig)
+	assert.Equal(t, c, *readConfig)
+}
+
 func assertEqual[T comparable](t testing.TB, result, expected T) {
 	t.Helper()
 

internal/cron.go

@@ -1,12 +1,22 @@
 package internal
 
+import (
+	"errors"
+	"fmt"
+)
+
 func RunCron() error {
 	c := GetConfig()
+	var errs []error
 	for name, l := range c.Locations {
 		l.name = name
 		if err := l.RunCron(); err != nil {
-			return err
+			errs = append(errs, err)
 		}
 	}
+
+	if len(errs) > 0 {
+		return fmt.Errorf("Encountered errors during cron process:\n%w", errors.Join(errs...))
+	}
 	return nil
 }

internal/location.go

@@ -1,6 +1,7 @@
 package internal
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -33,26 +34,26 @@ const (
 )
 
 type Hooks struct {
-	Dir         string    `mapstructure:"dir"`
+	Dir         string    `mapstructure:"dir" yaml:"dir"`
-	PreValidate HookArray `mapstructure:"prevalidate,omitempty"`
+	PreValidate HookArray `mapstructure:"prevalidate,omitempty" yaml:"prevalidate,omitempty"`
-	Before      HookArray `mapstructure:"before,omitempty"`
+	Before      HookArray `mapstructure:"before,omitempty" yaml:"before,omitempty"`
-	After       HookArray `mapstructure:"after,omitempty"`
+	After       HookArray `mapstructure:"after,omitempty" yaml:"after,omitempty"`
-	Success     HookArray `mapstructure:"success,omitempty"`
+	Success     HookArray `mapstructure:"success,omitempty" yaml:"success,omitempty"`
-	Failure     HookArray `mapstructure:"failure,omitempty"`
+	Failure     HookArray `mapstructure:"failure,omitempty" yaml:"failure,omitempty"`
 }
 
 type LocationCopy = map[string][]string
 
 type Location struct {
-	name         string               `mapstructure:",omitempty"`
+	name         string               `mapstructure:",omitempty" yaml:",omitempty"`
-	From         []string             `mapstructure:"from,omitempty"`
+	From         []string             `mapstructure:"from,omitempty" yaml:"from,omitempty"`
-	Type         string               `mapstructure:"type,omitempty"`
+	Type         string               `mapstructure:"type,omitempty" yaml:"type,omitempty"`
-	To           []string             `mapstructure:"to,omitempty"`
+	To           []string             `mapstructure:"to,omitempty" yaml:"to,omitempty"`
-	Hooks        Hooks                `mapstructure:"hooks,omitempty"`
+	Hooks        Hooks                `mapstructure:"hooks,omitempty" yaml:"hooks,omitempty"`
-	Cron         string               `mapstructure:"cron,omitempty"`
+	Cron         string               `mapstructure:"cron,omitempty" yaml:"cron,omitempty"`
-	Options      Options              `mapstructure:"options,omitempty"`
+	Options      Options              `mapstructure:"options,omitempty" yaml:"options,omitempty"`
-	ForgetOption LocationForgetOption `mapstructure:"forget,omitempty"`
+	ForgetOption LocationForgetOption `mapstructure:"forget,omitempty" yaml:"forget,omitempty"`
-	CopyOption   LocationCopy         `mapstructure:"copy,omitempty"`
+	CopyOption   LocationCopy         `mapstructure:"copy,omitempty" yaml:"copy,omitempty"`
 }
 
 func GetLocation(name string) (Location, bool) {
@@ -446,7 +447,10 @@ func (l Location) RunCron() error {
 	now := time.Now()
 	if now.After(next) {
 		lock.SetCron(l.name, now.Unix())
-		l.Backup(true, "")
+		errs := l.Backup(true, "")
+		if len(errs) > 0 {
+			return fmt.Errorf("Failed to backup location \"%s\":\n%w", l.name, errors.Join(errs...))
+		}
 	} else {
 		if !flags.CRON_LEAN {
 			colors.Body.Printf("Skipping \"%s\", not due yet.\n", l.name)