From c2f9ed920427de77f3621b7212f40485d51ed991 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sat, 30 Oct 2021 13:01:31 +0200 Subject: [PATCH 01/24] multiple paths --- internal/config.go | 3 ++- internal/location.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/config.go b/internal/config.go index 3cf5cca..a1733c9 100644 --- a/internal/config.go +++ b/internal/config.go @@ -81,7 +81,8 @@ func (c *Config) Describe() { var tmp string colors.PrimaryPrint(`Location: "%s"`, name) - colors.PrintDescription("From", l.From) + // TODO: Add more info + // colors.PrintDescription("From", l.From) tmp = "" for _, to := range l.To { diff --git a/internal/location.go b/internal/location.go index cf75ce1..a786b66 100644 --- a/internal/location.go +++ b/internal/location.go @@ -20,6 +20,7 @@ const ( TypeLocal LocationType = "local" TypeVolume LocationType = "volume" VolumePrefix string = "volume:" + TagPrefix string = "ar:" ) type HookArray = []string @@ -33,7 +34,7 @@ type Hooks struct { type Location struct { name string `yaml:",omitempty"` - From string `yaml:"from,omitempty"` + From []string `yaml:"from,omitempty"` To []string `yaml:"to,omitempty"` Hooks Hooks `yaml:"hooks,omitempty"` Cron string `yaml:"cron,omitempty"` @@ -179,8 +180,9 @@ func (l Location) Backup(cron bool, specificBackend string) []error { cmd = append(cmd, lFlags...) cmd = append(cmd, bFlags...) if cron { - cmd = append(cmd, "--tag", "cron") + cmd = append(cmd, "--tag", TagPrefix+"cron") } + cmd = append(cmd, "--tag", TagPrefix+"location:"+l.name) cmd = append(cmd, ".") backupOptions := ExecuteOptions{ Dir: options.Dir, From ac756dfbde3a92cfc1ef836d0b98fafd270e05b6 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:31:31 +0100 Subject: [PATCH 02/24] bug not showing error messages --- cmd/backup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/backup.go b/cmd/backup.go index e33af34..38d3fe1 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -30,7 +30,7 @@ var backupCmd = &cobra.Command{ } location, _ := internal.GetLocation(splitted[0]) errs := location.Backup(false, specificBackend) - for err := range errs { + for _, err := range errs { colors.Error.Println(err) errors++ } From 2c46f0da0c65c5437bb8c957b4a2952916e0f869 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:31:47 +0100 Subject: [PATCH 03/24] restore arg help --- cmd/restore.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/restore.go b/cmd/restore.go index 8237199..f22a1dd 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -9,8 +9,9 @@ import ( ) var restoreCmd = &cobra.Command{ - Use: "restore", + Use: "restore [snapshot id]", Short: "Restore backup for location", + Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { err := lock.Lock() CheckErr(err) @@ -24,7 +25,11 @@ var restoreCmd = &cobra.Command{ target, _ := cmd.Flags().GetString("to") from, _ := cmd.Flags().GetString("from") force, _ := cmd.Flags().GetBool("force") - err = l.Restore(target, from, force) + snapshot := "" + if len(args) > 0 { + snapshot = args[0] + } + err = l.Restore(target, from, force, snapshot) CheckErr(err) }, } From 6817f494ef758e4b273e994a55dedf03c9df70bc Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:01 +0100 Subject: [PATCH 04/24] util to check if volume exists --- internal/utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/utils.go b/internal/utils.go index 7756677..52172c8 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -77,3 +77,9 @@ func CopyFile(from, to string) error { } return nil } + +func CheckIfVolumeExists(volume string) bool { + out, err := ExecuteCommand(ExecuteOptions{Command: "docker"}, "volume", "inspect", volume) + fmt.Println(out) + return err == nil +} From bcfc734cd1d47ec2fac923baf64dd6041bc074c7 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:30 +0100 Subject: [PATCH 05/24] describe multiple sources --- internal/config.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/config.go b/internal/config.go index a1733c9..b6ea2e0 100644 --- a/internal/config.go +++ b/internal/config.go @@ -81,8 +81,11 @@ func (c *Config) Describe() { var tmp string colors.PrimaryPrint(`Location: "%s"`, name) - // TODO: Add more info - // colors.PrintDescription("From", l.From) + tmp = "" + for _, path := range l.From { + tmp += fmt.Sprintf("\t%s %s\n", colors.Success.Sprint("←"), path) + } + colors.PrintDescription("From", tmp) tmp = "" for _, to := range l.To { From 14dd41d60df72a52ca3820557d6b4131ff6074db Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:34 +0100 Subject: [PATCH 06/24] docs --- docs/markdown/cli/restore.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/markdown/cli/restore.md b/docs/markdown/cli/restore.md index 2bd1008..17d2bc9 100644 --- a/docs/markdown/cli/restore.md +++ b/docs/markdown/cli/restore.md @@ -1,12 +1,12 @@ # Restore ```bash -autorestic restore [-l, --location] [--from backend] [--to ] [-f, --force] +autorestic restore [-l, --location] [--from backend] [--to ] [-f, --force] [snapshot] ``` -This will restore all the locations to the selected target. If for one location there are more than one backends specified autorestic will take the first one. +This will restore the location to the selected target. If for one location there are more than one backends specified autorestic will take the first one. If no specific snapshot is specified `autorestic` will use `latest`. -The `--to` path has to be empty as no data will be overwritten by default. If you are sure you can pass the `-f, --force` flag and the data will be overwritten in the destination. However note that this will overwrite all the data existent in the backup, not only the 1 file that is missing e.g. +If you are sure you can pass the `-f, --force` flag and the data will be overwritten in the destination. However note that this will overwrite all the data existent in the backup, not only the 1 file that is missing e.g. ## Example From d0b1b86fdd183bd8462531b559f1fe473173a5e6 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:55 +0100 Subject: [PATCH 07/24] docker runner --- internal/backend.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/backend.go b/internal/backend.go index e60c709..7327e0d 100644 --- a/internal/backend.go +++ b/internal/backend.go @@ -157,25 +157,32 @@ func (b Backend) ExecDocker(l Location, args []string) (string, error) { if err != nil { return "", err } - volume := l.getVolumeName() - path, _ := l.getPath() + volume := l.From[0] options := ExecuteOptions{ Command: "docker", Envs: env, } + dir := "/data" docker := []string{ "run", "--rm", "--entrypoint", "ash", - "--workdir", path, - "--volume", volume + ":" + path, + "--workdir", dir, + "--volume", volume + ":" + dir, } + // Use of docker host, not the container host if hostname, err := os.Hostname(); err == nil { docker = append(docker, "--hostname", hostname) } - if b.Type == "local" { + switch b.Type { + case "local": actual := env["RESTIC_REPOSITORY"] docker = append(docker, "--volume", actual+":"+"/repo") env["RESTIC_REPOSITORY"] = "/repo" + case "b2": + case "s3": + // No additional setup needed + default: + return "", fmt.Errorf("Backend type \"%s\" is not supported as volume endpoint", b.Type) } for key, value := range env { docker = append(docker, "--env", key+"="+value) From 4fe241e6f3b955bd7e08301f295790ec098d4918 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:33:02 +0100 Subject: [PATCH 08/24] support for multiple sources --- internal/location.go | 155 ++++++++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 67 deletions(-) diff --git a/internal/location.go b/internal/location.go index a786b66..0151802 100644 --- a/internal/location.go +++ b/internal/location.go @@ -17,15 +17,14 @@ import ( type LocationType string const ( - TypeLocal LocationType = "local" - TypeVolume LocationType = "volume" - VolumePrefix string = "volume:" - TagPrefix string = "ar:" + TypeLocal LocationType = "local" + TypeVolume LocationType = "volume" ) type HookArray = []string type Hooks struct { + Dir string `yaml:"dir"` Before HookArray `yaml:"before,omitempty"` After HookArray `yaml:"after,omitempty"` Success HookArray `yaml:"success,omitempty"` @@ -35,6 +34,7 @@ type Hooks struct { type Location struct { name string `yaml:",omitempty"` From []string `yaml:"from,omitempty"` + Type string `yaml:"type,omitempty"` To []string `yaml:"to,omitempty"` Hooks Hooks `yaml:"hooks,omitempty"` Cron string `yaml:"cron,omitempty"` @@ -48,21 +48,32 @@ func GetLocation(name string) (Location, bool) { } func (l Location) validate() error { - if l.From == "" { + if len(l.From) == 0 { return fmt.Errorf(`Location "%s" is missing "from" key`, l.name) } - if l.getType() == TypeLocal { - if from, err := GetPathRelativeToConfig(l.From); err != nil { - return err - } else { - if stat, err := os.Stat(from); err != nil { + t, err := l.getType() + if err != nil { + return err + } + switch t { + case TypeLocal: + for _, path := range l.From { + if from, err := GetPathRelativeToConfig(path); err != nil { return err } else { - if !stat.IsDir() { - return fmt.Errorf("\"%s\" is not valid directory for location \"%s\"", from, l.name) + if stat, err := os.Stat(from); err != nil { + return err + } else { + if !stat.IsDir() { + return fmt.Errorf("\"%s\" is not valid directory for location \"%s\"", from, l.name) + } } } } + case TypeVolume: + if len(l.From) > 1 { + return fmt.Errorf(`location "%s" has more than one docker volume`, l.name) + } } if len(l.To) == 0 { @@ -78,10 +89,17 @@ func (l Location) validate() error { return nil } -func ExecuteHooks(commands []string, options ExecuteOptions) error { +func (l Location) ExecuteHooks(commands []string, options ExecuteOptions) error { if len(commands) == 0 { return nil } + if l.Hooks.Dir != "" { + if dir, err := GetPathRelativeToConfig(l.Hooks.Dir); err != nil { + return err + } else { + options.Dir = dir + } + } colors.Secondary.Println("\nRunning hooks") for _, command := range commands { colors.Body.Println("> " + command) @@ -98,39 +116,38 @@ func ExecuteHooks(commands []string, options ExecuteOptions) error { return nil } -func (l Location) getType() LocationType { - if strings.HasPrefix(l.From, VolumePrefix) { - return TypeVolume +func (l Location) getType() (LocationType, error) { + t := strings.ToLower(l.Type) + if t == "" || t == "local" { + return TypeLocal, nil + } else if t == "volume" { + return TypeVolume, nil } - return TypeLocal + return "", fmt.Errorf("invalid location type \"%s\"", l.Type) } -func (l Location) getVolumeName() string { - return strings.TrimPrefix(l.From, VolumePrefix) +func (l Location) getTag(parts ...string) string { + parts = append([]string{"ar"}, parts...) + return strings.Join(parts, ":") } -func (l Location) getPath() (string, error) { - t := l.getType() - switch t { - case TypeLocal: - if path, err := GetPathRelativeToConfig(l.From); err != nil { - return "", err - } else { - return path, nil - } - case TypeVolume: - return "/volume/" + l.name + "/" + l.getVolumeName(), nil - } - return "", fmt.Errorf("could not get path for location \"%s\"", l.name) +func (l Location) getLocationTag() string { + return l.getTag("location", l.name) } func (l Location) Backup(cron bool, specificBackend string) []error { var errors []error var backends []string colors.PrimaryPrint(" Backing up location \"%s\" ", l.name) - t := l.getType() + t, err := l.getType() + if err != nil { + errors = append(errors, err) + return errors + } + cwd, _ := GetPathRelativeToConfig(".") options := ExecuteOptions{ Command: "bash", + Dir: cwd, Envs: map[string]string{ "AUTORESTIC_LOCATION": l.name, }, @@ -138,18 +155,11 @@ func (l Location) Backup(cron bool, specificBackend string) []error { if err := l.validate(); err != nil { errors = append(errors, err) - colors.Error.Print(err) goto after } - if t == TypeLocal { - dir, _ := GetPathRelativeToConfig(l.From) - colors.Faint.Printf("Executing under: \"%s\"\n", dir) - options.Dir = dir - } - // Hooks - if err := ExecuteHooks(l.Hooks.Before, options); err != nil { + if err := l.ExecuteHooks(l.Hooks.Before, options); err != nil { errors = append(errors, err) goto after } @@ -180,25 +190,35 @@ func (l Location) Backup(cron bool, specificBackend string) []error { cmd = append(cmd, lFlags...) cmd = append(cmd, bFlags...) if cron { - cmd = append(cmd, "--tag", TagPrefix+"cron") + cmd = append(cmd, "--tag", l.getTag("cron")) } - cmd = append(cmd, "--tag", TagPrefix+"location:"+l.name) - cmd = append(cmd, ".") + cmd = append(cmd, "--tag", l.getLocationTag()) backupOptions := ExecuteOptions{ - Dir: options.Dir, Envs: env, } var out string - switch t { case TypeLocal: + for _, from := range l.From { + path, err := GetPathRelativeToConfig(from) + if err != nil { + errors = append(errors, err) + goto after + } + cmd = append(cmd, path) + } out, err = ExecuteResticCommand(backupOptions, cmd...) case TypeVolume: + ok := CheckIfVolumeExists(l.From[0]) + if !ok { + errors = append(errors, fmt.Errorf("volume \"%s\" does not exist", l.From[0])) + continue + } + cmd = append(cmd, "/data") out, err = backend.ExecDocker(l, cmd) } if err != nil { - colors.Error.Println(out) errors = append(errors, err) continue } @@ -215,7 +235,7 @@ func (l Location) Backup(cron bool, specificBackend string) []error { } // After hooks - if err := ExecuteHooks(l.Hooks.After, options); err != nil { + if err := l.ExecuteHooks(l.Hooks.After, options); err != nil { errors = append(errors, err) } @@ -226,22 +246,19 @@ after: } else { commands = l.Hooks.Success } - if err := ExecuteHooks(commands, options); err != nil { + if err := l.ExecuteHooks(commands, options); err != nil { errors = append(errors, err) } - colors.Success.Println("Done") + if len(errors) == 0 { + colors.Success.Println("Done") + } return errors } func (l Location) Forget(prune bool, dry bool) error { colors.PrimaryPrint("Forgetting for location \"%s\"", l.name) - path, err := l.getPath() - if err != nil { - return err - } - for _, to := range l.To { backend, _ := GetBackend(to) colors.Secondary.Printf("For backend \"%s\"\n", backend.name) @@ -254,7 +271,7 @@ func (l Location) Forget(prune bool, dry bool) error { } lFlags := getOptions(l.Options, "forget") bFlags := getOptions(backend.Options, "forget") - cmd := []string{"forget", "--path", path} + cmd := []string{"forget", "--tag", l.getLocationTag()} if prune { cmd = append(cmd, "--prune") } @@ -284,7 +301,7 @@ func (l Location) hasBackend(backend string) bool { return false } -func (l Location) Restore(to, from string, force bool) error { +func (l Location) Restore(to, from string, force bool, snapshot string) error { if from == "" { from = l.To[0] } else if !l.hasBackend(from) { @@ -295,16 +312,20 @@ func (l Location) Restore(to, from string, force bool) error { if err != nil { return err } - colors.PrimaryPrint("Restoring location \"%s\"", l.name) - backend, _ := GetBackend(from) - path, err := l.getPath() - if err != nil { - return nil + if snapshot == "" { + snapshot = "latest" } - colors.Secondary.Println("Restoring lastest snapshot") - colors.Body.Printf("%s → %s.\n", from, path) - switch l.getType() { + + colors.PrimaryPrint("Restoring location \"%s\"", l.name) + backend, _ := GetBackend(from) + colors.Secondary.Printf("Restoring %s@%s → %s\n", snapshot, backend.name, to) + + t, err := l.getType() + if err != nil { + return err + } + switch t { case TypeLocal: // Check if target is empty if !force { @@ -324,9 +345,9 @@ func (l Location) Restore(to, from string, force bool) error { } } } - err = backend.Exec([]string{"restore", "--target", to, "--path", path, "latest"}) + err = backend.Exec([]string{"restore", "--target", to, "--tag", l.getLocationTag(), snapshot}) case TypeVolume: - _, err = backend.ExecDocker(l, []string{"restore", "--target", ".", "--path", path, "latest"}) + _, err = backend.ExecDocker(l, []string{"restore", "--target", "/", "--tag", l.getLocationTag(), snapshot}) } if err != nil { return err From a68e3e426efcf3f9c3b0ff0c41d2b063199c7408 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 23:07:12 +0100 Subject: [PATCH 09/24] simplify options handling --- internal/config.go | 13 +++++++++++++ internal/location.go | 10 ++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/internal/config.go b/internal/config.go index dce9331..be7118d 100644 --- a/internal/config.go +++ b/internal/config.go @@ -55,6 +55,7 @@ func GetConfig() *Config { config = &Config{} if err := viper.UnmarshalExact(config); err != nil { + colors.Error.Println(err) colors.Error.Println("Could not parse config file!") lock.Unlock() os.Exit(1) @@ -273,3 +274,15 @@ func getOptions(options Options, key string) []string { } return selected } + +func combineOptions(key string, l Location, b Backend) []string { + // Priority: location > backend > global + var options []string + gFlags := getOptions(GetConfig().Global, key) + bFlags := getOptions(b.Options, key) + lFlags := getOptions(l.Options, key) + options = append(options, gFlags...) + options = append(options, bFlags...) + options = append(options, lFlags...) + return options +} diff --git a/internal/location.go b/internal/location.go index 0151802..024252e 100644 --- a/internal/location.go +++ b/internal/location.go @@ -184,11 +184,8 @@ func (l Location) Backup(cron bool, specificBackend string) []error { continue } - lFlags := getOptions(l.Options, "backup") - bFlags := getOptions(backend.Options, "backup") cmd := []string{"backup"} - cmd = append(cmd, lFlags...) - cmd = append(cmd, bFlags...) + cmd = append(cmd, combineOptions("backup", l, backend)...) if cron { cmd = append(cmd, "--tag", l.getTag("cron")) } @@ -269,8 +266,6 @@ func (l Location) Forget(prune bool, dry bool) error { options := ExecuteOptions{ Envs: env, } - lFlags := getOptions(l.Options, "forget") - bFlags := getOptions(backend.Options, "forget") cmd := []string{"forget", "--tag", l.getLocationTag()} if prune { cmd = append(cmd, "--prune") @@ -278,8 +273,7 @@ func (l Location) Forget(prune bool, dry bool) error { if dry { cmd = append(cmd, "--dry-run") } - cmd = append(cmd, lFlags...) - cmd = append(cmd, bFlags...) + cmd = append(cmd, combineOptions("forget", l, backend)...) out, err := ExecuteResticCommand(options, cmd...) if VERBOSE { colors.Faint.Println(out) From cf92d400c2c97d9f8f166d4fdeb16f0a74179db3 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 23:11:06 +0100 Subject: [PATCH 10/24] changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9341c44..aacb2aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.0] - 2021-11 + +### Added + +- Support for multiple paths +- Improved error handling + +## [1.4.1] - 2021-10-31 + +### Fixes + +- Numeric values from config files not being passed to env. + ## [1.4.0] - 2021-10-30 ### Added From 0ae374cd450a387aaf3f7088456ba268e43829d9 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 23:35:46 +0100 Subject: [PATCH 11/24] docs --- docs/markdown/_toc.md | 5 +++ docs/markdown/location/docker.md | 12 ++---- .../{upgrade.md => migration/0.x_1.0.md} | 4 +- docs/markdown/migration/1.4_1.5.md | 39 +++++++++++++++++++ 4 files changed, 49 insertions(+), 11 deletions(-) rename docs/markdown/{upgrade.md => migration/0.x_1.0.md} (94%) create mode 100644 docs/markdown/migration/1.4_1.5.md diff --git a/docs/markdown/_toc.md b/docs/markdown/_toc.md index 600055a..bd4e321 100644 --- a/docs/markdown/_toc.md +++ b/docs/markdown/_toc.md @@ -40,6 +40,11 @@ > [Uninstall](/cli/uninstall) > [Upgrade](/cli/upgrade) +> :Collapse label=Migration +> +> [0.x → 1.0](/migration/0.x_1.0) +> [1.4 → 1.5](/migration/1.4_1.5) + [Examples](/examples) [QA](/qa) [Community](/community) diff --git a/docs/markdown/location/docker.md b/docs/markdown/location/docker.md index 3223779..5498dd0 100644 --- a/docs/markdown/location/docker.md +++ b/docs/markdown/location/docker.md @@ -3,7 +3,7 @@ autorestic supports docker volumes directly, without needing them to be mounted to the host filesystem. ```yaml | docker-compose.yml -version: '3.7' +version: '3.8' volumes: data: @@ -18,13 +18,9 @@ services: ```yaml | .autorestic.yml locations: - - name: hello - from: volume:my-data - to: - - remote - -backends: - - name: remote + foo: + from: my-data + type: volume # ... ``` diff --git a/docs/markdown/upgrade.md b/docs/markdown/migration/0.x_1.0.md similarity index 94% rename from docs/markdown/upgrade.md rename to docs/markdown/migration/0.x_1.0.md index 72d79ab..252922d 100644 --- a/docs/markdown/upgrade.md +++ b/docs/markdown/migration/0.x_1.0.md @@ -1,6 +1,4 @@ -# Upgrade - -## From `0.x` to `1.0` +# From `0.x` to `1.0` Most of the config file is remained compatible, however to clean up the backends custom environment variables were moved from the root object to an `env` object. diff --git a/docs/markdown/migration/1.4_1.5.md b/docs/markdown/migration/1.4_1.5.md new file mode 100644 index 0000000..dd74424 --- /dev/null +++ b/docs/markdown/migration/1.4_1.5.md @@ -0,0 +1,39 @@ +# Migration from `1.4` to `1.5` + +## Hooks + +Since `1.5` multiple sources for a location are possible. +For this reason, while before hooks where executed in the folder of the source, now they are executed in the directory of the config `.autorestic.yaml`. + +You can overwrite this behavior with the new `dir` option in the hook section of the config. + +```yaml +locations: + l1: + # ... + from: /foo/bar + hooks: + dir: /foo/bar + before: pwd +``` + +## Docker volumes + +The syntax with docker volumes has changed and needs to be adjusted. + +```yaml +# Before +locations: + foo: + from: volume:my-data +``` + +```yaml +# After +locations: + foo: + from: my-data + type: volume +``` + +> :ToCPrevNext From 90914d2078f881618b70bab58c4e002e066784f6 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 23:35:52 +0100 Subject: [PATCH 12/24] changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1029273..9c4058c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.0] - 2021-11 + +### Added + +- Support for multiple paths +- Improved error handling +- Allow for specific snapshot to be restored + +### Changed + +- [Breaking Change] Declaration of docker volumes. See: https://autorestic.vercel.app/migration/1.4_1.5 +- [Breaking Change] Hooks default executing directory now defaults to the config file directory. See: https://autorestic.vercel.app/migration/1.4_1.5 + ## [1.4.1] - 2021-10-31 ### Fixes From 59035da46a1eafcfc9be2a97bf5b999d93446101 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 23:58:08 +0100 Subject: [PATCH 13/24] remove output --- internal/utils.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/utils.go b/internal/utils.go index 52172c8..1bc2b36 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -79,7 +79,6 @@ func CopyFile(from, to string) error { } func CheckIfVolumeExists(volume string) bool { - out, err := ExecuteCommand(ExecuteOptions{Command: "docker"}, "volume", "inspect", volume) - fmt.Println(out) + _, err := ExecuteCommand(ExecuteOptions{Command: "docker"}, "volume", "inspect", volume) return err == nil } From 3dd3956d644071b7c6aa4e88f329e3fad0d342ab Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Mon, 1 Nov 2021 00:16:54 +0100 Subject: [PATCH 14/24] support for rclone --- internal/backend.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/backend.go b/internal/backend.go index 7327e0d..cf71f75 100644 --- a/internal/backend.go +++ b/internal/backend.go @@ -163,8 +163,10 @@ func (b Backend) ExecDocker(l Location, args []string) (string, error) { Envs: env, } dir := "/data" + args = append([]string{"restic"}, args...) docker := []string{ "run", "--rm", + "--pull", "always", "--entrypoint", "ash", "--workdir", dir, "--volume", volume + ":" + dir, @@ -181,13 +183,23 @@ func (b Backend) ExecDocker(l Location, args []string) (string, error) { case "b2": case "s3": // No additional setup needed + case "rclone": + // Read host rclone config and mount it into the container + configFile, err := ExecuteCommand(ExecuteOptions{Command: "rclone"}, "config", "file") + if err != nil { + return "", err + } + splitted := strings.Split(strings.TrimSpace(configFile), "\n") + configFilePath := splitted[len(splitted)-1] + docker = append(docker, "--volume", configFilePath+":"+"/root/.config/rclone/rclone.conf:ro") + args = append([]string{"apk", "add", "rclone", "&&"}, args...) default: return "", fmt.Errorf("Backend type \"%s\" is not supported as volume endpoint", b.Type) } for key, value := range env { docker = append(docker, "--env", key+"="+value) } - docker = append(docker, "restic/restic", "-c", "restic "+strings.Join(args, " ")) + docker = append(docker, "restic/restic", "-c", strings.Join(args, " ")) out, err := ExecuteCommand(options, docker...) return out, err } From cd7a5cbc137f4b4b3f083f6db62196fcc82a5dcf Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Mon, 1 Nov 2021 00:19:32 +0100 Subject: [PATCH 15/24] also enable azure and google cloud --- internal/backend.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/backend.go b/internal/backend.go index cf71f75..0eb8773 100644 --- a/internal/backend.go +++ b/internal/backend.go @@ -182,6 +182,8 @@ func (b Backend) ExecDocker(l Location, args []string) (string, error) { env["RESTIC_REPOSITORY"] = "/repo" case "b2": case "s3": + case "azure": + case "gs": // No additional setup needed case "rclone": // Read host rclone config and mount it into the container @@ -192,6 +194,7 @@ func (b Backend) ExecDocker(l Location, args []string) (string, error) { splitted := strings.Split(strings.TrimSpace(configFile), "\n") configFilePath := splitted[len(splitted)-1] docker = append(docker, "--volume", configFilePath+":"+"/root/.config/rclone/rclone.conf:ro") + // Install rclone in the container args = append([]string{"apk", "add", "rclone", "&&"}, args...) default: return "", fmt.Errorf("Backend type \"%s\" is not supported as volume endpoint", b.Type) From d3b4915d258209a85cc6a7323fcd4cadacf749b1 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Mon, 1 Nov 2021 09:09:29 +0100 Subject: [PATCH 16/24] deprecation notion --- docs/markdown/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/markdown/installation.md b/docs/markdown/installation.md index 044147e..4f6d0e2 100644 --- a/docs/markdown/installation.md +++ b/docs/markdown/installation.md @@ -20,6 +20,6 @@ If you are on macOS you can install through brew: `brew install autorestic`. ### AUR -If you are on Arch there is an [AUR Package](https://aur.archlinux.org/packages/autorestic-bin/) (looking for maintainers). +~~If you are on Arch there is an [AUR Package](https://aur.archlinux.org/packages/autorestic-bin/) (looking for maintainers).~~ - Deprecated > :ToCPrevNext From c250391f6756db21598530ac1d26f7ac892adbd0 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Mon, 1 Nov 2021 11:19:27 +0100 Subject: [PATCH 17/24] docs --- docs/markdown/location/options.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/markdown/location/options.md b/docs/markdown/location/options.md index 16d0260..af35269 100644 --- a/docs/markdown/location/options.md +++ b/docs/markdown/location/options.md @@ -26,7 +26,7 @@ locations: ## Example -In this example, whenever `autorestic` runs `restic backup` it will append a `--tag abc --tag` to the native command. +In this example, whenever `autorestic` runs `restic backup` it will append a `--tag foo --tag bar` to the native command. ```yaml locations: @@ -40,6 +40,12 @@ locations: - bar ``` +## Priority + +Options can be set globally, on the backends or on the locations. + +The priority is as follows: `location > backend > global`. + ## Global Options It is possible to specify global flags that will be run every time restic is invoked. To do so specify them under `global` in your config file. From 113a97c283fde02a9d4b8f56ccad1514296ebafe Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sat, 6 Nov 2021 22:00:44 +0100 Subject: [PATCH 18/24] add config version to ensure compatibility --- CHANGELOG.md | 6 +++++- docs/markdown/migration/1.4_1.5.md | 8 ++++++++ internal/config.go | 33 ++++++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c4058c..a179ef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Improved error handling - Allow for specific snapshot to be restored +### Fixed + +- rclone in docker volumes + ### Changed - [Breaking Change] Declaration of docker volumes. See: https://autorestic.vercel.app/migration/1.4_1.5 @@ -20,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.4.1] - 2021-10-31 -### Fixes +### Fixed - Numeric values from config files not being passed to env. diff --git a/docs/markdown/migration/1.4_1.5.md b/docs/markdown/migration/1.4_1.5.md index dd74424..a94b435 100644 --- a/docs/markdown/migration/1.4_1.5.md +++ b/docs/markdown/migration/1.4_1.5.md @@ -1,5 +1,13 @@ # Migration from `1.4` to `1.5` +## Config files + +- The config file now required to have a version number. This has to be added with `version: 2` at the root. +- Hooks now optionally support `dir: /some/dir` in the [options object](https://pkg.go.dev/github.com/cupcakearmy/autorestic/internal#Hooks). +- Docker volumes don't get prefixed with `volume:` anymore, rather you have to set the `type: volume` in the [location config](https://pkg.go.dev/github.com/cupcakearmy/autorestic/internal#Hooks). + +See detailed instructions below. + ## Hooks Since `1.5` multiple sources for a location are possible. diff --git a/internal/config.go b/internal/config.go index be7118d..82d4a9b 100644 --- a/internal/config.go +++ b/internal/config.go @@ -26,6 +26,7 @@ type OptionMap map[string][]interface{} type Options map[string]OptionMap type Config struct { + Version string `yaml:"version"` Extras interface{} `yaml:"extras"` Locations map[string]Location `yaml:"locations"` Backends map[string]Backend `yaml:"backends"` @@ -35,7 +36,19 @@ type Config struct { var once sync.Once var config *Config +func exitConfig(err error, msg string) { + if err != nil { + colors.Error.Println(err) + } + if msg != "" { + colors.Error.Println(msg) + } + lock.Unlock() + os.Exit(1) +} + func GetConfig() *Config { + if config == nil { once.Do(func() { if err := viper.ReadInConfig(); err == nil { @@ -53,12 +66,24 @@ func GetConfig() *Config { return } + var versionConfig interface{} + viper.UnmarshalKey("version", &versionConfig) + if versionConfig == nil { + exitConfig(nil, "no version specified in config file. please see docs on how to migrate") + } + version, ok := versionConfig.(int) + if !ok { + exitConfig(nil, "version specified in config file is not an int") + } else { + // Check for version + if version != 2 { + exitConfig(nil, "unsupported version number. please check the docs") + } + } + config = &Config{} if err := viper.UnmarshalExact(config); err != nil { - colors.Error.Println(err) - colors.Error.Println("Could not parse config file!") - lock.Unlock() - os.Exit(1) + exitConfig(err, "Could not parse config file!") } }) } From 4126436f7f11607ea274390e91bcc633e44458cf Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 7 Nov 2021 11:42:45 +0100 Subject: [PATCH 19/24] migration docs --- docs/markdown/migration/1.4_1.5.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/markdown/migration/1.4_1.5.md b/docs/markdown/migration/1.4_1.5.md index a94b435..349bc40 100644 --- a/docs/markdown/migration/1.4_1.5.md +++ b/docs/markdown/migration/1.4_1.5.md @@ -1,5 +1,9 @@ # Migration from `1.4` to `1.5` +## ⚠️ Important notes + +The way snapshots are referenced in the `restore` and `prune` commands has been changed. Before they were referenced by the path. Now every backup is tagged and those tags are then referenced in the cli. This means that when running restore and forget commands old backups are not taken into account anymore. + ## Config files - The config file now required to have a version number. This has to be added with `version: 2` at the root. @@ -8,6 +12,15 @@ See detailed instructions below. +## Config Version + +```yaml +version: 2 # Added + +backends: + # ... +``` + ## Hooks Since `1.5` multiple sources for a location are possible. From 170bdb81ad525614d709d99cb9b2f13efffafca0 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 7 Nov 2021 11:48:03 +0100 Subject: [PATCH 20/24] tags --- internal/location.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/location.go b/internal/location.go index 024252e..2b2e4a3 100644 --- a/internal/location.go +++ b/internal/location.go @@ -126,13 +126,13 @@ func (l Location) getType() (LocationType, error) { return "", fmt.Errorf("invalid location type \"%s\"", l.Type) } -func (l Location) getTag(parts ...string) string { +func buildTag(parts ...string) string { parts = append([]string{"ar"}, parts...) return strings.Join(parts, ":") } -func (l Location) getLocationTag() string { - return l.getTag("location", l.name) +func (l Location) getLocationTags() string { + return buildTag("location", l.name) } func (l Location) Backup(cron bool, specificBackend string) []error { @@ -187,9 +187,9 @@ func (l Location) Backup(cron bool, specificBackend string) []error { cmd := []string{"backup"} cmd = append(cmd, combineOptions("backup", l, backend)...) if cron { - cmd = append(cmd, "--tag", l.getTag("cron")) + cmd = append(cmd, "--tag", buildTag("cron")) } - cmd = append(cmd, "--tag", l.getLocationTag()) + cmd = append(cmd, "--tag", l.getLocationTags()) backupOptions := ExecuteOptions{ Envs: env, } @@ -266,7 +266,7 @@ func (l Location) Forget(prune bool, dry bool) error { options := ExecuteOptions{ Envs: env, } - cmd := []string{"forget", "--tag", l.getLocationTag()} + cmd := []string{"forget", "--tag", l.getLocationTags()} if prune { cmd = append(cmd, "--prune") } @@ -339,9 +339,9 @@ func (l Location) Restore(to, from string, force bool, snapshot string) error { } } } - err = backend.Exec([]string{"restore", "--target", to, "--tag", l.getLocationTag(), snapshot}) + err = backend.Exec([]string{"restore", "--target", to, "--tag", l.getLocationTags(), snapshot}) case TypeVolume: - _, err = backend.ExecDocker(l, []string{"restore", "--target", "/", "--tag", l.getLocationTag(), snapshot}) + _, err = backend.ExecDocker(l, []string{"restore", "--target", "/", "--tag", l.getLocationTags(), snapshot}) } if err != nil { return err From c55e91b8ff0bea3db8b32538dff3f9d8526a2727 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sat, 20 Nov 2021 16:55:20 +0100 Subject: [PATCH 21/24] version bump --- internal/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config.go b/internal/config.go index 82d4a9b..108bd00 100644 --- a/internal/config.go +++ b/internal/config.go @@ -16,7 +16,7 @@ import ( "github.com/spf13/viper" ) -const VERSION = "1.4.1" +const VERSION = "1.5.0" var CI bool = false var VERBOSE bool = false From e94e7030fcf94e9759cf65c52283588169f22e4a Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sat, 20 Nov 2021 16:59:13 +0100 Subject: [PATCH 22/24] docker image --- .dockerignore | 6 ++++++ .github/workflows/build.yml | 36 ++++++++++++++++++++++++++++++++---- Dockerfile | 12 ++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6d57110 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +* +!**/*.go +!build +!cmd +!internal +!go.* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4027577..e82998f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,16 +3,44 @@ name: Main on: push: tags: - - 'v*.*.*' + - "v*.*.*" jobs: - build: + docker: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Docker Labels + id: meta + uses: crazy-max/ghaction-docker-meta@v2 + with: + images: cupcakearmy/autorestic + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v2 + with: + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + + release: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '^1.16.3' + go-version: "^1.16.3" - name: Build run: go run build/build.go @@ -25,4 +53,4 @@ jobs: with: files: dist/* env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..65fae81 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.16-alpine as builder + +WORKDIR /app +COPY go.* . +RUN go mod download +COPY . . +RUN go build + +FROM alpine +RUN apk add --no-cache restic rclone +COPY --from=builder /app/autorestic /usr/bin/autorestic +CMD [ "autorestic" ] From 8802b74b47a127d8920396528e4251ee6dee76c6 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sat, 20 Nov 2021 17:03:54 +0100 Subject: [PATCH 23/24] changelog and docs --- CHANGELOG.md | 3 ++- docs/markdown/installation.md | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a179ef3..5b5d899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.5.0] - 2021-11 +## [1.5.0] - 2021-11-20 ### Added - Support for multiple paths - Improved error handling - Allow for specific snapshot to be restored +- Docker image ### Fixed diff --git a/docs/markdown/installation.md b/docs/markdown/installation.md index 4f6d0e2..071e9c5 100644 --- a/docs/markdown/installation.md +++ b/docs/markdown/installation.md @@ -10,6 +10,10 @@ wget -qO - https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/insta ## Alternatives +### Docker + +There is an official docker image over at [cupcakearmy/autorestic](https://hub.docker.com/r/cupcakearmy/autorestic). + ### Manual You can download the right binary from the release page and simply copy it to `/usr/local/bin` or whatever path you prefer. Autoupdates will still work. From ed3c17d678b27925287c0b71f31917abaf19e17f Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sat, 20 Nov 2021 17:09:42 +0100 Subject: [PATCH 24/24] migration docs --- docs/markdown/migration/index.md | 4 ++++ internal/config.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 docs/markdown/migration/index.md diff --git a/docs/markdown/migration/index.md b/docs/markdown/migration/index.md new file mode 100644 index 0000000..624561d --- /dev/null +++ b/docs/markdown/migration/index.md @@ -0,0 +1,4 @@ +# Migration + +- [From 0.x to 1.0](/migration/0.x_1.0) +- [From 1.4 to 1.5](/migration/1.4_1.5) diff --git a/internal/config.go b/internal/config.go index 108bd00..9d5c253 100644 --- a/internal/config.go +++ b/internal/config.go @@ -77,7 +77,7 @@ func GetConfig() *Config { } else { // Check for version if version != 2 { - exitConfig(nil, "unsupported version number. please check the docs") + exitConfig(nil, "unsupported config version number. please check the docs for migration\nhttps://autorestic.vercel.app/migration/") } }