From c2f9ed920427de77f3621b7212f40485d51ed991 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sat, 30 Oct 2021 13:01:31 +0200 Subject: [PATCH 01/13] multiple paths --- internal/config.go | 3 ++- internal/location.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/config.go b/internal/config.go index 3cf5cca..a1733c9 100644 --- a/internal/config.go +++ b/internal/config.go @@ -81,7 +81,8 @@ func (c *Config) Describe() { var tmp string colors.PrimaryPrint(`Location: "%s"`, name) - colors.PrintDescription("From", l.From) + // TODO: Add more info + // colors.PrintDescription("From", l.From) tmp = "" for _, to := range l.To { diff --git a/internal/location.go b/internal/location.go index cf75ce1..a786b66 100644 --- a/internal/location.go +++ b/internal/location.go @@ -20,6 +20,7 @@ const ( TypeLocal LocationType = "local" TypeVolume LocationType = "volume" VolumePrefix string = "volume:" + TagPrefix string = "ar:" ) type HookArray = []string @@ -33,7 +34,7 @@ type Hooks struct { type Location struct { name string `yaml:",omitempty"` - From string `yaml:"from,omitempty"` + From []string `yaml:"from,omitempty"` To []string `yaml:"to,omitempty"` Hooks Hooks `yaml:"hooks,omitempty"` Cron string `yaml:"cron,omitempty"` @@ -179,8 +180,9 @@ func (l Location) Backup(cron bool, specificBackend string) []error { cmd = append(cmd, lFlags...) cmd = append(cmd, bFlags...) if cron { - cmd = append(cmd, "--tag", "cron") + cmd = append(cmd, "--tag", TagPrefix+"cron") } + cmd = append(cmd, "--tag", TagPrefix+"location:"+l.name) cmd = append(cmd, ".") backupOptions := ExecuteOptions{ Dir: options.Dir, From d78fbb66502bdbdcc14c1e3e4b3dd1f416780580 Mon Sep 17 00:00:00 2001 From: n <67550725+n194@users.noreply.github.com> Date: Sun, 31 Oct 2021 19:40:53 +0900 Subject: [PATCH 02/13] Remove credit from AUR --- docs/markdown/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/markdown/installation.md b/docs/markdown/installation.md index a6a9a2a..044147e 100644 --- a/docs/markdown/installation.md +++ b/docs/markdown/installation.md @@ -20,6 +20,6 @@ If you are on macOS you can install through brew: `brew install autorestic`. ### AUR -If you are on Arch there is an [AUR Package](https://aur.archlinux.org/packages/autorestic-bin/) by @n194. +If you are on Arch there is an [AUR Package](https://aur.archlinux.org/packages/autorestic-bin/) (looking for maintainers). > :ToCPrevNext From ac756dfbde3a92cfc1ef836d0b98fafd270e05b6 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:31:31 +0100 Subject: [PATCH 03/13] bug not showing error messages --- cmd/backup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/backup.go b/cmd/backup.go index e33af34..38d3fe1 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -30,7 +30,7 @@ var backupCmd = &cobra.Command{ } location, _ := internal.GetLocation(splitted[0]) errs := location.Backup(false, specificBackend) - for err := range errs { + for _, err := range errs { colors.Error.Println(err) errors++ } From 2c46f0da0c65c5437bb8c957b4a2952916e0f869 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:31:47 +0100 Subject: [PATCH 04/13] restore arg help --- cmd/restore.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/restore.go b/cmd/restore.go index 8237199..f22a1dd 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -9,8 +9,9 @@ import ( ) var restoreCmd = &cobra.Command{ - Use: "restore", + Use: "restore [snapshot id]", Short: "Restore backup for location", + Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { err := lock.Lock() CheckErr(err) @@ -24,7 +25,11 @@ var restoreCmd = &cobra.Command{ target, _ := cmd.Flags().GetString("to") from, _ := cmd.Flags().GetString("from") force, _ := cmd.Flags().GetBool("force") - err = l.Restore(target, from, force) + snapshot := "" + if len(args) > 0 { + snapshot = args[0] + } + err = l.Restore(target, from, force, snapshot) CheckErr(err) }, } From 6817f494ef758e4b273e994a55dedf03c9df70bc Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:01 +0100 Subject: [PATCH 05/13] util to check if volume exists --- internal/utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/utils.go b/internal/utils.go index 7756677..52172c8 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -77,3 +77,9 @@ func CopyFile(from, to string) error { } return nil } + +func CheckIfVolumeExists(volume string) bool { + out, err := ExecuteCommand(ExecuteOptions{Command: "docker"}, "volume", "inspect", volume) + fmt.Println(out) + return err == nil +} From bcfc734cd1d47ec2fac923baf64dd6041bc074c7 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:30 +0100 Subject: [PATCH 06/13] describe multiple sources --- internal/config.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/config.go b/internal/config.go index a1733c9..b6ea2e0 100644 --- a/internal/config.go +++ b/internal/config.go @@ -81,8 +81,11 @@ func (c *Config) Describe() { var tmp string colors.PrimaryPrint(`Location: "%s"`, name) - // TODO: Add more info - // colors.PrintDescription("From", l.From) + tmp = "" + for _, path := range l.From { + tmp += fmt.Sprintf("\t%s %s\n", colors.Success.Sprint("←"), path) + } + colors.PrintDescription("From", tmp) tmp = "" for _, to := range l.To { From 14dd41d60df72a52ca3820557d6b4131ff6074db Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:34 +0100 Subject: [PATCH 07/13] docs --- docs/markdown/cli/restore.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/markdown/cli/restore.md b/docs/markdown/cli/restore.md index 2bd1008..17d2bc9 100644 --- a/docs/markdown/cli/restore.md +++ b/docs/markdown/cli/restore.md @@ -1,12 +1,12 @@ # Restore ```bash -autorestic restore [-l, --location] [--from backend] [--to ] [-f, --force] +autorestic restore [-l, --location] [--from backend] [--to ] [-f, --force] [snapshot] ``` -This will restore all the locations to the selected target. If for one location there are more than one backends specified autorestic will take the first one. +This will restore the location to the selected target. If for one location there are more than one backends specified autorestic will take the first one. If no specific snapshot is specified `autorestic` will use `latest`. -The `--to` path has to be empty as no data will be overwritten by default. If you are sure you can pass the `-f, --force` flag and the data will be overwritten in the destination. However note that this will overwrite all the data existent in the backup, not only the 1 file that is missing e.g. +If you are sure you can pass the `-f, --force` flag and the data will be overwritten in the destination. However note that this will overwrite all the data existent in the backup, not only the 1 file that is missing e.g. ## Example From d0b1b86fdd183bd8462531b559f1fe473173a5e6 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:32:55 +0100 Subject: [PATCH 08/13] docker runner --- internal/backend.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/backend.go b/internal/backend.go index e60c709..7327e0d 100644 --- a/internal/backend.go +++ b/internal/backend.go @@ -157,25 +157,32 @@ func (b Backend) ExecDocker(l Location, args []string) (string, error) { if err != nil { return "", err } - volume := l.getVolumeName() - path, _ := l.getPath() + volume := l.From[0] options := ExecuteOptions{ Command: "docker", Envs: env, } + dir := "/data" docker := []string{ "run", "--rm", "--entrypoint", "ash", - "--workdir", path, - "--volume", volume + ":" + path, + "--workdir", dir, + "--volume", volume + ":" + dir, } + // Use of docker host, not the container host if hostname, err := os.Hostname(); err == nil { docker = append(docker, "--hostname", hostname) } - if b.Type == "local" { + switch b.Type { + case "local": actual := env["RESTIC_REPOSITORY"] docker = append(docker, "--volume", actual+":"+"/repo") env["RESTIC_REPOSITORY"] = "/repo" + case "b2": + case "s3": + // No additional setup needed + default: + return "", fmt.Errorf("Backend type \"%s\" is not supported as volume endpoint", b.Type) } for key, value := range env { docker = append(docker, "--env", key+"="+value) From 4fe241e6f3b955bd7e08301f295790ec098d4918 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:33:02 +0100 Subject: [PATCH 09/13] support for multiple sources --- internal/location.go | 155 ++++++++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 67 deletions(-) diff --git a/internal/location.go b/internal/location.go index a786b66..0151802 100644 --- a/internal/location.go +++ b/internal/location.go @@ -17,15 +17,14 @@ import ( type LocationType string const ( - TypeLocal LocationType = "local" - TypeVolume LocationType = "volume" - VolumePrefix string = "volume:" - TagPrefix string = "ar:" + TypeLocal LocationType = "local" + TypeVolume LocationType = "volume" ) type HookArray = []string type Hooks struct { + Dir string `yaml:"dir"` Before HookArray `yaml:"before,omitempty"` After HookArray `yaml:"after,omitempty"` Success HookArray `yaml:"success,omitempty"` @@ -35,6 +34,7 @@ type Hooks struct { type Location struct { name string `yaml:",omitempty"` From []string `yaml:"from,omitempty"` + Type string `yaml:"type,omitempty"` To []string `yaml:"to,omitempty"` Hooks Hooks `yaml:"hooks,omitempty"` Cron string `yaml:"cron,omitempty"` @@ -48,21 +48,32 @@ func GetLocation(name string) (Location, bool) { } func (l Location) validate() error { - if l.From == "" { + if len(l.From) == 0 { return fmt.Errorf(`Location "%s" is missing "from" key`, l.name) } - if l.getType() == TypeLocal { - if from, err := GetPathRelativeToConfig(l.From); err != nil { - return err - } else { - if stat, err := os.Stat(from); err != nil { + t, err := l.getType() + if err != nil { + return err + } + switch t { + case TypeLocal: + for _, path := range l.From { + if from, err := GetPathRelativeToConfig(path); err != nil { return err } else { - if !stat.IsDir() { - return fmt.Errorf("\"%s\" is not valid directory for location \"%s\"", from, l.name) + if stat, err := os.Stat(from); err != nil { + return err + } else { + if !stat.IsDir() { + return fmt.Errorf("\"%s\" is not valid directory for location \"%s\"", from, l.name) + } } } } + case TypeVolume: + if len(l.From) > 1 { + return fmt.Errorf(`location "%s" has more than one docker volume`, l.name) + } } if len(l.To) == 0 { @@ -78,10 +89,17 @@ func (l Location) validate() error { return nil } -func ExecuteHooks(commands []string, options ExecuteOptions) error { +func (l Location) ExecuteHooks(commands []string, options ExecuteOptions) error { if len(commands) == 0 { return nil } + if l.Hooks.Dir != "" { + if dir, err := GetPathRelativeToConfig(l.Hooks.Dir); err != nil { + return err + } else { + options.Dir = dir + } + } colors.Secondary.Println("\nRunning hooks") for _, command := range commands { colors.Body.Println("> " + command) @@ -98,39 +116,38 @@ func ExecuteHooks(commands []string, options ExecuteOptions) error { return nil } -func (l Location) getType() LocationType { - if strings.HasPrefix(l.From, VolumePrefix) { - return TypeVolume +func (l Location) getType() (LocationType, error) { + t := strings.ToLower(l.Type) + if t == "" || t == "local" { + return TypeLocal, nil + } else if t == "volume" { + return TypeVolume, nil } - return TypeLocal + return "", fmt.Errorf("invalid location type \"%s\"", l.Type) } -func (l Location) getVolumeName() string { - return strings.TrimPrefix(l.From, VolumePrefix) +func (l Location) getTag(parts ...string) string { + parts = append([]string{"ar"}, parts...) + return strings.Join(parts, ":") } -func (l Location) getPath() (string, error) { - t := l.getType() - switch t { - case TypeLocal: - if path, err := GetPathRelativeToConfig(l.From); err != nil { - return "", err - } else { - return path, nil - } - case TypeVolume: - return "/volume/" + l.name + "/" + l.getVolumeName(), nil - } - return "", fmt.Errorf("could not get path for location \"%s\"", l.name) +func (l Location) getLocationTag() string { + return l.getTag("location", l.name) } func (l Location) Backup(cron bool, specificBackend string) []error { var errors []error var backends []string colors.PrimaryPrint(" Backing up location \"%s\" ", l.name) - t := l.getType() + t, err := l.getType() + if err != nil { + errors = append(errors, err) + return errors + } + cwd, _ := GetPathRelativeToConfig(".") options := ExecuteOptions{ Command: "bash", + Dir: cwd, Envs: map[string]string{ "AUTORESTIC_LOCATION": l.name, }, @@ -138,18 +155,11 @@ func (l Location) Backup(cron bool, specificBackend string) []error { if err := l.validate(); err != nil { errors = append(errors, err) - colors.Error.Print(err) goto after } - if t == TypeLocal { - dir, _ := GetPathRelativeToConfig(l.From) - colors.Faint.Printf("Executing under: \"%s\"\n", dir) - options.Dir = dir - } - // Hooks - if err := ExecuteHooks(l.Hooks.Before, options); err != nil { + if err := l.ExecuteHooks(l.Hooks.Before, options); err != nil { errors = append(errors, err) goto after } @@ -180,25 +190,35 @@ func (l Location) Backup(cron bool, specificBackend string) []error { cmd = append(cmd, lFlags...) cmd = append(cmd, bFlags...) if cron { - cmd = append(cmd, "--tag", TagPrefix+"cron") + cmd = append(cmd, "--tag", l.getTag("cron")) } - cmd = append(cmd, "--tag", TagPrefix+"location:"+l.name) - cmd = append(cmd, ".") + cmd = append(cmd, "--tag", l.getLocationTag()) backupOptions := ExecuteOptions{ - Dir: options.Dir, Envs: env, } var out string - switch t { case TypeLocal: + for _, from := range l.From { + path, err := GetPathRelativeToConfig(from) + if err != nil { + errors = append(errors, err) + goto after + } + cmd = append(cmd, path) + } out, err = ExecuteResticCommand(backupOptions, cmd...) case TypeVolume: + ok := CheckIfVolumeExists(l.From[0]) + if !ok { + errors = append(errors, fmt.Errorf("volume \"%s\" does not exist", l.From[0])) + continue + } + cmd = append(cmd, "/data") out, err = backend.ExecDocker(l, cmd) } if err != nil { - colors.Error.Println(out) errors = append(errors, err) continue } @@ -215,7 +235,7 @@ func (l Location) Backup(cron bool, specificBackend string) []error { } // After hooks - if err := ExecuteHooks(l.Hooks.After, options); err != nil { + if err := l.ExecuteHooks(l.Hooks.After, options); err != nil { errors = append(errors, err) } @@ -226,22 +246,19 @@ after: } else { commands = l.Hooks.Success } - if err := ExecuteHooks(commands, options); err != nil { + if err := l.ExecuteHooks(commands, options); err != nil { errors = append(errors, err) } - colors.Success.Println("Done") + if len(errors) == 0 { + colors.Success.Println("Done") + } return errors } func (l Location) Forget(prune bool, dry bool) error { colors.PrimaryPrint("Forgetting for location \"%s\"", l.name) - path, err := l.getPath() - if err != nil { - return err - } - for _, to := range l.To { backend, _ := GetBackend(to) colors.Secondary.Printf("For backend \"%s\"\n", backend.name) @@ -254,7 +271,7 @@ func (l Location) Forget(prune bool, dry bool) error { } lFlags := getOptions(l.Options, "forget") bFlags := getOptions(backend.Options, "forget") - cmd := []string{"forget", "--path", path} + cmd := []string{"forget", "--tag", l.getLocationTag()} if prune { cmd = append(cmd, "--prune") } @@ -284,7 +301,7 @@ func (l Location) hasBackend(backend string) bool { return false } -func (l Location) Restore(to, from string, force bool) error { +func (l Location) Restore(to, from string, force bool, snapshot string) error { if from == "" { from = l.To[0] } else if !l.hasBackend(from) { @@ -295,16 +312,20 @@ func (l Location) Restore(to, from string, force bool) error { if err != nil { return err } - colors.PrimaryPrint("Restoring location \"%s\"", l.name) - backend, _ := GetBackend(from) - path, err := l.getPath() - if err != nil { - return nil + if snapshot == "" { + snapshot = "latest" } - colors.Secondary.Println("Restoring lastest snapshot") - colors.Body.Printf("%s → %s.\n", from, path) - switch l.getType() { + + colors.PrimaryPrint("Restoring location \"%s\"", l.name) + backend, _ := GetBackend(from) + colors.Secondary.Printf("Restoring %s@%s → %s\n", snapshot, backend.name, to) + + t, err := l.getType() + if err != nil { + return err + } + switch t { case TypeLocal: // Check if target is empty if !force { @@ -324,9 +345,9 @@ func (l Location) Restore(to, from string, force bool) error { } } } - err = backend.Exec([]string{"restore", "--target", to, "--path", path, "latest"}) + err = backend.Exec([]string{"restore", "--target", to, "--tag", l.getLocationTag(), snapshot}) case TypeVolume: - _, err = backend.ExecDocker(l, []string{"restore", "--target", ".", "--path", path, "latest"}) + _, err = backend.ExecDocker(l, []string{"restore", "--target", "/", "--tag", l.getLocationTag(), snapshot}) } if err != nil { return err From 2826f9586d63f002cb891e3cbf17ee6b520c7bba Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:45:03 +0100 Subject: [PATCH 10/13] allow all values from envs --- internal/config.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/config.go b/internal/config.go index 3cf5cca..083f7d2 100644 --- a/internal/config.go +++ b/internal/config.go @@ -253,12 +253,7 @@ func appendOptionsToSlice(str *[]string, options OptionMap) { *str = append(*str, optionToString(key)) continue } - // String - asString, ok := value.(string) - if ok { - *str = append(*str, optionToString(key), asString) - continue - } + *str = append(*str, optionToString(key), fmt.Sprint(value)) } } } From 05c3458a9544ba0c60d0055f397aa4a548f09171 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:45:54 +0100 Subject: [PATCH 11/13] version bump --- internal/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config.go b/internal/config.go index 083f7d2..c51bb79 100644 --- a/internal/config.go +++ b/internal/config.go @@ -16,7 +16,7 @@ import ( "github.com/spf13/viper" ) -const VERSION = "1.4.0" +const VERSION = "1.4.1" var CI bool = false var VERBOSE bool = false From 09cfa4a98e6c7324dfd23d82733300a6b7b83151 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 22:46:37 +0100 Subject: [PATCH 12/13] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9341c44..1029273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.1] - 2021-10-31 + +### Fixes + +- Numeric values from config files not being passed to env. + ## [1.4.0] - 2021-10-30 ### Added From a68e3e426efcf3f9c3b0ff0c41d2b063199c7408 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Sun, 31 Oct 2021 23:07:12 +0100 Subject: [PATCH 13/13] simplify options handling --- internal/config.go | 13 +++++++++++++ internal/location.go | 10 ++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/internal/config.go b/internal/config.go index dce9331..be7118d 100644 --- a/internal/config.go +++ b/internal/config.go @@ -55,6 +55,7 @@ func GetConfig() *Config { config = &Config{} if err := viper.UnmarshalExact(config); err != nil { + colors.Error.Println(err) colors.Error.Println("Could not parse config file!") lock.Unlock() os.Exit(1) @@ -273,3 +274,15 @@ func getOptions(options Options, key string) []string { } return selected } + +func combineOptions(key string, l Location, b Backend) []string { + // Priority: location > backend > global + var options []string + gFlags := getOptions(GetConfig().Global, key) + bFlags := getOptions(b.Options, key) + lFlags := getOptions(l.Options, key) + options = append(options, gFlags...) + options = append(options, bFlags...) + options = append(options, lFlags...) + return options +} diff --git a/internal/location.go b/internal/location.go index 0151802..024252e 100644 --- a/internal/location.go +++ b/internal/location.go @@ -184,11 +184,8 @@ func (l Location) Backup(cron bool, specificBackend string) []error { continue } - lFlags := getOptions(l.Options, "backup") - bFlags := getOptions(backend.Options, "backup") cmd := []string{"backup"} - cmd = append(cmd, lFlags...) - cmd = append(cmd, bFlags...) + cmd = append(cmd, combineOptions("backup", l, backend)...) if cron { cmd = append(cmd, "--tag", l.getTag("cron")) } @@ -269,8 +266,6 @@ func (l Location) Forget(prune bool, dry bool) error { options := ExecuteOptions{ Envs: env, } - lFlags := getOptions(l.Options, "forget") - bFlags := getOptions(backend.Options, "forget") cmd := []string{"forget", "--tag", l.getLocationTag()} if prune { cmd = append(cmd, "--prune") @@ -278,8 +273,7 @@ func (l Location) Forget(prune bool, dry bool) error { if dry { cmd = append(cmd, "--dry-run") } - cmd = append(cmd, lFlags...) - cmd = append(cmd, bFlags...) + cmd = append(cmd, combineOptions("forget", l, backend)...) out, err := ExecuteResticCommand(options, cmd...) if VERBOSE { colors.Faint.Println(out)