From b52f2aa9a40351a4cb4e40706eecf7e4e7b3e5ed Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 23 Apr 2018 14:34:37 -0600 Subject: [PATCH 1/5] forget: Add policy to keep snapshots before a date --- cmd/restic/cmd_forget.go | 37 ++++--- internal/restic/snapshot_policy.go | 30 +++--- internal/restic/snapshot_policy_test.go | 1 + .../restic/testdata/policy_keep_snapshots_21 | 97 +++++++++++++++++++ 4 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 internal/restic/testdata/policy_keep_snapshots_21 diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 79cc9f449..989fe0975 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -5,6 +5,7 @@ import ( "encoding/json" "sort" "strings" + "time" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -27,13 +28,14 @@ data after 'forget' was run successfully, see the 'prune' command. `, // ForgetOptions collects all options for the forget command. type ForgetOptions struct { - Last int - Hourly int - Daily int - Weekly int - Monthly int - Yearly int - KeepTags restic.TagLists + Last int + Hourly int + Daily int + Weekly int + Monthly int + Yearly int + NewerThan time.Duration + KeepTags restic.TagLists Host string Tags restic.TagLists @@ -58,6 +60,7 @@ func init() { f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots") f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots") f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots") + f.DurationVar(&forgetOptions.NewerThan, "keep-newer-than", 0, "keep snapshots that were created within this timeframe") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") // Sadly the commonly used shortcut `H` is already used. @@ -163,14 +166,20 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { } } + var ageCutoff time.Time + if opts.NewerThan > 0 { + ageCutoff = time.Now().Add(-opts.NewerThan) + } + policy := restic.ExpirePolicy{ - Last: opts.Last, - Hourly: opts.Hourly, - Daily: opts.Daily, - Weekly: opts.Weekly, - Monthly: opts.Monthly, - Yearly: opts.Yearly, - Tags: opts.KeepTags, + Last: opts.Last, + Hourly: opts.Hourly, + Daily: opts.Daily, + Weekly: opts.Weekly, + Monthly: opts.Monthly, + Yearly: opts.Yearly, + NewerThan: ageCutoff, + Tags: opts.KeepTags, } if policy.Empty() && len(args) == 0 { diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 8dd7e5ed6..88533a2cb 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -10,13 +10,14 @@ import ( // ExpirePolicy configures which snapshots should be automatically removed. type ExpirePolicy struct { - Last int // keep the last n snapshots - Hourly int // keep the last n hourly snapshots - Daily int // keep the last n daily snapshots - Weekly int // keep the last n weekly snapshots - Monthly int // keep the last n monthly snapshots - Yearly int // keep the last n yearly snapshots - Tags []TagList // keep all snapshots that include at least one of the tag lists. + Last int // keep the last n snapshots + Hourly int // keep the last n hourly snapshots + Daily int // keep the last n daily snapshots + Weekly int // keep the last n weekly snapshots + Monthly int // keep the last n monthly snapshots + Yearly int // keep the last n yearly snapshots + NewerThan time.Time // keep snapshots newer than this time + Tags []TagList // keep all snapshots that include at least one of the tag lists. } func (e ExpirePolicy) String() (s string) { @@ -39,15 +40,11 @@ func (e ExpirePolicy) String() (s string) { if e.Yearly > 0 { keeps = append(keeps, fmt.Sprintf("%d yearly", e.Yearly)) } - - s = "keep the last " - for _, k := range keeps { - s += k + ", " + if !e.NewerThan.IsZero() { + keeps = append(keeps, fmt.Sprintf("snapshots newer than %s", e.NewerThan)) } - s = strings.Trim(s, ", ") - s += " snapshots" - return s + return fmt.Sprintf("keep the last %s snapshots", strings.Join(keeps, ", ")) } // Sum returns the maximum number of snapshots to be kept according to this @@ -133,6 +130,11 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { } } + // If a timestamp is specified, it's a hard cutoff for older snapshots. + if !p.NewerThan.IsZero() && cur.Time.After(p.NewerThan) { + keepSnap = true + } + // Now update the other buckets and see if they have some counts left. for i, b := range buckets { if b.Count > 0 { diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 69111fab1..b725dcbad 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -171,6 +171,7 @@ var expireTests = []restic.ExpirePolicy{ {Tags: []restic.TagList{{"foo"}}}, {Tags: []restic.TagList{{"foo", "bar"}}}, {Tags: []restic.TagList{{"foo"}, {"bar"}}}, + {NewerThan: parseTimeUTC("2016-01-01 01:00:00")}, } func TestApplyPolicy(t *testing.T) { diff --git a/internal/restic/testdata/policy_keep_snapshots_21 b/internal/restic/testdata/policy_keep_snapshots_21 new file mode 100644 index 000000000..11be139f5 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_21 @@ -0,0 +1,97 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file From 5a0f0e3faa552d374e44b0e6d554c2fbabaf6dc4 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 13 May 2018 11:06:17 +0200 Subject: [PATCH 2/5] Add support for keeping a range of snapshots --- cmd/restic/cmd_forget.go | 40 ++++---- internal/restic/snapshot_policy.go | 47 ++++++--- internal/restic/snapshot_policy_test.go | 14 ++- .../restic/testdata/policy_keep_snapshots_21 | 90 ----------------- .../restic/testdata/policy_keep_snapshots_22 | 7 ++ .../restic/testdata/policy_keep_snapshots_23 | 17 ++++ .../restic/testdata/policy_keep_snapshots_24 | 97 +++++++++++++++++++ 7 files changed, 185 insertions(+), 127 deletions(-) create mode 100644 internal/restic/testdata/policy_keep_snapshots_22 create mode 100644 internal/restic/testdata/policy_keep_snapshots_23 create mode 100644 internal/restic/testdata/policy_keep_snapshots_24 diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 989fe0975..4debaef3b 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -5,7 +5,6 @@ import ( "encoding/json" "sort" "strings" - "time" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -28,14 +27,14 @@ data after 'forget' was run successfully, see the 'prune' command. `, // ForgetOptions collects all options for the forget command. type ForgetOptions struct { - Last int - Hourly int - Daily int - Weekly int - Monthly int - Yearly int - NewerThan time.Duration - KeepTags restic.TagLists + Last int + Hourly int + Daily int + Weekly int + Monthly int + Yearly int + WithinDays int + KeepTags restic.TagLists Host string Tags restic.TagLists @@ -60,7 +59,7 @@ func init() { f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots") f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots") f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots") - f.DurationVar(&forgetOptions.NewerThan, "keep-newer-than", 0, "keep snapshots that were created within this timeframe") + f.IntVar(&forgetOptions.WithinDays, "keep-within", 0, "keep snapshots that were created within `days` before the newest") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") // Sadly the commonly used shortcut `H` is already used. @@ -166,20 +165,15 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { } } - var ageCutoff time.Time - if opts.NewerThan > 0 { - ageCutoff = time.Now().Add(-opts.NewerThan) - } - policy := restic.ExpirePolicy{ - Last: opts.Last, - Hourly: opts.Hourly, - Daily: opts.Daily, - Weekly: opts.Weekly, - Monthly: opts.Monthly, - Yearly: opts.Yearly, - NewerThan: ageCutoff, - Tags: opts.KeepTags, + Last: opts.Last, + Hourly: opts.Hourly, + Daily: opts.Daily, + Weekly: opts.Weekly, + Monthly: opts.Monthly, + Yearly: opts.Yearly, + Within: opts.WithinDays, + Tags: opts.KeepTags, } if policy.Empty() && len(args) == 0 { diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 88533a2cb..244c19140 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -10,14 +10,14 @@ import ( // ExpirePolicy configures which snapshots should be automatically removed. type ExpirePolicy struct { - Last int // keep the last n snapshots - Hourly int // keep the last n hourly snapshots - Daily int // keep the last n daily snapshots - Weekly int // keep the last n weekly snapshots - Monthly int // keep the last n monthly snapshots - Yearly int // keep the last n yearly snapshots - NewerThan time.Time // keep snapshots newer than this time - Tags []TagList // keep all snapshots that include at least one of the tag lists. + Last int // keep the last n snapshots + Hourly int // keep the last n hourly snapshots + Daily int // keep the last n daily snapshots + Weekly int // keep the last n weekly snapshots + Monthly int // keep the last n monthly snapshots + Yearly int // keep the last n yearly snapshots + Within int // keep snapshots made within this number of days since the newest snapshot + Tags []TagList // keep all snapshots that include at least one of the tag lists. } func (e ExpirePolicy) String() (s string) { @@ -40,8 +40,8 @@ func (e ExpirePolicy) String() (s string) { if e.Yearly > 0 { keeps = append(keeps, fmt.Sprintf("%d yearly", e.Yearly)) } - if !e.NewerThan.IsZero() { - keeps = append(keeps, fmt.Sprintf("snapshots newer than %s", e.NewerThan)) + if e.Within != 0 { + keeps = append(keeps, fmt.Sprintf("snapshots within %d days of the newest snapshot", e.Within)) } return fmt.Sprintf("keep the last %s snapshots", strings.Join(keeps, ", ")) @@ -94,6 +94,22 @@ func always(d time.Time, nr int) int { return nr } +// findLatestTimestamp returns the time stamp for the newest snapshot. +func findLatestTimestamp(list Snapshots) time.Time { + if len(list) == 0 { + panic("list of snapshots is empty") + } + + var latest time.Time + for _, sn := range list { + if sn.Time.After(latest) { + latest = sn.Time + } + } + + return latest +} + // ApplyPolicy returns the snapshots from list that are to be kept and removed // according to the policy p. list is sorted in the process. func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { @@ -120,6 +136,8 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { {p.Yearly, y, -1}, } + latest := findLatestTimestamp(list) + for nr, cur := range list { var keepSnap bool @@ -130,9 +148,12 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { } } - // If a timestamp is specified, it's a hard cutoff for older snapshots. - if !p.NewerThan.IsZero() && cur.Time.After(p.NewerThan) { - keepSnap = true + // If the timestamp of the snapshot is within the range, then keep it. + if p.Within != 0 { + t := latest.AddDate(0, 0, -p.Within) + if cur.Time.After(t) { + keepSnap = true + } } // Now update the other buckets and see if they have some counts left. diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index b725dcbad..9aa5200f1 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -21,6 +21,15 @@ func parseTimeUTC(s string) time.Time { return t.UTC() } +func parseDuration(s string) time.Duration { + d, err := time.ParseDuration(s) + if err != nil { + panic(err) + } + + return d +} + func TestExpireSnapshotOps(t *testing.T) { data := []struct { expectEmpty bool @@ -171,7 +180,10 @@ var expireTests = []restic.ExpirePolicy{ {Tags: []restic.TagList{{"foo"}}}, {Tags: []restic.TagList{{"foo", "bar"}}}, {Tags: []restic.TagList{{"foo"}, {"bar"}}}, - {NewerThan: parseTimeUTC("2016-01-01 01:00:00")}, + {Within: 1}, + {Within: 2}, + {Within: 7}, + {Within: 30}, } func TestApplyPolicy(t *testing.T) { diff --git a/internal/restic/testdata/policy_keep_snapshots_21 b/internal/restic/testdata/policy_keep_snapshots_21 index 11be139f5..319c9ab1c 100644 --- a/internal/restic/testdata/policy_keep_snapshots_21 +++ b/internal/restic/testdata/policy_keep_snapshots_21 @@ -3,95 +3,5 @@ "time": "2016-01-18T12:02:03Z", "tree": null, "paths": null - }, - { - "time": "2016-01-12T21:08:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-12T21:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-09T21:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-08T20:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-07T10:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-06T08:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-05T09:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T16:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:30:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:28:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:24:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T11:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T10:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-03T07:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-01T07:08:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-01T01:03:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-01T01:02:03Z", - "tree": null, - "paths": null } ] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_22 b/internal/restic/testdata/policy_keep_snapshots_22 new file mode 100644 index 000000000..319c9ab1c --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_22 @@ -0,0 +1,7 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_23 b/internal/restic/testdata/policy_keep_snapshots_23 new file mode 100644 index 000000000..667fb8b6d --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_23 @@ -0,0 +1,17 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_24 b/internal/restic/testdata/policy_keep_snapshots_24 new file mode 100644 index 000000000..11be139f5 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_24 @@ -0,0 +1,97 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file From cc627e832b251e93d2074ec36835289a11903590 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 13 May 2018 12:02:21 +0200 Subject: [PATCH 3/5] Add custom Duration type --- cmd/restic/cmd_forget.go | 20 ++-- internal/restic/duration.go | 131 ++++++++++++++++++++++++ internal/restic/duration_test.go | 82 +++++++++++++++ internal/restic/snapshot_policy.go | 27 +++-- internal/restic/snapshot_policy_test.go | 12 +-- 5 files changed, 250 insertions(+), 22 deletions(-) create mode 100644 internal/restic/duration.go create mode 100644 internal/restic/duration_test.go diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 4debaef3b..4afef1380 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -27,14 +27,14 @@ data after 'forget' was run successfully, see the 'prune' command. `, // ForgetOptions collects all options for the forget command. type ForgetOptions struct { - Last int - Hourly int - Daily int - Weekly int - Monthly int - Yearly int - WithinDays int - KeepTags restic.TagLists + Last int + Hourly int + Daily int + Weekly int + Monthly int + Yearly int + Within restic.Duration + KeepTags restic.TagLists Host string Tags restic.TagLists @@ -59,7 +59,7 @@ func init() { f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots") f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots") f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots") - f.IntVar(&forgetOptions.WithinDays, "keep-within", 0, "keep snapshots that were created within `days` before the newest") + f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that were created within `duration` before the newest (e.g. 1y5m7d)") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") // Sadly the commonly used shortcut `H` is already used. @@ -172,7 +172,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { Weekly: opts.Weekly, Monthly: opts.Monthly, Yearly: opts.Yearly, - Within: opts.WithinDays, + Within: opts.Within, Tags: opts.KeepTags, } diff --git a/internal/restic/duration.go b/internal/restic/duration.go new file mode 100644 index 000000000..09289849b --- /dev/null +++ b/internal/restic/duration.go @@ -0,0 +1,131 @@ +package restic + +import ( + "fmt" + "strconv" + "strings" + "unicode" + + "github.com/restic/restic/internal/errors" +) + +// Duration is similar to time.Duration, except it only supports larger ranges +// like days, months, and years. +type Duration struct { + Days, Months, Years int +} + +func (d Duration) String() string { + var s string + if d.Years != 0 { + s += fmt.Sprintf("%dy", d.Years) + } + + if d.Months != 0 { + s += fmt.Sprintf("%dm", d.Months) + } + + if d.Days != 0 { + s += fmt.Sprintf("%dd", d.Days) + } + + return s +} + +func nextNumber(input string) (num int, rest string, err error) { + if len(input) == 0 { + return 0, "", nil + } + + var ( + n string + negative bool + ) + + if input[0] == '-' { + negative = true + input = input[1:] + } + + for i, s := range input { + if !unicode.IsNumber(s) { + rest = input[i:] + break + } + + n += string(s) + } + + if len(n) == 0 { + return 0, input, errors.New("no number found") + } + + num, err = strconv.Atoi(n) + if err != nil { + panic(err) + } + + if negative { + num = -num + } + + return num, rest, nil +} + +// ParseDuration parses a duration from a string. The format is: +// 6y5m234d +func ParseDuration(s string) (Duration, error) { + var ( + d Duration + num int + err error + ) + + s = strings.TrimSpace(s) + + for s != "" { + num, s, err = nextNumber(s) + if err != nil { + return Duration{}, err + } + + if len(s) == 0 { + return Duration{}, errors.Errorf("no unit found after number %d", num) + } + + switch s[0] { + case 'y': + d.Years = num + case 'm': + d.Months = num + case 'd': + d.Days = num + } + + s = s[1:] + } + + return d, nil +} + +// Set calls ParseDuration and updates d. +func (d *Duration) Set(s string) error { + v, err := ParseDuration(s) + if err != nil { + return err + } + + *d = v + return nil +} + +// Type returns the type of Duration, usable within github.com/spf13/pflag and +// in help texts. +func (d Duration) Type() string { + return "duration" +} + +// Zero returns true if the duration is empty (all values are set to zero). +func (d Duration) Zero() bool { + return d.Years == 0 && d.Months == 0 && d.Days == 0 +} diff --git a/internal/restic/duration_test.go b/internal/restic/duration_test.go new file mode 100644 index 000000000..0d5306069 --- /dev/null +++ b/internal/restic/duration_test.go @@ -0,0 +1,82 @@ +package restic + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestNextNumber(t *testing.T) { + var tests = []struct { + input string + num int + rest string + err bool + }{ + { + input: "3d", num: 3, rest: "d", + }, + { + input: "7m5d", num: 7, rest: "m5d", + }, + { + input: "-23y7m5d", num: -23, rest: "y7m5d", + }, + { + input: " 5d", num: 0, rest: " 5d", err: true, + }, + { + input: "5d ", num: 5, rest: "d ", + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + num, rest, err := nextNumber(test.input) + + if err != nil && !test.err { + t.Fatal(err) + } + + if num != test.num { + t.Errorf("wrong num, want %d, got %d", test.num, num) + } + + if rest != test.rest { + t.Errorf("wrong rest, want %q, got %q", test.rest, rest) + } + }) + } +} + +func TestParseDuration(t *testing.T) { + var tests = []struct { + input string + d Duration + output string + }{ + {"3d", Duration{Days: 3}, "3d"}, + {"7m5d", Duration{Months: 7, Days: 5}, "7m5d"}, + {"5d7m", Duration{Months: 7, Days: 5}, "7m5d"}, + {"-7m5d", Duration{Months: -7, Days: 5}, "-7m5d"}, + {"2y7m-5d", Duration{Years: 2, Months: 7, Days: -5}, "2y7m-5d"}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + d, err := ParseDuration(test.input) + if err != nil { + t.Fatal(err) + } + + if !cmp.Equal(d, test.d) { + t.Error(cmp.Diff(test.d, d)) + } + + s := d.String() + if s != test.output { + t.Errorf("unexpected return of String(), want %q, got %q", test.output, s) + } + }) + } +} diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 244c19140..df142c0fb 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -16,7 +16,7 @@ type ExpirePolicy struct { Weekly int // keep the last n weekly snapshots Monthly int // keep the last n monthly snapshots Yearly int // keep the last n yearly snapshots - Within int // keep snapshots made within this number of days since the newest snapshot + Within Duration // keep snapshots made within this duration Tags []TagList // keep all snapshots that include at least one of the tag lists. } @@ -40,11 +40,26 @@ func (e ExpirePolicy) String() (s string) { if e.Yearly > 0 { keeps = append(keeps, fmt.Sprintf("%d yearly", e.Yearly)) } - if e.Within != 0 { - keeps = append(keeps, fmt.Sprintf("snapshots within %d days of the newest snapshot", e.Within)) + + if len(keeps) > 0 { + s = fmt.Sprintf("keep the last %s snapshots", strings.Join(keeps, ", ")) } - return fmt.Sprintf("keep the last %s snapshots", strings.Join(keeps, ", ")) + if len(e.Tags) > 0 { + if s != "" { + s += " and " + } + s += fmt.Sprintf("all snapshots with tags %s", e.Tags) + } + + if !e.Within.Zero() { + if s != "" { + s += " and " + } + s += fmt.Sprintf("all snapshots within %s of the newest", e.Within) + } + + return s } // Sum returns the maximum number of snapshots to be kept according to this @@ -149,8 +164,8 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { } // If the timestamp of the snapshot is within the range, then keep it. - if p.Within != 0 { - t := latest.AddDate(0, 0, -p.Within) + if !p.Within.Zero() { + t := latest.AddDate(-p.Within.Years, -p.Within.Months, -p.Within.Days) if cur.Time.After(t) { keepSnap = true } diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 9aa5200f1..f2e0605e3 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -21,8 +21,8 @@ func parseTimeUTC(s string) time.Time { return t.UTC() } -func parseDuration(s string) time.Duration { - d, err := time.ParseDuration(s) +func parseDuration(s string) restic.Duration { + d, err := restic.ParseDuration(s) if err != nil { panic(err) } @@ -180,10 +180,10 @@ var expireTests = []restic.ExpirePolicy{ {Tags: []restic.TagList{{"foo"}}}, {Tags: []restic.TagList{{"foo", "bar"}}}, {Tags: []restic.TagList{{"foo"}, {"bar"}}}, - {Within: 1}, - {Within: 2}, - {Within: 7}, - {Within: 30}, + {Within: parseDuration("1d")}, + {Within: parseDuration("2d")}, + {Within: parseDuration("7d")}, + {Within: parseDuration("1m")}, } func TestApplyPolicy(t *testing.T) { From 060d8b57e0caa6e49c9a842efb9d6da3aff1cd05 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 13 May 2018 12:07:05 +0200 Subject: [PATCH 4/5] Restructure TestApplyPolicy --- internal/restic/snapshot_policy_test.go | 353 +++++++++--------- .../restic/testdata/policy_keep_snapshots_25 | 97 +++++ .../restic/testdata/policy_keep_snapshots_26 | 332 ++++++++++++++++ 3 files changed, 606 insertions(+), 176 deletions(-) create mode 100644 internal/restic/testdata/policy_keep_snapshots_25 create mode 100644 internal/restic/testdata/policy_keep_snapshots_26 diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index f2e0605e3..7ac89e809 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -52,193 +52,194 @@ func TestExpireSnapshotOps(t *testing.T) { } } -var testExpireSnapshots = restic.Snapshots{ - {Time: parseTimeUTC("2014-09-01 10:20:30")}, - {Time: parseTimeUTC("2014-09-02 10:20:30")}, - {Time: parseTimeUTC("2014-09-05 10:20:30")}, - {Time: parseTimeUTC("2014-09-06 10:20:30")}, - {Time: parseTimeUTC("2014-09-08 10:20:30")}, - {Time: parseTimeUTC("2014-09-09 10:20:30")}, - {Time: parseTimeUTC("2014-09-10 10:20:30")}, - {Time: parseTimeUTC("2014-09-11 10:20:30")}, - {Time: parseTimeUTC("2014-09-20 10:20:30")}, - {Time: parseTimeUTC("2014-09-22 10:20:30")}, - {Time: parseTimeUTC("2014-08-08 10:20:30")}, - {Time: parseTimeUTC("2014-08-10 10:20:30")}, - {Time: parseTimeUTC("2014-08-12 10:20:30")}, - {Time: parseTimeUTC("2014-08-13 10:20:30")}, - {Time: parseTimeUTC("2014-08-13 10:20:30.1")}, - {Time: parseTimeUTC("2014-08-15 10:20:30")}, - {Time: parseTimeUTC("2014-08-18 10:20:30")}, - {Time: parseTimeUTC("2014-08-20 10:20:30")}, - {Time: parseTimeUTC("2014-08-21 10:20:30")}, - {Time: parseTimeUTC("2014-08-22 10:20:30")}, - {Time: parseTimeUTC("2014-10-01 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-02 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-05 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-06 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-08 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-09 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-10 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-11 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-20 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-22 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-08 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-10 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-12 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-13 10:20:30.1"), Tags: []string{"bar"}}, - {Time: parseTimeUTC("2014-11-15 10:20:30"), Tags: []string{"foo", "bar"}}, - {Time: parseTimeUTC("2014-11-18 10:20:30")}, - {Time: parseTimeUTC("2014-11-20 10:20:30")}, - {Time: parseTimeUTC("2014-11-21 10:20:30")}, - {Time: parseTimeUTC("2014-11-22 10:20:30")}, - {Time: parseTimeUTC("2015-09-01 10:20:30")}, - {Time: parseTimeUTC("2015-09-02 10:20:30")}, - {Time: parseTimeUTC("2015-09-05 10:20:30")}, - {Time: parseTimeUTC("2015-09-06 10:20:30")}, - {Time: parseTimeUTC("2015-09-08 10:20:30")}, - {Time: parseTimeUTC("2015-09-09 10:20:30")}, - {Time: parseTimeUTC("2015-09-10 10:20:30")}, - {Time: parseTimeUTC("2015-09-11 10:20:30")}, - {Time: parseTimeUTC("2015-09-20 10:20:30")}, - {Time: parseTimeUTC("2015-09-22 10:20:30")}, - {Time: parseTimeUTC("2015-08-08 10:20:30")}, - {Time: parseTimeUTC("2015-08-10 10:20:30")}, - {Time: parseTimeUTC("2015-08-12 10:20:30")}, - {Time: parseTimeUTC("2015-08-13 10:20:30")}, - {Time: parseTimeUTC("2015-08-13 10:20:30.1")}, - {Time: parseTimeUTC("2015-08-15 10:20:30")}, - {Time: parseTimeUTC("2015-08-18 10:20:30")}, - {Time: parseTimeUTC("2015-08-20 10:20:30")}, - {Time: parseTimeUTC("2015-08-21 10:20:30")}, - {Time: parseTimeUTC("2015-08-22 10:20:30")}, - {Time: parseTimeUTC("2015-10-01 10:20:30")}, - {Time: parseTimeUTC("2015-10-02 10:20:30")}, - {Time: parseTimeUTC("2015-10-05 10:20:30")}, - {Time: parseTimeUTC("2015-10-06 10:20:30")}, - {Time: parseTimeUTC("2015-10-08 10:20:30")}, - {Time: parseTimeUTC("2015-10-09 10:20:30")}, - {Time: parseTimeUTC("2015-10-10 10:20:30")}, - {Time: parseTimeUTC("2015-10-11 10:20:30")}, - {Time: parseTimeUTC("2015-10-20 10:20:30")}, - {Time: parseTimeUTC("2015-10-22 10:20:30")}, - {Time: parseTimeUTC("2015-10-22 10:20:30")}, - {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, - {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, - {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}, Paths: []string{"path1", "path2"}}, - {Time: parseTimeUTC("2015-11-08 10:20:30")}, - {Time: parseTimeUTC("2015-11-10 10:20:30")}, - {Time: parseTimeUTC("2015-11-12 10:20:30")}, - {Time: parseTimeUTC("2015-11-13 10:20:30")}, - {Time: parseTimeUTC("2015-11-13 10:20:30.1")}, - {Time: parseTimeUTC("2015-11-15 10:20:30")}, - {Time: parseTimeUTC("2015-11-18 10:20:30")}, - {Time: parseTimeUTC("2015-11-20 10:20:30")}, - {Time: parseTimeUTC("2015-11-21 10:20:30")}, - {Time: parseTimeUTC("2015-11-22 10:20:30")}, - {Time: parseTimeUTC("2016-01-01 01:02:03")}, - {Time: parseTimeUTC("2016-01-01 01:03:03")}, - {Time: parseTimeUTC("2016-01-01 07:08:03")}, - {Time: parseTimeUTC("2016-01-03 07:02:03")}, - {Time: parseTimeUTC("2016-01-04 10:23:03")}, - {Time: parseTimeUTC("2016-01-04 11:23:03")}, - {Time: parseTimeUTC("2016-01-04 12:23:03")}, - {Time: parseTimeUTC("2016-01-04 12:24:03")}, - {Time: parseTimeUTC("2016-01-04 12:28:03")}, - {Time: parseTimeUTC("2016-01-04 12:30:03")}, - {Time: parseTimeUTC("2016-01-04 16:23:03")}, - {Time: parseTimeUTC("2016-01-05 09:02:03")}, - {Time: parseTimeUTC("2016-01-06 08:02:03")}, - {Time: parseTimeUTC("2016-01-07 10:02:03")}, - {Time: parseTimeUTC("2016-01-08 20:02:03")}, - {Time: parseTimeUTC("2016-01-09 21:02:03")}, - {Time: parseTimeUTC("2016-01-12 21:02:03")}, - {Time: parseTimeUTC("2016-01-12 21:08:03")}, - {Time: parseTimeUTC("2016-01-18 12:02:03")}, -} - -var expireTests = []restic.ExpirePolicy{ - {}, - {Last: 10}, - {Last: 15}, - {Last: 99}, - {Last: 200}, - {Hourly: 20}, - {Daily: 3}, - {Daily: 10}, - {Daily: 30}, - {Last: 5, Daily: 5}, - {Last: 2, Daily: 10}, - {Weekly: 2}, - {Weekly: 4}, - {Daily: 3, Weekly: 4}, - {Monthly: 6}, - {Daily: 2, Weekly: 2, Monthly: 6}, - {Yearly: 10}, - {Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}, - {Tags: []restic.TagList{{"foo"}}}, - {Tags: []restic.TagList{{"foo", "bar"}}}, - {Tags: []restic.TagList{{"foo"}, {"bar"}}}, - {Within: parseDuration("1d")}, - {Within: parseDuration("2d")}, - {Within: parseDuration("7d")}, - {Within: parseDuration("1m")}, -} - func TestApplyPolicy(t *testing.T) { - for i, p := range expireTests { - keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) + var testExpireSnapshots = restic.Snapshots{ + {Time: parseTimeUTC("2014-09-01 10:20:30")}, + {Time: parseTimeUTC("2014-09-02 10:20:30")}, + {Time: parseTimeUTC("2014-09-05 10:20:30")}, + {Time: parseTimeUTC("2014-09-06 10:20:30")}, + {Time: parseTimeUTC("2014-09-08 10:20:30")}, + {Time: parseTimeUTC("2014-09-09 10:20:30")}, + {Time: parseTimeUTC("2014-09-10 10:20:30")}, + {Time: parseTimeUTC("2014-09-11 10:20:30")}, + {Time: parseTimeUTC("2014-09-20 10:20:30")}, + {Time: parseTimeUTC("2014-09-22 10:20:30")}, + {Time: parseTimeUTC("2014-08-08 10:20:30")}, + {Time: parseTimeUTC("2014-08-10 10:20:30")}, + {Time: parseTimeUTC("2014-08-12 10:20:30")}, + {Time: parseTimeUTC("2014-08-13 10:20:30")}, + {Time: parseTimeUTC("2014-08-13 10:20:30.1")}, + {Time: parseTimeUTC("2014-08-15 10:20:30")}, + {Time: parseTimeUTC("2014-08-18 10:20:30")}, + {Time: parseTimeUTC("2014-08-20 10:20:30")}, + {Time: parseTimeUTC("2014-08-21 10:20:30")}, + {Time: parseTimeUTC("2014-08-22 10:20:30")}, + {Time: parseTimeUTC("2014-10-01 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-02 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-05 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-06 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-08 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-09 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-10 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-11 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-20 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-22 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-08 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-10 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-12 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-13 10:20:30.1"), Tags: []string{"bar"}}, + {Time: parseTimeUTC("2014-11-15 10:20:30"), Tags: []string{"foo", "bar"}}, + {Time: parseTimeUTC("2014-11-18 10:20:30")}, + {Time: parseTimeUTC("2014-11-20 10:20:30")}, + {Time: parseTimeUTC("2014-11-21 10:20:30")}, + {Time: parseTimeUTC("2014-11-22 10:20:30")}, + {Time: parseTimeUTC("2015-09-01 10:20:30")}, + {Time: parseTimeUTC("2015-09-02 10:20:30")}, + {Time: parseTimeUTC("2015-09-05 10:20:30")}, + {Time: parseTimeUTC("2015-09-06 10:20:30")}, + {Time: parseTimeUTC("2015-09-08 10:20:30")}, + {Time: parseTimeUTC("2015-09-09 10:20:30")}, + {Time: parseTimeUTC("2015-09-10 10:20:30")}, + {Time: parseTimeUTC("2015-09-11 10:20:30")}, + {Time: parseTimeUTC("2015-09-20 10:20:30")}, + {Time: parseTimeUTC("2015-09-22 10:20:30")}, + {Time: parseTimeUTC("2015-08-08 10:20:30")}, + {Time: parseTimeUTC("2015-08-10 10:20:30")}, + {Time: parseTimeUTC("2015-08-12 10:20:30")}, + {Time: parseTimeUTC("2015-08-13 10:20:30")}, + {Time: parseTimeUTC("2015-08-13 10:20:30.1")}, + {Time: parseTimeUTC("2015-08-15 10:20:30")}, + {Time: parseTimeUTC("2015-08-18 10:20:30")}, + {Time: parseTimeUTC("2015-08-20 10:20:30")}, + {Time: parseTimeUTC("2015-08-21 10:20:30")}, + {Time: parseTimeUTC("2015-08-22 10:20:30")}, + {Time: parseTimeUTC("2015-10-01 10:20:30")}, + {Time: parseTimeUTC("2015-10-02 10:20:30")}, + {Time: parseTimeUTC("2015-10-05 10:20:30")}, + {Time: parseTimeUTC("2015-10-06 10:20:30")}, + {Time: parseTimeUTC("2015-10-08 10:20:30")}, + {Time: parseTimeUTC("2015-10-09 10:20:30")}, + {Time: parseTimeUTC("2015-10-10 10:20:30")}, + {Time: parseTimeUTC("2015-10-11 10:20:30")}, + {Time: parseTimeUTC("2015-10-20 10:20:30")}, + {Time: parseTimeUTC("2015-10-22 10:20:30")}, + {Time: parseTimeUTC("2015-10-22 10:20:30")}, + {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, + {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, + {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}, Paths: []string{"path1", "path2"}}, + {Time: parseTimeUTC("2015-11-08 10:20:30")}, + {Time: parseTimeUTC("2015-11-10 10:20:30")}, + {Time: parseTimeUTC("2015-11-12 10:20:30")}, + {Time: parseTimeUTC("2015-11-13 10:20:30")}, + {Time: parseTimeUTC("2015-11-13 10:20:30.1")}, + {Time: parseTimeUTC("2015-11-15 10:20:30")}, + {Time: parseTimeUTC("2015-11-18 10:20:30")}, + {Time: parseTimeUTC("2015-11-20 10:20:30")}, + {Time: parseTimeUTC("2015-11-21 10:20:30")}, + {Time: parseTimeUTC("2015-11-22 10:20:30")}, + {Time: parseTimeUTC("2016-01-01 01:02:03")}, + {Time: parseTimeUTC("2016-01-01 01:03:03")}, + {Time: parseTimeUTC("2016-01-01 07:08:03")}, + {Time: parseTimeUTC("2016-01-03 07:02:03")}, + {Time: parseTimeUTC("2016-01-04 10:23:03")}, + {Time: parseTimeUTC("2016-01-04 11:23:03")}, + {Time: parseTimeUTC("2016-01-04 12:23:03")}, + {Time: parseTimeUTC("2016-01-04 12:24:03")}, + {Time: parseTimeUTC("2016-01-04 12:28:03")}, + {Time: parseTimeUTC("2016-01-04 12:30:03")}, + {Time: parseTimeUTC("2016-01-04 16:23:03")}, + {Time: parseTimeUTC("2016-01-05 09:02:03")}, + {Time: parseTimeUTC("2016-01-06 08:02:03")}, + {Time: parseTimeUTC("2016-01-07 10:02:03")}, + {Time: parseTimeUTC("2016-01-08 20:02:03")}, + {Time: parseTimeUTC("2016-01-09 21:02:03")}, + {Time: parseTimeUTC("2016-01-12 21:02:03")}, + {Time: parseTimeUTC("2016-01-12 21:08:03")}, + {Time: parseTimeUTC("2016-01-18 12:02:03")}, + } - t.Logf("test %d: returned keep %v, remove %v (of %v) expired snapshots for policy %v", - i, len(keep), len(remove), len(testExpireSnapshots), p) + var tests = []restic.ExpirePolicy{ + {}, + {Last: 10}, + {Last: 15}, + {Last: 99}, + {Last: 200}, + {Hourly: 20}, + {Daily: 3}, + {Daily: 10}, + {Daily: 30}, + {Last: 5, Daily: 5}, + {Last: 2, Daily: 10}, + {Weekly: 2}, + {Weekly: 4}, + {Daily: 3, Weekly: 4}, + {Monthly: 6}, + {Daily: 2, Weekly: 2, Monthly: 6}, + {Yearly: 10}, + {Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}, + {Tags: []restic.TagList{{"foo"}}}, + {Tags: []restic.TagList{{"foo", "bar"}}}, + {Tags: []restic.TagList{{"foo"}, {"bar"}}}, + {Within: parseDuration("1d")}, + {Within: parseDuration("2d")}, + {Within: parseDuration("7d")}, + {Within: parseDuration("1m")}, + {Within: parseDuration("1m14d")}, + {Within: parseDuration("1y1d1m")}, + } - if len(keep)+len(remove) != len(testExpireSnapshots) { - t.Errorf("test %d: len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", - i, len(keep)+len(remove), len(testExpireSnapshots)) - } + for i, p := range tests { + t.Run("", func(t *testing.T) { + keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) - if p.Sum() > 0 && len(keep) > p.Sum() { - t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", - p.Sum(), len(keep)) - } + t.Logf("returned keep %v, remove %v (of %v) expired snapshots for policy %v", + len(keep), len(remove), len(testExpireSnapshots), p) - for _, sn := range keep { - t.Logf("test %d: keep snapshot at %v %s\n", i, sn.Time, sn.Tags) - } - for _, sn := range remove { - t.Logf("test %d: forget snapshot at %v %s\n", i, sn.Time, sn.Tags) - } + if len(keep)+len(remove) != len(testExpireSnapshots) { + t.Errorf("len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", + len(keep)+len(remove), len(testExpireSnapshots)) + } - goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) + if p.Sum() > 0 && len(keep) > p.Sum() { + t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", + p.Sum(), len(keep)) + } - if *updateGoldenFiles { - buf, err := json.MarshalIndent(keep, "", " ") + for _, sn := range keep { + t.Logf(" keep snapshot at %v %s", sn.Time, sn.Tags) + } + for _, sn := range remove { + t.Logf(" forget snapshot at %v %s", sn.Time, sn.Tags) + } + + goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) + + if *updateGoldenFiles { + buf, err := json.MarshalIndent(keep, "", " ") + if err != nil { + t.Fatalf("error marshaling result: %v", err) + } + + if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { + t.Fatalf("unable to update golden file: %v", err) + } + } + + buf, err := ioutil.ReadFile(goldenFilename) if err != nil { - t.Fatalf("error marshaling result: %v", err) + t.Fatalf("error loading golden file %v: %v", goldenFilename, err) } - if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { - t.Fatalf("unable to update golden file: %v", err) + var want restic.Snapshots + err = json.Unmarshal(buf, &want) + if err != nil { + t.Fatalf("error unmarshalling golden file %v: %v", goldenFilename, err) } - } - buf, err := ioutil.ReadFile(goldenFilename) - if err != nil { - t.Errorf("error loading golden file %v: %v", goldenFilename, err) - continue - } - - var want restic.Snapshots - err = json.Unmarshal(buf, &want) - if err != nil { - t.Errorf("error unmarshalling golden file %v: %v", goldenFilename, err) - continue - } - - if !reflect.DeepEqual(keep, want) { - t.Errorf("test %v: wrong result, want:\n %v\ngot:\n %v", i, want, keep) - continue - } + if !reflect.DeepEqual(keep, want) { + t.Fatalf("wrong result, want:\n %v\ngot:\n %v", want, keep) + } + }) } } diff --git a/internal/restic/testdata/policy_keep_snapshots_25 b/internal/restic/testdata/policy_keep_snapshots_25 new file mode 100644 index 000000000..11be139f5 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_25 @@ -0,0 +1,97 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_26 b/internal/restic/testdata/policy_keep_snapshots_26 new file mode 100644 index 000000000..f36ec66b7 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_26 @@ -0,0 +1,332 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file From 375868edcf60351ba3c8c3617a2f4640162b7eab Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 13 May 2018 12:54:23 +0200 Subject: [PATCH 5/5] Add documentation --- changelog/unreleased/pull-1735 | 9 +++++++++ doc/060_forget.rst | 4 ++++ 2 files changed, 13 insertions(+) create mode 100644 changelog/unreleased/pull-1735 diff --git a/changelog/unreleased/pull-1735 b/changelog/unreleased/pull-1735 new file mode 100644 index 000000000..2cfd115d8 --- /dev/null +++ b/changelog/unreleased/pull-1735 @@ -0,0 +1,9 @@ +Enhancement: Allow keeping a time range of snaphots + +We've added the `--keep-within` option to the `forget` command. It instructs +restic to keep all snapshots within the given duration since the newest +snapshot. For example, running `restic forget --keep-within 5m7d` will keep all +snapshots which have been made in the five months and seven days since the +latest snapshot. + +https://github.com/restic/restic/pull/1735 diff --git a/doc/060_forget.rst b/doc/060_forget.rst index ab5274758..1ca614a79 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -159,6 +159,10 @@ The ``forget`` command accepts the following parameters: snapshots, only keep the last one for that year. - ``--keep-tag`` keep all snapshots which have all tags specified by this option (can be specified multiple times). +- ``--keep-within duration`` keep all snapshots which have been made within + the duration of the latest snapshot. ``duration`` needs to be a number of + years, months, and days, e.g. ``2y5m7d`` will keep all snapshots made in the + two years, five months, and seven days before the latest snapshot. Additionally, you can restrict removing snapshots to those which have a particular hostname with the ``--hostname`` parameter, or tags with the