diff --git a/docs/misc/prune-example.txt b/docs/misc/prune-example.txt index 12ffeb6fd..bc6bb209a 100644 --- a/docs/misc/prune-example.txt +++ b/docs/misc/prune-example.txt @@ -1,14 +1,18 @@ borg prune visualized ===================== -Assume it is 2016-01-01, today's backup has not yet been made and you have +Assume it is 2016-01-01, today's backup has not yet been made, you have created at least one backup on each day in 2015 except on 2015-12-19 (no -backup made on that day). +backup made on that day), and you started backing up with borg on +2015-01-01. -This is what borg prune --keep-daily 14 --keep-monthly 6 would keep. +This is what borg prune --keep-daily 14 --keep-monthly 6 --keep-yearly 1 +would keep. Backups kept by the --keep-daily rule are marked by a "d" to the right, -backups kept by the --keep-monthly rule are marked by a "m" to the right. +backups kept by the --keep-monthly rule are marked by a "m" to the right, +and backups kept by the --keep-yearly rule are marked by a "y" to the +right. Calendar view ------------- @@ -16,7 +20,7 @@ Calendar view 2015 January February March Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 3 4 1 1 + 1y 2 3 4 1 1 5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8 12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15 19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22 @@ -53,16 +57,16 @@ Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su List view --------- ---keep-daily 14 --keep-monthly 6 -------------------------------------------------- - 1. 2015-12-31 (2015-12-31 kept by daily rule) - 2. 2015-12-30 1. 2015-11-30 - 3. 2015-12-29 2. 2015-10-31 - 4. 2015-12-28 3. 2015-09-30 - 5. 2015-12-27 4. 2015-08-31 - 6. 2015-12-26 5. 2015-07-31 - 7. 2015-12-25 6. 2015-06-30 - 8. 2015-12-24 +--keep-daily 14 --keep-monthly 6 --keep-yearly 1 +---------------------------------------------------------------- + 1. 2015-12-31 (2015-12-31 kept (2015-12-31 kept + 2. 2015-12-30 by daily rule) by daily rule) + 3. 2015-12-29 1. 2015-11-30 1. 2015-01-01 (oldest) + 4. 2015-12-28 2. 2015-10-31 + 5. 2015-12-27 3. 2015-09-30 + 6. 2015-12-26 4. 2015-08-31 + 7. 2015-12-25 5. 2015-07-31 + 8. 2015-12-24 6. 2015-06-30 9. 2015-12-23 10. 2015-12-22 11. 2015-12-21 @@ -76,18 +80,23 @@ Notes ----- 2015-12-31 is kept due to the --keep-daily 14 rule (because it is applied -first), not due to the --keep-monthly rule. +first), not due to the --keep-monthly or --keep-yearly rule. -Because of that, the --keep-monthly 6 rule keeps Nov, Oct, Sep, Aug, Jul and -Jun. December is not considered for this rule, because that backup was already -kept because of the daily rule. +The --keep-yearly 1 rule does not consider the December 31st backup because it +has already been kept due to the daily rule. There are no backups available +from previous years, so the --keep-yearly target of 1 backup is not satisfied. +Because of this, the 2015-01-01 archive (the oldest archive available) is kept. + +The --keep-monthly 6 rule keeps Nov, Oct, Sep, Aug, Jul and Jun. December is +not considered for this rule, because that backup was already kept because of +the daily rule. 2015-12-17 is kept to satisfy the --keep-daily 14 rule - because no backup was made on 2015-12-19. If a backup had been made on that day, it would not keep the one from 2015-12-17. -We did not include yearly, weekly, hourly, minutely or secondly rules to keep -this example simple. They all work in basically the same way. +We did not include weekly, hourly, minutely or secondly rules to keep this +example simple. They all work in basically the same way. The weekly rule is easy to understand roughly, but hard to understand in all details. If interested, read "ISO 8601:2000 standard week-based year". diff --git a/requirements.d/development.txt b/requirements.d/development.txt index a8f90b878..ec9098f23 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -10,3 +10,4 @@ pytest-cov pytest-benchmark Cython!=0.27 twine +python-dateutil diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 3c7d45e19..3cfead31a 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -4164,7 +4164,11 @@ def define_borg_mount(parser): rules do not count towards those of later rules. The time that each backup starts is used for pruning purposes. Dates and times are interpreted in the local timezone, and weeks go from Monday to Sunday. Specifying a - negative number of archives to keep means that there is no limit. + negative number of archives to keep means that there is no limit. As of borg + 1.2.0, borg will retain the oldest archive if any of the secondly, minutely, + hourly, daily, weekly, monthly, or yearly rules was not otherwise able to meet + its retention target. This enables the first chronological archive to continue + aging until it is replaced by a newer archive that meets the retention criteria. The ``--keep-last N`` option is doing the same as ``--keep-secondly N`` (and it will keep the last N archives under the assumption that you do not create more than one diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 4b41ec042..763ba9103 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1,4 +1,5 @@ import argparse +import dateutil.tz import errno import io import json @@ -2059,6 +2060,101 @@ def test_prune_repository(self): # the latest archive must be still there self.assert_in('test5', output) + # Given a date and time in local tz, create a UTC timestamp string suitable + # for create --timestamp command line option + def _to_utc_timestamp(self, year, month, day, hour, minute, second): + dtime = datetime(year, month, day, hour, minute, second, 0, dateutil.tz.gettz()) + return dtime.astimezone(dateutil.tz.UTC).strftime("%Y-%m-%dT%H:%M:%S") + + def _create_archive_ts(self, name, y, m, d, H=0, M=0, S=0): + loc = self.repository_location + '::' + name + self.cmd('create', '--timestamp', self._to_utc_timestamp(y, m, d, H, M, S), loc, src_dir) + + # This test must match docs/misc/prune-example.txt + def test_prune_repository_example(self): + self.cmd('init', '--encryption=repokey', self.repository_location) + # Archives that will be kept, per the example + # Oldest archive + self._create_archive_ts('test01', 2015, 1, 1) + # 6 monthly archives + self._create_archive_ts('test02', 2015, 6, 30) + self._create_archive_ts('test03', 2015, 7, 31) + self._create_archive_ts('test04', 2015, 8, 31) + self._create_archive_ts('test05', 2015, 9, 30) + self._create_archive_ts('test06', 2015, 10, 31) + self._create_archive_ts('test07', 2015, 11, 30) + # 14 daily archives + self._create_archive_ts('test08', 2015, 12, 17) + self._create_archive_ts('test09', 2015, 12, 18) + self._create_archive_ts('test10', 2015, 12, 20) + self._create_archive_ts('test11', 2015, 12, 21) + self._create_archive_ts('test12', 2015, 12, 22) + self._create_archive_ts('test13', 2015, 12, 23) + self._create_archive_ts('test14', 2015, 12, 24) + self._create_archive_ts('test15', 2015, 12, 25) + self._create_archive_ts('test16', 2015, 12, 26) + self._create_archive_ts('test17', 2015, 12, 27) + self._create_archive_ts('test18', 2015, 12, 28) + self._create_archive_ts('test19', 2015, 12, 29) + self._create_archive_ts('test20', 2015, 12, 30) + self._create_archive_ts('test21', 2015, 12, 31) + # Additional archives that would be pruned + # The second backup of the year + self._create_archive_ts('test22', 2015, 1, 2) + # The next older monthly backup + self._create_archive_ts('test23', 2015, 5, 31) + # The next older daily backup + self._create_archive_ts('test24', 2015, 12, 16) + output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=14', '--keep-monthly=6', '--keep-yearly=1') + # Prune second backup of the year + assert re.search(r'Would prune:\s+test22', output) + # Prune next older monthly and daily backups + assert re.search(r'Would prune:\s+test23', output) + assert re.search(r'Would prune:\s+test24', output) + # Must keep the other 21 backups + # Yearly is kept as oldest archive + assert re.search(r'Keeping archive \(rule: yearly\[oldest\] #1\):\s+test01', output) + for i in range(1, 7): + assert re.search(r'Keeping archive \(rule: monthly #' + str(i) + r'\):\s+test' + ("%02d" % (8-i)), output) + for i in range(1, 15): + assert re.search(r'Keeping archive \(rule: daily #' + str(i) + r'\):\s+test' + ("%02d" % (22-i)), output) + output = self.cmd('list', self.repository_location) + # Nothing pruned after dry run + for i in range(1, 25): + self.assert_in('test%02d' % i, output) + self.cmd('prune', self.repository_location, '--keep-daily=14', '--keep-monthly=6', '--keep-yearly=1') + output = self.cmd('list', self.repository_location) + # All matching backups plus oldest kept + for i in range(1, 22): + self.assert_in('test%02d' % i, output) + # Other backups have been pruned + for i in range(22, 25): + self.assert_not_in('test%02d' % i, output) + + # With an initial and daily backup, prune daily until oldest is replaced by a monthly backup + def test_prune_retain_and_expire_oldest(self): + self.cmd('init', '--encryption=repokey', self.repository_location) + # Initial backup + self._create_archive_ts('original_archive', 2020, 9, 1, 11, 15) + # Archive and prune daily for 30 days + for i in range(1, 31): + self._create_archive_ts('september%02d' % i, 2020, 9, i, 12) + self.cmd('prune', self.repository_location, '--keep-daily=7', '--keep-monthly=1') + # Archive and prune 6 days into the next month + for i in range(1, 7): + self._create_archive_ts('october%02d' % i, 2020, 10, i, 12) + self.cmd('prune', self.repository_location, '--keep-daily=7', '--keep-monthly=1') + # Oldest backup is still retained + output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=7', '--keep-monthly=1') + assert re.search(r'Keeping archive \(rule: monthly\[oldest\] #1' + r'\):\s+original_archive', output) + # Archive one more day and prune. + self._create_archive_ts('october07', 2020, 10, 7, 12) + self.cmd('prune', self.repository_location, '--keep-daily=7', '--keep-monthly=1') + # Last day of previous month is retained as monthly, and oldest is expired. + output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=7', '--keep-monthly=1') + assert re.search(r'Keeping archive \(rule: monthly #1\):\s+september30', output) + self.assert_not_in('original_archive', output) + def test_prune_repository_save_space(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.cmd('create', self.repository_location + '::test1', src_dir)