Merge pull request #6968 from ThomasWaldmann/cleanup-prune-master

move prune related code to borg.archiver.prune

src/borg/archiver/prune.py

@@ -1,5 +1,8 @@
 import argparse
+from collections import OrderedDict
+from datetime import datetime, timezone, timedelta
 import logging
+from operator import attrgetter
 import re
 
 from .common import with_repository
@@ -7,7 +10,7 @@ from ..archive import Archive, Statistics
 from ..cache import Cache
 from ..constants import *  # NOQA
 from ..helpers import format_archive
-from ..helpers import interval, prune_within, prune_split, PRUNING_PATTERNS
+from ..helpers import interval
 from ..helpers import Manifest, sig_int
 from ..helpers import log_multi
 from ..helpers import ProgressIndicatorPercent
@@ -17,6 +20,58 @@ from ..logger import create_logger
 logger = create_logger()
 
 
+def prune_within(archives, hours, kept_because):
+    target = datetime.now(timezone.utc) - timedelta(seconds=hours * 3600)
+    kept_counter = 0
+    result = []
+    for a in archives:
+        if a.ts > target:
+            kept_counter += 1
+            kept_because[a.id] = ("within", kept_counter)
+            result.append(a)
+    return result
+
+
+PRUNING_PATTERNS = OrderedDict(
+    [
+        ("secondly", "%Y-%m-%d %H:%M:%S"),
+        ("minutely", "%Y-%m-%d %H:%M"),
+        ("hourly", "%Y-%m-%d %H"),
+        ("daily", "%Y-%m-%d"),
+        ("weekly", "%G-%V"),
+        ("monthly", "%Y-%m"),
+        ("yearly", "%Y"),
+    ]
+)
+
+
+def prune_split(archives, rule, n, kept_because=None):
+    last = None
+    keep = []
+    pattern = PRUNING_PATTERNS[rule]
+    if kept_because is None:
+        kept_because = {}
+    if n == 0:
+        return keep
+
+    a = None
+    for a in sorted(archives, key=attrgetter("ts"), reverse=True):
+        # we compute the pruning in local time zone
+        period = a.ts.astimezone().strftime(pattern)
+        if period != last:
+            last = period
+            if a.id not in kept_because:
+                keep.append(a)
+                kept_because[a.id] = (rule, len(keep))
+                if len(keep) == n:
+                    break
+    # Keep oldest archive if we didn't reach the target retention count
+    if a is not None and len(keep) < n and a.id not in kept_because:
+        keep.append(a)
+        kept_because[a.id] = (rule + "[oldest]", len(keep))
+    return keep
+
+
 class PruneMixIn:
     @with_repository(exclusive=True, compatibility=(Manifest.Operation.DELETE,))
     def do_prune(self, args, repository, manifest, key):

src/borg/helpers/__init__.py

@@ -17,7 +17,7 @@ from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, umount
 from .fs import O_, flags_root, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime
 from .fs import HardLinkManager
 from .manifest import Manifest, NoManifestError, MandatoryFeatureUnsupported, AI_HUMAN_SORT_KEYS
-from .misc import prune_within, prune_split, PRUNING_PATTERNS, sysinfo, log_multi, consume, get_tar_filter
+from .misc import sysinfo, log_multi, consume, get_tar_filter
 from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
 from .parseformat import bin_to_hex, safe_encode, safe_decode
 from .parseformat import remove_surrogates, eval_escapes, decode_dict, positive_int_validator, interval

src/borg/helpers/misc.py

@@ -4,10 +4,8 @@ import os
 import os.path
 import platform
 import sys
-from collections import deque, OrderedDict
+from collections import deque
-from datetime import datetime, timezone, timedelta
 from itertools import islice
-from operator import attrgetter
 
 from ..logger import create_logger
 
@@ -18,58 +16,6 @@ from .. import __version__ as borg_version
 from .. import chunker
 
 
-def prune_within(archives, hours, kept_because):
-    target = datetime.now(timezone.utc) - timedelta(seconds=hours * 3600)
-    kept_counter = 0
-    result = []
-    for a in archives:
-        if a.ts > target:
-            kept_counter += 1
-            kept_because[a.id] = ("within", kept_counter)
-            result.append(a)
-    return result
-
-
-PRUNING_PATTERNS = OrderedDict(
-    [
-        ("secondly", "%Y-%m-%d %H:%M:%S"),
-        ("minutely", "%Y-%m-%d %H:%M"),
-        ("hourly", "%Y-%m-%d %H"),
-        ("daily", "%Y-%m-%d"),
-        ("weekly", "%G-%V"),
-        ("monthly", "%Y-%m"),
-        ("yearly", "%Y"),
-    ]
-)
-
-
-def prune_split(archives, rule, n, kept_because=None):
-    last = None
-    keep = []
-    pattern = PRUNING_PATTERNS[rule]
-    if kept_because is None:
-        kept_because = {}
-    if n == 0:
-        return keep
-
-    a = None
-    for a in sorted(archives, key=attrgetter("ts"), reverse=True):
-        # we compute the pruning in local time zone
-        period = a.ts.astimezone().strftime(pattern)
-        if period != last:
-            last = period
-            if a.id not in kept_because:
-                keep.append(a)
-                kept_because[a.id] = (rule, len(keep))
-                if len(keep) == n:
-                    break
-    # Keep oldest archive if we didn't reach the target retention count
-    if a is not None and len(keep) < n and a.id not in kept_because:
-        keep.append(a)
-        kept_because[a.id] = (rule + "[oldest]", len(keep))
-    return keep
-
-
 def sysinfo():
     show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower()
     if show_sysinfo == "no":

src/borg/testsuite/helpers.py

@@ -10,6 +10,7 @@ from io import StringIO, BytesIO
 
 import pytest
 
+from ..archiver.prune import prune_within, prune_split
 from .. import platform
 from ..constants import MAX_DATA_SIZE
 from ..helpers import Location
@@ -24,7 +25,7 @@ from ..helpers import (
     replace_placeholders,
 )
 from ..helpers import make_path_safe, clean_lines
-from ..helpers import interval, prune_within, prune_split
+from ..helpers import interval
 from ..helpers import get_base_dir, get_cache_dir, get_keys_dir, get_security_dir, get_config_dir
 from ..helpers import is_slow_msgpack
 from ..helpers import msgpack