mirror of https://github.com/borgbackup/borg.git
Merge pull request #6968 from ThomasWaldmann/cleanup-prune-master
move prune related code to borg.archiver.prune
This commit is contained in:
commit
900398f927
|
@ -1,5 +1,8 @@
|
|||
import argparse
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timezone, timedelta
|
||||
import logging
|
||||
from operator import attrgetter
|
||||
import re
|
||||
|
||||
from .common import with_repository
|
||||
|
@ -7,7 +10,7 @@ from ..archive import Archive, Statistics
|
|||
from ..cache import Cache
|
||||
from ..constants import * # NOQA
|
||||
from ..helpers import format_archive
|
||||
from ..helpers import interval, prune_within, prune_split, PRUNING_PATTERNS
|
||||
from ..helpers import interval
|
||||
from ..helpers import Manifest, sig_int
|
||||
from ..helpers import log_multi
|
||||
from ..helpers import ProgressIndicatorPercent
|
||||
|
@ -17,6 +20,58 @@ from ..logger import create_logger
|
|||
logger = create_logger()
|
||||
|
||||
|
||||
def prune_within(archives, hours, kept_because):
|
||||
target = datetime.now(timezone.utc) - timedelta(seconds=hours * 3600)
|
||||
kept_counter = 0
|
||||
result = []
|
||||
for a in archives:
|
||||
if a.ts > target:
|
||||
kept_counter += 1
|
||||
kept_because[a.id] = ("within", kept_counter)
|
||||
result.append(a)
|
||||
return result
|
||||
|
||||
|
||||
PRUNING_PATTERNS = OrderedDict(
|
||||
[
|
||||
("secondly", "%Y-%m-%d %H:%M:%S"),
|
||||
("minutely", "%Y-%m-%d %H:%M"),
|
||||
("hourly", "%Y-%m-%d %H"),
|
||||
("daily", "%Y-%m-%d"),
|
||||
("weekly", "%G-%V"),
|
||||
("monthly", "%Y-%m"),
|
||||
("yearly", "%Y"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def prune_split(archives, rule, n, kept_because=None):
|
||||
last = None
|
||||
keep = []
|
||||
pattern = PRUNING_PATTERNS[rule]
|
||||
if kept_because is None:
|
||||
kept_because = {}
|
||||
if n == 0:
|
||||
return keep
|
||||
|
||||
a = None
|
||||
for a in sorted(archives, key=attrgetter("ts"), reverse=True):
|
||||
# we compute the pruning in local time zone
|
||||
period = a.ts.astimezone().strftime(pattern)
|
||||
if period != last:
|
||||
last = period
|
||||
if a.id not in kept_because:
|
||||
keep.append(a)
|
||||
kept_because[a.id] = (rule, len(keep))
|
||||
if len(keep) == n:
|
||||
break
|
||||
# Keep oldest archive if we didn't reach the target retention count
|
||||
if a is not None and len(keep) < n and a.id not in kept_because:
|
||||
keep.append(a)
|
||||
kept_because[a.id] = (rule + "[oldest]", len(keep))
|
||||
return keep
|
||||
|
||||
|
||||
class PruneMixIn:
|
||||
@with_repository(exclusive=True, compatibility=(Manifest.Operation.DELETE,))
|
||||
def do_prune(self, args, repository, manifest, key):
|
||||
|
|
|
@ -17,7 +17,7 @@ from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, umount
|
|||
from .fs import O_, flags_root, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime
|
||||
from .fs import HardLinkManager
|
||||
from .manifest import Manifest, NoManifestError, MandatoryFeatureUnsupported, AI_HUMAN_SORT_KEYS
|
||||
from .misc import prune_within, prune_split, PRUNING_PATTERNS, sysinfo, log_multi, consume, get_tar_filter
|
||||
from .misc import sysinfo, log_multi, consume, get_tar_filter
|
||||
from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
|
||||
from .parseformat import bin_to_hex, safe_encode, safe_decode
|
||||
from .parseformat import remove_surrogates, eval_escapes, decode_dict, positive_int_validator, interval
|
||||
|
|
|
@ -4,10 +4,8 @@ import os
|
|||
import os.path
|
||||
import platform
|
||||
import sys
|
||||
from collections import deque, OrderedDict
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from collections import deque
|
||||
from itertools import islice
|
||||
from operator import attrgetter
|
||||
|
||||
from ..logger import create_logger
|
||||
|
||||
|
@ -18,58 +16,6 @@ from .. import __version__ as borg_version
|
|||
from .. import chunker
|
||||
|
||||
|
||||
def prune_within(archives, hours, kept_because):
|
||||
target = datetime.now(timezone.utc) - timedelta(seconds=hours * 3600)
|
||||
kept_counter = 0
|
||||
result = []
|
||||
for a in archives:
|
||||
if a.ts > target:
|
||||
kept_counter += 1
|
||||
kept_because[a.id] = ("within", kept_counter)
|
||||
result.append(a)
|
||||
return result
|
||||
|
||||
|
||||
PRUNING_PATTERNS = OrderedDict(
|
||||
[
|
||||
("secondly", "%Y-%m-%d %H:%M:%S"),
|
||||
("minutely", "%Y-%m-%d %H:%M"),
|
||||
("hourly", "%Y-%m-%d %H"),
|
||||
("daily", "%Y-%m-%d"),
|
||||
("weekly", "%G-%V"),
|
||||
("monthly", "%Y-%m"),
|
||||
("yearly", "%Y"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def prune_split(archives, rule, n, kept_because=None):
|
||||
last = None
|
||||
keep = []
|
||||
pattern = PRUNING_PATTERNS[rule]
|
||||
if kept_because is None:
|
||||
kept_because = {}
|
||||
if n == 0:
|
||||
return keep
|
||||
|
||||
a = None
|
||||
for a in sorted(archives, key=attrgetter("ts"), reverse=True):
|
||||
# we compute the pruning in local time zone
|
||||
period = a.ts.astimezone().strftime(pattern)
|
||||
if period != last:
|
||||
last = period
|
||||
if a.id not in kept_because:
|
||||
keep.append(a)
|
||||
kept_because[a.id] = (rule, len(keep))
|
||||
if len(keep) == n:
|
||||
break
|
||||
# Keep oldest archive if we didn't reach the target retention count
|
||||
if a is not None and len(keep) < n and a.id not in kept_because:
|
||||
keep.append(a)
|
||||
kept_because[a.id] = (rule + "[oldest]", len(keep))
|
||||
return keep
|
||||
|
||||
|
||||
def sysinfo():
|
||||
show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower()
|
||||
if show_sysinfo == "no":
|
||||
|
|
|
@ -10,6 +10,7 @@ from io import StringIO, BytesIO
|
|||
|
||||
import pytest
|
||||
|
||||
from ..archiver.prune import prune_within, prune_split
|
||||
from .. import platform
|
||||
from ..constants import MAX_DATA_SIZE
|
||||
from ..helpers import Location
|
||||
|
@ -24,7 +25,7 @@ from ..helpers import (
|
|||
replace_placeholders,
|
||||
)
|
||||
from ..helpers import make_path_safe, clean_lines
|
||||
from ..helpers import interval, prune_within, prune_split
|
||||
from ..helpers import interval
|
||||
from ..helpers import get_base_dir, get_cache_dir, get_keys_dir, get_security_dir, get_config_dir
|
||||
from ..helpers import is_slow_msgpack
|
||||
from ..helpers import msgpack
|
||||
|
|
Loading…
Reference in New Issue