diff --git a/src/borg/archive.py b/src/borg/archive.py index ac37288df..c40e419ee 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -54,8 +54,9 @@ has_link = hasattr(os, 'link') class Statistics: - def __init__(self, output_json=False): + def __init__(self, output_json=False, iec=False): self.output_json = output_json + self.iec = iec self.osize = self.csize = self.usize = self.nfiles = 0 self.osize_parts = self.csize_parts = self.usize_parts = self.nfiles_parts = 0 self.last_progress = 0 # timestamp when last progress was shown @@ -75,7 +76,7 @@ class Statistics: def __add__(self, other): if not isinstance(other, Statistics): raise TypeError('can only add Statistics objects') - stats = Statistics(self.output_json) + stats = Statistics(self.output_json, self.iec) stats.osize = self.osize + other.osize stats.csize = self.csize + other.csize stats.usize = self.usize + other.usize @@ -97,23 +98,23 @@ class Statistics: def as_dict(self): return { - 'original_size': FileSize(self.osize), - 'compressed_size': FileSize(self.csize), - 'deduplicated_size': FileSize(self.usize), + 'original_size': FileSize(self.osize, iec=self.iec), + 'compressed_size': FileSize(self.csize, iec=self.iec), + 'deduplicated_size': FileSize(self.usize, iec=self.iec), 'nfiles': self.nfiles, } @property def osize_fmt(self): - return format_file_size(self.osize) + return format_file_size(self.osize, iec=self.iec) @property def usize_fmt(self): - return format_file_size(self.usize) + return format_file_size(self.usize, iec=self.iec) @property def csize_fmt(self): - return format_file_size(self.csize) + return format_file_size(self.csize, iec=self.iec) def show_progress(self, item=None, final=False, stream=None, dt=None): now = time.monotonic() @@ -400,14 +401,15 @@ class Archive: checkpoint_interval=1800, numeric_ids=False, noatime=False, noctime=False, noflags=False, noacls=False, noxattrs=False, progress=False, chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None, - consider_part_files=False, log_json=False): + consider_part_files=False, log_json=False, iec=False): self.cwd = os.getcwd() self.key = key self.repository = repository self.cache = cache self.manifest = manifest self.hard_links = {} - self.stats = Statistics(output_json=log_json) + self.stats = Statistics(output_json=log_json, iec=iec) + self.iec = iec self.show_progress = progress self.name = name # overwritten later with name from archive metadata self.name_in_manifest = name # can differ from .name later (if borg check fixed duplicate archive names) @@ -644,7 +646,7 @@ Utilization of max. archive size: {csize_max:.0%} unique_csize = archive_index.stats_against(cache.chunks)[3] pi.finish() - stats = Statistics() + stats = Statistics(iec=self.iec) stats.usize = unique_csize # the part files use same chunks as the full file if not have_borg12_meta: if self.consider_part_files: @@ -1220,7 +1222,7 @@ class FilesystemObjectProcessors: def __init__(self, *, metadata_collector, cache, key, add_item, process_file_chunks, chunker_params, show_progress, sparse, - log_json): + log_json, iec): self.metadata_collector = metadata_collector self.cache = cache self.key = key @@ -1229,7 +1231,7 @@ class FilesystemObjectProcessors: self.show_progress = show_progress self.hard_links = {} - self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) + self.stats = Statistics(output_json=log_json, iec=iec) # threading: done by cache (including progress) self.cwd = os.getcwd() self.chunker = get_chunker(*chunker_params, seed=key.chunk_seed, sparse=sparse) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 0ed680d72..2f4bd8b83 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -175,7 +175,8 @@ def with_repository(fake=False, invert_fake=False, create=False, lock=True, with Cache(repository, kwargs['key'], kwargs['manifest'], progress=getattr(args, 'progress', False), lock_wait=self.lock_wait, cache_mode=getattr(args, 'files_cache_mode', DEFAULT_FILES_CACHE_MODE), - consider_part_files=getattr(args, 'consider_part_files', False)) as cache_: + consider_part_files=getattr(args, 'consider_part_files', False), + iec=getattr(args, 'iec', False)) as cache_: return method(self, args, repository=repository, cache=cache_, **kwargs) else: return method(self, args, repository=repository, **kwargs) @@ -192,7 +193,7 @@ def with_archive(method): noacls=getattr(args, 'noacls', False), noxattrs=getattr(args, 'noxattrs', False), cache=kwargs.get('cache'), - consider_part_files=args.consider_part_files, log_json=args.log_json) + consider_part_files=args.consider_part_files, log_json=args.log_json, iec=args.iec) return method(self, args, repository=repository, manifest=manifest, key=key, archive=archive, **kwargs) return wrapper @@ -647,13 +648,13 @@ class Archiver: if not dry_run: with Cache(repository, key, manifest, progress=args.progress, lock_wait=self.lock_wait, permit_adhoc_cache=args.no_cache_sync, - cache_mode=args.files_cache_mode) as cache: + cache_mode=args.files_cache_mode, iec=args.iec) as cache: archive = Archive(repository, key, manifest, args.location.archive, cache=cache, create=True, checkpoint_interval=args.checkpoint_interval, numeric_ids=args.numeric_ids, noatime=not args.atime, noctime=args.noctime, progress=args.progress, chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic, - log_json=args.log_json) + log_json=args.log_json, iec=args.iec) metadata_collector = MetadataCollector(noatime=not args.atime, noctime=args.noctime, noflags=args.nobsdflags or args.noflags, noacls=args.noacls, noxattrs=args.noxattrs, numeric_ids=args.numeric_ids, nobirthtime=args.nobirthtime) @@ -663,7 +664,7 @@ class Archiver: fso = FilesystemObjectProcessors(metadata_collector=metadata_collector, cache=cache, key=key, process_file_chunks=cp.process_file_chunks, add_item=archive.add_item, chunker_params=args.chunker_params, show_progress=args.progress, sparse=args.sparse, - log_json=args.log_json) + log_json=args.log_json, iec=args.iec) create_inner(archive, cache, fso) else: create_inner(None, None, None) @@ -1408,7 +1409,7 @@ class Archiver: format = "{archive}{NL}" else: format = "{archive:<36} {time} [{id}]{NL}" - formatter = ArchiveFormatter(format, repository, manifest, key, json=args.json) + formatter = ArchiveFormatter(format, repository, manifest, key, json=args.json, iec=args.iec) output_data = [] @@ -1449,7 +1450,7 @@ class Archiver: for i, archive_name in enumerate(archive_names, 1): archive = Archive(repository, key, manifest, archive_name, cache=cache, - consider_part_files=args.consider_part_files) + consider_part_files=args.consider_part_files, iec=args.iec) info = archive.info() if args.json: output_data.append(info) @@ -2730,6 +2731,8 @@ class Archiver: 'The logger path is borg.debug. if TOPIC is not fully qualified.') add_common_option('-p', '--progress', dest='progress', action='store_true', help='show progress information') + add_common_option('--iec', action='store_true', + help='format using IEC units (1KiB = 1024B)') add_common_option('--log-json', dest='log_json', action='store_true', help='Output one JSON object per log line instead of formatted text.') add_common_option('--lock-wait', metavar='SECONDS', dest='lock_wait', type=int, default=1, diff --git a/src/borg/cache.py b/src/borg/cache.py index 12684f798..0d7db7c8a 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -367,15 +367,15 @@ class Cache: def __new__(cls, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True, progress=False, lock_wait=None, permit_adhoc_cache=False, cache_mode=DEFAULT_FILES_CACHE_MODE, - consider_part_files=False): + consider_part_files=False, iec=False): def local(): return LocalCache(repository=repository, key=key, manifest=manifest, path=path, sync=sync, - warn_if_unencrypted=warn_if_unencrypted, progress=progress, + warn_if_unencrypted=warn_if_unencrypted, progress=progress, iec=iec, lock_wait=lock_wait, cache_mode=cache_mode, consider_part_files=consider_part_files) def adhoc(): - return AdHocCache(repository=repository, key=key, manifest=manifest, lock_wait=lock_wait, + return AdHocCache(repository=repository, key=key, manifest=manifest, lock_wait=lock_wait, iec=iec, consider_part_files=consider_part_files) if not permit_adhoc_cache: @@ -405,6 +405,9 @@ All archives: {0.total_size:>20s} {0.total_csize:>20s} {0.unique_csize:>20s} Unique chunks Total chunks Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" + def __init__(self, iec=False): + self.iec = iec + def __str__(self): return self.str_format.format(self.format_tuple()) @@ -435,7 +438,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" def format_tuple(self): stats = self.stats() for field in ['total_size', 'total_csize', 'unique_csize']: - stats[field] = format_file_size(stats[field]) + stats[field] = format_file_size(stats[field], iec=self.iec) return self.Summary(**stats) def chunks_stored_size(self): @@ -448,13 +451,15 @@ class LocalCache(CacheStatsMixin): """ def __init__(self, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True, - progress=False, lock_wait=None, cache_mode=DEFAULT_FILES_CACHE_MODE, consider_part_files=False): + progress=False, lock_wait=None, cache_mode=DEFAULT_FILES_CACHE_MODE, consider_part_files=False, + iec=False): """ :param warn_if_unencrypted: print warning if accessing unknown unencrypted repository :param lock_wait: timeout for lock acquisition (int [s] or None [wait forever]) :param sync: do :meth:`.sync` :param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison """ + CacheStatsMixin.__init__(self, iec=iec) self.repository = repository self.key = key self.manifest = manifest @@ -1049,7 +1054,9 @@ All archives: unknown unknown unknown Unique chunks Total chunks Chunk index: {0.total_unique_chunks:20d} unknown""" - def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False): + def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False, + iec=False): + CacheStatsMixin.__init__(self, iec=iec) self.repository = repository self.key = key self.manifest = manifest diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index f28ab43de..1c0fe0223 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -222,15 +222,21 @@ def SortBySpec(text): return text.replace('timestamp', 'ts') -def format_file_size(v, precision=2, sign=False): +def format_file_size(v, precision=2, sign=False, iec=False): """Format file size into a human friendly format """ - return sizeof_fmt_decimal(v, suffix='B', sep=' ', precision=precision, sign=sign) + fn = sizeof_fmt_iec if iec else sizeof_fmt_decimal + return fn(v, suffix='B', sep=' ', precision=precision, sign=sign) class FileSize(int): + def __new__(cls, value, iec=False): + obj = int.__new__(cls, value) + obj.iec = iec + return obj + def __format__(self, format_spec): - return format_file_size(int(self)).__format__(format_spec) + return format_file_size(int(self), iec=self.iec).__format__(format_spec) def parse_file_size(s): @@ -593,7 +599,7 @@ class ArchiveFormatter(BaseFormatter): assert not keys, str(keys) return "\n".join(help) - def __init__(self, format, repository, manifest, key, *, json=False): + def __init__(self, format, repository, manifest, key, *, json=False, iec=False): self.repository = repository self.manifest = manifest self.key = key @@ -601,6 +607,7 @@ class ArchiveFormatter(BaseFormatter): self.id = None self._archive = None self.json = json + self.iec = iec static_keys = {} # here could be stuff on repo level, above archive level static_keys.update(self.FIXED_KEYS) self.format = partial_format(format, static_keys) @@ -644,7 +651,7 @@ class ArchiveFormatter(BaseFormatter): """lazy load / update loaded archive""" if self._archive is None or self._archive.id != self.id: from ..archive import Archive - self._archive = Archive(self.repository, self.key, self.manifest, self.name) + self._archive = Archive(self.repository, self.key, self.manifest, self.name, iec=self.iec) return self._archive def get_meta(self, key, rs): diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index ed7ccbf1b..caf50ec79 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -610,6 +610,27 @@ def test_file_size(): assert format_file_size(size) == fmt +def test_file_size_iec(): + """test the size formatting routines""" + iec_size_map = { + 0: '0 B', + 2**0: '1 B', + 2**10: '1.00 KiB', + 2**20: '1.00 MiB', + 2**30: '1.00 GiB', + 2**40: '1.00 TiB', + 2**50: '1.00 PiB', + 2**60: '1.00 EiB', + 2**70: '1.00 ZiB', + 2**80: '1.00 YiB', + -2**0: '-1 B', + -2**10: '-1.00 KiB', + -2**20: '-1.00 MiB', + } + for size, fmt in iec_size_map.items(): + assert format_file_size(size, iec=True) == fmt + + def test_file_size_precision(): assert format_file_size(1234, precision=1) == '1.2 kB' # rounded down assert format_file_size(1254, precision=1) == '1.3 kB' # rounded up