From 29bc3965593dbcb62fa1725776da4a651652f9b0 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 6 Apr 2016 23:17:06 +0200 Subject: [PATCH] borg create: add --ignore-inode option This is mainly meant for use with network file systems like sshfs and possibly CIFS, which don't convey proper inode numbers. --- borg/archive.py | 4 ++-- borg/archiver.py | 12 +++++++++++- borg/cache.py | 5 +++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 94948e89f..83308e66d 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -517,7 +517,7 @@ Number of files: {0.stats.nfiles}'''.format( self.add_item(item) return 'i' # stdin - def process_file(self, path, st, cache): + def process_file(self, path, st, cache, ignore_inode=False): status = None safe_path = make_path_safe(path) # Is it a hard link? @@ -533,7 +533,7 @@ Number of files: {0.stats.nfiles}'''.format( self.hard_links[st.st_ino, st.st_dev] = safe_path path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape')) first_run = not cache.files - ids = cache.file_known_and_unchanged(path_hash, st) + ids = cache.file_known_and_unchanged(path_hash, st, ignore_inode) if first_run: logger.debug('Processing files ...') chunks = None diff --git a/borg/archiver.py b/borg/archiver.py index a790ea34a..c49747e7c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -231,6 +231,7 @@ class Archiver: self.output_filter = args.output_filter self.output_list = args.output_list + self.ignore_inode = args.ignore_inode dry_run = args.dry_run t0 = datetime.utcnow() if not dry_run: @@ -270,7 +271,7 @@ class Archiver: if stat.S_ISREG(st.st_mode) or read_special and not stat.S_ISDIR(st.st_mode): if not dry_run: try: - status = archive.process_file(path, st, cache) + status = archive.process_file(path, st, cache, self.ignore_inode) except OSError as e: status = 'E' self.print_warning('%s: %s', path, e) @@ -984,6 +985,12 @@ class Archiver: traversing all paths specified. The archive will consume almost no disk space for files or parts of files that have already been stored in other archives. + + To speed up pulling backups over sshfs and similar network file systems which do + not provide correct inode information the --ignore-inode flag can be used. This + potentially decreases reliability of change detection, while avoiding always reading + all files on these file systems. + See the output of the "borg help patterns" command for more help on exclude patterns. """) @@ -1039,6 +1046,9 @@ class Archiver: type=ChunkerParams, default=CHUNKER_PARAMS, metavar='CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE', help='specify the chunker parameters. default: %d,%d,%d,%d' % CHUNKER_PARAMS) + subparser.add_argument('--ignore-inode', dest='ignore_inode', + action='store_true', default=False, + help='ignore inode data in the file metadata cache used to detect unchanged files.') subparser.add_argument('-C', '--compression', dest='compression', type=CompressionSpec, default=dict(name='none'), metavar='COMPRESSION', help='select compression algorithm (and level): ' diff --git a/borg/cache.py b/borg/cache.py index 79f7bd2d5..3c1e11f22 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -394,7 +394,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" self.chunks[id] = (count - 1, size, csize) stats.update(-size, -csize, False) - def file_known_and_unchanged(self, path_hash, st): + def file_known_and_unchanged(self, path_hash, st, ignore_inode=False): if not (self.do_files and stat.S_ISREG(st.st_mode)): return None if self.files is None: @@ -403,7 +403,8 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" if not entry: return None entry = msgpack.unpackb(entry) - if entry[2] == st.st_size and bigint_to_int(entry[3]) == st.st_mtime_ns and entry[1] == st.st_ino: + if (entry[2] == st.st_size and bigint_to_int(entry[3]) == st.st_mtime_ns and + (ignore_inode or entry[1] == st.st_ino)): # reset entry age entry[0] = 0 self.files[path_hash] = msgpack.packb(entry)