mirror of https://github.com/borgbackup/borg.git
FilesystemObjectProcessors.process_file: clean up orphaned chunks in case of exceptions
Note: no changes inside the indented block, just added the try and the except block.
This commit is contained in:
Thomas Waldmann
parent
f1981715e4
commit
d0c61bbbf1
|
|
@ -1451,76 +1451,90 @@ class FilesystemObjectProcessors:
|
|||
# so it can be extracted / accessed in FUSE mount like a regular file.
|
||||
# this needs to be done early, so that part files also get the patched mode.
|
||||
item.mode = stat.S_IFREG | stat.S_IMODE(item.mode)
|
||||
if hl_chunks is not None: # create_helper gave us chunks from a previous hardlink
|
||||
item.chunks = []
|
||||
for chunk_id, chunk_size in hl_chunks:
|
||||
# process one-by-one, so we will know in item.chunks how far we got
|
||||
chunk_entry = cache.chunk_incref(chunk_id, self.stats)
|
||||
item.chunks.append(chunk_entry)
|
||||
else: # normal case, no "2nd+" hardlink
|
||||
if not is_special_file:
|
||||
hashed_path = safe_encode(os.path.join(self.cwd, path))
|
||||
started_hashing = time.monotonic()
|
||||
path_hash = self.key.id_hash(hashed_path)
|
||||
self.stats.hashing_time += time.monotonic() - started_hashing
|
||||
known, ids = cache.file_known_and_unchanged(hashed_path, path_hash, st)
|
||||
else:
|
||||
# in --read-special mode, we may be called for special files.
|
||||
# there should be no information in the cache about special files processed in
|
||||
# read-special mode, but we better play safe as this was wrong in the past:
|
||||
hashed_path = path_hash = None
|
||||
known, ids = False, None
|
||||
if ids is not None:
|
||||
# Make sure all ids are available
|
||||
for id_ in ids:
|
||||
if not cache.seen_chunk(id_):
|
||||
# cache said it is unmodified, but we lost a chunk: process file like modified
|
||||
status = "M"
|
||||
break
|
||||
# we begin processing chunks now (writing or incref'ing them to the repository),
|
||||
# which might require cleanup (see except-branch):
|
||||
try:
|
||||
if hl_chunks is not None: # create_helper gave us chunks from a previous hardlink
|
||||
item.chunks = []
|
||||
for chunk_id, chunk_size in hl_chunks:
|
||||
# process one-by-one, so we will know in item.chunks how far we got
|
||||
chunk_entry = cache.chunk_incref(chunk_id, self.stats)
|
||||
item.chunks.append(chunk_entry)
|
||||
else: # normal case, no "2nd+" hardlink
|
||||
if not is_special_file:
|
||||
hashed_path = safe_encode(os.path.join(self.cwd, path))
|
||||
started_hashing = time.monotonic()
|
||||
path_hash = self.key.id_hash(hashed_path)
|
||||
self.stats.hashing_time += time.monotonic() - started_hashing
|
||||
known, ids = cache.file_known_and_unchanged(hashed_path, path_hash, st)
|
||||
else:
|
||||
item.chunks = []
|
||||
for chunk_id in ids:
|
||||
# process one-by-one, so we will know in item.chunks how far we got
|
||||
chunk_entry = cache.chunk_incref(chunk_id, self.stats)
|
||||
item.chunks.append(chunk_entry)
|
||||
status = "U" # regular file, unchanged
|
||||
else:
|
||||
status = "M" if known else "A" # regular file, modified or added
|
||||
self.print_file_status(status, path)
|
||||
self.stats.files_stats[status] += 1
|
||||
status = None # we already printed the status
|
||||
# Only chunkify the file if needed
|
||||
if "chunks" not in item:
|
||||
with backup_io("read"):
|
||||
self.process_file_chunks(
|
||||
item,
|
||||
cache,
|
||||
self.stats,
|
||||
self.show_progress,
|
||||
backup_io_iter(self.chunker.chunkify(None, fd)),
|
||||
)
|
||||
self.stats.chunking_time = self.chunker.chunking_time
|
||||
if is_win32:
|
||||
changed_while_backup = False # TODO
|
||||
# in --read-special mode, we may be called for special files.
|
||||
# there should be no information in the cache about special files processed in
|
||||
# read-special mode, but we better play safe as this was wrong in the past:
|
||||
hashed_path = path_hash = None
|
||||
known, ids = False, None
|
||||
if ids is not None:
|
||||
# Make sure all ids are available
|
||||
for id_ in ids:
|
||||
if not cache.seen_chunk(id_):
|
||||
# cache said it is unmodified, but we lost a chunk: process file like modified
|
||||
status = "M"
|
||||
break
|
||||
else:
|
||||
item.chunks = []
|
||||
for chunk_id in ids:
|
||||
# process one-by-one, so we will know in item.chunks how far we got
|
||||
chunk_entry = cache.chunk_incref(chunk_id, self.stats)
|
||||
item.chunks.append(chunk_entry)
|
||||
status = "U" # regular file, unchanged
|
||||
else:
|
||||
with backup_io("fstat2"):
|
||||
st2 = os.fstat(fd)
|
||||
# special files:
|
||||
# - fifos change naturally, because they are fed from the other side. no problem.
|
||||
# - blk/chr devices don't change ctime anyway.
|
||||
changed_while_backup = not is_special_file and st.st_ctime_ns != st2.st_ctime_ns
|
||||
if changed_while_backup:
|
||||
status = "C" # regular file changed while we backed it up, might be inconsistent/corrupt!
|
||||
if not is_special_file and not changed_while_backup:
|
||||
# we must not memorize special files, because the contents of e.g. a
|
||||
# block or char device will change without its mtime/size/inode changing.
|
||||
# also, we must not memorize a potentially inconsistent/corrupt file that
|
||||
# changed while we backed it up.
|
||||
cache.memorize_file(hashed_path, path_hash, st, [c.id for c in item.chunks])
|
||||
self.stats.nfiles += 1
|
||||
item.update(self.metadata_collector.stat_ext_attrs(st, path, fd=fd))
|
||||
item.get_size(memorize=True)
|
||||
return status
|
||||
status = "M" if known else "A" # regular file, modified or added
|
||||
self.print_file_status(status, path)
|
||||
self.stats.files_stats[status] += 1
|
||||
status = None # we already printed the status
|
||||
# Only chunkify the file if needed
|
||||
if "chunks" not in item:
|
||||
with backup_io("read"):
|
||||
self.process_file_chunks(
|
||||
item,
|
||||
cache,
|
||||
self.stats,
|
||||
self.show_progress,
|
||||
backup_io_iter(self.chunker.chunkify(None, fd)),
|
||||
)
|
||||
self.stats.chunking_time = self.chunker.chunking_time
|
||||
if is_win32:
|
||||
changed_while_backup = False # TODO
|
||||
else:
|
||||
with backup_io("fstat2"):
|
||||
st2 = os.fstat(fd)
|
||||
# special files:
|
||||
# - fifos change naturally, because they are fed from the other side. no problem.
|
||||
# - blk/chr devices don't change ctime anyway.
|
||||
changed_while_backup = not is_special_file and st.st_ctime_ns != st2.st_ctime_ns
|
||||
if changed_while_backup:
|
||||
# regular file changed while we backed it up, might be inconsistent/corrupt!
|
||||
status = "C"
|
||||
if not is_special_file and not changed_while_backup:
|
||||
# we must not memorize special files, because the contents of e.g. a
|
||||
# block or char device will change without its mtime/size/inode changing.
|
||||
# also, we must not memorize a potentially inconsistent/corrupt file that
|
||||
# changed while we backed it up.
|
||||
cache.memorize_file(hashed_path, path_hash, st, [c.id for c in item.chunks])
|
||||
self.stats.nfiles += 1
|
||||
item.update(self.metadata_collector.stat_ext_attrs(st, path, fd=fd))
|
||||
item.get_size(memorize=True)
|
||||
return status
|
||||
except BackupOSError:
|
||||
# Something went wrong and we might need to clean up a bit.
|
||||
# Maybe we have already incref'ed some file content chunks in the repo -
|
||||
# but we will not add an item (see add_item in create_helper) and thus
|
||||
# they would be orphaned chunks in case that we commit the transaction.
|
||||
for chunk in item.get("chunks", []):
|
||||
cache.chunk_decref(chunk.id, self.stats, wait=False)
|
||||
# Now that we have cleaned up the chunk references, we can re-raise the exception.
|
||||
# This will skip processing of this file, but might retry or continue with the next one.
|
||||
raise
|
||||
|
||||
|
||||
class TarfileObjectProcessors:
|
||||
|
|
|
|||
Loading…
Reference in New Issue