mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-25 01:06:50 +00:00
Code Issues Releases Wiki Activity

Merge pull request #2988 from ThomasWaldmann/recover-segments-memory-usage

Browse source 
recover_segment: use mmap(), fixes #2982
This commit is contained in:
TW 2017-09-02 17:48:04 +02:00 committed by GitHub
commit 86c0b66de3
1 changed files with 21 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
src/borg/repository.py
View file

@ -1,4 +1,5 @@
import errno import errno
import mmap
import os import os
import shutil import shutil
import struct import struct
@ -1309,25 +1310,28 @@ def iter_objects(self, segment, offset=0, include_data=False, read_data=True):
header = fd.read(self.header_fmt.size) header = fd.read(self.header_fmt.size)
def recover_segment(self, segment, filename): def recover_segment(self, segment, filename):
logger.info('attempting to recover ' + filename)
if segment in self.fds: if segment in self.fds:
del self.fds[segment] del self.fds[segment]
with open(filename, 'rb') as fd: backup_filename = filename + '.beforerecover'
# XXX: Rather use mmap, this loads the entire segment (up to 500 MB by default) into memory. os.rename(filename, backup_filename)
data = memoryview(fd.read()) with open(backup_filename, 'rb') as backup_fd:
os.rename(filename, filename + '.beforerecover') # note: file must not be 0 size (windows can't create 0 size mapping)
logger.info('attempting to recover ' + filename) with mmap.mmap(backup_fd.fileno(), 0, access=mmap.ACCESS_READ) as mm:
with open(filename, 'wb') as fd: data = memoryview(mm)
fd.write(MAGIC) with open(filename, 'wb') as fd:
while len(data) >= self.header_fmt.size: fd.write(MAGIC)
crc, size, tag = self.header_fmt.unpack(data[:self.header_fmt.size]) while len(data) >= self.header_fmt.size:
if size < self.header_fmt.size or size > len(data): crc, size, tag = self.header_fmt.unpack(data[:self.header_fmt.size])
data = data[1:] if size < self.header_fmt.size or size > len(data):
continue data = data[1:]
if crc32(data[4:size]) & 0xffffffff != crc: continue
data = data[1:] if crc32(data[4:size]) & 0xffffffff != crc:
continue data = data[1:]
fd.write(data[:size]) continue
data = data[size:] fd.write(data[:size])
data = data[size:]
data.release()
def read(self, segment, offset, id, read_data=True): def read(self, segment, offset, id, read_data=True):
""" """