mirror of https://github.com/borgbackup/borg.git
borg export-tar (#2519)
This commit is contained in:
enkore
GitHub
parent
98e4e55141
commit
5788219ff4
|
|
@ -0,0 +1,142 @@
|
|||
.\" Man page generated from reStructuredText.
|
||||
.
|
||||
.TH BORG-EXPORT-TAR 1 "2017-05-16" "" "borg backup tool"
|
||||
.SH NAME
|
||||
borg-export-tar \- Export archive contents as a tarball
|
||||
.
|
||||
.nr rst2man-indent-level 0
|
||||
.
|
||||
.de1 rstReportMargin
|
||||
\\$1 \\n[an-margin]
|
||||
level \\n[rst2man-indent-level]
|
||||
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
|
||||
-
|
||||
\\n[rst2man-indent0]
|
||||
\\n[rst2man-indent1]
|
||||
\\n[rst2man-indent2]
|
||||
..
|
||||
.de1 INDENT
|
||||
.\" .rstReportMargin pre:
|
||||
. RS \\$1
|
||||
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
|
||||
. nr rst2man-indent-level +1
|
||||
.\" .rstReportMargin post:
|
||||
..
|
||||
.de UNINDENT
|
||||
. RE
|
||||
.\" indent \\n[an-margin]
|
||||
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
|
||||
.nr rst2man-indent-level -1
|
||||
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
|
||||
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
|
||||
..
|
||||
.SH SYNOPSIS
|
||||
.sp
|
||||
borg export\-tar <options> ARCHIVE FILE PATH
|
||||
.SH DESCRIPTION
|
||||
.sp
|
||||
This command creates a tarball from an archive.
|
||||
.sp
|
||||
When giving \(aq\-\(aq as the output FILE, Borg will write a tar stream to standard output.
|
||||
.sp
|
||||
By default (\-\-tar\-filter=auto) Borg will detect whether the FILE should be compressed
|
||||
based on its file extension and pipe the tarball through an appropriate filter
|
||||
before writing it to FILE:
|
||||
.INDENT 0.0
|
||||
.IP \(bu 2
|
||||
\&.tar.gz: gzip
|
||||
.IP \(bu 2
|
||||
\&.tar.bz2: bzip2
|
||||
.IP \(bu 2
|
||||
\&.tar.xz: xz
|
||||
.UNINDENT
|
||||
.sp
|
||||
Alternatively a \-\-tar\-filter program may be explicitly specified. It should
|
||||
read the uncompressed tar stream from stdin and write a compressed/filtered
|
||||
tar stream to stdout.
|
||||
.sp
|
||||
The generated tarball uses the GNU tar format.
|
||||
.sp
|
||||
export\-tar is a lossy conversion:
|
||||
BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported.
|
||||
Timestamp resolution is limited to whole seconds, not the nanosecond resolution
|
||||
otherwise supported by Borg.
|
||||
.sp
|
||||
A \-\-sparse option (as found in borg extract) is not supported.
|
||||
.sp
|
||||
By default the entire archive is extracted but a subset of files and directories
|
||||
can be selected by passing a list of \fBPATHs\fP as arguments.
|
||||
The file selection can further be restricted by using the \fB\-\-exclude\fP option.
|
||||
.sp
|
||||
See the output of the "borg help patterns" command for more help on exclude patterns.
|
||||
.sp
|
||||
\fB\-\-progress\fP can be slower than no progress display, since it makes one additional
|
||||
pass over the archive metadata.
|
||||
.SH OPTIONS
|
||||
.sp
|
||||
See \fIborg\-common(1)\fP for common options of Borg commands.
|
||||
.SS arguments
|
||||
.INDENT 0.0
|
||||
.TP
|
||||
.B ARCHIVE
|
||||
archive to export
|
||||
.TP
|
||||
.B FILE
|
||||
output tar file. "\-" to write to stdout instead.
|
||||
.TP
|
||||
.B PATH
|
||||
paths to extract; patterns are supported
|
||||
.UNINDENT
|
||||
.SS optional arguments
|
||||
.INDENT 0.0
|
||||
.TP
|
||||
.B \-\-tar\-filter
|
||||
filter program to pipe data through
|
||||
.TP
|
||||
.B \-\-list
|
||||
output verbose list of items (files, dirs, ...)
|
||||
.TP
|
||||
.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
|
||||
exclude paths matching PATTERN
|
||||
.TP
|
||||
.BI \-\-exclude\-from \ EXCLUDEFILE
|
||||
read exclude patterns from EXCLUDEFILE, one per line
|
||||
.TP
|
||||
.BI \-\-pattern \ PATTERN
|
||||
include/exclude paths matching PATTERN
|
||||
.TP
|
||||
.BI \-\-patterns\-from \ PATTERNFILE
|
||||
read include/exclude patterns from PATTERNFILE, one per line
|
||||
.TP
|
||||
.BI \-\-strip\-components \ NUMBER
|
||||
Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.
|
||||
.UNINDENT
|
||||
.SH EXAMPLES
|
||||
.INDENT 0.0
|
||||
.INDENT 3.5
|
||||
.sp
|
||||
.nf
|
||||
.ft C
|
||||
# export as uncompressed tar
|
||||
$ borg export\-tar /path/to/repo::Monday Monday.tar
|
||||
|
||||
# exclude some types, compress using gzip
|
||||
$ borg export\-tar /path/to/repo::Monday Monday.tar.gz \-\-exclude \(aq*.so\(aq
|
||||
|
||||
# use higher compression level with gzip
|
||||
$ borg export\-tar testrepo::linux \-\-tar\-filter="gzip \-9" Monday.tar.gz
|
||||
|
||||
# export a gzipped tar, but instead of storing it on disk,
|
||||
# upload it to a remote site using curl.
|
||||
$ borg export\-tar ... \-\-tar\-filter="gzip" \- | curl \-\-data\-binary @\- https://somewhere/to/POST
|
||||
.ft P
|
||||
.fi
|
||||
.UNINDENT
|
||||
.UNINDENT
|
||||
.SH SEE ALSO
|
||||
.sp
|
||||
\fIborg\-common(1)\fP
|
||||
.SH AUTHOR
|
||||
The Borg Collective
|
||||
.\" Generated by docutils manpage writer.
|
||||
.
|
||||
|
|
@ -492,6 +492,24 @@ Examples
|
|||
Comment: This is a better comment
|
||||
...
|
||||
|
||||
.. include:: usage/export-tar.rst.inc
|
||||
|
||||
Examples
|
||||
~~~~~~~~
|
||||
::
|
||||
|
||||
# export as uncompressed tar
|
||||
$ borg export-tar /path/to/repo::Monday Monday.tar
|
||||
|
||||
# exclude some types, compress using gzip
|
||||
$ borg export-tar /path/to/repo::Monday Monday.tar.gz --exclude '*.so'
|
||||
|
||||
# use higher compression level with gzip
|
||||
$ borg export-tar testrepo::linux --tar-filter="gzip -9" Monday.tar.gz
|
||||
|
||||
# export a gzipped tar, but instead of storing it on disk,
|
||||
# upload it to a remote site using curl.
|
||||
$ borg export-tar ... --tar-filter="gzip" - | curl --data-binary @- https://somewhere/to/POST
|
||||
|
||||
.. include:: usage/with-lock.rst.inc
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
|
||||
|
||||
.. _borg_export-tar:
|
||||
|
||||
borg export-tar
|
||||
---------------
|
||||
::
|
||||
|
||||
borg export-tar <options> ARCHIVE FILE PATH
|
||||
|
||||
positional arguments
|
||||
ARCHIVE
|
||||
archive to export
|
||||
FILE
|
||||
output tar file. "-" to write to stdout instead.
|
||||
PATH
|
||||
paths to extract; patterns are supported
|
||||
|
||||
optional arguments
|
||||
``--tar-filter``
|
||||
| filter program to pipe data through
|
||||
``--list``
|
||||
| output verbose list of items (files, dirs, ...)
|
||||
``-e PATTERN``, ``--exclude PATTERN``
|
||||
| exclude paths matching PATTERN
|
||||
``--exclude-from EXCLUDEFILE``
|
||||
| read exclude patterns from EXCLUDEFILE, one per line
|
||||
``--pattern PATTERN``
|
||||
| include/exclude paths matching PATTERN
|
||||
``--patterns-from PATTERNFILE``
|
||||
| read include/exclude patterns from PATTERNFILE, one per line
|
||||
``--strip-components NUMBER``
|
||||
| Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.
|
||||
|
||||
`Common options`_
|
||||
|
|
||||
|
||||
Description
|
||||
~~~~~~~~~~~
|
||||
|
||||
This command creates a tarball from an archive.
|
||||
|
||||
When giving '-' as the output FILE, Borg will write a tar stream to standard output.
|
||||
|
||||
By default (--tar-filter=auto) Borg will detect whether the FILE should be compressed
|
||||
based on its file extension and pipe the tarball through an appropriate filter
|
||||
before writing it to FILE:
|
||||
|
||||
- .tar.gz: gzip
|
||||
- .tar.bz2: bzip2
|
||||
- .tar.xz: xz
|
||||
|
||||
Alternatively a --tar-filter program may be explicitly specified. It should
|
||||
read the uncompressed tar stream from stdin and write a compressed/filtered
|
||||
tar stream to stdout.
|
||||
|
||||
The generated tarball uses the GNU tar format.
|
||||
|
||||
export-tar is a lossy conversion:
|
||||
BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported.
|
||||
Timestamp resolution is limited to whole seconds, not the nanosecond resolution
|
||||
otherwise supported by Borg.
|
||||
|
||||
A --sparse option (as found in borg extract) is not supported.
|
||||
|
||||
By default the entire archive is extracted but a subset of files and directories
|
||||
can be selected by passing a list of ``PATHs`` as arguments.
|
||||
The file selection can further be restricted by using the ``--exclude`` option.
|
||||
|
||||
See the output of the "borg help patterns" command for more help on exclude patterns.
|
||||
|
||||
``--progress`` can be slower than no progress display, since it makes one additional
|
||||
pass over the archive metadata.
|
||||
|
|
@ -15,6 +15,7 @@ import signal
|
|||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import textwrap
|
||||
import time
|
||||
import traceback
|
||||
|
|
@ -61,6 +62,7 @@ from .helpers import ErrorIgnoringTextIOWrapper
|
|||
from .helpers import ProgressIndicatorPercent
|
||||
from .helpers import basic_json_data, json_print
|
||||
from .helpers import replace_placeholders
|
||||
from .helpers import ChunkIteratorFileWrapper
|
||||
from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern
|
||||
from .patterns import PatternMatcher
|
||||
from .item import Item
|
||||
|
|
@ -694,6 +696,219 @@ class Archiver:
|
|||
pi.finish()
|
||||
return self.exit_code
|
||||
|
||||
@with_repository()
|
||||
@with_archive
|
||||
def do_export_tar(self, args, repository, manifest, key, archive):
|
||||
"""Export archive contents as a tarball"""
|
||||
self.output_list = args.output_list
|
||||
|
||||
# A quick note about the general design of tar_filter and tarfile;
|
||||
# The tarfile module of Python can provide some compression mechanisms
|
||||
# by itself, using the builtin gzip, bz2 and lzma modules (and "tarmodes"
|
||||
# such as "w:xz").
|
||||
#
|
||||
# Doing so would have three major drawbacks:
|
||||
# For one the compressor runs on the same thread as the program using the
|
||||
# tarfile, stealing valuable CPU time from Borg and thus reducing throughput.
|
||||
# Then this limits the available options - what about lz4? Brotli? zstd?
|
||||
# The third issue is that systems can ship more optimized versions than those
|
||||
# built into Python, e.g. pigz or pxz, which can use more than one thread for
|
||||
# compression.
|
||||
#
|
||||
# Therefore we externalize compression by using a filter program, which has
|
||||
# none of these drawbacks. The only issue of using an external filter is
|
||||
# that it has to be installed -- hardly a problem, considering that
|
||||
# the decompressor must be installed as well to make use of the exported tarball!
|
||||
|
||||
filter = None
|
||||
if args.tar_filter == 'auto':
|
||||
# Note that filter remains None if tarfile is '-'.
|
||||
if args.tarfile.endswith('.tar.gz'):
|
||||
filter = 'gzip'
|
||||
elif args.tarfile.endswith('.tar.bz2'):
|
||||
filter = 'bzip2'
|
||||
elif args.tarfile.endswith('.tar.xz'):
|
||||
filter = 'xz'
|
||||
logger.debug('Automatically determined tar filter: %s', filter)
|
||||
else:
|
||||
filter = args.tar_filter
|
||||
|
||||
if args.tarfile == '-':
|
||||
tarstream, tarstream_close = sys.stdout.buffer, False
|
||||
else:
|
||||
tarstream, tarstream_close = open(args.tarfile, 'wb'), True
|
||||
|
||||
if filter:
|
||||
# When we put a filter between us and the final destination,
|
||||
# the selected output (tarstream until now) becomes the output of the filter (=filterout).
|
||||
# The decision whether to close that or not remains the same.
|
||||
filterout = tarstream
|
||||
filterout_close = tarstream_close
|
||||
# There is no deadlock potential here (the subprocess docs warn about this), because
|
||||
# communication with the process is a one-way road, i.e. the process can never block
|
||||
# for us to do something while we block on the process for something different.
|
||||
filtercmd = shlex.split(filter)
|
||||
logger.debug('--tar-filter command line: %s', filtercmd)
|
||||
filterproc = subprocess.Popen(filtercmd, stdin=subprocess.PIPE, stdout=filterout)
|
||||
# Always close the pipe, otherwise the filter process would not notice when we are done.
|
||||
tarstream = filterproc.stdin
|
||||
tarstream_close = True
|
||||
|
||||
# The | (pipe) symbol instructs tarfile to use a streaming mode of operation
|
||||
# where it never seeks on the passed fileobj.
|
||||
tar = tarfile.open(fileobj=tarstream, mode='w|')
|
||||
|
||||
self._export_tar(args, archive, tar)
|
||||
|
||||
# This does not close the fileobj (tarstream) we passed to it -- a side effect of the | mode.
|
||||
tar.close()
|
||||
|
||||
if tarstream_close:
|
||||
tarstream.close()
|
||||
|
||||
if filter:
|
||||
logger.debug('Done creating tar, waiting for filter to die...')
|
||||
rc = filterproc.wait()
|
||||
if rc:
|
||||
logger.error('--tar-filter exited with code %d, output file is likely unusable!', rc)
|
||||
self.exit_code = set_ec(EXIT_ERROR)
|
||||
else:
|
||||
logger.debug('filter exited with code %d', rc)
|
||||
|
||||
if filterout_close:
|
||||
filterout.close()
|
||||
|
||||
return self.exit_code
|
||||
|
||||
def _export_tar(self, args, archive, tar):
|
||||
matcher = self.build_matcher(args.patterns, args.paths)
|
||||
|
||||
progress = args.progress
|
||||
output_list = args.output_list
|
||||
strip_components = args.strip_components
|
||||
partial_extract = not matcher.empty() or strip_components
|
||||
hardlink_masters = {} if partial_extract else None
|
||||
|
||||
def peek_and_store_hardlink_masters(item, matched):
|
||||
if (partial_extract and not matched and hardlinkable(item.mode) and
|
||||
item.get('hardlink_master', True) and 'source' not in item):
|
||||
hardlink_masters[item.get('path')] = (item.get('chunks'), None)
|
||||
|
||||
filter = self.build_filter(matcher, peek_and_store_hardlink_masters, strip_components)
|
||||
|
||||
if progress:
|
||||
pi = ProgressIndicatorPercent(msg='%5.1f%% Processing: %s', step=0.1, msgid='extract')
|
||||
pi.output('Calculating size')
|
||||
extracted_size = sum(item.get_size(hardlink_masters) for item in archive.iter_items(filter))
|
||||
pi.total = extracted_size
|
||||
else:
|
||||
pi = None
|
||||
|
||||
def item_content_stream(item):
|
||||
"""
|
||||
Return a file-like object that reads from the chunks of *item*.
|
||||
"""
|
||||
chunk_iterator = archive.pipeline.fetch_many([chunk_id for chunk_id, _, _ in item.chunks])
|
||||
if pi:
|
||||
info = [remove_surrogates(item.path)]
|
||||
return ChunkIteratorFileWrapper(chunk_iterator,
|
||||
lambda read_bytes: pi.show(increase=len(read_bytes), info=info))
|
||||
else:
|
||||
return ChunkIteratorFileWrapper(chunk_iterator)
|
||||
|
||||
def item_to_tarinfo(item, original_path):
|
||||
"""
|
||||
Transform a Borg *item* into a tarfile.TarInfo object.
|
||||
|
||||
Return a tuple (tarinfo, stream), where stream may be a file-like object that represents
|
||||
the file contents, if any, and is None otherwise. When *tarinfo* is None, the *item*
|
||||
cannot be represented as a TarInfo object and should be skipped.
|
||||
"""
|
||||
|
||||
# If we would use the PAX (POSIX) format (which we currently don't),
|
||||
# we can support most things that aren't possible with classic tar
|
||||
# formats, including GNU tar, such as:
|
||||
# atime, ctime, possibly Linux capabilities (security.* xattrs)
|
||||
# and various additions supported by GNU tar in POSIX mode.
|
||||
|
||||
stream = None
|
||||
tarinfo = tarfile.TarInfo()
|
||||
tarinfo.name = item.path
|
||||
tarinfo.mtime = item.mtime / 1e9
|
||||
tarinfo.mode = stat.S_IMODE(item.mode)
|
||||
tarinfo.uid = item.uid
|
||||
tarinfo.gid = item.gid
|
||||
tarinfo.uname = item.user or ''
|
||||
tarinfo.gname = item.group or ''
|
||||
# The linkname in tar has the same dual use the 'source' attribute of Borg items,
|
||||
# i.e. for symlinks it means the destination, while for hardlinks it refers to the
|
||||
# file.
|
||||
# Since hardlinks in tar have a different type code (LNKTYPE) the format might
|
||||
# support hardlinking arbitrary objects (including symlinks and directories), but
|
||||
# whether implementations actually support that is a whole different question...
|
||||
tarinfo.linkname = ""
|
||||
|
||||
modebits = stat.S_IFMT(item.mode)
|
||||
if modebits == stat.S_IFREG:
|
||||
tarinfo.type = tarfile.REGTYPE
|
||||
if 'source' in item:
|
||||
source = os.sep.join(item.source.split(os.sep)[strip_components:])
|
||||
if hardlink_masters is None:
|
||||
linkname = source
|
||||
else:
|
||||
chunks, linkname = hardlink_masters.get(item.source, (None, source))
|
||||
if linkname:
|
||||
# Master was already added to the archive, add a hardlink reference to it.
|
||||
tarinfo.type = tarfile.LNKTYPE
|
||||
tarinfo.linkname = linkname
|
||||
elif chunks is not None:
|
||||
# The item which has the chunks was not put into the tar, therefore
|
||||
# we do that now and update hardlink_masters to reflect that.
|
||||
item.chunks = chunks
|
||||
tarinfo.size = item.get_size()
|
||||
stream = item_content_stream(item)
|
||||
hardlink_masters[item.get('source') or original_path] = (None, item.path)
|
||||
else:
|
||||
tarinfo.size = item.get_size()
|
||||
stream = item_content_stream(item)
|
||||
elif modebits == stat.S_IFDIR:
|
||||
tarinfo.type = tarfile.DIRTYPE
|
||||
elif modebits == stat.S_IFLNK:
|
||||
tarinfo.type = tarfile.SYMTYPE
|
||||
tarinfo.linkname = item.source
|
||||
elif modebits == stat.S_IFBLK:
|
||||
tarinfo.type = tarfile.BLKTYPE
|
||||
tarinfo.devmajor = os.major(item.rdev)
|
||||
tarinfo.devminor = os.minor(item.rdev)
|
||||
elif modebits == stat.S_IFCHR:
|
||||
tarinfo.type = tarfile.CHRTYPE
|
||||
tarinfo.devmajor = os.major(item.rdev)
|
||||
tarinfo.devminor = os.minor(item.rdev)
|
||||
elif modebits == stat.S_IFIFO:
|
||||
tarinfo.type = tarfile.FIFOTYPE
|
||||
else:
|
||||
self.print_warning('%s: unsupported file type %o for tar export', remove_surrogates(item.path), modebits)
|
||||
set_ec(EXIT_WARNING)
|
||||
return None, stream
|
||||
return tarinfo, stream
|
||||
|
||||
for item in archive.iter_items(filter, preload=True):
|
||||
orig_path = item.path
|
||||
if strip_components:
|
||||
item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
|
||||
tarinfo, stream = item_to_tarinfo(item, orig_path)
|
||||
if tarinfo:
|
||||
if output_list:
|
||||
logging.getLogger('borg.output.list').info(remove_surrogates(orig_path))
|
||||
tar.addfile(tarinfo, stream)
|
||||
|
||||
if pi:
|
||||
pi.finish()
|
||||
|
||||
for pattern in matcher.get_unmatched_include_patterns():
|
||||
self.print_warning("Include pattern '%s' never matched.", pattern)
|
||||
return self.exit_code
|
||||
|
||||
@with_repository()
|
||||
@with_archive
|
||||
def do_diff(self, args, repository, manifest, key, archive):
|
||||
|
|
@ -2605,6 +2820,72 @@ class Archiver:
|
|||
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
|
||||
help='paths to extract; patterns are supported')
|
||||
|
||||
export_tar_epilog = process_epilog("""
|
||||
This command creates a tarball from an archive.
|
||||
|
||||
When giving '-' as the output FILE, Borg will write a tar stream to standard output.
|
||||
|
||||
By default (--tar-filter=auto) Borg will detect whether the FILE should be compressed
|
||||
based on its file extension and pipe the tarball through an appropriate filter
|
||||
before writing it to FILE:
|
||||
|
||||
- .tar.gz: gzip
|
||||
- .tar.bz2: bzip2
|
||||
- .tar.xz: xz
|
||||
|
||||
Alternatively a --tar-filter program may be explicitly specified. It should
|
||||
read the uncompressed tar stream from stdin and write a compressed/filtered
|
||||
tar stream to stdout.
|
||||
|
||||
The generated tarball uses the GNU tar format.
|
||||
|
||||
export-tar is a lossy conversion:
|
||||
BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported.
|
||||
Timestamp resolution is limited to whole seconds, not the nanosecond resolution
|
||||
otherwise supported by Borg.
|
||||
|
||||
A --sparse option (as found in borg extract) is not supported.
|
||||
|
||||
By default the entire archive is extracted but a subset of files and directories
|
||||
can be selected by passing a list of ``PATHs`` as arguments.
|
||||
The file selection can further be restricted by using the ``--exclude`` option.
|
||||
|
||||
See the output of the "borg help patterns" command for more help on exclude patterns.
|
||||
|
||||
``--progress`` can be slower than no progress display, since it makes one additional
|
||||
pass over the archive metadata.
|
||||
""")
|
||||
subparser = subparsers.add_parser('export-tar', parents=[common_parser], add_help=False,
|
||||
description=self.do_export_tar.__doc__,
|
||||
epilog=export_tar_epilog,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
help='create tarball from archive')
|
||||
subparser.set_defaults(func=self.do_export_tar)
|
||||
subparser.add_argument('--tar-filter', dest='tar_filter', default='auto',
|
||||
help='filter program to pipe data through')
|
||||
subparser.add_argument('--list', dest='output_list',
|
||||
action='store_true', default=False,
|
||||
help='output verbose list of items (files, dirs, ...)')
|
||||
subparser.add_argument('-e', '--exclude', dest='patterns',
|
||||
type=parse_exclude_pattern, action='append',
|
||||
metavar="PATTERN", help='exclude paths matching PATTERN')
|
||||
subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
|
||||
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
|
||||
subparser.add_argument('--pattern', action=ArgparsePatternAction,
|
||||
metavar="PATTERN", help='include/exclude paths matching PATTERN')
|
||||
subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction,
|
||||
metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
|
||||
subparser.add_argument('--strip-components', dest='strip_components',
|
||||
type=int, default=0, metavar='NUMBER',
|
||||
help='Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.')
|
||||
subparser.add_argument('location', metavar='ARCHIVE',
|
||||
type=location_validator(archive=True),
|
||||
help='archive to export')
|
||||
subparser.add_argument('tarfile', metavar='FILE',
|
||||
help='output tar file. "-" to write to stdout instead.')
|
||||
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
|
||||
help='paths to extract; patterns are supported')
|
||||
|
||||
diff_epilog = process_epilog("""
|
||||
This command finds differences (file contents, user/group/mode) between archives.
|
||||
|
||||
|
|
|
|||
|
|
@ -1622,11 +1622,20 @@ class ItemFormatter(BaseFormatter):
|
|||
class ChunkIteratorFileWrapper:
|
||||
"""File-like wrapper for chunk iterators"""
|
||||
|
||||
def __init__(self, chunk_iterator):
|
||||
def __init__(self, chunk_iterator, read_callback=None):
|
||||
"""
|
||||
*chunk_iterator* should be an iterator yielding bytes. These will be buffered
|
||||
internally as necessary to satisfy .read() calls.
|
||||
|
||||
*read_callback* will be called with one argument, some byte string that has
|
||||
just been read and will be subsequently returned to a caller of .read().
|
||||
It can be used to update a progress display.
|
||||
"""
|
||||
self.chunk_iterator = chunk_iterator
|
||||
self.chunk_offset = 0
|
||||
self.chunk = b''
|
||||
self.exhausted = False
|
||||
self.read_callback = read_callback
|
||||
|
||||
def _refill(self):
|
||||
remaining = len(self.chunk) - self.chunk_offset
|
||||
|
|
@ -1655,6 +1664,8 @@ class ChunkIteratorFileWrapper:
|
|||
read_data = self._read(nbytes)
|
||||
nbytes -= len(read_data)
|
||||
parts.append(read_data)
|
||||
if self.read_callback:
|
||||
self.read_callback(read_data)
|
||||
return b''.join(parts)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ class BaseTestCase(unittest.TestCase):
|
|||
diff = filecmp.dircmp(dir1, dir2)
|
||||
self._assert_dirs_equal_cmp(diff, **kwargs)
|
||||
|
||||
def _assert_dirs_equal_cmp(self, diff, ignore_bsdflags=False, ignore_xattrs=False):
|
||||
def _assert_dirs_equal_cmp(self, diff, ignore_bsdflags=False, ignore_xattrs=False, ignore_ns=False):
|
||||
self.assert_equal(diff.left_only, [])
|
||||
self.assert_equal(diff.right_only, [])
|
||||
self.assert_equal(diff.diff_files, [])
|
||||
|
|
@ -162,25 +162,30 @@ class BaseTestCase(unittest.TestCase):
|
|||
s2 = os.lstat(path2)
|
||||
# Assume path2 is on FUSE if st_dev is different
|
||||
fuse = s1.st_dev != s2.st_dev
|
||||
attrs = ['st_mode', 'st_uid', 'st_gid', 'st_rdev']
|
||||
attrs = ['st_uid', 'st_gid', 'st_rdev']
|
||||
if not fuse or not os.path.isdir(path1):
|
||||
# dir nlink is always 1 on our fuse filesystem
|
||||
attrs.append('st_nlink')
|
||||
d1 = [filename] + [getattr(s1, a) for a in attrs]
|
||||
d2 = [filename] + [getattr(s2, a) for a in attrs]
|
||||
d1.insert(1, oct(s1.st_mode))
|
||||
d2.insert(1, oct(s2.st_mode))
|
||||
if not ignore_bsdflags:
|
||||
d1.append(get_flags(path1, s1))
|
||||
d2.append(get_flags(path2, s2))
|
||||
# ignore st_rdev if file is not a block/char device, fixes #203
|
||||
if not stat.S_ISCHR(d1[1]) and not stat.S_ISBLK(d1[1]):
|
||||
if not stat.S_ISCHR(s1.st_mode) and not stat.S_ISBLK(s1.st_mode):
|
||||
d1[4] = None
|
||||
if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]):
|
||||
if not stat.S_ISCHR(s2.st_mode) and not stat.S_ISBLK(s2.st_mode):
|
||||
d2[4] = None
|
||||
# If utime isn't fully supported, borg can't set mtime.
|
||||
# Therefore, we shouldn't test it in that case.
|
||||
if is_utime_fully_supported():
|
||||
# Older versions of llfuse do not support ns precision properly
|
||||
if fuse and not have_fuse_mtime_ns:
|
||||
if ignore_ns:
|
||||
d1.append(int(s1.st_mtime_ns / 1e9))
|
||||
d2.append(int(s2.st_mtime_ns / 1e9))
|
||||
elif fuse and not have_fuse_mtime_ns:
|
||||
d1.append(round(s1.st_mtime_ns, -4))
|
||||
d2.append(round(s2.st_mtime_ns, -4))
|
||||
else:
|
||||
|
|
@ -191,7 +196,7 @@ class BaseTestCase(unittest.TestCase):
|
|||
d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
|
||||
self.assert_equal(d1, d2)
|
||||
for sub_diff in diff.subdirs.values():
|
||||
self._assert_dirs_equal_cmp(sub_diff, ignore_bsdflags=ignore_bsdflags, ignore_xattrs=ignore_xattrs)
|
||||
self._assert_dirs_equal_cmp(sub_diff, ignore_bsdflags=ignore_bsdflags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns)
|
||||
|
||||
@contextmanager
|
||||
def fuse_mount(self, location, mountpoint, *options):
|
||||
|
|
|
|||
|
|
@ -96,6 +96,14 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw):
|
|||
sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
|
||||
|
||||
|
||||
def have_gnutar():
|
||||
if not shutil.which('tar'):
|
||||
return False
|
||||
popen = subprocess.Popen(['tar', '--version'], stdout=subprocess.PIPE)
|
||||
stdout, stderr = popen.communicate()
|
||||
return b'GNU tar' in stdout
|
||||
|
||||
|
||||
# check if the binary "borg.exe" is available (for local testing a symlink to virtualenv/bin/borg should do)
|
||||
try:
|
||||
exec_cmd('help', exe='borg.exe', fork=True)
|
||||
|
|
@ -2354,6 +2362,35 @@ id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02
|
|||
assert '_meta' in result
|
||||
assert '_items' in result
|
||||
|
||||
requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.')
|
||||
requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.')
|
||||
|
||||
@requires_gnutar
|
||||
def test_export_tar(self):
|
||||
self.create_test_files()
|
||||
os.unlink('input/flagfile')
|
||||
self.cmd('init', '--encryption=repokey', self.repository_location)
|
||||
self.cmd('create', self.repository_location + '::test', 'input')
|
||||
self.cmd('export-tar', self.repository_location + '::test', 'simple.tar')
|
||||
with changedir('output'):
|
||||
# This probably assumes GNU tar. Note -p switch to extract permissions regardless of umask.
|
||||
subprocess.check_output(['tar', 'xpf', '../simple.tar'])
|
||||
self.assert_dirs_equal('input', 'output/input', ignore_bsdflags=True, ignore_xattrs=True, ignore_ns=True)
|
||||
|
||||
@requires_gnutar
|
||||
@requires_gzip
|
||||
def test_export_tar_gz(self):
|
||||
if not shutil.which('gzip'):
|
||||
pytest.skip('gzip is not installed')
|
||||
self.create_test_files()
|
||||
os.unlink('input/flagfile')
|
||||
self.cmd('init', '--encryption=repokey', self.repository_location)
|
||||
self.cmd('create', self.repository_location + '::test', 'input')
|
||||
self.cmd('export-tar', self.repository_location + '::test', 'simple.tar.gz')
|
||||
with changedir('output'):
|
||||
subprocess.check_output(['tar', 'xpf', '../simple.tar.gz'])
|
||||
self.assert_dirs_equal('input', 'output/input', ignore_bsdflags=True, ignore_xattrs=True, ignore_ns=True)
|
||||
|
||||
|
||||
@unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available')
|
||||
class ArchiverTestCaseBinary(ArchiverTestCase):
|
||||
|
|
|
|||
Loading…
Reference in New Issue