1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-03-15 00:21:56 +00:00

Merge pull request #1112 from ThomasWaldmann/refactor-helper-imports

split helpers import into multiple ones
This commit is contained in:
enkore 2016-05-31 15:07:14 +02:00
commit a1c4d45920
20 changed files with 181 additions and 136 deletions

View file

@ -1,37 +1,43 @@
from datetime import datetime, timezone
from getpass import getuser
from itertools import groupby
import errno import errno
from .logger import create_logger
logger = create_logger()
from .key import key_factory
from .remote import cache_if_remote
import os import os
from shutil import get_terminal_size
import socket import socket
import stat import stat
import sys import sys
import time import time
from datetime import datetime, timezone
from getpass import getuser
from io import BytesIO from io import BytesIO
from . import xattr from itertools import groupby
from .compress import COMPR_BUFFER from shutil import get_terminal_size
from .constants import * # NOQA
from .helpers import Chunk, Error, uid2user, user2uid, gid2group, group2gid, \
parse_timestamp, to_localtime, format_time, format_timedelta, safe_encode, safe_decode, \
Manifest, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, bin_to_hex, \
ProgressIndicatorPercent, ChunkIteratorFileWrapper, remove_surrogates, log_multi, \
PathPrefixPattern, FnmatchPattern, open_item, file_status, format_file_size, consume, \
CompressionDecider1, CompressionDecider2, CompressionSpec, \
IntegrityError
from .repository import Repository
from .platform import acl_get, acl_set, set_flags, get_flags, swidth
from .chunker import Chunker
from .hashindex import ChunkIndex, ChunkIndexEntry
from .cache import ChunkListEntry
import msgpack import msgpack
from .logger import create_logger
logger = create_logger()
from . import xattr
from .cache import ChunkListEntry
from .chunker import Chunker
from .constants import * # NOQA
from .hashindex import ChunkIndex, ChunkIndexEntry
from .helpers import Manifest
from .helpers import Chunk, ChunkIteratorFileWrapper, open_item
from .helpers import Error, IntegrityError
from .helpers import uid2user, user2uid, gid2group, group2gid
from .helpers import parse_timestamp, to_localtime
from .helpers import format_time, format_timedelta, format_file_size, file_status
from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
from .helpers import decode_dict, StableDict
from .helpers import int_to_bigint, bigint_to_int, bin_to_hex
from .helpers import ProgressIndicatorPercent, log_multi
from .helpers import PathPrefixPattern, FnmatchPattern
from .helpers import consume
from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec
from .key import key_factory
from .platform import acl_get, acl_set, set_flags, get_flags, swidth
from .remote import cache_if_remote
from .repository import Repository
has_lchmod = hasattr(os, 'lchmod') has_lchmod = hasattr(os, 'lchmod')
flags_normal = os.O_RDONLY | getattr(os, 'O_BINARY', 0) flags_normal = os.O_RDONLY | getattr(os, 'O_BINARY', 0)

View file

@ -1,7 +1,3 @@
from binascii import unhexlify
from datetime import datetime
from itertools import zip_longest
from operator import attrgetter
import argparse import argparse
import collections import collections
import functools import functools
@ -18,28 +14,36 @@ import subprocess
import sys import sys
import textwrap import textwrap
import traceback import traceback
from binascii import unhexlify
from datetime import datetime
from itertools import zip_longest
from . import __version__
from .helpers import Error, location_validator, archivename_validator, format_time, format_file_size, \
parse_pattern, PathPrefixPattern, to_localtime, timestamp, \
get_cache_dir, prune_within, prune_split, bin_to_hex, safe_encode, \
Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, \
dir_is_tagged, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \
log_multi, PatternMatcher, ItemFormatter
from .logger import create_logger, setup_logging from .logger import create_logger, setup_logging
logger = create_logger() logger = create_logger()
from . import __version__
from . import helpers from . import helpers
from .compress import Compressor, COMPR_BUFFER from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics
from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
from .repository import Repository
from .cache import Cache from .cache import Cache
from .constants import * # NOQA from .constants import * # NOQA
from .helpers import Error
from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec
from .helpers import ItemFormatter, format_time, format_file_size, format_archive
from .helpers import safe_encode, remove_surrogates, bin_to_hex
from .helpers import prune_within, prune_split
from .helpers import to_localtime, timestamp
from .helpers import get_cache_dir
from .helpers import Manifest
from .helpers import update_excludes, check_extension_modules
from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo
from .helpers import log_multi
from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern
from .key import key_creator, RepoKey, PassphraseKey from .key import key_creator, RepoKey, PassphraseKey
from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics
from .remote import RepositoryServer, RemoteRepository, cache_if_remote
from .selftest import selftest
from .hashindex import ChunkIndexEntry
from .platform import get_flags from .platform import get_flags
from .remote import RepositoryServer, RemoteRepository, cache_if_remote
from .repository import Repository
from .selftest import selftest
from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
def argument(args, str_or_bool): def argument(args, str_or_bool):

View file

@ -1,21 +1,25 @@
import configparser import configparser
from .remote import cache_if_remote
from collections import namedtuple
import os import os
import stat import stat
from binascii import unhexlify
import shutil import shutil
from binascii import unhexlify
from .key import PlaintextKey from collections import namedtuple
from .logger import create_logger
logger = create_logger()
from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \
bigint_to_int, bin_to_hex, format_file_size, yes
from .locking import UpgradableLock
from .hashindex import ChunkIndex, ChunkIndexEntry
import msgpack import msgpack
from .logger import create_logger
logger = create_logger()
from .hashindex import ChunkIndex, ChunkIndexEntry
from .helpers import Error
from .helpers import get_cache_dir
from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex
from .helpers import format_file_size
from .helpers import yes
from .key import PlaintextKey
from .locking import UpgradableLock
from .remote import cache_if_remote
ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize') ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize')
FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids') FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids')

View file

@ -1,20 +1,23 @@
from collections import defaultdict
import errno import errno
import io import io
import llfuse
import os import os
import stat import stat
import tempfile import tempfile
import time import time
from .archive import Archive from collections import defaultdict
from .helpers import daemonize, bigint_to_int
from .logger import create_logger
from .lrucache import LRUCache
from distutils.version import LooseVersion from distutils.version import LooseVersion
import llfuse
import msgpack import msgpack
from .logger import create_logger
logger = create_logger() logger = create_logger()
from .archive import Archive
from .helpers import daemonize
from .helpers import bigint_to_int
from .lrucache import LRUCache
# Does this version of llfuse support ns precision? # Does this version of llfuse support ns precision?
have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')

View file

@ -1,42 +1,40 @@
import argparse import argparse
from binascii import hexlify
from collections import namedtuple, deque
from functools import wraps, partial
import grp import grp
import hashlib import hashlib
from itertools import islice import logging
import os import os
import os.path import os.path
import stat import platform
import textwrap
import pwd import pwd
import re import re
import socket
import sys import sys
from string import Formatter import stat
import platform import textwrap
import time import time
import unicodedata import unicodedata
import logging from binascii import hexlify
from collections import namedtuple, deque
from datetime import datetime, timezone, timedelta
from fnmatch import translate
from functools import wraps, partial
from itertools import islice
from operator import attrgetter
from string import Formatter
import msgpack
import msgpack.fallback
from .logger import create_logger from .logger import create_logger
logger = create_logger() logger = create_logger()
from datetime import datetime, timezone, timedelta
from fnmatch import translate
from operator import attrgetter
from . import __version__ as borg_version from . import __version__ as borg_version
from . import hashindex
from . import chunker from . import chunker
from .constants import * # NOQA
from . import crypto from . import crypto
from .compress import COMPR_BUFFER, get_compressor from . import hashindex
from . import shellpattern from . import shellpattern
import msgpack from .constants import * # NOQA
import msgpack.fallback from .compress import COMPR_BUFFER, get_compressor
import socket
# meta dict, data bytes # meta dict, data bytes
_Chunk = namedtuple('_Chunk', 'meta data') _Chunk = namedtuple('_Chunk', 'meta data')

View file

@ -1,5 +1,7 @@
from .constants import ITEM_KEYS from .constants import ITEM_KEYS
from .helpers import safe_encode, safe_decode, bigint_to_int, int_to_bigint, StableDict from .helpers import safe_encode, safe_decode
from .helpers import bigint_to_int, int_to_bigint
from .helpers import StableDict
class PropDict: class PropDict:

View file

@ -1,20 +1,27 @@
from binascii import a2b_base64, b2a_base64, hexlify
import configparser import configparser
import getpass import getpass
import os import os
import sys import sys
import textwrap import textwrap
from hmac import compare_digest from binascii import a2b_base64, b2a_base64, hexlify
from hashlib import sha256, pbkdf2_hmac from hashlib import sha256, pbkdf2_hmac
from hmac import compare_digest
import msgpack
from .helpers import Chunk, IntegrityError, get_keys_dir, Error, yes, bin_to_hex, CompressionDecider2, CompressionSpec
from .logger import create_logger from .logger import create_logger
logger = create_logger() logger = create_logger()
from .constants import * # NOQA from .constants import * # NOQA
from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256
from .compress import Compressor, COMPR_BUFFER, get_compressor from .compress import Compressor, COMPR_BUFFER, get_compressor
import msgpack from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256
from .helpers import Chunk
from .helpers import Error, IntegrityError
from .helpers import yes
from .helpers import get_keys_dir
from .helpers import bin_to_hex
from .helpers import CompressionDecider2, CompressionSpec
PREFIX = b'\0' * 8 PREFIX = b'\0' * 8

View file

@ -3,7 +3,7 @@ import os
import socket import socket
import time import time
from borg.helpers import Error, ErrorWithTraceback from .helpers import Error, ErrorWithTraceback
ADD, REMOVE = 'add', 'remove' ADD, REMOVE = 'add', 'remove'
SHARED, EXCLUSIVE = 'shared', 'exclusive' SHARED, EXCLUSIVE = 'shared', 'exclusive'

View file

@ -1,5 +1,7 @@
import os import os
from .helpers import user2uid, group2gid, safe_decode, safe_encode
from .helpers import user2uid, group2gid
from .helpers import safe_decode, safe_encode
from .platform_posix import swidth from .platform_posix import swidth
API_VERSION = 3 API_VERSION = 3

View file

@ -1,5 +1,7 @@
import os import os
from .helpers import posix_acl_use_stored_uid_gid, safe_encode, safe_decode
from .helpers import posix_acl_use_stored_uid_gid
from .helpers import safe_encode, safe_decode
from .platform_posix import swidth from .platform_posix import swidth
API_VERSION = 3 API_VERSION = 3

View file

@ -3,7 +3,9 @@ import re
import resource import resource
import stat import stat
from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid, safe_decode, safe_encode from .helpers import posix_acl_use_stored_uid_gid
from .helpers import user2uid, group2gid
from .helpers import safe_decode, safe_encode
from .platform_base import SyncFile as BaseSyncFile from .platform_base import SyncFile as BaseSyncFile
from .platform_posix import swidth from .platform_posix import swidth

View file

@ -4,17 +4,19 @@ import logging
import os import os
import select import select
import shlex import shlex
from subprocess import Popen, PIPE
import sys import sys
import tempfile import tempfile
from subprocess import Popen, PIPE
from . import __version__
from .helpers import Error, IntegrityError, get_home_dir, sysinfo, bin_to_hex
from .repository import Repository
import msgpack import msgpack
from . import __version__
from .helpers import Error, IntegrityError
from .helpers import get_home_dir
from .helpers import sysinfo
from .helpers import bin_to_hex
from .repository import Repository
RPC_PROTOCOL_VERSION = 2 RPC_PROTOCOL_VERSION = 2
BUFSIZE = 10 * 1024 * 1024 BUFSIZE = 10 * 1024 * 1024

View file

@ -1,23 +1,26 @@
from configparser import ConfigParser
from binascii import unhexlify
from datetime import datetime
from itertools import islice
import errno import errno
import logging
logger = logging.getLogger(__name__)
import os import os
import shutil import shutil
import struct import struct
from binascii import unhexlify
from collections import defaultdict from collections import defaultdict
from configparser import ConfigParser
from datetime import datetime
from functools import partial from functools import partial
from itertools import islice
from zlib import crc32 from zlib import crc32
import msgpack import msgpack
import logging
logger = logging.getLogger(__name__)
from .constants import * # NOQA from .constants import * # NOQA
from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError, Location, ProgressIndicatorPercent, \
bin_to_hex
from .hashindex import NSIndex from .hashindex import NSIndex
from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError
from .helpers import Location
from .helpers import ProgressIndicatorPercent
from .helpers import bin_to_hex
from .locking import UpgradableLock, LockError, LockErrorT from .locking import UpgradableLock, LockError, LockErrorT
from .lrucache import LRUCache from .lrucache import LRUCache
from .platform import SyncFile, sync_dir from .platform import SyncFile, sync_dir

View file

@ -1,5 +1,5 @@
import re
import os import os
import re
def translate(pat): def translate(pat):

View file

@ -17,6 +17,10 @@ from unittest.mock import patch
from hashlib import sha256 from hashlib import sha256
import pytest import pytest
try:
import llfuse
except ImportError:
pass
from .. import xattr, helpers, platform from .. import xattr, helpers, platform
from ..archive import Archive, ChunkBuffer, ArchiveRecreater from ..archive import Archive, ChunkBuffer, ArchiveRecreater
@ -24,18 +28,15 @@ from ..archiver import Archiver
from ..cache import Cache from ..cache import Cache
from ..constants import * # NOQA from ..constants import * # NOQA
from ..crypto import bytes_to_long, num_aes_blocks from ..crypto import bytes_to_long, num_aes_blocks
from ..helpers import Chunk, Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex from ..helpers import Chunk, Manifest
from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
from ..helpers import bin_to_hex
from ..key import KeyfileKeyBase from ..key import KeyfileKeyBase
from ..remote import RemoteRepository, PathNotAllowed from ..remote import RemoteRepository, PathNotAllowed
from ..repository import Repository from ..repository import Repository
from . import has_lchflags, has_llfuse from . import has_lchflags, has_llfuse
from . import BaseTestCase, changedir, environment_variable from . import BaseTestCase, changedir, environment_variable
try:
import llfuse
except ImportError:
pass
src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

View file

@ -1,23 +1,27 @@
import hashlib import hashlib
import io
import logging import logging
from time import mktime, strptime
from datetime import datetime, timezone, timedelta
import os import os
import sys
from datetime import datetime, timezone, timedelta
from time import mktime, strptime, sleep
import pytest import pytest
import sys
import msgpack import msgpack
import msgpack.fallback import msgpack.fallback
import time
from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, clean_lines, \ from ..helpers import Location
prune_within, prune_split, get_cache_dir, get_keys_dir, is_slow_msgpack, \ from ..helpers import partial_format, format_file_size, format_timedelta
yes, TRUISH, FALSISH, DEFAULTISH, \ from ..helpers import make_path_safe, clean_lines
StableDict, int_to_bigint, bigint_to_int, bin_to_hex, parse_timestamp, ChunkerParams, Chunk, \ from ..helpers import prune_within, prune_split
ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \ from ..helpers import get_cache_dir, get_keys_dir
CompressionSpec, CompressionDecider1, CompressionDecider2, \ from ..helpers import is_slow_msgpack
PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, partial_format, ChunkIteratorFileWrapper from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH
from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex
from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams, Chunk
from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless
from ..helpers import load_excludes
from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2
from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern
from . import BaseTestCase, environment_variable, FakeInputs from . import BaseTestCase, environment_variable, FakeInputs
@ -115,7 +119,7 @@ class TestLocationWithoutEnv:
assert repr(Location('/some/path::archive{pid}')) == \ assert repr(Location('/some/path::archive{pid}')) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive{}')".format(test_pid) "Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive{}')".format(test_pid)
location_time1 = Location('/some/path::archive{now:%s}') location_time1 = Location('/some/path::archive{now:%s}')
time.sleep(1.1) sleep(1.1)
location_time2 = Location('/some/path::archive{now:%s}') location_time2 = Location('/some/path::archive{now:%s}')
assert location_time1.archive != location_time2.archive assert location_time1.archive != location_time2.archive

View file

@ -1,15 +1,15 @@
import getpass import getpass
import os
import re import re
import shutil
import tempfile import tempfile
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
import pytest import pytest
from ..crypto import bytes_to_long, num_aes_blocks from ..crypto import bytes_to_long, num_aes_blocks
from ..helpers import Location
from ..helpers import Chunk
from ..helpers import IntegrityError
from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex
from ..helpers import Location, Chunk, IntegrityError
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)

View file

@ -1,7 +1,9 @@
from ..lrucache import LRUCache
import pytest
from tempfile import TemporaryFile from tempfile import TemporaryFile
import pytest
from ..lrucache import LRUCache
class TestLRUCache: class TestLRUCache:

View file

@ -7,7 +7,8 @@ import tempfile
from unittest.mock import patch from unittest.mock import patch
from ..hashindex import NSIndex from ..hashindex import NSIndex
from ..helpers import Location, IntegrityError, InternalOSError from ..helpers import Location
from ..helpers import IntegrityError, InternalOSError
from ..locking import UpgradableLock, LockFailed from ..locking import UpgradableLock, LockFailed
from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint, handle_remote_line from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint, handle_remote_line
from ..repository import Repository, LoggedIO, MAGIC from ..repository import Repository, LoggedIO, MAGIC

View file

@ -1,14 +1,16 @@
import datetime import datetime
import logging
logger = logging.getLogger(__name__)
import os import os
import shutil import shutil
import time import time
from .helpers import get_home_dir, get_keys_dir, get_cache_dir, ProgressIndicatorPercent import logging
logger = logging.getLogger(__name__)
from .helpers import get_home_dir, get_keys_dir, get_cache_dir
from .helpers import ProgressIndicatorPercent
from .key import KeyfileKey, KeyfileNotFoundError
from .locking import UpgradableLock from .locking import UpgradableLock
from .repository import Repository, MAGIC from .repository import Repository, MAGIC
from .key import KeyfileKey, KeyfileNotFoundError
ATTIC_MAGIC = b'ATTICSEG' ATTIC_MAGIC = b'ATTICSEG'