1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-23 00:07:38 +00:00

make item native code

This makes an surprisingly large difference. Test case: ~70000 empty files.
(Ie. little data shoveling, lots of metadata shoveling). Before: 9.1 seconds
+- 0.1 seconds. After: 8.4 seconds +- 0.1 seconds.). That's a huge
win for changing a few lines.

I'd expect that this improves performance in almost all areas that touch
the items (list, delete, prune).
This commit is contained in:
Marian Beermann 2016-11-30 12:43:28 +01:00
parent e9d7f928e2
commit b885841c39
3 changed files with 11 additions and 2 deletions

View file

@ -50,6 +50,7 @@
crypto_source = 'src/borg/crypto.pyx' crypto_source = 'src/borg/crypto.pyx'
chunker_source = 'src/borg/chunker.pyx' chunker_source = 'src/borg/chunker.pyx'
hashindex_source = 'src/borg/hashindex.pyx' hashindex_source = 'src/borg/hashindex.pyx'
item_source = 'src/borg/item.pyx'
platform_posix_source = 'src/borg/platform/posix.pyx' platform_posix_source = 'src/borg/platform/posix.pyx'
platform_linux_source = 'src/borg/platform/linux.pyx' platform_linux_source = 'src/borg/platform/linux.pyx'
platform_darwin_source = 'src/borg/platform/darwin.pyx' platform_darwin_source = 'src/borg/platform/darwin.pyx'
@ -60,6 +61,7 @@
crypto_source, crypto_source,
chunker_source, chunker_source,
hashindex_source, hashindex_source,
item_source,
platform_posix_source, platform_posix_source,
platform_linux_source, platform_linux_source,
@ -83,6 +85,7 @@ def make_distribution(self):
'src/borg/crypto.c', 'src/borg/crypto.c',
'src/borg/chunker.c', 'src/borg/_chunker.c', 'src/borg/chunker.c', 'src/borg/_chunker.c',
'src/borg/hashindex.c', 'src/borg/_hashindex.c', 'src/borg/hashindex.c', 'src/borg/_hashindex.c',
'src/borg/item.c',
'src/borg/platform/posix.c', 'src/borg/platform/posix.c',
'src/borg/platform/linux.c', 'src/borg/platform/linux.c',
'src/borg/platform/freebsd.c', 'src/borg/platform/freebsd.c',
@ -99,6 +102,7 @@ def __init__(self, *args, **kwargs):
crypto_source = crypto_source.replace('.pyx', '.c') crypto_source = crypto_source.replace('.pyx', '.c')
chunker_source = chunker_source.replace('.pyx', '.c') chunker_source = chunker_source.replace('.pyx', '.c')
hashindex_source = hashindex_source.replace('.pyx', '.c') hashindex_source = hashindex_source.replace('.pyx', '.c')
item_source = item_source.replace('.pyx', '.c')
platform_posix_source = platform_posix_source.replace('.pyx', '.c') platform_posix_source = platform_posix_source.replace('.pyx', '.c')
platform_linux_source = platform_linux_source.replace('.pyx', '.c') platform_linux_source = platform_linux_source.replace('.pyx', '.c')
platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c') platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
@ -358,7 +362,8 @@ def run(self):
Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.chunker', [chunker_source]), Extension('borg.chunker', [chunker_source]),
Extension('borg.hashindex', [hashindex_source]) Extension('borg.hashindex', [hashindex_source]),
Extension('borg.item', [item_source]),
] ]
if sys.platform.startswith(('linux', 'freebsd', 'darwin')): if sys.platform.startswith(('linux', 'freebsd', 'darwin')):
ext_modules.append(Extension('borg.platform.posix', [platform_posix_source])) ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))

View file

@ -86,7 +86,7 @@ class PlaceholderError(Error):
def check_extension_modules(): def check_extension_modules():
from . import platform, compress from . import platform, compress, item
if hashindex.API_VERSION != 4: if hashindex.API_VERSION != 4:
raise ExtensionModuleError raise ExtensionModuleError
if chunker.API_VERSION != 2: if chunker.API_VERSION != 2:
@ -97,6 +97,8 @@ def check_extension_modules():
raise ExtensionModuleError raise ExtensionModuleError
if platform.API_VERSION != platform.OS_API_VERSION != 5: if platform.API_VERSION != platform.OS_API_VERSION != 5:
raise ExtensionModuleError raise ExtensionModuleError
if item.API_VERSION != 1:
raise ExtensionModuleError
ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts') ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts')

View file

@ -3,6 +3,8 @@
from .helpers import bigint_to_int, int_to_bigint from .helpers import bigint_to_int, int_to_bigint
from .helpers import StableDict from .helpers import StableDict
API_VERSION = 1
class PropDict: class PropDict:
""" """