mirror of
https://github.com/borgbackup/borg.git
synced 2024-12-23 00:07:38 +00:00
make item native code
This makes an surprisingly large difference. Test case: ~70000 empty files. (Ie. little data shoveling, lots of metadata shoveling). Before: 9.1 seconds +- 0.1 seconds. After: 8.4 seconds +- 0.1 seconds.). That's a huge win for changing a few lines. I'd expect that this improves performance in almost all areas that touch the items (list, delete, prune).
This commit is contained in:
parent
e9d7f928e2
commit
b885841c39
3 changed files with 11 additions and 2 deletions
7
setup.py
7
setup.py
|
@ -50,6 +50,7 @@
|
||||||
crypto_source = 'src/borg/crypto.pyx'
|
crypto_source = 'src/borg/crypto.pyx'
|
||||||
chunker_source = 'src/borg/chunker.pyx'
|
chunker_source = 'src/borg/chunker.pyx'
|
||||||
hashindex_source = 'src/borg/hashindex.pyx'
|
hashindex_source = 'src/borg/hashindex.pyx'
|
||||||
|
item_source = 'src/borg/item.pyx'
|
||||||
platform_posix_source = 'src/borg/platform/posix.pyx'
|
platform_posix_source = 'src/borg/platform/posix.pyx'
|
||||||
platform_linux_source = 'src/borg/platform/linux.pyx'
|
platform_linux_source = 'src/borg/platform/linux.pyx'
|
||||||
platform_darwin_source = 'src/borg/platform/darwin.pyx'
|
platform_darwin_source = 'src/borg/platform/darwin.pyx'
|
||||||
|
@ -60,6 +61,7 @@
|
||||||
crypto_source,
|
crypto_source,
|
||||||
chunker_source,
|
chunker_source,
|
||||||
hashindex_source,
|
hashindex_source,
|
||||||
|
item_source,
|
||||||
|
|
||||||
platform_posix_source,
|
platform_posix_source,
|
||||||
platform_linux_source,
|
platform_linux_source,
|
||||||
|
@ -83,6 +85,7 @@ def make_distribution(self):
|
||||||
'src/borg/crypto.c',
|
'src/borg/crypto.c',
|
||||||
'src/borg/chunker.c', 'src/borg/_chunker.c',
|
'src/borg/chunker.c', 'src/borg/_chunker.c',
|
||||||
'src/borg/hashindex.c', 'src/borg/_hashindex.c',
|
'src/borg/hashindex.c', 'src/borg/_hashindex.c',
|
||||||
|
'src/borg/item.c',
|
||||||
'src/borg/platform/posix.c',
|
'src/borg/platform/posix.c',
|
||||||
'src/borg/platform/linux.c',
|
'src/borg/platform/linux.c',
|
||||||
'src/borg/platform/freebsd.c',
|
'src/borg/platform/freebsd.c',
|
||||||
|
@ -99,6 +102,7 @@ def __init__(self, *args, **kwargs):
|
||||||
crypto_source = crypto_source.replace('.pyx', '.c')
|
crypto_source = crypto_source.replace('.pyx', '.c')
|
||||||
chunker_source = chunker_source.replace('.pyx', '.c')
|
chunker_source = chunker_source.replace('.pyx', '.c')
|
||||||
hashindex_source = hashindex_source.replace('.pyx', '.c')
|
hashindex_source = hashindex_source.replace('.pyx', '.c')
|
||||||
|
item_source = item_source.replace('.pyx', '.c')
|
||||||
platform_posix_source = platform_posix_source.replace('.pyx', '.c')
|
platform_posix_source = platform_posix_source.replace('.pyx', '.c')
|
||||||
platform_linux_source = platform_linux_source.replace('.pyx', '.c')
|
platform_linux_source = platform_linux_source.replace('.pyx', '.c')
|
||||||
platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
|
platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
|
||||||
|
@ -358,7 +362,8 @@ def run(self):
|
||||||
Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
|
Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
|
||||||
Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
|
Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
|
||||||
Extension('borg.chunker', [chunker_source]),
|
Extension('borg.chunker', [chunker_source]),
|
||||||
Extension('borg.hashindex', [hashindex_source])
|
Extension('borg.hashindex', [hashindex_source]),
|
||||||
|
Extension('borg.item', [item_source]),
|
||||||
]
|
]
|
||||||
if sys.platform.startswith(('linux', 'freebsd', 'darwin')):
|
if sys.platform.startswith(('linux', 'freebsd', 'darwin')):
|
||||||
ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))
|
ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))
|
||||||
|
|
|
@ -86,7 +86,7 @@ class PlaceholderError(Error):
|
||||||
|
|
||||||
|
|
||||||
def check_extension_modules():
|
def check_extension_modules():
|
||||||
from . import platform, compress
|
from . import platform, compress, item
|
||||||
if hashindex.API_VERSION != 4:
|
if hashindex.API_VERSION != 4:
|
||||||
raise ExtensionModuleError
|
raise ExtensionModuleError
|
||||||
if chunker.API_VERSION != 2:
|
if chunker.API_VERSION != 2:
|
||||||
|
@ -97,6 +97,8 @@ def check_extension_modules():
|
||||||
raise ExtensionModuleError
|
raise ExtensionModuleError
|
||||||
if platform.API_VERSION != platform.OS_API_VERSION != 5:
|
if platform.API_VERSION != platform.OS_API_VERSION != 5:
|
||||||
raise ExtensionModuleError
|
raise ExtensionModuleError
|
||||||
|
if item.API_VERSION != 1:
|
||||||
|
raise ExtensionModuleError
|
||||||
|
|
||||||
|
|
||||||
ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts')
|
ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts')
|
||||||
|
|
|
@ -3,6 +3,8 @@
|
||||||
from .helpers import bigint_to_int, int_to_bigint
|
from .helpers import bigint_to_int, int_to_bigint
|
||||||
from .helpers import StableDict
|
from .helpers import StableDict
|
||||||
|
|
||||||
|
API_VERSION = 1
|
||||||
|
|
||||||
|
|
||||||
class PropDict:
|
class PropDict:
|
||||||
"""
|
"""
|
Loading…
Reference in a new issue