From 7846ef2e6955e87282e944bcef8d38c5a31383a1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 6 Jun 2017 04:26:48 +0200 Subject: [PATCH] remove attic dependency of the tests, fixes #2505 attic.tar.gz contains a repo + corresponding keyfile - all the upgrader module tests need. .tar.gz because the .tar was 20x bigger. (cherry picked from commit fed5873e293188d00bd731e161e7878e7f083cca) --- borg/testsuite/attic.tar.gz | Bin 0 -> 2822 bytes borg/testsuite/upgrader.py | 113 +++++++++++++++++------------------- requirements.d/attic.txt | 5 -- tox.ini | 1 - 4 files changed, 53 insertions(+), 66 deletions(-) create mode 100644 borg/testsuite/attic.tar.gz delete mode 100644 requirements.d/attic.txt diff --git a/borg/testsuite/attic.tar.gz b/borg/testsuite/attic.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..5c0f8dfa43c5a297f25571a8c45904becad902dd GIT binary patch literal 2822 zcma)+dpy(oAICRxiEt>DTtoHyVa*YZEm@bjeAnag`}h3L_mB7c^ZC5r-}m?VdVk-Kk8!FhNLl^h zd642fSv}KpO1DFCJPBkPd<$J#R+!kHwcbfM?q~g$Ss#e~82Hxta%=7gKaiFi81;pD zrs(qVE3b>VDgBK+)8A3g3}@NiMV`h_tuH-G>W6y=S9u;mh|5pBN0+YL{MKUG9?3c| zA9%LQf%f>_UY5*ZRUliPau4FP!k%jBhq=Ti(T;kUi{Dx5M1P_gYWh#cx?uu6vJG!f zJ8I@&rspkCtbl7uIb8vo>6~W***)jvuh`zDCYu2h~Za1U!OYGbo0?X)f3m+MwjwD{BpEM?jFRtSZvd1 zC~K|oZAnklZmUFb6Q6 zNk`GCdnHJL)xf`ZqatC+@FYR@3x6y{tGq_Z?Tt zov>9(&l}%~T{Bo6SJy~KgNTBiT=BQSlXFyjp4c!EWA)p&?-fj>b5+IuYZ?w%>K{Aq z>ehtTNKmA-Kp|8HS_iHh0*=O}5+Yr_5ubMO={(aXf zll4t6?A?vgcgw0i9Mp)%XuTjc->#j$bu={e#UQTZNnR21q6ubj(75{`RBJ`Far9Uj z4EpAV&PU~tszWztVlqzLd!{CS@3ePLwvQ++U`TmAMD+d3+JmxZzh}wr`ZgZmdo}L5 zw_LDwQ_s~o*0K`v^)k_F9Bu=;_x!Qt7U8Htuevil2k-SFhEf#0{OLR|xM%LpN+1L2 zZyw}uqoHU;OL!f|Me*Z~=^RR|{FQ^_uFVjoMz^YZb~;~`Qn=agQ@alG&n*>feVEZr zRGN`Jdfn}_wMo#DZ9N#i5wUVA=4Jj`mEqez1&8k0xBzd^+z7#W(@xM#D99Kq`*T(H z=(uQ=NVV~PB6?t;IT&lzuHucfF(VoL+PNDr5$REBMz?pT+G>rq3>*VR@CS}w%{gK*SG^Qrdv0jgtyy#^C6e_GhF6_Wn!)@{rD z%iomCZzMz{Fjg-JKW{?vCqE9@gZlUTbgE(>8P>jQHGvVLs#5NQ=pP5+DQMcXc41gHQ1sQe?JDIrUX5dr{0}%F zxAm(6U(7dA?u8mD0= z<$K~WAQjPAPR9*#Ft(k*a9!lJCWgh$vUAv58QJTc^?qyDu37 ziy`KOn1px|grnFAdmtGEi(7*ubxefoB`nR%FOd}XZbh-7DIqE$ez**7pk^XG%+*rq zq~G!-YGj7*6GbUb`k-p7lsoBnBz0hMDHW>mPtkxpM3Q}eB-{Vh$k8>~P8es8Z>B^B z7Eig#j7mH7q2l>sVpeYLbQ!=2>O zTx4yOO6RJre0}4AyjJMG4Prt(>xUJ#NuBT!Md*ua zC&--maS(jWRPmK-ZqqAjR100>>35GOGPmx&k*TU9WiM=ozS9K=g4EK8vvfvvG%&zC~Fa`jgkO221eG^AV_CQms(u1(Mtt0oAiJyMipzWv*kIazr><%DORii zi=&rSl|Exua(DdML>E4>t+p7z7?2$@aszQ$ht+?|l5&{|^G2-zcV_oa&MkST)VYcP+quBzd zszkM&-ouIER({d{{jpNYcUwIJW?%DJS(*zWDE!v%yYJ5jB96xK%F+)dERj!Krr_Z0As<0C)>hif+=+EGiR}{#cnyJ}^Q2icEvzX( z-Gb?qVdOpFz-YWEO(Ka#mlVkU&F3lXVRani%b=_{DFcuA$!C$vPQt~|IL3`i%h2Ci n?JRK1HoFzjA_(*sNG_Wk literal 0 HcmV?d00001 diff --git a/borg/testsuite/upgrader.py b/borg/testsuite/upgrader.py index fb5b5e6f4..5ea47e06f 100644 --- a/borg/testsuite/upgrader.py +++ b/borg/testsuite/upgrader.py @@ -1,14 +1,8 @@ import os +import tarfile import pytest -try: - import attic.repository - import attic.key - import attic.helpers -except ImportError: - attic = None - from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey from ..helpers import get_keys_dir from ..key import KeyfileKey @@ -16,6 +10,28 @@ from ..archiver import UMASK_DEFAULT from ..repository import Repository +# tar with a repo and repo keyfile from attic +ATTIC_TAR = os.path.join(os.path.dirname(__file__), 'attic.tar.gz') + + +def untar(tarfname, path, what): + """ + extract tar archive to , all stuff starting with . + + return path to . + """ + + def files(members): + for tarinfo in members: + if tarinfo.name.startswith(what): + yield tarinfo + + with tarfile.open(tarfname, 'r') as tf: + tf.extractall(path, members=files(tf)) + + return os.path.join(path, what) + + def repo_valid(path): """ utility function to check if borg can open a repository @@ -47,15 +63,10 @@ def attic_repo(tmpdir): create an attic repo with some stuff in it :param tmpdir: path to the repository to be created - :returns: a attic.repository.Repository object + :returns: path to attic repository """ - attic_repo = attic.repository.Repository(str(tmpdir), create=True) - # throw some stuff in that repo, copied from `RepositoryTestCase.test1` - for x in range(100): - attic_repo.put(('%-32d' % x).encode('ascii'), b'SOMEDATA') - attic_repo.commit() - attic_repo.close() - return attic_repo + # there is some stuff in that repo, copied from `RepositoryTestCase.test1` + return untar(ATTIC_TAR, str(tmpdir), 'repo') @pytest.fixture(params=[True, False]) @@ -63,52 +74,36 @@ def inplace(request): return request.param -@pytest.mark.skipif(attic is None, reason='cannot find an attic install') -def test_convert_segments(tmpdir, attic_repo, inplace): +def test_convert_segments(attic_repo, inplace): """test segment conversion this will load the given attic repository, list all the segments then convert them one at a time. we need to close the repo before conversion otherwise we have errors from borg - :param tmpdir: a temporary directory to run the test in (builtin - fixture) :param attic_repo: a populated attic repository (fixture) """ + repo_path = attic_repo # check should fail because of magic number - assert not repo_valid(tmpdir) - repository = AtticRepositoryUpgrader(str(tmpdir), create=False) + assert not repo_valid(repo_path) + repository = AtticRepositoryUpgrader(repo_path, create=False) with repository: segments = [filename for i, filename in repository.io.segment_iterator()] repository.convert_segments(segments, dryrun=False, inplace=inplace) repository.convert_cache(dryrun=False) - assert repo_valid(tmpdir) - - -class MockArgs: - """ - mock attic location - - this is used to simulate a key location with a properly loaded - repository object to create a key file - """ - def __init__(self, path): - self.repository = attic.helpers.Location(path) + assert repo_valid(repo_path) @pytest.fixture() -def attic_key_file(attic_repo, tmpdir, monkeypatch): +def attic_key_file(tmpdir, monkeypatch): """ create an attic key file from the given repo, in the keys subdirectory of the given tmpdir - :param attic_repo: an attic.repository.Repository object (fixture - define above) :param tmpdir: a temporary directory (a builtin fixture) - :returns: the KeyfileKey object as returned by - attic.key.KeyfileKey.create() + :returns: path to key file """ - keys_dir = str(tmpdir.mkdir('keys')) + keys_dir = untar(ATTIC_TAR, str(tmpdir), 'keys') # we use the repo dir for the created keyfile, because we do # not want to clutter existing keyfiles @@ -119,44 +114,42 @@ def attic_key_file(attic_repo, tmpdir, monkeypatch): # about anyways. in real runs, the original key will be retained. monkeypatch.setenv('BORG_KEYS_DIR', keys_dir) monkeypatch.setenv('ATTIC_PASSPHRASE', 'test') - return attic.key.KeyfileKey.create(attic_repo, - MockArgs(keys_dir)) + + return os.path.join(keys_dir, 'repo') -@pytest.mark.skipif(attic is None, reason='cannot find an attic install') -def test_keys(tmpdir, attic_repo, attic_key_file): +def test_keys(attic_repo, attic_key_file): """test key conversion test that we can convert the given key to a properly formatted borg key. assumes that the ATTIC_KEYS_DIR and BORG_KEYS_DIR have been properly populated by the attic_key_file fixture. - :param tmpdir: a temporary directory (a builtin fixture) - :param attic_repo: an attic.repository.Repository object (fixture - define above) - :param attic_key_file: an attic.key.KeyfileKey (fixture created above) + :param attic_repo: path to an attic repository (fixture defined above) + :param attic_key_file: path to an attic key file (fixture defined above) """ - with AtticRepositoryUpgrader(str(tmpdir), create=False) as repository: + keyfile_path = attic_key_file + assert not key_valid(keyfile_path) # not upgraded yet + with AtticRepositoryUpgrader(attic_repo, create=False) as repository: keyfile = AtticKeyfileKey.find_key_file(repository) AtticRepositoryUpgrader.convert_keyfiles(keyfile, dryrun=False) - assert key_valid(attic_key_file.path) + assert key_valid(keyfile_path) -@pytest.mark.skipif(attic is None, reason='cannot find an attic install') -def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace): +def test_convert_all(attic_repo, attic_key_file, inplace): """test all conversion steps this runs everything. mostly redundant test, since everything is done above. yet we expect a NotImplementedError because we do not convert caches yet. - :param tmpdir: a temporary directory (a builtin fixture) - :param attic_repo: an attic.repository.Repository object (fixture - define above) - :param attic_key_file: an attic.key.KeyfileKey (fixture created above) + :param attic_repo: path to an attic repository (fixture defined above) + :param attic_key_file: path to an attic key file (fixture defined above) """ + repo_path = attic_repo + # check should fail because of magic number - assert not repo_valid(tmpdir) + assert not repo_valid(repo_path) def stat_segment(path): return os.stat(os.path.join(path, 'data', '0', '0')) @@ -164,8 +157,8 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace): def first_inode(path): return stat_segment(path).st_ino - orig_inode = first_inode(attic_repo.path) - with AtticRepositoryUpgrader(str(tmpdir), create=False) as repository: + orig_inode = first_inode(repo_path) + with AtticRepositoryUpgrader(repo_path, create=False) as repository: # replicate command dispatch, partly os.umask(UMASK_DEFAULT) backup = repository.upgrade(dryrun=False, inplace=inplace) @@ -179,8 +172,8 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace): # permissions, which is wrong assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0 - assert key_valid(attic_key_file.path) - assert repo_valid(tmpdir) + assert key_valid(attic_key_file) + assert repo_valid(repo_path) def test_hardlink(tmpdir, inplace): diff --git a/requirements.d/attic.txt b/requirements.d/attic.txt deleted file mode 100644 index b5068ffd4..000000000 --- a/requirements.d/attic.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Please note: -# attic only builds using OpenSSL 1.0.x, it can not be installed using OpenSSL >= 1.1.0. -# If attic is not installed, our unit tests will just skip the tests that require attic. -attic - diff --git a/tox.ini b/tox.ini index 8a73c41cc..aae13befd 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ envlist = py{34,35,36},flake8 changedir = {toxworkdir} deps = -rrequirements.d/development.txt - -rrequirements.d/attic.txt -rrequirements.d/fuse.txt commands = py.test -n {env:XDISTN:4} -rs --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: