From 2fc7abac3529257918a2b1930b6e0f43777f9955 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Feb 2021 23:32:51 +0100 Subject: [PATCH 1/3] copy: copy raw bytes of tree blobs This avoids problems when for some reason the JSON encoding changes. This also ensures forward compatibility with future restic versions which might e.g. add new fields to the tree metadata. --- cmd/restic/cmd_copy.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index cb8296d4b..d16cd1742 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -196,13 +196,16 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep // Do we already have this tree blob? if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) { - newTreeID, err := dstRepo.SaveTree(ctx, tree.Tree) + // copy raw tree bytes to avoid problems if the serialization changes + var err error + buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf) if err != nil { - return fmt.Errorf("SaveTree(%v) returned error %v", tree.ID.Str(), err) + return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err) } - // Assurance only. - if newTreeID != tree.ID { - return fmt.Errorf("SaveTree(%v) returned unexpected id %s", tree.ID.Str(), newTreeID.Str()) + + _, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false) + if err != nil { + return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err) } } From 427781928f73fc66ac87277ba63fd75343f4b5ec Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Feb 2021 23:30:02 +0100 Subject: [PATCH 2/3] copy: test that trees with unstable json encoding are properly copied --- cmd/restic/integration_test.go | 19 ++++++++++++++++++ cmd/restic/testdata/copy-unstable-json.tar.gz | Bin 0 -> 5235 bytes 2 files changed, 19 insertions(+) create mode 100644 cmd/restic/testdata/copy-unstable-json.tar.gz diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 7d198d336..ad1abcb91 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -799,6 +799,25 @@ func TestCopyIncremental(t *testing.T) { len(copiedSnapshotIDs), len(snapshotIDs)) } +func TestCopyUnstableJSON(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + env2, cleanup2 := withTestEnvironment(t) + defer cleanup2() + + // contains a symlink created using `ln -s '../i/'$'\355\246\361''d/samba' broken-symlink` + datafile := filepath.Join("testdata", "copy-unstable-json.tar.gz") + rtest.SetupTarTestFixture(t, env.base, datafile) + + testRunInit(t, env2.gopts) + testRunCopy(t, env.gopts, env2.gopts) + testRunCheck(t, env2.gopts) + + copiedSnapshotIDs := testRunList(t, "snapshots", env2.gopts) + rtest.Assert(t, 1 == len(copiedSnapshotIDs), "still expected %v snapshot, found %v", + 1, len(copiedSnapshotIDs)) +} + func TestInitCopyChunkerParams(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() diff --git a/cmd/restic/testdata/copy-unstable-json.tar.gz b/cmd/restic/testdata/copy-unstable-json.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..c2c79c90f62df638c32ba33533450128d89edebb GIT binary patch literal 5235 zcmXY#c|276|HsEtt|XPptz6TMlD0{*q;%4Tkz_5E#;xo$sU%9L%~g~=DlxJzp)A#n z4A<2tWy%^dl-*!3Gt8Npb3VV%`2OZGf1JlTXFi|L`}JO4&&QA>e%8pMin=6RGL9wvRh}cv6<%KQ z%QbKlh5ae2xMY8;=Rzn<$;?vy%>wx2E8{>O(0ul@C_mLJ@%HA2@n zePBD4)EhKH$!}!)=JRQ##BVLb#Rj_zX}5X`nZwfBndp5FiTtoQBEMdI_Gkuk?UJUR zV1<{&?=8a=@4JE^V*cDU-Fd@EvDXV^zM(kM!(QAlcco&xMNzt6%*SMFtl9!-XK;NI z!Pskwn)iv+!dBtJgkQ)Ixs9xl#(We~I`i=p%UZ>8@Z%L{^Rg&$Ey& zWWipPijl26a66KaMioacXh)BdFtWY)60Q-x3Rg{#qIPZ)i|2<>?JQ~We>iGF!o6G3 z4zOQtm=f2-;GxTW1jcrs0Cjs0*g%r#Isv=4M5KH@h$}(=V1Nxb2^tlnnfFPN1%~32 zmO^k5s4ZFp3bZrf?n@2D4G9xy5m+lH$Y}+vA!NZCBBLJ!0Ay(zVL?SFz<+WgkU5H; z0(El#0V8`cAD0DgWG_V8$Vy*f=w>o=rwLuz z0=`7}ICCPN4+}`7P^6iTVd#RqK`BU5*ksfK|XojVh}$H zdik=`SrvdtF1q2Sl(Gdx_rW7cm9qGZA!ZC|##RroaMbc9fl_A?Hqe$EW7tbse6rNvLnZF{y>z7`i@?h#UjYN$@2J zDuTBJAhraJ0s*91BoCvfLJ3U208sed6fj_T1?K97FW-eq^>*I*pAwfaF8&_yb9Eo!s2cPA92_fcyXDS zxs#YHMG=FI>g<(TKNsQ41K$H=Lm8a6}OFQgIbI*LMp zt`pjFVqi4@-U4-{ZNSd=^mxjxNm-o1tIa-dHHLeZl3?eqeL&F#>3Fc!|G@zO{5jA*oK!7jisl9?a_~z zN^c9TS|lf}6EcncuCffqJN;K^mzQwP9Q7-bR4To{ooahRf9QIhsU1Vm+3;uKA4*%Z zY=Zy#Z@oq2k0-2?=Twb`mQ@t89~(`LQKRYMyLtb(eS0@iKo9KX?5fc5KEk(sS6Fbq zKZ6+WJhuAOozq&uMQ2*`kG#3|<*@-?$6MX6VW=X?KHb}LjC%~v=&Ta`x}RO#eS*wS zJT~y!1?yHRpRQGnPRY;LIxPq}n&WXg8CftaPPKR*DT$P9EJnM}1PR|wKRc2$%hU!( zhYAWM-QvQk$|;YT-u_p{6BVxaUCl6`(mjRhk#98H(S+u=LJcQ_e-|L+iAj?}+Vx^i9rZ8|{zrx|^<|l0;?8SqWY-OI-hRr6Ig6ZG5 zZF+$d1GfHwUPEH{6M+E*kxA-4ti#WYt1bp9Y0MguyD1f+Wz`kt6@KhT6Iw2p%xX`a z@yJ$I+k>6?Hhk@m`C4Ddvrf&c-?Jo~Q2Lju3c6N#B0ZE|^Pjp(U#B+`c>FJ~Th=@IFWAa)eWs0uOjK8%W$)mQVTa4O ze#HU1cBM`haGp8o`_z74p&m8hA9+7e#k=P#-q3O0?nBEL&cpVix7YR~%-pb}njyw7 zMn=zP7d3Ogkl)OP@8v%F?!Y;%FFcQ~{oIxL3FCSuOAZz9!Ui^%hYa+udvB3=#c(sl zS40ckH^`x03wDh^X{RB+YM6V&f4tz)%y{{_b=i?1NKOM0jhz|aGoa;?`YOsyh1cG& zS#SK?^>Z#%e}+=Lv+>Tn(6|5kb`Be0?6=3x|3PC1vhIhZsOW0?Hf5v>u?IrWi4U8% zp2}zYCeB>IF11G`7(Hok52AM6w@G3z+{FC4c~gH_L14rp%H+Ey)v}S? z+tdWLK6o&>Ua(c~%c_(;*VoKocZJOwn^$=hZ#DJYxOzp%dH%Ri)iWh!oa`XL@I_nR z%uZb$|MEvhW6I{OTb^WEdpYQConpV9q$$DiTL;EK9|`)#IC>6FX6TQ4L?4d8NkqxQ z(I0RiLmO@d7#;+NOmjV$ltkx3lei4cTX(4&5or%7OmJ86E29Nx^T{SRMbS+-PZ|PQ z=PNkqO)z>;nQ5j82?K7l{7aNdka=+K{|7Rsf&38U8VRC@0Zr(&jk0n28ijsC$&%2T zUf~%eDQYhP-$AY1XV4rWv)FktAmcJ2!v;6vP8-IaMPwteO=`>r>7+pfdfyO;B@sv- zf$Ra$)R<3Vh*Q7<9zf?2IA#Jzy9gw3(8P05aIgx=C-^i!+1#gO&VXqN#pEVoOP!&7 zgf0dh&OQPtKIAetKum1P2N%E`1_ctO@LYpT=%^;}Efe4^1Od`;`C*{_2&n)6t!^mk zbJ8!wo{#telbcd4YCukdFoX=DF|uDvDYGC$Mj1HmnT#92(R}!}(o*=g zJd!#?#p6>2^|2z1X+0-uo)qSppDD24r-^6rE7qN9&uJ;k$woh z?+nz~AdNJQB+MWIxSFyXD4S+J1Lkrq@b~CdLLIvQLwhMDUv}>^8XEx}Peln@`!u{z zkU>cVI1mOc6GBWrIQf&HLswH<2N>gI`0!f^Is=n{p;SK84;r6{(MG5hLESj$T0>AH zf$1a2>m(Y>WyUH-EHNO{lAxUsB2_ZzLSmkR0kPVmUP8AOYr%{kTV31k-iNt$6EEt!KXf$jj9A}lmvrksIN zyDUKvN+DIm{%^mQ}a8Mo+^caK7G?`-)H698b8+f&%BDk6W#FRk({yA6w zs3Hg4m(K-|>-kI*h&lFncn@kW{~3*gf)kMnkHMdi(j~MDBq3ry3;7o=sg@-s^UwqG zV={=V!C+HKExbXbPAR17uZ%W{txOXsEfv zBk;5`SjA9To&{(OM9m=t(p5$R#5^_2n|zo#-6B{iV~iGzE$$JB~(H@aBxv}cFM&@TcRyMC>~cs9pXB+3bQioY{h zX=*gxHn~LY^#jRO_91%l*Mf?{JzW*%-c+~uavzk!=lh=wT?ZPn!>OsREnepxNAdc1 zHq^wcSG)ux*4pxUnC!|5sGWHfeFa^Qz9o0P?!doB>aLRb5yA3YH17sy%ow5GpOUT0|CU$0%xrI>q4 z26$5^hnVY$!{`u38G#h+mJq(P*5~r^Em~$~I_66&lc2r&YTadsPsQPtgn>eq1!veooQjKu);Im5dcBzN+_Z zEo~bT_K)AnHt#s1TZ_1R#kIz!XzD38ObrTdb$qCFtkGQV{`t}VaK#g$`ycgAK3|gF z9s68u_0pu+v_p3zmG3BgzH|NhvmT+{-9G6e>eG(Vq{SR#ZBx@No$b$~>s1f8>0)YS zl8D&aaPhoP`!(ZG%P6Obq{O_;<8ig>+be|+5Pp`(C8EcBo+bKN-;>x9C(LT- z9U#MlsXNxQ66{1?;@aD+rbgQHKEGUNLO)1+c}rG%GK0}am$)(ATu{~|Hu-m*R$o!* zQhZ+)U>WrS1nQriuOG_}Pi z_C>Ch-%xYbwe^18k3#pHaqo`36*zK^9{=A}qv+;MQJDh&$*;o1b zYfk$A4Q8)N+c3(ecZ@oa-iX2fMHHK6c)Oh!zu`z6!}b3OXy{&>y^!+p-t5$f(WpJ^ zM3u8fJ9>II*Od2lHH%$db}eMbIPYqzhLjA+v4eBpy=UqPAQkd-Od#o_n`J#>g zZaoK@Gr;AnX9BOEZQi8)Al_I4{0o}=`#Rr_p^YXN^96TQuZ6U#RD`SzYP8RZ~JN`HPieG^1igPM{)+W-H4MjwFrcNpiy Jq;2w~{{uIukgos$ literal 0 HcmV?d00001 From 88731d8c288fb2d2d880a6e1181fc2ce082122f1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 28 Feb 2021 00:22:06 +0100 Subject: [PATCH 3/3] Add changelog --- changelog/unreleased/issue-3267 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/issue-3267 diff --git a/changelog/unreleased/issue-3267 b/changelog/unreleased/issue-3267 new file mode 100644 index 000000000..88b7b7f63 --- /dev/null +++ b/changelog/unreleased/issue-3267 @@ -0,0 +1,11 @@ +Bugfix: `copy` failed to copy snapshots in rare cases + +The `copy` command could in rare cases fail with the error message `SaveTree(...) +returned unexpected id ...`. This has been fixed. + +On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 +target paths. Note that, due to limitations in the repository format, these are +not stored properly and should be avoided if possible. + +https://github.com/restic/restic/issues/3267 +https://github.com/restic/restic/pull/3310