From 3541d06d076baa7ccc04a8e8576971e1429a0e73 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 16 Jul 2017 20:16:02 +0200
Subject: [PATCH] repo: Split packers for tree and data

The code now bundles tree blobs and data blobs into different pack
files, so we'll end up with pack files that either only contain data or
trees. This is in preparation to adding a cache (#1040), because
tree-only pack files can easily be cached later on.
---
 internal/repository/packer_manager.go      |  4 +-
 internal/repository/packer_manager_test.go |  2 +-
 internal/repository/repository.go          | 53 +++++++++++++++-------
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go
index 3b2734f53..aa1885cc1 100644
--- a/internal/repository/packer_manager.go
+++ b/internal/repository/packer_manager.go
@@ -50,7 +50,7 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager {
 
 // findPacker returns a packer for a new blob of size bytes. Either a new one is
 // created or one is returned that already has some blobs.
-func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
+func (r *packerManager) findPacker() (packer *Packer, err error) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
@@ -62,7 +62,7 @@ func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
 	}
 
 	// no suitable packer found, return new
-	debug.Log("create new pack for %d bytes", size)
+	debug.Log("create new pack")
 	tmpfile, err := fs.TempFile("", "restic-temp-pack-")
 	if err != nil {
 		return nil, errors.Wrap(err, "fs.TempFile")
diff --git a/internal/repository/packer_manager_test.go b/internal/repository/packer_manager_test.go
index efdd7ff43..aface097b 100644
--- a/internal/repository/packer_manager_test.go
+++ b/internal/repository/packer_manager_test.go
@@ -72,7 +72,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 		l := rnd.rand.Intn(1 << 20)
 		seed := rnd.rand.Int63()
 
-		packer, err := pm.findPacker(uint(l))
+		packer, err := pm.findPacker()
 		if err != nil {
 			t.Fatal(err)
 		}
diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 7623cde83..ae35f78c0 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -24,15 +24,17 @@ type Repository struct {
 	keyName string
 	idx     *MasterIndex
 
-	*packerManager
+	treePM *packerManager
+	dataPM *packerManager
 }
 
 // New returns a new repository with backend be.
 func New(be restic.Backend) *Repository {
 	repo := &Repository{
-		be:            be,
-		idx:           NewMasterIndex(),
-		packerManager: newPackerManager(be, nil),
+		be:     be,
+		idx:    NewMasterIndex(),
+		dataPM: newPackerManager(be, nil),
+		treePM: newPackerManager(be, nil),
 	}
 
 	return repo
@@ -180,7 +182,18 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	}
 
 	// find suitable packer and add blob
-	packer, err := r.findPacker(uint(len(ciphertext)))
+	var pm *packerManager
+
+	switch t {
+	case restic.TreeBlob:
+		pm = r.treePM
+	case restic.DataBlob:
+		pm = r.dataPM
+	default:
+		panic(fmt.Sprintf("invalid type: %v", t))
+	}
+
+	packer, err := pm.findPacker()
 	if err != nil {
 		return restic.ID{}, err
 	}
@@ -194,7 +207,7 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	// if the pack is not full enough, put back to the list
 	if packer.Size() < minPackSize {
 		debug.Log("pack is not full enough (%d bytes)", packer.Size())
-		r.insertPacker(packer)
+		pm.insertPacker(packer)
 		return *id, nil
 	}
 
@@ -238,18 +251,22 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 
 // Flush saves all remaining packs.
 func (r *Repository) Flush() error {
-	r.pm.Lock()
-	defer r.pm.Unlock()
+	for _, pm := range []*packerManager{r.dataPM, r.treePM} {
+		pm.pm.Lock()
 
-	debug.Log("manually flushing %d packs", len(r.packerManager.packers))
-
-	for _, p := range r.packerManager.packers {
-		err := r.savePacker(p)
-		if err != nil {
-			return err
+		debug.Log("manually flushing %d packs", len(pm.packers))
+		for _, p := range pm.packers {
+			err := r.savePacker(p)
+			if err != nil {
+				pm.pm.Unlock()
+				return err
+			}
 		}
+		pm.packers = pm.packers[:0]
+
+		pm.pm.Unlock()
 	}
-	r.packerManager.packers = r.packerManager.packers[:0]
+
 	return nil
 }
 
@@ -371,7 +388,8 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg, err = restic.LoadConfig(ctx, r)
 	return err
@@ -405,7 +423,8 @@ func (r *Repository) init(ctx context.Context, password string, cfg restic.Confi
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg = cfg
 	_, err = r.SaveJSONUnpacked(ctx, restic.ConfigFile, cfg)