Merge pull request #3148 from aawsome/simplify-index-rebuild

rebuild-index: code simplification

cmd/restic/cmd_prune.go

@@ -479,11 +479,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile)
 	}
 
-	packsAddedByRepack := 0
 	if len(repackPacks) != 0 {
-		// Remember the number of unique packs before repacking
-		packsBeforeRepacking := len(repo.Index().Packs())
-
 		Verbosef("repacking packs\n")
 		bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked")
 		_, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar)
@@ -492,18 +488,12 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			return err
 		}
 
-		// Since repacking will only add new packs, we can calculate the number
-		// of packs like this:
-		packsAddedByRepack = len(repo.Index().Packs()) - packsBeforeRepacking
-
 		// Also remove repacked packs
 		removePacks.Merge(repackPacks)
 	}
 
 	if len(removePacks) != 0 {
-		totalpacks := int(stats.packs.used+stats.packs.partlyUsed+stats.packs.unused) -
+		err = rebuildIndexFiles(gopts, repo, removePacks, nil)
-			len(removePacks) + packsAddedByRepack
-		err = rebuildIndexFiles(gopts, repo, removePacks, nil, uint64(totalpacks))
 		if err != nil {
 			return err
 		}
@@ -516,12 +506,13 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	return nil
 }
 
-func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, packcount uint64) error {
+func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
 	Verbosef("rebuilding index\n")
 
+	idx := (repo.Index()).(*repository.MasterIndex)
+	packcount := uint64(len(idx.Packs(removePacks)))
 	bar := newProgressMax(!gopts.Quiet, packcount, "packs processed")
-	obsoleteIndexes, err := (repo.Index()).(*repository.MasterIndex).
+	obsoleteIndexes, err := idx.Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
-		Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
 	bar.Done()
 	if err != nil {
 		return err

cmd/restic/cmd_rebuild_index.go

@@ -59,11 +59,11 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 
 	var obsoleteIndexes restic.IDs
 	packSizeFromList := make(map[restic.ID]int64)
+	packSizeFromIndex := make(map[restic.ID]int64)
 	removePacks := restic.NewIDSet()
-	totalPacks := 0
 
 	if opts.ReadAllPacks {
-		// get old index files
+		// get list of old index files but start with empty index
 		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
 			obsoleteIndexes = append(obsoleteIndexes, id)
 			return nil
@@ -71,46 +71,33 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		if err != nil {
 			return err
 		}
-
-		Verbosef("finding pack files in repo...\n")
-		err = repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
-			packSizeFromList[id] = size
-			removePacks.Insert(id)
-			totalPacks++
-			return nil
-		})
-		if err != nil {
-			return err
-		}
 	} else {
 		Verbosef("loading indexes...\n")
 		err := repo.LoadIndex(gopts.ctx)
 		if err != nil {
 			return err
 		}
+		packSizeFromIndex = repo.Index().PackSize(ctx, false)
+	}
 
-		Verbosef("getting pack files to read...\n")
+	Verbosef("getting pack files to read...\n")
-		packSizeFromIndex := repo.Index().PackSize(ctx, false)
+	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
-
+		size, ok := packSizeFromIndex[id]
-		err = repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
+		if !ok || size != packSize {
-			size, ok := packSizeFromIndex[id]
+			// Pack was not referenced in index or size does not match
-			if !ok || size != packSize {
+			packSizeFromList[id] = packSize
-				// Pack was not referenced in index or size does not match
-				packSizeFromList[id] = packSize
-				removePacks.Insert(id)
-			}
-			totalPacks++
-			delete(packSizeFromIndex, id)
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-		for id := range packSizeFromIndex {
-			// forget pack files that are referenced in the index but do not exist
-			// when rebuilding the index
 			removePacks.Insert(id)
 		}
+		delete(packSizeFromIndex, id)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	for id := range packSizeFromIndex {
+		// forget pack files that are referenced in the index but do not exist
+		// when rebuilding the index
+		removePacks.Insert(id)
 	}
 
 	if len(packSizeFromList) > 0 {
@@ -123,11 +110,10 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 
 		for _, id := range invalidFiles {
 			Verboseff("skipped incomplete pack file: %v\n", id)
-			totalPacks--
 		}
 	}
 
-	err := rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes, uint64(totalPacks))
+	err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
 	if err != nil {
 		return err
 	}

internal/repository/master_index.go

@@ -100,13 +100,19 @@ func (mi *MasterIndex) Has(bh restic.BlobHandle) bool {
 }
 
 // Packs returns all packs that are covered by the index.
-func (mi *MasterIndex) Packs() restic.IDSet {
+// If packBlacklist is given, those packs are only contained in the
+// resulting IDSet if they are contained in a non-final (newly written) index.
+func (mi *MasterIndex) Packs(packBlacklist restic.IDSet) restic.IDSet {
 	mi.idxMutex.RLock()
 	defer mi.idxMutex.RUnlock()
 
 	packs := restic.NewIDSet()
 	for _, idx := range mi.idx {
-		packs.Merge(idx.Packs())
+		idxPacks := idx.Packs()
+		if idx.final {
+			idxPacks = idxPacks.Sub(packBlacklist)
+		}
+		packs.Merge(idxPacks)
 	}
 
 	return packs

internal/restic/repository.go

@@ -65,7 +65,6 @@ type MasterIndex interface {
 	Has(BlobHandle) bool
 	Lookup(BlobHandle) []PackedBlob
 	Count(BlobType) uint
-	Packs() IDSet
 	PackSize(ctx context.Context, onlyHdr bool) map[ID]int64
 
 	// Each returns a channel that yields all blobs known to the index. When