rebuild-index: handle not yet indexed packs

cmd/restic/cmd_rebuild_index.go

@@ -1,8 +1,13 @@
 package main
 
 import (
+	"bytes"
+	"io"
+	"io/ioutil"
+
 	"github.com/restic/restic/backend"
 	"github.com/restic/restic/debug"
+	"github.com/restic/restic/pack"
 	"github.com/restic/restic/repository"
 )
 
@@ -54,6 +59,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 	debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
 
 	combinedIndex := repository.NewIndex()
+	packsDone := backend.NewIDSet()
 
 	i := 0
 	for indexID := range indexIDs {
@@ -69,6 +75,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 
 		for packedBlob := range idx.Each(done) {
 			combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
+			packsDone.Insert(packedBlob.PackID)
 		}
 
 		combinedIndex.AddToSupersedes(indexID)
@@ -102,6 +109,64 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 		}
 	}
 
+	cmd.global.Printf("checking for additional packs\n")
+	newPacks := 0
+	for packID := range cmd.repo.List(backend.Data, done) {
+		if packsDone.Has(packID) {
+			continue
+		}
+
+		debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
+		newPacks++
+
+		rd, err := cmd.repo.Backend().GetReader(backend.Data, packID.String(), 0, 0)
+		if err != nil {
+			debug.Log("RebuildIndex.RebuildIndex", "GetReader returned error: %v", err)
+			return err
+		}
+
+		var readSeeker io.ReadSeeker
+		if r, ok := rd.(io.ReadSeeker); ok {
+			debug.Log("RebuildIndex.RebuildIndex", "reader is seekable")
+			readSeeker = r
+		} else {
+			debug.Log("RebuildIndex.RebuildIndex", "reader is not seekable, loading contents to ram")
+			buf, err := ioutil.ReadAll(rd)
+			if err != nil {
+				return err
+			}
+
+			readSeeker = bytes.NewReader(buf)
+		}
+
+		up, err := pack.NewUnpacker(cmd.repo.Key(), readSeeker)
+		if err != nil {
+			debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str())
+			return err
+		}
+
+		for _, blob := range up.Entries {
+			debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob)
+			combinedIndex.Store(blob.Type, blob.ID, packID, blob.Offset, blob.Length)
+		}
+
+		if repository.IndexFull(combinedIndex) {
+			combinedIndex, err = cmd.storeIndex(combinedIndex)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if combinedIndex.Length() > 0 {
+		combinedIndex, err = cmd.storeIndex(combinedIndex)
+		if err != nil {
+			return err
+		}
+	}
+
+	cmd.global.Printf("added %d packs to the index\n", newPacks)
+
 	debug.Log("RebuildIndex.RebuildIndex", "done")
 	return nil
 }

pack/pack.go

@@ -233,7 +233,7 @@ type Unpacker struct {
 
 // NewUnpacker returns a pointer to Unpacker which can be used to read
 // individual Blobs from a pack.
-func NewUnpacker(k *crypto.Key, entries []Blob, rd io.ReadSeeker) (*Unpacker, error) {
+func NewUnpacker(k *crypto.Key, rd io.ReadSeeker) (*Unpacker, error) {
 	var err error
 	ls := binary.Size(uint32(0))
 
@@ -261,28 +261,28 @@ func NewUnpacker(k *crypto.Key, entries []Blob, rd io.ReadSeeker) (*Unpacker, er
 		return nil, err
 	}
 
-	if entries == nil {
+	var entries []Blob
-		pos := uint(0)
-		for {
-			e := headerEntry{}
-			err = binary.Read(hrd, binary.LittleEndian, &e)
-			if err == io.EOF {
-				break
-			}
 
-			if err != nil {
+	pos := uint(0)
-				return nil, err
+	for {
-			}
+		e := headerEntry{}
-
+		err = binary.Read(hrd, binary.LittleEndian, &e)
-			entries = append(entries, Blob{
+		if err == io.EOF {
-				Type:   e.Type,
+			break
-				Length: uint(e.Length),
-				ID:     e.ID,
-				Offset: pos,
-			})
-
-			pos += uint(e.Length)
 		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		entries = append(entries, Blob{
+			Type:   e.Type,
+			Length: uint(e.Length),
+			ID:     e.ID,
+			Offset: pos,
+		})
+
+		pos += uint(e.Length)
 	}
 
 	p := &Unpacker{