mirror of
https://github.com/restic/restic.git
synced 2024-12-23 08:16:36 +00:00
rebuild-index: handle not yet indexed packs
This commit is contained in:
parent
88849c06a6
commit
efbce9f0fa
2 changed files with 86 additions and 21 deletions
|
@ -1,8 +1,13 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/debug"
|
||||
"github.com/restic/restic/pack"
|
||||
"github.com/restic/restic/repository"
|
||||
)
|
||||
|
||||
|
@ -54,6 +59,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
|
|||
debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
|
||||
|
||||
combinedIndex := repository.NewIndex()
|
||||
packsDone := backend.NewIDSet()
|
||||
|
||||
i := 0
|
||||
for indexID := range indexIDs {
|
||||
|
@ -69,6 +75,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
|
|||
|
||||
for packedBlob := range idx.Each(done) {
|
||||
combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
|
||||
packsDone.Insert(packedBlob.PackID)
|
||||
}
|
||||
|
||||
combinedIndex.AddToSupersedes(indexID)
|
||||
|
@ -102,6 +109,64 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
|
|||
}
|
||||
}
|
||||
|
||||
cmd.global.Printf("checking for additional packs\n")
|
||||
newPacks := 0
|
||||
for packID := range cmd.repo.List(backend.Data, done) {
|
||||
if packsDone.Has(packID) {
|
||||
continue
|
||||
}
|
||||
|
||||
debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
|
||||
newPacks++
|
||||
|
||||
rd, err := cmd.repo.Backend().GetReader(backend.Data, packID.String(), 0, 0)
|
||||
if err != nil {
|
||||
debug.Log("RebuildIndex.RebuildIndex", "GetReader returned error: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var readSeeker io.ReadSeeker
|
||||
if r, ok := rd.(io.ReadSeeker); ok {
|
||||
debug.Log("RebuildIndex.RebuildIndex", "reader is seekable")
|
||||
readSeeker = r
|
||||
} else {
|
||||
debug.Log("RebuildIndex.RebuildIndex", "reader is not seekable, loading contents to ram")
|
||||
buf, err := ioutil.ReadAll(rd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
readSeeker = bytes.NewReader(buf)
|
||||
}
|
||||
|
||||
up, err := pack.NewUnpacker(cmd.repo.Key(), readSeeker)
|
||||
if err != nil {
|
||||
debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str())
|
||||
return err
|
||||
}
|
||||
|
||||
for _, blob := range up.Entries {
|
||||
debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob)
|
||||
combinedIndex.Store(blob.Type, blob.ID, packID, blob.Offset, blob.Length)
|
||||
}
|
||||
|
||||
if repository.IndexFull(combinedIndex) {
|
||||
combinedIndex, err = cmd.storeIndex(combinedIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if combinedIndex.Length() > 0 {
|
||||
combinedIndex, err = cmd.storeIndex(combinedIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cmd.global.Printf("added %d packs to the index\n", newPacks)
|
||||
|
||||
debug.Log("RebuildIndex.RebuildIndex", "done")
|
||||
return nil
|
||||
}
|
||||
|
|
42
pack/pack.go
42
pack/pack.go
|
@ -233,7 +233,7 @@ type Unpacker struct {
|
|||
|
||||
// NewUnpacker returns a pointer to Unpacker which can be used to read
|
||||
// individual Blobs from a pack.
|
||||
func NewUnpacker(k *crypto.Key, entries []Blob, rd io.ReadSeeker) (*Unpacker, error) {
|
||||
func NewUnpacker(k *crypto.Key, rd io.ReadSeeker) (*Unpacker, error) {
|
||||
var err error
|
||||
ls := binary.Size(uint32(0))
|
||||
|
||||
|
@ -261,28 +261,28 @@ func NewUnpacker(k *crypto.Key, entries []Blob, rd io.ReadSeeker) (*Unpacker, er
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if entries == nil {
|
||||
pos := uint(0)
|
||||
for {
|
||||
e := headerEntry{}
|
||||
err = binary.Read(hrd, binary.LittleEndian, &e)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
var entries []Blob
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries = append(entries, Blob{
|
||||
Type: e.Type,
|
||||
Length: uint(e.Length),
|
||||
ID: e.ID,
|
||||
Offset: pos,
|
||||
})
|
||||
|
||||
pos += uint(e.Length)
|
||||
pos := uint(0)
|
||||
for {
|
||||
e := headerEntry{}
|
||||
err = binary.Read(hrd, binary.LittleEndian, &e)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries = append(entries, Blob{
|
||||
Type: e.Type,
|
||||
Length: uint(e.Length),
|
||||
ID: e.ID,
|
||||
Offset: pos,
|
||||
})
|
||||
|
||||
pos += uint(e.Length)
|
||||
}
|
||||
|
||||
p := &Unpacker{
|
||||
|
|
Loading…
Reference in a new issue