mirror of
https://github.com/restic/restic.git
synced 2024-12-23 08:16:36 +00:00
rebuild-index: handle not yet indexed packs
This commit is contained in:
parent
88849c06a6
commit
efbce9f0fa
2 changed files with 86 additions and 21 deletions
|
@ -1,8 +1,13 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
|
||||||
"github.com/restic/restic/backend"
|
"github.com/restic/restic/backend"
|
||||||
"github.com/restic/restic/debug"
|
"github.com/restic/restic/debug"
|
||||||
|
"github.com/restic/restic/pack"
|
||||||
"github.com/restic/restic/repository"
|
"github.com/restic/restic/repository"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -54,6 +59,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
|
||||||
debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
|
debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
|
||||||
|
|
||||||
combinedIndex := repository.NewIndex()
|
combinedIndex := repository.NewIndex()
|
||||||
|
packsDone := backend.NewIDSet()
|
||||||
|
|
||||||
i := 0
|
i := 0
|
||||||
for indexID := range indexIDs {
|
for indexID := range indexIDs {
|
||||||
|
@ -69,6 +75,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
|
||||||
|
|
||||||
for packedBlob := range idx.Each(done) {
|
for packedBlob := range idx.Each(done) {
|
||||||
combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
|
combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
|
||||||
|
packsDone.Insert(packedBlob.PackID)
|
||||||
}
|
}
|
||||||
|
|
||||||
combinedIndex.AddToSupersedes(indexID)
|
combinedIndex.AddToSupersedes(indexID)
|
||||||
|
@ -102,6 +109,64 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cmd.global.Printf("checking for additional packs\n")
|
||||||
|
newPacks := 0
|
||||||
|
for packID := range cmd.repo.List(backend.Data, done) {
|
||||||
|
if packsDone.Has(packID) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
|
||||||
|
newPacks++
|
||||||
|
|
||||||
|
rd, err := cmd.repo.Backend().GetReader(backend.Data, packID.String(), 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
debug.Log("RebuildIndex.RebuildIndex", "GetReader returned error: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var readSeeker io.ReadSeeker
|
||||||
|
if r, ok := rd.(io.ReadSeeker); ok {
|
||||||
|
debug.Log("RebuildIndex.RebuildIndex", "reader is seekable")
|
||||||
|
readSeeker = r
|
||||||
|
} else {
|
||||||
|
debug.Log("RebuildIndex.RebuildIndex", "reader is not seekable, loading contents to ram")
|
||||||
|
buf, err := ioutil.ReadAll(rd)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
readSeeker = bytes.NewReader(buf)
|
||||||
|
}
|
||||||
|
|
||||||
|
up, err := pack.NewUnpacker(cmd.repo.Key(), readSeeker)
|
||||||
|
if err != nil {
|
||||||
|
debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, blob := range up.Entries {
|
||||||
|
debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob)
|
||||||
|
combinedIndex.Store(blob.Type, blob.ID, packID, blob.Offset, blob.Length)
|
||||||
|
}
|
||||||
|
|
||||||
|
if repository.IndexFull(combinedIndex) {
|
||||||
|
combinedIndex, err = cmd.storeIndex(combinedIndex)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if combinedIndex.Length() > 0 {
|
||||||
|
combinedIndex, err = cmd.storeIndex(combinedIndex)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.global.Printf("added %d packs to the index\n", newPacks)
|
||||||
|
|
||||||
debug.Log("RebuildIndex.RebuildIndex", "done")
|
debug.Log("RebuildIndex.RebuildIndex", "done")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
42
pack/pack.go
42
pack/pack.go
|
@ -233,7 +233,7 @@ type Unpacker struct {
|
||||||
|
|
||||||
// NewUnpacker returns a pointer to Unpacker which can be used to read
|
// NewUnpacker returns a pointer to Unpacker which can be used to read
|
||||||
// individual Blobs from a pack.
|
// individual Blobs from a pack.
|
||||||
func NewUnpacker(k *crypto.Key, entries []Blob, rd io.ReadSeeker) (*Unpacker, error) {
|
func NewUnpacker(k *crypto.Key, rd io.ReadSeeker) (*Unpacker, error) {
|
||||||
var err error
|
var err error
|
||||||
ls := binary.Size(uint32(0))
|
ls := binary.Size(uint32(0))
|
||||||
|
|
||||||
|
@ -261,28 +261,28 @@ func NewUnpacker(k *crypto.Key, entries []Blob, rd io.ReadSeeker) (*Unpacker, er
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if entries == nil {
|
var entries []Blob
|
||||||
pos := uint(0)
|
|
||||||
for {
|
|
||||||
e := headerEntry{}
|
|
||||||
err = binary.Read(hrd, binary.LittleEndian, &e)
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
pos := uint(0)
|
||||||
return nil, err
|
for {
|
||||||
}
|
e := headerEntry{}
|
||||||
|
err = binary.Read(hrd, binary.LittleEndian, &e)
|
||||||
entries = append(entries, Blob{
|
if err == io.EOF {
|
||||||
Type: e.Type,
|
break
|
||||||
Length: uint(e.Length),
|
|
||||||
ID: e.ID,
|
|
||||||
Offset: pos,
|
|
||||||
})
|
|
||||||
|
|
||||||
pos += uint(e.Length)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
entries = append(entries, Blob{
|
||||||
|
Type: e.Type,
|
||||||
|
Length: uint(e.Length),
|
||||||
|
ID: e.ID,
|
||||||
|
Offset: pos,
|
||||||
|
})
|
||||||
|
|
||||||
|
pos += uint(e.Length)
|
||||||
}
|
}
|
||||||
|
|
||||||
p := &Unpacker{
|
p := &Unpacker{
|
||||||
|
|
Loading…
Reference in a new issue