restic/checker/repacker.go

164 lines
4.2 KiB
Go
Raw Normal View History

2015-07-25 12:20:02 +00:00
package checker
import (
"errors"
"github.com/restic/restic/backend"
"github.com/restic/restic/debug"
"github.com/restic/restic/repository"
)
// Repacker extracts still used blobs from packs with unused blobs and creates
// new packs.
type Repacker struct {
2015-11-01 21:45:52 +00:00
unusedBlobs backend.IDSet
2015-11-02 18:28:30 +00:00
repo *repository.Repository
2015-07-25 12:20:02 +00:00
}
// NewRepacker returns a new repacker that (when Repack() in run) cleans up the
// repository and creates new packs and indexs so that all blobs in unusedBlobs
// aren't used any more.
2015-11-02 18:28:30 +00:00
func NewRepacker(repo *repository.Repository, unusedBlobs backend.IDSet) *Repacker {
2015-07-25 12:20:02 +00:00
return &Repacker{
2015-11-02 18:28:30 +00:00
repo: repo,
2015-07-25 12:20:02 +00:00
unusedBlobs: unusedBlobs,
}
}
// Repack runs the process of finding still used blobs in packs with unused
// blobs, extracts them and creates new packs with just the still-in-use blobs.
func (r *Repacker) Repack() error {
debug.Log("Repacker.Repack", "searching packs for %v", r.unusedBlobs)
2015-11-01 21:45:52 +00:00
2015-11-02 18:28:30 +00:00
unneededPacks, err := FindPacksForBlobs(r.repo, r.unusedBlobs)
2015-07-25 12:20:02 +00:00
if err != nil {
return err
}
2015-11-02 18:28:30 +00:00
debug.Log("Repacker.Repack", "found packs: %v", unneededPacks)
2015-07-25 12:20:02 +00:00
2015-11-02 18:28:30 +00:00
blobs, err := FindBlobsForPacks(r.repo, unneededPacks)
2015-11-01 21:57:54 +00:00
if err != nil {
return err
}
debug.Log("Repacker.Repack", "found blobs: %v", blobs)
for id := range r.unusedBlobs {
debug.Log("Repacker.Repack", "remove unused blob %v", id.Str())
blobs.Delete(id)
}
debug.Log("Repacker.Repack", "need to repack blobs: %v", blobs)
2015-11-02 18:28:30 +00:00
err = RepackBlobs(r.repo, r.repo, blobs)
2015-11-01 21:57:54 +00:00
if err != nil {
return err
}
2015-11-02 18:28:30 +00:00
debug.Log("Repacker.Repack", "remove unneeded packs: %v", unneededPacks)
for packID := range unneededPacks {
err = r.repo.Backend().Remove(backend.Data, packID.String())
if err != nil {
return err
}
}
debug.Log("Repacker.Repack", "rebuild index, unneeded packs: %v", unneededPacks)
2015-11-02 18:28:30 +00:00
idx, err := r.repo.Index().RebuildIndex(unneededPacks)
newIndexID, err := repository.SaveIndex(r.repo, idx)
debug.Log("Repacker.Repack", "saved new index at %v, err %v", newIndexID.Str(), err)
2015-11-02 18:28:30 +00:00
if err != nil {
return err
}
debug.Log("Repacker.Repack", "remove old indexes: %v", idx.Supersedes())
for _, id := range idx.Supersedes() {
err = r.repo.Backend().Remove(backend.Index, id.String())
2015-11-01 21:57:54 +00:00
if err != nil {
2015-11-02 18:28:30 +00:00
debug.Log("Repacker.Repack", "error removing index %v: %v", id.Str(), err)
2015-11-01 21:57:54 +00:00
return err
}
2015-11-02 18:28:30 +00:00
debug.Log("Repacker.Repack", "removed index %v", id.Str())
2015-11-01 21:57:54 +00:00
}
2015-07-25 12:20:02 +00:00
return nil
}
2015-11-01 21:45:52 +00:00
// FindPacksForBlobs returns the set of packs that contain the blobs.
func FindPacksForBlobs(repo *repository.Repository, blobs backend.IDSet) (backend.IDSet, error) {
2015-07-25 12:20:02 +00:00
packs := backend.NewIDSet()
idx := repo.Index()
2015-11-01 21:45:52 +00:00
for id := range blobs {
2015-07-25 12:20:02 +00:00
blob, err := idx.Lookup(id)
if err != nil {
return nil, err
}
packs.Insert(blob.PackID)
}
return packs, nil
}
2015-11-01 21:45:52 +00:00
// FindBlobsForPacks returns the set of blobs contained in a pack of packs.
func FindBlobsForPacks(repo *repository.Repository, packs backend.IDSet) (backend.IDSet, error) {
blobs := backend.NewIDSet()
for packID := range packs {
for _, packedBlob := range repo.Index().ListPack(packID) {
blobs.Insert(packedBlob.ID)
}
}
return blobs, nil
}
2015-07-25 12:20:02 +00:00
// repackBlob loads a single blob from src and saves it in dst.
func repackBlob(src, dst *repository.Repository, id backend.ID) error {
blob, err := src.Index().Lookup(id)
if err != nil {
return err
}
debug.Log("RepackBlobs", "repacking blob %v, len %v", id.Str(), blob.PlaintextLength())
buf := make([]byte, 0, blob.PlaintextLength())
buf, err = src.LoadBlob(blob.Type, id, buf)
if err != nil {
return err
}
if uint(len(buf)) != blob.PlaintextLength() {
debug.Log("RepackBlobs", "repack blob %v: len(buf) isn't equal to length: %v = %v", id.Str(), len(buf), blob.PlaintextLength())
return errors.New("LoadBlob returned wrong data, len() doesn't match")
}
_, err = dst.SaveAndEncrypt(blob.Type, buf, &id)
if err != nil {
return err
}
return nil
}
// RepackBlobs reads all blobs in blobIDs from src and saves them into new pack
// files in dst. Source and destination repo may be the same.
2015-11-01 21:57:54 +00:00
func RepackBlobs(src, dst *repository.Repository, blobIDs backend.IDSet) (err error) {
for id := range blobIDs {
2015-07-25 12:20:02 +00:00
err = repackBlob(src, dst, id)
if err != nil {
return err
}
}
err = dst.Flush()
if err != nil {
return err
}
return nil
}