package repository import ( "encoding/json" "errors" "fmt" "io" "sync" "time" "github.com/restic/restic/backend" "github.com/restic/restic/crypto" "github.com/restic/restic/debug" "github.com/restic/restic/pack" ) // Index holds a lookup table for id -> pack. type Index struct { m sync.Mutex pack map[backend.ID]indexEntry final bool // set to true for all indexes read from the backend ("finalized") supersedes backend.IDs created time.Time } type indexEntry struct { tpe pack.BlobType packID backend.ID offset uint length uint } // NewIndex returns a new index. func NewIndex() *Index { return &Index{ pack: make(map[backend.ID]indexEntry), created: time.Now(), } } func (idx *Index) store(t pack.BlobType, id backend.ID, pack backend.ID, offset, length uint) { idx.pack[id] = indexEntry{ tpe: t, packID: pack, offset: offset, length: length, } } // Final returns true iff the index is already written to the repository, it is // finalized. func (idx *Index) Final() bool { idx.m.Lock() defer idx.m.Unlock() return idx.final } const ( indexMinBlobs = 20 indexMaxBlobs = 2000 indexMinAge = 2 * time.Minute indexMaxAge = 15 * time.Minute ) // IndexFull returns true iff the index is "full enough" to be saved as a preliminary index. var IndexFull = func(idx *Index) bool { idx.m.Lock() defer idx.m.Unlock() debug.Log("Index.Full", "checking whether index %p is full", idx) packs := len(idx.pack) age := time.Now().Sub(idx.created) if age > indexMaxAge { debug.Log("Index.Full", "index %p is old enough", idx, age) return true } if packs < indexMinBlobs || age < indexMinAge { debug.Log("Index.Full", "index %p only has %d packs or is too young (%v)", idx, packs, age) return false } if packs > indexMaxBlobs { debug.Log("Index.Full", "index %p has %d packs", idx, packs) return true } debug.Log("Index.Full", "index %p is not full", idx) return false } // Store remembers the id and pack in the index. An existing entry will be // silently overwritten. func (idx *Index) Store(t pack.BlobType, id backend.ID, pack backend.ID, offset, length uint) { idx.m.Lock() defer idx.m.Unlock() if idx.final { panic("store new item in finalized index") } debug.Log("Index.Store", "pack %v contains id %v (%v), offset %v, length %v", pack.Str(), id.Str(), t, offset, length) idx.store(t, id, pack, offset, length) } // Lookup queries the index for the blob ID and returns a PackedBlob. func (idx *Index) Lookup(id backend.ID) (pb PackedBlob, err error) { idx.m.Lock() defer idx.m.Unlock() if p, ok := idx.pack[id]; ok { debug.Log("Index.Lookup", "id %v found in pack %v at %d, length %d", id.Str(), p.packID.Str(), p.offset, p.length) pb := PackedBlob{ Type: p.tpe, Length: p.length, ID: id, Offset: p.offset, PackID: p.packID, } return pb, nil } debug.Log("Index.Lookup", "id %v not found", id.Str()) return PackedBlob{}, fmt.Errorf("id %v not found in index", id) } // Has returns true iff the id is listed in the index. func (idx *Index) Has(id backend.ID) bool { _, err := idx.Lookup(id) if err == nil { return true } return false } // LookupSize returns the length of the cleartext content behind the // given id func (idx *Index) LookupSize(id backend.ID) (cleartextLength uint, err error) { blob, err := idx.Lookup(id) if err != nil { return 0, err } return blob.PlaintextLength(), nil } // Merge loads all items from other into idx. func (idx *Index) Merge(other *Index) { debug.Log("Index.Merge", "Merge index with %p", other) idx.m.Lock() defer idx.m.Unlock() for k, v := range other.pack { if _, ok := idx.pack[k]; ok { debug.Log("Index.Merge", "index already has key %v, updating", k.Str()) } idx.pack[k] = v } debug.Log("Index.Merge", "done merging index") } // Supersedes returns the list of indexes this index supersedes, if any. func (idx *Index) Supersedes() backend.IDs { return idx.supersedes } // AddToSupersedes adds the ids to the list of indexes superseded by this // index. If the index has already been finalized, an error is returned. func (idx *Index) AddToSupersedes(ids ...backend.ID) error { idx.m.Lock() defer idx.m.Unlock() if idx.final { return errors.New("index already finalized") } idx.supersedes = append(idx.supersedes, ids...) return nil } // PackedBlob is a blob already saved within a pack. type PackedBlob struct { Type pack.BlobType Length uint ID backend.ID Offset uint PackID backend.ID } func (pb PackedBlob) String() string { return fmt.Sprintf("