From 33ce4e33aa49997064cb937dffc828b193e36e70 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 23 Nov 2014 22:26:01 +0100 Subject: [PATCH] Store maps in new subdir "maps" --- archiver.go | 11 ++++++++--- backend/interface.go | 1 + backend/local.go | 4 ++++ backend/local_test.go | 2 +- backend/sftp.go | 2 ++ bloblist.go | 12 ++++++++++++ cmd/khepri/cmd_cat.go | 17 ++++++++++++++++- contenthandler.go | 17 +++++++++++------ snapshot.go | 3 ++- 9 files changed, 57 insertions(+), 12 deletions(-) diff --git a/archiver.go b/archiver.go index b1c593ab8..71dc988d2 100644 --- a/archiver.go +++ b/archiver.go @@ -85,7 +85,7 @@ func NewArchiver(be backend.Server, key *Key) (*Archiver, error) { } // load all blobs from all snapshots - err = arch.ch.LoadAllSnapshots() + err = arch.ch.LoadAllMaps() if err != nil { return nil, err } @@ -371,11 +371,16 @@ func (arch *Archiver) Snapshot(dir string, t *Tree) (*Snapshot, backend.ID, erro if err != nil { return nil, nil, err } - sn.Content = blob.ID + // save bloblist + blob, err = arch.SaveJSON(backend.Map, arch.bl) + if err != nil { + return nil, nil, err + } + sn.Map = blob.Storage + // save snapshot - sn.BlobList = arch.bl blob, err = arch.SaveJSON(backend.Snapshot, sn) if err != nil { return nil, nil, err diff --git a/backend/interface.go b/backend/interface.go index b45f962f0..bfaabacb6 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -10,6 +10,7 @@ const ( Lock = "lock" Snapshot = "snapshot" Tree = "tree" + Map = "map" ) const ( diff --git a/backend/local.go b/backend/local.go index 9ba203fbc..b7e1508d1 100644 --- a/backend/local.go +++ b/backend/local.go @@ -17,6 +17,7 @@ const ( dataPath = "data" snapshotPath = "snapshots" treePath = "trees" + mapPath = "maps" lockPath = "locks" keyPath = "keys" tempPath = "tmp" @@ -86,6 +87,7 @@ func CreateLocal(dir string) (*Local, error) { filepath.Join(dir, dataPath), filepath.Join(dir, snapshotPath), filepath.Join(dir, treePath), + filepath.Join(dir, mapPath), filepath.Join(dir, lockPath), filepath.Join(dir, keyPath), filepath.Join(dir, tempPath), @@ -158,6 +160,8 @@ func (b *Local) dir(t Type) string { n = snapshotPath case Tree: n = treePath + case Map: + n = mapPath case Lock: n = lockPath case Key: diff --git a/backend/local_test.go b/backend/local_test.go index 056ba916f..0c309c063 100644 --- a/backend/local_test.go +++ b/backend/local_test.go @@ -45,7 +45,7 @@ func teardownBackend(t *testing.T, b *backend.Local) { } func testBackend(b backend.Server, t *testing.T) { - for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} { + for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree, backend.Map} { // detect non-existing files for _, test := range TestStrings { id, err := backend.ParseID(test.id) diff --git a/backend/sftp.go b/backend/sftp.go index ff5774295..0299cd133 100644 --- a/backend/sftp.go +++ b/backend/sftp.go @@ -238,6 +238,8 @@ func (r *SFTP) dir(t Type) string { n = snapshotPath case Tree: n = treePath + case Map: + n = mapPath case Lock: n = lockPath case Key: diff --git a/bloblist.go b/bloblist.go index 48820e95f..07da14cfa 100644 --- a/bloblist.go +++ b/bloblist.go @@ -6,6 +6,8 @@ import ( "errors" "sort" "sync" + + "github.com/fd0/khepri/backend" ) type BlobList struct { @@ -21,6 +23,16 @@ func NewBlobList() *BlobList { } } +func LoadBlobList(ch *ContentHandler, id backend.ID) (*BlobList, error) { + bl := &BlobList{} + err := ch.LoadJSONRaw(backend.Map, id, bl) + if err != nil { + return nil, err + } + + return bl, nil +} + func (bl *BlobList) find(blob Blob) (int, Blob, error) { pos := sort.Search(len(bl.list), func(i int) bool { return blob.ID.Compare(bl.list[i].ID) >= 0 diff --git a/cmd/khepri/cmd_cat.go b/cmd/khepri/cmd_cat.go index 0e3b67056..19d925900 100644 --- a/cmd/khepri/cmd_cat.go +++ b/cmd/khepri/cmd_cat.go @@ -27,7 +27,7 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error { return err } - err = ch.LoadAllSnapshots() + err = ch.LoadAllMaps() if err != nil { return err } @@ -87,6 +87,21 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error { fmt.Println(string(buf)) + return nil + case "map": + var bl khepri.BlobList + err := ch.LoadJSONRaw(backend.Map, id, &bl) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(&bl, "", " ") + if err != nil { + return err + } + + fmt.Println(string(buf)) + return nil case "snapshot": var sn khepri.Snapshot diff --git a/contenthandler.go b/contenthandler.go index 8ccf7654f..c4f802862 100644 --- a/contenthandler.go +++ b/contenthandler.go @@ -33,22 +33,27 @@ func (ch *ContentHandler) LoadSnapshot(id backend.ID) (*Snapshot, error) { return nil, err } - ch.bl.Merge(sn.BlobList) + sn.bl, err = LoadBlobList(ch, sn.Map) + if err != nil { + return nil, err + } + + ch.bl.Merge(sn.bl) return sn, nil } -// LoadAllSnapshots adds all blobs from all snapshots that can be decrypted +// LoadAllMaps adds all blobs from all snapshots that can be decrypted // into the content handler. -func (ch *ContentHandler) LoadAllSnapshots() error { +func (ch *ContentHandler) LoadAllMaps() error { // add all maps from all snapshots that can be decrypted to the storage map - err := backend.EachID(ch.be, backend.Snapshot, func(id backend.ID) { - sn, err := LoadSnapshot(ch, id) + err := backend.EachID(ch.be, backend.Map, func(id backend.ID) { + bl, err := LoadBlobList(ch, id) if err != nil { return } - ch.bl.Merge(sn.BlobList) + ch.bl.Merge(bl) }) if err != nil { return err diff --git a/snapshot.go b/snapshot.go index c0cc6c2bb..a92dd19ff 100644 --- a/snapshot.go +++ b/snapshot.go @@ -13,7 +13,7 @@ import ( type Snapshot struct { Time time.Time `json:"time"` Content backend.ID `json:"content"` - BlobList *BlobList `json:"blobs"` + Map backend.ID `json:"map"` Dir string `json:"dir"` Hostname string `json:"hostname,omitempty"` Username string `json:"username,omitempty"` @@ -21,6 +21,7 @@ type Snapshot struct { GID string `json:"gid,omitempty"` id backend.ID // plaintext ID, used during restore + bl *BlobList } func NewSnapshot(dir string) *Snapshot {