From b3c2d82331dbf2b5784781a1127cab65a8780f8a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 4 Aug 2014 20:47:04 +0200 Subject: [PATCH] Refactor, add Object and Snapshot --- cmd/khepri/cmd_backup.go | 11 +++- cmd/khepri/cmd_list.go | 2 +- cmd/khepri/cmd_restore.go | 6 +- cmd/khepri/main.go | 4 +- object.go | 119 ++++++++++++++++++++++++++++++++++++++ object_test.go | 31 ++++++++++ repository.go | 34 +++++------ repository_test.go | 6 +- snapshot.go | 71 +++++++++++++++++++++++ snapshot_test.go | 26 +++++++++ 10 files changed, 281 insertions(+), 29 deletions(-) create mode 100644 object.go create mode 100644 object_test.go create mode 100644 snapshot.go create mode 100644 snapshot_test.go diff --git a/cmd/khepri/cmd_backup.go b/cmd/khepri/cmd_backup.go index 1219bfda9..097a7822c 100644 --- a/cmd/khepri/cmd_backup.go +++ b/cmd/khepri/cmd_backup.go @@ -24,8 +24,9 @@ func hash(filename string) (khepri.ID, error) { return h.Sum([]byte{}), nil } -func archive_dir(repo *khepri.DirRepository, path string) (khepri.ID, error) { +func archive_dir(repo *khepri.Repository, path string) (khepri.ID, error) { log.Printf("archiving dir %q", path) + dir, err := os.Open(path) if err != nil { log.Printf("open(%q): %v\n", path, err) @@ -81,7 +82,7 @@ func archive_dir(repo *khepri.DirRepository, path string) (khepri.ID, error) { return id, nil } -func commandBackup(repo *khepri.DirRepository, args []string) error { +func commandBackup(repo *khepri.Repository, args []string) error { if len(args) != 1 { return errors.New("usage: backup dir") } @@ -93,7 +94,11 @@ func commandBackup(repo *khepri.DirRepository, args []string) error { return err } - fmt.Printf("%q archived as %v\n", target, id) + sn := repo.NewSnapshot(target) + sn.Tree = id + sn.Save() + + fmt.Printf("%q archived as %v\n", target, sn.ID()) return nil } diff --git a/cmd/khepri/cmd_list.go b/cmd/khepri/cmd_list.go index 5cbb91a9b..0b8e9fe4b 100644 --- a/cmd/khepri/cmd_list.go +++ b/cmd/khepri/cmd_list.go @@ -8,7 +8,7 @@ import ( "github.com/fd0/khepri" ) -func commandList(repo *khepri.DirRepository, args []string) error { +func commandList(repo *khepri.Repository, args []string) error { if len(args) != 1 { return errors.New("usage: list [blob|ref]") } diff --git a/cmd/khepri/cmd_restore.go b/cmd/khepri/cmd_restore.go index f0cf188ef..cbc69fba3 100644 --- a/cmd/khepri/cmd_restore.go +++ b/cmd/khepri/cmd_restore.go @@ -10,7 +10,7 @@ import ( "github.com/fd0/khepri" ) -func restore_file(repo *khepri.DirRepository, node khepri.Node, target string) error { +func restore_file(repo *khepri.Repository, node khepri.Node, target string) error { fmt.Printf(" restore file %q\n", target) rd, err := repo.Get(khepri.TYPE_BLOB, node.Content) @@ -47,7 +47,7 @@ func restore_file(repo *khepri.DirRepository, node khepri.Node, target string) e return nil } -func restore_dir(repo *khepri.DirRepository, id khepri.ID, target string) error { +func restore_dir(repo *khepri.Repository, id khepri.ID, target string) error { fmt.Printf(" restore dir %q\n", target) rd, err := repo.Get(khepri.TYPE_REF, id) if err != nil { @@ -104,7 +104,7 @@ func restore_dir(repo *khepri.DirRepository, id khepri.ID, target string) error return nil } -func commandRestore(repo *khepri.DirRepository, args []string) error { +func commandRestore(repo *khepri.Repository, args []string) error { if len(args) != 2 { return errors.New("usage: restore ID dir") } diff --git a/cmd/khepri/main.go b/cmd/khepri/main.go index 28d752540..bffcc2f8b 100644 --- a/cmd/khepri/main.go +++ b/cmd/khepri/main.go @@ -22,7 +22,7 @@ func errx(code int, format string, data ...interface{}) { os.Exit(code) } -type commandFunc func(*khepri.DirRepository, []string) error +type commandFunc func(*khepri.Repository, []string) error var commands map[string]commandFunc @@ -61,7 +61,7 @@ func main() { errx(1, "unknown command: %q\n", cmd) } - repo, err := khepri.NewDirRepository(Opts.Repo) + repo, err := khepri.NewRepository(Opts.Repo) if err != nil { errx(1, "unable to create/open repo: %v", err) diff --git a/object.go b/object.go new file mode 100644 index 000000000..03349e498 --- /dev/null +++ b/object.go @@ -0,0 +1,119 @@ +package khepri + +import "os" + +type Object struct { + repo *Repository + + id ID + tpe Type + + hw HashingWriter + file *os.File +} + +func (repo *Repository) NewObject(t Type) (*Object, error) { + obj := &Object{ + repo: repo, + tpe: t, + } + + return obj, obj.open() +} + +func (obj *Object) open() error { + if obj.isFinal() { + panic("object is finalized") + } + + if obj.isOpen() { + panic("object already open") + } + + // create tempfile in repository + if obj.hw == nil { + // save contents to tempfile, hash while writing + var err error + obj.file, err = obj.repo.tempFile() + if err != nil { + return err + } + + // create hashing writer + obj.hw = NewHashingWriter(obj.file, obj.repo.hash) + } + + return nil +} + +func (obj *Object) isOpen() bool { + return obj.file != nil && obj.hw != nil +} + +func (obj *Object) isFinal() bool { + return obj.id != nil +} + +func (obj *Object) Write(data []byte) (int, error) { + if !obj.isOpen() { + panic("object not open") + } + + return obj.hw.Write(data) +} + +func (obj *Object) Close() error { + if obj.file == nil || obj.hw == nil { + panic("object is not open") + } + + obj.file.Close() + + hash := obj.hw.Hash() + + // move file to final name using hash of contents + id := ID(hash) + err := obj.repo.renameFile(obj.file, obj.tpe, id) + if err != nil { + return err + } + + obj.hw = nil + obj.file = nil + + obj.id = id + return nil +} + +func (obj *Object) ID() ID { + if !obj.isFinal() { + panic("object not finalized") + } + + return obj.id +} + +func (obj *Object) Type() Type { + return obj.tpe +} + +func (obj *Object) Remove() error { + if obj.id != nil { + return obj.repo.Remove(obj.tpe, obj.id) + } + + if obj.file != nil { + file := obj.file + obj.hw = nil + obj.file = nil + + err := file.Close() + if err != nil { + return err + } + + return os.Remove(file.Name()) + } + + return nil +} diff --git a/object_test.go b/object_test.go new file mode 100644 index 000000000..62afe9fcd --- /dev/null +++ b/object_test.go @@ -0,0 +1,31 @@ +package khepri_test + +import ( + "testing" + + "github.com/fd0/khepri" +) + +func TestObjects(t *testing.T) { + repo, err := setupRepo() + ok(t, err) + + defer func() { + err = teardownRepo(repo) + ok(t, err) + }() + + for _, test := range TestStrings { + obj, err := repo.NewObject(khepri.TYPE_BLOB) + ok(t, err) + + _, err = obj.Write([]byte(test.data)) + ok(t, err) + + obj.Close() + id, err := khepri.ParseID(test.id) + ok(t, err) + + equals(t, id, obj.ID()) + } +} diff --git a/repository.go b/repository.go index f5f8adc5b..f6756ef2f 100644 --- a/repository.go +++ b/repository.go @@ -31,7 +31,7 @@ func (n Name) Encode() string { return url.QueryEscape(string(n)) } -type DirRepository struct { +type Repository struct { path string hash func() hash.Hash } @@ -66,8 +66,8 @@ func (t Type) String() string { } // NewDirRepository creates a new dir-baked repository at the given path. -func NewDirRepository(path string) (*DirRepository, error) { - d := &DirRepository{ +func NewRepository(path string) (*Repository, error) { + d := &Repository{ path: path, hash: sha256.New, } @@ -81,7 +81,7 @@ func NewDirRepository(path string) (*DirRepository, error) { return d, nil } -func (r *DirRepository) create() error { +func (r *Repository) create() error { dirs := []string{ r.path, path.Join(r.path, blobPath), @@ -100,28 +100,28 @@ func (r *DirRepository) create() error { } // SetHash changes the hash function used for deriving IDs. Default is SHA256. -func (r *DirRepository) SetHash(h func() hash.Hash) { +func (r *Repository) SetHash(h func() hash.Hash) { r.hash = h } // Path returns the directory used for this repository. -func (r *DirRepository) Path() string { +func (r *Repository) Path() string { return r.path } // Return temp directory in correct directory for this repository. -func (r *DirRepository) tempFile() (*os.File, error) { +func (r *Repository) tempFile() (*os.File, error) { return ioutil.TempFile(path.Join(r.path, tempPath), "temp-") } // Rename temp file to final name according to type and ID. -func (r *DirRepository) renameFile(file *os.File, t Type, id ID) error { +func (r *Repository) renameFile(file *os.File, t Type, id ID) error { filename := path.Join(r.dir(t), id.String()) return os.Rename(file.Name(), filename) } // Put saves content and returns the ID. -func (r *DirRepository) Put(t Type, reader io.Reader) (ID, error) { +func (r *Repository) Put(t Type, reader io.Reader) (ID, error) { // save contents to tempfile, hash while writing file, err := r.tempFile() if err != nil { @@ -150,7 +150,7 @@ func (r *DirRepository) Put(t Type, reader io.Reader) (ID, error) { } // Construct directory for given Type. -func (r *DirRepository) dir(t Type) string { +func (r *Repository) dir(t Type) string { switch t { case TYPE_BLOB: return path.Join(r.path, blobPath) @@ -162,12 +162,12 @@ func (r *DirRepository) dir(t Type) string { } // Construct path for given Type and ID. -func (r *DirRepository) filename(t Type, id ID) string { +func (r *Repository) filename(t Type, id ID) string { return path.Join(r.dir(t), id.String()) } // PutFile saves a file's content to the repository and returns the ID. -func (r *DirRepository) PutFile(path string) (ID, error) { +func (r *Repository) PutFile(path string) (ID, error) { f, err := os.Open(path) defer f.Close() if err != nil { @@ -178,7 +178,7 @@ func (r *DirRepository) PutFile(path string) (ID, error) { } // PutRaw saves a []byte's content to the repository and returns the ID. -func (r *DirRepository) PutRaw(t Type, buf []byte) (ID, error) { +func (r *Repository) PutRaw(t Type, buf []byte) (ID, error) { // save contents to tempfile, hash while writing file, err := r.tempFile() if err != nil { @@ -206,7 +206,7 @@ func (r *DirRepository) PutRaw(t Type, buf []byte) (ID, error) { } // Test returns true if the given ID exists in the repository. -func (r *DirRepository) Test(t Type, id ID) (bool, error) { +func (r *Repository) Test(t Type, id ID) (bool, error) { // try to open file file, err := os.Open(r.filename(t, id)) defer func() { @@ -224,7 +224,7 @@ func (r *DirRepository) Test(t Type, id ID) (bool, error) { } // Get returns a reader for the content stored under the given ID. -func (r *DirRepository) Get(t Type, id ID) (io.Reader, error) { +func (r *Repository) Get(t Type, id ID) (io.Reader, error) { // try to open file file, err := os.Open(r.filename(t, id)) if err != nil { @@ -235,14 +235,14 @@ func (r *DirRepository) Get(t Type, id ID) (io.Reader, error) { } // Remove removes the content stored at ID. -func (r *DirRepository) Remove(t Type, id ID) error { +func (r *Repository) Remove(t Type, id ID) error { return os.Remove(r.filename(t, id)) } type IDs []ID // Lists all objects of a given type. -func (r *DirRepository) ListIDs(t Type) (IDs, error) { +func (r *Repository) ListIDs(t Type) (IDs, error) { // TODO: use os.Open() and d.Readdirnames() instead of Glob() pattern := path.Join(r.dir(t), "*") diff --git a/repository_test.go b/repository_test.go index 1429c5b06..aff39d09c 100644 --- a/repository_test.go +++ b/repository_test.go @@ -25,13 +25,13 @@ var TestStrings = []struct { {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", khepri.TYPE_BLOB, "foo/../../baz"}, } -func setupRepo() (*khepri.DirRepository, error) { +func setupRepo() (*khepri.Repository, error) { tempdir, err := ioutil.TempDir("", "khepri-test-") if err != nil { return nil, err } - repo, err := khepri.NewDirRepository(tempdir) + repo, err := khepri.NewRepository(tempdir) if err != nil { return nil, err } @@ -39,7 +39,7 @@ func setupRepo() (*khepri.DirRepository, error) { return repo, nil } -func teardownRepo(repo *khepri.DirRepository) error { +func teardownRepo(repo *khepri.Repository) error { if !*testCleanup { fmt.Fprintf(os.Stderr, "leaving repository at %s\n", repo.Path()) return nil diff --git a/snapshot.go b/snapshot.go new file mode 100644 index 000000000..f55f00551 --- /dev/null +++ b/snapshot.go @@ -0,0 +1,71 @@ +package khepri + +import ( + "encoding/json" + "os" + "os/user" + "time" +) + +type Snapshot struct { + Time time.Time `json:"time"` + Tree ID `json:"tree"` + Dir string `json:"dir"` + Hostname string `json:"hostname,omitempty"` + Username string `json:"username,omitempty"` + UID string `json:"uid,omitempty"` + GID string `json:"gid,omitempty"` + id ID + repo *Repository +} + +func (repo *Repository) NewSnapshot(dir string) *Snapshot { + sn := &Snapshot{ + Dir: dir, + repo: repo, + Time: time.Now(), + } + + hn, err := os.Hostname() + if err == nil { + sn.Hostname = hn + } + + usr, err := user.Current() + if err == nil { + sn.Username = usr.Username + sn.UID = usr.Uid + sn.GID = usr.Gid + } + + return sn +} + +func (sn *Snapshot) Save() error { + if sn.Tree == nil { + panic("Snapshot.Save() called with nil tree id") + } + + obj, err := sn.repo.NewObject(TYPE_REF) + if err != nil { + return err + } + + enc := json.NewEncoder(obj) + err = enc.Encode(sn) + if err != nil { + return err + } + + err = obj.Close() + if err != nil { + return err + } + + sn.id = obj.ID() + return nil +} + +func (sn *Snapshot) ID() ID { + return sn.id +} diff --git a/snapshot_test.go b/snapshot_test.go new file mode 100644 index 000000000..c3cc001aa --- /dev/null +++ b/snapshot_test.go @@ -0,0 +1,26 @@ +package khepri_test + +import ( + "testing" + "time" + + "github.com/fd0/khepri" +) + +func TestSnapshot(t *testing.T) { + repo, err := setupRepo() + ok(t, err) + + defer func() { + err = teardownRepo(repo) + ok(t, err) + }() + + sn := repo.NewSnapshot("/home/foobar") + sn.Tree, err = khepri.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2") + ok(t, err) + sn.Time, err = time.Parse(time.RFC3339Nano, "2014-08-03T17:49:05.378595539+02:00") + ok(t, err) + + ok(t, sn.Save()) +}