From a71b49ebb9411a7097a91d4668c8d82c408b3c87 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 30 Jul 2014 23:11:23 +0200 Subject: [PATCH] Introduce Type for DirRepository objects --- repository.go | 191 ++++++++++++++++++++++++++++++--------------- repository_test.go | 66 ++++++++++------ 2 files changed, 170 insertions(+), 87 deletions(-) diff --git a/repository.go b/repository.go index 7c9e95dcf..1438cbc0d 100644 --- a/repository.go +++ b/repository.go @@ -2,21 +2,22 @@ package khepri import ( "crypto/sha256" - "encoding/hex" "errors" + "fmt" "hash" "io" "io/ioutil" "net/url" "os" "path" + "path/filepath" ) const ( - dirMode = 0700 - objectPath = "objects" - refPath = "refs" - tempPath = "tmp" + dirMode = 0700 + blobPath = "blobs" + refPath = "refs" + tempPath = "tmp" ) var ( @@ -35,6 +36,36 @@ type DirRepository struct { hash func() hash.Hash } +type Type int + +const ( + TypeUnknown = iota + TypeBlob + TypeRef +) + +func NewTypeFromString(s string) Type { + switch s { + case "blob": + return TypeBlob + case "ref": + return TypeRef + } + + panic(fmt.Sprintf("unknown type %q", s)) +} + +func (t Type) String() string { + switch t { + case TypeBlob: + return "blob" + case TypeRef: + return "ref" + } + + panic(fmt.Sprintf("unknown type %d", t)) +} + // NewDirRepository creates a new dir-baked repository at the given path. func NewDirRepository(path string) (*DirRepository, error) { d := &DirRepository{ @@ -54,7 +85,7 @@ func NewDirRepository(path string) (*DirRepository, error) { func (r *DirRepository) create() error { dirs := []string{ r.path, - path.Join(r.path, objectPath), + path.Join(r.path, blobPath), path.Join(r.path, refPath), path.Join(r.path, tempPath), } @@ -79,10 +110,21 @@ func (r *DirRepository) Path() string { return r.path } +// Return temp directory in correct directory for this repository. +func (r *DirRepository) tempFile() (*os.File, error) { + return ioutil.TempFile(path.Join(r.path, tempPath), "temp-") +} + +// Rename temp file to final name according to type and ID. +func (r *DirRepository) renameFile(file *os.File, t Type, id ID) error { + filename := path.Join(r.dir(t), id.String()) + return os.Rename(file.Name(), filename) +} + // Put saves content and returns the ID. -func (r *DirRepository) Put(reader io.Reader) (ID, error) { +func (r *DirRepository) Put(t Type, reader io.Reader) (ID, error) { // save contents to tempfile, hash while writing - file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-") + file, err := r.tempFile() if err != nil { return nil, err } @@ -100,8 +142,7 @@ func (r *DirRepository) Put(reader io.Reader) (ID, error) { // move file to final name using hash of contents id := ID(rd.Hash()) - filename := path.Join(r.path, objectPath, id.String()) - err = os.Rename(file.Name(), filename) + err = r.renameFile(file, t, id) if err != nil { return nil, err } @@ -109,6 +150,23 @@ func (r *DirRepository) Put(reader io.Reader) (ID, error) { return id, nil } +// Construct directory for given Type. +func (r *DirRepository) dir(t Type) string { + switch t { + case TypeBlob: + return path.Join(r.path, blobPath) + case TypeRef: + return path.Join(r.path, refPath) + } + + panic(fmt.Sprintf("unknown type %d", t)) +} + +// Construct path for given Type and ID. +func (r *DirRepository) filename(t Type, id ID) string { + return path.Join(r.dir(t), id.String()) +} + // PutFile saves a file's content to the repository and returns the ID. func (r *DirRepository) PutFile(path string) (ID, error) { f, err := os.Open(path) @@ -117,13 +175,13 @@ func (r *DirRepository) PutFile(path string) (ID, error) { return nil, err } - return r.Put(f) + return r.Put(TypeBlob, f) } // PutRaw saves a []byte's content to the repository and returns the ID. -func (r *DirRepository) PutRaw(buf []byte) (ID, error) { +func (r *DirRepository) PutRaw(t Type, buf []byte) (ID, error) { // save contents to tempfile, hash while writing - file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-") + file, err := r.tempFile() if err != nil { return nil, err } @@ -140,8 +198,7 @@ func (r *DirRepository) PutRaw(buf []byte) (ID, error) { // move file to final name using hash of contents id := ID(wr.Hash()) - filename := path.Join(r.path, objectPath, id.String()) - err = os.Rename(file.Name(), filename) + err = r.renameFile(file, t, id) if err != nil { return nil, err } @@ -150,9 +207,9 @@ func (r *DirRepository) PutRaw(buf []byte) (ID, error) { } // Test returns true if the given ID exists in the repository. -func (r *DirRepository) Test(id ID) (bool, error) { +func (r *DirRepository) Test(t Type, id ID) (bool, error) { // try to open file - file, err := os.Open(path.Join(r.path, objectPath, id.String())) + file, err := os.Open(r.filename(t, id)) defer func() { file.Close() }() @@ -168,9 +225,9 @@ func (r *DirRepository) Test(id ID) (bool, error) { } // Get returns a reader for the content stored under the given ID. -func (r *DirRepository) Get(id ID) (io.Reader, error) { +func (r *DirRepository) Get(t Type, id ID) (io.Reader, error) { // try to open file - file, err := os.Open(path.Join(r.path, objectPath, id.String())) + file, err := os.Open(r.filename(t, id)) if err != nil { return nil, err } @@ -179,60 +236,66 @@ func (r *DirRepository) Get(id ID) (io.Reader, error) { } // Remove removes the content stored at ID. -func (r *DirRepository) Remove(id ID) error { - return os.Remove(path.Join(r.path, objectPath, id.String())) +func (r *DirRepository) Remove(t Type, id ID) error { + return os.Remove(r.filename(t, id)) } -// Unlink removes a named ID. -func (r *DirRepository) Unlink(name string) error { - return os.Remove(path.Join(r.path, refPath, Name(name).Encode())) -} +type IDs []ID -// Link assigns a name to an ID. Name must be unique in this repository and ID must exist. -func (r *DirRepository) Link(name string, id ID) error { - exist, err := r.Test(id) - if err != nil { - return err - } +// Lists all objects of a given type. +func (r *DirRepository) ListIDs(t Type) (IDs, error) { + // TODO: use os.Open() and d.Readdirnames() instead of Glob() + pattern := path.Join(r.dir(t), "*") - if !exist { - return ErrIDDoesNotExist - } - - // create file, write id - f, err := os.Create(path.Join(r.path, refPath, Name(name).Encode())) - defer f.Close() - - if err != nil { - return err - } - - f.Write([]byte(hex.EncodeToString(id))) - return nil -} - -// Resolve returns the ID associated with the given name. -func (r *DirRepository) Resolve(name string) (ID, error) { - f, err := os.Open(path.Join(r.path, refPath, Name(name).Encode())) - defer f.Close() + matches, err := filepath.Glob(pattern) if err != nil { return nil, err } - // read hex string - l := r.hash().Size() - buf := make([]byte, l*2) - _, err = io.ReadFull(f, buf) + ids := make(IDs, 0, len(matches)) - if err != nil { - return nil, err + for _, m := range matches { + base := filepath.Base(m) + + if base == "" { + continue + } + id, err := ParseID(base) + + if err != nil { + continue + } + + ids = append(ids, id) } - id := make([]byte, l) - _, err = hex.Decode(id, buf) - if err != nil { - return nil, err - } - - return ID(id), nil + return ids, nil +} + +func (ids IDs) Len() int { + return len(ids) +} + +func (ids IDs) Less(i, j int) bool { + if len(ids[i]) < len(ids[j]) { + return true + } + + for k, b := range ids[i] { + if b == ids[j][k] { + continue + } + + if b < ids[j][k] { + return true + } else { + return false + } + } + + return false +} + +func (ids IDs) Swap(i, j int) { + ids[i], ids[j] = ids[j], ids[i] } diff --git a/repository_test.go b/repository_test.go index 15013fa83..c41758431 100644 --- a/repository_test.go +++ b/repository_test.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "os" + "sort" "strings" "github.com/fd0/khepri" @@ -14,12 +15,13 @@ import ( var TestStrings = []struct { id string + t khepri.Type data string }{ - {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"}, - {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"}, - {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, + {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", khepri.TypeBlob, "foobar"}, + {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", khepri.TypeBlob, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", khepri.TypeRef, "foo/bar"}, + {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", khepri.TypeBlob, "foo/../../baz"}, } var _ = Describe("Storage", func() { @@ -30,7 +32,7 @@ var _ = Describe("Storage", func() { id khepri.ID ) - BeforeEach(func() { + var _ = BeforeSuite(func() { tempdir, err = ioutil.TempDir("", "khepri-test-") if err != nil { panic(err) @@ -41,7 +43,7 @@ var _ = Describe("Storage", func() { } }) - AfterEach(func() { + AfterSuite(func() { err = os.RemoveAll(tempdir) if err != nil { panic(err) @@ -58,7 +60,7 @@ var _ = Describe("Storage", func() { Expect(err).NotTo(HaveOccurred()) // try to get string out, should fail - ret, err := repo.Test(id) + ret, err := repo.Test(test.t, id) Expect(ret).Should(Equal(false)) } }) @@ -66,44 +68,62 @@ var _ = Describe("Storage", func() { It("Should Add File", func() { for _, test := range TestStrings { // store string in repository - id, err = repo.Put(strings.NewReader(test.data)) + id, err = repo.Put(test.t, strings.NewReader(test.data)) Expect(err).NotTo(HaveOccurred()) Expect(id.String()).Should(Equal(test.id)) // try to get it out again var buf bytes.Buffer - rd, err := repo.Get(id) + rd, err := repo.Get(test.t, id) Expect(err).NotTo(HaveOccurred()) Expect(rd).ShouldNot(BeNil()) // compare content Expect(io.Copy(&buf, rd)).Should(Equal(int64(len(test.data)))) Expect(buf.Bytes()).Should(Equal([]byte(test.data))) - - // store id under name - err = repo.Link(test.data, id) - Expect(err).NotTo(HaveOccurred()) - - // resolve again - Expect(repo.Resolve(test.data)).Should(Equal(id)) - - // remove link - Expect(repo.Unlink(test.data)).NotTo(HaveOccurred()) - - // remove string - Expect(repo.Remove(id)) } }) It("Should Add Buffer", func() { for _, test := range TestStrings { // store buf in repository - id, err := repo.PutRaw([]byte(test.data)) + id, err := repo.PutRaw(test.t, []byte(test.data)) Expect(err).NotTo(HaveOccurred()) Expect(id.String()).To(Equal(test.id)) } }) + + It("Should List IDs", func() { + for _, t := range []khepri.Type{khepri.TypeBlob, khepri.TypeRef} { + IDs := khepri.IDs{} + for _, test := range TestStrings { + if test.t == t { + id, err := khepri.ParseID(test.id) + Expect(err).NotTo(HaveOccurred()) + IDs = append(IDs, id) + } + } + + ids, err := repo.ListIDs(t) + + sort.Sort(ids) + sort.Sort(IDs) + Expect(err).NotTo(HaveOccurred()) + Expect(ids).Should(Equal(IDs)) + } + }) + + It("Should Remove Content", func() { + for _, test := range TestStrings { + id, err := khepri.ParseID(test.id) + Expect(err).ShouldNot(HaveOccurred()) + Expect(repo.Test(test.t, id)).To(Equal(true)) + Expect(repo.Remove(test.t, id)) + Expect(repo.Test(test.t, id)).To(Equal(false)) + } + }) }) }) + })