restic/internal/pack/pack.go

316 lines
6.9 KiB
Go
Raw Normal View History

2015-04-26 13:36:49 +00:00
package pack
import (
2016-01-24 18:30:14 +00:00
"bytes"
2015-04-26 13:36:49 +00:00
"encoding/binary"
"fmt"
"io"
"sync"
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal"
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/crypto"
2015-04-26 13:36:49 +00:00
)
// Packer is used to create a new Pack.
type Packer struct {
2016-08-31 20:39:36 +00:00
blobs []restic.Blob
2015-04-26 13:36:49 +00:00
bytes uint
k *crypto.Key
wr io.Writer
2015-04-26 13:36:49 +00:00
m sync.Mutex
}
// NewPacker returns a new Packer that can be used to pack blobs
// together. If wr is nil, a bytes.Buffer is used.
func NewPacker(k *crypto.Key, wr io.Writer) *Packer {
if wr == nil {
wr = bytes.NewBuffer(nil)
}
return &Packer{k: k, wr: wr}
2015-04-26 13:36:49 +00:00
}
// Add saves the data read from rd as a new blob to the packer. Returned is the
// number of bytes written to the pack.
2016-08-31 18:58:57 +00:00
func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte) (int, error) {
2015-04-26 13:36:49 +00:00
p.m.Lock()
defer p.m.Unlock()
2016-08-31 20:39:36 +00:00
c := restic.Blob{Type: t, ID: id}
2015-04-26 13:36:49 +00:00
n, err := p.wr.Write(data)
c.Length = uint(n)
2015-04-26 13:36:49 +00:00
c.Offset = p.bytes
p.bytes += uint(n)
p.blobs = append(p.blobs, c)
2016-08-29 20:16:58 +00:00
return n, errors.Wrap(err, "Write")
2015-04-26 13:36:49 +00:00
}
var entrySize = uint(binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
2015-04-26 13:36:49 +00:00
// headerEntry is used with encoding/binary to read and write header entries
type headerEntry struct {
Type uint8
2015-04-26 13:36:49 +00:00
Length uint32
ID restic.ID
2015-04-26 13:36:49 +00:00
}
// Finalize writes the header for all added blobs and finalizes the pack.
// Returned are the number of bytes written, including the header. If the
// underlying writer implements io.Closer, it is closed.
func (p *Packer) Finalize() (uint, error) {
2015-04-26 13:36:49 +00:00
p.m.Lock()
defer p.m.Unlock()
2016-01-24 18:30:14 +00:00
bytesWritten := p.bytes
2015-04-26 13:36:49 +00:00
2016-01-24 18:30:14 +00:00
hdrBuf := bytes.NewBuffer(nil)
bytesHeader, err := p.writeHeader(hdrBuf)
2015-04-29 22:36:36 +00:00
if err != nil {
return 0, err
2015-04-26 13:36:49 +00:00
}
encryptedHeader, err := p.k.Encrypt(nil, hdrBuf.Bytes())
2016-01-24 18:30:14 +00:00
if err != nil {
return 0, err
2016-01-24 18:30:14 +00:00
}
2015-04-29 22:41:11 +00:00
2016-01-24 18:30:14 +00:00
// append the header
n, err := p.wr.Write(encryptedHeader)
2015-04-26 13:36:49 +00:00
if err != nil {
2016-08-29 20:16:58 +00:00
return 0, errors.Wrap(err, "Write")
2016-01-24 18:30:14 +00:00
}
hdrBytes := restic.CiphertextLength(int(bytesHeader))
if n != hdrBytes {
return 0, errors.New("wrong number of bytes written")
2015-04-26 13:36:49 +00:00
}
bytesWritten += uint(hdrBytes)
2015-04-26 13:36:49 +00:00
// write length
err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(entrySize))))
2015-04-26 13:36:49 +00:00
if err != nil {
2016-08-29 20:16:58 +00:00
return 0, errors.Wrap(err, "binary.Write")
2015-04-26 13:36:49 +00:00
}
2015-04-29 22:19:56 +00:00
bytesWritten += uint(binary.Size(uint32(0)))
2015-04-26 13:36:49 +00:00
p.bytes = uint(bytesWritten)
2015-04-26 13:36:49 +00:00
if w, ok := p.wr.(io.Closer); ok {
return bytesWritten, w.Close()
}
return bytesWritten, nil
2015-04-26 13:36:49 +00:00
}
2015-04-29 22:36:36 +00:00
// writeHeader constructs and writes the header to wr.
func (p *Packer) writeHeader(wr io.Writer) (bytesWritten uint, err error) {
for _, b := range p.blobs {
entry := headerEntry{
Length: uint32(b.Length),
ID: b.ID,
2015-04-29 22:36:36 +00:00
}
switch b.Type {
2016-08-31 18:58:57 +00:00
case restic.DataBlob:
entry.Type = 0
2016-08-31 18:58:57 +00:00
case restic.TreeBlob:
entry.Type = 1
default:
return 0, errors.Errorf("invalid blob type %v", b.Type)
}
2015-04-29 22:36:36 +00:00
err := binary.Write(wr, binary.LittleEndian, entry)
if err != nil {
2016-08-29 20:16:58 +00:00
return bytesWritten, errors.Wrap(err, "binary.Write")
2015-04-29 22:36:36 +00:00
}
bytesWritten += entrySize
}
return
}
2015-04-26 13:36:49 +00:00
// Size returns the number of bytes written so far.
func (p *Packer) Size() uint {
p.m.Lock()
defer p.m.Unlock()
return p.bytes
}
// Count returns the number of blobs in this packer.
func (p *Packer) Count() int {
p.m.Lock()
defer p.m.Unlock()
return len(p.blobs)
}
// Blobs returns the slice of blobs that have been written.
2016-08-31 20:39:36 +00:00
func (p *Packer) Blobs() []restic.Blob {
2015-04-26 13:36:49 +00:00
p.m.Lock()
defer p.m.Unlock()
return p.blobs
}
// Writer return the underlying writer.
func (p *Packer) Writer() io.Writer {
return p.wr
}
2015-04-26 13:36:49 +00:00
func (p *Packer) String() string {
return fmt.Sprintf("<Packer %d blobs, %d bytes>", len(p.blobs), p.bytes)
}
2016-08-25 19:51:07 +00:00
// readHeaderLength returns the header length read from the end of the file
// encoded in little endian.
func readHeaderLength(rd io.ReaderAt, size int64) (uint32, error) {
off := size - int64(binary.Size(uint32(0)))
2016-08-25 19:51:07 +00:00
buf := make([]byte, binary.Size(uint32(0)))
n, err := rd.ReadAt(buf, off)
if err != nil {
2016-08-29 20:16:58 +00:00
return 0, errors.Wrap(err, "ReadAt")
2016-08-25 19:51:07 +00:00
}
2016-08-07 14:24:02 +00:00
2016-08-25 19:51:07 +00:00
if n != len(buf) {
return 0, errors.New("not enough bytes read")
2016-08-07 14:24:02 +00:00
}
2016-08-25 19:51:07 +00:00
return binary.LittleEndian.Uint32(buf), nil
}
const maxHeaderSize = 16 * 1024 * 1024
2017-06-08 18:40:12 +00:00
// we require at least one entry in the header, and one blob for a pack file
var minFileSize = entrySize + crypto.Extension
2016-08-25 19:51:07 +00:00
// readHeader reads the header at the end of rd. size is the length of the
// whole data accessible in rd.
func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
2017-06-08 19:04:07 +00:00
debug.Log("size: %v", size)
2017-06-08 18:40:12 +00:00
if size == 0 {
2017-06-08 19:04:07 +00:00
err := InvalidFileError{Message: "file is empty"}
2017-06-08 18:40:12 +00:00
return nil, errors.Wrap(err, "readHeader")
}
if size < int64(minFileSize) {
2017-06-08 19:04:07 +00:00
err := InvalidFileError{Message: "file is too small"}
2017-06-08 18:40:12 +00:00
return nil, errors.Wrap(err, "readHeader")
}
2016-08-25 19:51:07 +00:00
hl, err := readHeaderLength(rd, size)
if err != nil {
2016-08-25 19:51:07 +00:00
return nil, err
}
2017-06-08 19:04:07 +00:00
debug.Log("header length: %v", size)
if hl == 0 {
err := InvalidFileError{Message: "header length is zero"}
return nil, errors.Wrap(err, "readHeader")
}
if hl < crypto.Extension {
err := InvalidFileError{Message: "header length is too small"}
return nil, errors.Wrap(err, "readHeader")
}
if (hl-crypto.Extension)%uint32(entrySize) != 0 {
err := InvalidFileError{Message: "header length is invalid"}
return nil, errors.Wrap(err, "readHeader")
}
2016-08-25 19:51:07 +00:00
if int64(hl) > size-int64(binary.Size(hl)) {
err := InvalidFileError{Message: "header is larger than file"}
return nil, errors.Wrap(err, "readHeader")
2016-08-25 19:51:07 +00:00
}
2015-04-26 13:36:49 +00:00
2016-08-25 19:51:07 +00:00
if int64(hl) > maxHeaderSize {
err := InvalidFileError{Message: "header is larger than maxHeaderSize"}
return nil, errors.Wrap(err, "readHeader")
2016-08-23 20:21:29 +00:00
}
2016-08-25 19:51:07 +00:00
buf := make([]byte, int(hl))
n, err := rd.ReadAt(buf, size-int64(hl)-int64(binary.Size(hl)))
if err != nil {
2016-08-29 20:16:58 +00:00
return nil, errors.Wrap(err, "ReadAt")
2016-08-25 19:51:07 +00:00
}
2016-08-23 20:21:29 +00:00
2016-08-25 19:51:07 +00:00
if n != len(buf) {
return nil, errors.New("not enough bytes read")
}
2016-08-25 19:51:07 +00:00
return buf, nil
}
2017-06-08 18:40:12 +00:00
// InvalidFileError is return when a file is found that is not a pack file.
type InvalidFileError struct {
Message string
}
func (e InvalidFileError) Error() string {
return e.Message
}
2016-08-25 19:51:07 +00:00
// List returns the list of entries found in a pack file.
2016-08-31 20:39:36 +00:00
func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err error) {
2016-08-25 19:51:07 +00:00
buf, err := readHeader(rd, size)
if err != nil {
return nil, err
}
2015-04-26 13:36:49 +00:00
n, err := k.Decrypt(buf, buf)
2015-04-26 13:36:49 +00:00
if err != nil {
return nil, err
}
buf = buf[:n]
2015-04-26 13:36:49 +00:00
hdrRd := bytes.NewReader(buf)
2016-08-07 12:50:24 +00:00
2017-01-15 14:27:58 +00:00
entries = make([]restic.Blob, 0, uint(n)/entrySize)
pos := uint(0)
for {
e := headerEntry{}
2016-08-25 19:51:07 +00:00
err = binary.Read(hdrRd, binary.LittleEndian, &e)
if errors.Cause(err) == io.EOF {
break
}
if err != nil {
2016-08-29 20:16:58 +00:00
return nil, errors.Wrap(err, "binary.Read")
2015-04-26 13:36:49 +00:00
}
2016-08-31 20:39:36 +00:00
entry := restic.Blob{
Length: uint(e.Length),
ID: e.ID,
Offset: pos,
}
switch e.Type {
case 0:
2016-08-31 18:58:57 +00:00
entry.Type = restic.DataBlob
case 1:
2016-08-31 18:58:57 +00:00
entry.Type = restic.TreeBlob
default:
return nil, errors.Errorf("invalid type %d", e.Type)
}
entries = append(entries, entry)
pos += uint(e.Length)
2015-04-26 13:36:49 +00:00
}
2016-08-25 19:08:16 +00:00
return entries, nil
2015-04-26 13:36:49 +00:00
}