Update the vendor folder (#53)

* Update the vendor folder

* Update the Gopkg.lock by running dep ensure

vendor/github.com/vbatts/tar-split/tar/storage/doc.go

@@ -0,0 +1,12 @@
+/*
+Package storage is for metadata of a tar archive.
+
+Packing and unpacking the Entries of the stream. The types of streams are
+either segments of raw bytes (for the raw headers and various padding) and for
+an entry marking a file payload.
+
+The raw bytes are stored precisely in the packed (marshalled) Entry, whereas
+the file payload marker include the name of the file, size, and crc64 checksum
+(for basic file integrity).
+*/
+package storage

vendor/github.com/vbatts/tar-split/tar/storage/entry.go

@@ -0,0 +1,78 @@
+package storage
+
+import "unicode/utf8"
+
+// Entries is for sorting by Position
+type Entries []Entry
+
+func (e Entries) Len() int           { return len(e) }
+func (e Entries) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
+func (e Entries) Less(i, j int) bool { return e[i].Position < e[j].Position }
+
+// Type of Entry
+type Type int
+
+const (
+	// FileType represents a file payload from the tar stream.
+	//
+	// This will be used to map to relative paths on disk. Only Size > 0 will get
+	// read into a resulting output stream (due to hardlinks).
+	FileType Type = 1 + iota
+	// SegmentType represents a raw bytes segment from the archive stream. These raw
+	// byte segments consist of the raw headers and various padding.
+	//
+	// Its payload is to be marshalled base64 encoded.
+	SegmentType
+)
+
+// Entry is the structure for packing and unpacking the information read from
+// the Tar archive.
+//
+// FileType Payload checksum is using `hash/crc64` for basic file integrity,
+// _not_ for cryptography.
+// From http://www.backplane.com/matt/crc64.html, CRC32 has almost 40,000
+// collisions in a sample of 18.2 million, CRC64 had none.
+type Entry struct {
+	Type     Type   `json:"type"`
+	Name     string `json:"name,omitempty"`
+	NameRaw  []byte `json:"name_raw,omitempty"`
+	Size     int64  `json:"size,omitempty"`
+	Payload  []byte `json:"payload"` // SegmentType stores payload here; FileType stores crc64 checksum here;
+	Position int    `json:"position"`
+}
+
+// SetName will check name for valid UTF-8 string, and set the appropriate
+// field. See https://github.com/vbatts/tar-split/issues/17
+func (e *Entry) SetName(name string) {
+	if utf8.ValidString(name) {
+		e.Name = name
+	} else {
+		e.NameRaw = []byte(name)
+	}
+}
+
+// SetNameBytes will check name for valid UTF-8 string, and set the appropriate
+// field
+func (e *Entry) SetNameBytes(name []byte) {
+	if utf8.Valid(name) {
+		e.Name = string(name)
+	} else {
+		e.NameRaw = name
+	}
+}
+
+// GetName returns the string for the entry's name, regardless of the field stored in
+func (e *Entry) GetName() string {
+	if len(e.NameRaw) > 0 {
+		return string(e.NameRaw)
+	}
+	return e.Name
+}
+
+// GetNameBytes returns the bytes for the entry's name, regardless of the field stored in
+func (e *Entry) GetNameBytes() []byte {
+	if len(e.NameRaw) > 0 {
+		return e.NameRaw
+	}
+	return []byte(e.Name)
+}

vendor/github.com/vbatts/tar-split/tar/storage/entry_test.go

@@ -0,0 +1,95 @@
+package storage
+
+import (
+	"encoding/json"
+	"sort"
+	"testing"
+)
+
+func TestEntries(t *testing.T) {
+	e := Entries{
+		Entry{
+			Type:     SegmentType,
+			Payload:  []byte("y'all"),
+			Position: 1,
+		},
+		Entry{
+			Type:     SegmentType,
+			Payload:  []byte("doin"),
+			Position: 3,
+		},
+		Entry{
+			Type:     FileType,
+			Name:     "./hurr.txt",
+			Payload:  []byte("deadbeef"),
+			Position: 2,
+		},
+		Entry{
+			Type:     SegmentType,
+			Payload:  []byte("how"),
+			Position: 0,
+		},
+	}
+	sort.Sort(e)
+	if e[0].Position != 0 {
+		t.Errorf("expected Position 0, but got %d", e[0].Position)
+	}
+}
+
+func TestFile(t *testing.T) {
+	f := Entry{
+		Type:     FileType,
+		Size:     100,
+		Position: 2,
+	}
+	f.SetName("./hello.txt")
+
+	buf, err := json.Marshal(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f1 := Entry{}
+	if err = json.Unmarshal(buf, &f1); err != nil {
+		t.Fatal(err)
+	}
+
+	if f.GetName() != f1.GetName() {
+		t.Errorf("expected Name %q, got %q", f.GetName(), f1.GetName())
+	}
+	if f.Size != f1.Size {
+		t.Errorf("expected Size %q, got %q", f.Size, f1.Size)
+	}
+	if f.Position != f1.Position {
+		t.Errorf("expected Position %q, got %q", f.Position, f1.Position)
+	}
+}
+
+func TestFileRaw(t *testing.T) {
+	f := Entry{
+		Type:     FileType,
+		Size:     100,
+		Position: 2,
+	}
+	f.SetNameBytes([]byte{0x2E, 0x2F, 0x68, 0x65, 0x6C, 0x6C, 0x6F, 0xE4, 0x2E, 0x74, 0x78, 0x74})
+
+	buf, err := json.Marshal(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f1 := Entry{}
+	if err = json.Unmarshal(buf, &f1); err != nil {
+		t.Fatal(err)
+	}
+
+	if f.GetName() != f1.GetName() {
+		t.Errorf("expected Name %q, got %q", f.GetName(), f1.GetName())
+	}
+	if f.Size != f1.Size {
+		t.Errorf("expected Size %q, got %q", f.Size, f1.Size)
+	}
+	if f.Position != f1.Position {
+		t.Errorf("expected Position %q, got %q", f.Position, f1.Position)
+	}
+}

vendor/github.com/vbatts/tar-split/tar/storage/getter.go

@@ -0,0 +1,104 @@
+package storage
+
+import (
+	"bytes"
+	"errors"
+	"hash/crc64"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// FileGetter is the interface for getting a stream of a file payload,
+// addressed by name/filename. Presumably, the names will be scoped to relative
+// file paths.
+type FileGetter interface {
+	// Get returns a stream for the provided file path
+	Get(filename string) (output io.ReadCloser, err error)
+}
+
+// FilePutter is the interface for storing a stream of a file payload,
+// addressed by name/filename.
+type FilePutter interface {
+	// Put returns the size of the stream received, and the crc64 checksum for
+	// the provided stream
+	Put(filename string, input io.Reader) (size int64, checksum []byte, err error)
+}
+
+// FileGetPutter is the interface that groups both Getting and Putting file
+// payloads.
+type FileGetPutter interface {
+	FileGetter
+	FilePutter
+}
+
+// NewPathFileGetter returns a FileGetter that is for files relative to path
+// relpath.
+func NewPathFileGetter(relpath string) FileGetter {
+	return &pathFileGetter{root: relpath}
+}
+
+type pathFileGetter struct {
+	root string
+}
+
+func (pfg pathFileGetter) Get(filename string) (io.ReadCloser, error) {
+	return os.Open(filepath.Join(pfg.root, filename))
+}
+
+type bufferFileGetPutter struct {
+	files map[string][]byte
+}
+
+func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) {
+	if _, ok := bfgp.files[name]; !ok {
+		return nil, errors.New("no such file")
+	}
+	b := bytes.NewBuffer(bfgp.files[name])
+	return &readCloserWrapper{b}, nil
+}
+
+func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) (int64, []byte, error) {
+	crc := crc64.New(CRCTable)
+	buf := bytes.NewBuffer(nil)
+	cw := io.MultiWriter(crc, buf)
+	i, err := io.Copy(cw, r)
+	if err != nil {
+		return 0, nil, err
+	}
+	bfgp.files[name] = buf.Bytes()
+	return i, crc.Sum(nil), nil
+}
+
+type readCloserWrapper struct {
+	io.Reader
+}
+
+func (w *readCloserWrapper) Close() error { return nil }
+
+// NewBufferFileGetPutter is a simple in-memory FileGetPutter
+//
+// Implication is this is memory intensive...
+// Probably best for testing or light weight cases.
+func NewBufferFileGetPutter() FileGetPutter {
+	return &bufferFileGetPutter{
+		files: map[string][]byte{},
+	}
+}
+
+// NewDiscardFilePutter is a bit bucket FilePutter
+func NewDiscardFilePutter() FilePutter {
+	return &bitBucketFilePutter{}
+}
+
+type bitBucketFilePutter struct {
+}
+
+func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) (int64, []byte, error) {
+	c := crc64.New(CRCTable)
+	i, err := io.Copy(c, r)
+	return i, c.Sum(nil), err
+}
+
+// CRCTable is the default table used for crc64 sum calculations
+var CRCTable = crc64.MakeTable(crc64.ISO)

vendor/github.com/vbatts/tar-split/tar/storage/getter_test.go

@@ -0,0 +1,84 @@
+package storage
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+func TestGetter(t *testing.T) {
+	fgp := NewBufferFileGetPutter()
+	files := map[string]map[string][]byte{
+		"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
+		"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
+	}
+	for n, b := range files {
+		for body, sum := range b {
+			_, csum, err := fgp.Put(n, bytes.NewBufferString(body))
+			if err != nil {
+				t.Error(err)
+			}
+			if !bytes.Equal(csum, sum) {
+				t.Errorf("checksum: expected 0x%x; got 0x%x", sum, csum)
+			}
+		}
+	}
+	for n, b := range files {
+		for body := range b {
+			r, err := fgp.Get(n)
+			if err != nil {
+				t.Error(err)
+			}
+			buf, err := ioutil.ReadAll(r)
+			if err != nil {
+				t.Error(err)
+			}
+			if body != string(buf) {
+				t.Errorf("expected %q, got %q", body, string(buf))
+			}
+		}
+	}
+}
+
+func TestPutter(t *testing.T) {
+	fp := NewDiscardFilePutter()
+	// map[filename]map[body]crc64sum
+	files := map[string]map[string][]byte{
+		"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
+		"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
+		"file3.txt": {"baz": []byte{32, 68, 22, 240, 0, 0, 0, 0}},
+		"file4.txt": {"bif": []byte{48, 9, 150, 240, 0, 0, 0, 0}},
+	}
+	for n, b := range files {
+		for body, sum := range b {
+			_, csum, err := fp.Put(n, bytes.NewBufferString(body))
+			if err != nil {
+				t.Error(err)
+			}
+			if !bytes.Equal(csum, sum) {
+				t.Errorf("checksum on %q: expected %v; got %v", n, sum, csum)
+			}
+		}
+	}
+}
+
+func BenchmarkPutter(b *testing.B) {
+	files := []string{
+		strings.Repeat("foo", 1000),
+		strings.Repeat("bar", 1000),
+		strings.Repeat("baz", 1000),
+		strings.Repeat("fooz", 1000),
+		strings.Repeat("vbatts", 1000),
+		strings.Repeat("systemd", 1000),
+	}
+	for i := 0; i < b.N; i++ {
+		fgp := NewBufferFileGetPutter()
+		for n, body := range files {
+			if _, _, err := fgp.Put(fmt.Sprintf("%d", n), bytes.NewBufferString(body)); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}

vendor/github.com/vbatts/tar-split/tar/storage/packer.go

@@ -0,0 +1,127 @@
+package storage
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"path/filepath"
+	"unicode/utf8"
+)
+
+// ErrDuplicatePath occurs when a tar archive has more than one entry for the
+// same file path
+var ErrDuplicatePath = errors.New("duplicates of file paths not supported")
+
+// Packer describes the methods to pack Entries to a storage destination
+type Packer interface {
+	// AddEntry packs the Entry and returns its position
+	AddEntry(e Entry) (int, error)
+}
+
+// Unpacker describes the methods to read Entries from a source
+type Unpacker interface {
+	// Next returns the next Entry being unpacked, or error, until io.EOF
+	Next() (*Entry, error)
+}
+
+/* TODO(vbatts) figure out a good model for this
+type PackUnpacker interface {
+	Packer
+	Unpacker
+}
+*/
+
+type jsonUnpacker struct {
+	seen seenNames
+	dec  *json.Decoder
+}
+
+func (jup *jsonUnpacker) Next() (*Entry, error) {
+	var e Entry
+	err := jup.dec.Decode(&e)
+	if err != nil {
+		return nil, err
+	}
+
+	// check for dup name
+	if e.Type == FileType {
+		cName := filepath.Clean(e.GetName())
+		if _, ok := jup.seen[cName]; ok {
+			return nil, ErrDuplicatePath
+		}
+		jup.seen[cName] = struct{}{}
+	}
+
+	return &e, err
+}
+
+// NewJSONUnpacker provides an Unpacker that reads Entries (SegmentType and
+// FileType) as a json document.
+//
+// Each Entry read are expected to be delimited by new line.
+func NewJSONUnpacker(r io.Reader) Unpacker {
+	return &jsonUnpacker{
+		dec:  json.NewDecoder(r),
+		seen: seenNames{},
+	}
+}
+
+type jsonPacker struct {
+	w    io.Writer
+	e    *json.Encoder
+	pos  int
+	seen seenNames
+}
+
+type seenNames map[string]struct{}
+
+func (jp *jsonPacker) AddEntry(e Entry) (int, error) {
+	// if Name is not valid utf8, switch it to raw first.
+	if e.Name != "" {
+		if !utf8.ValidString(e.Name) {
+			e.NameRaw = []byte(e.Name)
+			e.Name = ""
+		}
+	}
+
+	// check early for dup name
+	if e.Type == FileType {
+		cName := filepath.Clean(e.GetName())
+		if _, ok := jp.seen[cName]; ok {
+			return -1, ErrDuplicatePath
+		}
+		jp.seen[cName] = struct{}{}
+	}
+
+	e.Position = jp.pos
+	err := jp.e.Encode(e)
+	if err != nil {
+		return -1, err
+	}
+
+	// made it this far, increment now
+	jp.pos++
+	return e.Position, nil
+}
+
+// NewJSONPacker provides a Packer that writes each Entry (SegmentType and
+// FileType) as a json document.
+//
+// The Entries are delimited by new line.
+func NewJSONPacker(w io.Writer) Packer {
+	return &jsonPacker{
+		w:    w,
+		e:    json.NewEncoder(w),
+		seen: seenNames{},
+	}
+}
+
+/*
+TODO(vbatts) perhaps have a more compact packer/unpacker, maybe using msgapck
+(https://github.com/ugorji/go)
+
+
+Even though, since our jsonUnpacker and jsonPacker just take
+io.Reader/io.Writer, then we can get away with passing them a
+gzip.Reader/gzip.Writer
+*/

vendor/github.com/vbatts/tar-split/tar/storage/packer_test.go

@@ -0,0 +1,218 @@
+package storage
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestDuplicateFail(t *testing.T) {
+	e := []Entry{
+		Entry{
+			Type:    FileType,
+			Name:    "./hurr.txt",
+			Payload: []byte("abcde"),
+		},
+		Entry{
+			Type:    FileType,
+			Name:    "./hurr.txt",
+			Payload: []byte("deadbeef"),
+		},
+		Entry{
+			Type:    FileType,
+			Name:    "hurr.txt", // slightly different path, same file though
+			Payload: []byte("deadbeef"),
+		},
+	}
+	buf := []byte{}
+	b := bytes.NewBuffer(buf)
+
+	jp := NewJSONPacker(b)
+	if _, err := jp.AddEntry(e[0]); err != nil {
+		t.Error(err)
+	}
+	if _, err := jp.AddEntry(e[1]); err != ErrDuplicatePath {
+		t.Errorf("expected failure on duplicate path")
+	}
+	if _, err := jp.AddEntry(e[2]); err != ErrDuplicatePath {
+		t.Errorf("expected failure on duplicate path")
+	}
+}
+
+func TestJSONPackerUnpacker(t *testing.T) {
+	e := []Entry{
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("how"),
+		},
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("y'all"),
+		},
+		Entry{
+			Type:    FileType,
+			Name:    "./hurr.txt",
+			Payload: []byte("deadbeef"),
+		},
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("doin"),
+		},
+	}
+
+	buf := []byte{}
+	b := bytes.NewBuffer(buf)
+
+	func() {
+		jp := NewJSONPacker(b)
+		for i := range e {
+			if _, err := jp.AddEntry(e[i]); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	// >> packer_test.go:43: uncompressed: 266
+	//t.Errorf("uncompressed: %d", len(b.Bytes()))
+
+	b = bytes.NewBuffer(b.Bytes())
+	entries := Entries{}
+	func() {
+		jup := NewJSONUnpacker(b)
+		for {
+			entry, err := jup.Next()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				t.Error(err)
+			}
+			entries = append(entries, *entry)
+			t.Logf("got %#v", entry)
+		}
+	}()
+	if len(entries) != len(e) {
+		t.Errorf("expected %d entries, got %d", len(e), len(entries))
+	}
+}
+
+// you can use a compress Reader/Writer and make nice savings.
+//
+// For these two tests that are using the same set, it the difference of 266
+// bytes uncompressed vs 138 bytes compressed.
+func TestGzip(t *testing.T) {
+	e := []Entry{
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("how"),
+		},
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("y'all"),
+		},
+		Entry{
+			Type:    FileType,
+			Name:    "./hurr.txt",
+			Payload: []byte("deadbeef"),
+		},
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("doin"),
+		},
+	}
+
+	buf := []byte{}
+	b := bytes.NewBuffer(buf)
+	gzW := gzip.NewWriter(b)
+	jp := NewJSONPacker(gzW)
+	for i := range e {
+		if _, err := jp.AddEntry(e[i]); err != nil {
+			t.Error(err)
+		}
+	}
+	gzW.Close()
+
+	// >> packer_test.go:99: compressed: 138
+	//t.Errorf("compressed: %d", len(b.Bytes()))
+
+	b = bytes.NewBuffer(b.Bytes())
+	gzR, err := gzip.NewReader(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	entries := Entries{}
+	func() {
+		jup := NewJSONUnpacker(gzR)
+		for {
+			entry, err := jup.Next()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				t.Error(err)
+			}
+			entries = append(entries, *entry)
+			t.Logf("got %#v", entry)
+		}
+	}()
+	if len(entries) != len(e) {
+		t.Errorf("expected %d entries, got %d", len(e), len(entries))
+	}
+}
+
+func BenchmarkGetPut(b *testing.B) {
+	e := []Entry{
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("how"),
+		},
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("y'all"),
+		},
+		Entry{
+			Type:    FileType,
+			Name:    "./hurr.txt",
+			Payload: []byte("deadbeef"),
+		},
+		Entry{
+			Type:    SegmentType,
+			Payload: []byte("doin"),
+		},
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			func() {
+				fh, err := ioutil.TempFile("", "tar-split.")
+				if err != nil {
+					b.Fatal(err)
+				}
+				defer os.Remove(fh.Name())
+				defer fh.Close()
+
+				jp := NewJSONPacker(fh)
+				for i := range e {
+					if _, err := jp.AddEntry(e[i]); err != nil {
+						b.Fatal(err)
+					}
+				}
+				fh.Sync()
+
+				up := NewJSONUnpacker(fh)
+				for {
+					_, err := up.Next()
+					if err != nil {
+						if err == io.EOF {
+							break
+						}
+						b.Fatal(err)
+					}
+				}
+
+			}()
+		}
+	})
+}