Update the vendor folder (#53)

* Update the vendor folder

* Update the Gopkg.lock by running dep ensure

vendor/github.com/vbatts/tar-split/tar/asm/README.md

@@ -0,0 +1,44 @@
+asm
+===
+
+This library for assembly and disassembly of tar archives, facilitated by
+`github.com/vbatts/tar-split/tar/storage`.
+
+
+Concerns
+--------
+
+For completely safe assembly/disassembly, there will need to be a Content
+Addressable Storage (CAS) directory, that maps to a checksum in the
+`storage.Entity` of `storage.FileType`.
+
+This is due to the fact that tar archives _can_ allow multiple records for the
+same path, but the last one effectively wins. Even if the prior records had a
+different payload. 
+
+In this way, when assembling an archive from relative paths, if the archive has
+multiple entries for the same path, then all payloads read in from a relative
+path would be identical.
+
+
+Thoughts
+--------
+
+Have a look-aside directory or storage. This way when a clobbering record is
+encountered from the tar stream, then the payload of the prior/existing file is
+stored to the CAS. This way the clobbering record's file payload can be
+extracted, but we'll have preserved the payload needed to reassemble a precise
+tar archive.
+
+clobbered/path/to/file.[0-N]
+
+*alternatively*
+
+We could just _not_ support tar streams that have clobbering file paths.
+Appending records to the archive is not incredibly common, and doesn't happen
+by default for most implementations.  Not supporting them wouldn't be a
+security concern either, as if it did occur, we would reassemble an archive
+that doesn't validate signature/checksum, so it shouldn't be trusted anyway.
+
+Otherwise, this will allow us to defer support for appended files as a FUTURE FEATURE.
+

vendor/github.com/vbatts/tar-split/tar/asm/assemble.go

@@ -0,0 +1,130 @@
+package asm
+
+import (
+	"bytes"
+	"fmt"
+	"hash"
+	"hash/crc64"
+	"io"
+	"sync"
+
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+// NewOutputTarStream returns an io.ReadCloser that is an assembled tar archive
+// stream.
+//
+// It takes a storage.FileGetter, for mapping the file payloads that are to be read in,
+// and a storage.Unpacker, which has access to the rawbytes and file order
+// metadata. With the combination of these two items, a precise assembled Tar
+// archive is possible.
+func NewOutputTarStream(fg storage.FileGetter, up storage.Unpacker) io.ReadCloser {
+	// ... Since these are interfaces, this is possible, so let's not have a nil pointer
+	if fg == nil || up == nil {
+		return nil
+	}
+	pr, pw := io.Pipe()
+	go func() {
+		err := WriteOutputTarStream(fg, up, pw)
+		if err != nil {
+			pw.CloseWithError(err)
+		} else {
+			pw.Close()
+		}
+	}()
+	return pr
+}
+
+// WriteOutputTarStream writes assembled tar archive to a writer.
+func WriteOutputTarStream(fg storage.FileGetter, up storage.Unpacker, w io.Writer) error {
+	// ... Since these are interfaces, this is possible, so let's not have a nil pointer
+	if fg == nil || up == nil {
+		return nil
+	}
+	var copyBuffer []byte
+	var crcHash hash.Hash
+	var crcSum []byte
+	var multiWriter io.Writer
+	for {
+		entry, err := up.Next()
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+		switch entry.Type {
+		case storage.SegmentType:
+			if _, err := w.Write(entry.Payload); err != nil {
+				return err
+			}
+		case storage.FileType:
+			if entry.Size == 0 {
+				continue
+			}
+			fh, err := fg.Get(entry.GetName())
+			if err != nil {
+				return err
+			}
+			if crcHash == nil {
+				crcHash = crc64.New(storage.CRCTable)
+				crcSum = make([]byte, 8)
+				multiWriter = io.MultiWriter(w, crcHash)
+				copyBuffer = byteBufferPool.Get().([]byte)
+				defer byteBufferPool.Put(copyBuffer)
+			} else {
+				crcHash.Reset()
+			}
+
+			if _, err := copyWithBuffer(multiWriter, fh, copyBuffer); err != nil {
+				fh.Close()
+				return err
+			}
+
+			if !bytes.Equal(crcHash.Sum(crcSum[:0]), entry.Payload) {
+				// I would rather this be a comparable ErrInvalidChecksum or such,
+				// but since it's coming through the PipeReader, the context of
+				// _which_ file would be lost...
+				fh.Close()
+				return fmt.Errorf("file integrity checksum failed for %q", entry.GetName())
+			}
+			fh.Close()
+		}
+	}
+}
+
+var byteBufferPool = &sync.Pool{
+	New: func() interface{} {
+		return make([]byte, 32*1024)
+	},
+}
+
+// copyWithBuffer is taken from stdlib io.Copy implementation
+// https://github.com/golang/go/blob/go1.5.1/src/io/io.go#L367
+func copyWithBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
+	for {
+		nr, er := src.Read(buf)
+		if nr > 0 {
+			nw, ew := dst.Write(buf[0:nr])
+			if nw > 0 {
+				written += int64(nw)
+			}
+			if ew != nil {
+				err = ew
+				break
+			}
+			if nr != nw {
+				err = io.ErrShortWrite
+				break
+			}
+		}
+		if er == io.EOF {
+			break
+		}
+		if er != nil {
+			err = er
+			break
+		}
+	}
+	return written, err
+}

vendor/github.com/vbatts/tar-split/tar/asm/assemble_test.go

@@ -0,0 +1,256 @@
+package asm
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/sha1"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+var entries = []struct {
+	Entry storage.Entry
+	Body  []byte
+}{
+	{
+		Entry: storage.Entry{
+			Type:    storage.FileType,
+			Name:    "./hurr.txt",
+			Payload: []byte{2, 116, 164, 177, 171, 236, 107, 78},
+			Size:    20,
+		},
+		Body: []byte("imma hurr til I derp"),
+	},
+	{
+		Entry: storage.Entry{
+			Type:    storage.FileType,
+			Name:    "./ermahgerd.txt",
+			Payload: []byte{126, 72, 89, 239, 230, 252, 160, 187},
+			Size:    26,
+		},
+		Body: []byte("café con leche, por favor"),
+	},
+	{
+		Entry: storage.Entry{
+			Type:    storage.FileType,
+			NameRaw: []byte{0x66, 0x69, 0x6c, 0x65, 0x2d, 0xe4}, // this is invalid UTF-8. Just checking the round trip.
+			Payload: []byte{126, 72, 89, 239, 230, 252, 160, 187},
+			Size:    26,
+		},
+		Body: []byte("café con leche, por favor"),
+	},
+}
+var entriesMangled = []struct {
+	Entry storage.Entry
+	Body  []byte
+}{
+	{
+		Entry: storage.Entry{
+			Type:    storage.FileType,
+			Name:    "./hurr.txt",
+			Payload: []byte{3, 116, 164, 177, 171, 236, 107, 78},
+			Size:    20,
+		},
+		// switch
+		Body: []byte("imma derp til I hurr"),
+	},
+	{
+		Entry: storage.Entry{
+			Type:    storage.FileType,
+			Name:    "./ermahgerd.txt",
+			Payload: []byte{127, 72, 89, 239, 230, 252, 160, 187},
+			Size:    26,
+		},
+		// san not con
+		Body: []byte("café sans leche, por favor"),
+	},
+	{
+		Entry: storage.Entry{
+			Type:    storage.FileType,
+			NameRaw: []byte{0x66, 0x69, 0x6c, 0x65, 0x2d, 0xe4},
+			Payload: []byte{127, 72, 89, 239, 230, 252, 160, 187},
+			Size:    26,
+		},
+		Body: []byte("café con leche, por favor"),
+	},
+}
+
+func TestTarStreamMangledGetterPutter(t *testing.T) {
+	fgp := storage.NewBufferFileGetPutter()
+
+	// first lets prep a GetPutter and Packer
+	for i := range entries {
+		if entries[i].Entry.Type == storage.FileType {
+			j, csum, err := fgp.Put(entries[i].Entry.GetName(), bytes.NewBuffer(entries[i].Body))
+			if err != nil {
+				t.Error(err)
+			}
+			if j != entries[i].Entry.Size {
+				t.Errorf("size %q: expected %d; got %d",
+					entries[i].Entry.GetName(),
+					entries[i].Entry.Size,
+					j)
+			}
+			if !bytes.Equal(csum, entries[i].Entry.Payload) {
+				t.Errorf("checksum %q: expected %v; got %v",
+					entries[i].Entry.GetName(),
+					entries[i].Entry.Payload,
+					csum)
+			}
+		}
+	}
+
+	for _, e := range entriesMangled {
+		if e.Entry.Type == storage.FileType {
+			rdr, err := fgp.Get(e.Entry.GetName())
+			if err != nil {
+				t.Error(err)
+			}
+			c := crc64.New(storage.CRCTable)
+			i, err := io.Copy(c, rdr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rdr.Close()
+
+			csum := c.Sum(nil)
+			if bytes.Equal(csum, e.Entry.Payload) {
+				t.Errorf("wrote %d bytes. checksum for %q should not have matched! %v",
+					i,
+					e.Entry.GetName(),
+					csum)
+			}
+		}
+	}
+}
+
+var testCases = []struct {
+	path            string
+	expectedSHA1Sum string
+	expectedSize    int64
+}{
+	{"./testdata/t.tar.gz", "1eb237ff69bca6e22789ecb05b45d35ca307adbd", 10240},
+	{"./testdata/longlink.tar.gz", "d9f6babe107b7247953dff6b5b5ae31a3a880add", 20480},
+	{"./testdata/fatlonglink.tar.gz", "8537f03f89aeef537382f8b0bb065d93e03b0be8", 26234880},
+	{"./testdata/iso-8859.tar.gz", "ddafa51cb03c74ec117ab366ee2240d13bba1ec3", 10240},
+	{"./testdata/extranils.tar.gz", "e187b4b3e739deaccc257342f4940f34403dc588", 10648},
+	{"./testdata/notenoughnils.tar.gz", "72f93f41efd95290baa5c174c234f5d4c22ce601", 512},
+}
+
+func TestTarStream(t *testing.T) {
+
+	for _, tc := range testCases {
+		fh, err := os.Open(tc.path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer fh.Close()
+		gzRdr, err := gzip.NewReader(fh)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer gzRdr.Close()
+
+		// Setup where we'll store the metadata
+		w := bytes.NewBuffer([]byte{})
+		sp := storage.NewJSONPacker(w)
+		fgp := storage.NewBufferFileGetPutter()
+
+		// wrap the disassembly stream
+		tarStream, err := NewInputTarStream(gzRdr, sp, fgp)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// get a sum of the stream after it has passed through to ensure it's the same.
+		h0 := sha1.New()
+		i, err := io.Copy(h0, tarStream)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if i != tc.expectedSize {
+			t.Errorf("size of tar: expected %d; got %d", tc.expectedSize, i)
+		}
+		if fmt.Sprintf("%x", h0.Sum(nil)) != tc.expectedSHA1Sum {
+			t.Fatalf("checksum of tar: expected %s; got %x", tc.expectedSHA1Sum, h0.Sum(nil))
+		}
+
+		//t.Logf("%s", w.String()) // if we fail, then show the packed info
+
+		// If we've made it this far, then we'll turn it around and create a tar
+		// stream from the packed metadata and buffered file contents.
+		r := bytes.NewBuffer(w.Bytes())
+		sup := storage.NewJSONUnpacker(r)
+		// and reuse the fgp that we Put the payloads to.
+
+		rc := NewOutputTarStream(fgp, sup)
+		h1 := sha1.New()
+		i, err = io.Copy(h1, rc)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if i != tc.expectedSize {
+			t.Errorf("size of output tar: expected %d; got %d", tc.expectedSize, i)
+		}
+		if fmt.Sprintf("%x", h1.Sum(nil)) != tc.expectedSHA1Sum {
+			t.Fatalf("checksum of output tar: expected %s; got %x", tc.expectedSHA1Sum, h1.Sum(nil))
+		}
+	}
+}
+
+func BenchmarkAsm(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		for _, tc := range testCases {
+			func() {
+				fh, err := os.Open(tc.path)
+				if err != nil {
+					b.Fatal(err)
+				}
+				defer fh.Close()
+				gzRdr, err := gzip.NewReader(fh)
+				if err != nil {
+					b.Fatal(err)
+				}
+				defer gzRdr.Close()
+
+				// Setup where we'll store the metadata
+				w := bytes.NewBuffer([]byte{})
+				sp := storage.NewJSONPacker(w)
+				fgp := storage.NewBufferFileGetPutter()
+
+				// wrap the disassembly stream
+				tarStream, err := NewInputTarStream(gzRdr, sp, fgp)
+				if err != nil {
+					b.Fatal(err)
+				}
+				// read it all to the bit bucket
+				i1, err := io.Copy(ioutil.Discard, tarStream)
+				if err != nil {
+					b.Fatal(err)
+				}
+
+				r := bytes.NewBuffer(w.Bytes())
+				sup := storage.NewJSONUnpacker(r)
+				// and reuse the fgp that we Put the payloads to.
+
+				rc := NewOutputTarStream(fgp, sup)
+
+				i2, err := io.Copy(ioutil.Discard, rc)
+				if err != nil {
+					b.Fatal(err)
+				}
+				if i1 != i2 {
+					b.Errorf("%s: input(%d) and ouput(%d) byte count didn't match", tc.path, i1, i2)
+				}
+			}()
+		}
+	}
+}

vendor/github.com/vbatts/tar-split/tar/asm/disassemble.go

@@ -0,0 +1,154 @@
+package asm
+
+import (
+	"io"
+
+	"github.com/vbatts/tar-split/archive/tar"
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+// NewInputTarStream wraps the Reader stream of a tar archive and provides a
+// Reader stream of the same.
+//
+// In the middle it will pack the segments and file metadata to storage.Packer
+// `p`.
+//
+// The the storage.FilePutter is where payload of files in the stream are
+// stashed. If this stashing is not needed, you can provide a nil
+// storage.FilePutter. Since the checksumming is still needed, then a default
+// of NewDiscardFilePutter will be used internally
+func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io.Reader, error) {
+	// What to do here... folks will want their own access to the Reader that is
+	// their tar archive stream, but we'll need that same stream to use our
+	// forked 'archive/tar'.
+	// Perhaps do an io.TeeReader that hands back an io.Reader for them to read
+	// from, and we'll MITM the stream to store metadata.
+	// We'll need a storage.FilePutter too ...
+
+	// Another concern, whether to do any storage.FilePutter operations, such that we
+	// don't extract any amount of the archive. But then again, we're not making
+	// files/directories, hardlinks, etc. Just writing the io to the storage.FilePutter.
+	// Perhaps we have a DiscardFilePutter that is a bit bucket.
+
+	// we'll return the pipe reader, since TeeReader does not buffer and will
+	// only read what the outputRdr Read's. Since Tar archives have padding on
+	// the end, we want to be the one reading the padding, even if the user's
+	// `archive/tar` doesn't care.
+	pR, pW := io.Pipe()
+	outputRdr := io.TeeReader(r, pW)
+
+	// we need a putter that will generate the crc64 sums of file payloads
+	if fp == nil {
+		fp = storage.NewDiscardFilePutter()
+	}
+
+	go func() {
+		tr := tar.NewReader(outputRdr)
+		tr.RawAccounting = true
+		for {
+			hdr, err := tr.Next()
+			if err != nil {
+				if err != io.EOF {
+					pW.CloseWithError(err)
+					return
+				}
+				// even when an EOF is reached, there is often 1024 null bytes on
+				// the end of an archive. Collect them too.
+				if b := tr.RawBytes(); len(b) > 0 {
+					_, err := p.AddEntry(storage.Entry{
+						Type:    storage.SegmentType,
+						Payload: b,
+					})
+					if err != nil {
+						pW.CloseWithError(err)
+						return
+					}
+				}
+				break // not return. We need the end of the reader.
+			}
+			if hdr == nil {
+				break // not return. We need the end of the reader.
+			}
+
+			if b := tr.RawBytes(); len(b) > 0 {
+				_, err := p.AddEntry(storage.Entry{
+					Type:    storage.SegmentType,
+					Payload: b,
+				})
+				if err != nil {
+					pW.CloseWithError(err)
+					return
+				}
+			}
+
+			var csum []byte
+			if hdr.Size > 0 {
+				var err error
+				_, csum, err = fp.Put(hdr.Name, tr)
+				if err != nil {
+					pW.CloseWithError(err)
+					return
+				}
+			}
+
+			entry := storage.Entry{
+				Type:    storage.FileType,
+				Size:    hdr.Size,
+				Payload: csum,
+			}
+			// For proper marshalling of non-utf8 characters
+			entry.SetName(hdr.Name)
+
+			// File entries added, regardless of size
+			_, err = p.AddEntry(entry)
+			if err != nil {
+				pW.CloseWithError(err)
+				return
+			}
+
+			if b := tr.RawBytes(); len(b) > 0 {
+				_, err = p.AddEntry(storage.Entry{
+					Type:    storage.SegmentType,
+					Payload: b,
+				})
+				if err != nil {
+					pW.CloseWithError(err)
+					return
+				}
+			}
+		}
+
+		// It is allowable, and not uncommon that there is further padding on
+		// the end of an archive, apart from the expected 1024 null bytes. We
+		// do this in chunks rather than in one go to avoid cases where a
+		// maliciously crafted tar file tries to trick us into reading many GBs
+		// into memory.
+		const paddingChunkSize = 1024 * 1024
+		var paddingChunk [paddingChunkSize]byte
+		for {
+			var isEOF bool
+			n, err := outputRdr.Read(paddingChunk[:])
+			if err != nil {
+				if err != io.EOF {
+					pW.CloseWithError(err)
+					return
+				}
+				isEOF = true
+			}
+			_, err = p.AddEntry(storage.Entry{
+				Type:    storage.SegmentType,
+				Payload: paddingChunk[:n],
+			})
+			if err != nil {
+				pW.CloseWithError(err)
+				return
+			}
+			if isEOF {
+				break
+			}
+		}
+		pW.Close()
+	}()
+
+	return pR, nil
+}

vendor/github.com/vbatts/tar-split/tar/asm/disassemble_test.go

@@ -0,0 +1,72 @@
+package asm
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+// This test failing causes the binary to crash due to memory overcommitment.
+func TestLargeJunkPadding(t *testing.T) {
+	pR, pW := io.Pipe()
+
+	// Write a normal tar file into the pipe and then load it full of junk
+	// bytes as padding. We have to do this in a goroutine because we can't
+	// store 20GB of junk in-memory.
+	go func() {
+		// Empty archive.
+		tw := tar.NewWriter(pW)
+		if err := tw.Close(); err != nil {
+			pW.CloseWithError(err)
+			t.Fatal(err)
+			return
+		}
+
+		// Write junk.
+		const (
+			junkChunkSize = 64 * 1024 * 1024
+			junkChunkNum  = 20 * 16
+		)
+		devZero, err := os.Open("/dev/zero")
+		if err != nil {
+			pW.CloseWithError(err)
+			t.Fatal(err)
+			return
+		}
+		defer devZero.Close()
+		for i := 0; i < junkChunkNum; i++ {
+			if i%32 == 0 {
+				fmt.Fprintf(os.Stderr, "[TestLargeJunkPadding] junk chunk #%d/#%d\n", i, junkChunkNum)
+			}
+			if _, err := io.CopyN(pW, devZero, junkChunkSize); err != nil {
+				pW.CloseWithError(err)
+				t.Fatal(err)
+				return
+			}
+		}
+
+		fmt.Fprintln(os.Stderr, "[TestLargeJunkPadding] junk chunk finished")
+		pW.Close()
+	}()
+
+	// Disassemble our junk file.
+	nilPacker := storage.NewJSONPacker(ioutil.Discard)
+	rdr, err := NewInputTarStream(pR, nilPacker, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Copy the entire rdr.
+	_, err = io.Copy(ioutil.Discard, rdr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// At this point, if we haven't crashed then we are not vulnerable to
+	// CVE-2017-14992.
+}

vendor/github.com/vbatts/tar-split/tar/asm/doc.go

@@ -0,0 +1,9 @@
+/*
+Package asm provides the API for streaming assembly and disassembly of tar
+archives.
+
+Using the `github.com/vbatts/tar-split/tar/storage` for Packing/Unpacking the
+metadata for a stream, as well as an implementation of Getting/Putting the file
+entries' payload.
+*/
+package asm