Initial commit

vendor/github.com/hyperhq/hypercli/pkg/tarsum/builder_context.go

@@ -0,0 +1,21 @@
+package tarsum
+
+// BuilderContext is an interface extending TarSum by adding the Remove method.
+// In general there was concern about adding this method to TarSum itself
+// so instead it is being added just to "BuilderContext" which will then
+// only be used during the .dockerignore file processing
+// - see builder/evaluator.go
+type BuilderContext interface {
+	TarSum
+	Remove(string)
+}
+
+func (bc *tarSum) Remove(filename string) {
+	for i, fis := range bc.sums {
+		if fis.Name() == filename {
+			bc.sums = append(bc.sums[:i], bc.sums[i+1:]...)
+			// Note, we don't just return because there could be
+			// more than one with this name
+		}
+	}
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/builder_context_test.go

@@ -0,0 +1,63 @@
+package tarsum
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+// Try to remove tarsum (in the BuilderContext) that do not exists, won't change a thing
+func TestTarSumRemoveNonExistent(t *testing.T) {
+	filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar"
+	reader, err := os.Open(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ts, err := NewTarSum(reader, false, Version0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Read and discard bytes so that it populates sums
+	_, err = io.Copy(ioutil.Discard, ts)
+	if err != nil {
+		t.Errorf("failed to read from %s: %s", filename, err)
+	}
+
+	expected := len(ts.GetSums())
+
+	ts.(BuilderContext).Remove("")
+	ts.(BuilderContext).Remove("Anything")
+
+	if len(ts.GetSums()) != expected {
+		t.Fatalf("Expected %v sums, go %v.", expected, ts.GetSums())
+	}
+}
+
+// Remove a tarsum (in the BuilderContext)
+func TestTarSumRemove(t *testing.T) {
+	filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar"
+	reader, err := os.Open(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ts, err := NewTarSum(reader, false, Version0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Read and discard bytes so that it populates sums
+	_, err = io.Copy(ioutil.Discard, ts)
+	if err != nil {
+		t.Errorf("failed to read from %s: %s", filename, err)
+	}
+
+	expected := len(ts.GetSums()) - 1
+
+	ts.(BuilderContext).Remove("etc/sudoers")
+
+	if len(ts.GetSums()) != expected {
+		t.Fatalf("Expected %v sums, go %v.", expected, len(ts.GetSums()))
+	}
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/fileinfosums.go

@@ -0,0 +1,126 @@
+package tarsum
+
+import "sort"
+
+// FileInfoSumInterface provides an interface for accessing file checksum
+// information within a tar file. This info is accessed through interface
+// so the actual name and sum cannot be melded with.
+type FileInfoSumInterface interface {
+	// File name
+	Name() string
+	// Checksum of this particular file and its headers
+	Sum() string
+	// Position of file in the tar
+	Pos() int64
+}
+
+type fileInfoSum struct {
+	name string
+	sum  string
+	pos  int64
+}
+
+func (fis fileInfoSum) Name() string {
+	return fis.name
+}
+func (fis fileInfoSum) Sum() string {
+	return fis.sum
+}
+func (fis fileInfoSum) Pos() int64 {
+	return fis.pos
+}
+
+// FileInfoSums provides a list of FileInfoSumInterfaces.
+type FileInfoSums []FileInfoSumInterface
+
+// GetFile returns the first FileInfoSumInterface with a matching name.
+func (fis FileInfoSums) GetFile(name string) FileInfoSumInterface {
+	for i := range fis {
+		if fis[i].Name() == name {
+			return fis[i]
+		}
+	}
+	return nil
+}
+
+// GetAllFile returns a FileInfoSums with all matching names.
+func (fis FileInfoSums) GetAllFile(name string) FileInfoSums {
+	f := FileInfoSums{}
+	for i := range fis {
+		if fis[i].Name() == name {
+			f = append(f, fis[i])
+		}
+	}
+	return f
+}
+
+// GetDuplicatePaths returns a FileInfoSums with all duplicated paths.
+func (fis FileInfoSums) GetDuplicatePaths() (dups FileInfoSums) {
+	seen := make(map[string]int, len(fis)) // allocate earl. no need to grow this map.
+	for i := range fis {
+		f := fis[i]
+		if _, ok := seen[f.Name()]; ok {
+			dups = append(dups, f)
+		} else {
+			seen[f.Name()] = 0
+		}
+	}
+	return dups
+}
+
+// Len returns the size of the FileInfoSums.
+func (fis FileInfoSums) Len() int { return len(fis) }
+
+// Swap swaps two FileInfoSum values if a FileInfoSums list.
+func (fis FileInfoSums) Swap(i, j int) { fis[i], fis[j] = fis[j], fis[i] }
+
+// SortByPos sorts FileInfoSums content by position.
+func (fis FileInfoSums) SortByPos() {
+	sort.Sort(byPos{fis})
+}
+
+// SortByNames sorts FileInfoSums content by name.
+func (fis FileInfoSums) SortByNames() {
+	sort.Sort(byName{fis})
+}
+
+// SortBySums sorts FileInfoSums content by sums.
+func (fis FileInfoSums) SortBySums() {
+	dups := fis.GetDuplicatePaths()
+	if len(dups) > 0 {
+		sort.Sort(bySum{fis, dups})
+	} else {
+		sort.Sort(bySum{fis, nil})
+	}
+}
+
+// byName is a sort.Sort helper for sorting by file names.
+// If names are the same, order them by their appearance in the tar archive
+type byName struct{ FileInfoSums }
+
+func (bn byName) Less(i, j int) bool {
+	if bn.FileInfoSums[i].Name() == bn.FileInfoSums[j].Name() {
+		return bn.FileInfoSums[i].Pos() < bn.FileInfoSums[j].Pos()
+	}
+	return bn.FileInfoSums[i].Name() < bn.FileInfoSums[j].Name()
+}
+
+// bySum is a sort.Sort helper for sorting by the sums of all the fileinfos in the tar archive
+type bySum struct {
+	FileInfoSums
+	dups FileInfoSums
+}
+
+func (bs bySum) Less(i, j int) bool {
+	if bs.dups != nil && bs.FileInfoSums[i].Name() == bs.FileInfoSums[j].Name() {
+		return bs.FileInfoSums[i].Pos() < bs.FileInfoSums[j].Pos()
+	}
+	return bs.FileInfoSums[i].Sum() < bs.FileInfoSums[j].Sum()
+}
+
+// byPos is a sort.Sort helper for sorting by the sums of all the fileinfos by their original order
+type byPos struct{ FileInfoSums }
+
+func (bp byPos) Less(i, j int) bool {
+	return bp.FileInfoSums[i].Pos() < bp.FileInfoSums[j].Pos()
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/fileinfosums_test.go

@@ -0,0 +1,62 @@
+package tarsum
+
+import "testing"
+
+func newFileInfoSums() FileInfoSums {
+	return FileInfoSums{
+		fileInfoSum{name: "file3", sum: "2abcdef1234567890", pos: 2},
+		fileInfoSum{name: "dup1", sum: "deadbeef1", pos: 5},
+		fileInfoSum{name: "file1", sum: "0abcdef1234567890", pos: 0},
+		fileInfoSum{name: "file4", sum: "3abcdef1234567890", pos: 3},
+		fileInfoSum{name: "dup1", sum: "deadbeef0", pos: 4},
+		fileInfoSum{name: "file2", sum: "1abcdef1234567890", pos: 1},
+	}
+}
+
+func TestSortFileInfoSums(t *testing.T) {
+	dups := newFileInfoSums().GetAllFile("dup1")
+	if len(dups) != 2 {
+		t.Errorf("expected length 2, got %d", len(dups))
+	}
+	dups.SortByNames()
+	if dups[0].Pos() != 4 {
+		t.Errorf("sorted dups should be ordered by position. Expected 4, got %d", dups[0].Pos())
+	}
+
+	fis := newFileInfoSums()
+	expected := "0abcdef1234567890"
+	fis.SortBySums()
+	got := fis[0].Sum()
+	if got != expected {
+		t.Errorf("Expected %q, got %q", expected, got)
+	}
+
+	fis = newFileInfoSums()
+	expected = "dup1"
+	fis.SortByNames()
+	gotFis := fis[0]
+	if gotFis.Name() != expected {
+		t.Errorf("Expected %q, got %q", expected, gotFis.Name())
+	}
+	// since a duplicate is first, ensure it is ordered first by position too
+	if gotFis.Pos() != 4 {
+		t.Errorf("Expected %d, got %d", 4, gotFis.Pos())
+	}
+
+	fis = newFileInfoSums()
+	fis.SortByPos()
+	if fis[0].Pos() != 0 {
+		t.Errorf("sorted fileInfoSums by Pos should order them by position.")
+	}
+
+	fis = newFileInfoSums()
+	expected = "deadbeef1"
+	gotFileInfoSum := fis.GetFile("dup1")
+	if gotFileInfoSum.Sum() != expected {
+		t.Errorf("Expected %q, got %q", expected, gotFileInfoSum)
+	}
+	if fis.GetFile("noPresent") != nil {
+		t.Errorf("Should have return nil if name not found.")
+	}
+
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/tarsum.go

@@ -0,0 +1,294 @@
+// Package tarsum provides algorithms to perform checksum calculation on
+// filesystem layers.
+//
+// The transportation of filesystems, regarding Docker, is done with tar(1)
+// archives. There are a variety of tar serialization formats [2], and a key
+// concern here is ensuring a repeatable checksum given a set of inputs from a
+// generic tar archive. Types of transportation include distribution to and from a
+// registry endpoint, saving and loading through commands or Docker daemon APIs,
+// transferring the build context from client to Docker daemon, and committing the
+// filesystem of a container to become an image.
+//
+// As tar archives are used for transit, but not preserved in many situations, the
+// focus of the algorithm is to ensure the integrity of the preserved filesystem,
+// while maintaining a deterministic accountability. This includes neither
+// constraining the ordering or manipulation of the files during the creation or
+// unpacking of the archive, nor include additional metadata state about the file
+// system attributes.
+package tarsum
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"strings"
+)
+
+const (
+	buf8K  = 8 * 1024
+	buf16K = 16 * 1024
+	buf32K = 32 * 1024
+)
+
+// NewTarSum creates a new interface for calculating a fixed time checksum of a
+// tar archive.
+//
+// This is used for calculating checksums of layers of an image, in some cases
+// including the byte payload of the image's json metadata as well, and for
+// calculating the checksums for buildcache.
+func NewTarSum(r io.Reader, dc bool, v Version) (TarSum, error) {
+	return NewTarSumHash(r, dc, v, DefaultTHash)
+}
+
+// NewTarSumHash creates a new TarSum, providing a THash to use rather than
+// the DefaultTHash.
+func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) {
+	headerSelector, err := getTarHeaderSelector(v)
+	if err != nil {
+		return nil, err
+	}
+	ts := &tarSum{Reader: r, DisableCompression: dc, tarSumVersion: v, headerSelector: headerSelector, tHash: tHash}
+	err = ts.initTarSum()
+	return ts, err
+}
+
+// NewTarSumForLabel creates a new TarSum using the provided TarSum version+hash label.
+func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) {
+	parts := strings.SplitN(label, "+", 2)
+	if len(parts) != 2 {
+		return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}")
+	}
+
+	versionName, hashName := parts[0], parts[1]
+
+	version, ok := tarSumVersionsByName[versionName]
+	if !ok {
+		return nil, fmt.Errorf("unknown TarSum version name: %q", versionName)
+	}
+
+	hashConfig, ok := standardHashConfigs[hashName]
+	if !ok {
+		return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName)
+	}
+
+	tHash := NewTHash(hashConfig.name, hashConfig.hash.New)
+
+	return NewTarSumHash(r, disableCompression, version, tHash)
+}
+
+// TarSum is the generic interface for calculating fixed time
+// checksums of a tar archive.
+type TarSum interface {
+	io.Reader
+	GetSums() FileInfoSums
+	Sum([]byte) string
+	Version() Version
+	Hash() THash
+}
+
+// tarSum struct is the structure for a Version0 checksum calculation.
+type tarSum struct {
+	io.Reader
+	tarR               *tar.Reader
+	tarW               *tar.Writer
+	writer             writeCloseFlusher
+	bufTar             *bytes.Buffer
+	bufWriter          *bytes.Buffer
+	bufData            []byte
+	h                  hash.Hash
+	tHash              THash
+	sums               FileInfoSums
+	fileCounter        int64
+	currentFile        string
+	finished           bool
+	first              bool
+	DisableCompression bool              // false by default. When false, the output gzip compressed.
+	tarSumVersion      Version           // this field is not exported so it can not be mutated during use
+	headerSelector     tarHeaderSelector // handles selecting and ordering headers for files in the archive
+}
+
+func (ts tarSum) Hash() THash {
+	return ts.tHash
+}
+
+func (ts tarSum) Version() Version {
+	return ts.tarSumVersion
+}
+
+// THash provides a hash.Hash type generator and its name.
+type THash interface {
+	Hash() hash.Hash
+	Name() string
+}
+
+// NewTHash is a convenience method for creating a THash.
+func NewTHash(name string, h func() hash.Hash) THash {
+	return simpleTHash{n: name, h: h}
+}
+
+type tHashConfig struct {
+	name string
+	hash crypto.Hash
+}
+
+var (
+	// NOTE: DO NOT include MD5 or SHA1, which are considered insecure.
+	standardHashConfigs = map[string]tHashConfig{
+		"sha256": {name: "sha256", hash: crypto.SHA256},
+		"sha512": {name: "sha512", hash: crypto.SHA512},
+	}
+)
+
+// DefaultTHash is default TarSum hashing algorithm - "sha256".
+var DefaultTHash = NewTHash("sha256", sha256.New)
+
+type simpleTHash struct {
+	n string
+	h func() hash.Hash
+}
+
+func (sth simpleTHash) Name() string    { return sth.n }
+func (sth simpleTHash) Hash() hash.Hash { return sth.h() }
+
+func (ts *tarSum) encodeHeader(h *tar.Header) error {
+	for _, elem := range ts.headerSelector.selectHeaders(h) {
+		if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ts *tarSum) initTarSum() error {
+	ts.bufTar = bytes.NewBuffer([]byte{})
+	ts.bufWriter = bytes.NewBuffer([]byte{})
+	ts.tarR = tar.NewReader(ts.Reader)
+	ts.tarW = tar.NewWriter(ts.bufTar)
+	if !ts.DisableCompression {
+		ts.writer = gzip.NewWriter(ts.bufWriter)
+	} else {
+		ts.writer = &nopCloseFlusher{Writer: ts.bufWriter}
+	}
+	if ts.tHash == nil {
+		ts.tHash = DefaultTHash
+	}
+	ts.h = ts.tHash.Hash()
+	ts.h.Reset()
+	ts.first = true
+	ts.sums = FileInfoSums{}
+	return nil
+}
+
+func (ts *tarSum) Read(buf []byte) (int, error) {
+	if ts.finished {
+		return ts.bufWriter.Read(buf)
+	}
+	if len(ts.bufData) < len(buf) {
+		switch {
+		case len(buf) <= buf8K:
+			ts.bufData = make([]byte, buf8K)
+		case len(buf) <= buf16K:
+			ts.bufData = make([]byte, buf16K)
+		case len(buf) <= buf32K:
+			ts.bufData = make([]byte, buf32K)
+		default:
+			ts.bufData = make([]byte, len(buf))
+		}
+	}
+	buf2 := ts.bufData[:len(buf)]
+
+	n, err := ts.tarR.Read(buf2)
+	if err != nil {
+		if err == io.EOF {
+			if _, err := ts.h.Write(buf2[:n]); err != nil {
+				return 0, err
+			}
+			if !ts.first {
+				ts.sums = append(ts.sums, fileInfoSum{name: ts.currentFile, sum: hex.EncodeToString(ts.h.Sum(nil)), pos: ts.fileCounter})
+				ts.fileCounter++
+				ts.h.Reset()
+			} else {
+				ts.first = false
+			}
+
+			currentHeader, err := ts.tarR.Next()
+			if err != nil {
+				if err == io.EOF {
+					if err := ts.tarW.Close(); err != nil {
+						return 0, err
+					}
+					if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
+						return 0, err
+					}
+					if err := ts.writer.Close(); err != nil {
+						return 0, err
+					}
+					ts.finished = true
+					return n, nil
+				}
+				return n, err
+			}
+			ts.currentFile = strings.TrimSuffix(strings.TrimPrefix(currentHeader.Name, "./"), "/")
+			if err := ts.encodeHeader(currentHeader); err != nil {
+				return 0, err
+			}
+			if err := ts.tarW.WriteHeader(currentHeader); err != nil {
+				return 0, err
+			}
+			if _, err := ts.tarW.Write(buf2[:n]); err != nil {
+				return 0, err
+			}
+			ts.tarW.Flush()
+			if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
+				return 0, err
+			}
+			ts.writer.Flush()
+
+			return ts.bufWriter.Read(buf)
+		}
+		return n, err
+	}
+
+	// Filling the hash buffer
+	if _, err = ts.h.Write(buf2[:n]); err != nil {
+		return 0, err
+	}
+
+	// Filling the tar writer
+	if _, err = ts.tarW.Write(buf2[:n]); err != nil {
+		return 0, err
+	}
+	ts.tarW.Flush()
+
+	// Filling the output writer
+	if _, err = io.Copy(ts.writer, ts.bufTar); err != nil {
+		return 0, err
+	}
+	ts.writer.Flush()
+
+	return ts.bufWriter.Read(buf)
+}
+
+func (ts *tarSum) Sum(extra []byte) string {
+	ts.sums.SortBySums()
+	h := ts.tHash.Hash()
+	if extra != nil {
+		h.Write(extra)
+	}
+	for _, fis := range ts.sums {
+		h.Write([]byte(fis.Sum()))
+	}
+	checksum := ts.Version().String() + "+" + ts.tHash.Name() + ":" + hex.EncodeToString(h.Sum(nil))
+	return checksum
+}
+
+func (ts *tarSum) GetSums() FileInfoSums {
+	return ts.sums
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/tarsum_spec.md

@@ -0,0 +1,230 @@
+page_title: TarSum checksum specification
+page_description: Documentation for algorithms used in the TarSum checksum calculation
+page_keywords: docker, checksum, validation, tarsum
+
+# TarSum Checksum Specification
+
+## Abstract
+
+This document describes the algorithms used in performing the TarSum checksum
+calculation on filesystem layers, the need for this method over existing
+methods, and the versioning of this calculation.
+
+## Warning
+
+This checksum algorithm is for best-effort comparison of file trees with fuzzy logic.
+
+This is _not_ a cryptographic attestation, and should not be considered secure.
+
+## Introduction
+
+The transportation of filesystems, regarding Docker, is done with tar(1)
+archives. There are a variety of tar serialization formats [2], and a key
+concern here is ensuring a repeatable checksum given a set of inputs from a
+generic tar archive. Types of transportation include distribution to and from a
+registry endpoint, saving and loading through commands or Docker daemon APIs,
+transferring the build context from client to Docker daemon, and committing the
+filesystem of a container to become an image.
+
+As tar archives are used for transit, but not preserved in many situations, the
+focus of the algorithm is to ensure the integrity of the preserved filesystem,
+while maintaining a deterministic accountability. This includes neither
+constraining the ordering or manipulation of the files during the creation or
+unpacking of the archive, nor include additional metadata state about the file
+system attributes.
+
+## Intended Audience
+
+This document is outlining the methods used for consistent checksum calculation
+for filesystems transported via tar archives.
+
+Auditing these methodologies is an open and iterative process. This document
+should accommodate the review of source code. Ultimately, this document should
+be the starting point of further refinements to the algorithm and its future
+versions.
+
+## Concept
+
+The checksum mechanism must ensure the integrity and assurance of the
+filesystem payload.
+
+## Checksum Algorithm Profile
+
+A checksum mechanism must define the following operations and attributes:
+
+* Associated hashing cipher - used to checksum each file payload and attribute
+  information.
+* Checksum list - each file of the filesystem archive has its checksum
+  calculated from the payload and attributes of the file. The final checksum is
+  calculated from this list, with specific ordering.
+* Version - as the algorithm adapts to requirements, there are behaviors of the
+  algorithm to manage by versioning.
+* Archive being calculated - the tar archive having its checksum calculated
+
+## Elements of TarSum checksum
+
+The calculated sum output is a text string. The elements included in the output
+of the calculated sum comprise the information needed for validation of the sum
+(TarSum version and hashing cipher used) and the expected checksum in hexadecimal
+form.
+
+There are two delimiters used:
+* '+' separates TarSum version from hashing cipher
+* ':' separates calculation mechanics from expected hash
+
+Example:
+
+```
+	"tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e"
+	|         |       \                                                               |
+	|         |        \                                                              |
+	|_version_|_cipher__|__                                                           |
+	|                      \                                                          |
+	|_calculation_mechanics_|______________________expected_sum_______________________|
+```
+
+## Versioning
+
+Versioning was introduced [0] to accommodate differences in calculation needed,
+and ability to maintain reverse compatibility.
+
+The general algorithm will be describe further in the 'Calculation'.
+
+### Version0
+
+This is the initial version of TarSum.
+
+Its element in the TarSum checksum string is `tarsum`.
+
+### Version1
+
+Its element in the TarSum checksum is `tarsum.v1`.
+
+The notable changes in this version:
+* Exclusion of file `mtime` from the file information headers, in each file
+  checksum calculation
+* Inclusion of extended attributes (`xattrs`. Also seen as `SCHILY.xattr.` prefixed Pax
+  tar file info headers) keys and values in each file checksum calculation
+
+### VersionDev
+
+*Do not use unless validating refinements to the checksum algorithm*
+
+Its element in the TarSum checksum is `tarsum.dev`.
+
+This is a floating place holder for a next version and grounds for testing
+changes. The methods used for calculation are subject to change without notice,
+and this version is for testing and not for production use.
+
+## Ciphers
+
+The official default and standard hashing cipher used in the calculation mechanic
+is `sha256`. This refers to SHA256 hash algorithm as defined in FIPS 180-4.
+
+Though the TarSum algorithm itself is not exclusively bound to the single
+hashing cipher `sha256`, support for alternate hashing ciphers was later added
+[1]. Use cases for alternate cipher could include future-proofing TarSum
+checksum format and using faster cipher hashes for tar filesystem checksums.
+
+## Calculation
+
+### Requirement
+
+As mentioned earlier, the calculation is such that it takes into consideration
+the lifecycle of the tar archive. In that the tar archive is not an immutable,
+permanent artifact. Otherwise options like relying on a known hashing cipher
+checksum of the archive itself would be reliable enough. The tar archive of the
+filesystem is used as a transportation medium for Docker images, and the
+archive is discarded once its contents are extracted. Therefore, for consistent
+validation items such as order of files in the tar archive and time stamps are
+subject to change once an image is received.
+
+### Process
+
+The method is typically iterative due to reading tar info headers from the
+archive stream, though this is not a strict requirement.
+
+#### Files
+
+Each file in the tar archive have their contents (headers and body) checksummed
+individually using the designated associated hashing cipher. The ordered
+headers of the file are written to the checksum calculation first, and then the
+payload of the file body.
+
+The resulting checksum of the file is appended to the list of file sums. The
+sum is encoded as a string of the hexadecimal digest. Additionally, the file
+name and position in the archive is kept as reference for special ordering.
+
+#### Headers
+
+The following headers are read, in this
+order ( and the corresponding representation of its value):
+* 'name' - string
+* 'mode' - string of the base10 integer
+* 'uid' - string of the integer
+* 'gid' - string of the integer
+* 'size' - string of the integer
+* 'mtime' (_Version0 only_) - string of integer of the seconds since 1970-01-01 00:00:00 UTC
+* 'typeflag' - string of the char
+* 'linkname' - string
+* 'uname' - string
+* 'gname' - string
+* 'devmajor' - string of the integer
+* 'devminor' - string of the integer
+
+For >= Version1, the extended attribute headers ("SCHILY.xattr." prefixed pax
+headers) included after the above list. These xattrs key/values are first
+sorted by the keys.
+
+#### Header Format
+
+The ordered headers are written to the hash in the format of
+
+	"{.key}{.value}"
+
+with no newline.
+
+#### Body
+
+After the order headers of the file have been added to the checksum for the
+file, the body of the file is written to the hash.
+
+#### List of file sums
+
+The list of file sums is sorted by the string of the hexadecimal digest.
+
+If there are two files in the tar with matching paths, the order of occurrence
+for that path is reflected for the sums of the corresponding file header and
+body.
+
+#### Final Checksum
+
+Begin with a fresh or initial state of the associated hash cipher. If there is
+additional payload to include in the TarSum calculation for the archive, it is
+written first. Then each checksum from the ordered list of file sums is written
+to the hash.
+
+The resulting digest is formatted per the Elements of TarSum checksum,
+including the TarSum version, the associated hash cipher and the hexadecimal
+encoded checksum digest.
+
+## Security Considerations
+
+The initial version of TarSum has undergone one update that could invalidate
+handcrafted tar archives. The tar archive format supports appending of files
+with same names as prior files in the archive. The latter file will clobber the
+prior file of the same path. Due to this the algorithm now accounts for files
+with matching paths, and orders the list of file sums accordingly [3].
+
+## Footnotes
+
+* [0] Versioning https://github.com/docker/docker/commit/747f89cd327db9d50251b17797c4d825162226d0
+* [1] Alternate ciphers https://github.com/docker/docker/commit/4e9925d780665149b8bc940d5ba242ada1973c4e
+* [2] Tar http://en.wikipedia.org/wiki/Tar_%28computing%29
+* [3] Name collision https://github.com/docker/docker/commit/c5e6362c53cbbc09ddbabd5a7323e04438b57d31
+
+## Acknowledgements
+
+Joffrey F (shin-) and Guillaume J. Charmes (creack) on the initial work of the
+TarSum calculation.
+

vendor/github.com/hyperhq/hypercli/pkg/tarsum/tarsum_test.go

@@ -0,0 +1,648 @@
+package tarsum
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/md5"
+	"crypto/rand"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+type testLayer struct {
+	filename string
+	options  *sizedOptions
+	jsonfile string
+	gzip     bool
+	tarsum   string
+	version  Version
+	hash     THash
+}
+
+var testLayers = []testLayer{
+	{
+		filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
+		jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
+		version:  Version0,
+		tarsum:   "tarsum+sha256:4095cc12fa5fdb1ab2760377e1cd0c4ecdd3e61b4f9b82319d96fcea6c9a41c6"},
+	{
+		filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
+		jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
+		version:  VersionDev,
+		tarsum:   "tarsum.dev+sha256:db56e35eec6ce65ba1588c20ba6b1ea23743b59e81fb6b7f358ccbde5580345c"},
+	{
+		filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
+		jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
+		gzip:     true,
+		tarsum:   "tarsum+sha256:4095cc12fa5fdb1ab2760377e1cd0c4ecdd3e61b4f9b82319d96fcea6c9a41c6"},
+	{
+		// Tests existing version of TarSum when xattrs are present
+		filename: "testdata/xattr/layer.tar",
+		jsonfile: "testdata/xattr/json",
+		version:  Version0,
+		tarsum:   "tarsum+sha256:07e304a8dbcb215b37649fde1a699f8aeea47e60815707f1cdf4d55d25ff6ab4"},
+	{
+		// Tests next version of TarSum when xattrs are present
+		filename: "testdata/xattr/layer.tar",
+		jsonfile: "testdata/xattr/json",
+		version:  VersionDev,
+		tarsum:   "tarsum.dev+sha256:6c58917892d77b3b357b0f9ad1e28e1f4ae4de3a8006bd3beb8beda214d8fd16"},
+	{
+		filename: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar",
+		jsonfile: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json",
+		tarsum:   "tarsum+sha256:c66bd5ec9f87b8f4c6135ca37684618f486a3dd1d113b138d0a177bfa39c2571"},
+	{
+		options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
+		tarsum:  "tarsum+sha256:8bf12d7e67c51ee2e8306cba569398b1b9f419969521a12ffb9d8875e8836738"},
+	{
+		// this tar has two files with the same path
+		filename: "testdata/collision/collision-0.tar",
+		tarsum:   "tarsum+sha256:08653904a68d3ab5c59e65ef58c49c1581caa3c34744f8d354b3f575ea04424a"},
+	{
+		// this tar has the same two files (with the same path), but reversed order. ensuring is has different hash than above
+		filename: "testdata/collision/collision-1.tar",
+		tarsum:   "tarsum+sha256:b51c13fbefe158b5ce420d2b930eef54c5cd55c50a2ee4abdddea8fa9f081e0d"},
+	{
+		// this tar has newer of collider-0.tar, ensuring is has different hash
+		filename: "testdata/collision/collision-2.tar",
+		tarsum:   "tarsum+sha256:381547080919bb82691e995508ae20ed33ce0f6948d41cafbeb70ce20c73ee8e"},
+	{
+		// this tar has newer of collider-1.tar, ensuring is has different hash
+		filename: "testdata/collision/collision-3.tar",
+		tarsum:   "tarsum+sha256:f886e431c08143164a676805205979cd8fa535dfcef714db5515650eea5a7c0f"},
+	{
+		options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
+		tarsum:  "tarsum+md5:0d7529ec7a8360155b48134b8e599f53",
+		hash:    md5THash,
+	},
+	{
+		options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
+		tarsum:  "tarsum+sha1:f1fee39c5925807ff75ef1925e7a23be444ba4df",
+		hash:    sha1Hash,
+	},
+	{
+		options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
+		tarsum:  "tarsum+sha224:6319390c0b061d639085d8748b14cd55f697cf9313805218b21cf61c",
+		hash:    sha224Hash,
+	},
+	{
+		options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
+		tarsum:  "tarsum+sha384:a578ce3ce29a2ae03b8ed7c26f47d0f75b4fc849557c62454be4b5ffd66ba021e713b48ce71e947b43aab57afd5a7636",
+		hash:    sha384Hash,
+	},
+	{
+		options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
+		tarsum:  "tarsum+sha512:e9bfb90ca5a4dfc93c46ee061a5cf9837de6d2fdf82544d6460d3147290aecfabf7b5e415b9b6e72db9b8941f149d5d69fb17a394cbfaf2eac523bd9eae21855",
+		hash:    sha512Hash,
+	},
+}
+
+type sizedOptions struct {
+	num      int64
+	size     int64
+	isRand   bool
+	realFile bool
+}
+
+// make a tar:
+// * num is the number of files the tar should have
+// * size is the bytes per file
+// * isRand is whether the contents of the files should be a random chunk (otherwise it's all zeros)
+// * realFile will write to a TempFile, instead of an in memory buffer
+func sizedTar(opts sizedOptions) io.Reader {
+	var (
+		fh  io.ReadWriter
+		err error
+	)
+	if opts.realFile {
+		fh, err = ioutil.TempFile("", "tarsum")
+		if err != nil {
+			return nil
+		}
+	} else {
+		fh = bytes.NewBuffer([]byte{})
+	}
+	tarW := tar.NewWriter(fh)
+	defer tarW.Close()
+	for i := int64(0); i < opts.num; i++ {
+		err := tarW.WriteHeader(&tar.Header{
+			Name: fmt.Sprintf("/testdata%d", i),
+			Mode: 0755,
+			Uid:  0,
+			Gid:  0,
+			Size: opts.size,
+		})
+		if err != nil {
+			return nil
+		}
+		var rBuf []byte
+		if opts.isRand {
+			rBuf = make([]byte, 8)
+			_, err = rand.Read(rBuf)
+			if err != nil {
+				return nil
+			}
+		} else {
+			rBuf = []byte{0, 0, 0, 0, 0, 0, 0, 0}
+		}
+
+		for i := int64(0); i < opts.size/int64(8); i++ {
+			tarW.Write(rBuf)
+		}
+	}
+	return fh
+}
+
+func emptyTarSum(gzip bool) (TarSum, error) {
+	reader, writer := io.Pipe()
+	tarWriter := tar.NewWriter(writer)
+
+	// Immediately close tarWriter and write-end of the
+	// Pipe in a separate goroutine so we don't block.
+	go func() {
+		tarWriter.Close()
+		writer.Close()
+	}()
+
+	return NewTarSum(reader, !gzip, Version0)
+}
+
+// Test errors on NewTarsumForLabel
+func TestNewTarSumForLabelInvalid(t *testing.T) {
+	reader := strings.NewReader("")
+
+	if _, err := NewTarSumForLabel(reader, true, "invalidlabel"); err == nil {
+		t.Fatalf("Expected an error, got nothing.")
+	}
+
+	if _, err := NewTarSumForLabel(reader, true, "invalid+sha256"); err == nil {
+		t.Fatalf("Expected an error, got nothing.")
+	}
+	if _, err := NewTarSumForLabel(reader, true, "tarsum.v1+invalid"); err == nil {
+		t.Fatalf("Expected an error, got nothing.")
+	}
+}
+
+func TestNewTarSumForLabel(t *testing.T) {
+
+	layer := testLayers[0]
+
+	reader, err := os.Open(layer.filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	label := strings.Split(layer.tarsum, ":")[0]
+	ts, err := NewTarSumForLabel(reader, false, label)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure it actually worked by reading a little bit of it
+	nbByteToRead := 8 * 1024
+	dBuf := make([]byte, nbByteToRead)
+	_, err = ts.Read(dBuf)
+	if err != nil {
+		t.Errorf("failed to read %vKB from %s: %s", nbByteToRead, layer.filename, err)
+	}
+}
+
+// TestEmptyTar tests that tarsum does not fail to read an empty tar
+// and correctly returns the hex digest of an empty hash.
+func TestEmptyTar(t *testing.T) {
+	// Test without gzip.
+	ts, err := emptyTarSum(false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	zeroBlock := make([]byte, 1024)
+	buf := new(bytes.Buffer)
+
+	n, err := io.Copy(buf, ts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if n != int64(len(zeroBlock)) || !bytes.Equal(buf.Bytes(), zeroBlock) {
+		t.Fatalf("tarSum did not write the correct number of zeroed bytes: %d", n)
+	}
+
+	expectedSum := ts.Version().String() + "+sha256:" + hex.EncodeToString(sha256.New().Sum(nil))
+	resultSum := ts.Sum(nil)
+
+	if resultSum != expectedSum {
+		t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum)
+	}
+
+	// Test with gzip.
+	ts, err = emptyTarSum(true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	buf.Reset()
+
+	n, err = io.Copy(buf, ts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bufgz := new(bytes.Buffer)
+	gz := gzip.NewWriter(bufgz)
+	n, err = io.Copy(gz, bytes.NewBuffer(zeroBlock))
+	gz.Close()
+	gzBytes := bufgz.Bytes()
+
+	if n != int64(len(zeroBlock)) || !bytes.Equal(buf.Bytes(), gzBytes) {
+		t.Fatalf("tarSum did not write the correct number of gzipped-zeroed bytes: %d", n)
+	}
+
+	resultSum = ts.Sum(nil)
+
+	if resultSum != expectedSum {
+		t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum)
+	}
+
+	// Test without ever actually writing anything.
+	if ts, err = NewTarSum(bytes.NewReader([]byte{}), true, Version0); err != nil {
+		t.Fatal(err)
+	}
+
+	resultSum = ts.Sum(nil)
+
+	if resultSum != expectedSum {
+		t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum)
+	}
+}
+
+var (
+	md5THash   = NewTHash("md5", md5.New)
+	sha1Hash   = NewTHash("sha1", sha1.New)
+	sha224Hash = NewTHash("sha224", sha256.New224)
+	sha384Hash = NewTHash("sha384", sha512.New384)
+	sha512Hash = NewTHash("sha512", sha512.New)
+)
+
+// Test all the build-in read size : buf8K, buf16K, buf32K and more
+func TestTarSumsReadSize(t *testing.T) {
+	// Test always on the same layer (that is big enough)
+	layer := testLayers[0]
+
+	for i := 0; i < 5; i++ {
+
+		reader, err := os.Open(layer.filename)
+		if err != nil {
+			t.Fatal(err)
+		}
+		ts, err := NewTarSum(reader, false, layer.version)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Read and discard bytes so that it populates sums
+		nbByteToRead := (i + 1) * 8 * 1024
+		dBuf := make([]byte, nbByteToRead)
+		_, err = ts.Read(dBuf)
+		if err != nil {
+			t.Errorf("failed to read %vKB from %s: %s", nbByteToRead, layer.filename, err)
+			continue
+		}
+	}
+}
+
+func TestTarSums(t *testing.T) {
+	for _, layer := range testLayers {
+		var (
+			fh  io.Reader
+			err error
+		)
+		if len(layer.filename) > 0 {
+			fh, err = os.Open(layer.filename)
+			if err != nil {
+				t.Errorf("failed to open %s: %s", layer.filename, err)
+				continue
+			}
+		} else if layer.options != nil {
+			fh = sizedTar(*layer.options)
+		} else {
+			// What else is there to test?
+			t.Errorf("what to do with %#v", layer)
+			continue
+		}
+		if file, ok := fh.(*os.File); ok {
+			defer file.Close()
+		}
+
+		var ts TarSum
+		if layer.hash == nil {
+			//                           double negatives!
+			ts, err = NewTarSum(fh, !layer.gzip, layer.version)
+		} else {
+			ts, err = NewTarSumHash(fh, !layer.gzip, layer.version, layer.hash)
+		}
+		if err != nil {
+			t.Errorf("%q :: %q", err, layer.filename)
+			continue
+		}
+
+		// Read variable number of bytes to test dynamic buffer
+		dBuf := make([]byte, 1)
+		_, err = ts.Read(dBuf)
+		if err != nil {
+			t.Errorf("failed to read 1B from %s: %s", layer.filename, err)
+			continue
+		}
+		dBuf = make([]byte, 16*1024)
+		_, err = ts.Read(dBuf)
+		if err != nil {
+			t.Errorf("failed to read 16KB from %s: %s", layer.filename, err)
+			continue
+		}
+
+		// Read and discard remaining bytes
+		_, err = io.Copy(ioutil.Discard, ts)
+		if err != nil {
+			t.Errorf("failed to copy from %s: %s", layer.filename, err)
+			continue
+		}
+		var gotSum string
+		if len(layer.jsonfile) > 0 {
+			jfh, err := os.Open(layer.jsonfile)
+			if err != nil {
+				t.Errorf("failed to open %s: %s", layer.jsonfile, err)
+				continue
+			}
+			buf, err := ioutil.ReadAll(jfh)
+			if err != nil {
+				t.Errorf("failed to readAll %s: %s", layer.jsonfile, err)
+				continue
+			}
+			gotSum = ts.Sum(buf)
+		} else {
+			gotSum = ts.Sum(nil)
+		}
+
+		if layer.tarsum != gotSum {
+			t.Errorf("expecting [%s], but got [%s]", layer.tarsum, gotSum)
+		}
+		var expectedHashName string
+		if layer.hash != nil {
+			expectedHashName = layer.hash.Name()
+		} else {
+			expectedHashName = DefaultTHash.Name()
+		}
+		if expectedHashName != ts.Hash().Name() {
+			t.Errorf("expecting hash [%v], but got [%s]", expectedHashName, ts.Hash().Name())
+		}
+	}
+}
+
+func TestIteration(t *testing.T) {
+	headerTests := []struct {
+		expectedSum string // TODO(vbatts) it would be nice to get individual sums of each
+		version     Version
+		hdr         *tar.Header
+		data        []byte
+	}{
+		{
+			"tarsum+sha256:626c4a2e9a467d65c33ae81f7f3dedd4de8ccaee72af73223c4bc4718cbc7bbd",
+			Version0,
+			&tar.Header{
+				Name:     "file.txt",
+				Size:     0,
+				Typeflag: tar.TypeReg,
+				Devminor: 0,
+				Devmajor: 0,
+			},
+			[]byte(""),
+		},
+		{
+			"tarsum.dev+sha256:6ffd43a1573a9913325b4918e124ee982a99c0f3cba90fc032a65f5e20bdd465",
+			VersionDev,
+			&tar.Header{
+				Name:     "file.txt",
+				Size:     0,
+				Typeflag: tar.TypeReg,
+				Devminor: 0,
+				Devmajor: 0,
+			},
+			[]byte(""),
+		},
+		{
+			"tarsum.dev+sha256:b38166c059e11fb77bef30bf16fba7584446e80fcc156ff46d47e36c5305d8ef",
+			VersionDev,
+			&tar.Header{
+				Name:     "another.txt",
+				Uid:      1000,
+				Gid:      1000,
+				Uname:    "slartibartfast",
+				Gname:    "users",
+				Size:     4,
+				Typeflag: tar.TypeReg,
+				Devminor: 0,
+				Devmajor: 0,
+			},
+			[]byte("test"),
+		},
+		{
+			"tarsum.dev+sha256:4cc2e71ac5d31833ab2be9b4f7842a14ce595ec96a37af4ed08f87bc374228cd",
+			VersionDev,
+			&tar.Header{
+				Name:     "xattrs.txt",
+				Uid:      1000,
+				Gid:      1000,
+				Uname:    "slartibartfast",
+				Gname:    "users",
+				Size:     4,
+				Typeflag: tar.TypeReg,
+				Xattrs: map[string]string{
+					"user.key1": "value1",
+					"user.key2": "value2",
+				},
+			},
+			[]byte("test"),
+		},
+		{
+			"tarsum.dev+sha256:65f4284fa32c0d4112dd93c3637697805866415b570587e4fd266af241503760",
+			VersionDev,
+			&tar.Header{
+				Name:     "xattrs.txt",
+				Uid:      1000,
+				Gid:      1000,
+				Uname:    "slartibartfast",
+				Gname:    "users",
+				Size:     4,
+				Typeflag: tar.TypeReg,
+				Xattrs: map[string]string{
+					"user.KEY1": "value1", // adding different case to ensure different sum
+					"user.key2": "value2",
+				},
+			},
+			[]byte("test"),
+		},
+		{
+			"tarsum+sha256:c12bb6f1303a9ddbf4576c52da74973c00d14c109bcfa76b708d5da1154a07fa",
+			Version0,
+			&tar.Header{
+				Name:     "xattrs.txt",
+				Uid:      1000,
+				Gid:      1000,
+				Uname:    "slartibartfast",
+				Gname:    "users",
+				Size:     4,
+				Typeflag: tar.TypeReg,
+				Xattrs: map[string]string{
+					"user.NOT": "CALCULATED",
+				},
+			},
+			[]byte("test"),
+		},
+	}
+	for _, htest := range headerTests {
+		s, err := renderSumForHeader(htest.version, htest.hdr, htest.data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if s != htest.expectedSum {
+			t.Errorf("expected sum: %q, got: %q", htest.expectedSum, s)
+		}
+	}
+
+}
+
+func renderSumForHeader(v Version, h *tar.Header, data []byte) (string, error) {
+	buf := bytes.NewBuffer(nil)
+	// first build our test tar
+	tw := tar.NewWriter(buf)
+	if err := tw.WriteHeader(h); err != nil {
+		return "", err
+	}
+	if _, err := tw.Write(data); err != nil {
+		return "", err
+	}
+	tw.Close()
+
+	ts, err := NewTarSum(buf, true, v)
+	if err != nil {
+		return "", err
+	}
+	tr := tar.NewReader(ts)
+	for {
+		hdr, err := tr.Next()
+		if hdr == nil || err == io.EOF {
+			// Signals the end of the archive.
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+		if _, err = io.Copy(ioutil.Discard, tr); err != nil {
+			return "", err
+		}
+	}
+	return ts.Sum(nil), nil
+}
+
+func Benchmark9kTar(b *testing.B) {
+	buf := bytes.NewBuffer([]byte{})
+	fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	n, err := io.Copy(buf, fh)
+	fh.Close()
+
+	reader := bytes.NewReader(buf.Bytes())
+
+	b.SetBytes(n)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		reader.Seek(0, 0)
+		ts, err := NewTarSum(reader, true, Version0)
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		io.Copy(ioutil.Discard, ts)
+		ts.Sum(nil)
+	}
+}
+
+func Benchmark9kTarGzip(b *testing.B) {
+	buf := bytes.NewBuffer([]byte{})
+	fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	n, err := io.Copy(buf, fh)
+	fh.Close()
+
+	reader := bytes.NewReader(buf.Bytes())
+
+	b.SetBytes(n)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		reader.Seek(0, 0)
+		ts, err := NewTarSum(reader, false, Version0)
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		io.Copy(ioutil.Discard, ts)
+		ts.Sum(nil)
+	}
+}
+
+// this is a single big file in the tar archive
+func Benchmark1mbSingleFileTar(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, false)
+}
+
+// this is a single big file in the tar archive
+func Benchmark1mbSingleFileTarGzip(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, true)
+}
+
+// this is 1024 1k files in the tar archive
+func Benchmark1kFilesTar(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1024, 1024, true, true}, false)
+}
+
+// this is 1024 1k files in the tar archive
+func Benchmark1kFilesTarGzip(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1024, 1024, true, true}, true)
+}
+
+func benchmarkTar(b *testing.B, opts sizedOptions, isGzip bool) {
+	var fh *os.File
+	tarReader := sizedTar(opts)
+	if br, ok := tarReader.(*os.File); ok {
+		fh = br
+	}
+	defer os.Remove(fh.Name())
+	defer fh.Close()
+
+	b.SetBytes(opts.size * opts.num)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ts, err := NewTarSum(fh, !isGzip, Version0)
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		io.Copy(ioutil.Discard, ts)
+		ts.Sum(nil)
+		fh.Seek(0, 0)
+	}
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json

@@ -0,0 +1 @@
+{"id":"46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457","parent":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","created":"2014-04-07T02:45:52.610504484Z","container":"e0f07f8d72cae171a3dcc35859960e7e956e0628bce6fedc4122bf55b2c287c7","container_config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","sed -ri 's/^(%wheel.*)(ALL)$/\\1NOPASSWD: \\2/' /etc/sudoers"],"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.9.1-dev","config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":3425}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json

@@ -0,0 +1 @@
+{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/testdata/xattr/json

@@ -0,0 +1 @@
+{"id":"4439c3c7f847954100b42b267e7e5529cac1d6934db082f65795c5ca2e594d93","parent":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","created":"2014-05-16T17:19:44.091534414Z","container":"5f92fb06cc58f357f0cde41394e2bbbb664e663974b2ac1693ab07b7a306749b","container_config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","setcap 'cap_setgid,cap_setuid+ep' ./file \u0026\u0026 getcap ./file"],"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.11.1-dev","config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":0}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/versioning.go

@@ -0,0 +1,150 @@
+package tarsum
+
+import (
+	"archive/tar"
+	"errors"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Version is used for versioning of the TarSum algorithm
+// based on the prefix of the hash used
+// i.e. "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b"
+type Version int
+
+// Prefix of "tarsum"
+const (
+	Version0 Version = iota
+	Version1
+	// VersionDev this constant will be either the latest or an unsettled next-version of the TarSum calculation
+	VersionDev
+)
+
+// VersionLabelForChecksum returns the label for the given tarsum
+// checksum, i.e., everything before the first `+` character in
+// the string or an empty string if no label separator is found.
+func VersionLabelForChecksum(checksum string) string {
+	// Checksums are in the form: {versionLabel}+{hashID}:{hex}
+	sepIndex := strings.Index(checksum, "+")
+	if sepIndex < 0 {
+		return ""
+	}
+	return checksum[:sepIndex]
+}
+
+// GetVersions gets a list of all known tarsum versions.
+func GetVersions() []Version {
+	v := []Version{}
+	for k := range tarSumVersions {
+		v = append(v, k)
+	}
+	return v
+}
+
+var (
+	tarSumVersions = map[Version]string{
+		Version0:   "tarsum",
+		Version1:   "tarsum.v1",
+		VersionDev: "tarsum.dev",
+	}
+	tarSumVersionsByName = map[string]Version{
+		"tarsum":     Version0,
+		"tarsum.v1":  Version1,
+		"tarsum.dev": VersionDev,
+	}
+)
+
+func (tsv Version) String() string {
+	return tarSumVersions[tsv]
+}
+
+// GetVersionFromTarsum returns the Version from the provided string.
+func GetVersionFromTarsum(tarsum string) (Version, error) {
+	tsv := tarsum
+	if strings.Contains(tarsum, "+") {
+		tsv = strings.SplitN(tarsum, "+", 2)[0]
+	}
+	for v, s := range tarSumVersions {
+		if s == tsv {
+			return v, nil
+		}
+	}
+	return -1, ErrNotVersion
+}
+
+// Errors that may be returned by functions in this package
+var (
+	ErrNotVersion            = errors.New("string does not include a TarSum Version")
+	ErrVersionNotImplemented = errors.New("TarSum Version is not yet implemented")
+)
+
+// tarHeaderSelector is the interface which different versions
+// of tarsum should use for selecting and ordering tar headers
+// for each item in the archive.
+type tarHeaderSelector interface {
+	selectHeaders(h *tar.Header) (orderedHeaders [][2]string)
+}
+
+type tarHeaderSelectFunc func(h *tar.Header) (orderedHeaders [][2]string)
+
+func (f tarHeaderSelectFunc) selectHeaders(h *tar.Header) (orderedHeaders [][2]string) {
+	return f(h)
+}
+
+func v0TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
+	return [][2]string{
+		{"name", h.Name},
+		{"mode", strconv.FormatInt(h.Mode, 10)},
+		{"uid", strconv.Itoa(h.Uid)},
+		{"gid", strconv.Itoa(h.Gid)},
+		{"size", strconv.FormatInt(h.Size, 10)},
+		{"mtime", strconv.FormatInt(h.ModTime.UTC().Unix(), 10)},
+		{"typeflag", string([]byte{h.Typeflag})},
+		{"linkname", h.Linkname},
+		{"uname", h.Uname},
+		{"gname", h.Gname},
+		{"devmajor", strconv.FormatInt(h.Devmajor, 10)},
+		{"devminor", strconv.FormatInt(h.Devminor, 10)},
+	}
+}
+
+func v1TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
+	// Get extended attributes.
+	xAttrKeys := make([]string, len(h.Xattrs))
+	for k := range h.Xattrs {
+		xAttrKeys = append(xAttrKeys, k)
+	}
+	sort.Strings(xAttrKeys)
+
+	// Make the slice with enough capacity to hold the 11 basic headers
+	// we want from the v0 selector plus however many xattrs we have.
+	orderedHeaders = make([][2]string, 0, 11+len(xAttrKeys))
+
+	// Copy all headers from v0 excluding the 'mtime' header (the 5th element).
+	v0headers := v0TarHeaderSelect(h)
+	orderedHeaders = append(orderedHeaders, v0headers[0:5]...)
+	orderedHeaders = append(orderedHeaders, v0headers[6:]...)
+
+	// Finally, append the sorted xattrs.
+	for _, k := range xAttrKeys {
+		orderedHeaders = append(orderedHeaders, [2]string{k, h.Xattrs[k]})
+	}
+
+	return
+}
+
+var registeredHeaderSelectors = map[Version]tarHeaderSelectFunc{
+	Version0:   v0TarHeaderSelect,
+	Version1:   v1TarHeaderSelect,
+	VersionDev: v1TarHeaderSelect,
+}
+
+func getTarHeaderSelector(v Version) (tarHeaderSelector, error) {
+	headerSelector, ok := registeredHeaderSelectors[v]
+	if !ok {
+		return nil, ErrVersionNotImplemented
+	}
+
+	return headerSelector, nil
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/versioning_test.go

@@ -0,0 +1,98 @@
+package tarsum
+
+import (
+	"testing"
+)
+
+func TestVersionLabelForChecksum(t *testing.T) {
+	version := VersionLabelForChecksum("tarsum+sha256:deadbeef")
+	if version != "tarsum" {
+		t.Fatalf("Version should have been 'tarsum', was %v", version)
+	}
+	version = VersionLabelForChecksum("tarsum.v1+sha256:deadbeef")
+	if version != "tarsum.v1" {
+		t.Fatalf("Version should have been 'tarsum.v1', was %v", version)
+	}
+	version = VersionLabelForChecksum("something+somethingelse")
+	if version != "something" {
+		t.Fatalf("Version should have been 'something', was %v", version)
+	}
+	version = VersionLabelForChecksum("invalidChecksum")
+	if version != "" {
+		t.Fatalf("Version should have been empty, was %v", version)
+	}
+}
+
+func TestVersion(t *testing.T) {
+	expected := "tarsum"
+	var v Version
+	if v.String() != expected {
+		t.Errorf("expected %q, got %q", expected, v.String())
+	}
+
+	expected = "tarsum.v1"
+	v = 1
+	if v.String() != expected {
+		t.Errorf("expected %q, got %q", expected, v.String())
+	}
+
+	expected = "tarsum.dev"
+	v = 2
+	if v.String() != expected {
+		t.Errorf("expected %q, got %q", expected, v.String())
+	}
+}
+
+func TestGetVersion(t *testing.T) {
+	testSet := []struct {
+		Str      string
+		Expected Version
+	}{
+		{"tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", Version0},
+		{"tarsum+sha256", Version0},
+		{"tarsum", Version0},
+		{"tarsum.dev", VersionDev},
+		{"tarsum.dev+sha256:deadbeef", VersionDev},
+	}
+
+	for _, ts := range testSet {
+		v, err := GetVersionFromTarsum(ts.Str)
+		if err != nil {
+			t.Fatalf("%q : %s", err, ts.Str)
+		}
+		if v != ts.Expected {
+			t.Errorf("expected %d (%q), got %d (%q)", ts.Expected, ts.Expected, v, v)
+		}
+	}
+
+	// test one that does not exist, to ensure it errors
+	str := "weak+md5:abcdeabcde"
+	_, err := GetVersionFromTarsum(str)
+	if err != ErrNotVersion {
+		t.Fatalf("%q : %s", err, str)
+	}
+}
+
+func TestGetVersions(t *testing.T) {
+	expected := []Version{
+		Version0,
+		Version1,
+		VersionDev,
+	}
+	versions := GetVersions()
+	if len(versions) != len(expected) {
+		t.Fatalf("Expected %v versions, got %v", len(expected), len(versions))
+	}
+	if !containsVersion(versions, expected[0]) || !containsVersion(versions, expected[1]) || !containsVersion(versions, expected[2]) {
+		t.Fatalf("Expected [%v], got [%v]", expected, versions)
+	}
+}
+
+func containsVersion(versions []Version, version Version) bool {
+	for _, v := range versions {
+		if v == version {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/hyperhq/hypercli/pkg/tarsum/writercloser.go

@@ -0,0 +1,22 @@
+package tarsum
+
+import (
+	"io"
+)
+
+type writeCloseFlusher interface {
+	io.WriteCloser
+	Flush() error
+}
+
+type nopCloseFlusher struct {
+	io.Writer
+}
+
+func (n *nopCloseFlusher) Close() error {
+	return nil
+}
+
+func (n *nopCloseFlusher) Flush() error {
+	return nil
+}