Fix the dependency issue (#231)

This commit is contained in:
Robbie Zhang
2018-06-21 12:09:42 -07:00
committed by GitHub
parent 027b76651d
commit 6ec1098bb8
16629 changed files with 74837 additions and 4975021 deletions

View File

@@ -1,629 +0,0 @@
// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package archive
import (
"archive/tar"
"bytes"
"context"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
log "github.com/Sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/vmware/vic/pkg/trace"
)
var (
newDir, oldDir string
a, b []byte
files map[string][]string
directories map[string]struct{}
err error
)
func TestMain(m *testing.M) {
var err error
directories = make(map[string]struct{}, 4)
files = make(map[string][]string, 6)
files["original"] = []string{"file1", "file2", "file3", "file4",
"original/file1", "original/file2", "original/remove",
"exclude/excludeme", "exclude/includeme", "excludeme", "include/excludeme", "include/includeme"}
files["added"] = []string{"added1", "added2", "add/file1", "add/file2"}
files["changed"] = []string{"file1", "original/file2",
"exclude/excludeme", "exclude/includeme",
"include/excludeme", "include/includeme",
"excludeme"}
files["removed"] = []string{"file2", "original/file1", "original/remove"}
files["excluded"] = []string{"exclude/", "excludeme", "include/excludeme"}
files["included"] = []string{"exclude/includeme", "include/"}
newDir, err = ioutil.TempDir("", "mnt")
if err != nil {
return
}
defer os.RemoveAll(newDir)
oldDir, err = ioutil.TempDir("", "mnt")
if err != nil {
return
}
defer os.RemoveAll(oldDir)
a = []byte("The mollusk lingers with its wandering eye\n")
b = []byte("The waking of all creatures that live on the land\n")
for _, dir := range []string{"original/", "add/", "exclude/", "include/"} {
directories[dir] = struct{}{}
if err = os.Mkdir(filepath.Join(oldDir, dir), 0777); err != nil {
log.Errorf("Failed to add directory: %s", err.Error())
return
}
if err = os.Mkdir(filepath.Join(newDir, dir), 0777); err != nil {
log.Errorf("Failed to add directory: %s", err.Error())
return
}
}
for _, file := range files["original"] {
if err = ioutil.WriteFile(filepath.Join(oldDir, file), a, 0777); err != nil {
log.Errorf("Failed to add file: %s", err.Error())
return
}
if err = ioutil.WriteFile(filepath.Join(newDir, file), a, 0777); err != nil {
log.Errorf("Failed to add file: %s", err.Error())
return
}
}
for _, file := range append(files["added"], files["changed"]...) {
if err = ioutil.WriteFile(filepath.Join(newDir, file), b, 0777); err != nil {
log.Errorf("Failed to add file: %s", err.Error())
return
}
}
for _, file := range files["removed"] {
if err = os.Remove(filepath.Join(newDir, file)); err != nil {
log.Errorf("Failed to remove file: %s", err.Error())
return
}
}
os.Exit(m.Run())
}
func TestDiff(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiff")
tarFile, err := Diff(op, newDir, oldDir, nil, true, true)
if !assert.NoError(t, err) {
return
}
tarredFiles := make(map[string][]byte)
tr := tar.NewReader(tarFile)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
op.Errorf("Error reading tar archive: %s", err.Error())
}
buf := bytes.NewBuffer([]byte{})
if _, err := io.Copy(buf, tr); !assert.NoError(t, err) {
return
}
tarredFiles[hdr.Name] = buf.Bytes()
}
all := append(files["added"], append(files["changed"], files["included"]...)...)
for _, file := range all {
if strings.HasSuffix(file, "/") {
continue
}
f, ok := tarredFiles[file]
assert.True(t, ok, "Expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if _, ok := directories[file]; !ok {
assert.Equal(t, b, f, "Expected file contents \"%s\", but found \"%s\"", b, f)
}
}
for _, file := range files["removed"] {
wh := filepath.Join(filepath.Dir(file), ".wh."+filepath.Base(file))
_, ok := tarredFiles[wh]
assert.True(t, ok, "Expected to find whiteout file in archive: %s", wh)
}
}
func TestDiffNoAncestor(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiffNoParent")
// test without ancestor
tarFile, err := Diff(op, newDir, "", nil, true, true)
if !assert.NoError(t, err) {
return
}
tarredFiles := make(map[string][]byte)
tr := tar.NewReader(tarFile)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
op.Errorf("Error reading tar header: %s", err.Error())
}
buf := bytes.NewBuffer([]byte{})
if _, err := io.Copy(buf, tr); !assert.NoError(t, err) {
return
}
tarredFiles[hdr.Name] = buf.Bytes()
if hdr.Typeflag == tar.TypeDir {
directories[hdr.Name] = struct{}{}
}
}
all := append(files["added"], append(files["changed"], files["included"]...)...)
for _, file := range all {
f, ok := tarredFiles[file]
assert.True(t, ok, "Expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if _, ok := directories[file]; !ok {
assert.Equal(t, b, f, "Expected file contents \"%s\", but found \"%s\"", b, f)
}
}
for _, file := range files["removed"] {
wh := filepath.Join(filepath.Dir(file), ".wh."+filepath.Base(file))
_, ok := tarredFiles[wh]
assert.False(t, ok, "Expected not to find whiteout file in archive: %s", wh)
}
}
func TestDiffNoData(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiffNoData")
tarFile, err := Diff(op, newDir, oldDir, nil, false, true)
if !assert.NoError(t, err) {
return
}
tarredFiles := make(map[string]string)
changeTypes := make(map[string]string)
tr := tar.NewReader(tarFile)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if !assert.NoError(t, err) {
op.Errorf("Error reading tar header: %s", err.Error())
}
buf := bytes.NewBuffer([]byte{})
n, err := io.Copy(buf, tr)
if !assert.NoError(t, err) {
return
}
assert.EqualValues(t, 0, n, "Expected 0 bytes copied, got %d instead", n)
tarredFiles[hdr.Name] = string(buf.Bytes())
changeTypes[hdr.Name] = hdr.Xattrs[ChangeTypeKey]
}
for _, file := range files["added"] {
f, ok := tarredFiles[file]
ctype := changeTypes[file]
assert.True(t, ok, "Expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if _, ok := directories[file]; !ok {
assert.Equal(t, "", f, "Expected file contents \"%s\", but found \"%s\"", "", f)
assert.Equal(t, "A", ctype, "Expected change type \"%s\", but found \"%s\"", "A", ctype)
}
}
for _, file := range append(files["changed"], files["included"]...) {
f, ok := tarredFiles[file]
ctype := changeTypes[file]
assert.True(t, ok, "expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if _, ok := directories[file]; !ok {
assert.Equal(t, "", f, "expected file contents \"%s\", but found \"%s\"", "", f)
assert.Equal(t, "C", ctype, "expected change type \"%s\", but found \"%s\"", "C", ctype)
}
}
for _, file := range files["removed"] {
wh := filepath.Join(filepath.Dir(file), ".wh."+filepath.Base(file))
_, ok := tarredFiles[wh]
ctype, cok := changeTypes[wh]
assert.True(t, ok, "Expected to find whiteout file in archive: %s", wh)
assert.True(t, cok, "Expected to find change type for %s", wh)
assert.Equal(t, "D", ctype, "Expected change type \"%s\" but found \"%s\"", "D", ctype)
}
}
func TestDiffFilterSpec(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiffFilterSpec")
filter := make(map[string]FilterType)
for _, path := range files["excluded"] {
p := strings.TrimSuffix(path, "/")
filter[p] = Exclude
}
for _, path := range files["included"] {
filter[path] = Include
}
spec, err := CreateFilterSpec(op, filter)
if !assert.NoError(t, err) {
return
}
tarFile, err := Diff(op, newDir, oldDir, spec, true, true)
if !assert.NoError(t, err) {
return
}
tarredFiles := make(map[string][]byte)
tr := tar.NewReader(tarFile)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
op.Errorf("Error reading tar archive: %s", err.Error())
}
buf := bytes.NewBuffer([]byte{})
if _, err := io.Copy(buf, tr); !assert.NoError(t, err) {
return
}
tarredFiles[hdr.Name] = buf.Bytes()
}
all := append(files["added"], files["included"]...)
for _, file := range all {
if file == string(filepath.Separator) {
continue
}
f, ok := tarredFiles[file]
assert.True(t, ok, "Expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if _, ok := directories[file]; !ok {
assert.Equal(t, b, f, "Expected file contents \"%s\" for %s, but found \"%s\"", b, file, f)
}
}
for _, file := range files["removed"] {
wh := filepath.Join(filepath.Dir(file), ".wh."+filepath.Base(file))
_, ok := tarredFiles[wh]
assert.True(t, ok, "Expected to find whiteout file in archive: %s", wh)
}
for _, file := range files["excluded"] {
_, ok := tarredFiles[file]
assert.False(t, ok, "Expected excluded file to be missing from archive: %s", file)
}
}
func TestDiffFilterSpecNoAncestor(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiffFilterSpecNoParent")
filter := make(map[string]FilterType)
for _, path := range files["excluded"] {
p := strings.TrimSuffix(path, "/")
filter[p] = Exclude
}
for _, path := range files["included"] {
filter[path] = Include
}
spec, err := CreateFilterSpec(op, filter)
if !assert.NoError(t, err) {
return
}
// test without ancestor
tarFile, err := Diff(op, newDir, "", spec, true, true)
if !assert.NoError(t, err) {
return
}
tarredFiles := make(map[string][]byte)
tr := tar.NewReader(tarFile)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
op.Errorf("Error reading tar archive: %s", err.Error())
}
buf := bytes.NewBuffer([]byte{})
if _, err := io.Copy(buf, tr); !assert.NoError(t, err) {
return
}
tarredFiles[hdr.Name] = buf.Bytes()
}
all := append(files["added"], files["included"]...)
for _, file := range all {
f, ok := tarredFiles[file]
if file == string(filepath.Separator) {
continue
}
assert.True(t, ok, "Expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if _, ok := directories[file]; !ok {
assert.Equal(t, b, f, "Expected file contents \"%s\", but found \"%s\" for target file (%s)", b, f, file)
}
}
for _, file := range files["removed"] {
wh := filepath.Join(filepath.Dir(file), ".wh."+filepath.Base(file))
_, ok := tarredFiles[wh]
assert.False(t, ok, "Expected not to find whiteout file in archive: %s", wh)
}
for _, file := range files["excluded"] {
_, ok := tarredFiles[file]
assert.False(t, ok, "Expected excluded file to be missing from archive: %s", file)
}
}
func TestDiffFilterSpecRebase(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiffFilterSpecRebase")
filter := make(map[string]FilterType)
for _, path := range files["excluded"] {
filter[path] = Exclude
}
for _, path := range files["included"] {
filter[path] = Include
}
rebasePath := "path/to/prefix"
filter[rebasePath] = Rebase
spec, err := CreateFilterSpec(op, filter)
if !assert.NoError(t, err) {
return
}
// test without ancestor
tarFile, err := Diff(op, newDir, oldDir, spec, true, true)
if !assert.NoError(t, err) {
return
}
tarredFiles := make(map[string][]byte)
tr := tar.NewReader(tarFile)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
op.Errorf("Error reading tar archive: %s", err.Error())
}
buf := bytes.NewBuffer([]byte{})
if _, err := io.Copy(buf, tr); !assert.NoError(t, err) {
return
}
tarredFiles[strings.TrimSuffix(hdr.Name, "/")] = buf.Bytes()
}
all := append(files["added"], files["included"]...)
for _, file := range all {
if file == string(filepath.Separator) {
continue
}
rebasedPath := filepath.Join(rebasePath, file)
var isDir bool
_, isDir = directories[file]
f, ok := tarredFiles[rebasedPath]
assert.True(t, ok, "Expected to find %s in tar archive", rebasedPath)
if !isDir {
assert.Equal(t, b, f, "Expected file contents \"%s\", but found \"%s\" for target file (%s)", b, f, rebasedPath)
}
}
for _, file := range files["removed"] {
wh := filepath.Join(filepath.Dir(file), ".wh."+filepath.Base(file))
wh = filepath.Join(rebasePath, wh)
_, ok := tarredFiles[wh]
assert.True(t, ok, "Expected not to find whiteout file in archive: %s", wh)
}
for _, file := range files["excluded"] {
_, ok := tarredFiles[file]
assert.False(t, ok, "Expected excluded file to be missing from archive: %s", file)
}
}
func TestDiffFilterSpecRebaseNoData(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiffFilterSpecRebase")
filter := make(map[string]FilterType)
for _, path := range files["excluded"] {
filter[path] = Exclude
}
for _, path := range files["included"] {
filter[path] = Include
}
rebasePath := "path/to/prefix"
filter[rebasePath] = Rebase
spec, err := CreateFilterSpec(op, filter)
if !assert.NoError(t, err) {
return
}
tarFile, err := Diff(op, newDir, oldDir, spec, false, true)
if !assert.NoError(t, err) {
return
}
tarredFiles := make(map[string]struct{})
changeTypes := make(map[string]string)
tr := tar.NewReader(tarFile)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if !assert.NoError(t, err) {
op.Errorf("Error reading tar header: %s", err.Error())
}
buf := bytes.NewBuffer([]byte{})
n, err := io.Copy(buf, tr)
if !assert.NoError(t, err) {
return
}
assert.EqualValues(t, 0, n, "Expected 0 bytes copied, got %d instead", n)
tarredFiles[strings.TrimSuffix(hdr.Name, "/")] = struct{}{}
changeTypes[hdr.Name] = hdr.Xattrs[ChangeTypeKey]
}
for _, file := range files["added"] {
rebasedPath := filepath.Join(rebasePath, file)
var isDir bool
_, isDir = directories[file]
_, ok := tarredFiles[rebasedPath]
ctype := changeTypes[rebasedPath]
assert.True(t, ok, "Expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if !isDir {
assert.Equal(t, "A", ctype, "Expected change type \"%s\", but found \"%s\"", "A", ctype)
}
}
for _, file := range append(files["changed"], files["included"]...) {
// don't check for excludes or whiteouts yet
base := filepath.Base(file)
if strings.HasSuffix(file, "excludeme") || strings.HasPrefix(base, ".wh.") {
continue
}
rebasedPath := filepath.Join(rebasePath, file)
var isDir bool
_, isDir = directories[file]
_, ok := tarredFiles[rebasedPath]
ctype := changeTypes[rebasedPath]
assert.True(t, ok, "expected to find %s in tar archive", file)
// don't try to check the contents if its a directory
if !isDir {
assert.Equal(t, "C", ctype, "expected change type \"%s\", but found \"%s\"", "C", ctype)
}
}
for _, file := range files["removed"] {
wh := filepath.Join(filepath.Dir(file), ".wh."+filepath.Base(file))
wh = filepath.Join(rebasePath, wh)
_, ok := tarredFiles[wh]
ctype, cok := changeTypes[wh]
assert.True(t, ok, "Expected to find whiteout file in archive: %s", wh)
assert.True(t, cok, "Expected to find change type for %s", wh)
assert.Equal(t, "D", ctype, "Expected change type \"%s\" but found \"%s\"", "D", ctype)
}
for _, file := range files["excluded"] {
_, ok := tarredFiles[file]
assert.False(t, ok, "Expected excluded file to be missing from archive: %s", file)
}
}
func TestDiffCreateFilterSpec(t *testing.T) {
op := trace.NewOperation(context.Background(), "TestDiffCreateFilterSpec")
filter := make(map[string]FilterType)
for _, path := range files["excluded"] {
filter[path] = Exclude
}
for _, path := range files["included"] {
filter[path] = Include
}
rebasePath := "/path/to/prefix"
filter[rebasePath] = Rebase
stripPath := "/path/to/strip"
filter[stripPath] = Strip
spec, err := CreateFilterSpec(op, filter)
if !assert.NoError(t, err) {
return
}
for _, path := range files["included"] {
_, ok := spec.Inclusions[path]
assert.True(t, ok, "Expected to find %s in inclusions set", path)
_, ok = spec.Exclusions[path]
assert.False(t, ok, "Expected not to find %s in exclusions set", path)
}
for _, path := range files["excluded"] {
_, ok := spec.Exclusions[path]
assert.True(t, ok, "Expected to find %s in exclusions set", path)
_, ok = spec.Inclusions[path]
assert.False(t, ok, "Expected not to find %s in inclusions set", path)
}
assert.Equal(t, rebasePath, spec.RebasePath)
assert.Equal(t, stripPath, spec.StripPath)
e := "/path/to/extra/rebase"
filter[e] = Rebase
spec, err = CreateFilterSpec(op, filter)
assert.Nil(t, spec, "Expected nil spec")
assert.Error(t, err)
assert.EqualError(t, err, "error creating filter spec: only one rebase path allowed")
delete(filter, e)
s := "/path/to/extra/strippath"
filter[s] = Strip
spec, err = CreateFilterSpec(op, filter)
assert.Nil(t, spec, "Expected nil spec")
assert.Error(t, err)
assert.EqualError(t, err, "error creating filter spec: only one strip path allowed")
delete(filter, s)
filter[s] = 20
spec, err = CreateFilterSpec(op, filter)
assert.Nil(t, spec, "Expected nil spec")
assert.Error(t, err)
assert.EqualError(t, err, "invalid filter specification: 20")
}

View File

@@ -1,320 +0,0 @@
// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package archive
import (
"archive/tar"
"context"
"fmt"
"io"
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/vmware/vic/pkg/trace"
"github.com/vmware/vic/pkg/uid"
)
// generateArchive takes a number of files and a file size and generates a tar archive
// based on that data. It returns:
// index of entry names in the archive
// archive byte stream
func generateArchive(name string, num, size int) ([]string, io.Reader) {
r, w := io.Pipe()
tw := tar.NewWriter(w)
// stable reference for expected archive entries
index := make([]string, num)
for i := 0; i < num; i++ {
index[i] = string(uid.New())
}
// our file contents are zeros - this is the worst case for stripping trailing headers
zeros := make([]byte, size)
go func() {
for i := 0; i < num; i++ {
// we only really care about two things in the header
hdr := &tar.Header{
Name: index[i],
Size: int64(size),
}
fmt.Printf("Writing header for file %s:%d\n", name, i)
err := tw.WriteHeader(hdr)
if err != nil {
panic(err)
}
fmt.Printf("Writing data for file %s:%d\n", name, i)
n, err := tw.Write(zeros)
if err != nil {
panic(err)
}
if n != size {
panic(fmt.Sprintf("Failed to write all bytes: %d != %d", n, size))
}
}
// add the end-of-archive
tw.Close()
w.Close()
}()
return index, r
}
// TestSingleStripper ensures that basic function (stripping end-of-archive footer) works as
// expected. I found no real way, when using the TarReader to actually assert that the footer
// has been dropped so this is left to assert correct passthrough of archive data and the
// dropping of the footer is asserted by the multistream cases.
func TestSingleStripper(t *testing.T) {
size := 2048
count := 5
index, reader := generateArchive("single", count, size)
source := tar.NewReader(reader)
stripper := NewStripper(trace.NewOperation(context.Background(), "strip"), source, nil)
pr, pw := io.Pipe()
tr := tar.NewReader(pr)
var wg sync.WaitGroup
wg.Add(1)
go func() {
n, err := io.Copy(pw, stripper)
pw.Close()
wg.Done()
fmt.Printf("Done copying from stripper: %d, %s\n", n, err)
if !assert.NoError(t, err, "Expected nil error from io.Copy on end-of-file") {
t.FailNow()
}
}()
for i := 0; i <= len(index); i++ {
fmt.Printf("Reading header for file %d\n", i)
header, err := tr.Next()
if err == io.EOF {
fmt.Printf("End-of-file")
// TODO: is this pass or fail?
return
}
if err != nil {
fmt.Printf("Error from archive: %s\n", err)
t.FailNow()
}
if !assert.NotEqual(t, i, len(index), "Expected EOF after index exhausted") {
t.FailNow()
}
if !assert.Equal(t, header.Name, index[i], "Expected header name to match index") {
t.FailNow()
}
if !assert.Equal(t, header.Size, int64(size), "Expected file size in header to match target generated size") {
t.FailNow()
}
// make the buf just that little bit bigger to allow for errrors in the copy if they would occur
buf := make([]byte, size+10)
fmt.Printf("Reading data for file %d\n", i)
n, err := tr.Read(buf)
if !assert.NoError(t, err, "No expected errors from file data copy") {
t.FailNow()
}
if !assert.Equal(t, n, size, "Expected file data size to match target generated size") {
t.FailNow()
}
}
wg.Wait()
}
// TestConjoinedStrippers ensures that the footer is correctly dropped from a stripped archive
// and that a TarReader continues to provide headers from the following conjoined streams.
func TestConjoinedStrippers(t *testing.T) {
size := 2048
count := 3
multiplicity := 3
indices := make([][]string, multiplicity)
strippers := make([]io.Reader, multiplicity)
for m := 0; m < multiplicity; m++ {
var reader io.Reader
indices[m], reader = generateArchive(fmt.Sprintf("archive-%d", m), count, size)
source := tar.NewReader(reader)
strippers[m] = NewStripper(trace.NewOperation(context.Background(), fmt.Sprintf("strip-%d", m)), source, nil)
}
conjoined := MultiReader(strippers...)
pr, pw := io.Pipe()
tr := tar.NewReader(pr)
var wg sync.WaitGroup
wg.Add(1)
go func() {
n, err := io.Copy(pw, conjoined)
pw.Close()
wg.Done()
fmt.Printf("Done copying from strippers: %d, %s\n", n, err)
if !assert.NoError(t, err, "Expected nil error from io.Copy on end-of-file") {
t.FailNow()
}
}()
expectedEntries := count * multiplicity
for i := 0; i <= expectedEntries; i++ {
fmt.Printf("Reading header for file %d\n", i)
header, err := tr.Next()
if err == io.EOF {
fmt.Printf("End-of-file\n")
// TODO: is this pass or fail?
return
}
if err != nil {
fmt.Printf("Error from archive: %s\n", err)
t.FailNow()
}
if !assert.NotEqual(t, i, expectedEntries, "Expected EOF after index exhausted") {
t.FailNow()
}
member := i / count
entry := i % count
if !assert.Equal(t, header.Name, indices[member][entry], "Expected header name to match index") {
t.FailNow()
}
if !assert.Equal(t, header.Size, int64(size), "Expected file size in header to match target generated size") {
t.FailNow()
}
// make the buf just that little bit bigger to allow for errrors in the copy if they would occur
buf := make([]byte, size+10)
fmt.Printf("Reading data for file %d\n", i)
n, err := tr.Read(buf)
if !assert.NoError(t, err, "No expected errors from file data copy") {
t.FailNow()
}
if !assert.Equal(t, n, size, "Expected file data size to match target generated size") {
t.FailNow()
}
}
wg.Wait()
}
// TestConjoinedStrippersWithCloser ensures that we can conjoin readers, multiple strippers and a regular, in order to get
// the end-of-archive footer as the finale. We have a wait group to ensure that all routines have finished before declaring
// success.
func TestConjoinedStrippersWithCloser(t *testing.T) {
size := 2048
count := 3
multiplicity := 3
indices := make([][]string, multiplicity)
readers := make([]io.Reader, multiplicity)
for m := 0; m < multiplicity; m++ {
var reader io.Reader
indices[m], reader = generateArchive(fmt.Sprintf("archive-%d", m), count, size)
source := tar.NewReader(reader)
if m < multiplicity-1 {
fmt.Printf("added stripper\n")
readers[m] = NewStripper(trace.NewOperation(context.Background(), fmt.Sprintf("strip-%d", m)), source, nil)
} else {
fmt.Printf("added raw\n")
readers[m] = reader
}
}
conjoined := MultiReader(readers...)
pr, pw := io.Pipe()
tr := tar.NewReader(pr)
var wg sync.WaitGroup
wg.Add(1)
go func() {
n, err := io.Copy(pw, conjoined)
pw.Close()
wg.Done()
fmt.Printf("Done copying from all sources: %d, %s\n", n, err)
if !assert.NoError(t, err, "Expected nil error from io.Copy on end-of-file") {
t.FailNow()
}
}()
expectedEntries := count * multiplicity
for i := 0; i <= expectedEntries; i++ {
fmt.Printf("Reading header for file %d\n", i)
header, err := tr.Next()
if err == io.EOF {
fmt.Printf("End-of-file\n")
wg.Wait()
return
}
if err != nil {
fmt.Printf("Error from archive: %s\n", err)
t.FailNow()
}
if !assert.NotEqual(t, i, expectedEntries, "Expected EOF after index exhausted") {
t.FailNow()
}
member := i / count
entry := i % count
if !assert.Equal(t, header.Name, indices[member][entry], "Expected header name to match index") {
t.FailNow()
}
if !assert.Equal(t, header.Size, int64(size), "Expected file size in header to match target generated size") {
t.FailNow()
}
// make the buf just that little bit bigger to allow for errrors in the copy if they would occur
buf := make([]byte, size+10)
fmt.Printf("Reading data for file %d\n", i)
n, err := tr.Read(buf)
if !assert.NoError(t, err, "No expected errors from file data copy") {
t.FailNow()
}
if !assert.Equal(t, n, size, "Expected file data size to match target generated size") {
t.FailNow()
}
}
wg.Wait()
}

View File

@@ -30,6 +30,8 @@ import (
"github.com/vmware/vic/pkg/trace"
)
type binaryPath string
const (
// fileWriteFlags is a collection of flags configuring our writes for general tar behavior
//
@@ -37,9 +39,15 @@ const (
// O_TRUNC = truncate file to 0 length if it does exist(overwrite the file)
// O_WRONLY = We use this since we do not intend to read, we only need to write.
fileWriteFlags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY
// Location of the unpack binary inside of containers
containerBinaryPath binaryPath = "/.tether/unpack"
// Location of the unpack binary inside the endpoint VM
applianceBinaryPath binaryPath = "/bin/unpack"
)
// InvokeUnpack will unpack the given tarstream(if it is a tar stream) on the local filesystem based on the specified root
// UnpackNoChroot will unpack the given tarstream(if it is a tar stream) on the local filesystem based on the specified root
// combined with any rebase from the path spec
//
// the pathSpec will include the following elements
@@ -48,7 +56,7 @@ const (
// - exlude : marks paths that are to be excluded from the write
// - rebase : marks the the write path that will be tacked onto (appended or prepended? TODO improve this comment) the "root". e.g /tmp/unpack + /my/target/path = /tmp/unpack/my/target/path
// N.B. tarStream MUST BE TERMINATED WITH EOF or this function will hang forever!
func InvokeUnpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string) error {
func UnpackNoChroot(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string) error {
op.Debugf("unpacking archive to root: %s, filter: %+v", root, filter)
// Online datasource is sending a tar reader instead of an io reader.
@@ -131,71 +139,158 @@ func InvokeUnpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, r
return nil
}
// Unpack hooks into a binary present in the appliance vm called unpack in order to execute InvokeUnpack inside of a chroot. This method works identically to InvokeUnpack, except that it will not function in areas where the binary is not present at /bin/unpack
func Unpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string) error {
// OfflineUnpack wraps Unpack for usage in contexts without a childReaper, namely when copying to an offline container with docker cp
func OfflineUnpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string) error {
var cmd *exec.Cmd
var err error
if cmd, err = unpack(op, tarStream, filter, root, applianceBinaryPath); err != nil {
return err
}
if err = cmd.Wait(); err != nil {
return err
}
return nil
}
// OnlineUnpack will extract a tar stream tarStream to folder root inside of a running container
func OnlineUnpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string) (*exec.Cmd, error) {
return unpack(op, tarStream, filter, root, containerBinaryPath)
}
func streamCopy(op trace.Operation, stdin io.WriteCloser, tarStream io.Reader) error {
// if we're passed a stream that doesn't cast to a tar.Reader copy the tarStream to the binary via stdin; the binary will stream it to InvokeUnpack unchanged
var err error
tr, ok := tarStream.(*tar.Reader)
if !ok {
defer stdin.Close()
if _, err := io.Copy(stdin, tarStream); err != nil {
op.Errorf("Error copying tarStream: %s", err.Error())
return err
}
return nil
}
tw := tar.NewWriter(stdin)
defer tw.Close()
var th *tar.Header
for {
th, err = tr.Next()
if err == io.EOF {
tw.Close()
return nil
}
if err != nil {
op.Errorf("error reading tar header %s", err)
return err
}
op.Debugf("processing tar header: asset(%s), size(%d)", th.Name, th.Size)
err = tw.WriteHeader(th)
if err != nil {
op.Errorf("error writing tar header %s", err)
return err
}
var k int64
k, err = io.Copy(tw, tr)
op.Debugf("wrote %d bytes", k)
if err != nil {
op.Errorf("error writing file body bytes to stdin %s", err)
return err
}
}
}
func DockerUnpack(op trace.Operation, root string, tarStream io.Reader) (int64, error) {
fi, err := os.Stat(root)
if err != nil {
// the target unpack path does not exist. We should not get here.
return 0, err
}
if !fi.IsDir() {
return 0, fmt.Errorf("unpack root target is not a directory: %s", root)
}
// #nosec: 193 applianceBinaryPath is a constant, not a variable
cmd := exec.Command(string(applianceBinaryPath), root)
stdin, err := cmd.StdinPipe()
if err != nil {
return 0, err
}
if stdin == nil {
err = errors.New("stdin was nil")
return 0, err
}
if err = cmd.Start(); err != nil {
return 0, err
}
bytesWritten := make(chan int64, 1)
go func() {
defer stdin.Close()
var n int64
if n, err = io.Copy(stdin, tarStream); err != nil {
op.Errorf("Error copying tarStream: %s", err.Error())
}
bytesWritten <- n
}()
if err = cmd.Wait(); err != nil {
return 0, err
}
return <-bytesWritten, nil
}
// Unpack runs the binary compiled in cmd/unpack.go which creates a chroot at `root` and passes `op`, `tarStream`, and `filter` to InvokeUnpack for extraction of the tar on the filesystem. `binPath` should be either ApplianceBinaryPath or ContainerBinaryPath. Unpack returns a `Cmd` to allow use in conjunction with the tether's `LaunchUtility`, so it is necessary to call `cmd.Wait` after `Unpack` exits e.g. OfflineUnpack, if not being used in conjunction with LaunchUtility and the childReaper.
func unpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string, binPath binaryPath) (*exec.Cmd, error) {
fi, err := os.Stat(root)
if err != nil {
// the target unpack path does not exist. We should not get here.
op.Errorf("tar unpack target does not exist: %s", root)
return err
return nil, err
}
if !fi.IsDir() {
err := fmt.Errorf("unpack root target is not a directory: %s", root)
op.Error(err)
return err
return nil, err
}
encodedFilter, err := EncodeFilterSpec(op, filter)
if err != nil {
op.Error(err)
return err
return nil, err
}
// Prepare to launch the binary, which will create a chroot at root and then invoke InvokeUnpack
// #nosec: Subprocess launching with variable. -- neither variable is user input & both are bounded inputs so this is fine
cmd := exec.Command("/bin/unpack", op.ID(), root, *encodedFilter)
// "/bin/unpack" on appliance
// "/.tether/unpack" inside container
cmd := exec.Command(string(binPath), root, *encodedFilter)
//stdin
stdin, err := cmd.StdinPipe()
if err != nil {
op.Error(err)
return err
return nil, err
}
if stdin == nil {
err = errors.New("stdin was nil")
op.Error(err)
return err
}
done := make(chan error)
go func() {
// copy the tarStream to the binary via stdin; the binary will stream it to InvokeUnpack unchanged
defer stdin.Close()
if _, err := io.Copy(stdin, tarStream); err != nil {
op.Errorf("Error copying tarStream: %s", err.Error())
}
done <- err
}()
out, err := cmd.CombinedOutput()
if len(out) == 0 {
op.Debug("No output from command")
} else {
// output should just be trace messages
op.Debugf("%s", string(out))
return nil, err
}
if err != nil {
stdin.Close()
op.Errorf("Command returned error %s", err.Error())
return err
}
go streamCopy(op, stdin, tarStream)
return cmd, cmd.Start()
// This error gets logged by the goroutine if it is non-nil.
// This receive is just functioning as a wait
err = <-done
return err
}

File diff suppressed because it is too large Load Diff

View File

@@ -43,7 +43,7 @@ const (
// - strip : The strip string will indicate the
// - exlude : marks paths that are to be excluded from the write operation
// - rebase : marks the the write path that will be tacked onto the "root". e.g /tmp/unpack + /my/target/path = /tmp/unpack/my/target/path
func Unpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string) error {
func Unpack(op trace.Operation, tarStream io.Reader, filter *FilterSpec, root string, _ string) error {
op.Debugf("unpacking archive to root: %s, filter: %+v", root, filter)
// Online datasource is sending a tar reader instead of an io reader.

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff