d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

VMware vSphere Integrated Containers provider (#206)

Browse Source 
* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR
This commit is contained in:
Loc Nguyen
2018-06-04 15:41:32 -07:00
committed by Ria Bhatia
parent 98a111e8b7
commit 513cebe7b7
6296 changed files with 1123685 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

485
vendor/github.com/vmware/vic/lib/imagec/docker.go generated vendored Normal file
View File

@@ -0,0 +1,485 @@
// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package imagec
import (
"archive/tar"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
log "github.com/Sirupsen/logrus"
ddigest "github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
dlayer "github.com/docker/docker/layer"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/reference"
"github.com/docker/libtrust"
urlfetcher "github.com/vmware/vic/pkg/fetcher"
registryutils "github.com/vmware/vic/pkg/registry"
"github.com/vmware/vic/pkg/trace"
)
const (
// DigestSHA256EmptyTar is the canonical sha256 digest of empty tar file -
// (1024 NULL bytes)
DigestSHA256EmptyTar = string(dlayer.DigestSHA256EmptyTar)
)
// FSLayer is a container struct for BlobSums defined in an image manifest
type FSLayer struct {
// BlobSum is the tarsum of the referenced filesystem image layer
BlobSum string `json:"blobSum"`
}
// History is a container struct for V1Compatibility defined in an image manifest
type History struct {
V1Compatibility string `json:"v1Compatibility"`
}
// Manifest represents the Docker Manifest file
type Manifest struct {
Name string `json:"name"`
Tag string `json:"tag"`
Digest string `json:"digest,omitempty"`
FSLayers []FSLayer `json:"fsLayers"`
History []History `json:"history"`
// ignoring signatures
}
// LearnRegistryURL returns the registry URL after making sure that it responds to queries
func LearnRegistryURL(options *Options) (string, error) {
defer trace.End(trace.Begin(options.Registry))
log.Debugf("Trying https scheme for %#v", options)
registry, err := registryutils.Reachable(options.Registry, "https", options.Username, options.Password, options.RegistryCAs, options.Timeout, options.InsecureSkipVerify)
if err != nil && options.InsecureAllowHTTP {
// try https without verification
log.Debugf("Trying https without verification, last error: %+v", err)
registry, err = registryutils.Reachable(options.Registry, "https", options.Username, options.Password, options.RegistryCAs, options.Timeout, true)
if err == nil {
// Success, set InsecureSkipVerify to true
options.InsecureSkipVerify = true
} else {
// try http
log.Debugf("Falling back to http")
registry, err = registryutils.Reachable(options.Registry, "http", options.Username, options.Password, options.RegistryCAs, options.Timeout, options.InsecureSkipVerify)
}
}
return registry, err
}
// LearnAuthURL returns the URL of the OAuth endpoint
func LearnAuthURL(options Options) (*url.URL, error) {
defer trace.End(trace.Begin(options.Reference.String()))
url, err := url.Parse(options.Registry)
if err != nil {
return nil, err
}
tagOrDigest := tagOrDigest(options.Reference, options.Tag)
manifestURL, err := url.Parse(path.Join(url.Path, options.Image, "manifests", tagOrDigest))
if err != nil {
return nil, err
}
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
InsecureSkipVerify: options.InsecureSkipVerify,
RootCAs: options.RegistryCAs,
})
// We expect docker registry to return a 401 to us - with a WWW-Authenticate header
// We parse that header and learn the OAuth endpoint to fetch OAuth token.
log.Debugf("Pinging %s", manifestURL.String())
hdr, err := fetcher.Ping(manifestURL)
if err == nil && fetcher.IsStatusUnauthorized() {
return fetcher.ExtractOAuthURL(hdr.Get("www-authenticate"), nil)
}
if !fetcher.IsStatusOK() {
// Try with just the registry url. This works better with some registries (e.g.
// Artifactory)
log.Debugf("Pinging %s", url.String())
hdr, err = fetcher.Ping(url)
if err == nil && fetcher.IsStatusUnauthorized() {
return fetcher.ExtractOAuthURL(hdr.Get("www-authenticate"), nil)
}
}
// Private registry returned the manifest directly as auth option is optional.
// https://github.com/docker/distribution/blob/master/docs/configuration.md#auth
if err == nil && options.Registry != DefaultDockerURL && fetcher.IsStatusOK() {
log.Debugf("%s does not support OAuth", url)
return nil, nil
}
// Do we even have the image on that registry
if err != nil && fetcher.IsStatusNotFound() {
err = fmt.Errorf("image not found")
return nil, urlfetcher.ImageNotFoundError{Err: err}
}
return nil, fmt.Errorf("%s returned an unexpected response: %s", url, err)
}
// FetchToken fetches the OAuth token from OAuth endpoint
func FetchToken(ctx context.Context, options Options, url *url.URL, progressOutput progress.Output) (*urlfetcher.Token, error) {
defer trace.End(trace.Begin(url.String()))
log.Debugf("URL: %s", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
InsecureSkipVerify: options.InsecureSkipVerify,
RootCAs: options.RegistryCAs,
})
token, err := fetcher.FetchAuthToken(url)
if err != nil {
err := fmt.Errorf("FetchToken (%s) failed: %s", url, err)
log.Error(err)
return nil, err
}
return token, nil
}
// FetchImageBlob fetches the image blob
func FetchImageBlob(ctx context.Context, options Options, image *ImageWithMeta, progressOutput progress.Output) (string, error) {
defer trace.End(trace.Begin(options.Image + "/" + image.Layer.BlobSum))
id := image.ID
layer := image.Layer.BlobSum
meta := image.Meta
diffID := ""
url, err := url.Parse(options.Registry)
if err != nil {
return diffID, err
}
url.Path = path.Join(url.Path, options.Image, "blobs", layer)
log.Debugf("URL: %s\n ", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
Token: options.Token,
InsecureSkipVerify: options.InsecureSkipVerify,
RootCAs: options.RegistryCAs,
})
// ctx
ctx, cancel := context.WithTimeout(ctx, options.Timeout)
defer cancel()
imageFileName, err := fetcher.Fetch(ctx, url, nil, true, progressOutput, image.String())
if err != nil {
return diffID, err
}
// Cleanup function for the error case
defer func() {
if err != nil {
os.Remove(imageFileName)
}
}()
// Open the file so that we can use it as a io.Reader for sha256 calculation
imageFile, err := os.Open(string(imageFileName))
if err != nil {
return diffID, err
}
defer imageFile.Close()
// blobSum is the sha of the compressed layer
blobSum := sha256.New()
// diffIDSum is the sha of the uncompressed layer
diffIDSum := sha256.New()
// blobTr is an io.TeeReader that writes bytes to blobSum that it reads from imageFile
// see https://golang.org/pkg/io/#TeeReader
blobTr := io.TeeReader(imageFile, blobSum)
progress.Update(progressOutput, image.String(), "Verifying Checksum")
decompressedTar, err := archive.DecompressStream(blobTr)
if err != nil {
return diffID, err
}
// Copy bytes from decompressed layer into diffIDSum to calculate diffID
_, cerr := io.Copy(diffIDSum, decompressedTar)
if cerr != nil {
return diffID, cerr
}
bs := fmt.Sprintf("sha256:%x", blobSum.Sum(nil))
if bs != layer {
return diffID, fmt.Errorf("Failed to validate layer checksum. Expected %s got %s", layer, bs)
}
diffID = fmt.Sprintf("sha256:%x", diffIDSum.Sum(nil))
// this isn't an empty layer, so we need to calculate the size
if diffID != string(DigestSHA256EmptyTar) {
var layerSize int64
// seek to the beginning of the file
imageFile.Seek(0, 0)
// recreate the decompressed tar Reader
decompressedTar, err := archive.DecompressStream(imageFile)
if err != nil {
return "", err
}
// get a tar reader for access to the files in the archive
tr := tar.NewReader(decompressedTar)
// iterate through tar headers to get file sizes
for {
tarHeader, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return "", err
}
layerSize += tarHeader.Size
}
image.Size = layerSize
}
log.Infof("diffID for layer %s: %s", id, diffID)
// Ensure the parent directory exists
destination := path.Join(DestinationDirectory(options), id)
err = os.MkdirAll(destination, 0755) /* #nosec */
if err != nil {
return diffID, err
}
// Move(rename) the temporary file to its final destination
err = os.Rename(string(imageFileName), path.Join(destination, id+".tar"))
if err != nil {
return diffID, err
}
// Dump the history next to it
err = ioutil.WriteFile(path.Join(destination, id+".json"), []byte(meta), 0644)
if err != nil {
return diffID, err
}
progress.Update(progressOutput, image.String(), "Download complete")
return diffID, nil
}
// tagOrDigest returns an image's digest if it's pulled by digest, or its tag
// otherwise.
func tagOrDigest(r reference.Named, tag string) string {
if digested, ok := r.(reference.Canonical); ok {
return digested.Digest().String()
}
return tag
}
// FetchImageManifest fetches the image manifest file
func FetchImageManifest(ctx context.Context, options Options, schemaVersion int, progressOutput progress.Output) (interface{}, string, error) {
defer trace.End(trace.Begin(options.Reference.String()))
if schemaVersion != 1 && schemaVersion != 2 {
return nil, "", fmt.Errorf("Unknown schema version %d requested!", schemaVersion)
}
url, err := url.Parse(options.Registry)
if err != nil {
return nil, "", err
}
tagOrDigest := tagOrDigest(options.Reference, options.Tag)
url.Path = path.Join(url.Path, options.Image, "manifests", tagOrDigest)
log.Debugf("URL: %s", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
Token: options.Token,
InsecureSkipVerify: options.InsecureSkipVerify,
RootCAs: options.RegistryCAs,
})
reqHeaders := make(http.Header)
if schemaVersion == 2 {
reqHeaders.Add("Accept", schema2.MediaTypeManifest)
reqHeaders.Add("Accept", schema1.MediaTypeManifest)
}
manifestFileName, err := fetcher.Fetch(ctx, url, &reqHeaders, true, progressOutput)
if err != nil {
return nil, "", err
}
// Cleanup function for the error case
defer func() {
if err != nil {
os.Remove(manifestFileName)
}
}()
switch schemaVersion {
case 1: //schema 1, signed manifest
return decodeManifestSchema1(manifestFileName, options, url.Hostname())
case 2: //schema 2
return decodeManifestSchema2(manifestFileName, options)
}
//We shouldn't really get here
return nil, "", fmt.Errorf("Unknown schema version %d requested!", schemaVersion)
}
// decodeManifestSchema1() reads a manifest schema 1 and creates an imageC
// defined Manifest structure and returns the digest of the manifest as a string.
// For historical reason, we did not use the Docker's defined schema1.Manifest
// instead of our own and probably should do so in the future.
func decodeManifestSchema1(filename string, options Options, registry string) (interface{}, string, error) {
// Read the entire file into []byte for json.Unmarshal
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, "", err
}
manifest := &Manifest{}
err = json.Unmarshal(content, manifest)
if err != nil {
return nil, "", err
}
digest, err := getManifestDigest(content, options.Reference)
if err != nil {
return nil, "", err
}
manifest.Digest = digest
// Verify schema 1 manifest's fields per docker/docker/distribution/pull_v2.go
numFSLayers := len(manifest.FSLayers)
if numFSLayers == 0 {
return nil, "", fmt.Errorf("no FSLayers in manifest")
}
if numFSLayers != len(manifest.History) {
return nil, "", fmt.Errorf("length of history not equal to number of layers")
}
return manifest, digest, nil
}
// verifyManifestDigest checks the manifest digest against the received payload.
func verifyManifestDigest(digested reference.Canonical, bytes []byte) error {
verifier, err := ddigest.NewDigestVerifier(digested.Digest())
if err != nil {
return err
}
if _, err = verifier.Write(bytes); err != nil {
return err
}
if !verifier.Verified() {
return fmt.Errorf("image manifest verification failed for digest %s", digested.Digest())
}
return nil
}
// decodeManifestSchema2() reads a manifest schema 2 and creates a Docker
// defined Manifest structure and returns the digest of the manifest as a string.
func decodeManifestSchema2(filename string, options Options) (interface{}, string, error) {
// Read the entire file into []byte for json.Unmarshal
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, "", err
}
manifest := &schema2.DeserializedManifest{}
err = json.Unmarshal(content, manifest)
if err != nil {
return nil, "", err
}
_, canonical, err := manifest.Payload()
if err != nil {
return nil, "", err
}
digest := ddigest.FromBytes(canonical)
return manifest, string(digest), nil
}
func getManifestDigest(content []byte, ref reference.Named) (string, error) {
jsonSig, err := libtrust.ParsePrettySignature(content, "signatures")
if err != nil {
return "", err
}
// Resolve the payload in the manifest.
bytes, err := jsonSig.Payload()
if err != nil {
return "", err
}
log.Debugf("Canonical Bytes: %d", len(bytes))
// Verify the manifest digest if the image is pulled by digest. If the image
// is not pulled by digest, we proceed without this check because we don't
// have a digest to verify the received content with.
// https://docs.docker.com/registry/spec/api/#content-digests
if digested, ok := ref.(reference.Canonical); ok {
if err := verifyManifestDigest(digested, bytes); err != nil {
return "", err
}
}
digest := ddigest.FromBytes(bytes)
// Correct Manifest Digest
log.Debugf("Manifest Digest: %v", digest)
return string(digest), nil
}

105
vendor/github.com/vmware/vic/lib/imagec/docker_test.go generated vendored Normal file
View File

@@ -0,0 +1,105 @@
// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package imagec
import (
"testing"
"github.com/docker/docker/reference"
"github.com/stretchr/testify/assert"
)
const (
UbuntuTaggedRef = "library/ubuntu:latest"
UbuntuDigest = "ubuntu@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2"
UbuntuDigestSHA = "sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2"
UbuntuDigestManifest = `{
"schemaVersion": 1,
"name": "library/ubuntu",
"tag": "14.04",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:28a2f68d1120598986362662445c47dce7ec13c2662479e7aab9f0ecad4a7416"
},
{
"blobSum": "sha256:fd2731e4c50ce221d785d4ce26a8430bca9a95bfe4162fafc997a1cc65682cce"
},
{
"blobSum": "sha256:5a132a7e7af11f304041e93efb9cb2a0a7839bccaec5a03cfbdc9a3f5d0eb481"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"56063ad57855f2632f641a622efa81a0feda1731bfadda497b1288d11feef4da\",\"parent\":\"4e1f7c524148bf80fcc4ce9991e88d708048d38440e3e3a14d56e72c17ddccc7\",\"created\":\"2016-03-03T21:38:53.80360049Z\",\"container\":\"b6361ab0a2a82f71c5bd3becbb9c854331f8259b9c3fe466bf6e7e073c735a2c\",\"container_config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/bin/bash\\\"]\"],\"Image\":\"4e1f7c524148bf80fcc4ce9991e88d708048d38440e3e3a14d56e72c17ddccc7\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.9.1\",\"config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":[\"/bin/bash\"],\"Image\":\"4e1f7c524148bf80fcc4ce9991e88d708048d38440e3e3a14d56e72c17ddccc7\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\"}"
},
{
"v1Compatibility": "{\"id\":\"4e1f7c524148bf80fcc4ce9991e88d708048d38440e3e3a14d56e72c17ddccc7\",\"parent\":\"38112156678df7d8001ae944f118d283009565540dc0cd88fb39fccc88c3c7f2\",\"created\":\"2016-03-03T21:38:53.085760873Z\",\"container\":\"ccc6ec8b31df981344b4107bd890394be35564adb8d13df34d1cb1849c7f0c3e\",\"container_config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":[\"/bin/sh\",\"-c\",\"sed -i 's/^#\\\\s*\\\\(deb.*universe\\\\)$/\\\\1/g' /etc/apt/sources.list\"],\"Image\":\"38112156678df7d8001ae944f118d283009565540dc0cd88fb39fccc88c3c7f2\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.9.1\",\"config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":null,\"Image\":\"38112156678df7d8001ae944f118d283009565540dc0cd88fb39fccc88c3c7f2\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":1895}"
},
{
"v1Compatibility": "{\"id\":\"38112156678df7d8001ae944f118d283009565540dc0cd88fb39fccc88c3c7f2\",\"parent\":\"454970bd163ba95435b50e963edd63b2b2fff4c1845e5d3cd03d5ba8afb8a08d\",\"created\":\"2016-03-03T21:38:51.45368726Z\",\"container\":\"3c8556d1a209f22cfbc87f3cbd9bcb6674c5f9645a14aa488756d129f6987f40\",\"container_config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":[\"/bin/sh\",\"-c\",\"echo '#!/bin/sh' \\u003e /usr/sbin/policy-rc.d \\t\\u0026\\u0026 echo 'exit 101' \\u003e\\u003e /usr/sbin/policy-rc.d \\t\\u0026\\u0026 chmod +x /usr/sbin/policy-rc.d \\t\\t\\u0026\\u0026 dpkg-divert --local --rename --add /sbin/initctl \\t\\u0026\\u0026 cp -a /usr/sbin/policy-rc.d /sbin/initctl \\t\\u0026\\u0026 sed -i 's/^exit.*/exit 0/' /sbin/initctl \\t\\t\\u0026\\u0026 echo 'force-unsafe-io' \\u003e /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\t\\t\\u0026\\u0026 echo 'DPkg::Post-Invoke { \\\"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\\\"; };' \\u003e /etc/apt/apt.conf.d/docker-clean \\t\\u0026\\u0026 echo 'APT::Update::Post-Invoke { \\\"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\\\"; };' \\u003e\\u003e /etc/apt/apt.conf.d/docker-clean \\t\\u0026\\u0026 echo 'Dir::Cache::pkgcache \\\"\\\"; Dir::Cache::srcpkgcache \\\"\\\";' \\u003e\\u003e /etc/apt/apt.conf.d/docker-clean \\t\\t\\u0026\\u0026 echo 'Acquire::Languages \\\"none\\\";' \\u003e /etc/apt/apt.conf.d/docker-no-languages \\t\\t\\u0026\\u0026 echo 'Acquire::GzipIndexes \\\"true\\\"; Acquire::CompressionTypes::Order:: \\\"gz\\\";' \\u003e /etc/apt/apt.conf.d/docker-gzip-indexes\"],\"Image\":\"454970bd163ba95435b50e963edd63b2b2fff4c1845e5d3cd03d5ba8afb8a08d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.9.1\",\"config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":null,\"Image\":\"454970bd163ba95435b50e963edd63b2b2fff4c1845e5d3cd03d5ba8afb8a08d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":194533}"
},
{
"v1Compatibility": "{\"id\":\"454970bd163ba95435b50e963edd63b2b2fff4c1845e5d3cd03d5ba8afb8a08d\",\"created\":\"2016-03-03T21:38:46.169812943Z\",\"container\":\"c24ffee5b2b808674d335e2c42c9c47aa9aff1b368eb5920018cde7dd26f2046\",\"container_config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ADD file:b9504126dc55908988977286e89c43c7ea73a506d43fae82c29ef132e21b6ece in /\"],\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.9.1\",\"config\":{\"Hostname\":\"c24ffee5b2b8\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":187763841}"
}
],
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "G2JA:NPRD:EWRM:EFEJ:4PHQ:TRZR:6W6O:D5LC:UJ36:RHOE:ZN7D:N55I",
"kty": "EC",
"x": "NrETepARqTLeOBcTdBCE8K8jbQJgiTH1p7XJ78zBxjk",
"y": "ay0SmJatkJs-JdnW80807CcNPWHElsh6MW_JTh7NdbU"
},
"alg": "ES256"
},
"signature": "kHbWdD1NQw2RIAQ8uYnKmolU3Z_WeUW8DfRtJRprVDzK7AZaF-ChI4V9Lh74HnjSNwoNZ_QRhUQDl_Nezb0Hgw",
"protected": "eyJmb3JtYXRMZW5ndGgiOjY3NzksImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNy0wNS0xN1QxODowMjozMFoifQ"
}
]
}
`
)
func TestGetManifestDigest(t *testing.T) {
// Get the manifest content when the image is not pulled by digest
ref, err := reference.ParseNamed(UbuntuTaggedRef)
if err != nil {
t.Errorf(err.Error())
}
digest, err := getManifestDigest([]byte(UbuntuDigestManifest), ref)
assert.NoError(t, err)
assert.Equal(t, digest, UbuntuDigestSHA)
// Get and verify the manifest content with the correct digest
ref, err = reference.ParseNamed(UbuntuDigest)
if err != nil {
t.Errorf(err.Error())
}
_, ok := ref.(reference.Canonical)
assert.True(t, ok)
digest, err = getManifestDigest([]byte(UbuntuDigestManifest), ref)
assert.NoError(t, err)
assert.Equal(t, digest, UbuntuDigestSHA)
// Attempt to get and verify an incorrect manifest content with the digest
digest, err = getManifestDigest([]byte(DefaultManifest), ref)
assert.NotNil(t, err)
}

388
vendor/github.com/vmware/vic/lib/imagec/download.go generated vendored Normal file
View File

@@ -0,0 +1,388 @@
// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissi[ons and
// limitations under the License.
package imagec
import (
"context"
"errors"
"fmt"
"sync"
"github.com/vmware/vic/lib/apiservers/engine/backends/cache"
"github.com/vmware/vic/pkg/trace"
"github.com/docker/docker/distribution/xfer"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
)
const (
maxDownloadAttempts = 5
maxConcurrentDownloads = 3
)
type downloadTransfer struct {
xfer.Transfer
layer *ImageWithMeta
err error
}
// result returns the layer resulting from the download, if the download
// and registration were successful.
func (d *downloadTransfer) result() error {
return d.err
}
// LayerDownloader keeps track of what layers are currently being downloaded
type LayerDownloader struct {
m sync.Mutex
downloadsByID map[string]*downloadTransfer
tm xfer.TransferManager
}
// NewLayerDownloader creates and returns a new LayerDownloadManager
func NewLayerDownloader() *LayerDownloader {
return &LayerDownloader{
tm: xfer.NewTransferManager(maxConcurrentDownloads),
downloadsByID: make(map[string]*downloadTransfer),
}
}
func (ldm *LayerDownloader) registerDownload(download *downloadTransfer) {
ldm.downloadsByID[download.layer.ID] = download
}
func (ldm *LayerDownloader) unregisterDownload(layer *ImageWithMeta) {
// stop tracking the download transfer
delete(ldm.downloadsByID, layer.ID)
}
type prog struct {
err chan error
p progress.Progress
}
type serialProgressOutput struct {
c chan prog
out progress.Output
}
func (s *serialProgressOutput) WriteProgress(p progress.Progress) error {
pr := prog{err: make(chan error, 1), p: p}
s.c <- pr
err := <-pr.err
close(pr.err)
return err
}
func (s *serialProgressOutput) run() {
for pr := range s.c {
err := s.out.WriteProgress(pr.p)
pr.err <- err
}
}
func (s *serialProgressOutput) stop() {
close(s.c)
}
// DownloadLayers ensures layers end up in the portlayer's image store
// It handles existing and simultaneous layer download de-duplication
// This code is utilizes Docker's xfer package: https://github.com/docker/docker/tree/v1.11.2/distribution/xfer
func (ldm *LayerDownloader) DownloadLayers(ctx context.Context, ic *ImageC) error {
defer trace.End(trace.Begin(""))
var (
topDownload *downloadTransfer
watcher *xfer.Watcher
d xfer.Transfer
layerCount = 0
sf = streamformatter.NewJSONStreamFormatter()
progressOutput = &serialProgressOutput{
c: make(chan prog, 100),
out: sf.NewProgressOutput(ic.Outstream, false),
}
)
go progressOutput.run()
defer progressOutput.stop()
// lock here so that we get all layers in flight before another client comes along
ldm.m.Lock()
// Grab the imageLayers
layers := ic.ImageLayers
// iterate backwards through layers to download
for i := len(layers) - 1; i >= 0; i-- {
layer := layers[i]
id := layer.ID
layerConfig, err := LayerCache().Get(id)
if err != nil {
switch err := err.(type) {
case LayerNotFoundError:
layerCount++
// layer does not already exist in store and is not currently in flight, so download it
progress.Update(progressOutput, layer.String(), "Pulling fs layer")
xferFunc := ldm.makeDownloadFunc(layer, ic, topDownload, layers)
d, watcher = ldm.tm.Transfer(id, xferFunc, progressOutput)
topDownload = d.(*downloadTransfer)
defer topDownload.Transfer.Release(watcher)
ldm.registerDownload(topDownload)
layer.Downloading = true
LayerCache().Add(layer)
continue
default:
return err
}
}
if layerConfig.Downloading {
layerCount++
if existingDownload, ok := ldm.downloadsByID[id]; ok {
xferFunc := ldm.makeDownloadFuncFromDownload(layer, existingDownload, topDownload, layers)
d, watcher = ldm.tm.Transfer(id, xferFunc, progressOutput)
topDownload = d.(*downloadTransfer)
defer topDownload.Transfer.Release(watcher)
}
continue
}
progress.Update(progressOutput, layer.String(), "Already exists")
}
ldm.m.Unlock()
// each layer download will block until the parent download finishes,
// so this will block until the child-most layer, and thus all layers, have finished downloading
if layerCount > 0 {
select {
case <-ctx.Done():
return ctx.Err()
case <-topDownload.Done():
default:
<-topDownload.Done()
}
err := topDownload.result()
if err != nil {
return err
}
} else {
if err := UpdateRepoCache(ic); err != nil {
return err
}
}
progress.Message(progressOutput, "", "Digest: "+ic.ManifestDigest)
tagOrDigest := tagOrDigest(ic.Reference, ic.Tag)
if layerCount > 0 {
progress.Message(progressOutput, "", "Status: Downloaded newer image for "+ic.Image+":"+tagOrDigest)
} else {
progress.Message(progressOutput, "", "Status: Image is up to date for "+ic.Image+":"+tagOrDigest)
}
return nil
}
// makeDownloadFunc returns a func used by xfer.TransferManager to download a layer
func (ldm *LayerDownloader) makeDownloadFunc(layer *ImageWithMeta, ic *ImageC, parentDownload *downloadTransfer, layers []*ImageWithMeta) xfer.DoFunc {
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) xfer.Transfer {
d := &downloadTransfer{
Transfer: xfer.NewTransfer(),
layer: layer,
}
go func() {
defer func() {
close(progressChan)
// remove layer from cache if there was an error attempting to download
if d.err != nil {
LayerCache().Remove(layer.ID)
}
}()
progressOutput := progress.ChanOutput(progressChan)
// wait for TransferManager to give the go-ahead
select {
case <-start:
default:
progress.Update(progressOutput, layer.String(), "Waiting")
<-start
}
if parentDownload != nil {
// bail if parent download failed or was cancelled
select {
case <-parentDownload.Done():
if err := parentDownload.result(); err != nil {
d.err = err
return
}
default:
}
}
// fetch blob
diffID, err := FetchImageBlob(d.Transfer.Context(), ic.Options, layer, progressOutput)
if err != nil {
d.err = fmt.Errorf("%s/%s returned %s", ic.Image, layer.ID, err)
return
}
layer.DiffID = diffID
close(inactive)
if parentDownload != nil {
select {
case <-d.Transfer.Context().Done():
d.err = errors.New("layer download cancelled")
return
default:
<-parentDownload.Done() // block until parent download completes
}
if err := parentDownload.result(); err != nil {
d.err = err
return
}
}
// is this the leaf layer?
imageLayer := layer.ID == layers[0].ID
if !ic.Standalone {
// if this is the leaf layer, we are done and can now create the image config
if imageLayer {
imageConfig, err := ic.CreateImageConfig(layers)
if err != nil {
d.err = err
return
}
// cache and persist the image
cache.ImageCache().Add(&imageConfig)
if err := cache.ImageCache().Save(); err != nil {
d.err = fmt.Errorf("error saving image cache: %s", err)
return
}
// place calculated ImageID in struct
ic.ImageID = imageConfig.ImageID
if err = UpdateRepoCache(ic); err != nil {
d.err = err
return
}
}
}
ldm.m.Lock()
defer ldm.m.Unlock()
// Write blob to the storage layer
if err := ic.WriteImageBlob(layer, progressOutput, imageLayer); err != nil {
d.err = err
return
}
if !ic.Standalone {
// mark the layer as finished downloading
LayerCache().Commit(layer)
}
ldm.unregisterDownload(layer)
}()
return d
}
}
// makeDownloadFuncFromDownload returns a func used by the TransferManager to
// handle a layer that was already seen in this image pull, or is currently being downloaded
func (ldm *LayerDownloader) makeDownloadFuncFromDownload(layer *ImageWithMeta, sourceDownload, parentDownload *downloadTransfer, layers []*ImageWithMeta) xfer.DoFunc {
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) xfer.Transfer {
d := &downloadTransfer{
Transfer: xfer.NewTransfer(),
layer: layer,
}
go func() {
defer close(progressChan)
<-start
close(inactive)
if parentDownload != nil {
select {
case <-d.Transfer.Context().Done():
d.err = errors.New("layer download cancelled")
return
case <-parentDownload.Done(): // wait for parent layer download to complete
}
if err := parentDownload.result(); err != nil {
d.err = err
return
}
}
// sourceDownload should have already finished if
// parentDownload finished, but wait for it explicitly
// to be sure.
select {
case <-d.Transfer.Context().Done():
d.err = errors.New("layer download cancelled")
return
case <-sourceDownload.Done():
}
if err := sourceDownload.result(); err != nil {
d.err = err
return
}
}()
return d
}
}

648
vendor/github.com/vmware/vic/lib/imagec/imagec.go generated vendored Normal file
View File

@@ -0,0 +1,648 @@
// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package imagec
import (
"crypto/sha256"
"crypto/x509"
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"path"
"strings"
"time"
log "github.com/Sirupsen/logrus"
"golang.org/x/net/context"
"github.com/docker/distribution/manifest/schema2"
docker "github.com/docker/docker/image"
dockerLayer "github.com/docker/docker/layer"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/reference"
"github.com/vmware/vic/lib/apiservers/engine/backends/cache"
"github.com/vmware/vic/lib/apiservers/portlayer/models"
"github.com/vmware/vic/lib/constants"
"github.com/vmware/vic/lib/metadata"
urlfetcher "github.com/vmware/vic/pkg/fetcher"
"github.com/vmware/vic/pkg/trace"
"github.com/vmware/vic/pkg/vsphere/sys"
)
// ImageC is responsible for pulling docker images from a repository
type ImageC struct {
Options
// https://raw.githubusercontent.com/docker/docker/master/distribution/pull_v2.go
sf *streamformatter.StreamFormatter
progressOutput progress.Output
// ImageLayers are sourced from the manifest file
ImageLayers []*ImageWithMeta
// ImageID is the docker ImageID calculated during download
ImageID string
}
// NewImageC returns a new instance of ImageC
func NewImageC(options Options, strfmtr *streamformatter.StreamFormatter) *ImageC {
return &ImageC{
Options: options,
sf: strfmtr,
progressOutput: strfmtr.NewProgressOutput(options.Outstream, false),
}
}
// Options contain all options for a single instance of imagec
type Options struct {
Reference reference.Named
Registry string
Image string
Tag string
Destination string
Host string
Storename string
Username string
Password string
Token *urlfetcher.Token
Timeout time.Duration
Outstream io.Writer
InsecureSkipVerify bool
InsecureAllowHTTP bool
// Get both schema 1 and schema 2 manifests. Schema 1 is used to get history
// and was imageC implementation predated schema 2. Schema 2 is used to
// calculate digest.
ImageManifestSchema1 *Manifest
ImageManifestSchema2 *schema2.DeserializedManifest
//Digest of manifest schema 2 or schema 1; therefore, it may not match hash
//of the above manifest.
ManifestDigest string
// RegistryCAs will not be modified by imagec
RegistryCAs *x509.CertPool
// If true, do not bother portlayer or persona
Standalone bool
// image store name or url
ImageStore string
}
// ImageWithMeta wraps the models.Image with some additional metadata
type ImageWithMeta struct {
*models.Image
DiffID string
Layer FSLayer
Meta string
Size int64
Downloading bool
}
func (i *ImageWithMeta) String() string {
return stringid.TruncateID(i.Layer.BlobSum)
}
var (
ldm *LayerDownloader
)
const (
// DefaultDockerURL holds the URL of Docker registry
DefaultDockerURL = "registry.hub.docker.com"
// DefaultDestination specifies the default directory to use
DefaultDestination = "images"
// DefaultPortLayerHost specifies the default port layer server
DefaultPortLayerHost = "localhost:2377"
// DefaultLogfile specifies the default log file name
DefaultLogfile = "imagec.log"
// DefaultHTTPTimeout specifies the default HTTP timeout
DefaultHTTPTimeout = 3600 * time.Second
// attribute update actions
Add = iota + 1
)
func init() {
ldm = NewLayerDownloader()
}
// ParseReference parses the -reference parameter and populate options struct
func (ic *ImageC) ParseReference() {
if reference.IsNameOnly(ic.Reference) {
ic.Tag = reference.DefaultTag
} else {
if tagged, isTagged := ic.Reference.(reference.NamedTagged); isTagged {
ic.Tag = tagged.Tag()
}
}
ic.Registry = ic.Reference.Hostname()
if ic.Registry == reference.DefaultHostname {
ic.Registry = DefaultDockerURL
}
ic.Image = ic.Reference.RemoteName()
}
// DestinationDirectory returns the path of the output directory
func DestinationDirectory(options Options) string {
// #nosec: Errors unhandled.
u, _ := url.Parse(options.Registry)
// Use a hierarchy like following so that we can support multiple schemes, registries and versions
/*
https/
├── 192.168.218.5:5000
│ └── v2
│ └── busybox
│ └── latest
...
│ ├── fef924a0204a00b3ec67318e2ed337b189c99ea19e2bf10ed30a13b87c5e17ab
│ │ ├── fef924a0204a00b3ec67318e2ed337b189c99ea19e2bf10ed30a13b87c5e17ab.json
│ │ └── fef924a0204a00b3ec67318e2ed337b189c99ea19e2bf10ed30a13b87c5e17ab.tar
│ └── manifest.json
└── registry-1.docker.io
└── v2
└── library
└── golang
└── latest
...
├── f61ebe2817bb4e6a7f0a4cf249a5316223f7ecc886feac24b9887a490feaed57
│ ├── f61ebe2817bb4e6a7f0a4cf249a5316223f7ecc886feac24b9887a490feaed57.json
│ └── f61ebe2817bb4e6a7f0a4cf249a5316223f7ecc886feac24b9887a490feaed57.tar
└── manifest.json
*/
if u.Scheme == "" && u.Host == "" && u.Path == "" {
return path.Join(
options.Destination,
"localhost",
options.Image,
options.Tag,
)
}
return path.Join(
options.Destination,
u.Scheme,
u.Host,
u.Path,
options.Image,
options.Tag,
)
}
// LayersToDownload creates a slice of ImageWithMeta for the layers that need to be downloaded
func (ic *ImageC) LayersToDownload() ([]*ImageWithMeta, error) {
images := make([]*ImageWithMeta, len(ic.ImageManifestSchema1.FSLayers))
manifest := ic.ImageManifestSchema1
v1 := docker.V1Image{}
// iterate from parent to children
for i := len(ic.ImageManifestSchema1.History) - 1; i >= 0; i-- {
history := manifest.History[i]
layer := manifest.FSLayers[i]
// unmarshall V1Compatibility to get the image ID
if err := json.Unmarshal([]byte(history.V1Compatibility), &v1); err != nil {
return nil, fmt.Errorf("Failed to unmarshall image history: %s", err)
}
// if parent is empty set it to scratch
parent := constants.ScratchLayerID
if v1.Parent != "" {
parent = v1.Parent
}
// add image to ImageWithMeta list
images[i] = &ImageWithMeta{
Image: &models.Image{
ID: v1.ID,
Parent: parent,
Store: ic.Storename,
},
Meta: history.V1Compatibility,
Layer: layer,
}
// populate manifest layer with existing cached data
if layer, err := LayerCache().Get(images[i].ID); err == nil {
if !layer.Downloading { // possibly unnecessary but won't hurt anything
images[i] = layer
}
}
}
return images, nil
}
// UpdateRepositoryCache will update the repository cache
// that resides in the docker persona. This will add image tag,
// digest and layer information.
func UpdateRepoCache(ic *ImageC) error {
// if standalone then no persona, so exit
if ic.Standalone {
return nil
}
// LayerID for the image layer
imageLayerID := ic.ImageLayers[0].ID
// get the repoCache
repoCache := cache.RepositoryCache()
// In the case that we don't have the ImageID, then we need
// to go to the RepositoryCache to get it.
if ic.ImageID == "" {
// call to repository cache for the imageID for this layer
ic.ImageID = repoCache.GetImageID(imageLayerID)
// if we still don't have an imageID we can't continue
if ic.ImageID == "" {
return fmt.Errorf("ImageID not found by LayerID(%s) in RepositoryCache", imageLayerID)
}
}
// AddReference will add the repo:tag to the repositoryCache and save to the portLayer
err := repoCache.AddReference(ic.Reference, ic.ImageID, true, imageLayerID, true)
if err != nil {
return fmt.Errorf("Unable to Add Image Reference(%s): %s", ic.Reference.String(), err.Error())
}
dig, err := reference.ParseNamed(fmt.Sprintf("%s@%s", ic.Reference.Name(), ic.ManifestDigest))
if err != nil {
return fmt.Errorf("Unable to parse digest: %s", err.Error())
}
// AddReference will add the digest and persist to the portLayer
err = repoCache.AddReference(dig, ic.ImageID, true, imageLayerID, true)
if err != nil {
return fmt.Errorf("Unable to Add Image Digest(%s): %s", dig.String(), err.Error())
}
return nil
}
// WriteImageBlob writes the image blob to the storage layer
func (ic *ImageC) WriteImageBlob(image *ImageWithMeta, progressOutput progress.Output, cleanup bool) error {
defer trace.End(trace.Begin(image.Image.ID))
destination := DestinationDirectory(ic.Options)
id := image.Image.ID
filePath := path.Join(destination, id, id+".tar")
log.Infof("Path: %s", filePath)
f, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("Failed to open file: %s", err)
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return fmt.Errorf("Failed to stat file: %s", err)
}
in := progress.NewProgressReader(
ioutils.NewCancelReadCloser(context.Background(), f),
progressOutput,
fi.Size(),
image.String(),
"Extracting",
)
defer in.Close()
if !ic.Standalone {
// Write the image
err = WriteImage(ic.Host, image, in)
if err != nil {
return fmt.Errorf("Failed to write to image store: %s", err)
}
} else {
// If standalone, write to a local directory
cleanup = false
}
progress.Update(progressOutput, image.String(), "Pull complete")
if cleanup {
if err := os.RemoveAll(destination); err != nil {
return fmt.Errorf("Failed to remove download directory: %s", err)
}
}
return nil
}
// CreateImageConfig constructs the image metadata from layers that compose the image
func (ic *ImageC) CreateImageConfig(images []*ImageWithMeta) (metadata.ImageConfig, error) {
imageLayer := images[0] // the layer that represents the actual image
// if we already have an imageID associated with this layerID, we don't need
// to calculate imageID and can just grab the image config from the cache
id := cache.RepositoryCache().GetImageID(imageLayer.ID)
if image, err := cache.ImageCache().Get(id); err == nil {
return *image, nil
}
manifest := ic.ImageManifestSchema1
image := docker.V1Image{}
rootFS := docker.NewRootFS()
history := make([]docker.History, 0, len(images))
diffIDs := make(map[string]string)
var size int64
// step through layers to get command history and diffID from oldest to newest
for i := len(images) - 1; i >= 0; i-- {
layer := images[i]
if err := json.Unmarshal([]byte(layer.Meta), &image); err != nil {
return metadata.ImageConfig{}, fmt.Errorf("Failed to unmarshall layer history: %s", err)
}
h := docker.History{
Created: image.Created,
Author: image.Author,
CreatedBy: strings.Join(image.ContainerConfig.Cmd, " "),
Comment: image.Comment,
}
// is this an empty layer?
if layer.DiffID == dockerLayer.DigestSHA256EmptyTar.String() {
h.EmptyLayer = true
} else {
// if not empty, add diffID to rootFS
rootFS.DiffIDs = append(rootFS.DiffIDs, dockerLayer.DiffID(layer.DiffID))
}
history = append(history, h)
size += layer.Size
}
// result is constructed without unused fields
result := docker.Image{
V1Image: docker.V1Image{
Comment: image.Comment,
Created: image.Created,
Container: image.Container,
ContainerConfig: image.ContainerConfig,
DockerVersion: image.DockerVersion,
Author: image.Author,
Config: image.Config,
Architecture: image.Architecture,
OS: image.OS,
},
RootFS: rootFS,
History: history,
}
imageConfigBytes, err := result.MarshalJSON()
if err != nil {
return metadata.ImageConfig{}, fmt.Errorf("Failed to marshall image metadata: %s", err)
}
// calculate image ID
sum := fmt.Sprintf("%x", sha256.Sum256(imageConfigBytes))
log.Infof("Image ID: sha256:%s", sum)
// prepare metadata
result.V1Image.Parent = image.Parent
result.Size = size
result.V1Image.ID = imageLayer.ID
imageConfig := metadata.ImageConfig{
V1Image: result.V1Image,
ImageID: sum,
DiffIDs: diffIDs,
History: history,
}
if ic.Tag != "" {
imageConfig.Tags = []string{ic.Tag}
}
if manifest != nil {
imageConfig.Name = manifest.Name
}
if ic.Reference != nil {
imageConfig.Reference = ic.Reference.String()
}
if _, ok := ic.Reference.(reference.Canonical); ok {
log.Debugf("Populating digest in imageConfig for image: %s", ic.Reference.String())
imageConfig.Digests = []string{ic.ManifestDigest}
}
return imageConfig, nil
}
// PullImage pulls an image from docker hub
func (ic *ImageC) PullImage() error {
ctx, cancel := context.WithTimeout(ctx, ic.Options.Timeout)
defer cancel()
// Authenticate, get URL, get token
if err := ic.prepareTransfer(ctx); err != nil {
return err
}
// Output message
tagOrDigest := tagOrDigest(ic.Reference, ic.Tag)
progress.Message(ic.progressOutput, "", tagOrDigest+": Pulling from "+ic.Image)
// Pull the image manifest
if err := ic.pullManifest(ctx); err != nil {
return err
}
log.Infof("Manifest for image = %#v", ic.ImageManifestSchema1)
// Get layers to download from manifest
layers, err := ic.LayersToDownload()
if err != nil {
return err
}
ic.ImageLayers = layers
// Download all the layers
if err := ldm.DownloadLayers(ctx, ic); err != nil {
return err
}
return nil
}
// ListLayer prints out the layers for an image to progress. This is used by imagec standalone binary
// for debug/validation.
func (ic *ImageC) ListLayers() error {
defer trace.End(trace.Begin(""))
ctx, cancel := context.WithTimeout(ctx, ic.Options.Timeout)
defer cancel()
// Authenticate, get URL, get token
if err := ic.prepareTransfer(ctx); err != nil {
return err
}
// Output message
tagOrDigest := tagOrDigest(ic.Reference, ic.Tag)
progress.Message(ic.progressOutput, "", tagOrDigest+": Fetching layers from "+ic.Image)
// Pull the image manifest
if err := ic.pullManifest(ctx); err != nil {
return err
}
// Get layers to download from manifest
layers, err := ic.LayersToDownload()
if err != nil {
return err
}
progress.Message(ic.progressOutput, "", constants.ScratchLayerID)
for i := len(layers) - 1; i >= 0; i-- {
progress.Message(ic.progressOutput, "", layers[i].ID)
}
return nil
}
// prepareTransfer Looks up URLs and fetch auth token
func (ic *ImageC) prepareTransfer(ctx context.Context) error {
// Parse the -reference parameter
ic.ParseReference()
// Host is either the host's UUID (if run on vsphere) or the hostname of
// the system (if run standalone)
host, err := sys.UUID()
if err != nil {
log.Errorf("Failed to return host name: %s", err)
return err
}
if host != "" {
log.Infof("Using UUID (%s) for imagestore name", host)
} else if ic.Standalone {
host, err = os.Hostname()
log.Infof("Using host (%s) for imagestore name", host)
}
ic.Storename = host
if !ic.Standalone {
log.Debugf("Running with portlayer")
// Ping the server to ensure it's at least running
ok, err := PingPortLayer(ic.Host)
if err != nil || !ok {
log.Errorf("Failed to ping portlayer: %s", err)
return err
}
} else {
log.Debugf("Running standalone")
}
// Calculate (and overwrite) the registry URL and make sure that it responds to requests
ic.Registry, err = LearnRegistryURL(&ic.Options)
if err != nil {
log.Errorf("Error while pulling image: %s", err)
return err
}
// Get the URL of the OAuth endpoint
url, err := LearnAuthURL(ic.Options)
if err != nil {
log.Warnf("LearnAuthURL returned %s", err.Error())
switch err := err.(type) {
case urlfetcher.ImageNotFoundError:
return fmt.Errorf("Error: image %s not found", ic.Reference)
default:
return fmt.Errorf("Failed to obtain OAuth endpoint: %s", err)
}
}
// Get the OAuth token - if only we have a URL
if url != nil {
token, err := FetchToken(ctx, ic.Options, url, ic.progressOutput)
if err != nil {
log.Errorf("Failed to fetch OAuth token: %s", err)
return err
}
ic.Token = token
}
return nil
}
// pullManifest attempts to pull manifest for an image. Attempts to get schema 2 but will fall back to schema 1.
func (ic *ImageC) pullManifest(ctx context.Context) error {
// Get the schema1 manifest
manifest, digest, err := FetchImageManifest(ctx, ic.Options, 1, ic.progressOutput)
if err != nil {
log.Infof(err.Error())
switch err := err.(type) {
case urlfetcher.ImageNotFoundError:
return fmt.Errorf("Error: image %s not found", ic.Image)
case urlfetcher.TagNotFoundError:
return fmt.Errorf("Tag %s not found in repository %s", ic.Tag, ic.Image)
default:
return fmt.Errorf("Error while pulling image manifest: %s", err)
}
}
schema1, ok := manifest.(*Manifest)
if !ok {
return fmt.Errorf("Error pulling manifest schema 1")
}
ic.ImageManifestSchema1 = schema1
ic.ManifestDigest = digest
// Attempt to get schema2 manifest
manifest, digest, err = FetchImageManifest(ctx, ic.Options, 2, ic.progressOutput)
if err == nil {
if schema2, ok := manifest.(*schema2.DeserializedManifest); ok {
if schema2 != nil {
log.Infof("pullManifest - schema 2: %#v", schema2)
}
ic.ImageManifestSchema2 = schema2
// Override the manifest digest as Docker uses schema 2, unless the image
// is pulled by digest since we only support pull-by-digest for schema 1.
// TODO(anchal): this check should be removed once issue #5187 is implemented.
if _, ok := ic.Reference.(reference.Canonical); !ok {
ic.ManifestDigest = digest
}
}
}
return nil
}

616
vendor/github.com/vmware/vic/lib/imagec/imagec_test.go generated vendored Normal file
View File

@@ -0,0 +1,616 @@
// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package imagec
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path"
"strings"
"testing"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/reference"
"github.com/stretchr/testify/assert"
"github.com/vmware/vic/lib/apiservers/portlayer/models"
urlfetcher "github.com/vmware/vic/pkg/fetcher"
)
const (
OAuthToken = "Top_Secret_Token"
Image = "library/photon"
Tag = "latest"
Reference = Image + ":" + Tag
BusyboxImage = "library/busybox"
// fake content
LayerContent = "Cannot_Contain_Myself"
// fake ID
LayerID = "f9767cae14f372c98900f15bb07cb40b8e1a6d1507912489e1342db499313d32"
// fake history
LayerHistory = "{\"id\":\"f9767cae14f372c98900f15bb07cb40b8e1a6d1507912489e1342db499313d32\"" + "," +
"\"parent\":\"09a5baea69e9c781d64df5366c36492d53d507048035abd68632264dc23a1edb\"}"
// fake store
Storename = "PetStore"
// sha256 sum of LayerContent
DigestSHA256LayerContent = "sha256:1f1f9635040c465c7f7b32a396e56e26c1396dbadfbed744b0aab9337a24ad5a"
//DigestSHA256EmptyData is the canonical sha256 digest of empty data
DigestSHA256EmptyData = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
DefaultManifest = `
{
"schemaVersion": 1,
"name": "library/photon",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
}
],
"history": [
{
"v1Compatibility": "{\"architecture\":\"amd64\",\"config\":{\"Hostname\":\"156e10b83429\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"sh\"],\"Image\":\"56ed16bd6310cca65920c653a9bb22de6b235990dcaa1742ff839867aed730e5\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"container\":\"5f8098ec29947b5bea80483cd3275008911ce87438fed628e34ec0c522665510\",\"container_config\":{\"Hostname\":\"156e10b83429\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"sh\\\"]\"],\"Image\":\"56ed16bd6310cca65920c653a9bb22de6b235990dcaa1742ff839867aed730e5\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"created\":\"2016-03-18T18:22:48.810791943Z\",\"docker_version\":\"1.9.1\",\"id\":\"437595becdebaaaf3a4fc3db02c59a980f955dee825c153308c670610bb694e1\",\"os\":\"linux\",\"parent\":\"920777304d1d5e337bc59877253e946f224df5aae64c72538672eb74637b3c9e\"}"
}
],
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "LUQI:WBTB:JRDU:TTD2:FUVY:EMCB:64HP:MZF6:SGFS:XAB6:JPUK:6PK4",
"kty": "EC",
"x": "zjkAuFGpCuWOBl-iMMzZqgl_1cid-04S04-k-A1qEeU",
"y": "9HWcOMfVFUMXJGeNajIAlPicL4UOsCJSpqRcIxpUl0Q"
},
"alg": "ES256"
},
"signature": "dTXnnt3IkTScpZhyyqRlmZFcQV1QzD7lWDqnjlD4Cj-KsMuGd1pl5QpFL2Cadw-8KeTBlSleSecxjHU4t3yhCQ",
"protected": "eyJmb3JtYXRMZW5ndGgiOjE5MjIsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNi0wNi0wOVQxNzoyMzo1NFoifQ"
}
]
}
`
)
func TestParseReference(t *testing.T) {
options := Options{
Outstream: os.Stdout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
ref, err := reference.ParseNamed("index.docker.io/library/busybox")
ic.Options.Reference = ref
ic.ParseReference()
assert.Equal(t, ic.Tag, reference.DefaultTag)
assert.Equal(t, ic.Image, BusyboxImage)
assert.Equal(t, ic.Registry, DefaultDockerURL)
ref, err = reference.ParseNamed("vmware/photon")
ic.Options.Reference = ref
ic.ParseReference()
assert.Equal(t, ic.Tag, reference.DefaultTag)
assert.Equal(t, ic.Image, "vmware/photon")
assert.Equal(t, ic.Registry, DefaultDockerURL)
ref, err = reference.ParseNamed("busybox")
if err != nil {
t.Errorf(err.Error())
}
ic.Options.Reference = ref
ic.ParseReference()
assert.Equal(t, ic.Tag, reference.DefaultTag)
assert.Equal(t, ic.Image, BusyboxImage)
assert.Equal(t, ic.Registry, DefaultDockerURL)
ref, err = reference.ParseNamed("library/busybox")
if err != nil {
t.Errorf(err.Error())
}
ic.Options.Reference = ref
ic.ParseReference()
assert.Equal(t, ic.Tag, reference.DefaultTag)
assert.Equal(t, ic.Image, BusyboxImage)
assert.Equal(t, ic.Registry, DefaultDockerURL)
ref, err = reference.ParseNamed("library/busybox:latest")
if err != nil {
t.Errorf(err.Error())
}
ic.Options.Reference = ref
ic.ParseReference()
assert.Equal(t, ic.Tag, reference.DefaultTag)
assert.Equal(t, ic.Image, BusyboxImage)
assert.Equal(t, ic.Registry, DefaultDockerURL)
ic = NewImageC(options, streamformatter.NewJSONStreamFormatter())
ref, err = reference.ParseNamed("busybox")
if err != nil {
t.Errorf(err.Error())
}
digest, err := digest.ParseDigest("sha256:c79345819a6882c31b41bc771d9a94fc52872fa651b36771fbe0c8461d7ee558")
if err != nil {
t.Errorf(err.Error())
}
ref, err = reference.WithDigest(reference.TrimNamed(ref), digest)
if err != nil {
t.Errorf(err.Error())
}
ic.Options.Reference = ref
ic.ParseReference()
assert.Equal(t, ic.Tag, "")
assert.Equal(t, ic.Image, BusyboxImage)
assert.Equal(t, ic.Registry, DefaultDockerURL)
}
func TestLearnRegistryURL(t *testing.T) {
options := Options{
Outstream: os.Stdout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Docker-Distribution-API-Version", "registry/2.0")
http.Error(w, "You shall not pass", http.StatusUnauthorized)
}))
defer s.Close()
ic.Options.Registry = s.URL[7:]
ic.Options.Image = Image
ic.Options.Tag = Tag
ic.Options.Timeout = DefaultHTTPTimeout
// should fail
_, err := LearnRegistryURL(&ic.Options)
if err == nil {
t.Errorf(err.Error())
}
// should pass
ic.Options.InsecureAllowHTTP = true
_, err = LearnRegistryURL(&ic.Options)
if err != nil {
t.Errorf(err.Error())
}
}
func TestLearnAuthURL(t *testing.T) {
var err error
options := Options{
Outstream: os.Stdout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("www-authenticate",
"Bearer realm=\"https://auth.docker.io/token\",service=\"registry.docker.io\",scope=\"repository:library/photon:pull\"")
http.Error(w, "You shall not pass", http.StatusUnauthorized)
}))
defer s.Close()
ic.Options.Registry = s.URL
ic.Options.Image = Image
ic.Options.Tag = Tag
ic.Options.Timeout = DefaultHTTPTimeout
ic.Options.Reference, err = reference.ParseNamed(Reference)
if err != nil {
t.Errorf(err.Error())
}
url, err := LearnAuthURL(ic.Options)
if err != nil {
t.Errorf(err.Error())
}
if url.String() != "https://auth.docker.io/token?scope=repository%3Alibrary%2Fphoton%3Apull&service=registry.docker.io" {
t.Errorf("Returned url %s is different than expected", url)
}
}
func TestFetchToken(t *testing.T) {
options := Options{
Outstream: os.Stdout,
Timeout: DefaultHTTPTimeout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
body, err := json.Marshal(&urlfetcher.Token{Token: OAuthToken})
if err != nil {
t.Errorf(err.Error())
}
w.Write(body)
}))
defer s.Close()
url, err := url.Parse(s.URL)
if err != nil {
t.Errorf(err.Error())
}
url.Path = path.Join(url.Path, "token?scope=repository%3Alibrary%2Fphoton%3Apull&service=registry.docker.io")
ctx := context.TODO()
token, err := FetchToken(ctx, ic.Options, url, ic.progressOutput)
if err != nil {
t.Errorf(err.Error())
}
if token.Token != OAuthToken {
t.Errorf("Returned token %s is different than expected", token.Token)
}
}
func TestFetchImageManifest(t *testing.T) {
var err error
options := Options{
Outstream: os.Stdout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(DefaultManifest))
}))
defer s.Close()
ic.Options.Registry = s.URL
ic.Options.Image = Image
ic.Options.Tag = Tag
ic.Options.Timeout = DefaultHTTPTimeout
ic.Options.Token = &urlfetcher.Token{Token: OAuthToken}
ic.Options.Reference, err = reference.ParseNamed(Reference)
if err != nil {
t.Errorf(err.Error())
}
// create a temporary directory
dir, err := ioutil.TempDir("", "imagec")
if err != nil {
t.Errorf(err.Error())
}
defer os.RemoveAll(dir)
ic.Options.Destination = dir
ctx := context.TODO()
manifest, _, err := FetchImageManifest(ctx, ic.Options, 1, ic.progressOutput)
if err != nil {
t.Errorf(err.Error())
}
if schema1, ok := manifest.(*Manifest); ok {
if schema1.FSLayers[0].BlobSum != DigestSHA256EmptyData {
t.Errorf("Returned manifest %#v is different than expected", manifest)
}
}
}
func TestFetchImageBlob(t *testing.T) {
options := Options{
Outstream: os.Stdout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
// create a tar archive from our dummy data
r := strings.NewReader(LayerContent)
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
defer tw.Close()
// create and populate tar header
header := new(tar.Header)
header.Size = r.Size()
header.Name = LayerID
header.Mode = 0644
// write the header
if err := tw.WriteHeader(header); err != nil {
t.Errorf(err.Error())
}
// write the file into the tar archive
if _, err := io.Copy(tw, r); err != nil {
t.Error(err.Error())
}
attempt := 3
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempt--
if attempt == 0 {
w.Header().Set("Content-Type", "application/x-gzip")
// return our tar archive
w.Write(buf.Bytes())
} else {
// return an error to force caller to retry
w.WriteHeader(http.StatusInternalServerError)
}
}))
defer s.Close()
ic.Options.Registry = s.URL
ic.Options.Image = Image
ic.Options.Tag = Tag
ic.Options.Timeout = DefaultHTTPTimeout
ic.Options.Token = &urlfetcher.Token{Token: OAuthToken}
// create a temporary directory
dir, err := ioutil.TempDir("", "imagec")
if err != nil {
t.Errorf(err.Error())
}
defer os.RemoveAll(dir)
ic.Options.Destination = dir
parent := "scratch"
image := ImageWithMeta{
Image: &models.Image{
ID: LayerID,
Parent: parent,
Store: Storename,
},
Meta: LayerHistory,
Layer: FSLayer{BlobSum: DigestSHA256LayerContent},
}
diffID, err := FetchImageBlob(ctx, ic.Options, &image, ic.progressOutput)
if err != nil {
t.Errorf(err.Error())
}
if diffID == "" {
t.Errorf("Expected a diffID, got nil.")
}
tarFile, err := ioutil.ReadFile(path.Join(DestinationDirectory(ic.Options), LayerID, LayerID+".tar"))
if err != nil {
t.Errorf(err.Error())
}
br := bytes.NewReader(tarFile)
tr := tar.NewReader(br)
out := new(bytes.Buffer)
// extract the tar file
for {
_, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Errorf(err.Error())
}
if _, err := io.Copy(out, tr); err != nil {
t.Errorf(err.Error())
}
}
// compare contents of tar file to dummy data
if out.String() != LayerContent {
t.Errorf(err.Error())
}
hist, err := ioutil.ReadFile(path.Join(DestinationDirectory(ic.Options), LayerID, LayerID+".json"))
if err != nil {
t.Errorf(err.Error())
}
if string(hist) != LayerHistory {
t.Errorf(err.Error())
}
}
func TestPingPortLayer(t *testing.T) {
options := Options{
Outstream: os.Stdout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte("\"OK\""))
}))
defer s.Close()
ic.Options.Host = s.URL[7:]
b, err := PingPortLayer(ic.Host)
if err != nil || b != true {
t.Errorf(err.Error())
}
}
func TestListImages(t *testing.T) {
options := Options{
Outstream: os.Stdout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
resp := `
[
{
"ID": "7bd023c8937ded982c1b98da453b1a5afec86f390ffad8fa0f4fba244a6155f1",
"Metadata": {
"v1Compatibility": "{\"id\":\"7bd023c8937ded982c1b98da453b1a5afec86f390ffad8fa0f4fba244a6155f1\",\"parent\":\"b873f334fa5259acb24cf0e2cd2639d3a9fb3eb9bafbca06ed4f702c289b31c0\",\"created\":\"2016-05-27T14:15:02.359284074Z\",\"container\":\"b8bd6a8e8874a87f626871ce370f4775bdf598865637082da2949ee0f4786432\",\"container_config\":{\"Hostname\":\"914cf42a3e15\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/bin/bash\\\"]\"],\"Image\":\"b873f334fa5259acb24cf0e2cd2639d3a9fb3eb9bafbca06ed4f702c289b31c0\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.9.1\",\"config\":{\"Hostname\":\"914cf42a3e15\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[],\"Cmd\":[\"/bin/bash\"],\"Image\":\"b873f334fa5259acb24cf0e2cd2639d3a9fb3eb9bafbca06ed4f702c289b31c0\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\"}"
},
"SelfLink": "http://Photon/storage/7bd023c8937ded982c1b98da453b1a5afec86f390ffad8fa0f4fba244a6155f1",
"Store": "http://Photon/storage/PetStore"
}
]
`
w.Write([]byte(resp))
}))
defer s.Close()
ic.Options.Host = s.URL[7:]
m, err := ListImages(ic.Host, ic.Storename, nil)
if err != nil {
t.Errorf(err.Error())
}
if m["7bd023c8937ded982c1b98da453b1a5afec86f390ffad8fa0f4fba244a6155f1"].ID != "7bd023c8937ded982c1b98da453b1a5afec86f390ffad8fa0f4fba244a6155f1" {
t.Errorf("Returned list %#v is different than expected", m)
}
}
func TestFetchScenarios(t *testing.T) {
var err error
ctx := context.TODO()
options := Options{
Outstream: os.Stdout,
Timeout: DefaultHTTPTimeout,
}
ic := NewImageC(options, streamformatter.NewJSONStreamFormatter())
wwwAuthenticate := false
insufficientScope := false
invalidToken := false
s := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
auth := r.Header.Get("Authorization")
if auth == "" {
if wwwAuthenticate {
w.Header().Set("www-authenticate", "Bearer realm=\"https://auth.docker.io/token\",service=\"registry.docker.io\",scope=\"repository:library/busybox:pull\"")
}
w.WriteHeader(http.StatusUnauthorized)
} else {
if !insufficientScope && !invalidToken {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(DefaultManifest))
} else if insufficientScope {
w.Header().Set("www-authenticate", "error=\"insufficient_scope\"")
w.WriteHeader(http.StatusUnauthorized)
} else if invalidToken {
w.Header().Set("www-authenticate", "error=\"invalid_token\"")
w.WriteHeader(http.StatusUnauthorized)
} else {
w.WriteHeader(http.StatusUnauthorized)
}
}
}))
defer s.Close()
ic.Options.Registry = s.URL
ic.Options.Image = Image
ic.Options.Tag = Tag
ic.Options.InsecureAllowHTTP = true
ic.Options.Reference, err = reference.ParseNamed(Reference)
if err != nil {
t.Errorf(err.Error())
}
// create a temporary directory
dir, err := ioutil.TempDir("", "imagec")
if err != nil {
t.Errorf(err.Error())
}
defer os.RemoveAll(dir)
ic.Options.Destination = dir
ic.Options.Token = nil
// try without token, response will miss www-authenticate so we will retry (and fail eventually)
_, _, err = FetchImageManifest(ctx, ic.Options, 1, ic.progressOutput)
if err == nil {
t.Errorf("Condition didn't fail (testing failure)")
}
// set the www-authenticate
wwwAuthenticate = true
// try without token, response will carry a valid www-authenticate so we won't retry
_, _, err = FetchImageManifest(ctx, ic.Options, 1, ic.progressOutput)
if err != nil {
// we should get a DNR error
if _, isDNR := err.(urlfetcher.DoNotRetry); !isDNR {
t.Errorf(err.Error())
}
}
wwwAuthenticate = false
// set a valid token
ic.Options.Token = &urlfetcher.Token{Token: OAuthToken}
// enable invalid token test
invalidToken = true
// valid token but faulty hub, we should retry but eventually fail
_, _, err = FetchImageManifest(ctx, ic.Options, 1, ic.progressOutput)
if err == nil {
t.Errorf("Condition didn't fail (testing failure)")
}
invalidToken = false
// enable insufficient_scope test
insufficientScope = true
// valid token but image is missing we shouldn't retry
_, _, err = FetchImageManifest(ctx, ic.Options, 1, ic.progressOutput)
if err != nil {
// we should get a ImageNotFoundError
if _, imageErr := err.(urlfetcher.ImageNotFoundError); !imageErr {
t.Errorf(err.Error())
}
}
insufficientScope = false
// valid token, existing image, correct header. We should succeed
_, _, err = FetchImageManifest(ctx, ic.Options, 1, ic.progressOutput)
if err != nil {
t.Errorf(err.Error())
}
}

180
vendor/github.com/vmware/vic/lib/imagec/layer_cache.go generated vendored Normal file
View File

@@ -0,0 +1,180 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package imagec
import (
"encoding/json"
"fmt"
"sync"
log "github.com/Sirupsen/logrus"
"github.com/vmware/vic/lib/apiservers/engine/backends/kv"
"github.com/vmware/vic/lib/apiservers/portlayer/client"
"github.com/vmware/vic/pkg/trace"
)
// LCache is an in-memory cache to account for existing image layers
// It is used primarily by imagec when coordinating layer downloads
// The cache is initially hydrated by way of the image cache at startup
type LCache struct {
m sync.RWMutex
layers map[string]*ImageWithMeta
client *client.PortLayer
dirty bool
}
// LayerNotFoundError is returned when a layer does not exist in the cache
type LayerNotFoundError struct{}
func (e LayerNotFoundError) Error() string {
return "Layer does not exist"
}
const (
layerCacheKey = "layers"
)
var (
layerCache *LCache
)
func init() {
layerCache = &LCache{
layers: make(map[string]*ImageWithMeta),
}
}
// LayerCache returns a reference to the layer cache
func LayerCache() *LCache {
return layerCache
}
// InitializeLayerCache will create a new layer cache or rehydrate an
// existing layer cache from the portlayer k/v store
func InitializeLayerCache(client *client.PortLayer) error {
defer trace.End(trace.Begin(""))
log.Debugf("Initializing layer cache")
layerCache.client = client
val, err := kv.Get(client, layerCacheKey)
if err != nil && err != kv.ErrKeyNotFound {
return err
}
l := struct {
Layers map[string]*ImageWithMeta
}{}
if val != "" {
if err = json.Unmarshal([]byte(val), &l); err != nil {
return fmt.Errorf("Failed to unmarshal layer cache: %s", err)
}
layerCache.layers = l.Layers
}
return nil
}
// Add adds a new layer to the cache
func (lc *LCache) Add(layer *ImageWithMeta) {
defer trace.End(trace.Begin(""))
lc.m.Lock()
defer lc.m.Unlock()
lc.layers[layer.ID] = layer
lc.dirty = true
}
// Remove removes a layer from the cache
func (lc *LCache) Remove(id string) {
defer trace.End(trace.Begin(""))
lc.m.Lock()
defer lc.m.Unlock()
if _, ok := lc.layers[id]; ok {
delete(lc.layers, id)
lc.dirty = true
}
}
// Commit marks a layer as downloaded
func (lc *LCache) Commit(layer *ImageWithMeta) error {
defer trace.End(trace.Begin(""))
lc.m.Lock()
defer lc.m.Unlock()
lc.layers[layer.ID] = layer
lc.layers[layer.ID].Downloading = false
lc.dirty = true
return lc.save()
}
// Get returns a cached layer, or LayerNotFoundError if it doesn't exist
func (lc *LCache) Get(id string) (*ImageWithMeta, error) {
defer trace.End(trace.Begin(""))
lc.m.RLock()
defer lc.m.RUnlock()
layer, ok := lc.layers[id]
if !ok {
return nil, LayerNotFoundError{}
}
return layer, nil
}
func (lc *LCache) save() error {
defer trace.End(trace.Begin(""))
if !lc.dirty {
return nil
}
m := struct {
Layers map[string]*ImageWithMeta
}{
Layers: lc.layers,
}
bytes, err := json.Marshal(m)
if err != nil {
log.Errorf("Unable to marshal layer cache: %s", err.Error())
return err
}
err = kv.Put(lc.client, layerCacheKey, string(bytes))
if err != nil {
log.Errorf("Unable to save layer cache: %s", err.Error())
return err
}
lc.dirty = false
return nil
}
// Save will persist the image cache to the portlayer k/v store
func (lc *LCache) Save() error {
defer trace.End(trace.Begin(""))
lc.m.Lock()
defer lc.m.Unlock()
return lc.save()
}

116
vendor/github.com/vmware/vic/lib/imagec/storage.go generated vendored Normal file
View File

@@ -0,0 +1,116 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package imagec
import (
"context"
"io"
log "github.com/Sirupsen/logrus"
"github.com/go-openapi/runtime"
rc "github.com/go-openapi/runtime/client"
apiclient "github.com/vmware/vic/lib/apiservers/portlayer/client"
"github.com/vmware/vic/lib/apiservers/portlayer/client/misc"
"github.com/vmware/vic/lib/apiservers/portlayer/client/storage"
"github.com/vmware/vic/lib/apiservers/portlayer/models"
"github.com/vmware/vic/lib/metadata"
"github.com/vmware/vic/pkg/trace"
)
var (
ctx = context.TODO()
)
// PingPortLayer calls the _ping endpoint of the portlayer
func PingPortLayer(host string) (bool, error) {
defer trace.End(trace.Begin(host))
transport := rc.New(host, "/", []string{"http"})
client := apiclient.New(transport, nil)
ok, err := client.Misc.Ping(misc.NewPingParamsWithContext(ctx))
if err != nil {
return false, err
}
return ok.Payload == "OK", nil
}
// ListImages lists the images from given image store
func ListImages(host, storename string, images []*ImageWithMeta) (map[string]*models.Image, error) {
defer trace.End(trace.Begin(storename))
transport := rc.New(host, "/", []string{"http"})
client := apiclient.New(transport, nil)
ids := make([]string, len(images))
for i := range images {
ids = append(ids, images[i].ID)
}
imageList, err := client.Storage.ListImages(
storage.NewListImagesParamsWithContext(ctx).WithStoreName(storename).WithIds(ids),
)
if err != nil {
return nil, err
}
existingImages := make(map[string]*models.Image)
for i := range imageList.Payload {
v := imageList.Payload[i]
existingImages[v.ID] = v
}
return existingImages, nil
}
// WriteImage writes the image to given image store
func WriteImage(host string, image *ImageWithMeta, data io.ReadCloser) error {
defer trace.End(trace.Begin(image.ID))
transport := rc.New(host, "/", []string{"http"})
client := apiclient.New(transport, nil)
transport.Consumers["application/json"] = runtime.JSONConsumer()
transport.Producers["application/json"] = runtime.JSONProducer()
transport.Consumers["application/octet-stream"] = runtime.ByteStreamConsumer()
transport.Producers["application/octet-stream"] = runtime.ByteStreamProducer()
key := new(string)
blob := new(string)
*key = metadata.MetaDataKey
*blob = image.Meta
r, err := client.Storage.WriteImage(
storage.NewWriteImageParamsWithContext(ctx).
WithImageID(image.ID).
WithParentID(image.Parent).
WithStoreName(image.Store).
WithMetadatakey(key).
WithMetadataval(blob).
WithImageFile(data).
WithSum(image.Layer.BlobSum),
)
if err != nil {
log.Debugf("Creating an image failed: %s", err)
return err
}
log.Printf("Created an image %#v", r.Payload)
return nil
}