Initial commit
This commit is contained in:
Ria Bhatia
38
vendor/github.com/hyperhq/hypercli/distribution/fixtures/validate_manifest/bad_manifest
generated
vendored
Normal file
38
vendor/github.com/hyperhq/hypercli/distribution/fixtures/validate_manifest/bad_manifest
generated
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"name": "library/hello-world",
|
||||
"tag": "latest",
|
||||
"architecture": "amd64",
|
||||
"fsLayers": [
|
||||
{
|
||||
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
|
||||
}
|
||||
],
|
||||
"history": [
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
|
||||
},
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
|
||||
}
|
||||
],
|
||||
"signatures": [
|
||||
{
|
||||
"header": {
|
||||
"jwk": {
|
||||
"crv": "P-256",
|
||||
"kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
|
||||
"kty": "EC",
|
||||
"x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
|
||||
"y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
|
||||
},
|
||||
"alg": "ES256"
|
||||
},
|
||||
"signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
|
||||
"protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
|
||||
}
|
||||
]
|
||||
}
|
||||
46
vendor/github.com/hyperhq/hypercli/distribution/fixtures/validate_manifest/extra_data_manifest
generated
vendored
Normal file
46
vendor/github.com/hyperhq/hypercli/distribution/fixtures/validate_manifest/extra_data_manifest
generated
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"name": "library/hello-world",
|
||||
"tag": "latest",
|
||||
"architecture": "amd64",
|
||||
"fsLayers": [
|
||||
{
|
||||
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
|
||||
}
|
||||
],
|
||||
"history": [
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
|
||||
},
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
|
||||
}
|
||||
],
|
||||
"fsLayers": [
|
||||
{
|
||||
"blobSum": "sha256:ffff95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:ffff658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
|
||||
}
|
||||
],
|
||||
"signatures": [
|
||||
{
|
||||
"header": {
|
||||
"jwk": {
|
||||
"crv": "P-256",
|
||||
"kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
|
||||
"kty": "EC",
|
||||
"x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
|
||||
"y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
|
||||
},
|
||||
"alg": "ES256"
|
||||
},
|
||||
"signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
|
||||
"protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
|
||||
}
|
||||
]
|
||||
}
|
||||
38
vendor/github.com/hyperhq/hypercli/distribution/fixtures/validate_manifest/good_manifest
generated
vendored
Normal file
38
vendor/github.com/hyperhq/hypercli/distribution/fixtures/validate_manifest/good_manifest
generated
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"name": "library/hello-world",
|
||||
"tag": "latest",
|
||||
"architecture": "amd64",
|
||||
"fsLayers": [
|
||||
{
|
||||
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
|
||||
}
|
||||
],
|
||||
"history": [
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
|
||||
},
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
|
||||
}
|
||||
],
|
||||
"signatures": [
|
||||
{
|
||||
"header": {
|
||||
"jwk": {
|
||||
"crv": "P-256",
|
||||
"kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
|
||||
"kty": "EC",
|
||||
"x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
|
||||
"y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
|
||||
},
|
||||
"alg": "ES256"
|
||||
},
|
||||
"signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
|
||||
"protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
|
||||
}
|
||||
]
|
||||
}
|
||||
77
vendor/github.com/hyperhq/hypercli/distribution/metadata/metadata.go
generated
vendored
Normal file
77
vendor/github.com/hyperhq/hypercli/distribution/metadata/metadata.go
generated
vendored
Normal file
@@ -0,0 +1,77 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Store implements a K/V store for mapping distribution-related IDs
|
||||
// to on-disk layer IDs and image IDs. The namespace identifies the type of
|
||||
// mapping (i.e. "v1ids" or "artifacts"). MetadataStore is goroutine-safe.
|
||||
type Store interface {
|
||||
// Get retrieves data by namespace and key.
|
||||
Get(namespace string, key string) ([]byte, error)
|
||||
// Set writes data indexed by namespace and key.
|
||||
Set(namespace, key string, value []byte) error
|
||||
// Delete removes data indexed by namespace and key.
|
||||
Delete(namespace, key string) error
|
||||
}
|
||||
|
||||
// FSMetadataStore uses the filesystem to associate metadata with layer and
|
||||
// image IDs.
|
||||
type FSMetadataStore struct {
|
||||
sync.RWMutex
|
||||
basePath string
|
||||
}
|
||||
|
||||
// NewFSMetadataStore creates a new filesystem-based metadata store.
|
||||
func NewFSMetadataStore(basePath string) (*FSMetadataStore, error) {
|
||||
if err := os.MkdirAll(basePath, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &FSMetadataStore{
|
||||
basePath: basePath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (store *FSMetadataStore) path(namespace, key string) string {
|
||||
return filepath.Join(store.basePath, namespace, key)
|
||||
}
|
||||
|
||||
// Get retrieves data by namespace and key. The data is read from a file named
|
||||
// after the key, stored in the namespace's directory.
|
||||
func (store *FSMetadataStore) Get(namespace string, key string) ([]byte, error) {
|
||||
store.RLock()
|
||||
defer store.RUnlock()
|
||||
|
||||
return ioutil.ReadFile(store.path(namespace, key))
|
||||
}
|
||||
|
||||
// Set writes data indexed by namespace and key. The data is written to a file
|
||||
// named after the key, stored in the namespace's directory.
|
||||
func (store *FSMetadataStore) Set(namespace, key string, value []byte) error {
|
||||
store.Lock()
|
||||
defer store.Unlock()
|
||||
|
||||
path := store.path(namespace, key)
|
||||
tempFilePath := path + ".tmp"
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ioutil.WriteFile(tempFilePath, value, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tempFilePath, path)
|
||||
}
|
||||
|
||||
// Delete removes data indexed by namespace and key. The data file named after
|
||||
// the key, stored in the namespace's directory is deleted.
|
||||
func (store *FSMetadataStore) Delete(namespace, key string) error {
|
||||
store.Lock()
|
||||
defer store.Unlock()
|
||||
|
||||
path := store.path(namespace, key)
|
||||
return os.Remove(path)
|
||||
}
|
||||
44
vendor/github.com/hyperhq/hypercli/distribution/metadata/v1_id_service.go
generated
vendored
Normal file
44
vendor/github.com/hyperhq/hypercli/distribution/metadata/v1_id_service.go
generated
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"github.com/hyperhq/hypercli/image/v1"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
)
|
||||
|
||||
// V1IDService maps v1 IDs to layers on disk.
|
||||
type V1IDService struct {
|
||||
store Store
|
||||
}
|
||||
|
||||
// NewV1IDService creates a new V1 ID mapping service.
|
||||
func NewV1IDService(store Store) *V1IDService {
|
||||
return &V1IDService{
|
||||
store: store,
|
||||
}
|
||||
}
|
||||
|
||||
// namespace returns the namespace used by this service.
|
||||
func (idserv *V1IDService) namespace() string {
|
||||
return "v1id"
|
||||
}
|
||||
|
||||
// Get finds a layer by its V1 ID.
|
||||
func (idserv *V1IDService) Get(v1ID, registry string) (layer.DiffID, error) {
|
||||
if err := v1.ValidateID(v1ID); err != nil {
|
||||
return layer.DiffID(""), err
|
||||
}
|
||||
|
||||
idBytes, err := idserv.store.Get(idserv.namespace(), registry+","+v1ID)
|
||||
if err != nil {
|
||||
return layer.DiffID(""), err
|
||||
}
|
||||
return layer.DiffID(idBytes), nil
|
||||
}
|
||||
|
||||
// Set associates an image with a V1 ID.
|
||||
func (idserv *V1IDService) Set(v1ID, registry string, id layer.DiffID) error {
|
||||
if err := v1.ValidateID(v1ID); err != nil {
|
||||
return err
|
||||
}
|
||||
return idserv.store.Set(idserv.namespace(), registry+","+v1ID, []byte(id))
|
||||
}
|
||||
83
vendor/github.com/hyperhq/hypercli/distribution/metadata/v1_id_service_test.go
generated
vendored
Normal file
83
vendor/github.com/hyperhq/hypercli/distribution/metadata/v1_id_service_test.go
generated
vendored
Normal file
@@ -0,0 +1,83 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
)
|
||||
|
||||
func TestV1IDService(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "v1-id-service-test")
|
||||
if err != nil {
|
||||
t.Fatalf("could not create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
metadataStore, err := NewFSMetadataStore(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("could not create metadata store: %v", err)
|
||||
}
|
||||
v1IDService := NewV1IDService(metadataStore)
|
||||
|
||||
testVectors := []struct {
|
||||
registry string
|
||||
v1ID string
|
||||
layerID layer.DiffID
|
||||
}{
|
||||
{
|
||||
registry: "registry1",
|
||||
v1ID: "f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937",
|
||||
layerID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
||||
},
|
||||
{
|
||||
registry: "registry2",
|
||||
v1ID: "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
|
||||
layerID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
|
||||
},
|
||||
{
|
||||
registry: "registry1",
|
||||
v1ID: "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
|
||||
layerID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
|
||||
},
|
||||
}
|
||||
|
||||
// Set some associations
|
||||
for _, vec := range testVectors {
|
||||
err := v1IDService.Set(vec.v1ID, vec.registry, vec.layerID)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Set: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Check the correct values are read back
|
||||
for _, vec := range testVectors {
|
||||
layerID, err := v1IDService.Get(vec.v1ID, vec.registry)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Get: %v", err)
|
||||
}
|
||||
if layerID != vec.layerID {
|
||||
t.Fatal("Get returned incorrect layer ID")
|
||||
}
|
||||
}
|
||||
|
||||
// Test Get on a nonexistent entry
|
||||
_, err = v1IDService.Get("82379823067823853223359023576437723560923756b03560378f4497753917", "registry1")
|
||||
if err == nil {
|
||||
t.Fatal("expected error looking up nonexistent entry")
|
||||
}
|
||||
|
||||
// Overwrite one of the entries and read it back
|
||||
err = v1IDService.Set(testVectors[0].v1ID, testVectors[0].registry, testVectors[1].layerID)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Set: %v", err)
|
||||
}
|
||||
layerID, err := v1IDService.Get(testVectors[0].v1ID, testVectors[0].registry)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Get: %v", err)
|
||||
}
|
||||
if layerID != testVectors[1].layerID {
|
||||
t.Fatal("Get returned incorrect layer ID")
|
||||
}
|
||||
}
|
||||
137
vendor/github.com/hyperhq/hypercli/distribution/metadata/v2_metadata_service.go
generated
vendored
Normal file
137
vendor/github.com/hyperhq/hypercli/distribution/metadata/v2_metadata_service.go
generated
vendored
Normal file
@@ -0,0 +1,137 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
)
|
||||
|
||||
// V2MetadataService maps layer IDs to a set of known metadata for
|
||||
// the layer.
|
||||
type V2MetadataService struct {
|
||||
store Store
|
||||
}
|
||||
|
||||
// V2Metadata contains the digest and source repository information for a layer.
|
||||
type V2Metadata struct {
|
||||
Digest digest.Digest
|
||||
SourceRepository string
|
||||
}
|
||||
|
||||
// maxMetadata is the number of metadata entries to keep per layer DiffID.
|
||||
const maxMetadata = 50
|
||||
|
||||
// NewV2MetadataService creates a new diff ID to v2 metadata mapping service.
|
||||
func NewV2MetadataService(store Store) *V2MetadataService {
|
||||
return &V2MetadataService{
|
||||
store: store,
|
||||
}
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) diffIDNamespace() string {
|
||||
return "v2metadata-by-diffid"
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) digestNamespace() string {
|
||||
return "diffid-by-digest"
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) diffIDKey(diffID layer.DiffID) string {
|
||||
return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex()
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) digestKey(dgst digest.Digest) string {
|
||||
return string(dgst.Algorithm()) + "/" + dgst.Hex()
|
||||
}
|
||||
|
||||
// GetMetadata finds the metadata associated with a layer DiffID.
|
||||
func (serv *V2MetadataService) GetMetadata(diffID layer.DiffID) ([]V2Metadata, error) {
|
||||
jsonBytes, err := serv.store.Get(serv.diffIDNamespace(), serv.diffIDKey(diffID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var metadata []V2Metadata
|
||||
if err := json.Unmarshal(jsonBytes, &metadata); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// GetDiffID finds a layer DiffID from a digest.
|
||||
func (serv *V2MetadataService) GetDiffID(dgst digest.Digest) (layer.DiffID, error) {
|
||||
diffIDBytes, err := serv.store.Get(serv.digestNamespace(), serv.digestKey(dgst))
|
||||
if err != nil {
|
||||
return layer.DiffID(""), err
|
||||
}
|
||||
|
||||
return layer.DiffID(diffIDBytes), nil
|
||||
}
|
||||
|
||||
// Add associates metadata with a layer DiffID. If too many metadata entries are
|
||||
// present, the oldest one is dropped.
|
||||
func (serv *V2MetadataService) Add(diffID layer.DiffID, metadata V2Metadata) error {
|
||||
oldMetadata, err := serv.GetMetadata(diffID)
|
||||
if err != nil {
|
||||
oldMetadata = nil
|
||||
}
|
||||
newMetadata := make([]V2Metadata, 0, len(oldMetadata)+1)
|
||||
|
||||
// Copy all other metadata to new slice
|
||||
for _, oldMeta := range oldMetadata {
|
||||
if oldMeta != metadata {
|
||||
newMetadata = append(newMetadata, oldMeta)
|
||||
}
|
||||
}
|
||||
|
||||
newMetadata = append(newMetadata, metadata)
|
||||
|
||||
if len(newMetadata) > maxMetadata {
|
||||
newMetadata = newMetadata[len(newMetadata)-maxMetadata:]
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return serv.store.Set(serv.digestNamespace(), serv.digestKey(metadata.Digest), []byte(diffID))
|
||||
}
|
||||
|
||||
// Remove unassociates a metadata entry from a layer DiffID.
|
||||
func (serv *V2MetadataService) Remove(metadata V2Metadata) error {
|
||||
diffID, err := serv.GetDiffID(metadata.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldMetadata, err := serv.GetMetadata(diffID)
|
||||
if err != nil {
|
||||
oldMetadata = nil
|
||||
}
|
||||
newMetadata := make([]V2Metadata, 0, len(oldMetadata))
|
||||
|
||||
// Copy all other metadata to new slice
|
||||
for _, oldMeta := range oldMetadata {
|
||||
if oldMeta != metadata {
|
||||
newMetadata = append(newMetadata, oldMeta)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newMetadata) == 0 {
|
||||
return serv.store.Delete(serv.diffIDNamespace(), serv.diffIDKey(diffID))
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes)
|
||||
}
|
||||
115
vendor/github.com/hyperhq/hypercli/distribution/metadata/v2_metadata_service_test.go
generated
vendored
Normal file
115
vendor/github.com/hyperhq/hypercli/distribution/metadata/v2_metadata_service_test.go
generated
vendored
Normal file
@@ -0,0 +1,115 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
)
|
||||
|
||||
func TestV2MetadataService(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test")
|
||||
if err != nil {
|
||||
t.Fatalf("could not create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
metadataStore, err := NewFSMetadataStore(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("could not create metadata store: %v", err)
|
||||
}
|
||||
V2MetadataService := NewV2MetadataService(metadataStore)
|
||||
|
||||
tooManyBlobSums := make([]V2Metadata, 100)
|
||||
for i := range tooManyBlobSums {
|
||||
randDigest := randomDigest()
|
||||
tooManyBlobSums[i] = V2Metadata{Digest: randDigest}
|
||||
}
|
||||
|
||||
testVectors := []struct {
|
||||
diffID layer.DiffID
|
||||
metadata []V2Metadata
|
||||
}{
|
||||
{
|
||||
diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
||||
metadata: []V2Metadata{
|
||||
{Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
||||
},
|
||||
},
|
||||
{
|
||||
diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
|
||||
metadata: []V2Metadata{
|
||||
{Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
||||
{Digest: digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e")},
|
||||
},
|
||||
},
|
||||
{
|
||||
diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
|
||||
metadata: tooManyBlobSums,
|
||||
},
|
||||
}
|
||||
|
||||
// Set some associations
|
||||
for _, vec := range testVectors {
|
||||
for _, blobsum := range vec.metadata {
|
||||
err := V2MetadataService.Add(vec.diffID, blobsum)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Set: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check the correct values are read back
|
||||
for _, vec := range testVectors {
|
||||
metadata, err := V2MetadataService.GetMetadata(vec.diffID)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Get: %v", err)
|
||||
}
|
||||
expectedMetadataEntries := len(vec.metadata)
|
||||
if expectedMetadataEntries > 50 {
|
||||
expectedMetadataEntries = 50
|
||||
}
|
||||
if !reflect.DeepEqual(metadata, vec.metadata[len(vec.metadata)-expectedMetadataEntries:len(vec.metadata)]) {
|
||||
t.Fatal("Get returned incorrect layer ID")
|
||||
}
|
||||
}
|
||||
|
||||
// Test GetMetadata on a nonexistent entry
|
||||
_, err = V2MetadataService.GetMetadata(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
||||
if err == nil {
|
||||
t.Fatal("expected error looking up nonexistent entry")
|
||||
}
|
||||
|
||||
// Test GetDiffID on a nonexistent entry
|
||||
_, err = V2MetadataService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
||||
if err == nil {
|
||||
t.Fatal("expected error looking up nonexistent entry")
|
||||
}
|
||||
|
||||
// Overwrite one of the entries and read it back
|
||||
err = V2MetadataService.Add(testVectors[1].diffID, testVectors[0].metadata[0])
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Add: %v", err)
|
||||
}
|
||||
diffID, err := V2MetadataService.GetDiffID(testVectors[0].metadata[0].Digest)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling GetDiffID: %v", err)
|
||||
}
|
||||
if diffID != testVectors[1].diffID {
|
||||
t.Fatal("GetDiffID returned incorrect diffID")
|
||||
}
|
||||
}
|
||||
|
||||
func randomDigest() digest.Digest {
|
||||
b := [32]byte{}
|
||||
for i := 0; i < len(b); i++ {
|
||||
b[i] = byte(rand.Intn(256))
|
||||
}
|
||||
d := hex.EncodeToString(b[:])
|
||||
return digest.Digest("sha256:" + d)
|
||||
}
|
||||
202
vendor/github.com/hyperhq/hypercli/distribution/pull.go
generated
vendored
Normal file
202
vendor/github.com/hyperhq/hypercli/distribution/pull.go
generated
vendored
Normal file
@@ -0,0 +1,202 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/hyperhq/hypercli/api"
|
||||
"github.com/hyperhq/hypercli/distribution/metadata"
|
||||
"github.com/hyperhq/hypercli/distribution/xfer"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"github.com/docker/engine-api/types"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// ImagePullConfig stores pull configuration.
|
||||
type ImagePullConfig struct {
|
||||
// MetaHeaders stores HTTP headers with metadata about the image
|
||||
MetaHeaders map[string][]string
|
||||
// AuthConfig holds authentication credentials for authenticating with
|
||||
// the registry.
|
||||
AuthConfig *types.AuthConfig
|
||||
// ProgressOutput is the interface for showing the status of the pull
|
||||
// operation.
|
||||
ProgressOutput progress.Output
|
||||
// RegistryService is the registry service to use for TLS configuration
|
||||
// and endpoint lookup.
|
||||
RegistryService *registry.Service
|
||||
// ImageEventLogger notifies events for a given image
|
||||
ImageEventLogger func(id, name, action string)
|
||||
// MetadataStore is the storage backend for distribution-specific
|
||||
// metadata.
|
||||
MetadataStore metadata.Store
|
||||
// ImageStore manages images.
|
||||
ImageStore image.Store
|
||||
// ReferenceStore manages tags.
|
||||
ReferenceStore reference.Store
|
||||
// DownloadManager manages concurrent pulls.
|
||||
DownloadManager *xfer.LayerDownloadManager
|
||||
}
|
||||
|
||||
// Puller is an interface that abstracts pulling for different API versions.
|
||||
type Puller interface {
|
||||
// Pull tries to pull the image referenced by `tag`
|
||||
// Pull returns an error if any, as well as a boolean that determines whether to retry Pull on the next configured endpoint.
|
||||
//
|
||||
Pull(ctx context.Context, ref reference.Named) error
|
||||
}
|
||||
|
||||
// newPuller returns a Puller interface that will pull from either a v1 or v2
|
||||
// registry. The endpoint argument contains a Version field that determines
|
||||
// whether a v1 or v2 puller will be created. The other parameters are passed
|
||||
// through to the underlying puller implementation for use during the actual
|
||||
// pull operation.
|
||||
func newPuller(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePullConfig *ImagePullConfig) (Puller, error) {
|
||||
switch endpoint.Version {
|
||||
case registry.APIVersion2:
|
||||
return &v2Puller{
|
||||
V2MetadataService: metadata.NewV2MetadataService(imagePullConfig.MetadataStore),
|
||||
endpoint: endpoint,
|
||||
config: imagePullConfig,
|
||||
repoInfo: repoInfo,
|
||||
}, nil
|
||||
case registry.APIVersion1:
|
||||
return &v1Puller{
|
||||
v1IDService: metadata.NewV1IDService(imagePullConfig.MetadataStore),
|
||||
endpoint: endpoint,
|
||||
config: imagePullConfig,
|
||||
repoInfo: repoInfo,
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
|
||||
}
|
||||
|
||||
// Pull initiates a pull operation. image is the repository name to pull, and
|
||||
// tag may be either empty, or indicate a specific tag to pull.
|
||||
func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullConfig) error {
|
||||
// Resolve the Repository name from fqn to RepositoryInfo
|
||||
repoInfo, err := imagePullConfig.RegistryService.ResolveRepository(ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// makes sure name is not empty or `scratch`
|
||||
if err := validateRepoName(repoInfo.Name()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
endpoints, err := imagePullConfig.RegistryService.LookupPullEndpoints(repoInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
lastErr error
|
||||
|
||||
// discardNoSupportErrors is used to track whether an endpoint encountered an error of type registry.ErrNoSupport
|
||||
// By default it is false, which means that if a ErrNoSupport error is encountered, it will be saved in lastErr.
|
||||
// As soon as another kind of error is encountered, discardNoSupportErrors is set to true, avoiding the saving of
|
||||
// any subsequent ErrNoSupport errors in lastErr.
|
||||
// It's needed for pull-by-digest on v1 endpoints: if there are only v1 endpoints configured, the error should be
|
||||
// returned and displayed, but if there was a v2 endpoint which supports pull-by-digest, then the last relevant
|
||||
// error is the ones from v2 endpoints not v1.
|
||||
discardNoSupportErrors bool
|
||||
|
||||
// confirmedV2 is set to true if a pull attempt managed to
|
||||
// confirm that it was talking to a v2 registry. This will
|
||||
// prevent fallback to the v1 protocol.
|
||||
confirmedV2 bool
|
||||
)
|
||||
for _, endpoint := range endpoints {
|
||||
if confirmedV2 && endpoint.Version == registry.APIVersion1 {
|
||||
logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL)
|
||||
continue
|
||||
}
|
||||
logrus.Debugf("Trying to pull %s from %s %s", repoInfo.Name(), endpoint.URL, endpoint.Version)
|
||||
|
||||
puller, err := newPuller(endpoint, repoInfo, imagePullConfig)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if err := puller.Pull(ctx, ref); err != nil {
|
||||
// Was this pull cancelled? If so, don't try to fall
|
||||
// back.
|
||||
fallback := false
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
default:
|
||||
if fallbackErr, ok := err.(fallbackError); ok {
|
||||
fallback = true
|
||||
confirmedV2 = confirmedV2 || fallbackErr.confirmedV2
|
||||
err = fallbackErr.err
|
||||
}
|
||||
}
|
||||
if fallback {
|
||||
if _, ok := err.(registry.ErrNoSupport); !ok {
|
||||
// Because we found an error that's not ErrNoSupport, discard all subsequent ErrNoSupport errors.
|
||||
discardNoSupportErrors = true
|
||||
// append subsequent errors
|
||||
lastErr = err
|
||||
} else if !discardNoSupportErrors {
|
||||
// Save the ErrNoSupport error, because it's either the first error or all encountered errors
|
||||
// were also ErrNoSupport errors.
|
||||
// append subsequent errors
|
||||
lastErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
logrus.Debugf("Not continuing with error: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
imagePullConfig.ImageEventLogger(ref.String(), repoInfo.Name(), "pull")
|
||||
return nil
|
||||
}
|
||||
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("no endpoints found for %s", ref.String())
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// writeStatus writes a status message to out. If layersDownloaded is true, the
|
||||
// status message indicates that a newer image was downloaded. Otherwise, it
|
||||
// indicates that the image is up to date. requestedTag is the tag the message
|
||||
// will refer to.
|
||||
func writeStatus(requestedTag string, out progress.Output, layersDownloaded bool) {
|
||||
if layersDownloaded {
|
||||
progress.Message(out, "", "Status: Downloaded newer image for "+requestedTag)
|
||||
} else {
|
||||
progress.Message(out, "", "Status: Image is up to date for "+requestedTag)
|
||||
}
|
||||
}
|
||||
|
||||
// validateRepoName validates the name of a repository.
|
||||
func validateRepoName(name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("Repository name can't be empty")
|
||||
}
|
||||
if name == api.NoBaseImageSpecifier {
|
||||
return fmt.Errorf("'%s' is a reserved name", api.NoBaseImageSpecifier)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tmpFileClose creates a closer function for a temporary file that closes the file
|
||||
// and also deletes it.
|
||||
func tmpFileCloser(tmpFile *os.File) func() error {
|
||||
return func() error {
|
||||
tmpFile.Close()
|
||||
if err := os.RemoveAll(tmpFile.Name()); err != nil {
|
||||
logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
336
vendor/github.com/hyperhq/hypercli/distribution/pull_v1.go
generated
vendored
Normal file
336
vendor/github.com/hyperhq/hypercli/distribution/pull_v1.go
generated
vendored
Normal file
@@ -0,0 +1,336 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution/registry/client/transport"
|
||||
"github.com/hyperhq/hypercli/distribution/metadata"
|
||||
"github.com/hyperhq/hypercli/distribution/xfer"
|
||||
"github.com/hyperhq/hypercli/dockerversion"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/image/v1"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/ioutils"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"github.com/hyperhq/hypercli/pkg/stringid"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
type v1Puller struct {
|
||||
v1IDService *metadata.V1IDService
|
||||
endpoint registry.APIEndpoint
|
||||
config *ImagePullConfig
|
||||
repoInfo *registry.RepositoryInfo
|
||||
session *registry.Session
|
||||
}
|
||||
|
||||
func (p *v1Puller) Pull(ctx context.Context, ref reference.Named) error {
|
||||
if _, isCanonical := ref.(reference.Canonical); isCanonical {
|
||||
// Allowing fallback, because HTTPS v1 is before HTTP v2
|
||||
return fallbackError{err: registry.ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}}
|
||||
}
|
||||
|
||||
tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
|
||||
tr := transport.NewTransport(
|
||||
// TODO(tiborvass): was ReceiveTimeout
|
||||
registry.NewTransport(tlsConfig),
|
||||
registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)...,
|
||||
)
|
||||
client := registry.HTTPClient(tr)
|
||||
v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders)
|
||||
if err != nil {
|
||||
logrus.Debugf("Could not get v1 endpoint: %v", err)
|
||||
return fallbackError{err: err}
|
||||
}
|
||||
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
|
||||
if err != nil {
|
||||
// TODO(dmcgowan): Check if should fallback
|
||||
logrus.Debugf("Fallback from error: %s", err)
|
||||
return fallbackError{err: err}
|
||||
}
|
||||
if err := p.pullRepository(ctx, ref); err != nil {
|
||||
// TODO(dmcgowan): Check if should fallback
|
||||
return err
|
||||
}
|
||||
progress.Message(p.config.ProgressOutput, "", p.repoInfo.FullName()+": this image was pulled from a legacy registry. Important: This registry version will not be supported in future versions of docker.")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *v1Puller) pullRepository(ctx context.Context, ref reference.Named) error {
|
||||
progress.Message(p.config.ProgressOutput, "", "Pulling repository "+p.repoInfo.FullName())
|
||||
|
||||
repoData, err := p.session.GetRepositoryData(p.repoInfo)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "HTTP code: 404") {
|
||||
return fmt.Errorf("Error: image %s not found", p.repoInfo.RemoteName())
|
||||
}
|
||||
// Unexpected HTTP error
|
||||
return err
|
||||
}
|
||||
|
||||
logrus.Debugf("Retrieving the tag list")
|
||||
var tagsList map[string]string
|
||||
tagged, isTagged := ref.(reference.NamedTagged)
|
||||
if !isTagged {
|
||||
tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo)
|
||||
} else {
|
||||
var tagID string
|
||||
tagsList = make(map[string]string)
|
||||
tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo, tagged.Tag())
|
||||
if err == registry.ErrRepoNotFound {
|
||||
return fmt.Errorf("Tag %s not found in repository %s", tagged.Tag(), p.repoInfo.FullName())
|
||||
}
|
||||
tagsList[tagged.Tag()] = tagID
|
||||
}
|
||||
if err != nil {
|
||||
logrus.Errorf("unable to get remote tags: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
for tag, id := range tagsList {
|
||||
repoData.ImgList[id] = ®istry.ImgData{
|
||||
ID: id,
|
||||
Tag: tag,
|
||||
Checksum: "",
|
||||
}
|
||||
}
|
||||
|
||||
layersDownloaded := false
|
||||
for _, imgData := range repoData.ImgList {
|
||||
if isTagged && imgData.Tag != tagged.Tag() {
|
||||
continue
|
||||
}
|
||||
|
||||
err := p.downloadImage(ctx, repoData, imgData, &layersDownloaded)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writeStatus(ref.String(), p.config.ProgressOutput, layersDownloaded)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *v1Puller) downloadImage(ctx context.Context, repoData *registry.RepositoryData, img *registry.ImgData, layersDownloaded *bool) error {
|
||||
if img.Tag == "" {
|
||||
logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
localNameRef, err := reference.WithTag(p.repoInfo, img.Tag)
|
||||
if err != nil {
|
||||
retErr := fmt.Errorf("Image (id: %s) has invalid tag: %s", img.ID, img.Tag)
|
||||
logrus.Debug(retErr.Error())
|
||||
return retErr
|
||||
}
|
||||
|
||||
if err := v1.ValidateID(img.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, p.repoInfo.FullName())
|
||||
success := false
|
||||
var lastErr error
|
||||
for _, ep := range p.repoInfo.Index.Mirrors {
|
||||
ep += "v1/"
|
||||
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.FullName(), ep))
|
||||
if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil {
|
||||
// Don't report errors when pulling from mirrors.
|
||||
logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.FullName(), ep, err)
|
||||
continue
|
||||
}
|
||||
success = true
|
||||
break
|
||||
}
|
||||
if !success {
|
||||
for _, ep := range repoData.Endpoints {
|
||||
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.FullName(), ep)
|
||||
if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil {
|
||||
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
|
||||
// As the error is also given to the output stream the user will see the error.
|
||||
lastErr = err
|
||||
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.FullName(), ep, err)
|
||||
continue
|
||||
}
|
||||
success = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !success {
|
||||
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.FullName(), lastErr)
|
||||
progress.Update(p.config.ProgressOutput, stringid.TruncateID(img.ID), err.Error())
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *v1Puller) pullImage(ctx context.Context, v1ID, endpoint string, localNameRef reference.Named, layersDownloaded *bool) (err error) {
|
||||
var history []string
|
||||
history, err = p.session.GetRemoteHistory(v1ID, endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(history) < 1 {
|
||||
return fmt.Errorf("empty history for image %s", v1ID)
|
||||
}
|
||||
progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1ID), "Pulling dependent layers")
|
||||
|
||||
var (
|
||||
descriptors []xfer.DownloadDescriptor
|
||||
newHistory []image.History
|
||||
imgJSON []byte
|
||||
imgSize int64
|
||||
)
|
||||
|
||||
// Iterate over layers, in order from bottom-most to top-most. Download
|
||||
// config for all layers and create descriptors.
|
||||
for i := len(history) - 1; i >= 0; i-- {
|
||||
v1LayerID := history[i]
|
||||
imgJSON, imgSize, err = p.downloadLayerConfig(v1LayerID, endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create a new-style config from the legacy configs
|
||||
h, err := v1.HistoryFromConfig(imgJSON, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newHistory = append(newHistory, h)
|
||||
|
||||
layerDescriptor := &v1LayerDescriptor{
|
||||
v1LayerID: v1LayerID,
|
||||
indexName: p.repoInfo.Index.Name,
|
||||
endpoint: endpoint,
|
||||
v1IDService: p.v1IDService,
|
||||
layersDownloaded: layersDownloaded,
|
||||
layerSize: imgSize,
|
||||
session: p.session,
|
||||
}
|
||||
|
||||
descriptors = append(descriptors, layerDescriptor)
|
||||
}
|
||||
|
||||
rootFS := image.NewRootFS()
|
||||
resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
|
||||
config, err := v1.MakeConfigFromV1Config(imgJSON, &resultRootFS, newHistory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imageID, err := p.config.ImageStore.Create(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := p.config.ReferenceStore.AddTag(localNameRef, imageID, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *v1Puller) downloadLayerConfig(v1LayerID, endpoint string) (imgJSON []byte, imgSize int64, err error) {
|
||||
progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1LayerID), "Pulling metadata")
|
||||
|
||||
retries := 5
|
||||
for j := 1; j <= retries; j++ {
|
||||
imgJSON, imgSize, err := p.session.GetRemoteImageJSON(v1LayerID, endpoint)
|
||||
if err != nil && j == retries {
|
||||
progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1LayerID), "Error pulling layer metadata")
|
||||
return nil, 0, err
|
||||
} else if err != nil {
|
||||
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
|
||||
return imgJSON, imgSize, nil
|
||||
}
|
||||
|
||||
// not reached
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
type v1LayerDescriptor struct {
|
||||
v1LayerID string
|
||||
indexName string
|
||||
endpoint string
|
||||
v1IDService *metadata.V1IDService
|
||||
layersDownloaded *bool
|
||||
layerSize int64
|
||||
session *registry.Session
|
||||
}
|
||||
|
||||
func (ld *v1LayerDescriptor) Key() string {
|
||||
return "v1:" + ld.v1LayerID
|
||||
}
|
||||
|
||||
func (ld *v1LayerDescriptor) ID() string {
|
||||
return stringid.TruncateID(ld.v1LayerID)
|
||||
}
|
||||
|
||||
func (ld *v1LayerDescriptor) DiffID() (layer.DiffID, error) {
|
||||
return ld.v1IDService.Get(ld.v1LayerID, ld.indexName)
|
||||
}
|
||||
|
||||
func (ld *v1LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
|
||||
progress.Update(progressOutput, ld.ID(), "Pulling fs layer")
|
||||
layerReader, err := ld.session.GetRemoteImageLayer(ld.v1LayerID, ld.endpoint, ld.layerSize)
|
||||
if err != nil {
|
||||
progress.Update(progressOutput, ld.ID(), "Error pulling dependent layers")
|
||||
if uerr, ok := err.(*url.Error); ok {
|
||||
err = uerr.Err
|
||||
}
|
||||
if terr, ok := err.(net.Error); ok && terr.Timeout() {
|
||||
return nil, 0, err
|
||||
}
|
||||
return nil, 0, xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
*ld.layersDownloaded = true
|
||||
|
||||
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
|
||||
if err != nil {
|
||||
layerReader.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerReader), progressOutput, ld.layerSize, ld.ID(), "Downloading")
|
||||
defer reader.Close()
|
||||
|
||||
_, err = io.Copy(tmpFile, reader)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
progress.Update(progressOutput, ld.ID(), "Download complete")
|
||||
|
||||
logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), tmpFile.Name())
|
||||
|
||||
tmpFile.Seek(0, 0)
|
||||
return ioutils.NewReadCloserWrapper(tmpFile, tmpFileCloser(tmpFile)), ld.layerSize, nil
|
||||
}
|
||||
|
||||
func (ld *v1LayerDescriptor) Registered(diffID layer.DiffID) {
|
||||
// Cache mapping from this layer's DiffID to the blobsum
|
||||
ld.v1IDService.Set(ld.v1LayerID, ld.indexName, diffID)
|
||||
}
|
||||
713
vendor/github.com/hyperhq/hypercli/distribution/pull_v2.go
generated
vendored
Normal file
713
vendor/github.com/hyperhq/hypercli/distribution/pull_v2.go
generated
vendored
Normal file
@@ -0,0 +1,713 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"runtime"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/manifest/manifestlist"
|
||||
"github.com/docker/distribution/manifest/schema1"
|
||||
"github.com/docker/distribution/manifest/schema2"
|
||||
"github.com/docker/distribution/registry/api/errcode"
|
||||
"github.com/docker/distribution/registry/client"
|
||||
"github.com/hyperhq/hypercli/distribution/metadata"
|
||||
"github.com/hyperhq/hypercli/distribution/xfer"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/image/v1"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/ioutils"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"github.com/hyperhq/hypercli/pkg/stringid"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var errRootFSMismatch = errors.New("layers from manifest don't match image configuration")
|
||||
|
||||
type v2Puller struct {
|
||||
V2MetadataService *metadata.V2MetadataService
|
||||
endpoint registry.APIEndpoint
|
||||
config *ImagePullConfig
|
||||
repoInfo *registry.RepositoryInfo
|
||||
repo distribution.Repository
|
||||
// confirmedV2 is set to true if we confirm we're talking to a v2
|
||||
// registry. This is used to limit fallbacks to the v1 protocol.
|
||||
confirmedV2 bool
|
||||
}
|
||||
|
||||
func (p *v2Puller) Pull(ctx context.Context, ref reference.Named) (err error) {
|
||||
// TODO(tiborvass): was ReceiveTimeout
|
||||
p.repo, p.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
|
||||
if err != nil {
|
||||
logrus.Warnf("Error getting v2 registry: %v", err)
|
||||
return fallbackError{err: err, confirmedV2: p.confirmedV2}
|
||||
}
|
||||
|
||||
if err = p.pullV2Repository(ctx, ref); err != nil {
|
||||
if _, ok := err.(fallbackError); ok {
|
||||
return err
|
||||
}
|
||||
if registry.ContinueOnError(err) {
|
||||
logrus.Debugf("Error trying v2 registry: %v", err)
|
||||
return fallbackError{err: err, confirmedV2: p.confirmedV2}
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (err error) {
|
||||
var layersDownloaded bool
|
||||
if !reference.IsNameOnly(ref) {
|
||||
layersDownloaded, err = p.pullV2Tag(ctx, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
tags, err := p.repo.Tags(ctx).All(ctx)
|
||||
if err != nil {
|
||||
// If this repository doesn't exist on V2, we should
|
||||
// permit a fallback to V1.
|
||||
return allowV1Fallback(err)
|
||||
}
|
||||
|
||||
// The v2 registry knows about this repository, so we will not
|
||||
// allow fallback to the v1 protocol even if we encounter an
|
||||
// error later on.
|
||||
p.confirmedV2 = true
|
||||
|
||||
for _, tag := range tags {
|
||||
tagRef, err := reference.WithTag(ref, tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pulledNew, err := p.pullV2Tag(ctx, tagRef)
|
||||
if err != nil {
|
||||
// Since this is the pull-all-tags case, don't
|
||||
// allow an error pulling a particular tag to
|
||||
// make the whole pull fall back to v1.
|
||||
if fallbackErr, ok := err.(fallbackError); ok {
|
||||
return fallbackErr.err
|
||||
}
|
||||
return err
|
||||
}
|
||||
// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
|
||||
// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
|
||||
layersDownloaded = layersDownloaded || pulledNew
|
||||
}
|
||||
}
|
||||
|
||||
writeStatus(ref.String(), p.config.ProgressOutput, layersDownloaded)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type v2LayerDescriptor struct {
|
||||
digest digest.Digest
|
||||
repoInfo *registry.RepositoryInfo
|
||||
repo distribution.Repository
|
||||
V2MetadataService *metadata.V2MetadataService
|
||||
}
|
||||
|
||||
func (ld *v2LayerDescriptor) Key() string {
|
||||
return "v2:" + ld.digest.String()
|
||||
}
|
||||
|
||||
func (ld *v2LayerDescriptor) ID() string {
|
||||
return stringid.TruncateID(ld.digest.String())
|
||||
}
|
||||
|
||||
func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) {
|
||||
return ld.V2MetadataService.GetDiffID(ld.digest)
|
||||
}
|
||||
|
||||
func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
|
||||
logrus.Debugf("pulling blob %q", ld.digest)
|
||||
|
||||
blobs := ld.repo.Blobs(ctx)
|
||||
|
||||
layerDownload, err := blobs.Open(ctx, ld.digest)
|
||||
if err != nil {
|
||||
logrus.Debugf("Error statting layer: %v", err)
|
||||
if err == distribution.ErrBlobUnknown {
|
||||
return nil, 0, xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
return nil, 0, retryOnError(err)
|
||||
}
|
||||
|
||||
size, err := layerDownload.Seek(0, os.SEEK_END)
|
||||
if err != nil {
|
||||
// Seek failed, perhaps because there was no Content-Length
|
||||
// header. This shouldn't fail the download, because we can
|
||||
// still continue without a progress bar.
|
||||
size = 0
|
||||
} else {
|
||||
// Restore the seek offset at the beginning of the stream.
|
||||
_, err = layerDownload.Seek(0, os.SEEK_SET)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size, ld.ID(), "Downloading")
|
||||
defer reader.Close()
|
||||
|
||||
verifier, err := digest.NewDigestVerifier(ld.digest)
|
||||
if err != nil {
|
||||
return nil, 0, xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
|
||||
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
|
||||
if err != nil {
|
||||
return nil, 0, xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
|
||||
_, err = io.Copy(tmpFile, io.TeeReader(reader, verifier))
|
||||
if err != nil {
|
||||
tmpFile.Close()
|
||||
if err := os.Remove(tmpFile.Name()); err != nil {
|
||||
logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
|
||||
}
|
||||
return nil, 0, retryOnError(err)
|
||||
}
|
||||
|
||||
progress.Update(progressOutput, ld.ID(), "Verifying Checksum")
|
||||
|
||||
if !verifier.Verified() {
|
||||
err = fmt.Errorf("filesystem layer verification failed for digest %s", ld.digest)
|
||||
logrus.Error(err)
|
||||
|
||||
tmpFile.Close()
|
||||
if err := os.Remove(tmpFile.Name()); err != nil {
|
||||
logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
|
||||
}
|
||||
|
||||
return nil, 0, xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
|
||||
progress.Update(progressOutput, ld.ID(), "Download complete")
|
||||
|
||||
logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), tmpFile.Name())
|
||||
|
||||
_, err = tmpFile.Seek(0, os.SEEK_SET)
|
||||
if err != nil {
|
||||
tmpFile.Close()
|
||||
if err := os.Remove(tmpFile.Name()); err != nil {
|
||||
logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
|
||||
}
|
||||
return nil, 0, xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
return ioutils.NewReadCloserWrapper(tmpFile, tmpFileCloser(tmpFile)), size, nil
|
||||
}
|
||||
|
||||
func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
|
||||
// Cache mapping from this layer's DiffID to the blobsum
|
||||
ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})
|
||||
}
|
||||
|
||||
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
|
||||
manSvc, err := p.repo.Manifests(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var (
|
||||
manifest distribution.Manifest
|
||||
tagOrDigest string // Used for logging/progress only
|
||||
)
|
||||
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
|
||||
// NOTE: not using TagService.Get, since it uses HEAD requests
|
||||
// against the manifests endpoint, which are not supported by
|
||||
// all registry versions.
|
||||
manifest, err = manSvc.Get(ctx, "", client.WithTag(tagged.Tag()))
|
||||
if err != nil {
|
||||
return false, allowV1Fallback(err)
|
||||
}
|
||||
tagOrDigest = tagged.Tag()
|
||||
} else if digested, isDigested := ref.(reference.Canonical); isDigested {
|
||||
manifest, err = manSvc.Get(ctx, digested.Digest())
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
tagOrDigest = digested.Digest().String()
|
||||
} else {
|
||||
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
|
||||
}
|
||||
|
||||
if manifest == nil {
|
||||
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
|
||||
}
|
||||
|
||||
// If manSvc.Get succeeded, we can be confident that the registry on
|
||||
// the other side speaks the v2 protocol.
|
||||
p.confirmedV2 = true
|
||||
|
||||
logrus.Debugf("Pulling ref from V2 registry: %s", ref.String())
|
||||
progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Name())
|
||||
|
||||
var (
|
||||
imageID image.ID
|
||||
manifestDigest digest.Digest
|
||||
)
|
||||
|
||||
switch v := manifest.(type) {
|
||||
case *schema1.SignedManifest:
|
||||
imageID, manifestDigest, err = p.pullSchema1(ctx, ref, v)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
case *schema2.DeserializedManifest:
|
||||
imageID, manifestDigest, err = p.pullSchema2(ctx, ref, v)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
case *manifestlist.DeserializedManifestList:
|
||||
imageID, manifestDigest, err = p.pullManifestList(ctx, ref, v)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
default:
|
||||
return false, errors.New("unsupported manifest format")
|
||||
}
|
||||
|
||||
progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
|
||||
|
||||
oldTagImageID, err := p.config.ReferenceStore.Get(ref)
|
||||
if err == nil {
|
||||
if oldTagImageID == imageID {
|
||||
return false, nil
|
||||
}
|
||||
} else if err != reference.ErrDoesNotExist {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if canonical, ok := ref.(reference.Canonical); ok {
|
||||
if err = p.config.ReferenceStore.AddDigest(canonical, imageID, true); err != nil {
|
||||
return false, err
|
||||
}
|
||||
} else if err = p.config.ReferenceStore.AddTag(ref, imageID, true); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
|
||||
var verifiedManifest *schema1.Manifest
|
||||
verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
rootFS := image.NewRootFS()
|
||||
|
||||
if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// remove duplicate layers and check parent chain validity
|
||||
err = fixManifestLayers(verifiedManifest)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
var descriptors []xfer.DownloadDescriptor
|
||||
|
||||
// Image history converted to the new format
|
||||
var history []image.History
|
||||
|
||||
// Note that the order of this loop is in the direction of bottom-most
|
||||
// to top-most, so that the downloads slice gets ordered correctly.
|
||||
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
|
||||
blobSum := verifiedManifest.FSLayers[i].BlobSum
|
||||
|
||||
var throwAway struct {
|
||||
ThrowAway bool `json:"throwaway,omitempty"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
history = append(history, h)
|
||||
|
||||
if throwAway.ThrowAway {
|
||||
continue
|
||||
}
|
||||
|
||||
layerDescriptor := &v2LayerDescriptor{
|
||||
digest: blobSum,
|
||||
repoInfo: p.repoInfo,
|
||||
repo: p.repo,
|
||||
V2MetadataService: p.V2MetadataService,
|
||||
}
|
||||
|
||||
descriptors = append(descriptors, layerDescriptor)
|
||||
}
|
||||
|
||||
resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer release()
|
||||
|
||||
config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
imageID, err = p.config.ImageStore.Create(config)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
|
||||
|
||||
return imageID, manifestDigest, nil
|
||||
}
|
||||
|
||||
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
|
||||
manifestDigest, err = schema2ManifestDigest(ref, mfst)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
target := mfst.Target()
|
||||
imageID = image.ID(target.Digest)
|
||||
if _, err := p.config.ImageStore.Get(imageID); err == nil {
|
||||
// If the image already exists locally, no need to pull
|
||||
// anything.
|
||||
return imageID, manifestDigest, nil
|
||||
}
|
||||
|
||||
configChan := make(chan []byte, 1)
|
||||
errChan := make(chan error, 1)
|
||||
var cancel func()
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
|
||||
// Pull the image config
|
||||
go func() {
|
||||
configJSON, err := p.pullSchema2ImageConfig(ctx, target.Digest)
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
configChan <- configJSON
|
||||
}()
|
||||
|
||||
var descriptors []xfer.DownloadDescriptor
|
||||
|
||||
// Note that the order of this loop is in the direction of bottom-most
|
||||
// to top-most, so that the downloads slice gets ordered correctly.
|
||||
for _, d := range mfst.References() {
|
||||
layerDescriptor := &v2LayerDescriptor{
|
||||
digest: d.Digest,
|
||||
repo: p.repo,
|
||||
repoInfo: p.repoInfo,
|
||||
V2MetadataService: p.V2MetadataService,
|
||||
}
|
||||
|
||||
descriptors = append(descriptors, layerDescriptor)
|
||||
}
|
||||
|
||||
var (
|
||||
configJSON []byte // raw serialized image config
|
||||
unmarshalledConfig image.Image // deserialized image config
|
||||
downloadRootFS image.RootFS // rootFS to use for registering layers.
|
||||
)
|
||||
if runtime.GOOS == "windows" {
|
||||
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if unmarshalledConfig.RootFS == nil {
|
||||
return "", "", errors.New("image config has no rootfs section")
|
||||
}
|
||||
downloadRootFS = *unmarshalledConfig.RootFS
|
||||
downloadRootFS.DiffIDs = []layer.DiffID{}
|
||||
} else {
|
||||
downloadRootFS = *image.NewRootFS()
|
||||
}
|
||||
|
||||
rootFS, release, err := p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
|
||||
if err != nil {
|
||||
if configJSON != nil {
|
||||
// Already received the config
|
||||
return "", "", err
|
||||
}
|
||||
select {
|
||||
case err = <-errChan:
|
||||
return "", "", err
|
||||
default:
|
||||
cancel()
|
||||
select {
|
||||
case <-configChan:
|
||||
case <-errChan:
|
||||
}
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
defer release()
|
||||
|
||||
if configJSON == nil {
|
||||
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
|
||||
// The DiffIDs returned in rootFS MUST match those in the config.
|
||||
// Otherwise the image config could be referencing layers that aren't
|
||||
// included in the manifest.
|
||||
if len(rootFS.DiffIDs) != len(unmarshalledConfig.RootFS.DiffIDs) {
|
||||
return "", "", errRootFSMismatch
|
||||
}
|
||||
|
||||
for i := range rootFS.DiffIDs {
|
||||
if rootFS.DiffIDs[i] != unmarshalledConfig.RootFS.DiffIDs[i] {
|
||||
return "", "", errRootFSMismatch
|
||||
}
|
||||
}
|
||||
|
||||
imageID, err = p.config.ImageStore.Create(configJSON)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
return imageID, manifestDigest, nil
|
||||
}
|
||||
|
||||
func receiveConfig(configChan <-chan []byte, errChan <-chan error) ([]byte, image.Image, error) {
|
||||
select {
|
||||
case configJSON := <-configChan:
|
||||
var unmarshalledConfig image.Image
|
||||
if err := json.Unmarshal(configJSON, &unmarshalledConfig); err != nil {
|
||||
return nil, image.Image{}, err
|
||||
}
|
||||
return configJSON, unmarshalledConfig, nil
|
||||
case err := <-errChan:
|
||||
return nil, image.Image{}, err
|
||||
// Don't need a case for ctx.Done in the select because cancellation
|
||||
// will trigger an error in p.pullSchema2ImageConfig.
|
||||
}
|
||||
}
|
||||
|
||||
// pullManifestList handles "manifest lists" which point to various
|
||||
// platform-specifc manifests.
|
||||
func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList) (imageID image.ID, manifestListDigest digest.Digest, err error) {
|
||||
manifestListDigest, err = schema2ManifestDigest(ref, mfstList)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
var manifestDigest digest.Digest
|
||||
for _, manifestDescriptor := range mfstList.Manifests {
|
||||
// TODO(aaronl): The manifest list spec supports optional
|
||||
// "features" and "variant" fields. These are not yet used.
|
||||
// Once they are, their values should be interpreted here.
|
||||
if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == runtime.GOOS {
|
||||
manifestDigest = manifestDescriptor.Digest
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if manifestDigest == "" {
|
||||
return "", "", errors.New("no supported platform found in manifest list")
|
||||
}
|
||||
|
||||
manSvc, err := p.repo.Manifests(ctx)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
manifest, err := manSvc.Get(ctx, manifestDigest)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
manifestRef, err := reference.WithDigest(ref, manifestDigest)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
switch v := manifest.(type) {
|
||||
case *schema1.SignedManifest:
|
||||
imageID, _, err = p.pullSchema1(ctx, manifestRef, v)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
case *schema2.DeserializedManifest:
|
||||
imageID, _, err = p.pullSchema2(ctx, manifestRef, v)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
default:
|
||||
return "", "", errors.New("unsupported manifest format")
|
||||
}
|
||||
|
||||
return imageID, manifestListDigest, err
|
||||
}
|
||||
|
||||
func (p *v2Puller) pullSchema2ImageConfig(ctx context.Context, dgst digest.Digest) (configJSON []byte, err error) {
|
||||
blobs := p.repo.Blobs(ctx)
|
||||
configJSON, err = blobs.Get(ctx, dgst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Verify image config digest
|
||||
verifier, err := digest.NewDigestVerifier(dgst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := verifier.Write(configJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !verifier.Verified() {
|
||||
err := fmt.Errorf("image config verification failed for digest %s", dgst)
|
||||
logrus.Error(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return configJSON, nil
|
||||
}
|
||||
|
||||
// schema2ManifestDigest computes the manifest digest, and, if pulling by
|
||||
// digest, ensures that it matches the requested digest.
|
||||
func schema2ManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) {
|
||||
_, canonical, err := mfst.Payload()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// If pull by digest, then verify the manifest digest.
|
||||
if digested, isDigested := ref.(reference.Canonical); isDigested {
|
||||
verifier, err := digest.NewDigestVerifier(digested.Digest())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := verifier.Write(canonical); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !verifier.Verified() {
|
||||
err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
|
||||
logrus.Error(err)
|
||||
return "", err
|
||||
}
|
||||
return digested.Digest(), nil
|
||||
}
|
||||
|
||||
return digest.FromBytes(canonical), nil
|
||||
}
|
||||
|
||||
// allowV1Fallback checks if the error is a possible reason to fallback to v1
|
||||
// (even if confirmedV2 has been set already), and if so, wraps the error in
|
||||
// a fallbackError with confirmedV2 set to false. Otherwise, it returns the
|
||||
// error unmodified.
|
||||
func allowV1Fallback(err error) error {
|
||||
switch v := err.(type) {
|
||||
case errcode.Errors:
|
||||
if len(v) != 0 {
|
||||
if v0, ok := v[0].(errcode.Error); ok && registry.ShouldV2Fallback(v0) {
|
||||
return fallbackError{err: err, confirmedV2: false}
|
||||
}
|
||||
}
|
||||
case errcode.Error:
|
||||
if registry.ShouldV2Fallback(v) {
|
||||
return fallbackError{err: err, confirmedV2: false}
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func verifySchema1Manifest(signedManifest *schema1.SignedManifest, ref reference.Named) (m *schema1.Manifest, err error) {
|
||||
// If pull by digest, then verify the manifest digest. NOTE: It is
|
||||
// important to do this first, before any other content validation. If the
|
||||
// digest cannot be verified, don't even bother with those other things.
|
||||
if digested, isCanonical := ref.(reference.Canonical); isCanonical {
|
||||
verifier, err := digest.NewDigestVerifier(digested.Digest())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := verifier.Write(signedManifest.Canonical); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !verifier.Verified() {
|
||||
err := fmt.Errorf("image verification failed for digest %s", digested.Digest())
|
||||
logrus.Error(err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
m = &signedManifest.Manifest
|
||||
|
||||
if m.SchemaVersion != 1 {
|
||||
return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, ref.String())
|
||||
}
|
||||
if len(m.FSLayers) != len(m.History) {
|
||||
return nil, fmt.Errorf("length of history not equal to number of layers for %q", ref.String())
|
||||
}
|
||||
if len(m.FSLayers) == 0 {
|
||||
return nil, fmt.Errorf("no FSLayers in manifest for %q", ref.String())
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// fixManifestLayers removes repeated layers from the manifest and checks the
|
||||
// correctness of the parent chain.
|
||||
func fixManifestLayers(m *schema1.Manifest) error {
|
||||
imgs := make([]*image.V1Image, len(m.FSLayers))
|
||||
for i := range m.FSLayers {
|
||||
img := &image.V1Image{}
|
||||
|
||||
if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imgs[i] = img
|
||||
if err := v1.ValidateID(img.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" {
|
||||
// Windows base layer can point to a base layer parent that is not in manifest.
|
||||
return errors.New("Invalid parent ID in the base layer of the image.")
|
||||
}
|
||||
|
||||
// check general duplicates to error instead of a deadlock
|
||||
idmap := make(map[string]struct{})
|
||||
|
||||
var lastID string
|
||||
for _, img := range imgs {
|
||||
// skip IDs that appear after each other, we handle those later
|
||||
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
|
||||
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
|
||||
}
|
||||
lastID = img.ID
|
||||
idmap[lastID] = struct{}{}
|
||||
}
|
||||
|
||||
// backwards loop so that we keep the remaining indexes after removing items
|
||||
for i := len(imgs) - 2; i >= 0; i-- {
|
||||
if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
|
||||
m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
|
||||
m.History = append(m.History[:i], m.History[i+1:]...)
|
||||
} else if imgs[i].Parent != imgs[i+1].ID {
|
||||
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
174
vendor/github.com/hyperhq/hypercli/distribution/pull_v2_test.go
generated
vendored
Normal file
174
vendor/github.com/hyperhq/hypercli/distribution/pull_v2_test.go
generated
vendored
Normal file
@@ -0,0 +1,174 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/manifest/schema1"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
)
|
||||
|
||||
// TestFixManifestLayers checks that fixManifestLayers removes a duplicate
|
||||
// layer, and that it makes no changes to the manifest when called a second
|
||||
// time, after the duplicate is removed.
|
||||
func TestFixManifestLayers(t *testing.T) {
|
||||
duplicateLayerManifest := schema1.Manifest{
|
||||
FSLayers: []schema1.FSLayer{
|
||||
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
||||
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
||||
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
|
||||
},
|
||||
History: []schema1.History{
|
||||
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
|
||||
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
|
||||
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
|
||||
},
|
||||
}
|
||||
|
||||
duplicateLayerManifestExpectedOutput := schema1.Manifest{
|
||||
FSLayers: []schema1.FSLayer{
|
||||
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
||||
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
|
||||
},
|
||||
History: []schema1.History{
|
||||
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
|
||||
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
|
||||
},
|
||||
}
|
||||
|
||||
if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
|
||||
t.Fatalf("unexpected error from fixManifestLayers: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
|
||||
t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest")
|
||||
}
|
||||
|
||||
// Run fixManifestLayers again and confirm that it doesn't change the
|
||||
// manifest (which no longer has duplicate layers).
|
||||
if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
|
||||
t.Fatalf("unexpected error from fixManifestLayers: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
|
||||
t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest (second pass)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFixManifestLayersBaseLayerParent makes sure that fixManifestLayers fails
|
||||
// if the base layer configuration specifies a parent.
|
||||
func TestFixManifestLayersBaseLayerParent(t *testing.T) {
|
||||
duplicateLayerManifest := schema1.Manifest{
|
||||
FSLayers: []schema1.FSLayer{
|
||||
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
||||
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
||||
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
|
||||
},
|
||||
History: []schema1.History{
|
||||
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
|
||||
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
|
||||
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"parent\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
|
||||
},
|
||||
}
|
||||
|
||||
if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID in the base layer of the image.") {
|
||||
t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFixManifestLayersBadParent makes sure that fixManifestLayers fails
|
||||
// if an image configuration specifies a parent that doesn't directly follow
|
||||
// that (deduplicated) image in the image history.
|
||||
func TestFixManifestLayersBadParent(t *testing.T) {
|
||||
duplicateLayerManifest := schema1.Manifest{
|
||||
FSLayers: []schema1.FSLayer{
|
||||
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
||||
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
||||
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
|
||||
},
|
||||
History: []schema1.History{
|
||||
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
|
||||
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
|
||||
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
|
||||
},
|
||||
}
|
||||
|
||||
if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID.") {
|
||||
t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateManifest verifies the validateManifest function
|
||||
func TestValidateManifest(t *testing.T) {
|
||||
expectedDigest, err := reference.ParseNamed("repo@sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd")
|
||||
if err != nil {
|
||||
t.Fatal("could not parse reference")
|
||||
}
|
||||
expectedFSLayer0 := digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
|
||||
|
||||
// Good manifest
|
||||
|
||||
goodManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/good_manifest")
|
||||
if err != nil {
|
||||
t.Fatal("error reading fixture:", err)
|
||||
}
|
||||
|
||||
var goodSignedManifest schema1.SignedManifest
|
||||
err = json.Unmarshal(goodManifestBytes, &goodSignedManifest)
|
||||
if err != nil {
|
||||
t.Fatal("error unmarshaling manifest:", err)
|
||||
}
|
||||
|
||||
verifiedManifest, err := verifySchema1Manifest(&goodSignedManifest, expectedDigest)
|
||||
if err != nil {
|
||||
t.Fatal("validateManifest failed:", err)
|
||||
}
|
||||
|
||||
if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
|
||||
t.Fatal("unexpected FSLayer in good manifest")
|
||||
}
|
||||
|
||||
// "Extra data" manifest
|
||||
|
||||
extraDataManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/extra_data_manifest")
|
||||
if err != nil {
|
||||
t.Fatal("error reading fixture:", err)
|
||||
}
|
||||
|
||||
var extraDataSignedManifest schema1.SignedManifest
|
||||
err = json.Unmarshal(extraDataManifestBytes, &extraDataSignedManifest)
|
||||
if err != nil {
|
||||
t.Fatal("error unmarshaling manifest:", err)
|
||||
}
|
||||
|
||||
verifiedManifest, err = verifySchema1Manifest(&extraDataSignedManifest, expectedDigest)
|
||||
if err != nil {
|
||||
t.Fatal("validateManifest failed:", err)
|
||||
}
|
||||
|
||||
if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
|
||||
t.Fatal("unexpected FSLayer in extra data manifest")
|
||||
}
|
||||
|
||||
// Bad manifest
|
||||
|
||||
badManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/bad_manifest")
|
||||
if err != nil {
|
||||
t.Fatal("error reading fixture:", err)
|
||||
}
|
||||
|
||||
var badSignedManifest schema1.SignedManifest
|
||||
err = json.Unmarshal(badManifestBytes, &badSignedManifest)
|
||||
if err != nil {
|
||||
t.Fatal("error unmarshaling manifest:", err)
|
||||
}
|
||||
|
||||
verifiedManifest, err = verifySchema1Manifest(&badSignedManifest, expectedDigest)
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "image verification failed for digest") {
|
||||
t.Fatal("expected validateManifest to fail with digest error")
|
||||
}
|
||||
}
|
||||
12
vendor/github.com/hyperhq/hypercli/distribution/pull_v2_unix.go
generated
vendored
Normal file
12
vendor/github.com/hyperhq/hypercli/distribution/pull_v2_unix.go
generated
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
// +build !windows
|
||||
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"github.com/docker/distribution/manifest/schema1"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
)
|
||||
|
||||
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
|
||||
return nil
|
||||
}
|
||||
29
vendor/github.com/hyperhq/hypercli/distribution/pull_v2_windows.go
generated
vendored
Normal file
29
vendor/github.com/hyperhq/hypercli/distribution/pull_v2_windows.go
generated
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
// +build windows
|
||||
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/docker/distribution/manifest/schema1"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
)
|
||||
|
||||
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
|
||||
v1img := &image.V1Image{}
|
||||
if err := json.Unmarshal([]byte(m.History[len(m.History)-1].V1Compatibility), v1img); err != nil {
|
||||
return err
|
||||
}
|
||||
if v1img.Parent == "" {
|
||||
return fmt.Errorf("Last layer %q does not have a base layer reference", v1img.ID)
|
||||
}
|
||||
// There must be an image that already references the baselayer.
|
||||
for _, img := range is.Map() {
|
||||
if img.RootFS.BaseLayerID() == v1img.Parent {
|
||||
rootFS.BaseLayer = img.RootFS.BaseLayer
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Invalid base layer %q", v1img.Parent)
|
||||
}
|
||||
203
vendor/github.com/hyperhq/hypercli/distribution/push.go
generated
vendored
Normal file
203
vendor/github.com/hyperhq/hypercli/distribution/push.go
generated
vendored
Normal file
@@ -0,0 +1,203 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/hyperhq/hypercli/distribution/metadata"
|
||||
"github.com/hyperhq/hypercli/distribution/xfer"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"github.com/docker/engine-api/types"
|
||||
"github.com/docker/libtrust"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// ImagePushConfig stores push configuration.
|
||||
type ImagePushConfig struct {
|
||||
// MetaHeaders store HTTP headers with metadata about the image
|
||||
MetaHeaders map[string][]string
|
||||
// AuthConfig holds authentication credentials for authenticating with
|
||||
// the registry.
|
||||
AuthConfig *types.AuthConfig
|
||||
// ProgressOutput is the interface for showing the status of the push
|
||||
// operation.
|
||||
ProgressOutput progress.Output
|
||||
// RegistryService is the registry service to use for TLS configuration
|
||||
// and endpoint lookup.
|
||||
RegistryService *registry.Service
|
||||
// ImageEventLogger notifies events for a given image
|
||||
ImageEventLogger func(id, name, action string)
|
||||
// MetadataStore is the storage backend for distribution-specific
|
||||
// metadata.
|
||||
MetadataStore metadata.Store
|
||||
// LayerStore manages layers.
|
||||
LayerStore layer.Store
|
||||
// ImageStore manages images.
|
||||
ImageStore image.Store
|
||||
// ReferenceStore manages tags.
|
||||
ReferenceStore reference.Store
|
||||
// TrustKey is the private key for legacy signatures. This is typically
|
||||
// an ephemeral key, since these signatures are no longer verified.
|
||||
TrustKey libtrust.PrivateKey
|
||||
// UploadManager dispatches uploads.
|
||||
UploadManager *xfer.LayerUploadManager
|
||||
}
|
||||
|
||||
// Pusher is an interface that abstracts pushing for different API versions.
|
||||
type Pusher interface {
|
||||
// Push tries to push the image configured at the creation of Pusher.
|
||||
// Push returns an error if any, as well as a boolean that determines whether to retry Push on the next configured endpoint.
|
||||
//
|
||||
// TODO(tiborvass): have Push() take a reference to repository + tag, so that the pusher itself is repository-agnostic.
|
||||
Push(ctx context.Context) error
|
||||
}
|
||||
|
||||
const compressionBufSize = 32768
|
||||
|
||||
// NewPusher creates a new Pusher interface that will push to either a v1 or v2
|
||||
// registry. The endpoint argument contains a Version field that determines
|
||||
// whether a v1 or v2 pusher will be created. The other parameters are passed
|
||||
// through to the underlying pusher implementation for use during the actual
|
||||
// push operation.
|
||||
func NewPusher(ref reference.Named, endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePushConfig *ImagePushConfig) (Pusher, error) {
|
||||
switch endpoint.Version {
|
||||
case registry.APIVersion2:
|
||||
return &v2Pusher{
|
||||
v2MetadataService: metadata.NewV2MetadataService(imagePushConfig.MetadataStore),
|
||||
ref: ref,
|
||||
endpoint: endpoint,
|
||||
repoInfo: repoInfo,
|
||||
config: imagePushConfig,
|
||||
}, nil
|
||||
case registry.APIVersion1:
|
||||
return &v1Pusher{
|
||||
v1IDService: metadata.NewV1IDService(imagePushConfig.MetadataStore),
|
||||
ref: ref,
|
||||
endpoint: endpoint,
|
||||
repoInfo: repoInfo,
|
||||
config: imagePushConfig,
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
|
||||
}
|
||||
|
||||
// Push initiates a push operation on the repository named localName.
|
||||
// ref is the specific variant of the image to be pushed.
|
||||
// If no tag is provided, all tags will be pushed.
|
||||
func Push(ctx context.Context, ref reference.Named, imagePushConfig *ImagePushConfig) error {
|
||||
// FIXME: Allow to interrupt current push when new push of same image is done.
|
||||
|
||||
// Resolve the Repository name from fqn to RepositoryInfo
|
||||
repoInfo, err := imagePushConfig.RegistryService.ResolveRepository(ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
endpoints, err := imagePushConfig.RegistryService.LookupPushEndpoints(repoInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
progress.Messagef(imagePushConfig.ProgressOutput, "", "The push refers to a repository [%s]", repoInfo.FullName())
|
||||
|
||||
associations := imagePushConfig.ReferenceStore.ReferencesByName(repoInfo)
|
||||
if len(associations) == 0 {
|
||||
return fmt.Errorf("Repository does not exist: %s", repoInfo.Name())
|
||||
}
|
||||
|
||||
var (
|
||||
lastErr error
|
||||
|
||||
// confirmedV2 is set to true if a push attempt managed to
|
||||
// confirm that it was talking to a v2 registry. This will
|
||||
// prevent fallback to the v1 protocol.
|
||||
confirmedV2 bool
|
||||
)
|
||||
|
||||
for _, endpoint := range endpoints {
|
||||
if confirmedV2 && endpoint.Version == registry.APIVersion1 {
|
||||
logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL)
|
||||
continue
|
||||
}
|
||||
|
||||
logrus.Debugf("Trying to push %s to %s %s", repoInfo.FullName(), endpoint.URL, endpoint.Version)
|
||||
|
||||
pusher, err := NewPusher(ref, endpoint, repoInfo, imagePushConfig)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if err := pusher.Push(ctx); err != nil {
|
||||
// Was this push cancelled? If so, don't try to fall
|
||||
// back.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
default:
|
||||
if fallbackErr, ok := err.(fallbackError); ok {
|
||||
confirmedV2 = confirmedV2 || fallbackErr.confirmedV2
|
||||
err = fallbackErr.err
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
logrus.Debugf("Not continuing with error: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
imagePushConfig.ImageEventLogger(ref.String(), repoInfo.Name(), "push")
|
||||
return nil
|
||||
}
|
||||
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("no endpoints found for %s", repoInfo.FullName())
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// compress returns an io.ReadCloser which will supply a compressed version of
|
||||
// the provided Reader. The caller must close the ReadCloser after reading the
|
||||
// compressed data.
|
||||
//
|
||||
// Note that this function returns a reader instead of taking a writer as an
|
||||
// argument so that it can be used with httpBlobWriter's ReadFrom method.
|
||||
// Using httpBlobWriter's Write method would send a PATCH request for every
|
||||
// Write call.
|
||||
//
|
||||
// The second return value is a channel that gets closed when the goroutine
|
||||
// is finished. This allows the caller to make sure the goroutine finishes
|
||||
// before it releases any resources connected with the reader that was
|
||||
// passed in.
|
||||
func compress(in io.Reader) (io.ReadCloser, chan struct{}) {
|
||||
compressionDone := make(chan struct{})
|
||||
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
// Use a bufio.Writer to avoid excessive chunking in HTTP request.
|
||||
bufWriter := bufio.NewWriterSize(pipeWriter, compressionBufSize)
|
||||
compressor := gzip.NewWriter(bufWriter)
|
||||
|
||||
go func() {
|
||||
_, err := io.Copy(compressor, in)
|
||||
if err == nil {
|
||||
err = compressor.Close()
|
||||
}
|
||||
if err == nil {
|
||||
err = bufWriter.Flush()
|
||||
}
|
||||
if err != nil {
|
||||
pipeWriter.CloseWithError(err)
|
||||
} else {
|
||||
pipeWriter.Close()
|
||||
}
|
||||
close(compressionDone)
|
||||
}()
|
||||
|
||||
return pipeReader, compressionDone
|
||||
}
|
||||
455
vendor/github.com/hyperhq/hypercli/distribution/push_v1.go
generated
vendored
Normal file
455
vendor/github.com/hyperhq/hypercli/distribution/push_v1.go
generated
vendored
Normal file
@@ -0,0 +1,455 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/registry/client/transport"
|
||||
"github.com/hyperhq/hypercli/distribution/metadata"
|
||||
"github.com/hyperhq/hypercli/dockerversion"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/image/v1"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/ioutils"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"github.com/hyperhq/hypercli/pkg/stringid"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
type v1Pusher struct {
|
||||
ctx context.Context
|
||||
v1IDService *metadata.V1IDService
|
||||
endpoint registry.APIEndpoint
|
||||
ref reference.Named
|
||||
repoInfo *registry.RepositoryInfo
|
||||
config *ImagePushConfig
|
||||
session *registry.Session
|
||||
}
|
||||
|
||||
func (p *v1Pusher) Push(ctx context.Context) error {
|
||||
tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
|
||||
tr := transport.NewTransport(
|
||||
// TODO(tiborvass): was NoTimeout
|
||||
registry.NewTransport(tlsConfig),
|
||||
registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)...,
|
||||
)
|
||||
client := registry.HTTPClient(tr)
|
||||
v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders)
|
||||
if err != nil {
|
||||
logrus.Debugf("Could not get v1 endpoint: %v", err)
|
||||
return fallbackError{err: err}
|
||||
}
|
||||
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
|
||||
if err != nil {
|
||||
// TODO(dmcgowan): Check if should fallback
|
||||
return fallbackError{err: err}
|
||||
}
|
||||
if err := p.pushRepository(ctx); err != nil {
|
||||
// TODO(dmcgowan): Check if should fallback
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// v1Image exposes the configuration, filesystem layer ID, and a v1 ID for an
|
||||
// image being pushed to a v1 registry.
|
||||
type v1Image interface {
|
||||
Config() []byte
|
||||
Layer() layer.Layer
|
||||
V1ID() string
|
||||
}
|
||||
|
||||
type v1ImageCommon struct {
|
||||
layer layer.Layer
|
||||
config []byte
|
||||
v1ID string
|
||||
}
|
||||
|
||||
func (common *v1ImageCommon) Config() []byte {
|
||||
return common.config
|
||||
}
|
||||
|
||||
func (common *v1ImageCommon) V1ID() string {
|
||||
return common.v1ID
|
||||
}
|
||||
|
||||
func (common *v1ImageCommon) Layer() layer.Layer {
|
||||
return common.layer
|
||||
}
|
||||
|
||||
// v1TopImage defines a runnable (top layer) image being pushed to a v1
|
||||
// registry.
|
||||
type v1TopImage struct {
|
||||
v1ImageCommon
|
||||
imageID image.ID
|
||||
}
|
||||
|
||||
func newV1TopImage(imageID image.ID, img *image.Image, l layer.Layer, parent *v1DependencyImage) (*v1TopImage, error) {
|
||||
v1ID := digest.Digest(imageID).Hex()
|
||||
parentV1ID := ""
|
||||
if parent != nil {
|
||||
parentV1ID = parent.V1ID()
|
||||
}
|
||||
|
||||
config, err := v1.MakeV1ConfigFromConfig(img, v1ID, parentV1ID, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &v1TopImage{
|
||||
v1ImageCommon: v1ImageCommon{
|
||||
v1ID: v1ID,
|
||||
config: config,
|
||||
layer: l,
|
||||
},
|
||||
imageID: imageID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// v1DependencyImage defines a dependency layer being pushed to a v1 registry.
|
||||
type v1DependencyImage struct {
|
||||
v1ImageCommon
|
||||
}
|
||||
|
||||
func newV1DependencyImage(l layer.Layer, parent *v1DependencyImage) (*v1DependencyImage, error) {
|
||||
v1ID := digest.Digest(l.ChainID()).Hex()
|
||||
|
||||
config := ""
|
||||
if parent != nil {
|
||||
config = fmt.Sprintf(`{"id":"%s","parent":"%s"}`, v1ID, parent.V1ID())
|
||||
} else {
|
||||
config = fmt.Sprintf(`{"id":"%s"}`, v1ID)
|
||||
}
|
||||
return &v1DependencyImage{
|
||||
v1ImageCommon: v1ImageCommon{
|
||||
v1ID: v1ID,
|
||||
config: []byte(config),
|
||||
layer: l,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Retrieve the all the images to be uploaded in the correct order
|
||||
func (p *v1Pusher) getImageList() (imageList []v1Image, tagsByImage map[image.ID][]string, referencedLayers []layer.Layer, err error) {
|
||||
tagsByImage = make(map[image.ID][]string)
|
||||
|
||||
// Ignore digest references
|
||||
if _, isCanonical := p.ref.(reference.Canonical); isCanonical {
|
||||
return
|
||||
}
|
||||
|
||||
tagged, isTagged := p.ref.(reference.NamedTagged)
|
||||
if isTagged {
|
||||
// Push a specific tag
|
||||
var imgID image.ID
|
||||
imgID, err = p.config.ReferenceStore.Get(p.ref)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
imageList, err = p.imageListForTag(imgID, nil, &referencedLayers)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
tagsByImage[imgID] = []string{tagged.Tag()}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
imagesSeen := make(map[image.ID]struct{})
|
||||
dependenciesSeen := make(map[layer.ChainID]*v1DependencyImage)
|
||||
|
||||
associations := p.config.ReferenceStore.ReferencesByName(p.ref)
|
||||
for _, association := range associations {
|
||||
if tagged, isTagged = association.Ref.(reference.NamedTagged); !isTagged {
|
||||
// Ignore digest references.
|
||||
continue
|
||||
}
|
||||
|
||||
tagsByImage[association.ImageID] = append(tagsByImage[association.ImageID], tagged.Tag())
|
||||
|
||||
if _, present := imagesSeen[association.ImageID]; present {
|
||||
// Skip generating image list for already-seen image
|
||||
continue
|
||||
}
|
||||
imagesSeen[association.ImageID] = struct{}{}
|
||||
|
||||
imageListForThisTag, err := p.imageListForTag(association.ImageID, dependenciesSeen, &referencedLayers)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// append to main image list
|
||||
imageList = append(imageList, imageListForThisTag...)
|
||||
}
|
||||
if len(imageList) == 0 {
|
||||
return nil, nil, nil, fmt.Errorf("No images found for the requested repository / tag")
|
||||
}
|
||||
logrus.Debugf("Image list: %v", imageList)
|
||||
logrus.Debugf("Tags by image: %v", tagsByImage)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (p *v1Pusher) imageListForTag(imgID image.ID, dependenciesSeen map[layer.ChainID]*v1DependencyImage, referencedLayers *[]layer.Layer) (imageListForThisTag []v1Image, err error) {
|
||||
img, err := p.config.ImageStore.Get(imgID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
topLayerID := img.RootFS.ChainID()
|
||||
|
||||
var l layer.Layer
|
||||
if topLayerID == "" {
|
||||
l = layer.EmptyLayer
|
||||
} else {
|
||||
l, err = p.config.LayerStore.Get(topLayerID)
|
||||
*referencedLayers = append(*referencedLayers, l)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get top layer from image: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
dependencyImages, parent, err := generateDependencyImages(l.Parent(), dependenciesSeen)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
topImage, err := newV1TopImage(imgID, img, l, parent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
imageListForThisTag = append(dependencyImages, topImage)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func generateDependencyImages(l layer.Layer, dependenciesSeen map[layer.ChainID]*v1DependencyImage) (imageListForThisTag []v1Image, parent *v1DependencyImage, err error) {
|
||||
if l == nil {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
imageListForThisTag, parent, err = generateDependencyImages(l.Parent(), dependenciesSeen)
|
||||
|
||||
if dependenciesSeen != nil {
|
||||
if dependencyImage, present := dependenciesSeen[l.ChainID()]; present {
|
||||
// This layer is already on the list, we can ignore it
|
||||
// and all its parents.
|
||||
return imageListForThisTag, dependencyImage, nil
|
||||
}
|
||||
}
|
||||
|
||||
dependencyImage, err := newV1DependencyImage(l, parent)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
imageListForThisTag = append(imageListForThisTag, dependencyImage)
|
||||
|
||||
if dependenciesSeen != nil {
|
||||
dependenciesSeen[l.ChainID()] = dependencyImage
|
||||
}
|
||||
|
||||
return imageListForThisTag, dependencyImage, nil
|
||||
}
|
||||
|
||||
// createImageIndex returns an index of an image's layer IDs and tags.
|
||||
func createImageIndex(images []v1Image, tags map[image.ID][]string) []*registry.ImgData {
|
||||
var imageIndex []*registry.ImgData
|
||||
for _, img := range images {
|
||||
v1ID := img.V1ID()
|
||||
|
||||
if topImage, isTopImage := img.(*v1TopImage); isTopImage {
|
||||
if tags, hasTags := tags[topImage.imageID]; hasTags {
|
||||
// If an image has tags you must add an entry in the image index
|
||||
// for each tag
|
||||
for _, tag := range tags {
|
||||
imageIndex = append(imageIndex, ®istry.ImgData{
|
||||
ID: v1ID,
|
||||
Tag: tag,
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// If the image does not have a tag it still needs to be sent to the
|
||||
// registry with an empty tag so that it is associated with the repository
|
||||
imageIndex = append(imageIndex, ®istry.ImgData{
|
||||
ID: v1ID,
|
||||
Tag: "",
|
||||
})
|
||||
}
|
||||
return imageIndex
|
||||
}
|
||||
|
||||
// lookupImageOnEndpoint checks the specified endpoint to see if an image exists
|
||||
// and if it is absent then it sends the image id to the channel to be pushed.
|
||||
func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, endpoint string, images chan v1Image, imagesToPush chan string) {
|
||||
defer wg.Done()
|
||||
for image := range images {
|
||||
v1ID := image.V1ID()
|
||||
truncID := stringid.TruncateID(image.Layer().DiffID().String())
|
||||
if err := p.session.LookupRemoteImage(v1ID, endpoint); err != nil {
|
||||
logrus.Errorf("Error in LookupRemoteImage: %s", err)
|
||||
imagesToPush <- v1ID
|
||||
progress.Update(p.config.ProgressOutput, truncID, "Waiting")
|
||||
} else {
|
||||
progress.Update(p.config.ProgressOutput, truncID, "Already exists")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *v1Pusher) pushImageToEndpoint(ctx context.Context, endpoint string, imageList []v1Image, tags map[image.ID][]string, repo *registry.RepositoryData) error {
|
||||
workerCount := len(imageList)
|
||||
// start a maximum of 5 workers to check if images exist on the specified endpoint.
|
||||
if workerCount > 5 {
|
||||
workerCount = 5
|
||||
}
|
||||
var (
|
||||
wg = &sync.WaitGroup{}
|
||||
imageData = make(chan v1Image, workerCount*2)
|
||||
imagesToPush = make(chan string, workerCount*2)
|
||||
pushes = make(chan map[string]struct{}, 1)
|
||||
)
|
||||
for i := 0; i < workerCount; i++ {
|
||||
wg.Add(1)
|
||||
go p.lookupImageOnEndpoint(wg, endpoint, imageData, imagesToPush)
|
||||
}
|
||||
// start a go routine that consumes the images to push
|
||||
go func() {
|
||||
shouldPush := make(map[string]struct{})
|
||||
for id := range imagesToPush {
|
||||
shouldPush[id] = struct{}{}
|
||||
}
|
||||
pushes <- shouldPush
|
||||
}()
|
||||
for _, v1Image := range imageList {
|
||||
imageData <- v1Image
|
||||
}
|
||||
// close the channel to notify the workers that there will be no more images to check.
|
||||
close(imageData)
|
||||
wg.Wait()
|
||||
close(imagesToPush)
|
||||
// wait for all the images that require pushes to be collected into a consumable map.
|
||||
shouldPush := <-pushes
|
||||
// finish by pushing any images and tags to the endpoint. The order that the images are pushed
|
||||
// is very important that is why we are still iterating over the ordered list of imageIDs.
|
||||
for _, img := range imageList {
|
||||
v1ID := img.V1ID()
|
||||
if _, push := shouldPush[v1ID]; push {
|
||||
if _, err := p.pushImage(ctx, img, endpoint); err != nil {
|
||||
// FIXME: Continue on error?
|
||||
return err
|
||||
}
|
||||
}
|
||||
if topImage, isTopImage := img.(*v1TopImage); isTopImage {
|
||||
for _, tag := range tags[topImage.imageID] {
|
||||
progress.Messagef(p.config.ProgressOutput, "", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(v1ID), endpoint+"repositories/"+p.repoInfo.RemoteName()+"/tags/"+tag)
|
||||
if err := p.session.PushRegistryTag(p.repoInfo, v1ID, tag, endpoint); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pushRepository pushes layers that do not already exist on the registry.
|
||||
func (p *v1Pusher) pushRepository(ctx context.Context) error {
|
||||
imgList, tags, referencedLayers, err := p.getImageList()
|
||||
defer func() {
|
||||
for _, l := range referencedLayers {
|
||||
p.config.LayerStore.Release(l)
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imageIndex := createImageIndex(imgList, tags)
|
||||
for _, data := range imageIndex {
|
||||
logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
|
||||
}
|
||||
|
||||
// Register all the images in a repository with the registry
|
||||
// If an image is not in this list it will not be associated with the repository
|
||||
repoData, err := p.session.PushImageJSONIndex(p.repoInfo, imageIndex, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// push the repository to each of the endpoints only if it does not exist.
|
||||
for _, endpoint := range repoData.Endpoints {
|
||||
if err := p.pushImageToEndpoint(ctx, endpoint, imgList, tags, repoData); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = p.session.PushImageJSONIndex(p.repoInfo, imageIndex, true, repoData.Endpoints)
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *v1Pusher) pushImage(ctx context.Context, v1Image v1Image, ep string) (checksum string, err error) {
|
||||
l := v1Image.Layer()
|
||||
v1ID := v1Image.V1ID()
|
||||
truncID := stringid.TruncateID(l.DiffID().String())
|
||||
|
||||
jsonRaw := v1Image.Config()
|
||||
progress.Update(p.config.ProgressOutput, truncID, "Pushing")
|
||||
|
||||
// General rule is to use ID for graph accesses and compatibilityID for
|
||||
// calls to session.registry()
|
||||
imgData := ®istry.ImgData{
|
||||
ID: v1ID,
|
||||
}
|
||||
|
||||
// Send the json
|
||||
if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
|
||||
if err == registry.ErrAlreadyExists {
|
||||
progress.Update(p.config.ProgressOutput, truncID, "Image already pushed, skipping")
|
||||
return "", nil
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
arch, err := l.TarStream()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer arch.Close()
|
||||
|
||||
// don't care if this fails; best effort
|
||||
size, _ := l.DiffSize()
|
||||
|
||||
// Send the layer
|
||||
logrus.Debugf("rendered layer for %s of [%d] size", v1ID, size)
|
||||
|
||||
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), p.config.ProgressOutput, size, truncID, "Pushing")
|
||||
defer reader.Close()
|
||||
|
||||
checksum, checksumPayload, err := p.session.PushImageLayerRegistry(v1ID, reader, ep, jsonRaw)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
imgData.Checksum = checksum
|
||||
imgData.ChecksumPayload = checksumPayload
|
||||
// Send the checksum
|
||||
if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err := p.v1IDService.Set(v1ID, p.repoInfo.Index.Name, l.DiffID()); err != nil {
|
||||
logrus.Warnf("Could not set v1 ID mapping: %v", err)
|
||||
}
|
||||
|
||||
progress.Update(p.config.ProgressOutput, truncID, "Image successfully pushed")
|
||||
return imgData.Checksum, nil
|
||||
}
|
||||
422
vendor/github.com/hyperhq/hypercli/distribution/push_v2.go
generated
vendored
Normal file
422
vendor/github.com/hyperhq/hypercli/distribution/push_v2.go
generated
vendored
Normal file
@@ -0,0 +1,422 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/manifest/schema1"
|
||||
"github.com/docker/distribution/manifest/schema2"
|
||||
distreference "github.com/docker/distribution/reference"
|
||||
"github.com/docker/distribution/registry/client"
|
||||
"github.com/hyperhq/hypercli/distribution/metadata"
|
||||
"github.com/hyperhq/hypercli/distribution/xfer"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/ioutils"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"github.com/hyperhq/hypercli/pkg/stringid"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// PushResult contains the tag, manifest digest, and manifest size from the
|
||||
// push. It's used to signal this information to the trust code in the client
|
||||
// so it can sign the manifest if necessary.
|
||||
type PushResult struct {
|
||||
Tag string
|
||||
Digest digest.Digest
|
||||
Size int
|
||||
}
|
||||
|
||||
type v2Pusher struct {
|
||||
v2MetadataService *metadata.V2MetadataService
|
||||
ref reference.Named
|
||||
endpoint registry.APIEndpoint
|
||||
repoInfo *registry.RepositoryInfo
|
||||
config *ImagePushConfig
|
||||
repo distribution.Repository
|
||||
|
||||
// pushState is state built by the Download functions.
|
||||
pushState pushState
|
||||
}
|
||||
|
||||
type pushState struct {
|
||||
sync.Mutex
|
||||
// remoteLayers is the set of layers known to exist on the remote side.
|
||||
// This avoids redundant queries when pushing multiple tags that
|
||||
// involve the same layers. It is also used to fill in digest and size
|
||||
// information when building the manifest.
|
||||
remoteLayers map[layer.DiffID]distribution.Descriptor
|
||||
// confirmedV2 is set to true if we confirm we're talking to a v2
|
||||
// registry. This is used to limit fallbacks to the v1 protocol.
|
||||
confirmedV2 bool
|
||||
}
|
||||
|
||||
func (p *v2Pusher) Push(ctx context.Context) (err error) {
|
||||
p.pushState.remoteLayers = make(map[layer.DiffID]distribution.Descriptor)
|
||||
|
||||
p.repo, p.pushState.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
|
||||
if err != nil {
|
||||
logrus.Debugf("Error getting v2 registry: %v", err)
|
||||
return fallbackError{err: err, confirmedV2: p.pushState.confirmedV2}
|
||||
}
|
||||
|
||||
if err = p.pushV2Repository(ctx); err != nil {
|
||||
if registry.ContinueOnError(err) {
|
||||
return fallbackError{err: err, confirmedV2: p.pushState.confirmedV2}
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *v2Pusher) pushV2Repository(ctx context.Context) (err error) {
|
||||
if namedTagged, isNamedTagged := p.ref.(reference.NamedTagged); isNamedTagged {
|
||||
imageID, err := p.config.ReferenceStore.Get(p.ref)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tag does not exist: %s", p.ref.String())
|
||||
}
|
||||
|
||||
return p.pushV2Tag(ctx, namedTagged, imageID)
|
||||
}
|
||||
|
||||
if !reference.IsNameOnly(p.ref) {
|
||||
return errors.New("cannot push a digest reference")
|
||||
}
|
||||
|
||||
// Pull all tags
|
||||
pushed := 0
|
||||
for _, association := range p.config.ReferenceStore.ReferencesByName(p.ref) {
|
||||
if namedTagged, isNamedTagged := association.Ref.(reference.NamedTagged); isNamedTagged {
|
||||
pushed++
|
||||
if err := p.pushV2Tag(ctx, namedTagged, association.ImageID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if pushed == 0 {
|
||||
return fmt.Errorf("no tags to push for %s", p.repoInfo.Name())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, imageID image.ID) error {
|
||||
logrus.Debugf("Pushing repository: %s", ref.String())
|
||||
|
||||
img, err := p.config.ImageStore.Get(imageID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
|
||||
}
|
||||
|
||||
var l layer.Layer
|
||||
|
||||
topLayerID := img.RootFS.ChainID()
|
||||
if topLayerID == "" {
|
||||
l = layer.EmptyLayer
|
||||
} else {
|
||||
l, err = p.config.LayerStore.Get(topLayerID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get top layer from image: %v", err)
|
||||
}
|
||||
defer layer.ReleaseAndLog(p.config.LayerStore, l)
|
||||
}
|
||||
|
||||
var descriptors []xfer.UploadDescriptor
|
||||
|
||||
descriptorTemplate := v2PushDescriptor{
|
||||
v2MetadataService: p.v2MetadataService,
|
||||
repoInfo: p.repoInfo,
|
||||
repo: p.repo,
|
||||
pushState: &p.pushState,
|
||||
}
|
||||
|
||||
// Loop bounds condition is to avoid pushing the base layer on Windows.
|
||||
for i := 0; i < len(img.RootFS.DiffIDs); i++ {
|
||||
descriptor := descriptorTemplate
|
||||
descriptor.layer = l
|
||||
descriptors = append(descriptors, &descriptor)
|
||||
|
||||
l = l.Parent()
|
||||
}
|
||||
|
||||
if err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Try schema2 first
|
||||
builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), img.RawJSON())
|
||||
manifest, err := manifestFromBuilder(ctx, builder, descriptors)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manSvc, err := p.repo.Manifests(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
putOptions := []distribution.ManifestServiceOption{client.WithTag(ref.Tag())}
|
||||
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
|
||||
logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err)
|
||||
|
||||
builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, p.repo.Name(), ref.Tag(), img.RawJSON())
|
||||
manifest, err = manifestFromBuilder(ctx, builder, descriptors)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var canonicalManifest []byte
|
||||
|
||||
switch v := manifest.(type) {
|
||||
case *schema1.SignedManifest:
|
||||
canonicalManifest = v.Canonical
|
||||
case *schema2.DeserializedManifest:
|
||||
_, canonicalManifest, err = v.Payload()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
manifestDigest := digest.FromBytes(canonicalManifest)
|
||||
progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", ref.Tag(), manifestDigest, len(canonicalManifest))
|
||||
// Signal digest to the trust client so it can sign the
|
||||
// push, if appropriate.
|
||||
progress.Aux(p.config.ProgressOutput, PushResult{Tag: ref.Tag(), Digest: manifestDigest, Size: len(canonicalManifest)})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuilder, descriptors []xfer.UploadDescriptor) (distribution.Manifest, error) {
|
||||
// descriptors is in reverse order; iterate backwards to get references
|
||||
// appended in the right order.
|
||||
for i := len(descriptors) - 1; i >= 0; i-- {
|
||||
if err := builder.AppendReference(descriptors[i].(*v2PushDescriptor)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return builder.Build(ctx)
|
||||
}
|
||||
|
||||
type v2PushDescriptor struct {
|
||||
layer layer.Layer
|
||||
v2MetadataService *metadata.V2MetadataService
|
||||
repoInfo reference.Named
|
||||
repo distribution.Repository
|
||||
pushState *pushState
|
||||
}
|
||||
|
||||
func (pd *v2PushDescriptor) Key() string {
|
||||
return "v2push:" + pd.repo.Name() + " " + pd.layer.DiffID().String()
|
||||
}
|
||||
|
||||
func (pd *v2PushDescriptor) ID() string {
|
||||
return stringid.TruncateID(pd.layer.DiffID().String())
|
||||
}
|
||||
|
||||
func (pd *v2PushDescriptor) DiffID() layer.DiffID {
|
||||
return pd.layer.DiffID()
|
||||
}
|
||||
|
||||
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) error {
|
||||
diffID := pd.DiffID()
|
||||
|
||||
pd.pushState.Lock()
|
||||
if _, ok := pd.pushState.remoteLayers[diffID]; ok {
|
||||
// it is already known that the push is not needed and
|
||||
// therefore doing a stat is unnecessary
|
||||
pd.pushState.Unlock()
|
||||
progress.Update(progressOutput, pd.ID(), "Layer already exists")
|
||||
return nil
|
||||
}
|
||||
pd.pushState.Unlock()
|
||||
|
||||
// Do we have any metadata associated with this layer's DiffID?
|
||||
v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID)
|
||||
if err == nil {
|
||||
descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState)
|
||||
if err != nil {
|
||||
progress.Update(progressOutput, pd.ID(), "Image push failed")
|
||||
return retryOnError(err)
|
||||
}
|
||||
if exists {
|
||||
progress.Update(progressOutput, pd.ID(), "Layer already exists")
|
||||
pd.pushState.Lock()
|
||||
pd.pushState.remoteLayers[diffID] = descriptor
|
||||
pd.pushState.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
logrus.Debugf("Pushing layer: %s", diffID)
|
||||
|
||||
// if digest was empty or not saved, or if blob does not exist on the remote repository,
|
||||
// then push the blob.
|
||||
bs := pd.repo.Blobs(ctx)
|
||||
|
||||
var mountFrom metadata.V2Metadata
|
||||
|
||||
// Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload
|
||||
for _, metadata := range v2Metadata {
|
||||
sourceRepo, err := reference.ParseNamed(metadata.SourceRepository)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pd.repoInfo.Hostname() == sourceRepo.Hostname() {
|
||||
logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, metadata.Digest, sourceRepo.FullName())
|
||||
mountFrom = metadata
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var createOpts []distribution.BlobCreateOption
|
||||
|
||||
if mountFrom.SourceRepository != "" {
|
||||
namedRef, err := reference.WithName(mountFrom.SourceRepository)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO (brianbland): We need to construct a reference where the Name is
|
||||
// only the full remote name, so clean this up when distribution has a
|
||||
// richer reference package
|
||||
remoteRef, err := distreference.WithName(namedRef.RemoteName())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
createOpts = append(createOpts, client.WithMountFrom(canonicalRef))
|
||||
}
|
||||
|
||||
// Send the layer
|
||||
layerUpload, err := bs.Create(ctx, createOpts...)
|
||||
switch err := err.(type) {
|
||||
case distribution.ErrBlobMounted:
|
||||
progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name())
|
||||
|
||||
err.Descriptor.MediaType = schema2.MediaTypeLayer
|
||||
|
||||
pd.pushState.Lock()
|
||||
pd.pushState.confirmedV2 = true
|
||||
pd.pushState.remoteLayers[diffID] = err.Descriptor
|
||||
pd.pushState.Unlock()
|
||||
|
||||
// Cache mapping from this layer's DiffID to the blobsum
|
||||
if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
||||
return xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
if mountFrom.SourceRepository != "" {
|
||||
// unable to mount layer from this repository, so this source mapping is no longer valid
|
||||
logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository)
|
||||
pd.v2MetadataService.Remove(mountFrom)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return retryOnError(err)
|
||||
}
|
||||
defer layerUpload.Close()
|
||||
|
||||
arch, err := pd.layer.TarStream()
|
||||
if err != nil {
|
||||
return xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
|
||||
// don't care if this fails; best effort
|
||||
size, _ := pd.layer.DiffSize()
|
||||
|
||||
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), progressOutput, size, pd.ID(), "Pushing")
|
||||
compressedReader, compressionDone := compress(reader)
|
||||
defer func() {
|
||||
reader.Close()
|
||||
<-compressionDone
|
||||
}()
|
||||
|
||||
digester := digest.Canonical.New()
|
||||
tee := io.TeeReader(compressedReader, digester.Hash())
|
||||
|
||||
nn, err := layerUpload.ReadFrom(tee)
|
||||
compressedReader.Close()
|
||||
if err != nil {
|
||||
return retryOnError(err)
|
||||
}
|
||||
|
||||
pushDigest := digester.Digest()
|
||||
if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil {
|
||||
return retryOnError(err)
|
||||
}
|
||||
|
||||
logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn)
|
||||
progress.Update(progressOutput, pd.ID(), "Pushed")
|
||||
|
||||
// Cache mapping from this layer's DiffID to the blobsum
|
||||
if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
||||
return xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
|
||||
pd.pushState.Lock()
|
||||
|
||||
// If Commit succeded, that's an indication that the remote registry
|
||||
// speaks the v2 protocol.
|
||||
pd.pushState.confirmedV2 = true
|
||||
|
||||
pd.pushState.remoteLayers[diffID] = distribution.Descriptor{
|
||||
Digest: pushDigest,
|
||||
MediaType: schema2.MediaTypeLayer,
|
||||
Size: nn,
|
||||
}
|
||||
|
||||
pd.pushState.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor {
|
||||
// Not necessary to lock pushStatus because this is always
|
||||
// called after all the mutation in pushStatus.
|
||||
// By the time this function is called, every layer will have
|
||||
// an entry in remoteLayers.
|
||||
return pd.pushState.remoteLayers[pd.DiffID()]
|
||||
}
|
||||
|
||||
// layerAlreadyExists checks if the registry already know about any of the
|
||||
// metadata passed in the "metadata" slice. If it finds one that the registry
|
||||
// knows about, it returns the known digest and "true".
|
||||
func layerAlreadyExists(ctx context.Context, metadata []metadata.V2Metadata, repoInfo reference.Named, repo distribution.Repository, pushState *pushState) (distribution.Descriptor, bool, error) {
|
||||
for _, meta := range metadata {
|
||||
// Only check blobsums that are known to this repository or have an unknown source
|
||||
if meta.SourceRepository != "" && meta.SourceRepository != repoInfo.FullName() {
|
||||
continue
|
||||
}
|
||||
descriptor, err := repo.Blobs(ctx).Stat(ctx, meta.Digest)
|
||||
switch err {
|
||||
case nil:
|
||||
descriptor.MediaType = schema2.MediaTypeLayer
|
||||
return descriptor, true, nil
|
||||
case distribution.ErrBlobUnknown:
|
||||
// nop
|
||||
default:
|
||||
return distribution.Descriptor{}, false, err
|
||||
}
|
||||
}
|
||||
return distribution.Descriptor{}, false, nil
|
||||
}
|
||||
164
vendor/github.com/hyperhq/hypercli/distribution/registry.go
generated
vendored
Normal file
164
vendor/github.com/hyperhq/hypercli/distribution/registry.go
generated
vendored
Normal file
@@ -0,0 +1,164 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
"github.com/docker/distribution/registry/api/errcode"
|
||||
"github.com/docker/distribution/registry/client"
|
||||
"github.com/docker/distribution/registry/client/auth"
|
||||
"github.com/docker/distribution/registry/client/transport"
|
||||
"github.com/hyperhq/hypercli/distribution/xfer"
|
||||
"github.com/hyperhq/hypercli/dockerversion"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"github.com/docker/engine-api/types"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// fallbackError wraps an error that can possibly allow fallback to a different
|
||||
// endpoint.
|
||||
type fallbackError struct {
|
||||
// err is the error being wrapped.
|
||||
err error
|
||||
// confirmedV2 is set to true if it was confirmed that the registry
|
||||
// supports the v2 protocol. This is used to limit fallbacks to the v1
|
||||
// protocol.
|
||||
confirmedV2 bool
|
||||
}
|
||||
|
||||
// Error renders the FallbackError as a string.
|
||||
func (f fallbackError) Error() string {
|
||||
return f.err.Error()
|
||||
}
|
||||
|
||||
type dumbCredentialStore struct {
|
||||
auth *types.AuthConfig
|
||||
}
|
||||
|
||||
func (dcs dumbCredentialStore) Basic(*url.URL) (string, string) {
|
||||
return dcs.auth.Username, dcs.auth.Password
|
||||
}
|
||||
|
||||
// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
|
||||
// providing timeout settings and authentication support, and also verifies the
|
||||
// remote API version.
|
||||
func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
|
||||
repoName := repoInfo.FullName()
|
||||
// If endpoint does not support CanonicalName, use the RemoteName instead
|
||||
if endpoint.TrimHostname {
|
||||
repoName = repoInfo.RemoteName()
|
||||
}
|
||||
|
||||
// TODO(dmcgowan): Call close idle connections when complete, use keep alive
|
||||
base := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
Dial: (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
DualStack: true,
|
||||
}).Dial,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
TLSClientConfig: endpoint.TLSConfig,
|
||||
// TODO(dmcgowan): Call close idle connections when complete and use keep alive
|
||||
DisableKeepAlives: true,
|
||||
}
|
||||
|
||||
modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), metaHeaders)
|
||||
authTransport := transport.NewTransport(base, modifiers...)
|
||||
pingClient := &http.Client{
|
||||
Transport: authTransport,
|
||||
Timeout: 15 * time.Second,
|
||||
}
|
||||
endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
|
||||
req, err := http.NewRequest("GET", endpointStr, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
resp, err := pingClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
v2Version := auth.APIVersion{
|
||||
Type: "registry",
|
||||
Version: "2.0",
|
||||
}
|
||||
|
||||
versions := auth.APIVersions(resp, registry.DefaultRegistryVersionHeader)
|
||||
for _, pingVersion := range versions {
|
||||
if pingVersion == v2Version {
|
||||
// The version header indicates we're definitely
|
||||
// talking to a v2 registry. So don't allow future
|
||||
// fallbacks to the v1 protocol.
|
||||
|
||||
foundVersion = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
challengeManager := auth.NewSimpleChallengeManager()
|
||||
if err := challengeManager.AddResponse(resp); err != nil {
|
||||
return nil, foundVersion, err
|
||||
}
|
||||
|
||||
if authConfig.RegistryToken != "" {
|
||||
passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
|
||||
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
|
||||
} else {
|
||||
creds := dumbCredentialStore{auth: authConfig}
|
||||
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
|
||||
basicHandler := auth.NewBasicHandler(creds)
|
||||
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
|
||||
}
|
||||
tr := transport.NewTransport(base, modifiers...)
|
||||
|
||||
repo, err = client.NewRepository(ctx, repoName, endpoint.URL, tr)
|
||||
return repo, foundVersion, err
|
||||
}
|
||||
|
||||
type existingTokenHandler struct {
|
||||
token string
|
||||
}
|
||||
|
||||
func (th *existingTokenHandler) Scheme() string {
|
||||
return "bearer"
|
||||
}
|
||||
|
||||
func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token))
|
||||
return nil
|
||||
}
|
||||
|
||||
// retryOnError wraps the error in xfer.DoNotRetry if we should not retry the
|
||||
// operation after this error.
|
||||
func retryOnError(err error) error {
|
||||
switch v := err.(type) {
|
||||
case errcode.Errors:
|
||||
return retryOnError(v[0])
|
||||
case errcode.Error:
|
||||
switch v.Code {
|
||||
case errcode.ErrorCodeUnauthorized, errcode.ErrorCodeUnsupported, errcode.ErrorCodeDenied:
|
||||
return xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
case *url.Error:
|
||||
return retryOnError(v.Err)
|
||||
case *client.UnexpectedHTTPResponseError:
|
||||
return xfer.DoNotRetry{Err: err}
|
||||
case error:
|
||||
if strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) {
|
||||
return xfer.DoNotRetry{Err: err}
|
||||
}
|
||||
}
|
||||
// let's be nice and fallback if the error is a completely
|
||||
// unexpected one.
|
||||
// If new errors have to be handled in some way, please
|
||||
// add them to the switch above.
|
||||
return err
|
||||
}
|
||||
127
vendor/github.com/hyperhq/hypercli/distribution/registry_unit_test.go
generated
vendored
Normal file
127
vendor/github.com/hyperhq/hypercli/distribution/registry_unit_test.go
generated
vendored
Normal file
@@ -0,0 +1,127 @@
|
||||
package distribution
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/hyperhq/hypercli/reference"
|
||||
"github.com/hyperhq/hypercli/registry"
|
||||
"github.com/hyperhq/hypercli/utils"
|
||||
"github.com/docker/engine-api/types"
|
||||
registrytypes "github.com/docker/engine-api/types/registry"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const secretRegistryToken = "mysecrettoken"
|
||||
|
||||
type tokenPassThruHandler struct {
|
||||
reached bool
|
||||
gotToken bool
|
||||
shouldSend401 func(url string) bool
|
||||
}
|
||||
|
||||
func (h *tokenPassThruHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
h.reached = true
|
||||
if strings.Contains(r.Header.Get("Authorization"), secretRegistryToken) {
|
||||
logrus.Debug("Detected registry token in auth header")
|
||||
h.gotToken = true
|
||||
}
|
||||
if h.shouldSend401 == nil || h.shouldSend401(r.RequestURI) {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="foorealm"`)
|
||||
w.WriteHeader(401)
|
||||
}
|
||||
}
|
||||
|
||||
func testTokenPassThru(t *testing.T, ts *httptest.Server) {
|
||||
tmp, err := utils.TestDirectory("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
|
||||
endpoint := registry.APIEndpoint{
|
||||
Mirror: false,
|
||||
URL: ts.URL,
|
||||
Version: 2,
|
||||
Official: false,
|
||||
TrimHostname: false,
|
||||
TLSConfig: nil,
|
||||
//VersionHeader: "verheader",
|
||||
}
|
||||
n, _ := reference.ParseNamed("testremotename")
|
||||
repoInfo := ®istry.RepositoryInfo{
|
||||
Named: n,
|
||||
Index: ®istrytypes.IndexInfo{
|
||||
Name: "testrepo",
|
||||
Mirrors: nil,
|
||||
Secure: false,
|
||||
Official: false,
|
||||
},
|
||||
Official: false,
|
||||
}
|
||||
imagePullConfig := &ImagePullConfig{
|
||||
MetaHeaders: http.Header{},
|
||||
AuthConfig: &types.AuthConfig{
|
||||
RegistryToken: secretRegistryToken,
|
||||
},
|
||||
}
|
||||
puller, err := newPuller(endpoint, repoInfo, imagePullConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := puller.(*v2Puller)
|
||||
ctx := context.Background()
|
||||
p.repo, _, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
logrus.Debug("About to pull")
|
||||
// We expect it to fail, since we haven't mock'd the full registry exchange in our handler above
|
||||
tag, _ := reference.WithTag(n, "tag_goes_here")
|
||||
_ = p.pullV2Repository(ctx, tag)
|
||||
}
|
||||
|
||||
func TestTokenPassThru(t *testing.T) {
|
||||
handler := &tokenPassThruHandler{shouldSend401: func(url string) bool { return url == "/v2/" }}
|
||||
ts := httptest.NewServer(handler)
|
||||
defer ts.Close()
|
||||
|
||||
testTokenPassThru(t, ts)
|
||||
|
||||
if !handler.reached {
|
||||
t.Fatal("Handler not reached")
|
||||
}
|
||||
if !handler.gotToken {
|
||||
t.Fatal("Failed to receive registry token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenPassThruDifferentHost(t *testing.T) {
|
||||
handler := new(tokenPassThruHandler)
|
||||
ts := httptest.NewServer(handler)
|
||||
defer ts.Close()
|
||||
|
||||
tsredirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.RequestURI == "/v2/" {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="foorealm"`)
|
||||
w.WriteHeader(401)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, ts.URL+r.URL.Path, http.StatusMovedPermanently)
|
||||
}))
|
||||
defer tsredirect.Close()
|
||||
|
||||
testTokenPassThru(t, tsredirect)
|
||||
|
||||
if !handler.reached {
|
||||
t.Fatal("Handler not reached")
|
||||
}
|
||||
if handler.gotToken {
|
||||
t.Fatal("Redirect should not forward Authorization header to another host")
|
||||
}
|
||||
}
|
||||
420
vendor/github.com/hyperhq/hypercli/distribution/xfer/download.go
generated
vendored
Normal file
420
vendor/github.com/hyperhq/hypercli/distribution/xfer/download.go
generated
vendored
Normal file
@@ -0,0 +1,420 @@
|
||||
package xfer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/archive"
|
||||
"github.com/hyperhq/hypercli/pkg/ioutils"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const maxDownloadAttempts = 5
|
||||
|
||||
// LayerDownloadManager figures out which layers need to be downloaded, then
|
||||
// registers and downloads those, taking into account dependencies between
|
||||
// layers.
|
||||
type LayerDownloadManager struct {
|
||||
layerStore layer.Store
|
||||
tm TransferManager
|
||||
}
|
||||
|
||||
// NewLayerDownloadManager returns a new LayerDownloadManager.
|
||||
func NewLayerDownloadManager(layerStore layer.Store, concurrencyLimit int) *LayerDownloadManager {
|
||||
return &LayerDownloadManager{
|
||||
layerStore: layerStore,
|
||||
tm: NewTransferManager(concurrencyLimit),
|
||||
}
|
||||
}
|
||||
|
||||
type downloadTransfer struct {
|
||||
Transfer
|
||||
|
||||
layerStore layer.Store
|
||||
layer layer.Layer
|
||||
err error
|
||||
}
|
||||
|
||||
// result returns the layer resulting from the download, if the download
|
||||
// and registration were successful.
|
||||
func (d *downloadTransfer) result() (layer.Layer, error) {
|
||||
return d.layer, d.err
|
||||
}
|
||||
|
||||
// A DownloadDescriptor references a layer that may need to be downloaded.
|
||||
type DownloadDescriptor interface {
|
||||
// Key returns the key used to deduplicate downloads.
|
||||
Key() string
|
||||
// ID returns the ID for display purposes.
|
||||
ID() string
|
||||
// DiffID should return the DiffID for this layer, or an error
|
||||
// if it is unknown (for example, if it has not been downloaded
|
||||
// before).
|
||||
DiffID() (layer.DiffID, error)
|
||||
// Download is called to perform the download.
|
||||
Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error)
|
||||
}
|
||||
|
||||
// DownloadDescriptorWithRegistered is a DownloadDescriptor that has an
|
||||
// additional Registered method which gets called after a downloaded layer is
|
||||
// registered. This allows the user of the download manager to know the DiffID
|
||||
// of each registered layer. This method is called if a cast to
|
||||
// DownloadDescriptorWithRegistered is successful.
|
||||
type DownloadDescriptorWithRegistered interface {
|
||||
DownloadDescriptor
|
||||
Registered(diffID layer.DiffID)
|
||||
}
|
||||
|
||||
// Download is a blocking function which ensures the requested layers are
|
||||
// present in the layer store. It uses the string returned by the Key method to
|
||||
// deduplicate downloads. If a given layer is not already known to present in
|
||||
// the layer store, and the key is not used by an in-progress download, the
|
||||
// Download method is called to get the layer tar data. Layers are then
|
||||
// registered in the appropriate order. The caller must call the returned
|
||||
// release function once it is is done with the returned RootFS object.
|
||||
func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS image.RootFS, layers []DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error) {
|
||||
var (
|
||||
topLayer layer.Layer
|
||||
topDownload *downloadTransfer
|
||||
watcher *Watcher
|
||||
missingLayer bool
|
||||
transferKey = ""
|
||||
downloadsByKey = make(map[string]*downloadTransfer)
|
||||
)
|
||||
|
||||
rootFS := initialRootFS
|
||||
for _, descriptor := range layers {
|
||||
key := descriptor.Key()
|
||||
transferKey += key
|
||||
|
||||
if !missingLayer {
|
||||
missingLayer = true
|
||||
diffID, err := descriptor.DiffID()
|
||||
if err == nil {
|
||||
getRootFS := rootFS
|
||||
getRootFS.Append(diffID)
|
||||
l, err := ldm.layerStore.Get(getRootFS.ChainID())
|
||||
if err == nil {
|
||||
// Layer already exists.
|
||||
logrus.Debugf("Layer already exists: %s", descriptor.ID())
|
||||
progress.Update(progressOutput, descriptor.ID(), "Already exists")
|
||||
if topLayer != nil {
|
||||
layer.ReleaseAndLog(ldm.layerStore, topLayer)
|
||||
}
|
||||
topLayer = l
|
||||
missingLayer = false
|
||||
rootFS.Append(diffID)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Does this layer have the same data as a previous layer in
|
||||
// the stack? If so, avoid downloading it more than once.
|
||||
var topDownloadUncasted Transfer
|
||||
if existingDownload, ok := downloadsByKey[key]; ok {
|
||||
xferFunc := ldm.makeDownloadFuncFromDownload(descriptor, existingDownload, topDownload)
|
||||
defer topDownload.Transfer.Release(watcher)
|
||||
topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
|
||||
topDownload = topDownloadUncasted.(*downloadTransfer)
|
||||
continue
|
||||
}
|
||||
|
||||
// Layer is not known to exist - download and register it.
|
||||
progress.Update(progressOutput, descriptor.ID(), "Pulling fs layer")
|
||||
|
||||
var xferFunc DoFunc
|
||||
if topDownload != nil {
|
||||
xferFunc = ldm.makeDownloadFunc(descriptor, "", topDownload)
|
||||
defer topDownload.Transfer.Release(watcher)
|
||||
} else {
|
||||
xferFunc = ldm.makeDownloadFunc(descriptor, rootFS.ChainID(), nil)
|
||||
}
|
||||
topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
|
||||
topDownload = topDownloadUncasted.(*downloadTransfer)
|
||||
downloadsByKey[key] = topDownload
|
||||
}
|
||||
|
||||
if topDownload == nil {
|
||||
return rootFS, func() { layer.ReleaseAndLog(ldm.layerStore, topLayer) }, nil
|
||||
}
|
||||
|
||||
// Won't be using the list built up so far - will generate it
|
||||
// from downloaded layers instead.
|
||||
rootFS.DiffIDs = []layer.DiffID{}
|
||||
|
||||
defer func() {
|
||||
if topLayer != nil {
|
||||
layer.ReleaseAndLog(ldm.layerStore, topLayer)
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
topDownload.Transfer.Release(watcher)
|
||||
return rootFS, func() {}, ctx.Err()
|
||||
case <-topDownload.Done():
|
||||
break
|
||||
}
|
||||
|
||||
l, err := topDownload.result()
|
||||
if err != nil {
|
||||
topDownload.Transfer.Release(watcher)
|
||||
return rootFS, func() {}, err
|
||||
}
|
||||
|
||||
// Must do this exactly len(layers) times, so we don't include the
|
||||
// base layer on Windows.
|
||||
for range layers {
|
||||
if l == nil {
|
||||
topDownload.Transfer.Release(watcher)
|
||||
return rootFS, func() {}, errors.New("internal error: too few parent layers")
|
||||
}
|
||||
rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...)
|
||||
l = l.Parent()
|
||||
}
|
||||
return rootFS, func() { topDownload.Transfer.Release(watcher) }, err
|
||||
}
|
||||
|
||||
// makeDownloadFunc returns a function that performs the layer download and
|
||||
// registration. If parentDownload is non-nil, it waits for that download to
|
||||
// complete before the registration step, and registers the downloaded data
|
||||
// on top of parentDownload's resulting layer. Otherwise, it registers the
|
||||
// layer on top of the ChainID given by parentLayer.
|
||||
func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, parentLayer layer.ChainID, parentDownload *downloadTransfer) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
d := &downloadTransfer{
|
||||
Transfer: NewTransfer(),
|
||||
layerStore: ldm.layerStore,
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
close(progressChan)
|
||||
}()
|
||||
|
||||
progressOutput := progress.ChanOutput(progressChan)
|
||||
|
||||
select {
|
||||
case <-start:
|
||||
default:
|
||||
progress.Update(progressOutput, descriptor.ID(), "Waiting")
|
||||
<-start
|
||||
}
|
||||
|
||||
if parentDownload != nil {
|
||||
// Did the parent download already fail or get
|
||||
// cancelled?
|
||||
select {
|
||||
case <-parentDownload.Done():
|
||||
_, err := parentDownload.result()
|
||||
if err != nil {
|
||||
d.err = err
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
downloadReader io.ReadCloser
|
||||
size int64
|
||||
err error
|
||||
retries int
|
||||
)
|
||||
|
||||
for {
|
||||
downloadReader, size, err = descriptor.Download(d.Transfer.Context(), progressOutput)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
// If an error was returned because the context
|
||||
// was cancelled, we shouldn't retry.
|
||||
select {
|
||||
case <-d.Transfer.Context().Done():
|
||||
d.err = err
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
retries++
|
||||
if _, isDNR := err.(DoNotRetry); isDNR || retries == maxDownloadAttempts {
|
||||
logrus.Errorf("Download failed: %v", err)
|
||||
d.err = err
|
||||
return
|
||||
}
|
||||
|
||||
logrus.Errorf("Download failed, retrying: %v", err)
|
||||
delay := retries * 5
|
||||
ticker := time.NewTicker(time.Second)
|
||||
|
||||
selectLoop:
|
||||
for {
|
||||
progress.Updatef(progressOutput, descriptor.ID(), "Retrying in %d seconds", delay)
|
||||
select {
|
||||
case <-ticker.C:
|
||||
delay--
|
||||
if delay == 0 {
|
||||
ticker.Stop()
|
||||
break selectLoop
|
||||
}
|
||||
case <-d.Transfer.Context().Done():
|
||||
ticker.Stop()
|
||||
d.err = errors.New("download cancelled during retry delay")
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
close(inactive)
|
||||
|
||||
if parentDownload != nil {
|
||||
select {
|
||||
case <-d.Transfer.Context().Done():
|
||||
d.err = errors.New("layer registration cancelled")
|
||||
downloadReader.Close()
|
||||
return
|
||||
case <-parentDownload.Done():
|
||||
}
|
||||
|
||||
l, err := parentDownload.result()
|
||||
if err != nil {
|
||||
d.err = err
|
||||
downloadReader.Close()
|
||||
return
|
||||
}
|
||||
parentLayer = l.ChainID()
|
||||
}
|
||||
|
||||
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(d.Transfer.Context(), downloadReader), progressOutput, size, descriptor.ID(), "Extracting")
|
||||
defer reader.Close()
|
||||
|
||||
inflatedLayerData, err := archive.DecompressStream(reader)
|
||||
if err != nil {
|
||||
d.err = fmt.Errorf("could not get decompression stream: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer)
|
||||
if err != nil {
|
||||
select {
|
||||
case <-d.Transfer.Context().Done():
|
||||
d.err = errors.New("layer registration cancelled")
|
||||
default:
|
||||
d.err = fmt.Errorf("failed to register layer: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
progress.Update(progressOutput, descriptor.ID(), "Pull complete")
|
||||
withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered)
|
||||
if hasRegistered {
|
||||
withRegistered.Registered(d.layer.DiffID())
|
||||
}
|
||||
|
||||
// Doesn't actually need to be its own goroutine, but
|
||||
// done like this so we can defer close(c).
|
||||
go func() {
|
||||
<-d.Transfer.Released()
|
||||
if d.layer != nil {
|
||||
layer.ReleaseAndLog(d.layerStore, d.layer)
|
||||
}
|
||||
}()
|
||||
}()
|
||||
|
||||
return d
|
||||
}
|
||||
}
|
||||
|
||||
// makeDownloadFuncFromDownload returns a function that performs the layer
|
||||
// registration when the layer data is coming from an existing download. It
|
||||
// waits for sourceDownload and parentDownload to complete, and then
|
||||
// reregisters the data from sourceDownload's top layer on top of
|
||||
// parentDownload. This function does not log progress output because it would
|
||||
// interfere with the progress reporting for sourceDownload, which has the same
|
||||
// Key.
|
||||
func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor DownloadDescriptor, sourceDownload *downloadTransfer, parentDownload *downloadTransfer) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
d := &downloadTransfer{
|
||||
Transfer: NewTransfer(),
|
||||
layerStore: ldm.layerStore,
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
close(progressChan)
|
||||
}()
|
||||
|
||||
<-start
|
||||
|
||||
close(inactive)
|
||||
|
||||
select {
|
||||
case <-d.Transfer.Context().Done():
|
||||
d.err = errors.New("layer registration cancelled")
|
||||
return
|
||||
case <-parentDownload.Done():
|
||||
}
|
||||
|
||||
l, err := parentDownload.result()
|
||||
if err != nil {
|
||||
d.err = err
|
||||
return
|
||||
}
|
||||
parentLayer := l.ChainID()
|
||||
|
||||
// sourceDownload should have already finished if
|
||||
// parentDownload finished, but wait for it explicitly
|
||||
// to be sure.
|
||||
select {
|
||||
case <-d.Transfer.Context().Done():
|
||||
d.err = errors.New("layer registration cancelled")
|
||||
return
|
||||
case <-sourceDownload.Done():
|
||||
}
|
||||
|
||||
l, err = sourceDownload.result()
|
||||
if err != nil {
|
||||
d.err = err
|
||||
return
|
||||
}
|
||||
|
||||
layerReader, err := l.TarStream()
|
||||
if err != nil {
|
||||
d.err = err
|
||||
return
|
||||
}
|
||||
defer layerReader.Close()
|
||||
|
||||
d.layer, err = d.layerStore.Register(layerReader, parentLayer)
|
||||
if err != nil {
|
||||
d.err = fmt.Errorf("failed to register layer: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered)
|
||||
if hasRegistered {
|
||||
withRegistered.Registered(d.layer.DiffID())
|
||||
}
|
||||
|
||||
// Doesn't actually need to be its own goroutine, but
|
||||
// done like this so we can defer close(c).
|
||||
go func() {
|
||||
<-d.Transfer.Released()
|
||||
if d.layer != nil {
|
||||
layer.ReleaseAndLog(d.layerStore, d.layer)
|
||||
}
|
||||
}()
|
||||
}()
|
||||
|
||||
return d
|
||||
}
|
||||
}
|
||||
327
vendor/github.com/hyperhq/hypercli/distribution/xfer/download_test.go
generated
vendored
Normal file
327
vendor/github.com/hyperhq/hypercli/distribution/xfer/download_test.go
generated
vendored
Normal file
@@ -0,0 +1,327 @@
|
||||
package xfer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/hyperhq/hypercli/image"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const maxDownloadConcurrency = 3
|
||||
|
||||
type mockLayer struct {
|
||||
layerData bytes.Buffer
|
||||
diffID layer.DiffID
|
||||
chainID layer.ChainID
|
||||
parent layer.Layer
|
||||
}
|
||||
|
||||
func (ml *mockLayer) TarStream() (io.ReadCloser, error) {
|
||||
return ioutil.NopCloser(bytes.NewBuffer(ml.layerData.Bytes())), nil
|
||||
}
|
||||
|
||||
func (ml *mockLayer) ChainID() layer.ChainID {
|
||||
return ml.chainID
|
||||
}
|
||||
|
||||
func (ml *mockLayer) DiffID() layer.DiffID {
|
||||
return ml.diffID
|
||||
}
|
||||
|
||||
func (ml *mockLayer) Parent() layer.Layer {
|
||||
return ml.parent
|
||||
}
|
||||
|
||||
func (ml *mockLayer) Size() (size int64, err error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (ml *mockLayer) DiffSize() (size int64, err error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (ml *mockLayer) Metadata() (map[string]string, error) {
|
||||
return make(map[string]string), nil
|
||||
}
|
||||
|
||||
type mockLayerStore struct {
|
||||
layers map[layer.ChainID]*mockLayer
|
||||
}
|
||||
|
||||
func createChainIDFromParent(parent layer.ChainID, dgsts ...layer.DiffID) layer.ChainID {
|
||||
if len(dgsts) == 0 {
|
||||
return parent
|
||||
}
|
||||
if parent == "" {
|
||||
return createChainIDFromParent(layer.ChainID(dgsts[0]), dgsts[1:]...)
|
||||
}
|
||||
// H = "H(n-1) SHA256(n)"
|
||||
dgst := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
|
||||
return createChainIDFromParent(layer.ChainID(dgst), dgsts[1:]...)
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (layer.Layer, error) {
|
||||
var (
|
||||
parent layer.Layer
|
||||
err error
|
||||
)
|
||||
|
||||
if parentID != "" {
|
||||
parent, err = ls.Get(parentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
l := &mockLayer{parent: parent}
|
||||
_, err = l.layerData.ReadFrom(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l.diffID = layer.DiffID(digest.FromBytes(l.layerData.Bytes()))
|
||||
l.chainID = createChainIDFromParent(parentID, l.diffID)
|
||||
|
||||
ls.layers[l.chainID] = l
|
||||
return l, nil
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) Get(chainID layer.ChainID) (layer.Layer, error) {
|
||||
l, ok := ls.layers[chainID]
|
||||
if !ok {
|
||||
return nil, layer.ErrLayerDoesNotExist
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) Release(l layer.Layer) ([]layer.Metadata, error) {
|
||||
return []layer.Metadata{}, nil
|
||||
}
|
||||
func (ls *mockLayerStore) CreateRWLayer(string, layer.ChainID, string, layer.MountInit) (layer.RWLayer, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) GetRWLayer(string) (layer.RWLayer, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) ReleaseRWLayer(layer.RWLayer) ([]layer.Metadata, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) Cleanup() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) DriverStatus() [][2]string {
|
||||
return [][2]string{}
|
||||
}
|
||||
|
||||
func (ls *mockLayerStore) DriverName() string {
|
||||
return "mock"
|
||||
}
|
||||
|
||||
type mockDownloadDescriptor struct {
|
||||
currentDownloads *int32
|
||||
id string
|
||||
diffID layer.DiffID
|
||||
registeredDiffID layer.DiffID
|
||||
expectedDiffID layer.DiffID
|
||||
simulateRetries int
|
||||
}
|
||||
|
||||
// Key returns the key used to deduplicate downloads.
|
||||
func (d *mockDownloadDescriptor) Key() string {
|
||||
return d.id
|
||||
}
|
||||
|
||||
// ID returns the ID for display purposes.
|
||||
func (d *mockDownloadDescriptor) ID() string {
|
||||
return d.id
|
||||
}
|
||||
|
||||
// DiffID should return the DiffID for this layer, or an error
|
||||
// if it is unknown (for example, if it has not been downloaded
|
||||
// before).
|
||||
func (d *mockDownloadDescriptor) DiffID() (layer.DiffID, error) {
|
||||
if d.diffID != "" {
|
||||
return d.diffID, nil
|
||||
}
|
||||
return "", errors.New("no diffID available")
|
||||
}
|
||||
|
||||
func (d *mockDownloadDescriptor) Registered(diffID layer.DiffID) {
|
||||
d.registeredDiffID = diffID
|
||||
}
|
||||
|
||||
func (d *mockDownloadDescriptor) mockTarStream() io.ReadCloser {
|
||||
// The mock implementation returns the ID repeated 5 times as a tar
|
||||
// stream instead of actual tar data. The data is ignored except for
|
||||
// computing IDs.
|
||||
return ioutil.NopCloser(bytes.NewBuffer([]byte(d.id + d.id + d.id + d.id + d.id)))
|
||||
}
|
||||
|
||||
// Download is called to perform the download.
|
||||
func (d *mockDownloadDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
|
||||
if d.currentDownloads != nil {
|
||||
defer atomic.AddInt32(d.currentDownloads, -1)
|
||||
|
||||
if atomic.AddInt32(d.currentDownloads, 1) > maxDownloadConcurrency {
|
||||
return nil, 0, errors.New("concurrency limit exceeded")
|
||||
}
|
||||
}
|
||||
|
||||
// Sleep a bit to simulate a time-consuming download.
|
||||
for i := int64(0); i <= 10; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, 0, ctx.Err()
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
progressOutput.WriteProgress(progress.Progress{ID: d.ID(), Action: "Downloading", Current: i, Total: 10})
|
||||
}
|
||||
}
|
||||
|
||||
if d.simulateRetries != 0 {
|
||||
d.simulateRetries--
|
||||
return nil, 0, errors.New("simulating retry")
|
||||
}
|
||||
|
||||
return d.mockTarStream(), 0, nil
|
||||
}
|
||||
|
||||
func downloadDescriptors(currentDownloads *int32) []DownloadDescriptor {
|
||||
return []DownloadDescriptor{
|
||||
&mockDownloadDescriptor{
|
||||
currentDownloads: currentDownloads,
|
||||
id: "id1",
|
||||
expectedDiffID: layer.DiffID("sha256:68e2c75dc5c78ea9240689c60d7599766c213ae210434c53af18470ae8c53ec1"),
|
||||
},
|
||||
&mockDownloadDescriptor{
|
||||
currentDownloads: currentDownloads,
|
||||
id: "id2",
|
||||
expectedDiffID: layer.DiffID("sha256:64a636223116aa837973a5d9c2bdd17d9b204e4f95ac423e20e65dfbb3655473"),
|
||||
},
|
||||
&mockDownloadDescriptor{
|
||||
currentDownloads: currentDownloads,
|
||||
id: "id3",
|
||||
expectedDiffID: layer.DiffID("sha256:58745a8bbd669c25213e9de578c4da5c8ee1c836b3581432c2b50e38a6753300"),
|
||||
},
|
||||
&mockDownloadDescriptor{
|
||||
currentDownloads: currentDownloads,
|
||||
id: "id2",
|
||||
expectedDiffID: layer.DiffID("sha256:64a636223116aa837973a5d9c2bdd17d9b204e4f95ac423e20e65dfbb3655473"),
|
||||
},
|
||||
&mockDownloadDescriptor{
|
||||
currentDownloads: currentDownloads,
|
||||
id: "id4",
|
||||
expectedDiffID: layer.DiffID("sha256:0dfb5b9577716cc173e95af7c10289322c29a6453a1718addc00c0c5b1330936"),
|
||||
simulateRetries: 1,
|
||||
},
|
||||
&mockDownloadDescriptor{
|
||||
currentDownloads: currentDownloads,
|
||||
id: "id5",
|
||||
expectedDiffID: layer.DiffID("sha256:0a5f25fa1acbc647f6112a6276735d0fa01e4ee2aa7ec33015e337350e1ea23d"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuccessfulDownload(t *testing.T) {
|
||||
layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
|
||||
ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency)
|
||||
|
||||
progressChan := make(chan progress.Progress)
|
||||
progressDone := make(chan struct{})
|
||||
receivedProgress := make(map[string]progress.Progress)
|
||||
|
||||
go func() {
|
||||
for p := range progressChan {
|
||||
receivedProgress[p.ID] = p
|
||||
}
|
||||
close(progressDone)
|
||||
}()
|
||||
|
||||
var currentDownloads int32
|
||||
descriptors := downloadDescriptors(¤tDownloads)
|
||||
|
||||
firstDescriptor := descriptors[0].(*mockDownloadDescriptor)
|
||||
|
||||
// Pre-register the first layer to simulate an already-existing layer
|
||||
l, err := layerStore.Register(firstDescriptor.mockTarStream(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstDescriptor.diffID = l.DiffID()
|
||||
|
||||
rootFS, releaseFunc, err := ldm.Download(context.Background(), *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
|
||||
if err != nil {
|
||||
t.Fatalf("download error: %v", err)
|
||||
}
|
||||
|
||||
releaseFunc()
|
||||
|
||||
close(progressChan)
|
||||
<-progressDone
|
||||
|
||||
if len(rootFS.DiffIDs) != len(descriptors) {
|
||||
t.Fatal("got wrong number of diffIDs in rootfs")
|
||||
}
|
||||
|
||||
for i, d := range descriptors {
|
||||
descriptor := d.(*mockDownloadDescriptor)
|
||||
|
||||
if descriptor.diffID != "" {
|
||||
if receivedProgress[d.ID()].Action != "Already exists" {
|
||||
t.Fatalf("did not get 'Already exists' message for %v", d.ID())
|
||||
}
|
||||
} else if receivedProgress[d.ID()].Action != "Pull complete" {
|
||||
t.Fatalf("did not get 'Pull complete' message for %v", d.ID())
|
||||
}
|
||||
|
||||
if rootFS.DiffIDs[i] != descriptor.expectedDiffID {
|
||||
t.Fatalf("rootFS item %d has the wrong diffID (expected: %v got: %v)", i, descriptor.expectedDiffID, rootFS.DiffIDs[i])
|
||||
}
|
||||
|
||||
if descriptor.diffID == "" && descriptor.registeredDiffID != rootFS.DiffIDs[i] {
|
||||
t.Fatal("diffID mismatch between rootFS and Registered callback")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelledDownload(t *testing.T) {
|
||||
ldm := NewLayerDownloadManager(&mockLayerStore{make(map[layer.ChainID]*mockLayer)}, maxDownloadConcurrency)
|
||||
|
||||
progressChan := make(chan progress.Progress)
|
||||
progressDone := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
for range progressChan {
|
||||
}
|
||||
close(progressDone)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
go func() {
|
||||
<-time.After(time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
descriptors := downloadDescriptors(nil)
|
||||
_, _, err := ldm.Download(ctx, *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
|
||||
if err != context.Canceled {
|
||||
t.Fatal("expected download to be cancelled")
|
||||
}
|
||||
|
||||
close(progressChan)
|
||||
<-progressDone
|
||||
}
|
||||
392
vendor/github.com/hyperhq/hypercli/distribution/xfer/transfer.go
generated
vendored
Normal file
392
vendor/github.com/hyperhq/hypercli/distribution/xfer/transfer.go
generated
vendored
Normal file
@@ -0,0 +1,392 @@
|
||||
package xfer
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// DoNotRetry is an error wrapper indicating that the error cannot be resolved
|
||||
// with a retry.
|
||||
type DoNotRetry struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
// Error returns the stringified representation of the encapsulated error.
|
||||
func (e DoNotRetry) Error() string {
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
// Watcher is returned by Watch and can be passed to Release to stop watching.
|
||||
type Watcher struct {
|
||||
// signalChan is used to signal to the watcher goroutine that
|
||||
// new progress information is available, or that the transfer
|
||||
// has finished.
|
||||
signalChan chan struct{}
|
||||
// releaseChan signals to the watcher goroutine that the watcher
|
||||
// should be detached.
|
||||
releaseChan chan struct{}
|
||||
// running remains open as long as the watcher is watching the
|
||||
// transfer. It gets closed if the transfer finishes or the
|
||||
// watcher is detached.
|
||||
running chan struct{}
|
||||
}
|
||||
|
||||
// Transfer represents an in-progress transfer.
|
||||
type Transfer interface {
|
||||
Watch(progressOutput progress.Output) *Watcher
|
||||
Release(*Watcher)
|
||||
Context() context.Context
|
||||
Close()
|
||||
Done() <-chan struct{}
|
||||
Released() <-chan struct{}
|
||||
Broadcast(masterProgressChan <-chan progress.Progress)
|
||||
}
|
||||
|
||||
type transfer struct {
|
||||
mu sync.Mutex
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
// watchers keeps track of the goroutines monitoring progress output,
|
||||
// indexed by the channels that release them.
|
||||
watchers map[chan struct{}]*Watcher
|
||||
|
||||
// lastProgress is the most recently received progress event.
|
||||
lastProgress progress.Progress
|
||||
// hasLastProgress is true when lastProgress has been set.
|
||||
hasLastProgress bool
|
||||
|
||||
// running remains open as long as the transfer is in progress.
|
||||
running chan struct{}
|
||||
// released stays open until all watchers release the transfer and
|
||||
// the transfer is no longer tracked by the transfer manager.
|
||||
released chan struct{}
|
||||
|
||||
// broadcastDone is true if the master progress channel has closed.
|
||||
broadcastDone bool
|
||||
// closed is true if Close has been called
|
||||
closed bool
|
||||
// broadcastSyncChan allows watchers to "ping" the broadcasting
|
||||
// goroutine to wait for it for deplete its input channel. This ensures
|
||||
// a detaching watcher won't miss an event that was sent before it
|
||||
// started detaching.
|
||||
broadcastSyncChan chan struct{}
|
||||
}
|
||||
|
||||
// NewTransfer creates a new transfer.
|
||||
func NewTransfer() Transfer {
|
||||
t := &transfer{
|
||||
watchers: make(map[chan struct{}]*Watcher),
|
||||
running: make(chan struct{}),
|
||||
released: make(chan struct{}),
|
||||
broadcastSyncChan: make(chan struct{}),
|
||||
}
|
||||
|
||||
// This uses context.Background instead of a caller-supplied context
|
||||
// so that a transfer won't be cancelled automatically if the client
|
||||
// which requested it is ^C'd (there could be other viewers).
|
||||
t.ctx, t.cancel = context.WithCancel(context.Background())
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// Broadcast copies the progress and error output to all viewers.
|
||||
func (t *transfer) Broadcast(masterProgressChan <-chan progress.Progress) {
|
||||
for {
|
||||
var (
|
||||
p progress.Progress
|
||||
ok bool
|
||||
)
|
||||
select {
|
||||
case p, ok = <-masterProgressChan:
|
||||
default:
|
||||
// We've depleted the channel, so now we can handle
|
||||
// reads on broadcastSyncChan to let detaching watchers
|
||||
// know we're caught up.
|
||||
select {
|
||||
case <-t.broadcastSyncChan:
|
||||
continue
|
||||
case p, ok = <-masterProgressChan:
|
||||
}
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
if ok {
|
||||
t.lastProgress = p
|
||||
t.hasLastProgress = true
|
||||
for _, w := range t.watchers {
|
||||
select {
|
||||
case w.signalChan <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.broadcastDone = true
|
||||
}
|
||||
t.mu.Unlock()
|
||||
if !ok {
|
||||
close(t.running)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Watch adds a watcher to the transfer. The supplied channel gets progress
|
||||
// updates and is closed when the transfer finishes.
|
||||
func (t *transfer) Watch(progressOutput progress.Output) *Watcher {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
w := &Watcher{
|
||||
releaseChan: make(chan struct{}),
|
||||
signalChan: make(chan struct{}),
|
||||
running: make(chan struct{}),
|
||||
}
|
||||
|
||||
t.watchers[w.releaseChan] = w
|
||||
|
||||
if t.broadcastDone {
|
||||
close(w.running)
|
||||
return w
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
close(w.running)
|
||||
}()
|
||||
var (
|
||||
done bool
|
||||
lastWritten progress.Progress
|
||||
hasLastWritten bool
|
||||
)
|
||||
for {
|
||||
t.mu.Lock()
|
||||
hasLastProgress := t.hasLastProgress
|
||||
lastProgress := t.lastProgress
|
||||
t.mu.Unlock()
|
||||
|
||||
// Make sure we don't write the last progress item
|
||||
// twice.
|
||||
if hasLastProgress && (!done || !hasLastWritten || lastProgress != lastWritten) {
|
||||
progressOutput.WriteProgress(lastProgress)
|
||||
lastWritten = lastProgress
|
||||
hasLastWritten = true
|
||||
}
|
||||
|
||||
if done {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case <-w.signalChan:
|
||||
case <-w.releaseChan:
|
||||
done = true
|
||||
// Since the watcher is going to detach, make
|
||||
// sure the broadcaster is caught up so we
|
||||
// don't miss anything.
|
||||
select {
|
||||
case t.broadcastSyncChan <- struct{}{}:
|
||||
case <-t.running:
|
||||
}
|
||||
case <-t.running:
|
||||
done = true
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return w
|
||||
}
|
||||
|
||||
// Release is the inverse of Watch; indicating that the watcher no longer wants
|
||||
// to be notified about the progress of the transfer. All calls to Watch must
|
||||
// be paired with later calls to Release so that the lifecycle of the transfer
|
||||
// is properly managed.
|
||||
func (t *transfer) Release(watcher *Watcher) {
|
||||
t.mu.Lock()
|
||||
delete(t.watchers, watcher.releaseChan)
|
||||
|
||||
if len(t.watchers) == 0 {
|
||||
if t.closed {
|
||||
// released may have been closed already if all
|
||||
// watchers were released, then another one was added
|
||||
// while waiting for a previous watcher goroutine to
|
||||
// finish.
|
||||
select {
|
||||
case <-t.released:
|
||||
default:
|
||||
close(t.released)
|
||||
}
|
||||
} else {
|
||||
t.cancel()
|
||||
}
|
||||
}
|
||||
t.mu.Unlock()
|
||||
|
||||
close(watcher.releaseChan)
|
||||
// Block until the watcher goroutine completes
|
||||
<-watcher.running
|
||||
}
|
||||
|
||||
// Done returns a channel which is closed if the transfer completes or is
|
||||
// cancelled. Note that having 0 watchers causes a transfer to be cancelled.
|
||||
func (t *transfer) Done() <-chan struct{} {
|
||||
// Note that this doesn't return t.ctx.Done() because that channel will
|
||||
// be closed the moment Cancel is called, and we need to return a
|
||||
// channel that blocks until a cancellation is actually acknowledged by
|
||||
// the transfer function.
|
||||
return t.running
|
||||
}
|
||||
|
||||
// Released returns a channel which is closed once all watchers release the
|
||||
// transfer AND the transfer is no longer tracked by the transfer manager.
|
||||
func (t *transfer) Released() <-chan struct{} {
|
||||
return t.released
|
||||
}
|
||||
|
||||
// Context returns the context associated with the transfer.
|
||||
func (t *transfer) Context() context.Context {
|
||||
return t.ctx
|
||||
}
|
||||
|
||||
// Close is called by the transfer manager when the transfer is no longer
|
||||
// being tracked.
|
||||
func (t *transfer) Close() {
|
||||
t.mu.Lock()
|
||||
t.closed = true
|
||||
if len(t.watchers) == 0 {
|
||||
close(t.released)
|
||||
}
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
// DoFunc is a function called by the transfer manager to actually perform
|
||||
// a transfer. It should be non-blocking. It should wait until the start channel
|
||||
// is closed before transferring any data. If the function closes inactive, that
|
||||
// signals to the transfer manager that the job is no longer actively moving
|
||||
// data - for example, it may be waiting for a dependent transfer to finish.
|
||||
// This prevents it from taking up a slot.
|
||||
type DoFunc func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer
|
||||
|
||||
// TransferManager is used by LayerDownloadManager and LayerUploadManager to
|
||||
// schedule and deduplicate transfers. It is up to the TransferManager
|
||||
// implementation to make the scheduling and concurrency decisions.
|
||||
type TransferManager interface {
|
||||
// Transfer checks if a transfer with the given key is in progress. If
|
||||
// so, it returns progress and error output from that transfer.
|
||||
// Otherwise, it will call xferFunc to initiate the transfer.
|
||||
Transfer(key string, xferFunc DoFunc, progressOutput progress.Output) (Transfer, *Watcher)
|
||||
}
|
||||
|
||||
type transferManager struct {
|
||||
mu sync.Mutex
|
||||
|
||||
concurrencyLimit int
|
||||
activeTransfers int
|
||||
transfers map[string]Transfer
|
||||
waitingTransfers []chan struct{}
|
||||
}
|
||||
|
||||
// NewTransferManager returns a new TransferManager.
|
||||
func NewTransferManager(concurrencyLimit int) TransferManager {
|
||||
return &transferManager{
|
||||
concurrencyLimit: concurrencyLimit,
|
||||
transfers: make(map[string]Transfer),
|
||||
}
|
||||
}
|
||||
|
||||
// Transfer checks if a transfer matching the given key is in progress. If not,
|
||||
// it starts one by calling xferFunc. The caller supplies a channel which
|
||||
// receives progress output from the transfer.
|
||||
func (tm *transferManager) Transfer(key string, xferFunc DoFunc, progressOutput progress.Output) (Transfer, *Watcher) {
|
||||
tm.mu.Lock()
|
||||
defer tm.mu.Unlock()
|
||||
|
||||
for {
|
||||
xfer, present := tm.transfers[key]
|
||||
if !present {
|
||||
break
|
||||
}
|
||||
// Transfer is already in progress.
|
||||
watcher := xfer.Watch(progressOutput)
|
||||
|
||||
select {
|
||||
case <-xfer.Context().Done():
|
||||
// We don't want to watch a transfer that has been cancelled.
|
||||
// Wait for it to be removed from the map and try again.
|
||||
xfer.Release(watcher)
|
||||
tm.mu.Unlock()
|
||||
// The goroutine that removes this transfer from the
|
||||
// map is also waiting for xfer.Done(), so yield to it.
|
||||
// This could be avoided by adding a Closed method
|
||||
// to Transfer to allow explicitly waiting for it to be
|
||||
// removed the map, but forcing a scheduling round in
|
||||
// this very rare case seems better than bloating the
|
||||
// interface definition.
|
||||
runtime.Gosched()
|
||||
<-xfer.Done()
|
||||
tm.mu.Lock()
|
||||
default:
|
||||
return xfer, watcher
|
||||
}
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
inactive := make(chan struct{})
|
||||
|
||||
if tm.activeTransfers < tm.concurrencyLimit {
|
||||
close(start)
|
||||
tm.activeTransfers++
|
||||
} else {
|
||||
tm.waitingTransfers = append(tm.waitingTransfers, start)
|
||||
}
|
||||
|
||||
masterProgressChan := make(chan progress.Progress)
|
||||
xfer := xferFunc(masterProgressChan, start, inactive)
|
||||
watcher := xfer.Watch(progressOutput)
|
||||
go xfer.Broadcast(masterProgressChan)
|
||||
tm.transfers[key] = xfer
|
||||
|
||||
// When the transfer is finished, remove from the map.
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-inactive:
|
||||
tm.mu.Lock()
|
||||
tm.inactivate(start)
|
||||
tm.mu.Unlock()
|
||||
inactive = nil
|
||||
case <-xfer.Done():
|
||||
tm.mu.Lock()
|
||||
if inactive != nil {
|
||||
tm.inactivate(start)
|
||||
}
|
||||
delete(tm.transfers, key)
|
||||
tm.mu.Unlock()
|
||||
xfer.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return xfer, watcher
|
||||
}
|
||||
|
||||
func (tm *transferManager) inactivate(start chan struct{}) {
|
||||
// If the transfer was started, remove it from the activeTransfers
|
||||
// count.
|
||||
select {
|
||||
case <-start:
|
||||
// Start next transfer if any are waiting
|
||||
if len(tm.waitingTransfers) != 0 {
|
||||
close(tm.waitingTransfers[0])
|
||||
tm.waitingTransfers = tm.waitingTransfers[1:]
|
||||
} else {
|
||||
tm.activeTransfers--
|
||||
}
|
||||
default:
|
||||
}
|
||||
}
|
||||
414
vendor/github.com/hyperhq/hypercli/distribution/xfer/transfer_test.go
generated
vendored
Normal file
414
vendor/github.com/hyperhq/hypercli/distribution/xfer/transfer_test.go
generated
vendored
Normal file
@@ -0,0 +1,414 @@
|
||||
package xfer
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
)
|
||||
|
||||
func TestTransfer(t *testing.T) {
|
||||
makeXferFunc := func(id string) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
select {
|
||||
case <-start:
|
||||
default:
|
||||
t.Fatalf("transfer function not started even though concurrency limit not reached")
|
||||
}
|
||||
|
||||
xfer := NewTransfer()
|
||||
go func() {
|
||||
for i := 0; i <= 10; i++ {
|
||||
progressChan <- progress.Progress{ID: id, Action: "testing", Current: int64(i), Total: 10}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
close(progressChan)
|
||||
}()
|
||||
return xfer
|
||||
}
|
||||
}
|
||||
|
||||
tm := NewTransferManager(5)
|
||||
progressChan := make(chan progress.Progress)
|
||||
progressDone := make(chan struct{})
|
||||
receivedProgress := make(map[string]int64)
|
||||
|
||||
go func() {
|
||||
for p := range progressChan {
|
||||
val, present := receivedProgress[p.ID]
|
||||
if !present {
|
||||
if p.Current != 0 {
|
||||
t.Fatalf("got unexpected progress value: %d (expected 0)", p.Current)
|
||||
}
|
||||
} else if p.Current != val+1 {
|
||||
t.Fatalf("got unexpected progress value: %d (expected %d)", p.Current, val+1)
|
||||
}
|
||||
receivedProgress[p.ID] = p.Current
|
||||
}
|
||||
close(progressDone)
|
||||
}()
|
||||
|
||||
// Start a few transfers
|
||||
ids := []string{"id1", "id2", "id3"}
|
||||
xfers := make([]Transfer, len(ids))
|
||||
watchers := make([]*Watcher, len(ids))
|
||||
for i, id := range ids {
|
||||
xfers[i], watchers[i] = tm.Transfer(id, makeXferFunc(id), progress.ChanOutput(progressChan))
|
||||
}
|
||||
|
||||
for i, xfer := range xfers {
|
||||
<-xfer.Done()
|
||||
xfer.Release(watchers[i])
|
||||
}
|
||||
close(progressChan)
|
||||
<-progressDone
|
||||
|
||||
for _, id := range ids {
|
||||
if receivedProgress[id] != 10 {
|
||||
t.Fatalf("final progress value %d instead of 10", receivedProgress[id])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrencyLimit(t *testing.T) {
|
||||
concurrencyLimit := 3
|
||||
var runningJobs int32
|
||||
|
||||
makeXferFunc := func(id string) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
xfer := NewTransfer()
|
||||
go func() {
|
||||
<-start
|
||||
totalJobs := atomic.AddInt32(&runningJobs, 1)
|
||||
if int(totalJobs) > concurrencyLimit {
|
||||
t.Fatalf("too many jobs running")
|
||||
}
|
||||
for i := 0; i <= 10; i++ {
|
||||
progressChan <- progress.Progress{ID: id, Action: "testing", Current: int64(i), Total: 10}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
atomic.AddInt32(&runningJobs, -1)
|
||||
close(progressChan)
|
||||
}()
|
||||
return xfer
|
||||
}
|
||||
}
|
||||
|
||||
tm := NewTransferManager(concurrencyLimit)
|
||||
progressChan := make(chan progress.Progress)
|
||||
progressDone := make(chan struct{})
|
||||
receivedProgress := make(map[string]int64)
|
||||
|
||||
go func() {
|
||||
for p := range progressChan {
|
||||
receivedProgress[p.ID] = p.Current
|
||||
}
|
||||
close(progressDone)
|
||||
}()
|
||||
|
||||
// Start more transfers than the concurrency limit
|
||||
ids := []string{"id1", "id2", "id3", "id4", "id5", "id6", "id7", "id8"}
|
||||
xfers := make([]Transfer, len(ids))
|
||||
watchers := make([]*Watcher, len(ids))
|
||||
for i, id := range ids {
|
||||
xfers[i], watchers[i] = tm.Transfer(id, makeXferFunc(id), progress.ChanOutput(progressChan))
|
||||
}
|
||||
|
||||
for i, xfer := range xfers {
|
||||
<-xfer.Done()
|
||||
xfer.Release(watchers[i])
|
||||
}
|
||||
close(progressChan)
|
||||
<-progressDone
|
||||
|
||||
for _, id := range ids {
|
||||
if receivedProgress[id] != 10 {
|
||||
t.Fatalf("final progress value %d instead of 10", receivedProgress[id])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInactiveJobs(t *testing.T) {
|
||||
concurrencyLimit := 3
|
||||
var runningJobs int32
|
||||
testDone := make(chan struct{})
|
||||
|
||||
makeXferFunc := func(id string) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
xfer := NewTransfer()
|
||||
go func() {
|
||||
<-start
|
||||
totalJobs := atomic.AddInt32(&runningJobs, 1)
|
||||
if int(totalJobs) > concurrencyLimit {
|
||||
t.Fatalf("too many jobs running")
|
||||
}
|
||||
for i := 0; i <= 10; i++ {
|
||||
progressChan <- progress.Progress{ID: id, Action: "testing", Current: int64(i), Total: 10}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
atomic.AddInt32(&runningJobs, -1)
|
||||
close(inactive)
|
||||
<-testDone
|
||||
close(progressChan)
|
||||
}()
|
||||
return xfer
|
||||
}
|
||||
}
|
||||
|
||||
tm := NewTransferManager(concurrencyLimit)
|
||||
progressChan := make(chan progress.Progress)
|
||||
progressDone := make(chan struct{})
|
||||
receivedProgress := make(map[string]int64)
|
||||
|
||||
go func() {
|
||||
for p := range progressChan {
|
||||
receivedProgress[p.ID] = p.Current
|
||||
}
|
||||
close(progressDone)
|
||||
}()
|
||||
|
||||
// Start more transfers than the concurrency limit
|
||||
ids := []string{"id1", "id2", "id3", "id4", "id5", "id6", "id7", "id8"}
|
||||
xfers := make([]Transfer, len(ids))
|
||||
watchers := make([]*Watcher, len(ids))
|
||||
for i, id := range ids {
|
||||
xfers[i], watchers[i] = tm.Transfer(id, makeXferFunc(id), progress.ChanOutput(progressChan))
|
||||
}
|
||||
|
||||
close(testDone)
|
||||
for i, xfer := range xfers {
|
||||
<-xfer.Done()
|
||||
xfer.Release(watchers[i])
|
||||
}
|
||||
close(progressChan)
|
||||
<-progressDone
|
||||
|
||||
for _, id := range ids {
|
||||
if receivedProgress[id] != 10 {
|
||||
t.Fatalf("final progress value %d instead of 10", receivedProgress[id])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchRelease(t *testing.T) {
|
||||
ready := make(chan struct{})
|
||||
|
||||
makeXferFunc := func(id string) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
xfer := NewTransfer()
|
||||
go func() {
|
||||
defer func() {
|
||||
close(progressChan)
|
||||
}()
|
||||
<-ready
|
||||
for i := int64(0); ; i++ {
|
||||
select {
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
case <-xfer.Context().Done():
|
||||
return
|
||||
}
|
||||
progressChan <- progress.Progress{ID: id, Action: "testing", Current: i, Total: 10}
|
||||
}
|
||||
}()
|
||||
return xfer
|
||||
}
|
||||
}
|
||||
|
||||
tm := NewTransferManager(5)
|
||||
|
||||
type watcherInfo struct {
|
||||
watcher *Watcher
|
||||
progressChan chan progress.Progress
|
||||
progressDone chan struct{}
|
||||
receivedFirstProgress chan struct{}
|
||||
}
|
||||
|
||||
progressConsumer := func(w watcherInfo) {
|
||||
first := true
|
||||
for range w.progressChan {
|
||||
if first {
|
||||
close(w.receivedFirstProgress)
|
||||
}
|
||||
first = false
|
||||
}
|
||||
close(w.progressDone)
|
||||
}
|
||||
|
||||
// Start a transfer
|
||||
watchers := make([]watcherInfo, 5)
|
||||
var xfer Transfer
|
||||
watchers[0].progressChan = make(chan progress.Progress)
|
||||
watchers[0].progressDone = make(chan struct{})
|
||||
watchers[0].receivedFirstProgress = make(chan struct{})
|
||||
xfer, watchers[0].watcher = tm.Transfer("id1", makeXferFunc("id1"), progress.ChanOutput(watchers[0].progressChan))
|
||||
go progressConsumer(watchers[0])
|
||||
|
||||
// Give it multiple watchers
|
||||
for i := 1; i != len(watchers); i++ {
|
||||
watchers[i].progressChan = make(chan progress.Progress)
|
||||
watchers[i].progressDone = make(chan struct{})
|
||||
watchers[i].receivedFirstProgress = make(chan struct{})
|
||||
watchers[i].watcher = xfer.Watch(progress.ChanOutput(watchers[i].progressChan))
|
||||
go progressConsumer(watchers[i])
|
||||
}
|
||||
|
||||
// Now that the watchers are set up, allow the transfer goroutine to
|
||||
// proceed.
|
||||
close(ready)
|
||||
|
||||
// Confirm that each watcher gets progress output.
|
||||
for _, w := range watchers {
|
||||
<-w.receivedFirstProgress
|
||||
}
|
||||
|
||||
// Release one watcher every 5ms
|
||||
for _, w := range watchers {
|
||||
xfer.Release(w.watcher)
|
||||
<-time.After(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Now that all watchers have been released, Released() should
|
||||
// return a closed channel.
|
||||
<-xfer.Released()
|
||||
|
||||
// Done() should return a closed channel because the xfer func returned
|
||||
// due to cancellation.
|
||||
<-xfer.Done()
|
||||
|
||||
for _, w := range watchers {
|
||||
close(w.progressChan)
|
||||
<-w.progressDone
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchFinishedTransfer(t *testing.T) {
|
||||
makeXferFunc := func(id string) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
xfer := NewTransfer()
|
||||
go func() {
|
||||
// Finish immediately
|
||||
close(progressChan)
|
||||
}()
|
||||
return xfer
|
||||
}
|
||||
}
|
||||
|
||||
tm := NewTransferManager(5)
|
||||
|
||||
// Start a transfer
|
||||
watchers := make([]*Watcher, 3)
|
||||
var xfer Transfer
|
||||
xfer, watchers[0] = tm.Transfer("id1", makeXferFunc("id1"), progress.ChanOutput(make(chan progress.Progress)))
|
||||
|
||||
// Give it a watcher immediately
|
||||
watchers[1] = xfer.Watch(progress.ChanOutput(make(chan progress.Progress)))
|
||||
|
||||
// Wait for the transfer to complete
|
||||
<-xfer.Done()
|
||||
|
||||
// Set up another watcher
|
||||
watchers[2] = xfer.Watch(progress.ChanOutput(make(chan progress.Progress)))
|
||||
|
||||
// Release the watchers
|
||||
for _, w := range watchers {
|
||||
xfer.Release(w)
|
||||
}
|
||||
|
||||
// Now that all watchers have been released, Released() should
|
||||
// return a closed channel.
|
||||
<-xfer.Released()
|
||||
}
|
||||
|
||||
func TestDuplicateTransfer(t *testing.T) {
|
||||
ready := make(chan struct{})
|
||||
|
||||
var xferFuncCalls int32
|
||||
|
||||
makeXferFunc := func(id string) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
atomic.AddInt32(&xferFuncCalls, 1)
|
||||
xfer := NewTransfer()
|
||||
go func() {
|
||||
defer func() {
|
||||
close(progressChan)
|
||||
}()
|
||||
<-ready
|
||||
for i := int64(0); ; i++ {
|
||||
select {
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
case <-xfer.Context().Done():
|
||||
return
|
||||
}
|
||||
progressChan <- progress.Progress{ID: id, Action: "testing", Current: i, Total: 10}
|
||||
}
|
||||
}()
|
||||
return xfer
|
||||
}
|
||||
}
|
||||
|
||||
tm := NewTransferManager(5)
|
||||
|
||||
type transferInfo struct {
|
||||
xfer Transfer
|
||||
watcher *Watcher
|
||||
progressChan chan progress.Progress
|
||||
progressDone chan struct{}
|
||||
receivedFirstProgress chan struct{}
|
||||
}
|
||||
|
||||
progressConsumer := func(t transferInfo) {
|
||||
first := true
|
||||
for range t.progressChan {
|
||||
if first {
|
||||
close(t.receivedFirstProgress)
|
||||
}
|
||||
first = false
|
||||
}
|
||||
close(t.progressDone)
|
||||
}
|
||||
|
||||
// Try to start multiple transfers with the same ID
|
||||
transfers := make([]transferInfo, 5)
|
||||
for i := range transfers {
|
||||
t := &transfers[i]
|
||||
t.progressChan = make(chan progress.Progress)
|
||||
t.progressDone = make(chan struct{})
|
||||
t.receivedFirstProgress = make(chan struct{})
|
||||
t.xfer, t.watcher = tm.Transfer("id1", makeXferFunc("id1"), progress.ChanOutput(t.progressChan))
|
||||
go progressConsumer(*t)
|
||||
}
|
||||
|
||||
// Allow the transfer goroutine to proceed.
|
||||
close(ready)
|
||||
|
||||
// Confirm that each watcher gets progress output.
|
||||
for _, t := range transfers {
|
||||
<-t.receivedFirstProgress
|
||||
}
|
||||
|
||||
// Confirm that the transfer function was called exactly once.
|
||||
if xferFuncCalls != 1 {
|
||||
t.Fatal("transfer function wasn't called exactly once")
|
||||
}
|
||||
|
||||
// Release one watcher every 5ms
|
||||
for _, t := range transfers {
|
||||
t.xfer.Release(t.watcher)
|
||||
<-time.After(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
for _, t := range transfers {
|
||||
// Now that all watchers have been released, Released() should
|
||||
// return a closed channel.
|
||||
<-t.xfer.Released()
|
||||
// Done() should return a closed channel because the xfer func returned
|
||||
// due to cancellation.
|
||||
<-t.xfer.Done()
|
||||
}
|
||||
|
||||
for _, t := range transfers {
|
||||
close(t.progressChan)
|
||||
<-t.progressDone
|
||||
}
|
||||
}
|
||||
154
vendor/github.com/hyperhq/hypercli/distribution/xfer/upload.go
generated
vendored
Normal file
154
vendor/github.com/hyperhq/hypercli/distribution/xfer/upload.go
generated
vendored
Normal file
@@ -0,0 +1,154 @@
|
||||
package xfer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const maxUploadAttempts = 5
|
||||
|
||||
// LayerUploadManager provides task management and progress reporting for
|
||||
// uploads.
|
||||
type LayerUploadManager struct {
|
||||
tm TransferManager
|
||||
}
|
||||
|
||||
// NewLayerUploadManager returns a new LayerUploadManager.
|
||||
func NewLayerUploadManager(concurrencyLimit int) *LayerUploadManager {
|
||||
return &LayerUploadManager{
|
||||
tm: NewTransferManager(concurrencyLimit),
|
||||
}
|
||||
}
|
||||
|
||||
type uploadTransfer struct {
|
||||
Transfer
|
||||
|
||||
diffID layer.DiffID
|
||||
err error
|
||||
}
|
||||
|
||||
// An UploadDescriptor references a layer that may need to be uploaded.
|
||||
type UploadDescriptor interface {
|
||||
// Key returns the key used to deduplicate uploads.
|
||||
Key() string
|
||||
// ID returns the ID for display purposes.
|
||||
ID() string
|
||||
// DiffID should return the DiffID for this layer.
|
||||
DiffID() layer.DiffID
|
||||
// Upload is called to perform the Upload.
|
||||
Upload(ctx context.Context, progressOutput progress.Output) error
|
||||
}
|
||||
|
||||
// Upload is a blocking function which ensures the listed layers are present on
|
||||
// the remote registry. It uses the string returned by the Key method to
|
||||
// deduplicate uploads.
|
||||
func (lum *LayerUploadManager) Upload(ctx context.Context, layers []UploadDescriptor, progressOutput progress.Output) error {
|
||||
var (
|
||||
uploads []*uploadTransfer
|
||||
dedupDescriptors = make(map[string]struct{})
|
||||
)
|
||||
|
||||
for _, descriptor := range layers {
|
||||
progress.Update(progressOutput, descriptor.ID(), "Preparing")
|
||||
|
||||
key := descriptor.Key()
|
||||
if _, present := dedupDescriptors[key]; present {
|
||||
continue
|
||||
}
|
||||
dedupDescriptors[key] = struct{}{}
|
||||
|
||||
xferFunc := lum.makeUploadFunc(descriptor)
|
||||
upload, watcher := lum.tm.Transfer(descriptor.Key(), xferFunc, progressOutput)
|
||||
defer upload.Release(watcher)
|
||||
uploads = append(uploads, upload.(*uploadTransfer))
|
||||
}
|
||||
|
||||
for _, upload := range uploads {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-upload.Transfer.Done():
|
||||
if upload.err != nil {
|
||||
return upload.err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lum *LayerUploadManager) makeUploadFunc(descriptor UploadDescriptor) DoFunc {
|
||||
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
|
||||
u := &uploadTransfer{
|
||||
Transfer: NewTransfer(),
|
||||
diffID: descriptor.DiffID(),
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
close(progressChan)
|
||||
}()
|
||||
|
||||
progressOutput := progress.ChanOutput(progressChan)
|
||||
|
||||
select {
|
||||
case <-start:
|
||||
default:
|
||||
progress.Update(progressOutput, descriptor.ID(), "Waiting")
|
||||
<-start
|
||||
}
|
||||
|
||||
retries := 0
|
||||
for {
|
||||
err := descriptor.Upload(u.Transfer.Context(), progressOutput)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
// If an error was returned because the context
|
||||
// was cancelled, we shouldn't retry.
|
||||
select {
|
||||
case <-u.Transfer.Context().Done():
|
||||
u.err = err
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
retries++
|
||||
if _, isDNR := err.(DoNotRetry); isDNR || retries == maxUploadAttempts {
|
||||
logrus.Errorf("Upload failed: %v", err)
|
||||
u.err = err
|
||||
return
|
||||
}
|
||||
|
||||
logrus.Errorf("Upload failed, retrying: %v", err)
|
||||
delay := retries * 5
|
||||
ticker := time.NewTicker(time.Second)
|
||||
|
||||
selectLoop:
|
||||
for {
|
||||
progress.Updatef(progressOutput, descriptor.ID(), "Retrying in %d seconds", delay)
|
||||
select {
|
||||
case <-ticker.C:
|
||||
delay--
|
||||
if delay == 0 {
|
||||
ticker.Stop()
|
||||
break selectLoop
|
||||
}
|
||||
case <-u.Transfer.Context().Done():
|
||||
ticker.Stop()
|
||||
u.err = errors.New("upload cancelled during retry delay")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return u
|
||||
}
|
||||
}
|
||||
138
vendor/github.com/hyperhq/hypercli/distribution/xfer/upload_test.go
generated
vendored
Normal file
138
vendor/github.com/hyperhq/hypercli/distribution/xfer/upload_test.go
generated
vendored
Normal file
@@ -0,0 +1,138 @@
|
||||
package xfer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/hyperhq/hypercli/layer"
|
||||
"github.com/hyperhq/hypercli/pkg/progress"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const maxUploadConcurrency = 3
|
||||
|
||||
type mockUploadDescriptor struct {
|
||||
currentUploads *int32
|
||||
diffID layer.DiffID
|
||||
simulateRetries int
|
||||
}
|
||||
|
||||
// Key returns the key used to deduplicate downloads.
|
||||
func (u *mockUploadDescriptor) Key() string {
|
||||
return u.diffID.String()
|
||||
}
|
||||
|
||||
// ID returns the ID for display purposes.
|
||||
func (u *mockUploadDescriptor) ID() string {
|
||||
return u.diffID.String()
|
||||
}
|
||||
|
||||
// DiffID should return the DiffID for this layer.
|
||||
func (u *mockUploadDescriptor) DiffID() layer.DiffID {
|
||||
return u.diffID
|
||||
}
|
||||
|
||||
// Upload is called to perform the upload.
|
||||
func (u *mockUploadDescriptor) Upload(ctx context.Context, progressOutput progress.Output) error {
|
||||
if u.currentUploads != nil {
|
||||
defer atomic.AddInt32(u.currentUploads, -1)
|
||||
|
||||
if atomic.AddInt32(u.currentUploads, 1) > maxUploadConcurrency {
|
||||
return errors.New("concurrency limit exceeded")
|
||||
}
|
||||
}
|
||||
|
||||
// Sleep a bit to simulate a time-consuming upload.
|
||||
for i := int64(0); i <= 10; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
progressOutput.WriteProgress(progress.Progress{ID: u.ID(), Current: i, Total: 10})
|
||||
}
|
||||
}
|
||||
|
||||
if u.simulateRetries != 0 {
|
||||
u.simulateRetries--
|
||||
return errors.New("simulating retry")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func uploadDescriptors(currentUploads *int32) []UploadDescriptor {
|
||||
return []UploadDescriptor{
|
||||
&mockUploadDescriptor{currentUploads, layer.DiffID("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"), 0},
|
||||
&mockUploadDescriptor{currentUploads, layer.DiffID("sha256:1515325234325236634634608943609283523908626098235490238423902343"), 0},
|
||||
&mockUploadDescriptor{currentUploads, layer.DiffID("sha256:6929356290463485374960346430698374523437683470934634534953453453"), 0},
|
||||
&mockUploadDescriptor{currentUploads, layer.DiffID("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"), 0},
|
||||
&mockUploadDescriptor{currentUploads, layer.DiffID("sha256:8159352387436803946235346346368745389534789534897538734598734987"), 1},
|
||||
&mockUploadDescriptor{currentUploads, layer.DiffID("sha256:4637863963478346897346987346987346789346789364879364897364987346"), 0},
|
||||
}
|
||||
}
|
||||
|
||||
var expectedDigests = map[layer.DiffID]digest.Digest{
|
||||
layer.DiffID("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"): digest.Digest("sha256:c5095d6cf7ee42b7b064371dcc1dc3fb4af197f04d01a60009d484bd432724fc"),
|
||||
layer.DiffID("sha256:1515325234325236634634608943609283523908626098235490238423902343"): digest.Digest("sha256:968cbfe2ff5269ea1729b3804767a1f57ffbc442d3bc86f47edbf7e688a4f36e"),
|
||||
layer.DiffID("sha256:6929356290463485374960346430698374523437683470934634534953453453"): digest.Digest("sha256:8a5e56ab4b477a400470a7d5d4c1ca0c91235fd723ab19cc862636a06f3a735d"),
|
||||
layer.DiffID("sha256:8159352387436803946235346346368745389534789534897538734598734987"): digest.Digest("sha256:5e733e5cd3688512fc240bd5c178e72671c9915947d17bb8451750d827944cb2"),
|
||||
layer.DiffID("sha256:4637863963478346897346987346987346789346789364879364897364987346"): digest.Digest("sha256:ec4bb98d15e554a9f66c3ef9296cf46772c0ded3b1592bd8324d96e2f60f460c"),
|
||||
}
|
||||
|
||||
func TestSuccessfulUpload(t *testing.T) {
|
||||
lum := NewLayerUploadManager(maxUploadConcurrency)
|
||||
|
||||
progressChan := make(chan progress.Progress)
|
||||
progressDone := make(chan struct{})
|
||||
receivedProgress := make(map[string]int64)
|
||||
|
||||
go func() {
|
||||
for p := range progressChan {
|
||||
receivedProgress[p.ID] = p.Current
|
||||
}
|
||||
close(progressDone)
|
||||
}()
|
||||
|
||||
var currentUploads int32
|
||||
descriptors := uploadDescriptors(¤tUploads)
|
||||
|
||||
err := lum.Upload(context.Background(), descriptors, progress.ChanOutput(progressChan))
|
||||
if err != nil {
|
||||
t.Fatalf("upload error: %v", err)
|
||||
}
|
||||
|
||||
close(progressChan)
|
||||
<-progressDone
|
||||
}
|
||||
|
||||
func TestCancelledUpload(t *testing.T) {
|
||||
lum := NewLayerUploadManager(maxUploadConcurrency)
|
||||
|
||||
progressChan := make(chan progress.Progress)
|
||||
progressDone := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
for range progressChan {
|
||||
}
|
||||
close(progressDone)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
go func() {
|
||||
<-time.After(time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
descriptors := uploadDescriptors(nil)
|
||||
err := lum.Upload(ctx, descriptors, progress.ChanOutput(progressChan))
|
||||
if err != context.Canceled {
|
||||
t.Fatal("expected upload to be cancelled")
|
||||
}
|
||||
|
||||
close(progressChan)
|
||||
<-progressDone
|
||||
}
|
||||
Reference in New Issue
Block a user