virtual-kubelet/vendor/github.com/vmware/vic/cmd/imagec/main.go

// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"runtime/debug"
	"runtime/trace"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/reference"
	"github.com/go-openapi/runtime"
	rc "github.com/go-openapi/runtime/client"
	"github.com/pkg/profile"

	"github.com/vmware/vic/lib/apiservers/engine/proxy"
	apiclient "github.com/vmware/vic/lib/apiservers/portlayer/client"
	vicarchive "github.com/vmware/vic/lib/archive"
	"github.com/vmware/vic/lib/constants"
	"github.com/vmware/vic/lib/imagec"
	optrace "github.com/vmware/vic/pkg/trace"
	"github.com/vmware/vic/pkg/version"
)

var (
	imageCOptions = ImageCOptions{}

	// https://raw.githubusercontent.com/docker/docker/master/distribution/pull_v2.go
	sf = streamformatter.NewJSONStreamFormatter()
)

const (
	PullImage  = "pull"
	PushImage  = "push"
	ListLayers = "listlayers"
	Save       = "save"
)

// ImageCOptions wraps the cli arguments
type ImageCOptions struct {
	imageName string

	options imagec.Options

	logfile string

	stdout bool
	debug  bool

	profiling string
	tracing   bool

	operation string
}

func init() {

	flag.StringVar(&imageCOptions.imageName, "reference", "", "Name of the reference")

	flag.StringVar(&imageCOptions.options.Destination, "destination", imagec.DefaultDestination, "Destination directory")

	flag.StringVar(&imageCOptions.options.Host, "host", imagec.DefaultPortLayerHost, "Host that runs portlayer API (FQDN:port format)")

	flag.StringVar(&imageCOptions.logfile, "logfile", imagec.DefaultLogfile, "Path of the imagec log file")

	flag.StringVar(&imageCOptions.options.Username, "username", "", "Username")
	flag.StringVar(&imageCOptions.options.Password, "password", "", "Password")

	flag.DurationVar(&imageCOptions.options.Timeout, "timeout", imagec.DefaultHTTPTimeout, "HTTP timeout")

	flag.BoolVar(&imageCOptions.stdout, "stdout", false, "Enable writing to stdout")
	flag.BoolVar(&imageCOptions.debug, "debug", false, "Show debug logging")
	flag.BoolVar(&imageCOptions.options.InsecureSkipVerify, "insecure-skip-verify", false, "Don't verify certificates when fetching images")
	flag.BoolVar(&imageCOptions.options.InsecureAllowHTTP, "insecure-allow-http", false, "Uses unencrypted connections when fetching images")
	flag.BoolVar(&imageCOptions.options.Standalone, "standalone", false, "Disable port-layer integration")

	flag.StringVar(&imageCOptions.profiling, "profile.mode", "", "Enable profiling mode, one of [cpu, mem, block]")
	flag.BoolVar(&imageCOptions.tracing, "tracing", false, "Enable runtime tracing")

	flag.StringVar(&imageCOptions.operation, "operation", "pull", "Pull image/push image/listlayers/save image")

	flag.StringVar(&imageCOptions.options.Registry, "registry", imagec.DefaultDockerURL, "Registry to pull/push images (default: registry-1.docker.io)")

	flag.StringVar(&imageCOptions.options.ImageStore, "image-store", imagec.DefaultDockerURL, "portlayer image store name or url used to query image data")

	flag.Parse()

	var err error
	if imageCOptions.options.Reference, err = reference.ParseNamed(imageCOptions.imageName); err != nil {
		panic(err)
	}
}

func main() {
	defer func() {
		if r := recover(); r != nil {
			_, err := fmt.Fprintf(os.Stderr, string(sf.FormatError(fmt.Errorf("%s : %s", r, debug.Stack()))))
			if err != nil {
				//do this to pass security check
			}
		}
	}()

	if version.Show() {
		_, err := fmt.Fprintf(os.Stdout, "%s\n", version.String())
		if err != nil {
			panic(err)
		}
		return
	}

	// Enable profiling if mode is set
	switch imageCOptions.profiling {
	case "cpu":
		defer profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.Quiet).Stop()
	case "mem":
		defer profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.Quiet).Stop()
	case "block":
		defer profile.Start(profile.BlockProfile, profile.ProfilePath("."), profile.Quiet).Stop()
	default:
		// do nothing
	}

	// Register our custom Error hook
	log.AddHook(NewErrorHook(os.Stderr))

	// Enable runtime tracing if tracing is true
	if imageCOptions.tracing {
		tracing, err := os.Create(time.Now().Format("2006-01-02T150405.pprof"))
		if err != nil {
			log.Fatalf("Failed to create tracing logfile: %s", err)
		}
		defer tracing.Close()

		if err := trace.Start(tracing); err != nil {
			log.Fatalf("Failed to start tracing: %s", err)
		}
		defer trace.Stop()
	}

	// Open the log file
	// #nosec: Expect file permissions to be 0600 or less
	f, err := os.OpenFile(imageCOptions.logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatalf("Failed to open the logfile %s: %s", imageCOptions.logfile, err)
	}
	defer f.Close()

	// Initiliaze logger with default TextFormatter
	log.SetFormatter(&log.TextFormatter{DisableColors: true, FullTimestamp: true})

	// Set the log level
	if imageCOptions.debug {
		log.SetLevel(log.DebugLevel)
	}

	// SetOutput to log file and/or stdout
	log.SetOutput(f)
	if imageCOptions.stdout {
		log.SetOutput(io.MultiWriter(os.Stdout, f))
	}

	switch imageCOptions.operation {
	case PullImage:
		options := imageCOptions.options

		options.Outstream = os.Stdout

		ic := imagec.NewImageC(options, streamformatter.NewJSONStreamFormatter())
		if err := ic.PullImage(); err != nil {
			log.Fatalf("Pulling image failed due to %s\n", err)
		}
	case PushImage:
		log.Errorf("The operation '%s' is not implemented\n", PushImage)
	case ListLayers:
		options := imageCOptions.options

		options.Outstream = os.Stdout

		ic := imagec.NewImageC(options, streamformatter.NewJSONStreamFormatter())
		if err := ic.ListLayers(); err != nil {
			log.Fatalf("Listing layers for image failed due to %s\n", err)
		}
	case Save:
		options := imageCOptions.options

		options.Outstream = os.Stdout

		ap := archiveProxy(options.Host)
		ic := imagec.NewImageC(options, streamformatter.NewJSONStreamFormatter())
		if err := saveImage(ap, ic); err != nil {
			log.Fatalf("Saving image %s failed due to %s\n", options.Reference.String(), err)
		}
	default:
		log.Errorf("The operation '%s' is not valid\n", imageCOptions.operation)
	}
}

func saveImage(ap proxy.VicArchiveProxy, ic *imagec.ImageC) error {
	log.Debugf("Save image %s", ic.Options.Reference)
	err := ic.ListLayers()
	if err != nil {
		return err
	}
	layers, err := ic.LayersToDownload()
	if err != nil {
		return err
	}

	dest := imagec.DestinationDirectory(ic.Options)
	var wg sync.WaitGroup
	errChan := make(chan error, len(layers))

	for i := len(layers) - 1; i >= 0; i-- {
		if layers[i].ID == constants.ScratchLayerID {
			continue
		}
		wg.Add(1)
		go func(pid, cid string) {
			defer wg.Done()
			var err error
			log.Debugf("parent id: %s, layer id: %s", pid, cid)

			defer func() {
				errChan <- err
			}()

			fileDir := path.Join(dest, cid)
			err = os.MkdirAll(fileDir, 0755) /* #nosec */
			if err != nil {
				return
			}
			filePath := path.Join(fileDir, cid+".tar")
			log.Debugf("save layer %s to file %s", cid, filePath)
			err = writeArchiveFile(ap, ic.ImageStore, ic.ImageStore, cid, pid, filePath)
		}(layers[i].Parent, layers[i].ID)
	}
	wg.Wait()
	close(errChan)

	var errs []string
	for e := range errChan {
		if e == nil {
			continue
		}
		errs = append(errs, e.Error())
	}
	if len(errs) > 0 {
		return fmt.Errorf("save image layers failed: %s", strings.Join(errs, ","))
	}
	return nil
}

func portlayerClient(portLayerAddr string) *apiclient.PortLayer {
	t := rc.New(portLayerAddr, "/", []string{"http"})
	t.Consumers["application/x-tar"] = runtime.ByteStreamConsumer()
	t.Consumers["application/octet-stream"] = runtime.ByteStreamConsumer()
	t.Producers["application/x-tar"] = runtime.ByteStreamProducer()
	t.Producers["application/octet-stream"] = runtime.ByteStreamProducer()

	client := apiclient.New(t, nil)
	return client
}

func archiveProxy(portLayerAddr string) proxy.VicArchiveProxy {
	plClient := portlayerClient(portLayerAddr)
	archiveProxy := proxy.NewArchiveProxy(plClient)

	return archiveProxy
}

func writeArchiveFile(archiveProxy proxy.VicArchiveProxy, store, ancestorStore, layerID, ancestorID, archivePath string) error {
	var filterSpec vicarchive.FilterSpec

	op := optrace.NewOperation(context.Background(), "export layer %s:%s", layerID, ancestorID)
	//Initialize an archive stream from the portlayer for the layer
	ar, err := archiveProxy.ArchiveExportReader(op, store, ancestorStore, layerID, ancestorID, true, filterSpec)
	if err != nil || ar == nil {
		return fmt.Errorf("Failed to get reader for layer %s", layerID)
	}

	log.Infof("Obtain archive reader for layer %s, parent %s", layerID, ancestorID)

	// #nosec - there's nothing secret/sensitive about these image layer files
	tarFile, err := os.OpenFile(archivePath, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		msg := fmt.Sprintf("Failed to create tmp file: %s", err.Error())
		log.Info(msg)
		return fmt.Errorf(msg)
	}
	defer tarFile.Close()

	_, err = io.Copy(tarFile, ar)
	if err != nil {
		msg := fmt.Sprintf("Failed to read from acrhive stream: %s", err.Error())
		log.Info(msg)
		return fmt.Errorf(msg)
	}

	ar.Close()

	if err = tarFile.Sync(); err != nil {
		msg := fmt.Sprintf("Failed to flush tar file: %s", err.Error())
		log.Info(msg)
		return fmt.Errorf(msg)
	}

	return nil
}