Initial commit

vendor/github.com/hyperhq/hypercli/image/tarexport/load.go

@@ -0,0 +1,289 @@
+package tarexport
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/hyperhq/hypercli/image"
+	"github.com/hyperhq/hypercli/image/v1"
+	"github.com/hyperhq/hypercli/layer"
+	"github.com/hyperhq/hypercli/pkg/archive"
+	"github.com/hyperhq/hypercli/pkg/chrootarchive"
+	"github.com/hyperhq/hypercli/pkg/symlink"
+	"github.com/hyperhq/hypercli/reference"
+)
+
+func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer) error {
+	tmpDir, err := ioutil.TempDir("", "docker-import-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
+		return err
+	}
+	// read manifest, if no file then load in legacy mode
+	manifestPath, err := safePath(tmpDir, manifestFileName)
+	if err != nil {
+		return err
+	}
+	manifestFile, err := os.Open(manifestPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return l.legacyLoad(tmpDir, outStream)
+		}
+		return manifestFile.Close()
+	}
+	defer manifestFile.Close()
+
+	var manifest []manifestItem
+	if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
+		return err
+	}
+
+	for _, m := range manifest {
+		configPath, err := safePath(tmpDir, m.Config)
+		if err != nil {
+			return err
+		}
+		config, err := ioutil.ReadFile(configPath)
+		if err != nil {
+			return err
+		}
+		img, err := image.NewFromJSON(config)
+		if err != nil {
+			return err
+		}
+		var rootFS image.RootFS
+		rootFS = *img.RootFS
+		rootFS.DiffIDs = nil
+
+		if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
+			return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
+		}
+
+		for i, diffID := range img.RootFS.DiffIDs {
+			layerPath, err := safePath(tmpDir, m.Layers[i])
+			if err != nil {
+				return err
+			}
+			r := rootFS
+			r.Append(diffID)
+			newLayer, err := l.ls.Get(r.ChainID())
+			if err != nil {
+				newLayer, err = l.loadLayer(layerPath, rootFS)
+				if err != nil {
+					return err
+				}
+			}
+			defer layer.ReleaseAndLog(l.ls, newLayer)
+			if expected, actual := diffID, newLayer.DiffID(); expected != actual {
+				return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
+			}
+			rootFS.Append(diffID)
+		}
+
+		imgID, err := l.is.Create(config)
+		if err != nil {
+			return err
+		}
+
+		for _, repoTag := range m.RepoTags {
+			named, err := reference.ParseNamed(repoTag)
+			if err != nil {
+				return err
+			}
+			ref, ok := named.(reference.NamedTagged)
+			if !ok {
+				return fmt.Errorf("invalid tag %q", repoTag)
+			}
+			l.setLoadedTag(ref, imgID, outStream)
+		}
+
+	}
+
+	return nil
+}
+
+func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS) (layer.Layer, error) {
+	rawTar, err := os.Open(filename)
+	if err != nil {
+		logrus.Debugf("Error reading embedded tar: %v", err)
+		return nil, err
+	}
+	defer rawTar.Close()
+
+	inflatedLayerData, err := archive.DecompressStream(rawTar)
+	if err != nil {
+		return nil, err
+	}
+	defer inflatedLayerData.Close()
+
+	return l.ls.Register(inflatedLayerData, rootFS.ChainID())
+}
+
+func (l *tarexporter) setLoadedTag(ref reference.NamedTagged, imgID image.ID, outStream io.Writer) error {
+	if prevID, err := l.rs.Get(ref); err == nil && prevID != imgID {
+		fmt.Fprintf(outStream, "The image %s already exists, renaming the old one with ID %s to empty string\n", ref.String(), string(prevID)) // todo: this message is wrong in case of multiple tags
+	}
+
+	if err := l.rs.AddTag(ref, imgID, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (l *tarexporter) legacyLoad(tmpDir string, outStream io.Writer) error {
+	legacyLoadedMap := make(map[string]image.ID)
+
+	dirs, err := ioutil.ReadDir(tmpDir)
+	if err != nil {
+		return err
+	}
+
+	// every dir represents an image
+	for _, d := range dirs {
+		if d.IsDir() {
+			if err := l.legacyLoadImage(d.Name(), tmpDir, legacyLoadedMap); err != nil {
+				return err
+			}
+		}
+	}
+
+	// load tags from repositories file
+	repositoriesPath, err := safePath(tmpDir, legacyRepositoriesFileName)
+	if err != nil {
+		return err
+	}
+	repositoriesFile, err := os.Open(repositoriesPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+		return repositoriesFile.Close()
+	}
+	defer repositoriesFile.Close()
+
+	repositories := make(map[string]map[string]string)
+	if err := json.NewDecoder(repositoriesFile).Decode(&repositories); err != nil {
+		return err
+	}
+
+	for name, tagMap := range repositories {
+		for tag, oldID := range tagMap {
+			imgID, ok := legacyLoadedMap[oldID]
+			if !ok {
+				return fmt.Errorf("invalid target ID: %v", oldID)
+			}
+			named, err := reference.WithName(name)
+			if err != nil {
+				return err
+			}
+			ref, err := reference.WithTag(named, tag)
+			if err != nil {
+				return err
+			}
+			l.setLoadedTag(ref, imgID, outStream)
+		}
+	}
+
+	return nil
+}
+
+func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[string]image.ID) error {
+	if _, loaded := loadedMap[oldID]; loaded {
+		return nil
+	}
+	configPath, err := safePath(sourceDir, filepath.Join(oldID, legacyConfigFileName))
+	if err != nil {
+		return err
+	}
+	imageJSON, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		logrus.Debugf("Error reading json: %v", err)
+		return err
+	}
+
+	var img struct{ Parent string }
+	if err := json.Unmarshal(imageJSON, &img); err != nil {
+		return err
+	}
+
+	var parentID image.ID
+	if img.Parent != "" {
+		for {
+			var loaded bool
+			if parentID, loaded = loadedMap[img.Parent]; !loaded {
+				if err := l.legacyLoadImage(img.Parent, sourceDir, loadedMap); err != nil {
+					return err
+				}
+			} else {
+				break
+			}
+		}
+	}
+
+	// todo: try to connect with migrate code
+	rootFS := image.NewRootFS()
+	var history []image.History
+
+	if parentID != "" {
+		parentImg, err := l.is.Get(parentID)
+		if err != nil {
+			return err
+		}
+
+		rootFS = parentImg.RootFS
+		history = parentImg.History
+	}
+
+	layerPath, err := safePath(sourceDir, filepath.Join(oldID, legacyLayerFileName))
+	if err != nil {
+		return err
+	}
+	newLayer, err := l.loadLayer(layerPath, *rootFS)
+	if err != nil {
+		return err
+	}
+	rootFS.Append(newLayer.DiffID())
+
+	h, err := v1.HistoryFromConfig(imageJSON, false)
+	if err != nil {
+		return err
+	}
+	history = append(history, h)
+
+	config, err := v1.MakeConfigFromV1Config(imageJSON, rootFS, history)
+	if err != nil {
+		return err
+	}
+	imgID, err := l.is.Create(config)
+	if err != nil {
+		return err
+	}
+
+	metadata, err := l.ls.Release(newLayer)
+	layer.LogReleaseMetadata(metadata)
+	if err != nil {
+		return err
+	}
+
+	if parentID != "" {
+		if err := l.is.SetParent(imgID, parentID); err != nil {
+			return err
+		}
+	}
+
+	loadedMap[oldID] = imgID
+	return nil
+}
+
+func safePath(base, path string) (string, error) {
+	return symlink.FollowSymlinkInScope(filepath.Join(base, path), base)
+}

vendor/github.com/hyperhq/hypercli/image/tarexport/save.go

@@ -0,0 +1,300 @@
+package tarexport
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/docker/distribution/digest"
+	"github.com/hyperhq/hypercli/image"
+	"github.com/hyperhq/hypercli/image/v1"
+	"github.com/hyperhq/hypercli/layer"
+	"github.com/hyperhq/hypercli/pkg/archive"
+	"github.com/hyperhq/hypercli/reference"
+)
+
+type imageDescriptor struct {
+	refs   []reference.NamedTagged
+	layers []string
+}
+
+type saveSession struct {
+	*tarexporter
+	outDir      string
+	images      map[image.ID]*imageDescriptor
+	savedLayers map[string]struct{}
+}
+
+func (l *tarexporter) Save(names []string, outStream io.Writer) error {
+	images, err := l.parseNames(names)
+	if err != nil {
+		return err
+	}
+
+	return (&saveSession{tarexporter: l, images: images}).save(outStream)
+}
+
+func (l *tarexporter) parseNames(names []string) (map[image.ID]*imageDescriptor, error) {
+	imgDescr := make(map[image.ID]*imageDescriptor)
+
+	addAssoc := func(id image.ID, ref reference.Named) {
+		if _, ok := imgDescr[id]; !ok {
+			imgDescr[id] = &imageDescriptor{}
+		}
+
+		if ref != nil {
+			var tagged reference.NamedTagged
+			if _, ok := ref.(reference.Canonical); ok {
+				return
+			}
+			var ok bool
+			if tagged, ok = ref.(reference.NamedTagged); !ok {
+				var err error
+				if tagged, err = reference.WithTag(ref, reference.DefaultTag); err != nil {
+					return
+				}
+			}
+
+			for _, t := range imgDescr[id].refs {
+				if tagged.String() == t.String() {
+					return
+				}
+			}
+			imgDescr[id].refs = append(imgDescr[id].refs, tagged)
+		}
+	}
+
+	for _, name := range names {
+		ref, err := reference.ParseNamed(name)
+		if err != nil {
+			return nil, err
+		}
+		if ref.Name() == string(digest.Canonical) {
+			imgID, err := l.is.Search(name)
+			if err != nil {
+				return nil, err
+			}
+			addAssoc(imgID, nil)
+			continue
+		}
+		if reference.IsNameOnly(ref) {
+			assocs := l.rs.ReferencesByName(ref)
+			for _, assoc := range assocs {
+				addAssoc(assoc.ImageID, assoc.Ref)
+			}
+			if len(assocs) == 0 {
+				imgID, err := l.is.Search(name)
+				if err != nil {
+					return nil, err
+				}
+				addAssoc(imgID, nil)
+			}
+			continue
+		}
+		var imgID image.ID
+		if imgID, err = l.rs.Get(ref); err != nil {
+			return nil, err
+		}
+		addAssoc(imgID, ref)
+
+	}
+	return imgDescr, nil
+}
+
+func (s *saveSession) save(outStream io.Writer) error {
+	s.savedLayers = make(map[string]struct{})
+
+	// get image json
+	tempDir, err := ioutil.TempDir("", "docker-export-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	s.outDir = tempDir
+	reposLegacy := make(map[string]map[string]string)
+
+	var manifest []manifestItem
+
+	for id, imageDescr := range s.images {
+		if err = s.saveImage(id); err != nil {
+			return err
+		}
+
+		var repoTags []string
+		var layers []string
+
+		for _, ref := range imageDescr.refs {
+			if _, ok := reposLegacy[ref.Name()]; !ok {
+				reposLegacy[ref.Name()] = make(map[string]string)
+			}
+			reposLegacy[ref.Name()][ref.Tag()] = imageDescr.layers[len(imageDescr.layers)-1]
+			repoTags = append(repoTags, ref.String())
+		}
+
+		for _, l := range imageDescr.layers {
+			layers = append(layers, filepath.Join(l, legacyLayerFileName))
+		}
+
+		manifest = append(manifest, manifestItem{
+			Config:   digest.Digest(id).Hex() + ".json",
+			RepoTags: repoTags,
+			Layers:   layers,
+		})
+	}
+
+	if len(reposLegacy) > 0 {
+		reposFile := filepath.Join(tempDir, legacyRepositoriesFileName)
+		f, err := os.OpenFile(reposFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		if err != nil {
+			f.Close()
+			return err
+		}
+		if err := json.NewEncoder(f).Encode(reposLegacy); err != nil {
+			return err
+		}
+		if err := f.Close(); err != nil {
+			return err
+		}
+		if err := os.Chtimes(reposFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
+			return err
+		}
+	}
+
+	manifestFileName := filepath.Join(tempDir, manifestFileName)
+	f, err := os.OpenFile(manifestFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		f.Close()
+		return err
+	}
+	if err := json.NewEncoder(f).Encode(manifest); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	if err := os.Chtimes(manifestFileName, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
+		return err
+	}
+
+	fs, err := archive.Tar(tempDir, archive.Uncompressed)
+	if err != nil {
+		return err
+	}
+	defer fs.Close()
+
+	if _, err := io.Copy(outStream, fs); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *saveSession) saveImage(id image.ID) error {
+	img, err := s.is.Get(id)
+	if err != nil {
+		return err
+	}
+
+	if len(img.RootFS.DiffIDs) == 0 {
+		return fmt.Errorf("empty export - not implemented")
+	}
+
+	var parent digest.Digest
+	var layers []string
+	for i := range img.RootFS.DiffIDs {
+		v1Img := image.V1Image{}
+		if i == len(img.RootFS.DiffIDs)-1 {
+			v1Img = img.V1Image
+		}
+		rootFS := *img.RootFS
+		rootFS.DiffIDs = rootFS.DiffIDs[:i+1]
+		v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent)
+		if err != nil {
+			return err
+		}
+
+		v1Img.ID = v1ID.Hex()
+		if parent != "" {
+			v1Img.Parent = parent.Hex()
+		}
+
+		if err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created); err != nil {
+			return err
+		}
+		layers = append(layers, v1Img.ID)
+		parent = v1ID
+	}
+
+	configFile := filepath.Join(s.outDir, digest.Digest(id).Hex()+".json")
+	if err := ioutil.WriteFile(configFile, img.RawJSON(), 0644); err != nil {
+		return err
+	}
+	if err := os.Chtimes(configFile, img.Created, img.Created); err != nil {
+		return err
+	}
+
+	s.images[id].layers = layers
+	return nil
+}
+
+func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) error {
+	if _, exists := s.savedLayers[legacyImg.ID]; exists {
+		return nil
+	}
+
+	outDir := filepath.Join(s.outDir, legacyImg.ID)
+	if err := os.Mkdir(outDir, 0755); err != nil {
+		return err
+	}
+
+	// todo: why is this version file here?
+	if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil {
+		return err
+	}
+
+	imageConfig, err := json.Marshal(legacyImg)
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil {
+		return err
+	}
+
+	// serialize filesystem
+	tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName))
+	if err != nil {
+		return err
+	}
+	defer tarFile.Close()
+
+	l, err := s.ls.Get(id)
+	if err != nil {
+		return err
+	}
+	defer layer.ReleaseAndLog(s.ls, l)
+
+	arch, err := l.TarStream()
+	if err != nil {
+		return err
+	}
+	defer arch.Close()
+
+	if _, err := io.Copy(tarFile, arch); err != nil {
+		return err
+	}
+
+	for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
+		// todo: maybe save layer created timestamp?
+		if err := os.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
+			return err
+		}
+	}
+
+	s.savedLayers[legacyImg.ID] = struct{}{}
+	return nil
+}

vendor/github.com/hyperhq/hypercli/image/tarexport/tarexport.go

@@ -0,0 +1,36 @@
+package tarexport
+
+import (
+	"github.com/hyperhq/hypercli/image"
+	"github.com/hyperhq/hypercli/layer"
+	"github.com/hyperhq/hypercli/reference"
+)
+
+const (
+	manifestFileName           = "manifest.json"
+	legacyLayerFileName        = "layer.tar"
+	legacyConfigFileName       = "json"
+	legacyVersionFileName      = "VERSION"
+	legacyRepositoriesFileName = "repositories"
+)
+
+type manifestItem struct {
+	Config   string
+	RepoTags []string
+	Layers   []string
+}
+
+type tarexporter struct {
+	is image.Store
+	ls layer.Store
+	rs reference.Store
+}
+
+// NewTarExporter returns new ImageExporter for tar packages
+func NewTarExporter(is image.Store, ls layer.Store, rs reference.Store) image.Exporter {
+	return &tarexporter{
+		is: is,
+		ls: ls,
+		rs: rs,
+	}
+}