Initial commit

vendor/github.com/hyperhq/hypercli/daemon/logger/jsonfilelog/jsonfilelog.go

@@ -0,0 +1,147 @@
+// Package jsonfilelog provides the default Logger implementation for
+// Docker logging. This logger logs to files on the host server in the
+// JSON format.
+package jsonfilelog
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/hyperhq/hypercli/daemon/logger"
+	"github.com/hyperhq/hypercli/daemon/logger/loggerutils"
+	"github.com/hyperhq/hypercli/pkg/jsonlog"
+	"github.com/docker/go-units"
+)
+
+// Name is the name of the file that the jsonlogger logs to.
+const Name = "json-file"
+
+// JSONFileLogger is Logger implementation for default Docker logging.
+type JSONFileLogger struct {
+	buf     *bytes.Buffer
+	writer  *loggerutils.RotateFileWriter
+	mu      sync.Mutex
+	ctx     logger.Context
+	readers map[*logger.LogWatcher]struct{} // stores the active log followers
+	extra   []byte                          // json-encoded extra attributes
+}
+
+func init() {
+	if err := logger.RegisterLogDriver(Name, New); err != nil {
+		logrus.Fatal(err)
+	}
+	if err := logger.RegisterLogOptValidator(Name, ValidateLogOpt); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+// New creates new JSONFileLogger which writes to filename passed in
+// on given context.
+func New(ctx logger.Context) (logger.Logger, error) {
+	var capval int64 = -1
+	if capacity, ok := ctx.Config["max-size"]; ok {
+		var err error
+		capval, err = units.FromHumanSize(capacity)
+		if err != nil {
+			return nil, err
+		}
+	}
+	var maxFiles = 1
+	if maxFileString, ok := ctx.Config["max-file"]; ok {
+		var err error
+		maxFiles, err = strconv.Atoi(maxFileString)
+		if err != nil {
+			return nil, err
+		}
+		if maxFiles < 1 {
+			return nil, fmt.Errorf("max-file cannot be less than 1")
+		}
+	}
+
+	writer, err := loggerutils.NewRotateFileWriter(ctx.LogPath, capval, maxFiles)
+	if err != nil {
+		return nil, err
+	}
+
+	var extra []byte
+	if attrs := ctx.ExtraAttributes(nil); len(attrs) > 0 {
+		var err error
+		extra, err = json.Marshal(attrs)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &JSONFileLogger{
+		buf:     bytes.NewBuffer(nil),
+		writer:  writer,
+		readers: make(map[*logger.LogWatcher]struct{}),
+		extra:   extra,
+	}, nil
+}
+
+// Log converts logger.Message to jsonlog.JSONLog and serializes it to file.
+func (l *JSONFileLogger) Log(msg *logger.Message) error {
+	timestamp, err := jsonlog.FastTimeMarshalJSON(msg.Timestamp)
+	if err != nil {
+		return err
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	err = (&jsonlog.JSONLogs{
+		Log:      append(msg.Line, '\n'),
+		Stream:   msg.Source,
+		Created:  timestamp,
+		RawAttrs: l.extra,
+	}).MarshalJSONBuf(l.buf)
+	if err != nil {
+		return err
+	}
+
+	l.buf.WriteByte('\n')
+	_, err = l.writer.Write(l.buf.Bytes())
+	l.buf.Reset()
+
+	return err
+}
+
+// ValidateLogOpt looks for json specific log options max-file & max-size.
+func ValidateLogOpt(cfg map[string]string) error {
+	for key := range cfg {
+		switch key {
+		case "max-file":
+		case "max-size":
+		case "labels":
+		case "env":
+		default:
+			return fmt.Errorf("unknown log opt '%s' for json-file log driver", key)
+		}
+	}
+	return nil
+}
+
+// LogPath returns the location the given json logger logs to.
+func (l *JSONFileLogger) LogPath() string {
+	return l.writer.LogPath()
+}
+
+// Close closes underlying file and signals all readers to stop.
+func (l *JSONFileLogger) Close() error {
+	l.mu.Lock()
+	err := l.writer.Close()
+	for r := range l.readers {
+		r.Close()
+		delete(l.readers, r)
+	}
+	l.mu.Unlock()
+	return err
+}
+
+// Name returns name of this logger.
+func (l *JSONFileLogger) Name() string {
+	return Name
+}

vendor/github.com/hyperhq/hypercli/daemon/logger/jsonfilelog/jsonfilelog_test.go

@@ -0,0 +1,201 @@
+package jsonfilelog
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/hyperhq/hypercli/daemon/logger"
+	"github.com/hyperhq/hypercli/pkg/jsonlog"
+)
+
+func TestJSONFileLogger(t *testing.T) {
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
+	tmp, err := ioutil.TempDir("", "docker-logger-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	filename := filepath.Join(tmp, "container.log")
+	l, err := New(logger.Context{
+		ContainerID: cid,
+		LogPath:     filename,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line1"), Source: "src1"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line2"), Source: "src2"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line3"), Source: "src3"}); err != nil {
+		t.Fatal(err)
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := `{"log":"line1\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line2\n","stream":"src2","time":"0001-01-01T00:00:00Z"}
+{"log":"line3\n","stream":"src3","time":"0001-01-01T00:00:00Z"}
+`
+
+	if string(res) != expected {
+		t.Fatalf("Wrong log content: %q, expected %q", res, expected)
+	}
+}
+
+func BenchmarkJSONFileLogger(b *testing.B) {
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
+	tmp, err := ioutil.TempDir("", "docker-logger-")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	filename := filepath.Join(tmp, "container.log")
+	l, err := New(logger.Context{
+		ContainerID: cid,
+		LogPath:     filename,
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer l.Close()
+
+	testLine := "Line that thinks that it is log line from docker\n"
+	msg := &logger.Message{ContainerID: cid, Line: []byte(testLine), Source: "stderr", Timestamp: time.Now().UTC()}
+	jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON()
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.SetBytes(int64(len(jsonlog)+1) * 30)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for j := 0; j < 30; j++ {
+			if err := l.Log(msg); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func TestJSONFileLoggerWithOpts(t *testing.T) {
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
+	tmp, err := ioutil.TempDir("", "docker-logger-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	filename := filepath.Join(tmp, "container.log")
+	config := map[string]string{"max-file": "2", "max-size": "1k"}
+	l, err := New(logger.Context{
+		ContainerID: cid,
+		LogPath:     filename,
+		Config:      config,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	for i := 0; i < 20; i++ {
+		if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line" + strconv.Itoa(i)), Source: "src1"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	penUlt, err := ioutil.ReadFile(filename + ".1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedPenultimate := `{"log":"line0\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line1\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line2\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line3\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line4\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line5\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line6\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line7\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line8\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line9\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line10\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line11\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line12\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line13\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line14\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line15\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+`
+	expected := `{"log":"line16\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line17\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line18\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line19\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+`
+
+	if string(res) != expected {
+		t.Fatalf("Wrong log content: %q, expected %q", res, expected)
+	}
+	if string(penUlt) != expectedPenultimate {
+		t.Fatalf("Wrong log content: %q, expected %q", penUlt, expectedPenultimate)
+	}
+
+}
+
+func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
+	tmp, err := ioutil.TempDir("", "docker-logger-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	filename := filepath.Join(tmp, "container.log")
+	config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl"}
+	l, err := New(logger.Context{
+		ContainerID:     cid,
+		LogPath:         filename,
+		Config:          config,
+		ContainerLabels: map[string]string{"rack": "101", "dc": "lhr"},
+		ContainerEnv:    []string{"environ=production", "debug=false", "port=10001", "ssl=true"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line"), Source: "src1"}); err != nil {
+		t.Fatal(err)
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var jsonLog jsonlog.JSONLogs
+	if err := json.Unmarshal(res, &jsonLog); err != nil {
+		t.Fatal(err)
+	}
+	extra := make(map[string]string)
+	if err := json.Unmarshal(jsonLog.RawAttrs, &extra); err != nil {
+		t.Fatal(err)
+	}
+	expected := map[string]string{
+		"rack":    "101",
+		"dc":      "lhr",
+		"environ": "production",
+		"debug":   "false",
+		"ssl":     "true",
+	}
+	if !reflect.DeepEqual(extra, expected) {
+		t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected)
+	}
+}

vendor/github.com/hyperhq/hypercli/daemon/logger/jsonfilelog/read.go

@@ -0,0 +1,216 @@
+package jsonfilelog
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/hyperhq/hypercli/daemon/logger"
+	"github.com/hyperhq/hypercli/pkg/filenotify"
+	"github.com/hyperhq/hypercli/pkg/ioutils"
+	"github.com/hyperhq/hypercli/pkg/jsonlog"
+	"github.com/hyperhq/hypercli/pkg/tailfile"
+)
+
+const maxJSONDecodeRetry = 20000
+
+func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, error) {
+	l.Reset()
+	if err := dec.Decode(l); err != nil {
+		return nil, err
+	}
+	msg := &logger.Message{
+		Source:    l.Stream,
+		Timestamp: l.Created,
+		Line:      []byte(l.Log),
+	}
+	return msg, nil
+}
+
+// ReadLogs implements the logger's LogReader interface for the logs
+// created by this driver.
+func (l *JSONFileLogger) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
+	logWatcher := logger.NewLogWatcher()
+
+	go l.readLogs(logWatcher, config)
+	return logWatcher
+}
+
+func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.ReadConfig) {
+	defer close(logWatcher.Msg)
+
+	pth := l.writer.LogPath()
+	var files []io.ReadSeeker
+	for i := l.writer.MaxFiles(); i > 1; i-- {
+		f, err := os.Open(fmt.Sprintf("%s.%d", pth, i-1))
+		if err != nil {
+			if !os.IsNotExist(err) {
+				logWatcher.Err <- err
+				break
+			}
+			continue
+		}
+		defer f.Close()
+		files = append(files, f)
+	}
+
+	latestFile, err := os.Open(pth)
+	if err != nil {
+		logWatcher.Err <- err
+		return
+	}
+	defer latestFile.Close()
+
+	files = append(files, latestFile)
+	tailer := ioutils.MultiReadSeeker(files...)
+
+	if config.Tail != 0 {
+		tailFile(tailer, logWatcher, config.Tail, config.Since)
+	}
+
+	if !config.Follow {
+		return
+	}
+
+	if config.Tail >= 0 {
+		latestFile.Seek(0, os.SEEK_END)
+	}
+
+	l.mu.Lock()
+	l.readers[logWatcher] = struct{}{}
+	l.mu.Unlock()
+
+	notifyRotate := l.writer.NotifyRotate()
+	followLogs(latestFile, logWatcher, notifyRotate, config.Since)
+
+	l.mu.Lock()
+	delete(l.readers, logWatcher)
+	l.mu.Unlock()
+
+	l.writer.NotifyRotateEvict(notifyRotate)
+}
+
+func tailFile(f io.ReadSeeker, logWatcher *logger.LogWatcher, tail int, since time.Time) {
+	var rdr io.Reader = f
+	if tail > 0 {
+		ls, err := tailfile.TailFile(f, tail)
+		if err != nil {
+			logWatcher.Err <- err
+			return
+		}
+		rdr = bytes.NewBuffer(bytes.Join(ls, []byte("\n")))
+	}
+	dec := json.NewDecoder(rdr)
+	l := &jsonlog.JSONLog{}
+	for {
+		msg, err := decodeLogLine(dec, l)
+		if err != nil {
+			if err != io.EOF {
+				logWatcher.Err <- err
+			}
+			return
+		}
+		if !since.IsZero() && msg.Timestamp.Before(since) {
+			continue
+		}
+		logWatcher.Msg <- msg
+	}
+}
+
+func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan interface{}, since time.Time) {
+	dec := json.NewDecoder(f)
+	l := &jsonlog.JSONLog{}
+
+	fileWatcher, err := filenotify.New()
+	if err != nil {
+		logWatcher.Err <- err
+	}
+	defer fileWatcher.Close()
+
+	var retries int
+	for {
+		msg, err := decodeLogLine(dec, l)
+		if err != nil {
+			if err != io.EOF {
+				// try again because this shouldn't happen
+				if _, ok := err.(*json.SyntaxError); ok && retries <= maxJSONDecodeRetry {
+					dec = json.NewDecoder(f)
+					retries++
+					continue
+				}
+
+				// io.ErrUnexpectedEOF is returned from json.Decoder when there is
+				// remaining data in the parser's buffer while an io.EOF occurs.
+				// If the json logger writes a partial json log entry to the disk
+				// while at the same time the decoder tries to decode it, the race condition happens.
+				if err == io.ErrUnexpectedEOF && retries <= maxJSONDecodeRetry {
+					reader := io.MultiReader(dec.Buffered(), f)
+					dec = json.NewDecoder(reader)
+					retries++
+					continue
+				}
+				logWatcher.Err <- err
+				return
+			}
+
+			logrus.WithField("logger", "json-file").Debugf("waiting for events")
+			if err := fileWatcher.Add(f.Name()); err != nil {
+				logrus.WithField("logger", "json-file").Warn("falling back to file poller")
+				fileWatcher.Close()
+				fileWatcher = filenotify.NewPollingWatcher()
+				if err := fileWatcher.Add(f.Name()); err != nil {
+					logrus.Errorf("error watching log file for modifications: %v", err)
+					logWatcher.Err <- err
+				}
+			}
+			select {
+			case <-fileWatcher.Events():
+				dec = json.NewDecoder(f)
+				fileWatcher.Remove(f.Name())
+				continue
+			case <-fileWatcher.Errors():
+				fileWatcher.Remove(f.Name())
+				logWatcher.Err <- err
+				return
+			case <-logWatcher.WatchClose():
+				fileWatcher.Remove(f.Name())
+				return
+			case <-notifyRotate:
+				f, err = os.Open(f.Name())
+				if err != nil {
+					logWatcher.Err <- err
+					return
+				}
+
+				dec = json.NewDecoder(f)
+				fileWatcher.Remove(f.Name())
+				fileWatcher.Add(f.Name())
+				continue
+			}
+		}
+
+		retries = 0 // reset retries since we've succeeded
+		if !since.IsZero() && msg.Timestamp.Before(since) {
+			continue
+		}
+		select {
+		case logWatcher.Msg <- msg:
+		case <-logWatcher.WatchClose():
+			logWatcher.Msg <- msg
+			for {
+				msg, err := decodeLogLine(dec, l)
+				if err != nil {
+					return
+				}
+				if !since.IsZero() && msg.Timestamp.Before(since) {
+					continue
+				}
+				logWatcher.Msg <- msg
+			}
+		}
+	}
+}