Add HashiCorp Nomad provider (#483)

* provider: adding Nomad provider

* updating CONTRIBUTING.md with Nomad provider

* updated README.md by adding the Nomad provider

* fix typo

* adding nomad/api and nomad/testutil deps

* adding Nomad binary dependency for provider tests

* fixed the nomad binary download command step and added tolerations to the nomad provider.

* adding nomad provider demo gif

* adding my name to authors

* adding two missing go-rootcerts files after dep ensure

* delete pod comment

vendor/github.com/hashicorp/nomad/testutil/server.go

@@ -0,0 +1,378 @@
+package testutil
+
+// TestServer is a test helper. It uses a fork/exec model to create
+// a test Nomad server instance in the background and initialize it
+// with some data and/or services. The test server can then be used
+// to run a unit test, and offers an easy API to tear itself down
+// when the test has completed. The only prerequisite is to have a nomad
+// binary available on the $PATH.
+//
+// This package does not use Nomad's official API client. This is
+// because we use TestServer to test the API client, which would
+// otherwise cause an import cycle.
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+
+	"github.com/hashicorp/consul/lib/freeport"
+	cleanhttp "github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/nomad/helper/discover"
+	testing "github.com/mitchellh/go-testing-interface"
+)
+
+// TestServerConfig is the main server configuration struct.
+type TestServerConfig struct {
+	NodeName          string        `json:"name,omitempty"`
+	DataDir           string        `json:"data_dir,omitempty"`
+	Region            string        `json:"region,omitempty"`
+	DisableCheckpoint bool          `json:"disable_update_check"`
+	LogLevel          string        `json:"log_level,omitempty"`
+	Consul            *Consul       `json:"consul,omitempty"`
+	AdvertiseAddrs    *Advertise    `json:"advertise,omitempty"`
+	Ports             *PortsConfig  `json:"ports,omitempty"`
+	Server            *ServerConfig `json:"server,omitempty"`
+	Client            *ClientConfig `json:"client,omitempty"`
+	Vault             *VaultConfig  `json:"vault,omitempty"`
+	ACL               *ACLConfig    `json:"acl,omitempty"`
+	DevMode           bool          `json:"-"`
+	Stdout, Stderr    io.Writer     `json:"-"`
+}
+
+// Consul is used to configure the communication with Consul
+type Consul struct {
+	Address string `json:"address,omitempty"`
+	Auth    string `json:"auth,omitempty"`
+	Token   string `json:"token,omitempty"`
+}
+
+// Advertise is used to configure the addresses to advertise
+type Advertise struct {
+	HTTP string `json:"http,omitempty"`
+	RPC  string `json:"rpc,omitempty"`
+	Serf string `json:"serf,omitempty"`
+}
+
+// PortsConfig is used to configure the network ports we use.
+type PortsConfig struct {
+	HTTP int `json:"http,omitempty"`
+	RPC  int `json:"rpc,omitempty"`
+	Serf int `json:"serf,omitempty"`
+}
+
+// ServerConfig is used to configure the nomad server.
+type ServerConfig struct {
+	Enabled         bool `json:"enabled"`
+	BootstrapExpect int  `json:"bootstrap_expect"`
+	RaftProtocol    int  `json:"raft_protocol,omitempty"`
+}
+
+// ClientConfig is used to configure the client
+type ClientConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+// VaultConfig is used to configure Vault
+type VaultConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+// ACLConfig is used to configure ACLs
+type ACLConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+// ServerConfigCallback is a function interface which can be
+// passed to NewTestServerConfig to modify the server config.
+type ServerConfigCallback func(c *TestServerConfig)
+
+// defaultServerConfig returns a new TestServerConfig struct
+// with all of the listen ports incremented by one.
+func defaultServerConfig(t testing.T) *TestServerConfig {
+	ports := freeport.GetT(t, 3)
+	return &TestServerConfig{
+		NodeName:          fmt.Sprintf("node-%d", ports[0]),
+		DisableCheckpoint: true,
+		LogLevel:          "DEBUG",
+		Ports: &PortsConfig{
+			HTTP: ports[0],
+			RPC:  ports[1],
+			Serf: ports[2],
+		},
+		Server: &ServerConfig{
+			Enabled:         true,
+			BootstrapExpect: 1,
+		},
+		Client: &ClientConfig{
+			Enabled: false,
+		},
+		Vault: &VaultConfig{
+			Enabled: false,
+		},
+		ACL: &ACLConfig{
+			Enabled: false,
+		},
+	}
+}
+
+// TestServer is the main server wrapper struct.
+type TestServer struct {
+	cmd    *exec.Cmd
+	Config *TestServerConfig
+	t      testing.T
+
+	HTTPAddr   string
+	SerfAddr   string
+	HTTPClient *http.Client
+}
+
+// NewTestServer creates a new TestServer, and makes a call to
+// an optional callback function to modify the configuration.
+func NewTestServer(t testing.T, cb ServerConfigCallback) *TestServer {
+	path, err := discover.NomadExecutable()
+	if err != nil {
+		t.Skipf("nomad not found, skipping: %v", err)
+	}
+
+	// Do a sanity check that we are actually running nomad
+	vcmd := exec.Command(path, "-version")
+	vcmd.Stdout = nil
+	vcmd.Stderr = nil
+	if err := vcmd.Run(); err != nil {
+		t.Skipf("nomad version failed: %v", err)
+	}
+
+	dataDir, err := ioutil.TempDir("", "nomad")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	configFile, err := ioutil.TempFile(dataDir, "nomad")
+	if err != nil {
+		defer os.RemoveAll(dataDir)
+		t.Fatalf("err: %s", err)
+	}
+	defer configFile.Close()
+
+	nomadConfig := defaultServerConfig(t)
+	nomadConfig.DataDir = dataDir
+
+	if cb != nil {
+		cb(nomadConfig)
+	}
+
+	configContent, err := json.Marshal(nomadConfig)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if _, err := configFile.Write(configContent); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	configFile.Close()
+
+	stdout := io.Writer(os.Stdout)
+	if nomadConfig.Stdout != nil {
+		stdout = nomadConfig.Stdout
+	}
+
+	stderr := io.Writer(os.Stderr)
+	if nomadConfig.Stderr != nil {
+		stderr = nomadConfig.Stderr
+	}
+
+	args := []string{"agent", "-config", configFile.Name()}
+	if nomadConfig.DevMode {
+		args = append(args, "-dev")
+	}
+
+	// Start the server
+	cmd := exec.Command(path, args...)
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	client := cleanhttp.DefaultClient()
+
+	server := &TestServer{
+		Config: nomadConfig,
+		cmd:    cmd,
+		t:      t,
+
+		HTTPAddr:   fmt.Sprintf("127.0.0.1:%d", nomadConfig.Ports.HTTP),
+		SerfAddr:   fmt.Sprintf("127.0.0.1:%d", nomadConfig.Ports.Serf),
+		HTTPClient: client,
+	}
+
+	// Wait for the server to be ready
+	if nomadConfig.Server.Enabled && nomadConfig.Server.BootstrapExpect != 0 {
+		server.waitForLeader()
+	} else {
+		server.waitForAPI()
+	}
+
+	// Wait for the client to be ready
+	if nomadConfig.DevMode {
+		server.waitForClient()
+	}
+	return server
+}
+
+// Stop stops the test Nomad server, and removes the Nomad data
+// directory once we are done.
+func (s *TestServer) Stop() {
+	defer os.RemoveAll(s.Config.DataDir)
+
+	if err := s.cmd.Process.Kill(); err != nil {
+		s.t.Errorf("err: %s", err)
+	}
+
+	// wait for the process to exit to be sure that the data dir can be
+	// deleted on all platforms.
+	s.cmd.Wait()
+}
+
+// waitForAPI waits for only the agent HTTP endpoint to start
+// responding. This is an indication that the agent has started,
+// but will likely return before a leader is elected.
+func (s *TestServer) waitForAPI() {
+	WaitForResult(func() (bool, error) {
+		// Using this endpoint as it is does not have restricted access
+		resp, err := s.HTTPClient.Get(s.url("/v1/metrics"))
+		if err != nil {
+			return false, err
+		}
+		defer resp.Body.Close()
+		if err := s.requireOK(resp); err != nil {
+			return false, err
+		}
+		return true, nil
+	}, func(err error) {
+		defer s.Stop()
+		s.t.Fatalf("err: %s", err)
+	})
+}
+
+// waitForLeader waits for the Nomad server's HTTP API to become
+// available, and then waits for a known leader and an index of
+// 1 or more to be observed to confirm leader election is done.
+func (s *TestServer) waitForLeader() {
+	WaitForResult(func() (bool, error) {
+		// Query the API and check the status code
+		// Using this endpoint as it is does not have restricted access
+		resp, err := s.HTTPClient.Get(s.url("/v1/status/leader"))
+		if err != nil {
+			return false, err
+		}
+		defer resp.Body.Close()
+		if err := s.requireOK(resp); err != nil {
+			return false, err
+		}
+
+		return true, nil
+	}, func(err error) {
+		defer s.Stop()
+		s.t.Fatalf("err: %s", err)
+	})
+}
+
+// waitForClient waits for the Nomad client to be ready. The function returns
+// immediately if the server is not in dev mode.
+func (s *TestServer) waitForClient() {
+	if !s.Config.DevMode {
+		return
+	}
+
+	WaitForResult(func() (bool, error) {
+		resp, err := s.HTTPClient.Get(s.url("/v1/nodes"))
+		if err != nil {
+			return false, err
+		}
+		defer resp.Body.Close()
+		if err := s.requireOK(resp); err != nil {
+			return false, err
+		}
+
+		var decoded []struct {
+			ID     string
+			Status string
+		}
+
+		dec := json.NewDecoder(resp.Body)
+		if err := dec.Decode(&decoded); err != nil {
+			return false, err
+		}
+
+		if len(decoded) != 1 || decoded[0].Status != "ready" {
+			return false, fmt.Errorf("Node not ready: %v", decoded)
+		}
+
+		return true, nil
+	}, func(err error) {
+		defer s.Stop()
+		s.t.Fatalf("err: %s", err)
+	})
+}
+
+// url is a helper function which takes a relative URL and
+// makes it into a proper URL against the local Nomad server.
+func (s *TestServer) url(path string) string {
+	return fmt.Sprintf("http://%s%s", s.HTTPAddr, path)
+}
+
+// requireOK checks the HTTP response code and ensures it is acceptable.
+func (s *TestServer) requireOK(resp *http.Response) error {
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("Bad status code: %d", resp.StatusCode)
+	}
+	return nil
+}
+
+// put performs a new HTTP PUT request.
+func (s *TestServer) put(path string, body io.Reader) *http.Response {
+	req, err := http.NewRequest("PUT", s.url(path), body)
+	if err != nil {
+		s.t.Fatalf("err: %s", err)
+	}
+	resp, err := s.HTTPClient.Do(req)
+	if err != nil {
+		s.t.Fatalf("err: %s", err)
+	}
+	if err := s.requireOK(resp); err != nil {
+		defer resp.Body.Close()
+		s.t.Fatal(err)
+	}
+	return resp
+}
+
+// get performs a new HTTP GET request.
+func (s *TestServer) get(path string) *http.Response {
+	resp, err := s.HTTPClient.Get(s.url(path))
+	if err != nil {
+		s.t.Fatalf("err: %s", err)
+	}
+	if err := s.requireOK(resp); err != nil {
+		defer resp.Body.Close()
+		s.t.Fatal(err)
+	}
+	return resp
+}
+
+// encodePayload returns a new io.Reader wrapping the encoded contents
+// of the payload, suitable for passing directly to a new request.
+func (s *TestServer) encodePayload(payload interface{}) io.Reader {
+	var encoded bytes.Buffer
+	enc := json.NewEncoder(&encoded)
+	if err := enc.Encode(payload); err != nil {
+		s.t.Fatalf("err: %s", err)
+	}
+	return &encoded
+}

vendor/github.com/hashicorp/nomad/testutil/slow.go

@@ -0,0 +1,15 @@
+package testutil
+
+import (
+	"os"
+
+	testing "github.com/mitchellh/go-testing-interface"
+)
+
+// SkipSlow skips a slow test unless the NOMAD_SLOW_TEST environment variable
+// is set.
+func SkipSlow(t testing.T) {
+	if os.Getenv("NOMAD_SLOW_TEST") == "" {
+		t.Skip("Skipping slow test. Set NOMAD_SLOW_TEST=1 to run.")
+	}
+}

vendor/github.com/hashicorp/nomad/testutil/vault.go

@@ -0,0 +1,227 @@
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/hashicorp/consul/lib/freeport"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/structs/config"
+	vapi "github.com/hashicorp/vault/api"
+	"github.com/mitchellh/go-testing-interface"
+)
+
+// TestVault is a test helper. It uses a fork/exec model to create a test Vault
+// server instance in the background and can be initialized with policies, roles
+// and backends mounted. The test Vault instances can be used to run a unit test
+// and offers and easy API to tear itself down on test end. The only
+// prerequisite is that the Vault binary is on the $PATH.
+
+// TestVault wraps a test Vault server launched in dev mode, suitable for
+// testing.
+type TestVault struct {
+	cmd    *exec.Cmd
+	t      testing.T
+	waitCh chan error
+
+	Addr      string
+	HTTPAddr  string
+	RootToken string
+	Config    *config.VaultConfig
+	Client    *vapi.Client
+}
+
+func NewTestVaultFromPath(t testing.T, binary string) *TestVault {
+	for i := 10; i >= 0; i-- {
+		port := freeport.GetT(t, 1)[0]
+		token := uuid.Generate()
+		bind := fmt.Sprintf("-dev-listen-address=127.0.0.1:%d", port)
+		http := fmt.Sprintf("http://127.0.0.1:%d", port)
+		root := fmt.Sprintf("-dev-root-token-id=%s", token)
+
+		cmd := exec.Command(binary, "server", "-dev", bind, root)
+		cmd.Stdout = testlog.NewWriter(t)
+		cmd.Stderr = testlog.NewWriter(t)
+
+		// Build the config
+		conf := vapi.DefaultConfig()
+		conf.Address = http
+
+		// Make the client and set the token to the root token
+		client, err := vapi.NewClient(conf)
+		if err != nil {
+			t.Fatalf("failed to build Vault API client: %v", err)
+		}
+		client.SetToken(token)
+
+		enable := true
+		tv := &TestVault{
+			cmd:       cmd,
+			t:         t,
+			Addr:      bind,
+			HTTPAddr:  http,
+			RootToken: token,
+			Client:    client,
+			Config: &config.VaultConfig{
+				Enabled: &enable,
+				Token:   token,
+				Addr:    http,
+			},
+		}
+
+		if err := tv.cmd.Start(); err != nil {
+			tv.t.Fatalf("failed to start vault: %v", err)
+		}
+
+		// Start the waiter
+		tv.waitCh = make(chan error, 1)
+		go func() {
+			err := tv.cmd.Wait()
+			tv.waitCh <- err
+		}()
+
+		// Ensure Vault started
+		var startErr error
+		select {
+		case startErr = <-tv.waitCh:
+		case <-time.After(time.Duration(500*TestMultiplier()) * time.Millisecond):
+		}
+
+		if startErr != nil && i == 0 {
+			t.Fatalf("failed to start vault: %v", startErr)
+		} else if startErr != nil {
+			wait := time.Duration(rand.Int31n(2000)) * time.Millisecond
+			time.Sleep(wait)
+			continue
+		}
+
+		waitErr := tv.waitForAPI()
+		if waitErr != nil && i == 0 {
+			t.Fatalf("failed to start vault: %v", waitErr)
+		} else if waitErr != nil {
+			wait := time.Duration(rand.Int31n(2000)) * time.Millisecond
+			time.Sleep(wait)
+			continue
+		}
+
+		return tv
+	}
+
+	return nil
+
+}
+
+// NewTestVault returns a new TestVault instance that has yet to be started
+func NewTestVault(t testing.T) *TestVault {
+	// Lookup vault from the path
+	return NewTestVaultFromPath(t, "vault")
+}
+
+// NewTestVaultDelayed returns a test Vault server that has not been started.
+// Start must be called and it is the callers responsibility to deal with any
+// port conflicts that may occur and retry accordingly.
+func NewTestVaultDelayed(t testing.T) *TestVault {
+	port := freeport.GetT(t, 1)[0]
+	token := uuid.Generate()
+	bind := fmt.Sprintf("-dev-listen-address=127.0.0.1:%d", port)
+	http := fmt.Sprintf("http://127.0.0.1:%d", port)
+	root := fmt.Sprintf("-dev-root-token-id=%s", token)
+
+	cmd := exec.Command("vault", "server", "-dev", bind, root)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	// Build the config
+	conf := vapi.DefaultConfig()
+	conf.Address = http
+
+	// Make the client and set the token to the root token
+	client, err := vapi.NewClient(conf)
+	if err != nil {
+		t.Fatalf("failed to build Vault API client: %v", err)
+	}
+	client.SetToken(token)
+
+	enable := true
+	tv := &TestVault{
+		cmd:       cmd,
+		t:         t,
+		Addr:      bind,
+		HTTPAddr:  http,
+		RootToken: token,
+		Client:    client,
+		Config: &config.VaultConfig{
+			Enabled: &enable,
+			Token:   token,
+			Addr:    http,
+		},
+	}
+
+	return tv
+}
+
+// Start starts the test Vault server and waits for it to respond to its HTTP
+// API
+func (tv *TestVault) Start() error {
+	if err := tv.cmd.Start(); err != nil {
+		tv.t.Fatalf("failed to start vault: %v", err)
+	}
+
+	// Start the waiter
+	tv.waitCh = make(chan error, 1)
+	go func() {
+		err := tv.cmd.Wait()
+		tv.waitCh <- err
+	}()
+
+	// Ensure Vault started
+	select {
+	case err := <-tv.waitCh:
+		return err
+	case <-time.After(time.Duration(500*TestMultiplier()) * time.Millisecond):
+	}
+
+	return tv.waitForAPI()
+}
+
+// Stop stops the test Vault server
+func (tv *TestVault) Stop() {
+	if tv.cmd.Process == nil {
+		return
+	}
+
+	if err := tv.cmd.Process.Kill(); err != nil {
+		tv.t.Errorf("err: %s", err)
+	}
+	if tv.waitCh != nil {
+		<-tv.waitCh
+	}
+}
+
+// waitForAPI waits for the Vault HTTP endpoint to start
+// responding. This is an indication that the agent has started.
+func (tv *TestVault) waitForAPI() error {
+	var waitErr error
+	WaitForResult(func() (bool, error) {
+		inited, err := tv.Client.Sys().InitStatus()
+		if err != nil {
+			return false, err
+		}
+		return inited, nil
+	}, func(err error) {
+		waitErr = err
+	})
+	return waitErr
+}
+
+// VaultVersion returns the Vault version as a string or an error if it couldn't
+// be determined
+func VaultVersion() (string, error) {
+	cmd := exec.Command("vault", "version")
+	out, err := cmd.Output()
+	return string(out), err
+}

vendor/github.com/hashicorp/nomad/testutil/wait.go

@@ -0,0 +1,133 @@
+package testutil
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/mitchellh/go-testing-interface"
+)
+
+const (
+	// TravisRunEnv is an environment variable that is set if being run by
+	// Travis.
+	TravisRunEnv = "CI"
+)
+
+type testFn func() (bool, error)
+type errorFn func(error)
+
+func WaitForResult(test testFn, error errorFn) {
+	WaitForResultRetries(500*TestMultiplier(), test, error)
+}
+
+func WaitForResultRetries(retries int64, test testFn, error errorFn) {
+	for retries > 0 {
+		time.Sleep(10 * time.Millisecond)
+		retries--
+
+		success, err := test()
+		if success {
+			return
+		}
+
+		if retries == 0 {
+			error(err)
+		}
+	}
+}
+
+// AssertUntil asserts the test function passes throughout the given duration.
+// Otherwise error is called on failure.
+func AssertUntil(until time.Duration, test testFn, error errorFn) {
+	deadline := time.Now().Add(until)
+	for time.Now().Before(deadline) {
+		success, err := test()
+		if !success {
+			error(err)
+			return
+		}
+		// Sleep some arbitrary fraction of the deadline
+		time.Sleep(until / 30)
+	}
+}
+
+// TestMultiplier returns a multiplier for retries and waits given environment
+// the tests are being run under.
+func TestMultiplier() int64 {
+	if IsTravis() {
+		return 4
+	}
+
+	return 1
+}
+
+// Timeout takes the desired timeout and increases it if running in Travis
+func Timeout(original time.Duration) time.Duration {
+	return original * time.Duration(TestMultiplier())
+}
+
+func IsTravis() bool {
+	_, ok := os.LookupEnv(TravisRunEnv)
+	return ok
+}
+
+type rpcFn func(string, interface{}, interface{}) error
+
+// WaitForLeader blocks until a leader is elected.
+func WaitForLeader(t testing.T, rpc rpcFn) {
+	WaitForResult(func() (bool, error) {
+		args := &structs.GenericRequest{}
+		var leader string
+		err := rpc("Status.Leader", args, &leader)
+		return leader != "", err
+	}, func(err error) {
+		t.Fatalf("failed to find leader: %v", err)
+	})
+}
+
+// WaitForRunning runs a job and blocks until it is running.
+func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) {
+	registered := false
+	WaitForResult(func() (bool, error) {
+		if !registered {
+			args := &structs.JobRegisterRequest{}
+			args.Job = job
+			args.WriteRequest.Region = "global"
+			var jobResp structs.JobRegisterResponse
+			err := rpc("Job.Register", args, &jobResp)
+			if err != nil {
+				return false, fmt.Errorf("Job.Register error: %v", err)
+			}
+
+			// Only register once
+			registered = true
+		}
+
+		args := &structs.JobSummaryRequest{}
+		args.JobID = job.ID
+		args.QueryOptions.Region = "global"
+		var resp structs.JobSummaryResponse
+		err := rpc("Job.Summary", args, &resp)
+		if err != nil {
+			return false, fmt.Errorf("Job.Summary error: %v", err)
+		}
+
+		tgs := len(job.TaskGroups)
+		summaries := len(resp.JobSummary.Summary)
+		if tgs != summaries {
+			return false, fmt.Errorf("task_groups=%d summaries=%d", tgs, summaries)
+		}
+
+		for tg, summary := range resp.JobSummary.Summary {
+			if summary.Running == 0 {
+				return false, fmt.Errorf("task_group=%s %#v", tg, resp.JobSummary.Summary)
+			}
+		}
+
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("job not running: %v", err)
+	})
+}