VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR

vendor/github.com/docker/libnetwork/iptables/firewalld.go

@@ -0,0 +1,164 @@
+package iptables
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/godbus/dbus"
+)
+
+// IPV defines the table string
+type IPV string
+
+const (
+	// Iptables point ipv4 table
+	Iptables IPV = "ipv4"
+	// IP6Tables point to ipv6 table
+	IP6Tables IPV = "ipv6"
+	// Ebtables point to bridge table
+	Ebtables IPV = "eb"
+)
+const (
+	dbusInterface = "org.fedoraproject.FirewallD1"
+	dbusPath      = "/org/fedoraproject/FirewallD1"
+)
+
+// Conn is a connection to firewalld dbus endpoint.
+type Conn struct {
+	sysconn *dbus.Conn
+	sysobj  dbus.BusObject
+	signal  chan *dbus.Signal
+}
+
+var (
+	connection       *Conn
+	firewalldRunning bool      // is Firewalld service running
+	onReloaded       []*func() // callbacks when Firewalld has been reloaded
+)
+
+// FirewalldInit initializes firewalld management code.
+func FirewalldInit() error {
+	var err error
+
+	if connection, err = newConnection(); err != nil {
+		return fmt.Errorf("Failed to connect to D-Bus system bus: %v", err)
+	}
+	if connection != nil {
+		go signalHandler()
+	}
+
+	firewalldRunning = checkRunning()
+	return nil
+}
+
+// New() establishes a connection to the system bus.
+func newConnection() (*Conn, error) {
+	c := new(Conn)
+	if err := c.initConnection(); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// Innitialize D-Bus connection.
+func (c *Conn) initConnection() error {
+	var err error
+
+	c.sysconn, err = dbus.SystemBus()
+	if err != nil {
+		return err
+	}
+
+	// This never fails, even if the service is not running atm.
+	c.sysobj = c.sysconn.Object(dbusInterface, dbus.ObjectPath(dbusPath))
+
+	rule := fmt.Sprintf("type='signal',path='%s',interface='%s',sender='%s',member='Reloaded'",
+		dbusPath, dbusInterface, dbusInterface)
+	c.sysconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, rule)
+
+	rule = fmt.Sprintf("type='signal',interface='org.freedesktop.DBus',member='NameOwnerChanged',path='/org/freedesktop/DBus',sender='org.freedesktop.DBus',arg0='%s'",
+		dbusInterface)
+	c.sysconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, rule)
+
+	c.signal = make(chan *dbus.Signal, 10)
+	c.sysconn.Signal(c.signal)
+
+	return nil
+}
+
+func signalHandler() {
+	for signal := range connection.signal {
+		if strings.Contains(signal.Name, "NameOwnerChanged") {
+			firewalldRunning = checkRunning()
+			dbusConnectionChanged(signal.Body)
+		} else if strings.Contains(signal.Name, "Reloaded") {
+			reloaded()
+		}
+	}
+}
+
+func dbusConnectionChanged(args []interface{}) {
+	name := args[0].(string)
+	oldOwner := args[1].(string)
+	newOwner := args[2].(string)
+
+	if name != dbusInterface {
+		return
+	}
+
+	if len(newOwner) > 0 {
+		connectionEstablished()
+	} else if len(oldOwner) > 0 {
+		connectionLost()
+	}
+}
+
+func connectionEstablished() {
+	reloaded()
+}
+
+func connectionLost() {
+	// Doesn't do anything for now. Libvirt also doesn't react to this.
+}
+
+// call all callbacks
+func reloaded() {
+	for _, pf := range onReloaded {
+		(*pf)()
+	}
+}
+
+// OnReloaded add callback
+func OnReloaded(callback func()) {
+	for _, pf := range onReloaded {
+		if pf == &callback {
+			return
+		}
+	}
+	onReloaded = append(onReloaded, &callback)
+}
+
+// Call some remote method to see whether the service is actually running.
+func checkRunning() bool {
+	var zone string
+	var err error
+
+	if connection != nil {
+		err = connection.sysobj.Call(dbusInterface+".getDefaultZone", 0).Store(&zone)
+		logrus.Infof("Firewalld running: %t", err == nil)
+		return err == nil
+	}
+	return false
+}
+
+// Passthrough method simply passes args through to iptables/ip6tables
+func Passthrough(ipv IPV, args ...string) ([]byte, error) {
+	var output string
+	logrus.Debugf("Firewalld passthrough: %s, %s", ipv, args)
+	if err := connection.sysobj.Call(dbusInterface+".direct.passthrough", 0, ipv, args).Store(&output); err != nil {
+		return nil, err
+	}
+	return []byte(output), nil
+}

vendor/github.com/docker/libnetwork/iptables/firewalld_test.go

@@ -0,0 +1,86 @@
+package iptables
+
+import (
+	"net"
+	"strconv"
+	"testing"
+)
+
+func TestFirewalldInit(t *testing.T) {
+	if !checkRunning() {
+		t.Skip("firewalld is not running")
+	}
+	if err := FirewalldInit(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestReloaded(t *testing.T) {
+	var err error
+	var fwdChain *ChainInfo
+
+	fwdChain, err = NewChain("FWD", Filter, false)
+	bridgeName := "lo"
+
+	err = ProgramChain(fwdChain, bridgeName, false, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fwdChain.Remove()
+
+	// copy-pasted from iptables_test:TestLink
+	ip1 := net.ParseIP("192.168.1.1")
+	ip2 := net.ParseIP("192.168.1.2")
+	port := 1234
+	proto := "tcp"
+
+	err = fwdChain.Link(Append, ip1, ip2, port, proto, bridgeName)
+	if err != nil {
+		t.Fatal(err)
+	} else {
+		// to be re-called again later
+		OnReloaded(func() { fwdChain.Link(Append, ip1, ip2, port, proto, bridgeName) })
+	}
+
+	rule1 := []string{
+		"-i", bridgeName,
+		"-o", bridgeName,
+		"-p", proto,
+		"-s", ip1.String(),
+		"-d", ip2.String(),
+		"--dport", strconv.Itoa(port),
+		"-j", "ACCEPT"}
+
+	if !Exists(fwdChain.Table, fwdChain.Name, rule1...) {
+		t.Fatalf("rule1 does not exist")
+	}
+
+	// flush all rules
+	fwdChain.Remove()
+
+	reloaded()
+
+	// make sure the rules have been recreated
+	if !Exists(fwdChain.Table, fwdChain.Name, rule1...) {
+		t.Fatalf("rule1 hasn't been recreated")
+	}
+}
+
+func TestPassthrough(t *testing.T) {
+	rule1 := []string{
+		"-i", "lo",
+		"-p", "udp",
+		"--dport", "123",
+		"-j", "ACCEPT"}
+
+	if firewalldRunning {
+		_, err := Passthrough(Iptables, append([]string{"-A"}, rule1...)...)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !Exists(Filter, "INPUT", rule1...) {
+			t.Fatalf("rule1 does not exist")
+		}
+	}
+
+}

vendor/github.com/docker/libnetwork/iptables/iptables.go

@@ -0,0 +1,371 @@
+package iptables
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// Action signifies the iptable action.
+type Action string
+
+// Table refers to Nat, Filter or Mangle.
+type Table string
+
+const (
+	// Append appends the rule at the end of the chain.
+	Append Action = "-A"
+	// Delete deletes the rule from the chain.
+	Delete Action = "-D"
+	// Insert inserts the rule at the top of the chain.
+	Insert Action = "-I"
+	// Nat table is used for nat translation rules.
+	Nat Table = "nat"
+	// Filter table is used for filter rules.
+	Filter Table = "filter"
+	// Mangle table is used for mangling the packet.
+	Mangle Table = "mangle"
+)
+
+var (
+	iptablesPath  string
+	supportsXlock = false
+	// used to lock iptables commands if xtables lock is not supported
+	bestEffortLock sync.Mutex
+	// ErrIptablesNotFound is returned when the rule is not found.
+	ErrIptablesNotFound = errors.New("Iptables not found")
+)
+
+// ChainInfo defines the iptables chain.
+type ChainInfo struct {
+	Name        string
+	Table       Table
+	HairpinMode bool
+}
+
+// ChainError is returned to represent errors during ip table operation.
+type ChainError struct {
+	Chain  string
+	Output []byte
+}
+
+func (e ChainError) Error() string {
+	return fmt.Sprintf("Error iptables %s: %s", e.Chain, string(e.Output))
+}
+
+func initCheck() error {
+
+	if iptablesPath == "" {
+		path, err := exec.LookPath("iptables")
+		if err != nil {
+			return ErrIptablesNotFound
+		}
+		iptablesPath = path
+		supportsXlock = exec.Command(iptablesPath, "--wait", "-L", "-n").Run() == nil
+	}
+	return nil
+}
+
+// NewChain adds a new chain to ip table.
+func NewChain(name string, table Table, hairpinMode bool) (*ChainInfo, error) {
+	c := &ChainInfo{
+		Name:        name,
+		Table:       table,
+		HairpinMode: hairpinMode,
+	}
+	if string(c.Table) == "" {
+		c.Table = Filter
+	}
+
+	// Add chain if it doesn't exist
+	if _, err := Raw("-t", string(c.Table), "-n", "-L", c.Name); err != nil {
+		if output, err := Raw("-t", string(c.Table), "-N", c.Name); err != nil {
+			return nil, err
+		} else if len(output) != 0 {
+			return nil, fmt.Errorf("Could not create %s/%s chain: %s", c.Table, c.Name, output)
+		}
+	}
+	return c, nil
+}
+
+// ProgramChain is used to add rules to a chain
+func ProgramChain(c *ChainInfo, bridgeName string, hairpinMode, enable bool) error {
+	if c.Name == "" {
+		return fmt.Errorf("Could not program chain, missing chain name.")
+	}
+
+	switch c.Table {
+	case Nat:
+		preroute := []string{
+			"-m", "addrtype",
+			"--dst-type", "LOCAL",
+			"-j", c.Name}
+		if !Exists(Nat, "PREROUTING", preroute...) && enable {
+			if err := c.Prerouting(Append, preroute...); err != nil {
+				return fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err)
+			}
+		} else if Exists(Nat, "PREROUTING", preroute...) && !enable {
+			if err := c.Prerouting(Delete, preroute...); err != nil {
+				return fmt.Errorf("Failed to remove docker in PREROUTING chain: %s", err)
+			}
+		}
+		output := []string{
+			"-m", "addrtype",
+			"--dst-type", "LOCAL",
+			"-j", c.Name}
+		if !hairpinMode {
+			output = append(output, "!", "--dst", "127.0.0.0/8")
+		}
+		if !Exists(Nat, "OUTPUT", output...) && enable {
+			if err := c.Output(Append, output...); err != nil {
+				return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
+			}
+		} else if Exists(Nat, "OUTPUT", output...) && !enable {
+			if err := c.Output(Delete, output...); err != nil {
+				return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
+			}
+		}
+	case Filter:
+		if bridgeName == "" {
+			return fmt.Errorf("Could not program chain %s/%s, missing bridge name.",
+				c.Table, c.Name)
+		}
+		link := []string{
+			"-o", bridgeName,
+			"-j", c.Name}
+		if !Exists(Filter, "FORWARD", link...) && enable {
+			insert := append([]string{string(Insert), "FORWARD"}, link...)
+			if output, err := Raw(insert...); err != nil {
+				return err
+			} else if len(output) != 0 {
+				return fmt.Errorf("Could not create linking rule to %s/%s: %s", c.Table, c.Name, output)
+			}
+		} else if Exists(Filter, "FORWARD", link...) && !enable {
+			del := append([]string{string(Delete), "FORWARD"}, link...)
+			if output, err := Raw(del...); err != nil {
+				return err
+			} else if len(output) != 0 {
+				return fmt.Errorf("Could not delete linking rule from %s/%s: %s", c.Table, c.Name, output)
+			}
+
+		}
+	}
+	return nil
+}
+
+// RemoveExistingChain removes existing chain from the table.
+func RemoveExistingChain(name string, table Table) error {
+	c := &ChainInfo{
+		Name:  name,
+		Table: table,
+	}
+	if string(c.Table) == "" {
+		c.Table = Filter
+	}
+	return c.Remove()
+}
+
+// Forward adds forwarding rule to 'filter' table and corresponding nat rule to 'nat' table.
+func (c *ChainInfo) Forward(action Action, ip net.IP, port int, proto, destAddr string, destPort int, bridgeName string) error {
+	daddr := ip.String()
+	if ip.IsUnspecified() {
+		// iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we
+		// want "0.0.0.0/0". "0/0" is correctly interpreted as "any
+		// value" by both iptables and ip6tables.
+		daddr = "0/0"
+	}
+	args := []string{"-t", string(Nat), string(action), c.Name,
+		"-p", proto,
+		"-d", daddr,
+		"--dport", strconv.Itoa(port),
+		"-j", "DNAT",
+		"--to-destination", net.JoinHostPort(destAddr, strconv.Itoa(destPort))}
+	if !c.HairpinMode {
+		args = append(args, "!", "-i", bridgeName)
+	}
+	if output, err := Raw(args...); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return ChainError{Chain: "FORWARD", Output: output}
+	}
+
+	if output, err := Raw("-t", string(Filter), string(action), c.Name,
+		"!", "-i", bridgeName,
+		"-o", bridgeName,
+		"-p", proto,
+		"-d", destAddr,
+		"--dport", strconv.Itoa(destPort),
+		"-j", "ACCEPT"); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return ChainError{Chain: "FORWARD", Output: output}
+	}
+
+	if output, err := Raw("-t", string(Nat), string(action), "POSTROUTING",
+		"-p", proto,
+		"-s", destAddr,
+		"-d", destAddr,
+		"--dport", strconv.Itoa(destPort),
+		"-j", "MASQUERADE"); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return ChainError{Chain: "FORWARD", Output: output}
+	}
+
+	return nil
+}
+
+// Link adds reciprocal ACCEPT rule for two supplied IP addresses.
+// Traffic is allowed from ip1 to ip2 and vice-versa
+func (c *ChainInfo) Link(action Action, ip1, ip2 net.IP, port int, proto string, bridgeName string) error {
+	if output, err := Raw("-t", string(Filter), string(action), c.Name,
+		"-i", bridgeName, "-o", bridgeName,
+		"-p", proto,
+		"-s", ip1.String(),
+		"-d", ip2.String(),
+		"--dport", strconv.Itoa(port),
+		"-j", "ACCEPT"); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return fmt.Errorf("Error iptables forward: %s", output)
+	}
+	if output, err := Raw("-t", string(Filter), string(action), c.Name,
+		"-i", bridgeName, "-o", bridgeName,
+		"-p", proto,
+		"-s", ip2.String(),
+		"-d", ip1.String(),
+		"--sport", strconv.Itoa(port),
+		"-j", "ACCEPT"); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return fmt.Errorf("Error iptables forward: %s", output)
+	}
+	return nil
+}
+
+// Prerouting adds linking rule to nat/PREROUTING chain.
+func (c *ChainInfo) Prerouting(action Action, args ...string) error {
+	a := []string{"-t", string(Nat), string(action), "PREROUTING"}
+	if len(args) > 0 {
+		a = append(a, args...)
+	}
+	if output, err := Raw(a...); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return ChainError{Chain: "PREROUTING", Output: output}
+	}
+	return nil
+}
+
+// Output adds linking rule to an OUTPUT chain.
+func (c *ChainInfo) Output(action Action, args ...string) error {
+	a := []string{"-t", string(c.Table), string(action), "OUTPUT"}
+	if len(args) > 0 {
+		a = append(a, args...)
+	}
+	if output, err := Raw(a...); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return ChainError{Chain: "OUTPUT", Output: output}
+	}
+	return nil
+}
+
+// Remove removes the chain.
+func (c *ChainInfo) Remove() error {
+	// Ignore errors - This could mean the chains were never set up
+	if c.Table == Nat {
+		c.Prerouting(Delete, "-m", "addrtype", "--dst-type", "LOCAL", "-j", c.Name)
+		c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8", "-j", c.Name)
+		c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL", "-j", c.Name) // Created in versions <= 0.1.6
+
+		c.Prerouting(Delete)
+		c.Output(Delete)
+	}
+	Raw("-t", string(c.Table), "-F", c.Name)
+	Raw("-t", string(c.Table), "-X", c.Name)
+	return nil
+}
+
+// Exists checks if a rule exists
+func Exists(table Table, chain string, rule ...string) bool {
+	if string(table) == "" {
+		table = Filter
+	}
+
+	// iptables -C, --check option was added in v.1.4.11
+	// http://ftp.netfilter.org/pub/iptables/changes-iptables-1.4.11.txt
+
+	// try -C
+	// if exit status is 0 then return true, the rule exists
+	if _, err := Raw(append([]string{
+		"-t", string(table), "-C", chain}, rule...)...); err == nil {
+		return true
+	}
+
+	// parse "iptables -S" for the rule (this checks rules in a specific chain
+	// in a specific table)
+	ruleString := strings.Join(rule, " ")
+	ruleString = chain + " " + ruleString
+	existingRules, _ := exec.Command(iptablesPath, "-t", string(table), "-S", chain).Output()
+
+	return strings.Contains(string(existingRules), ruleString)
+}
+
+// Raw calls 'iptables' system command, passing supplied arguments.
+func Raw(args ...string) ([]byte, error) {
+	if firewalldRunning {
+		output, err := Passthrough(Iptables, args...)
+		if err == nil || !strings.Contains(err.Error(), "was not provided by any .service files") {
+			return output, err
+		}
+
+	}
+
+	if err := initCheck(); err != nil {
+		return nil, err
+	}
+	if supportsXlock {
+		args = append([]string{"--wait"}, args...)
+	} else {
+		bestEffortLock.Lock()
+		defer bestEffortLock.Unlock()
+	}
+
+	logrus.Debugf("%s, %v", iptablesPath, args)
+
+	output, err := exec.Command(iptablesPath, args...).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("iptables failed: iptables %v: %s (%s)", strings.Join(args, " "), output, err)
+	}
+
+	// ignore iptables' message about xtables lock
+	if strings.Contains(string(output), "waiting for it to exit") {
+		output = []byte("")
+	}
+
+	return output, err
+}
+
+// RawCombinedOutput inernally calls the Raw function and returns a non nil
+// error if Raw returned a non nil error or a non empty output
+func RawCombinedOutput(args ...string) error {
+	if output, err := Raw(args...); err != nil || len(output) != 0 {
+		return fmt.Errorf("%s (%v)", string(output), err)
+	}
+	return nil
+}
+
+// ExistChain checks if a chain exists
+func ExistChain(chain string, table Table) bool {
+	if _, err := Raw("-t", string(table), "-L", chain); err == nil {
+		return true
+	}
+	return false
+}

vendor/github.com/docker/libnetwork/iptables/iptables_test.go

@@ -0,0 +1,236 @@
+package iptables
+
+import (
+	"net"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	_ "github.com/docker/libnetwork/testutils"
+)
+
+const chainName = "DOCKEREST"
+
+var natChain *ChainInfo
+var filterChain *ChainInfo
+var bridgeName string
+
+func TestNewChain(t *testing.T) {
+	var err error
+
+	bridgeName = "lo"
+	natChain, err = NewChain(chainName, Nat, false)
+	err = ProgramChain(natChain, bridgeName, false, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	filterChain, err = NewChain(chainName, Filter, false)
+	err = ProgramChain(filterChain, bridgeName, false, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestForward(t *testing.T) {
+	ip := net.ParseIP("192.168.1.1")
+	port := 1234
+	dstAddr := "172.17.0.1"
+	dstPort := 4321
+	proto := "tcp"
+
+	bridgeName := "lo"
+	err := natChain.Forward(Insert, ip, port, proto, dstAddr, dstPort, bridgeName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dnatRule := []string{
+		"-d", ip.String(),
+		"-p", proto,
+		"--dport", strconv.Itoa(port),
+		"-j", "DNAT",
+		"--to-destination", dstAddr + ":" + strconv.Itoa(dstPort),
+		"!", "-i", bridgeName,
+	}
+
+	if !Exists(natChain.Table, natChain.Name, dnatRule...) {
+		t.Fatalf("DNAT rule does not exist")
+	}
+
+	filterRule := []string{
+		"!", "-i", bridgeName,
+		"-o", bridgeName,
+		"-d", dstAddr,
+		"-p", proto,
+		"--dport", strconv.Itoa(dstPort),
+		"-j", "ACCEPT",
+	}
+
+	if !Exists(filterChain.Table, filterChain.Name, filterRule...) {
+		t.Fatalf("filter rule does not exist")
+	}
+
+	masqRule := []string{
+		"-d", dstAddr,
+		"-s", dstAddr,
+		"-p", proto,
+		"--dport", strconv.Itoa(dstPort),
+		"-j", "MASQUERADE",
+	}
+
+	if !Exists(natChain.Table, "POSTROUTING", masqRule...) {
+		t.Fatalf("MASQUERADE rule does not exist")
+	}
+}
+
+func TestLink(t *testing.T) {
+	var err error
+
+	bridgeName := "lo"
+	ip1 := net.ParseIP("192.168.1.1")
+	ip2 := net.ParseIP("192.168.1.2")
+	port := 1234
+	proto := "tcp"
+
+	err = filterChain.Link(Append, ip1, ip2, port, proto, bridgeName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rule1 := []string{
+		"-i", bridgeName,
+		"-o", bridgeName,
+		"-p", proto,
+		"-s", ip1.String(),
+		"-d", ip2.String(),
+		"--dport", strconv.Itoa(port),
+		"-j", "ACCEPT"}
+
+	if !Exists(filterChain.Table, filterChain.Name, rule1...) {
+		t.Fatalf("rule1 does not exist")
+	}
+
+	rule2 := []string{
+		"-i", bridgeName,
+		"-o", bridgeName,
+		"-p", proto,
+		"-s", ip2.String(),
+		"-d", ip1.String(),
+		"--sport", strconv.Itoa(port),
+		"-j", "ACCEPT"}
+
+	if !Exists(filterChain.Table, filterChain.Name, rule2...) {
+		t.Fatalf("rule2 does not exist")
+	}
+}
+
+func TestPrerouting(t *testing.T) {
+	args := []string{
+		"-i", "lo",
+		"-d", "192.168.1.1"}
+
+	err := natChain.Prerouting(Insert, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !Exists(natChain.Table, "PREROUTING", args...) {
+		t.Fatalf("rule does not exist")
+	}
+
+	delRule := append([]string{"-D", "PREROUTING", "-t", string(Nat)}, args...)
+	if _, err = Raw(delRule...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestOutput(t *testing.T) {
+	args := []string{
+		"-o", "lo",
+		"-d", "192.168.1.1"}
+
+	err := natChain.Output(Insert, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !Exists(natChain.Table, "OUTPUT", args...) {
+		t.Fatalf("rule does not exist")
+	}
+
+	delRule := append([]string{"-D", "OUTPUT", "-t",
+		string(natChain.Table)}, args...)
+	if _, err = Raw(delRule...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestConcurrencyWithWait(t *testing.T) {
+	RunConcurrencyTest(t, true)
+}
+
+func TestConcurrencyNoWait(t *testing.T) {
+	RunConcurrencyTest(t, false)
+}
+
+// Runs 10 concurrent rule additions. This will fail if iptables
+// is actually invoked simultaneously without --wait.
+// Note that if iptables does not support the xtable lock on this
+// system, then allowXlock has no effect -- it will always be off.
+func RunConcurrencyTest(t *testing.T, allowXlock bool) {
+	var wg sync.WaitGroup
+
+	if !allowXlock && supportsXlock {
+		supportsXlock = false
+		defer func() { supportsXlock = true }()
+	}
+
+	ip := net.ParseIP("192.168.1.1")
+	port := 1234
+	dstAddr := "172.17.0.1"
+	dstPort := 4321
+	proto := "tcp"
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := natChain.Forward(Append, ip, port, proto, dstAddr, dstPort, "lo")
+			if err != nil {
+				t.Fatal(err)
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func TestCleanup(t *testing.T) {
+	var err error
+	var rules []byte
+
+	// Cleanup filter/FORWARD first otherwise output of iptables-save is dirty
+	link := []string{"-t", string(filterChain.Table),
+		string(Delete), "FORWARD",
+		"-o", bridgeName,
+		"-j", filterChain.Name}
+	if _, err = Raw(link...); err != nil {
+		t.Fatal(err)
+	}
+	filterChain.Remove()
+
+	err = RemoveExistingChain(chainName, Nat)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rules, err = exec.Command("iptables-save").Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(rules), chainName) {
+		t.Fatalf("Removing chain failed. %s found in iptables-save", chainName)
+	}
+}