virtual-kubelet/vendor/github.com/vmware/vic/cmd/tether/ops_linux.go

// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"path"
	"strconv"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/sys/unix"

	"github.com/vmware/vic/lib/config/executor"
	"github.com/vmware/vic/lib/constants"
	"github.com/vmware/vic/lib/iolog"
	"github.com/vmware/vic/lib/tether"
	"github.com/vmware/vic/lib/tether/netfilter"
	"github.com/vmware/vic/pkg/dio"
	"github.com/vmware/vic/pkg/trace"
)

const (
	runMountPoint = "/run"

	// default values to set for ulimit fields
	defaultNOFILE = 1024 * 1024
	defaultULimit = math.MaxUint64
)

type operations struct {
	tether.BaseOperations

	logging bool
}

func (t *operations) Log() (io.Writer, error) {
	defer trace.End(trace.Begin("operations.Log"))

	// redirect logging to the serial log
	log.Infof("opening %s/ttyS1 for debug log", pathPrefix)
	f, err := os.OpenFile(path.Join(pathPrefix, "ttyS1"), os.O_RDWR|os.O_SYNC|syscall.O_NOCTTY, 0)
	if err != nil {
		detail := fmt.Sprintf("failed to open serial port for debug log: %s", err)
		log.Error(detail)
		return nil, errors.New(detail)
	}

	if err := setTerminalSpeed(f.Fd()); err != nil {
		log.Errorf("Setting terminal speed failed with %s", err)
	}

	// enable raw mode
	_, err = terminal.MakeRaw(int(f.Fd()))
	if err != nil {
		detail := fmt.Sprintf("Making ttyS1 raw failed with %s", err)
		log.Error(detail)
		return nil, errors.New(detail)
	}

	return io.MultiWriter(f, os.Stdout), nil
}

// sessionLogWriter returns a writer that will persist the session output
func (t *operations) SessionLog(session *tether.SessionConfig) (dio.DynamicMultiWriter, dio.DynamicMultiWriter, error) {
	defer trace.End(trace.Begin("configure session log writer"))

	if t.logging {
		detail := "unable to log more than one session concurrently to persistent logging"
		log.Warn(detail)
		// use multi-writer so it's still viable for attach
		return dio.MultiWriter(), dio.MultiWriter(), nil
	}

	t.logging = true

	// open SttyS2 for session logging
	log.Info("opening ttyS2 for session logging")
	f, err := os.OpenFile(path.Join(pathPrefix, "ttyS2"), os.O_RDWR|os.O_SYNC|syscall.O_NOCTTY, 0)
	if err != nil {
		detail := fmt.Sprintf("failed to open serial port for session log: %s", err)
		log.Error(detail)
		return nil, nil, errors.New(detail)
	}

	if err := setTerminalSpeed(f.Fd()); err != nil {
		log.Errorf("Setting terminal speed failed with %s", err)
	}

	// enable raw mode
	_, err = terminal.MakeRaw(int(f.Fd()))
	if err != nil {
		detail := fmt.Sprintf("Making ttyS2 raw failed with %s", err)
		log.Error(detail)
		return nil, nil, errors.New(detail)
	}

	// wrap output in a LogWriter to serialize it into our persisted
	// containerVM output format, using iolog.LogClock for timestamps
	lw := iolog.NewLogWriter(f, iolog.LogClock{})

	// use multi-writer so it goes to both screen and session log
	return dio.MultiWriter(lw, os.Stdout), dio.MultiWriter(lw, os.Stderr), nil
}

func (t *operations) Setup(sink tether.Config) error {
	if err := t.BaseOperations.Setup(sink); err != nil {
		return err
	}

	// unmount /run - https://github.com/vmware/vic/issues/1643
	if err := tether.Sys.Syscall.Unmount(runMountPoint, syscall.MNT_DETACH); err != nil {
		if errno, ok := err.(syscall.Errno); !ok || errno != syscall.EINVAL {
			return err
		}
	}

	// NOTE: ulimit default values should change when we support ulimit configuration
	ApplyDefaultULimit()

	return nil
}

// ApplyDefaultULimit sets ulimit fields to their defined default value
func ApplyDefaultULimit() {
	var rLimit syscall.Rlimit

	// NOFILE does not support defaultULimit as a value due to kernel restriction on number of open files
	rLimit.Max = defaultNOFILE
	rLimit.Cur = rLimit.Max
	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
		log.Errorf("Cannot set ulimit for nofile: %s", err.Error())
	}

	rLimit.Max = defaultULimit
	rLimit.Cur = rLimit.Max
	if err := syscall.Setrlimit(syscall.RLIMIT_STACK, &rLimit); err != nil {
		log.Errorf("Cannot set ulimit for stack: %s ", err.Error())
	}

	if err := syscall.Setrlimit(syscall.RLIMIT_CORE, &rLimit); err != nil {
		log.Errorf("Cannot set ulimit for core blocks: %s", err.Error())
	}

	if err := syscall.Setrlimit(unix.RLIMIT_MEMLOCK, &rLimit); err != nil {
		log.Errorf("Cannot set ulimit for memlock: %s", err.Error())
	}

	if err := syscall.Setrlimit(unix.RLIMIT_NPROC, &rLimit); err != nil {
		log.Errorf("Cannot set ulimit for nproc: %s", err.Error())
	}
}

// invoke will invoke the closure returned from the tether netfilter prep and
// block until complete. It handles both potential preparation errors and invocation
// errors.
// the 'task' specified is used to construct error messages with the specific operation
// embedded
func invoke(t *tether.BaseOperations, fn tether.UtilityFn, task string) error {
	exitChan, err := t.LaunchUtility(fn)
	if err != nil {
		return fmt.Errorf("%s failed: %s", task, err)
	}

	exitCode := <-exitChan
	if exitCode != 0 {
		return fmt.Errorf("%s returned non-zero: %d", task, exitCode)
	}

	return nil
}

// SetupFirewall sets up firewall rules on the external scope only.  Any
// portmaps are honored as are port exposes.
func (t *operations) SetupFirewall(ctx context.Context, config *tether.ExecutorConfig) error {
	return setupFirewall(ctx, &t.BaseOperations, config)
}

// setupFirewall is broken out from SetupFirewall so that it can be referenced from the test code
func setupFirewall(ctx context.Context, t *tether.BaseOperations, config *tether.ExecutorConfig) error {
	fn := netfilter.Flush(ctx, "VIC")
	if err := invoke(t, fn, "flush"); err != nil {
		return err
	}

	for _, endpoint := range config.Networks {
		switch endpoint.Network.Type {
		case constants.ExternalScopeType:
			id, err := strconv.Atoi(endpoint.ID)
			if err != nil {
				log.Errorf("can't apply port rules: %s", err.Error())
				continue
			}

			iface, err := t.LinkBySlot(int32(id))
			if err != nil {
				log.Errorf("can't apply rules: %s", err.Error())
				continue
			}

			ifaceName := iface.Attrs().Name
			log.Debugf("slot %d -> %s", endpoint.ID, ifaceName)

			// ensure that we can pass DHCP traffic if it's necessary
			if endpoint.Network.TrustLevel != executor.Open && endpoint.IsDynamic() {
				allowDHCPTraffic(ctx, t, ifaceName)
			}

			switch endpoint.Network.TrustLevel {
			case executor.Open:
				// Configure port redirect in Open deployment
				if err := setupPorts(ctx, t, endpoint, ifaceName, true); err != nil {
					return err
				}

				// Accept all incoming and outgoing traffic
				for _, chain := range []netfilter.Chain{netfilter.Input, netfilter.Output, netfilter.Forward} {
					fn := (&netfilter.Rule{
						Chain:     chain,
						Target:    netfilter.Accept,
						Interface: ifaceName,
					}).Commit(ctx)
					if err := invoke(t, fn, "accept all"); err != nil {
						return err
					}
				}

			case executor.Closed:
				// Reject all incoming and outgoing traffic
				// Since our default policy is to drop traffic, nothing is needed here.

			case executor.Outbound:
				// Reject all incoming traffic, but allow outgoing
				if err := setupOutboundFirewall(ctx, t, ifaceName); err != nil {
					return err
				}

			case executor.Peers:
				// Outbound + all ports open to source addresses in --container-network-ip-range
				if err := setupOutboundFirewall(ctx, t, ifaceName); err != nil {
					return err
				}

				// Configure port redirect in Peer deployment
				if err := setupPorts(ctx, t, endpoint, ifaceName, true); err != nil {
					return err
				}

				sourceAddresses := make([]string, len(endpoint.Network.Pools))
				for i, v := range endpoint.Network.Pools {
					sourceAddresses[i] = v.String()
				}
				fn := (&netfilter.Rule{
					Chain:           netfilter.Input,
					Target:          netfilter.Accept,
					SourceAddresses: sourceAddresses,
					Interface:       ifaceName,
				}).Commit(ctx)
				if err := invoke(t, fn, "allow outbound and peers"); err != nil {
					return err
				}

				if err := allowPingTraffic(ctx, t, ifaceName, sourceAddresses); err != nil {
					return err
				}

			default:
				// covers executor.Published and executor.Unspecified as well as invalid values
				log.Infof("Applying published rules for configuration %v", endpoint.Network.TrustLevel)

				if err := setupOutboundFirewall(ctx, t, ifaceName); err != nil {
					return err
				}

				if err := allowPingTraffic(ctx, t, ifaceName, nil); err != nil {
					return err
				}

				if err := setupPorts(ctx, t, endpoint, ifaceName, false); err != nil {
					return err
				}
			}
		case constants.BridgeScopeType:
			id, err := strconv.Atoi(endpoint.ID)
			if err != nil {
				log.Errorf("can't apply port rules: %s", err.Error())
				continue
			}

			iface, err := t.LinkBySlot(int32(id))
			if err != nil {
				log.Errorf("can't apply rules: %s", err.Error())
				continue
			}

			ifaceName := iface.Attrs().Name
			log.Debugf("slot %d -> %s", endpoint.ID, ifaceName)

			// Traffic over container bridge network should be peers+outbound.
			if err := setupOutboundFirewall(ctx, t, ifaceName); err != nil {
				return err
			}
			sourceAddresses := make([]string, len(endpoint.Network.Pools))

			for i, v := range endpoint.Network.Pools {
				sourceAddresses[i] = v.String()
			}
			fn := (&netfilter.Rule{
				Chain:           netfilter.Input,
				Target:          netfilter.Accept,
				SourceAddresses: sourceAddresses,
				Interface:       ifaceName,
			}).Commit(ctx)
			if err := invoke(t, fn, "configure for bridge scope"); err != nil {
				return err
			}
		}
	}

	return invoke(t, netfilter.Return(ctx, "VIC"), "return from VIC chain")
}

func setupOutboundFirewall(ctx context.Context, t *tether.BaseOperations, ifaceName string) error {
	// All already established inputs are accepted
	fn := (&netfilter.Rule{
		Chain:     netfilter.Input,
		States:    []netfilter.State{netfilter.Established, netfilter.Related},
		Target:    netfilter.Accept,
		Interface: ifaceName,
	}).Commit(ctx)
	if err := invoke(t, fn, "permit established inbound"); err != nil {
		return err
	}

	// All output is accepted
	fn = (&netfilter.Rule{
		Chain:     netfilter.Output,
		Target:    netfilter.Accept,
		Interface: ifaceName,
	}).Commit(ctx)
	return invoke(t, fn, "permit all outbound")
}

func setupPorts(ctx context.Context, t *tether.BaseOperations, endpoint *tether.NetworkEndpoint, ifaceName string, redirectOnly bool) error {
	// handle the ports
	for _, p := range endpoint.Ports {
		// parse the port maps
		rules, err := portToRule(p, redirectOnly)
		if err != nil {
			log.Errorf("can't apply port rule (%s): %s", p, err.Error())
			continue
		}

		log.Infof("Applying %d rules for port %s", len(rules), p)
		for _, r := range rules {
			r.Interface = ifaceName
			fn := r.Commit(ctx)
			if err := invoke(t, fn, "allow incoming on published port"); err != nil {
				return err
			}
		}
	}

	return nil
}

func allowPingTraffic(ctx context.Context, t *tether.BaseOperations, ifaceName string, sourceAddresses []string) error {
	fn := (&netfilter.Rule{
		Chain:           netfilter.Input,
		Target:          netfilter.Accept,
		Interface:       ifaceName,
		Protocol:        netfilter.ICMP,
		ICMPType:        netfilter.EchoRequest,
		SourceAddresses: sourceAddresses,
	}).Commit(ctx)
	if err := invoke(t, fn, "allow ping inbound"); err != nil {
		return err
	}

	fn = (&netfilter.Rule{
		Chain:           netfilter.Output,
		Target:          netfilter.Accept,
		Interface:       ifaceName,
		Protocol:        netfilter.ICMP,
		ICMPType:        netfilter.EchoReply,
		SourceAddresses: sourceAddresses,
	}).Commit(ctx)
	return invoke(t, fn, "allow ping outbound")
}

func allowDHCPTraffic(ctx context.Context, t *tether.BaseOperations, ifaceName string) error {
	fn := (&netfilter.Rule{
		Chain:     netfilter.Input,
		Target:    netfilter.Accept,
		Interface: ifaceName,
		Protocol:  netfilter.UDP,
		FromPort:  "67:68",
		SrcPort:   "67:68",
	}).Commit(ctx)
	if err := invoke(t, fn, "allow dhcp inbound"); err != nil {
		return err
	}

	fn = (&netfilter.Rule{
		Chain:     netfilter.Output,
		Target:    netfilter.Accept,
		Interface: ifaceName,
		Protocol:  netfilter.UDP,
		FromPort:  "67:68",
		SrcPort:   "67:68",
	}).Commit(ctx)
	return invoke(t, fn, "allow dhcp outbound")
}

func portToRule(p string, redirectOnly bool) ([]*netfilter.Rule, error) {
	// 9999/tcp
	s := strings.Split(p, "/")
	if len(s) != 2 {
		return nil, errors.New("can't parse port spec: " + p)
	}

	proto := netfilter.Protocol(s[1])
	switch proto {
	case netfilter.UDP:
	case netfilter.TCP:
	default:
		return nil, errors.New("unknown protocol")
	}

	mapping := strings.Split(s[0], ":")
	directPort := mapping[len(mapping)-1]

	// publish the actual port directly
	var rules []*netfilter.Rule
	expose := &netfilter.Rule{
		Chain:     netfilter.Input,
		Interface: "external",
		Target:    netfilter.Accept,
		Protocol:  proto,
		// ranges are specified using a hyphen in docker
		FromPort: strings.Replace(directPort, "-", ":", -1),
	}

	if !redirectOnly {
		rules = append(rules, expose)
	}

	// if there's no redirection we're done
	if len(mapping) == 1 || mapping[0] == mapping[1] {
		return rules, nil
	}

	// redirect port
	// https://wiki.debian.org/Firewalls-local-port-redirection contains a useful reference

	if strings.Contains(s[0], "-") {
		return nil, errors.New("cannot port forward a range")
	}

	rules = append(rules, &netfilter.Rule{
		Table:     netfilter.Nat,
		Chain:     netfilter.Prerouting,
		Interface: "external",
		Target:    netfilter.Redirect,
		Protocol:  proto,
		FromPort:  mapping[0],
		ToPort:    mapping[1],
	})

	return rules, nil
}