From 589bf46e06f319e93632e32f5e13df4635abf0ef Mon Sep 17 00:00:00 2001 From: seborama Date: Thu, 24 May 2018 22:59:54 +0100 Subject: [PATCH] Added hyperkit options for enterprise VPN support The purpose of these changes is to enhance Hyperkit support from the minikube command line for better integration with enterprise networks behind a VPN. uuid: Provide VM UUID to restore MAC address (only supported with Hyperkit driver). vpnkitSock: Location of the VPNKit socket used for networking. If empty, disables Hyperkit VPNKitSock, if 'auto' uses Docker for Mac VPNKit connection, otherwise uses the specified VSock." vsockPorts: List of guest VSock ports that should be exposed as sockets on the host (Only supported on with hyperkit now). Note: tests pass but file: `vendor/github.com/google/certificate-transparency/go/x509/root_darwin.go` has to be edited to correct an issue - not committed since this is in the vendor directory. --- cmd/minikube/cmd/start.go | 10 ++- pkg/drivers/hyperkit/driver.go | 46 +++++++++++-- pkg/drivers/hyperkit/driver_test.go | 86 +++++++++++++++++++++++++ pkg/minikube/config/types.go | 2 + pkg/minikube/drivers/hyperkit/driver.go | 9 ++- 5 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 pkg/drivers/hyperkit/driver_test.go diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 6ed188f86523..5ffe4c668b54 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -70,6 +70,8 @@ const ( disableDriverMounts = "disable-driver-mounts" cacheImages = "cache-images" uuid = "uuid" + vpnkitSock = "hyperkit-vpnkit-sock" + vsockPorts = "hyperkit-vsock-ports" ) var ( @@ -135,6 +137,8 @@ func runStart(cmd *cobra.Command, args []string) { CPUs: viper.GetInt(cpus), DiskSize: diskSizeMB, VMDriver: viper.GetString(vmDriver), + HyperkitVpnKitSock: viper.GetString(vpnkitSock), + HyperkitVSockPorts: viper.GetStringSlice(vsockPorts), XhyveDiskDriver: viper.GetString(xhyveDiskDriver), NFSShare: viper.GetStringSlice(NFSShare), NFSSharesRoot: viper.GetString(NFSSharesRoot), @@ -321,8 +325,7 @@ func runStart(cmd *cobra.Command, args []string) { if viper.GetBool(cfg.WantNoneDriverWarning) { fmt.Println(`=================== WARNING: IT IS RECOMMENDED NOT TO RUN THE NONE DRIVER ON PERSONAL WORKSTATIONS - The 'none' driver will run an insecure kubernetes apiserver as root that may leave the host vulnerable to CSRF attacks -`) + The 'none' driver will run an insecure kubernetes apiserver as root that may leave the host vulnerable to CSRF attacks` + "\n") } if os.Getenv("CHANGE_MINIKUBE_NONE_USER") == "" { @@ -399,6 +402,9 @@ func init() { `A set of key=value pairs that describe configuration that may be passed to different components. The key should be '.' separated, and the first part before the dot is the component to apply the configuration to. Valid components are: kubelet, apiserver, controller-manager, etcd, proxy, scheduler.`) + startCmd.Flags().String(uuid, "", "Provide VM UUID to restore MAC address (only supported with Hyperkit driver).") + startCmd.Flags().String(vpnkitSock, "", "Location of the VPNKit socket used for networking. If empty, disables Hyperkit VPNKitSock, if 'auto' uses Docker for Mac VPNKit connection, otherwise uses the specified VSock.") + startCmd.Flags().StringSlice(vsockPorts, []string{}, "List of guest VSock ports that should be exposed as sockets on the host (Only supported on with hyperkit now).") viper.BindPFlags(startCmd.Flags()) RootCmd.AddCommand(startCmd) } diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go index 5fe4bceb811a..23abc304657a 100644 --- a/pkg/drivers/hyperkit/driver.go +++ b/pkg/drivers/hyperkit/driver.go @@ -25,6 +25,7 @@ import ( "os/user" "path" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -59,6 +60,8 @@ type Driver struct { NFSShares []string NFSSharesRoot string UUID string + VpnKitSock string + VSockPorts []string } func NewDriver(hostName, storePath string) *Driver { @@ -92,7 +95,7 @@ func (d *Driver) Create() error { isoPath := d.ResolveStorePath(isoFilename) if err := d.extractKernel(isoPath); err != nil { - return err + return errors.Wrap(err, "extracting kernel") } return d.Start() @@ -164,9 +167,9 @@ func (d *Driver) Restart() error { // Start a host func (d *Driver) Start() error { - h, err := hyperkit.New("", "", filepath.Join(d.StorePath, "machines", d.MachineName)) + h, err := hyperkit.New("", d.VpnKitSock, filepath.Join(d.StorePath, "machines", d.MachineName)) if err != nil { - return err + return errors.Wrap(err, "new-ing Hyperkit") } // TODO: handle the rest of our settings. @@ -179,10 +182,17 @@ func (d *Driver) Start() error { h.Memory = d.Memory h.UUID = d.UUID + if vsockPorts, err := d.extractVSockPorts(); err != nil { + return err + } else if len(vsockPorts) >= 1 { + h.VSock = true + h.VSockPorts = vsockPorts + } + log.Infof("Using UUID %s", h.UUID) mac, err := GetMACAddressFromUUID(h.UUID) if err != nil { - return err + return errors.Wrap(err, "getting MAC address from UUID") } // Need to strip 0's @@ -197,7 +207,7 @@ func (d *Driver) Start() error { } log.Infof("Starting with cmdline: %s", d.Cmdline) if err := h.Start(d.Cmdline); err != nil { - return err + return errors.Wrapf(err, "starting with cmd line: %s", d.Cmdline) } getIP := func() error { @@ -250,6 +260,32 @@ func (d *Driver) extractKernel(isoPath string) error { return nil } +// InvalidPortNumberError implements the Error interface. +// It is used when a VSockPorts port number cannot be recognised as an integer. +type InvalidPortNumberError string + +// Error returns an Error for InvalidPortNumberError +func (port InvalidPortNumberError) Error() string { + return fmt.Sprintf("vsock port '%s' is not an integer", string(port)) +} + +func (d *Driver) extractVSockPorts() ([]int, error) { + vsockPorts := make([]int, 0, len(d.VSockPorts)) + + for _, port := range d.VSockPorts { + p, err := strconv.Atoi(port) + if err != nil { + // return nil, errors.Wrapf(err, "vsock port '%s' is not an integer", port) + var err InvalidPortNumberError + err = InvalidPortNumberError(port) + return nil, err + } + vsockPorts = append(vsockPorts, p) + } + + return vsockPorts, nil +} + func (d *Driver) setupNFSShare() error { user, err := user.Current() if err != nil { diff --git a/pkg/drivers/hyperkit/driver_test.go b/pkg/drivers/hyperkit/driver_test.go new file mode 100644 index 000000000000..22d4ddbd39c0 --- /dev/null +++ b/pkg/drivers/hyperkit/driver_test.go @@ -0,0 +1,86 @@ +// +build darwin + +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hyperkit + +import ( + "testing" +) + +func Test_portExtraction(t *testing.T) { + tests := []struct { + name string + ports []string + want []int + wantErr error + }{ + { + "valid_empty", + []string{}, + []int{}, + nil, + }, + { + "valid_list", + []string{"10", "20", "30"}, + []int{10, 20, 30}, + nil, + }, + { + "invalid", + []string{"8080", "not_an_integer"}, + nil, + InvalidPortNumberError("not_an_integer"), + }, + } + + for _, tt := range tests { + d := NewDriver("", "") + d.VSockPorts = tt.ports + got, gotErr := d.extractVSockPorts() + if !testEq(got, tt.want) { + t.Errorf("extractVSockPorts() got: %v, want: %v", got, tt.want) + } + if gotErr != tt.wantErr { + t.Errorf("extractVSockPorts() gotErr: %s, wantErr: %s", gotErr.Error(), tt.wantErr.Error()) + } + } +} + +func testEq(a, b []int) bool { + + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 05238bb539e6..957454351d3a 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -35,6 +35,8 @@ type MachineConfig struct { CPUs int DiskSize int VMDriver string + HyperkitVpnKitSock string // Only used by the Hyperkit driver + HyperkitVSockPorts []string // Only used by the Hyperkit driver XhyveDiskDriver string // Only used by the xhyve driver DockerEnv []string // Each entry is formatted as KEY=VALUE. InsecureRegistry []string diff --git a/pkg/minikube/drivers/hyperkit/driver.go b/pkg/minikube/drivers/hyperkit/driver.go index e932296c9adf..4fe34962bd18 100644 --- a/pkg/minikube/drivers/hyperkit/driver.go +++ b/pkg/minikube/drivers/hyperkit/driver.go @@ -36,6 +36,11 @@ func init() { } func createHyperkitHost(config cfg.MachineConfig) interface{} { + uuID := config.UUID + if uuID == "" { + uuID = uuid.NewUUID().String() + } + return &hyperkit.Driver{ BaseDriver: &drivers.BaseDriver{ MachineName: cfg.GetMachineName(), @@ -48,7 +53,9 @@ func createHyperkitHost(config cfg.MachineConfig) interface{} { CPU: config.CPUs, NFSShares: config.NFSShare, NFSSharesRoot: config.NFSSharesRoot, - UUID: uuid.NewUUID().String(), + UUID: uuID, + VpnKitSock: config.HyperkitVpnKitSock, + VSockPorts: config.HyperkitVSockPorts, Cmdline: "loglevel=3 user=docker console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes base host=" + cfg.GetMachineName(), } }