From f390d4bf3053e79b5edf5268de2c81ec0a8bd1ee Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 19:29:34 -0300 Subject: [PATCH 1/6] kic on mac: add service cmd support --- cmd/minikube/cmd/service.go | 98 +++++++++++++++++------ pkg/minikube/tunnel/kic/service_tunnel.go | 78 ++++++++++++++++++ pkg/minikube/tunnel/kic/ssh_conn.go | 2 +- pkg/minikube/tunnel/kic/ssh_tunnel.go | 32 +++++++- 4 files changed, 185 insertions(+), 25 deletions(-) create mode 100644 pkg/minikube/tunnel/kic/service_tunnel.go diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 65e1f69297f8..b7ec05a6c39a 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -20,8 +20,13 @@ import ( "fmt" "net/url" "os" + "os/signal" + "path/filepath" "runtime" + "strconv" + "strings" "text/template" + "time" "github.com/golang/glog" "github.com/pkg/browser" @@ -32,9 +37,11 @@ import ( "k8s.io/minikube/pkg/minikube/config" pkg_config "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/service" + "k8s.io/minikube/pkg/minikube/tunnel/kic" ) const defaultServiceFormatTemplate = "http://{{.IP}}:{{.Port}}" @@ -85,34 +92,17 @@ var serviceCmd = &cobra.Command{ exit.WithError("Error getting config", err) } + if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { + startKicServiceTunnel(svc, cfg.Name) + return + } + urls, err := service.WaitForService(api, namespace, svc, serviceURLTemplate, serviceURLMode, https, wait, interval) if err != nil { exit.WithError("Error opening service", err) } - if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { - out.FailureT("Opening service in browser is not implemented yet for docker driver on Mac.\nThe following issue is tracking the in progress work:\nhttps://github.com/kubernetes/minikube/issues/6778") - exit.WithCodeT(exit.Unavailable, "Not yet implemented for docker driver on MacOS.") - } - - for _, u := range urls { - _, err := url.Parse(u) - if err != nil { - glog.Warningf("failed to parse url %q: %v (will not open)", u, err) - out.String(fmt.Sprintf("%s\n", u)) - continue - } - - if serviceURLMode { - out.String(fmt.Sprintf("%s\n", u)) - continue - } - - out.T(out.Celebrate, "Opening service {{.namespace_name}}/{{.service_name}} in default browser...", out.V{"namespace_name": namespace, "service_name": svc}) - if err := browser.OpenURL(u); err != nil { - exit.WithError(fmt.Sprintf("open url failed: %s", u), err) - } - } + openURLs(svc, urls) }, } @@ -126,3 +116,65 @@ func init() { serviceCmd.PersistentFlags().StringVar(&serviceURLFormat, "format", defaultServiceFormatTemplate, "Format to output service URL in. This format will be applied to each url individually and they will be printed one at a time.") } + +func startKicServiceTunnel(svc, configName string) { + ctrlC := make(chan os.Signal, 1) + signal.Notify(ctrlC, os.Interrupt) + + clientset, err := service.K8s.GetClientset(1 * time.Second) + if err != nil { + exit.WithError("error creating clientset", err) + } + + port, err := oci.HostPortBinding(oci.Docker, configName, 22) + if err != nil { + exit.WithError("error getting ssh port", err) + } + sshPort := strconv.Itoa(port) + sshKey := filepath.Join(localpath.MiniPath(), "machines", configName, "id_rsa") + + serviceTunnel := kic.NewServiceTunnel(sshPort, sshKey, clientset.CoreV1()) + urls, err := serviceTunnel.Start(svc, namespace) + if err != nil { + exit.WithError("error starting tunnel", err) + } + + // wait for tunnel to come up + time.Sleep(1 * time.Second) + + data := [][]string{{namespace, svc, "", strings.Join(urls, "\n")}} + service.PrintServiceList(os.Stdout, data) + + openURLs(svc, urls) + + <-ctrlC + + err = serviceTunnel.Stop() + if err != nil { + exit.WithError("error stopping tunnel", err) + } + + return +} + +func openURLs(svc string, urls []string) { + for _, u := range urls { + _, err := url.Parse(u) + if err != nil { + glog.Warningf("failed to parse url %q: %v (will not open)", u, err) + out.String(fmt.Sprintf("%s\n", u)) + continue + } + + if serviceURLMode { + out.String(fmt.Sprintf("%s\n", u)) + continue + } + + out.T(out.Celebrate, "Opening service {{.namespace_name}}/{{.service_name}} in default browser...", out.V{"namespace_name": namespace, "service_name": svc}) + if err := browser.OpenURL(u); err != nil { + exit.WithError(fmt.Sprintf("open url failed: %s", u), err) + } + } + +} diff --git a/pkg/minikube/tunnel/kic/service_tunnel.go b/pkg/minikube/tunnel/kic/service_tunnel.go new file mode 100644 index 000000000000..5cc6bff7ac9e --- /dev/null +++ b/pkg/minikube/tunnel/kic/service_tunnel.go @@ -0,0 +1,78 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kic + +import ( + "fmt" + + "github.com/golang/glog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + typed_core "k8s.io/client-go/kubernetes/typed/core/v1" +) + +// ServiceTunnel ... +type ServiceTunnel struct { + sshPort string + sshKey string + v1Core typed_core.CoreV1Interface + sshConn *sshConn +} + +// NewServiceTunnel ... +func NewServiceTunnel(sshPort, sshKey string, v1Core typed_core.CoreV1Interface) *ServiceTunnel { + return &ServiceTunnel{ + sshPort: sshPort, + sshKey: sshKey, + v1Core: v1Core, + } +} + +// Start ... +func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { + svc, err := t.v1Core.Services(namespace).Get(svcName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("error listing services: %v", err) + return nil, err + } + + t.sshConn = createSSHConn(svcName, t.sshPort, t.sshKey, svc) + + go func() { + err = t.sshConn.startAndWait() + if err != nil { + glog.Errorf("error starting ssh tunnel: %v", err) + } + }() + + urls := make([]string, 0, len(svc.Spec.Ports)) + for _, port := range svc.Spec.Ports { + urls = append(urls, fmt.Sprintf("http://127.0.0.1:%d", port.Port)) + } + + return urls, nil +} + +// Stop ... +func (t *ServiceTunnel) Stop() error { + err := t.sshConn.stop() + if err != nil { + glog.Errorf("error stopping ssh tunnel: %v", err) + return err + } + + return nil +} diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 53c7304a3c40..4c98f0836f9d 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -29,7 +29,7 @@ type sshConn struct { cmd *exec.Cmd } -func createSSHConn(name, sshPort, sshKey string, svc v1.Service) *sshConn { +func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { // extract sshArgs sshArgs := []string{ // TODO: document the options here diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index 3dca0a1d3460..65acaf325b2b 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -104,7 +104,7 @@ func (t *SSHTunnel) startConnection(svc v1.Service) { } // create new ssh conn - newSSHConn := createSSHConn(uniqName, t.sshPort, t.sshKey, svc) + newSSHConn := createSSHConn(uniqName, t.sshPort, t.sshKey, &svc) t.conns[newSSHConn.name] = newSSHConn go func() { @@ -147,3 +147,33 @@ func sshConnUniqName(service v1.Service) string { return strings.Join(n, "") } + +// StartServiceTunnel ... +func (t *SSHTunnel) StartServiceTunnel(svcName string) error { + svc, err := t.v1Core.Services("default").Get(svcName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("error listing services: %v", err) + } + + newSSHConn := createSSHConn(svcName, t.sshPort, t.sshKey, svc) + + go func() { + err := newSSHConn.startAndWait() + if err != nil { + glog.Errorf("error starting ssh tunnel: %v", err) + } + }() + + //err = t.LoadBalancerEmulator.PatchServiceIP(t.v1Core.RESTClient(), svc, "127.0.0.1") + //if err != nil { + // glog.Errorf("error patching service: %v", err) + //} + + <-t.ctx.Done() + _, err = t.LoadBalancerEmulator.Cleanup() + if err != nil { + glog.Errorf("error cleaning up: %v", err) + } + + return err +} From 5f73506ff6a09eb21e36de91af8f4d620f77ddf6 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 20:51:36 -0300 Subject: [PATCH 2/6] Remove old code --- pkg/minikube/tunnel/kic/ssh_tunnel.go | 30 --------------------------- 1 file changed, 30 deletions(-) diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index 65acaf325b2b..bc5fd53a4fbc 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -147,33 +147,3 @@ func sshConnUniqName(service v1.Service) string { return strings.Join(n, "") } - -// StartServiceTunnel ... -func (t *SSHTunnel) StartServiceTunnel(svcName string) error { - svc, err := t.v1Core.Services("default").Get(svcName, metav1.GetOptions{}) - if err != nil { - glog.Errorf("error listing services: %v", err) - } - - newSSHConn := createSSHConn(svcName, t.sshPort, t.sshKey, svc) - - go func() { - err := newSSHConn.startAndWait() - if err != nil { - glog.Errorf("error starting ssh tunnel: %v", err) - } - }() - - //err = t.LoadBalancerEmulator.PatchServiceIP(t.v1Core.RESTClient(), svc, "127.0.0.1") - //if err != nil { - // glog.Errorf("error patching service: %v", err) - //} - - <-t.ctx.Done() - _, err = t.LoadBalancerEmulator.Cleanup() - if err != nil { - glog.Errorf("error cleaning up: %v", err) - } - - return err -} From 1752d7c004f225ac9c4e0167c7a452e1e9a07006 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:09:48 -0300 Subject: [PATCH 3/6] kic on mac: add random port support to service cmd --- pkg/minikube/tunnel/kic/service_tunnel.go | 10 ++++-- pkg/minikube/tunnel/kic/ssh_conn.go | 44 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/tunnel/kic/service_tunnel.go b/pkg/minikube/tunnel/kic/service_tunnel.go index 5cc6bff7ac9e..6f4222f1b060 100644 --- a/pkg/minikube/tunnel/kic/service_tunnel.go +++ b/pkg/minikube/tunnel/kic/service_tunnel.go @@ -49,7 +49,11 @@ func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { return nil, err } - t.sshConn = createSSHConn(svcName, t.sshPort, t.sshKey, svc) + t.sshConn, err = createSSHConnWithRandomPorts(svcName, t.sshPort, t.sshKey, svc) + if err != nil { + glog.Errorf("error creating ssh conn: %v", err) + return nil, err + } go func() { err = t.sshConn.startAndWait() @@ -59,8 +63,8 @@ func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { }() urls := make([]string, 0, len(svc.Spec.Ports)) - for _, port := range svc.Spec.Ports { - urls = append(urls, fmt.Sprintf("http://127.0.0.1:%d", port.Port)) + for _, port := range t.sshConn.ports { + urls = append(urls, fmt.Sprintf("http://127.0.0.1:%d", port)) } return urls, nil diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 4c98f0836f9d..d619a44bc508 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -20,6 +20,8 @@ import ( "fmt" "os/exec" + "github.com/phayes/freeport" + v1 "k8s.io/api/core/v1" ) @@ -27,6 +29,7 @@ type sshConn struct { name string service string cmd *exec.Cmd + ports []int } func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { @@ -61,6 +64,47 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { } } +func createSSHConnWithRandomPorts(name, sshPort, sshKey string, svc *v1.Service) (*sshConn, error) { + // extract sshArgs + sshArgs := []string{ + // TODO: document the options here + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking no", + "-N", + "docker@127.0.0.1", + "-p", sshPort, + "-i", sshKey, + } + + usedPorts := make([]int, 0, len(svc.Spec.Ports)) + + for _, port := range svc.Spec.Ports { + freeport, err := freeport.GetFreePort() + if err != nil { + return nil, err + } + + arg := fmt.Sprintf( + "-L %d:%s:%d", + freeport, + svc.Spec.ClusterIP, + port.Port, + ) + + sshArgs = append(sshArgs, arg) + usedPorts = append(usedPorts, freeport) + } + + cmd := exec.Command("ssh", sshArgs...) + + return &sshConn{ + name: name, + service: svc.Name, + cmd: cmd, + ports: usedPorts, + }, nil +} + func (c *sshConn) startAndWait() error { fmt.Printf("starting tunnel for %s\n", c.service) err := c.cmd.Start() From 5d8f2541f6093b87a501e81fe43093917a165934 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:16:50 -0300 Subject: [PATCH 4/6] kic on mac: add service warn --- cmd/minikube/cmd/service.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index b7ec05a6c39a..357da1682735 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -175,6 +175,8 @@ func openURLs(svc string, urls []string) { if err := browser.OpenURL(u); err != nil { exit.WithError(fmt.Sprintf("open url failed: %s", u), err) } + + out.T(out.WarningType, "Because you are using docker driver on Mac, the terminal needs to be open to run it.") } } From 7a6ff5838469b5deebd48db3e43f924073dcefad Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:21:44 -0300 Subject: [PATCH 5/6] kic on mac: improve error reporting service tunnel --- pkg/minikube/tunnel/kic/service_tunnel.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/tunnel/kic/service_tunnel.go b/pkg/minikube/tunnel/kic/service_tunnel.go index 6f4222f1b060..e1be04d97d04 100644 --- a/pkg/minikube/tunnel/kic/service_tunnel.go +++ b/pkg/minikube/tunnel/kic/service_tunnel.go @@ -20,6 +20,8 @@ import ( "fmt" "github.com/golang/glog" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" typed_core "k8s.io/client-go/kubernetes/typed/core/v1" ) @@ -45,14 +47,12 @@ func NewServiceTunnel(sshPort, sshKey string, v1Core typed_core.CoreV1Interface) func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { svc, err := t.v1Core.Services(namespace).Get(svcName, metav1.GetOptions{}) if err != nil { - glog.Errorf("error listing services: %v", err) - return nil, err + return nil, errors.Wrap(err, "getting service") } t.sshConn, err = createSSHConnWithRandomPorts(svcName, t.sshPort, t.sshKey, svc) if err != nil { - glog.Errorf("error creating ssh conn: %v", err) - return nil, err + return nil, errors.Wrap(err, "creating ssh conn") } go func() { @@ -74,8 +74,7 @@ func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { func (t *ServiceTunnel) Stop() error { err := t.sshConn.stop() if err != nil { - glog.Errorf("error stopping ssh tunnel: %v", err) - return err + return errors.Wrap(err, "stopping ssh tunnel") } return nil From 5e30d2f8c93de2c9d54ab79c736e72d4ac616eb1 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:29:28 -0300 Subject: [PATCH 6/6] Fix linter --- cmd/minikube/cmd/service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 357da1682735..de0205fbcc2c 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -153,8 +153,6 @@ func startKicServiceTunnel(svc, configName string) { if err != nil { exit.WithError("error stopping tunnel", err) } - - return } func openURLs(svc string, urls []string) {