Add support for external etcd cluster

.github/workflows/go.yml

@@ -149,6 +149,7 @@ jobs:
           - check-ctr
           - check-customports
           - check-dualstack
+          - check-externaletcd
           - check-hacontrolplane
           # exists in inttest/Makefile.variables but there's no matching suite:
           #- check-install

cmd/api/api.go

@@ -86,16 +86,17 @@ func (c *CmdOpts) startAPI() error {
 	c.KubeClient = kc
 	prefix := "/v1beta1"
 	router := mux.NewRouter()
+	storage := c.NodeConfig.Spec.Storage
 
-	if c.NodeConfig.Spec.Storage.Type == v1beta1.EtcdStorageType {
-		// Only mount the etcd handler if we're running on etcd storage
+	if storage.Type == v1beta1.EtcdStorageType && !storage.Etcd.IsExternalClusterUsed() {
+		// Only mount the etcd handler if we're running on internal etcd storage
 		// by default the mux will return 404 back which the caller should handle
 		router.Path(prefix + "/etcd/members").Methods("POST").Handler(
 			c.controllerHandler(c.etcdHandler()),
 		)
 	}
 
-	if c.NodeConfig.Spec.Storage.IsJoinable() {
+	if storage.IsJoinable() {
 		router.Path(prefix + "/ca").Methods("GET").Handler(
 			c.controllerHandler(c.caHandler()),
 		)
@@ -135,7 +136,7 @@ func (c *CmdOpts) etcdHandler() http.Handler {
 			return
 		}
 
-		etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir)
+		etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, nil)
 		if err != nil {
 			sendError(err, resp)
 			return

cmd/backup/backup.go

@@ -48,6 +48,9 @@ func NewBackupCmd() *cobra.Command {
 				return err
 			}
 			c.ClusterConfig = cfg
+			if c.ClusterConfig.Spec.Storage.Etcd.IsExternalClusterUsed() {
+				return fmt.Errorf("command 'k0s backup' does not support external etcd cluster")
+			}
 			return c.backup()
 		},
 		PreRunE: preRunValidateConfig,

cmd/etcd/etcd.go

@@ -30,7 +30,7 @@ func NewEtcdCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "etcd",
 		Short: "Manage etcd cluster",
-		PreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			c := CmdOpts(config.GetCmdOpts())
 			cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars)
 			if err != nil {
@@ -40,6 +40,9 @@ func NewEtcdCmd() *cobra.Command {
 			if c.ClusterConfig.Spec.Storage.Type != v1beta1.EtcdStorageType {
 				return fmt.Errorf("wrong storage type: %s", c.ClusterConfig.Spec.Storage.Type)
 			}
+			if c.ClusterConfig.Spec.Storage.Etcd.IsExternalClusterUsed() {
+				return fmt.Errorf("command 'k0s etcd' does not support external etcd cluster")
+			}
 			return nil
 		},
 	}

cmd/etcd/leave.go

@@ -38,6 +38,7 @@ func etcdLeaveCmd() *cobra.Command {
 				return err
 			}
 			c.ClusterConfig = cfg
+
 			ctx := context.Background()
 			if etcdPeerAddress == "" {
 				etcdPeerAddress = c.ClusterConfig.Spec.Storage.Etcd.PeerAddress
@@ -47,7 +48,7 @@ func etcdLeaveCmd() *cobra.Command {
 			}
 
 			peerURL := fmt.Sprintf("https://%s:2380", etcdPeerAddress)
-			etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir)
+			etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.ClusterConfig.Spec.Storage.Etcd)
 			if err != nil {
 				return fmt.Errorf("can't connect to the etcd: %v", err)
 			}

cmd/etcd/list.go

@@ -33,8 +33,14 @@ func etcdListCmd() *cobra.Command {
 		Short: "Returns etcd cluster members list",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			c := CmdOpts(config.GetCmdOpts())
+			cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars)
+			if err != nil {
+				return err
+			}
+			c.ClusterConfig = cfg
+
 			ctx := context.Background()
-			etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir)
+			etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.ClusterConfig.Spec.Storage.Etcd)
 			if err != nil {
 				return fmt.Errorf("can't list etcd cluster members: %v", err)
 			}

inttest/Makefile

@@ -10,6 +10,12 @@ bins = bin/sonobuoy
 
 include ../embedded-bins/Makefile.variables
 
+ifeq ($(ARCH),amd64)
+etcd_arch = amd64
+else
+etcd_arch = arm64
+endif
+
 .PHONY: all
 all: $(bins) .footloose-alpine.stamp
 
@@ -20,7 +26,7 @@ bin/sonobuoy: | bin
 	$(curl) $(sonobuoy_url) | tar -C bin/ -zxv $(notdir $@)
 
 .footloose-alpine.stamp: footloose-alpine/Dockerfile
-	docker build -t footloose-alpine -f $< $(dir $<)
+	docker build --build-arg ETCD_ARCH=$(etcd_arch) -t footloose-alpine -f $< $(dir $<)
 	touch $@
 
 check-network: bin/sonobuoy .footloose-alpine.stamp

inttest/Makefile.variables

@@ -7,6 +7,7 @@ smoketests := \
 	check-ctr \
 	check-customports \
 	check-dualstack \
+	check-externaletcd \
 	check-hacontrolplane \
 	check-kine \
 	check-metrics \

inttest/common/footloosesuite.go

@@ -54,6 +54,16 @@ import (
 
 var defaultK0sBinPath = "/usr/bin/k0s"
 
+const (
+	// DefaultTimeout defines the default timeout for triggering custom teardown functionality
+	DefaultTimeout = 9 * time.Minute // The default golang test timeout is 10mins
+
+	controllerNodeNameFormat = "controller%d"
+	workerNodeNameFormat     = "worker%d"
+	lbNodeNameFormat         = "lb%d"
+	etcdNodeNameFormat       = "etcd%d"
+)
+
 // FootlooseSuite defines all the common stuff we need to be able to run k0s testing on footloose
 type FootlooseSuite struct {
 	suite.Suite
@@ -66,6 +76,7 @@ type FootlooseSuite struct {
 	KonnectivityAdminPort int
 	KonnectivityAgentPort int
 	KubeAPIExternalPort   int
+	WithExternalEtcd      bool
 	WithLB                bool
 	WorkerCount           int
 	ControllerUmask       int
@@ -178,12 +189,12 @@ func (s *FootlooseSuite) waitForSSH() {
 
 // ControllerNode gets the node name of given controller index
 func (s *FootlooseSuite) ControllerNode(idx int) string {
-	return fmt.Sprintf(s.footlooseConfig.Machines[0].Spec.Name, idx)
+	return fmt.Sprintf(controllerNodeNameFormat, idx)
 }
 
 // WorkerNode gets the node name of given worker index
 func (s *FootlooseSuite) WorkerNode(idx int) string {
-	return fmt.Sprintf(s.footlooseConfig.Machines[1].Spec.Name, idx)
+	return fmt.Sprintf(workerNodeNameFormat, idx)
 }
 
 // LBNode gets the node of given LB index
@@ -192,7 +203,15 @@ func (s *FootlooseSuite) LBNode(idx int) string {
 		s.T().Log("Can't get Loadbalancer address because LB is not enabled for this suit")
 		s.T().FailNow()
 	}
-	return fmt.Sprintf(s.footlooseConfig.Machines[2].Spec.Name, idx)
+	return fmt.Sprintf(lbNodeNameFormat, idx)
+}
+
+func (s *FootlooseSuite) ExternalEtcd(idx int) string {
+	if !s.WithExternalEtcd {
+		s.T().Log("Can't get etcd address because it is not enabled for this suit")
+		s.T().FailNow()
+	}
+	return fmt.Sprintf(etcdNodeNameFormat, idx)
 }
 
 // TearDownSuite does the cleanup work, namely destroy the footloose boxes
@@ -854,7 +873,7 @@ func (s *FootlooseSuite) createConfig() config.Config {
 				Count: s.ControllerCount,
 				Spec: config.Machine{
 					Image:        "footloose-alpine",
-					Name:         "controller%d",
+					Name:         controllerNodeNameFormat,
 					Privileged:   true,
 					Volumes:      volumes,
 					PortMappings: portMaps,
@@ -864,7 +883,7 @@ func (s *FootlooseSuite) createConfig() config.Config {
 				Count: s.WorkerCount,
 				Spec: config.Machine{
 					Image:        "footloose-alpine",
-					Name:         "worker%d",
+					Name:         workerNodeNameFormat,
 					Privileged:   true,
 					Volumes:      volumes,
 					PortMappings: portMaps,
@@ -876,7 +895,7 @@ func (s *FootlooseSuite) createConfig() config.Config {
 	if s.WithLB {
 		cfg.Machines = append(cfg.Machines, config.MachineReplicas{
 			Spec: config.Machine{
-				Name:         "lb%d",
+				Name:         lbNodeNameFormat,
 				Image:        "footloose-alpine",
 				Privileged:   true,
 				Volumes:      volumes,
@@ -886,12 +905,22 @@ func (s *FootlooseSuite) createConfig() config.Config {
 			Count: 1,
 		})
 	}
+
+	if s.WithExternalEtcd {
+		cfg.Machines = append(cfg.Machines, config.MachineReplicas{
+			Spec: config.Machine{
+				Name:         etcdNodeNameFormat,
+				Image:        "footloose-alpine",
+				Privileged:   true,
+				PortMappings: []config.PortMapping{{ContainerPort: 22}},
+			},
+			Count: 1,
+		})
+	}
+
 	return cfg
 }
 
-// DefaultTimeout defines the default timeout for triggering custom teardown functionality
-const DefaultTimeout = 9 * time.Minute // The default golang test timeout is 10mins
-
 func getTestTimeout() time.Duration {
 	for _, a := range os.Args {
 		if strings.HasPrefix(a, "-test.timeout") {
@@ -908,19 +937,21 @@ func getTestTimeout() time.Duration {
 	return DefaultTimeout
 }
 
-// GetMainIPAddress returns controller ip address
+// GetControllerIPAddress returns controller ip address
 func (s *FootlooseSuite) GetControllerIPAddress(idx int) string {
-	ssh, err := s.SSH(s.ControllerNode(idx))
-	s.Require().NoError(err)
-	defer ssh.Disconnect()
-
-	ipAddress, err := ssh.ExecWithOutput("hostname -i")
-	s.Require().NoError(err)
-	return ipAddress
+	return s.getIPAddress(s.ControllerNode(idx))
 }
 
 func (s *FootlooseSuite) GetLBAddress() string {
-	ssh, err := s.SSH(s.LBNode(0))
+	return s.getIPAddress(s.LBNode(0))
+}
+
+func (s *FootlooseSuite) GetExternalEtcdIPAddress() string {
+	return s.getIPAddress(s.ExternalEtcd(0))
+}
+
+func (s *FootlooseSuite) getIPAddress(nodeName string) string {
+	ssh, err := s.SSH(nodeName)
 	s.Require().NoError(err)
 	defer ssh.Disconnect()
 

inttest/externaletcd/external_etcd_test.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2021 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package externaletcd
+
+import (
+	"context"
+	"fmt"
+	"github.com/avast/retry-go"
+	"github.com/k0sproject/k0s/inttest/common"
+	"github.com/stretchr/testify/suite"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"testing"
+)
+
+const k0sConfig = `
+spec:
+  storage:
+    type: etcd
+    etcd:
+      externalCluster:
+        endpoints:
+        - http://etcd0:2379
+        etcdPrefix: k0s-tenant
+`
+
+type ExternalEtcdSuite struct {
+	common.FootlooseSuite
+}
+
+func (s *ExternalEtcdSuite) TestK0sWithExternalEtcdCluster() {
+	s.T().Log("starting etcd")
+	err := retry.Do(func() error {
+		ssh, err := s.SSH(s.ExternalEtcd(0))
+		if err != nil {
+			return err
+		}
+		defer ssh.Disconnect()
+
+		_, err = ssh.ExecWithOutput(
+			"ETCD_ADVERTISE_CLIENT_URLS=\"http://etcd0:2379\" " +
+				"ETCD_LISTEN_CLIENT_URLS=\"http://0.0.0.0:2379\" " +
+				"/opt/etcd > /var/log/etcd.log 2>&1 &")
+		return err
+	})
+	s.Require().NoError(err)
+
+	s.T().Log("configuring k0s controller to resolve etcd0 hostname")
+	k0sControllerSSH, err := s.SSH(s.ControllerNode(0))
+	s.Require().NoError(err)
+	defer k0sControllerSSH.Disconnect()
+
+	_, err = k0sControllerSSH.ExecWithOutput(fmt.Sprintf("echo '%s etcd0' >> /etc/hosts", s.GetExternalEtcdIPAddress()))
+	s.Require().NoError(err)
+
+	s.T().Log("starting k0s controller and worker")
+	s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfig)
+	s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml"))
+	s.Require().NoError(s.RunWorkers())
+
+	kc, err := s.KubeClient(s.ControllerNode(0))
+	s.NoError(err)
+
+	err = s.WaitForNodeReady(s.WorkerNode(0), kc)
+	s.NoError(err)
+
+	pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{
+		Limit: 100,
+	})
+	s.NoError(err)
+
+	podCount := len(pods.Items)
+	s.T().Logf("found %d pods in kube-system", podCount)
+	s.Greater(podCount, 0, "expecting to see few pods in kube-system namespace")
+
+	s.T().Log("checking if etcd contains keys")
+	etcdSSH, err := s.SSH(s.ExternalEtcd(0))
+	s.Require().NoError(err)
+	defer etcdSSH.Disconnect()
+
+	output, err := etcdSSH.ExecWithOutput(
+		"ETCDCTL_API=3 /opt/etcdctl --endpoints=http://127.0.0.1:2379 get /k0s-tenant/services/specs/kube-system/kube-dns --keys-only")
+	s.Require().NoError(err)
+	s.Require().Contains(output, "/k0s-tenant/services/specs/kube-system/kube-dns")
+
+	etcdLeaveOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd leave --config=/tmp/k0s.yaml")
+	s.Require().Error(err)
+	s.Require().Contains(etcdLeaveOutput, "command 'k0s etcd' does not support external etcd cluster")
+
+	etcdListOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd member-list --config=/tmp/k0s.yaml")
+	s.Require().Error(err)
+	s.Require().Contains(etcdListOutput, "command 'k0s etcd' does not support external etcd cluster")
+
+	backupOutput, err := k0sControllerSSH.ExecWithOutput("k0s backup --config=/tmp/k0s.yaml")
+	s.Require().Error(err)
+	s.Require().Contains(backupOutput, "command 'k0s backup' does not support external etcd cluster")
+}
+
+func TestExternalEtcdSuite(t *testing.T) {
+	s := ExternalEtcdSuite{
+		common.FootlooseSuite{
+			ControllerCount:  1,
+			WorkerCount:      1,
+			WithExternalEtcd: true,
+		},
+	}
+	suite.Run(t, &s)
+}