diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index fd395d15decf..ce64ab9fa312 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -149,6 +149,7 @@ jobs: - check-ctr - check-customports - check-dualstack + - check-externaletcd - check-hacontrolplane # exists in inttest/Makefile.variables but there's no matching suite: #- check-install diff --git a/cmd/api/api.go b/cmd/api/api.go index 571a3780dbea..e9b31b97fc27 100644 --- a/cmd/api/api.go +++ b/cmd/api/api.go @@ -86,16 +86,17 @@ func (c *CmdOpts) startAPI() error { c.KubeClient = kc prefix := "/v1beta1" router := mux.NewRouter() + storage := c.NodeConfig.Spec.Storage - if c.NodeConfig.Spec.Storage.Type == v1beta1.EtcdStorageType { - // Only mount the etcd handler if we're running on etcd storage + if storage.Type == v1beta1.EtcdStorageType && !storage.Etcd.IsExternalClusterUsed() { + // Only mount the etcd handler if we're running on internal etcd storage // by default the mux will return 404 back which the caller should handle router.Path(prefix + "/etcd/members").Methods("POST").Handler( c.controllerHandler(c.etcdHandler()), ) } - if c.NodeConfig.Spec.Storage.IsJoinable() { + if storage.IsJoinable() { router.Path(prefix + "/ca").Methods("GET").Handler( c.controllerHandler(c.caHandler()), ) @@ -135,7 +136,7 @@ func (c *CmdOpts) etcdHandler() http.Handler { return } - etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir) + etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, nil) if err != nil { sendError(err, resp) return diff --git a/cmd/backup/backup.go b/cmd/backup/backup.go index b68b38d8a9fb..43b2b3e01e17 100644 --- a/cmd/backup/backup.go +++ b/cmd/backup/backup.go @@ -48,6 +48,9 @@ func NewBackupCmd() *cobra.Command { return err } c.ClusterConfig = cfg + if c.ClusterConfig.Spec.Storage.Etcd.IsExternalClusterUsed() { + return fmt.Errorf("command 'k0s backup' does not support external etcd cluster") + } return c.backup() }, PreRunE: preRunValidateConfig, diff --git a/cmd/etcd/etcd.go b/cmd/etcd/etcd.go index e12fa156ea7a..206971829be8 100644 --- a/cmd/etcd/etcd.go +++ b/cmd/etcd/etcd.go @@ -30,7 +30,7 @@ func NewEtcdCmd() *cobra.Command { cmd := &cobra.Command{ Use: "etcd", Short: "Manage etcd cluster", - PreRunE: func(cmd *cobra.Command, args []string) error { + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) if err != nil { @@ -40,6 +40,9 @@ func NewEtcdCmd() *cobra.Command { if c.ClusterConfig.Spec.Storage.Type != v1beta1.EtcdStorageType { return fmt.Errorf("wrong storage type: %s", c.ClusterConfig.Spec.Storage.Type) } + if c.ClusterConfig.Spec.Storage.Etcd.IsExternalClusterUsed() { + return fmt.Errorf("command 'k0s etcd' does not support external etcd cluster") + } return nil }, } diff --git a/cmd/etcd/leave.go b/cmd/etcd/leave.go index 4e90033b204a..5dd243dbf52d 100644 --- a/cmd/etcd/leave.go +++ b/cmd/etcd/leave.go @@ -38,6 +38,7 @@ func etcdLeaveCmd() *cobra.Command { return err } c.ClusterConfig = cfg + ctx := context.Background() if etcdPeerAddress == "" { etcdPeerAddress = c.ClusterConfig.Spec.Storage.Etcd.PeerAddress @@ -47,7 +48,7 @@ func etcdLeaveCmd() *cobra.Command { } peerURL := fmt.Sprintf("https://%s:2380", etcdPeerAddress) - etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir) + etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.ClusterConfig.Spec.Storage.Etcd) if err != nil { return fmt.Errorf("can't connect to the etcd: %v", err) } diff --git a/cmd/etcd/list.go b/cmd/etcd/list.go index d66c387d7756..7cdd66a2db92 100644 --- a/cmd/etcd/list.go +++ b/cmd/etcd/list.go @@ -33,8 +33,14 @@ func etcdListCmd() *cobra.Command { Short: "Returns etcd cluster members list", RunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) + cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) + if err != nil { + return err + } + c.ClusterConfig = cfg + ctx := context.Background() - etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir) + etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.ClusterConfig.Spec.Storage.Etcd) if err != nil { return fmt.Errorf("can't list etcd cluster members: %v", err) } diff --git a/inttest/Makefile b/inttest/Makefile index 448d81300a3d..3a3a7120e042 100644 --- a/inttest/Makefile +++ b/inttest/Makefile @@ -10,6 +10,12 @@ bins = bin/sonobuoy include ../embedded-bins/Makefile.variables +ifeq ($(ARCH),amd64) +etcd_arch = amd64 +else +etcd_arch = arm64 +endif + .PHONY: all all: $(bins) .footloose-alpine.stamp @@ -20,7 +26,7 @@ bin/sonobuoy: | bin $(curl) $(sonobuoy_url) | tar -C bin/ -zxv $(notdir $@) .footloose-alpine.stamp: footloose-alpine/Dockerfile - docker build -t footloose-alpine -f $< $(dir $<) + docker build --build-arg ETCD_ARCH=$(etcd_arch) -t footloose-alpine -f $< $(dir $<) touch $@ check-network: bin/sonobuoy .footloose-alpine.stamp diff --git a/inttest/Makefile.variables b/inttest/Makefile.variables index 2d884d64cf34..eaad3884db86 100644 --- a/inttest/Makefile.variables +++ b/inttest/Makefile.variables @@ -7,6 +7,7 @@ smoketests := \ check-ctr \ check-customports \ check-dualstack \ + check-externaletcd \ check-hacontrolplane \ check-kine \ check-metrics \ diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index fe0905c42311..ddc8cba577fe 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -54,6 +54,16 @@ import ( var defaultK0sBinPath = "/usr/bin/k0s" +const ( + // DefaultTimeout defines the default timeout for triggering custom teardown functionality + DefaultTimeout = 9 * time.Minute // The default golang test timeout is 10mins + + controllerNodeNameFormat = "controller%d" + workerNodeNameFormat = "worker%d" + lbNodeNameFormat = "lb%d" + etcdNodeNameFormat = "etcd%d" +) + // FootlooseSuite defines all the common stuff we need to be able to run k0s testing on footloose type FootlooseSuite struct { suite.Suite @@ -66,6 +76,7 @@ type FootlooseSuite struct { KonnectivityAdminPort int KonnectivityAgentPort int KubeAPIExternalPort int + WithExternalEtcd bool WithLB bool WorkerCount int ControllerUmask int @@ -178,12 +189,12 @@ func (s *FootlooseSuite) waitForSSH() { // ControllerNode gets the node name of given controller index func (s *FootlooseSuite) ControllerNode(idx int) string { - return fmt.Sprintf(s.footlooseConfig.Machines[0].Spec.Name, idx) + return fmt.Sprintf(controllerNodeNameFormat, idx) } // WorkerNode gets the node name of given worker index func (s *FootlooseSuite) WorkerNode(idx int) string { - return fmt.Sprintf(s.footlooseConfig.Machines[1].Spec.Name, idx) + return fmt.Sprintf(workerNodeNameFormat, idx) } // LBNode gets the node of given LB index @@ -192,7 +203,15 @@ func (s *FootlooseSuite) LBNode(idx int) string { s.T().Log("Can't get Loadbalancer address because LB is not enabled for this suit") s.T().FailNow() } - return fmt.Sprintf(s.footlooseConfig.Machines[2].Spec.Name, idx) + return fmt.Sprintf(lbNodeNameFormat, idx) +} + +func (s *FootlooseSuite) ExternalEtcd(idx int) string { + if !s.WithExternalEtcd { + s.T().Log("Can't get etcd address because it is not enabled for this suit") + s.T().FailNow() + } + return fmt.Sprintf(etcdNodeNameFormat, idx) } // TearDownSuite does the cleanup work, namely destroy the footloose boxes @@ -854,7 +873,7 @@ func (s *FootlooseSuite) createConfig() config.Config { Count: s.ControllerCount, Spec: config.Machine{ Image: "footloose-alpine", - Name: "controller%d", + Name: controllerNodeNameFormat, Privileged: true, Volumes: volumes, PortMappings: portMaps, @@ -864,7 +883,7 @@ func (s *FootlooseSuite) createConfig() config.Config { Count: s.WorkerCount, Spec: config.Machine{ Image: "footloose-alpine", - Name: "worker%d", + Name: workerNodeNameFormat, Privileged: true, Volumes: volumes, PortMappings: portMaps, @@ -876,7 +895,7 @@ func (s *FootlooseSuite) createConfig() config.Config { if s.WithLB { cfg.Machines = append(cfg.Machines, config.MachineReplicas{ Spec: config.Machine{ - Name: "lb%d", + Name: lbNodeNameFormat, Image: "footloose-alpine", Privileged: true, Volumes: volumes, @@ -886,12 +905,22 @@ func (s *FootlooseSuite) createConfig() config.Config { Count: 1, }) } + + if s.WithExternalEtcd { + cfg.Machines = append(cfg.Machines, config.MachineReplicas{ + Spec: config.Machine{ + Name: etcdNodeNameFormat, + Image: "footloose-alpine", + Privileged: true, + PortMappings: []config.PortMapping{{ContainerPort: 22}}, + }, + Count: 1, + }) + } + return cfg } -// DefaultTimeout defines the default timeout for triggering custom teardown functionality -const DefaultTimeout = 9 * time.Minute // The default golang test timeout is 10mins - func getTestTimeout() time.Duration { for _, a := range os.Args { if strings.HasPrefix(a, "-test.timeout") { @@ -908,19 +937,21 @@ func getTestTimeout() time.Duration { return DefaultTimeout } -// GetMainIPAddress returns controller ip address +// GetControllerIPAddress returns controller ip address func (s *FootlooseSuite) GetControllerIPAddress(idx int) string { - ssh, err := s.SSH(s.ControllerNode(idx)) - s.Require().NoError(err) - defer ssh.Disconnect() - - ipAddress, err := ssh.ExecWithOutput("hostname -i") - s.Require().NoError(err) - return ipAddress + return s.getIPAddress(s.ControllerNode(idx)) } func (s *FootlooseSuite) GetLBAddress() string { - ssh, err := s.SSH(s.LBNode(0)) + return s.getIPAddress(s.LBNode(0)) +} + +func (s *FootlooseSuite) GetExternalEtcdIPAddress() string { + return s.getIPAddress(s.ExternalEtcd(0)) +} + +func (s *FootlooseSuite) getIPAddress(nodeName string) string { + ssh, err := s.SSH(nodeName) s.Require().NoError(err) defer ssh.Disconnect() diff --git a/inttest/externaletcd/external_etcd_test.go b/inttest/externaletcd/external_etcd_test.go new file mode 100644 index 000000000000..90ca93b9e6ba --- /dev/null +++ b/inttest/externaletcd/external_etcd_test.go @@ -0,0 +1,120 @@ +/* +Copyright 2021 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package externaletcd + +import ( + "context" + "fmt" + "github.com/avast/retry-go" + "github.com/k0sproject/k0s/inttest/common" + "github.com/stretchr/testify/suite" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +const k0sConfig = ` +spec: + storage: + type: etcd + etcd: + externalCluster: + endpoints: + - http://etcd0:2379 + etcdPrefix: k0s-tenant +` + +type ExternalEtcdSuite struct { + common.FootlooseSuite +} + +func (s *ExternalEtcdSuite) TestK0sWithExternalEtcdCluster() { + s.T().Log("starting etcd") + err := retry.Do(func() error { + ssh, err := s.SSH(s.ExternalEtcd(0)) + if err != nil { + return err + } + defer ssh.Disconnect() + + _, err = ssh.ExecWithOutput( + "ETCD_ADVERTISE_CLIENT_URLS=\"http://etcd0:2379\" " + + "ETCD_LISTEN_CLIENT_URLS=\"http://0.0.0.0:2379\" " + + "/opt/etcd > /var/log/etcd.log 2>&1 &") + return err + }) + s.Require().NoError(err) + + s.T().Log("configuring k0s controller to resolve etcd0 hostname") + k0sControllerSSH, err := s.SSH(s.ControllerNode(0)) + s.Require().NoError(err) + defer k0sControllerSSH.Disconnect() + + _, err = k0sControllerSSH.ExecWithOutput(fmt.Sprintf("echo '%s etcd0' >> /etc/hosts", s.GetExternalEtcdIPAddress())) + s.Require().NoError(err) + + s.T().Log("starting k0s controller and worker") + s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfig) + s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) + s.Require().NoError(s.RunWorkers()) + + kc, err := s.KubeClient(s.ControllerNode(0)) + s.NoError(err) + + err = s.WaitForNodeReady(s.WorkerNode(0), kc) + s.NoError(err) + + pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{ + Limit: 100, + }) + s.NoError(err) + + podCount := len(pods.Items) + s.T().Logf("found %d pods in kube-system", podCount) + s.Greater(podCount, 0, "expecting to see few pods in kube-system namespace") + + s.T().Log("checking if etcd contains keys") + etcdSSH, err := s.SSH(s.ExternalEtcd(0)) + s.Require().NoError(err) + defer etcdSSH.Disconnect() + + output, err := etcdSSH.ExecWithOutput( + "ETCDCTL_API=3 /opt/etcdctl --endpoints=http://127.0.0.1:2379 get /k0s-tenant/services/specs/kube-system/kube-dns --keys-only") + s.Require().NoError(err) + s.Require().Contains(output, "/k0s-tenant/services/specs/kube-system/kube-dns") + + etcdLeaveOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd leave --config=/tmp/k0s.yaml") + s.Require().Error(err) + s.Require().Contains(etcdLeaveOutput, "command 'k0s etcd' does not support external etcd cluster") + + etcdListOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd member-list --config=/tmp/k0s.yaml") + s.Require().Error(err) + s.Require().Contains(etcdListOutput, "command 'k0s etcd' does not support external etcd cluster") + + backupOutput, err := k0sControllerSSH.ExecWithOutput("k0s backup --config=/tmp/k0s.yaml") + s.Require().Error(err) + s.Require().Contains(backupOutput, "command 'k0s backup' does not support external etcd cluster") +} + +func TestExternalEtcdSuite(t *testing.T) { + s := ExternalEtcdSuite{ + common.FootlooseSuite{ + ControllerCount: 1, + WorkerCount: 1, + WithExternalEtcd: true, + }, + } + suite.Run(t, &s) +} diff --git a/inttest/footloose-alpine/Dockerfile b/inttest/footloose-alpine/Dockerfile index 5f09833d16d1..7fe67dcf1d59 100644 --- a/inttest/footloose-alpine/Dockerfile +++ b/inttest/footloose-alpine/Dockerfile @@ -1,5 +1,6 @@ FROM alpine:3.13 +ARG ETCD_ARCH ENV KUBE_VERSION=1.21.3 RUN apk add openrc openssh-server bash busybox-initscripts coreutils findutils curl haproxy @@ -21,5 +22,10 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v$KUBE_VE && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl ENV KUBECONFIG=/var/lib/k0s/pki/admin.conf + +# Install etcd for smoke tests with external etcd +RUN wget https://github.com/etcd-io/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-$ETCD_ARCH.tar.gz +RUN tar -xvf etcd-v3.5.1-linux-$ETCD_ARCH.tar.gz -C /opt --strip-components=1 + # This lets etcd start when running on arm in smokes ENV ETCD_UNSUPPORTED_ARCH=arm64 diff --git a/pkg/apis/k0s.k0sproject.io/v1beta1/storage.go b/pkg/apis/k0s.k0sproject.io/v1beta1/storage.go index ad128b0df3e8..a2c97002ebd5 100644 --- a/pkg/apis/k0s.k0sproject.io/v1beta1/storage.go +++ b/pkg/apis/k0s.k0sproject.io/v1beta1/storage.go @@ -17,6 +17,9 @@ package v1beta1 import ( "encoding/json" + "fmt" + "k8s.io/utils/strings/slices" + "path/filepath" "strings" "github.com/sirupsen/logrus" @@ -59,7 +62,7 @@ func DefaultStorageSpec() *StorageSpec { // IsJoinable returns true only if the storage config is such that another controller can join the cluster func (s *StorageSpec) IsJoinable() bool { if s.Type == EtcdStorageType { - return true + return !s.Etcd.IsExternalClusterUsed() } if strings.HasPrefix(s.Kine.DataSource, "sqlite://") { @@ -97,15 +100,43 @@ func (s *StorageSpec) UnmarshalJSON(data []byte) error { // Validate validates storage specs correctness func (s *StorageSpec) Validate() []error { - return nil + var errors []error + + if s.Etcd != nil && s.Etcd.ExternalCluster != nil { + errors = append(errors, validateRequiredProperties(s.Etcd.ExternalCluster)...) + errors = append(errors, validateOptionalTLSProperties(s.Etcd.ExternalCluster)...) + } + + return errors } // EtcdConfig defines etcd related config options type EtcdConfig struct { + // ExternalCluster defines external etcd cluster related config options + ExternalCluster *ExternalCluster `json:"externalCluster"` + // Node address used for etcd cluster peering PeerAddress string `json:"peerAddress"` } +// ExternalCluster defines external etcd cluster related config options +type ExternalCluster struct { + // Endpoints of external etcd cluster used to connect by k0s + Endpoints []string `json:"endpoints"` + + // EtcdPrefix is a prefix to prepend to all resource paths in etcd + EtcdPrefix string `json:"etcdPrefix"` + + // CaFile is the host path to a file with CA certificate + CaFile string `json:"caFile"` + + // ClientCertFile is the host path to a file with TLS certificate for etcd client + ClientCertFile string `json:"clientCertFile"` + + // ClientKeyFile is the host path to a file with TLS key for etcd client + ClientKeyFile string `json:"clientKeyFile"` +} + // DefaultEtcdConfig creates EtcdConfig with sane defaults func DefaultEtcdConfig() *EtcdConfig { addr, err := iface.FirstPublicAddress() @@ -114,7 +145,8 @@ func DefaultEtcdConfig() *EtcdConfig { addr = "127.0.0.1" } return &EtcdConfig{ - PeerAddress: addr, + ExternalCluster: nil, + PeerAddress: addr, } } @@ -124,3 +156,89 @@ func DefaultKineConfig(dataDir string) *KineConfig { DataSource: "sqlite://" + dataDir + "/db/state.db?more=rwc&_journal=WAL&cache=shared", } } + +// GetEndpointsAsString returns comma-separated list of external cluster endpoints if exist +// or internal etcd address which is https://127.0.0.1:2379 +func (e *EtcdConfig) GetEndpointsAsString() string { + if e != nil && e.IsExternalClusterUsed() { + return strings.Join(e.ExternalCluster.Endpoints, ",") + } + return "https://127.0.0.1:2379" +} + +// GetEndpointsAsString returns external cluster endpoints if exist +// or internal etcd address which is https://127.0.0.1:2379 +func (e *EtcdConfig) GetEndpoints() []string { + if e != nil && e.IsExternalClusterUsed() { + return e.ExternalCluster.Endpoints + } + return []string{"https://127.0.0.1:2379"} +} + +// IsExternalClusterUsed returns true if `spec.storage.etcd.externalCluster` is defined, otherwise returns false. +func (e *EtcdConfig) IsExternalClusterUsed() bool { + return e != nil && e.ExternalCluster != nil +} + +// IsTLSEnabled returns true if external cluster is not configured or external cluster is configured +// with all TLS properties: caFile, clientCertFile, clientKeyFile. Otherwise it returns false. +func (e *EtcdConfig) IsTLSEnabled() bool { + return !e.IsExternalClusterUsed() || e.ExternalCluster.hasAllTLSPropertiesDefined() +} + +// GetCaFilePath returns the host path to a file with CA certificate if external cluster has configured all TLS properties, +// otherwise it returns the host path to a default CA certificate in a given certDir directory. +func (e *EtcdConfig) GetCaFilePath(certDir string) string { + if e.IsExternalClusterUsed() && e.ExternalCluster.hasAllTLSPropertiesDefined() { + return e.ExternalCluster.CaFile + } + return filepath.Join(certDir, "ca.crt") +} + +// GetCertFilePath returns the host path to a file with a client certificate if external cluster has configured all TLS properties, +// otherwise it returns the host path to a default client certificate in a given certDir directory. +func (e *EtcdConfig) GetCertFilePath(certDir string) string { + if e.IsExternalClusterUsed() && e.ExternalCluster.hasAllTLSPropertiesDefined() { + return e.ExternalCluster.ClientCertFile + } + return filepath.Join(certDir, "apiserver-etcd-client.crt") +} + +// GetCaFilePath returns the host path to a file with client private key if external cluster has configured all TLS properties, +// otherwise it returns the host path to a default client private key in a given certDir directory. +func (e *EtcdConfig) GetKeyFilePath(certDir string) string { + if e.IsExternalClusterUsed() && e.ExternalCluster.hasAllTLSPropertiesDefined() { + return e.ExternalCluster.ClientKeyFile + } + return filepath.Join(certDir, "apiserver-etcd-client.key") +} + +func validateRequiredProperties(e *ExternalCluster) []error { + var errors []error + + if e.Endpoints == nil || len(e.Endpoints) == 0 { + errors = append(errors, fmt.Errorf("spec.storage.etcd.externalCluster.endpoints cannot be null or empty")) + } else if slices.Contains(e.Endpoints, "") { + errors = append(errors, fmt.Errorf("spec.storage.etcd.externalCluster.endpoints cannot contain empty strings")) + } + + if e.EtcdPrefix == "" { + errors = append(errors, fmt.Errorf("spec.storage.etcd.externalCluster.etcdPrefix cannot be empty")) + } + + return errors +} + +func validateOptionalTLSProperties(e *ExternalCluster) []error { + noTLSPropertyDefined := e.CaFile == "" && e.ClientCertFile == "" && e.ClientKeyFile == "" + + if noTLSPropertyDefined || e.hasAllTLSPropertiesDefined() { + return nil + } + return []error{fmt.Errorf("spec.storage.etcd.externalCluster is invalid: " + + "all TLS properties [caFile,clientCertFile,clientKeyFile] must be defined or none of those")} +} + +func (e *ExternalCluster) hasAllTLSPropertiesDefined() bool { + return e.CaFile != "" && e.ClientCertFile != "" && e.ClientKeyFile != "" +} diff --git a/pkg/apis/k0s.k0sproject.io/v1beta1/storage_test.go b/pkg/apis/k0s.k0sproject.io/v1beta1/storage_test.go index 8f0383a5d82b..0e36e6514d14 100644 --- a/pkg/apis/k0s.k0sproject.io/v1beta1/storage_test.go +++ b/pkg/apis/k0s.k0sproject.io/v1beta1/storage_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" ) func TestStorageSpec_IsJoinable(t *testing.T) { @@ -34,6 +35,20 @@ func TestStorageSpec_IsJoinable(t *testing.T) { }, want: true, }, + { + name: "etcd", + storage: StorageSpec{ + Type: "etcd", + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"https://192.168.10.2:2379"}, + EtcdPrefix: "k0s-tenant-1", + }, + PeerAddress: "", + }, + }, + want: false, + }, { name: "kine-sqlite", storage: StorageSpec{ @@ -96,3 +111,212 @@ spec: assert.NotNil(t, c.Spec.Storage.Kine) assert.Equal(t, "sqlite:///var/lib/k0s/db/state.db?more=rwc&_journal=WAL&cache=shared", c.Spec.Storage.Kine.DataSource) } + +type storageSuite struct { + suite.Suite +} + +func (s *storageSuite) TestValidation() { + var validStorageSpecs = []struct { + desc string + spec *StorageSpec + }{ + { + desc: "default_storage_spec_is_valid", + spec: DefaultStorageSpec(), + }, + { + desc: "internal_cluster_spec_is_valid", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + PeerAddress: "192.168.10.10", + }, + }, + }, + { + desc: "external_cluster_spec_without_tls_is_valid", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"http://192.168.10.10"}, + EtcdPrefix: "tenant-1", + }, + }, + }, + }, + { + desc: "external_cluster_spec_with_tls_is_valid", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"http://192.168.10.10"}, + EtcdPrefix: "tenant-1", + CaFile: "/etc/pki/CA/ca.crt", + ClientCertFile: "/etc/pki/tls/certs/etcd-client.crt", + ClientKeyFile: "/etc/pki/tls/private/etcd-client.key", + }, + }, + }, + }, + } + + for _, tt := range validStorageSpecs { + s.T().Run(tt.desc, func(t *testing.T) { + s.Nil(tt.spec.Validate()) + }) + } + + var singleValidationErrorCases = []struct { + desc string + spec *StorageSpec + expectedErrMsg string + }{ + { + desc: "external_cluster_endpoints_cannot_be_null", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: nil, + EtcdPrefix: "tenant-1", + }, + }, + }, + expectedErrMsg: "spec.storage.etcd.externalCluster.endpoints cannot be null or empty", + }, + { + desc: "external_cluster_endpoints_cannot_contain_empty_strings", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"http://192.168.10.2:2379", ""}, + EtcdPrefix: "tenant-1", + }, + }, + }, + expectedErrMsg: "spec.storage.etcd.externalCluster.endpoints cannot contain empty strings", + }, + { + desc: "external_cluster_must_have_configured_all_tls_properties_or_none_of_them", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"http://192.168.10.10"}, + EtcdPrefix: "tenant-1", + CaFile: "", + ClientCertFile: "/etc/pki/tls/certs/etcd-client.crt", + ClientKeyFile: "", + }, + }, + }, + expectedErrMsg: "spec.storage.etcd.externalCluster is invalid: all TLS properties [caFile,clientCertFile,clientKeyFile] must be defined or none of those", + }, + } + + for _, tt := range singleValidationErrorCases { + s.T().Run(tt.desc, func(t *testing.T) { + errs := tt.spec.Validate() + s.NotNil(errs) + s.Len(errs, 1) + s.Contains(errs[0].Error(), tt.expectedErrMsg) + }) + } + + s.T().Run("external_cluster_endpoints_and_etcd_prefix_cannot_be_empty", func(t *testing.T) { + spec := &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{}, + EtcdPrefix: "", + }, + }, + } + + errs := spec.Validate() + s.NotNil(errs) + s.Len(errs, 2) + s.Contains(errs[0].Error(), "spec.storage.etcd.externalCluster.endpoints cannot be null or empty") + s.Contains(errs[1].Error(), "spec.storage.etcd.externalCluster.etcdPrefix cannot be empty") + }) +} + +func (s *storageSuite) TestIsTLSEnabled() { + var storageSpecs = []struct { + desc string + spec *StorageSpec + expectedResult bool + }{ + { + desc: "is_TLS_enabled_returns_true_when_internal_cluster_is_used", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + PeerAddress: "192.168.10.10", + }, + }, + expectedResult: true, + }, + { + desc: "is_TLS_enabled_returns_true_when_external_cluster_is_used_and_has_set_all_TLS_properties", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"http://192.168.10.10"}, + EtcdPrefix: "tenant-1", + CaFile: "/etc/pki/CA/ca.crt", + ClientCertFile: "/etc/pki/tls/certs/etcd-client.crt", + ClientKeyFile: "/etc/pki/tls/private/etcd-client.key", + }, + }, + }, + expectedResult: true, + }, + { + desc: "is_TLS_enabled_returns_false_when_external_cluster_is_used_but_has_no_TLS_properties", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"http://192.168.10.10"}, + EtcdPrefix: "tenant-1", + }, + }, + }, + expectedResult: false, + }, + { + desc: "is_TLS_enabled_returns_false_when_external_cluster_is_used_but_TLS_properties_are_configured_partially", + spec: &StorageSpec{ + Type: EtcdStorageType, + Etcd: &EtcdConfig{ + ExternalCluster: &ExternalCluster{ + Endpoints: []string{"http://192.168.10.10"}, + EtcdPrefix: "tenant-1", + CaFile: "/etc/pki/CA/ca.crt", + }, + }, + }, + expectedResult: false, + }, + } + + for _, tt := range storageSpecs { + s.T().Run(tt.desc, func(t *testing.T) { + result := tt.spec.Etcd.IsTLSEnabled() + s.Equal(result, tt.expectedResult) + }) + } +} + +func TestStorageSuite(t *testing.T) { + storageSuite := &storageSuite{} + + suite.Run(t, storageSuite) +} diff --git a/pkg/apis/k0s.k0sproject.io/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s.k0sproject.io/v1beta1/zz_generated.deepcopy.go index 6818d636610c..903e429e5fd6 100644 --- a/pkg/apis/k0s.k0sproject.io/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s.k0sproject.io/v1beta1/zz_generated.deepcopy.go @@ -370,6 +370,11 @@ func (in *DualStack) DeepCopy() *DualStack { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { *out = *in + if in.ExternalCluster != nil { + in, out := &in.ExternalCluster, &out.ExternalCluster + *out = new(ExternalCluster) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdConfig. @@ -418,6 +423,26 @@ func (in *EtcdResponse) DeepCopy() *EtcdResponse { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalCluster) DeepCopyInto(out *ExternalCluster) { + *out = *in + if in.Endpoints != nil { + in, out := &in.Endpoints, &out.Endpoints + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalCluster. +func (in *ExternalCluster) DeepCopy() *ExternalCluster { + if in == nil { + return nil + } + out := new(ExternalCluster) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HelmExtensions) DeepCopyInto(out *HelmExtensions) { *out = *in @@ -644,7 +669,7 @@ func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { if in.Etcd != nil { in, out := &in.Etcd, &out.Etcd *out = new(EtcdConfig) - **out = **in + (*in).DeepCopyInto(*out) } if in.Kine != nil { in, out := &in.Kine, &out.Kine diff --git a/pkg/backup/etcd.go b/pkg/backup/etcd.go index 8408d8d69e92..4caba015a4e4 100644 --- a/pkg/backup/etcd.go +++ b/pkg/backup/etcd.go @@ -54,7 +54,7 @@ func (e etcdStep) Name() string { func (e etcdStep) Backup() (StepResult, error) { ctx := context.TODO() - etcdClient, err := etcd.NewClient(e.certRootDir, e.etcdCertDir) + etcdClient, err := etcd.NewClient(e.certRootDir, e.etcdCertDir, nil) if err != nil { return StepResult{}, err } diff --git a/pkg/backup/manager.go b/pkg/backup/manager.go index 0e18036079ab..9ee8d4948b8c 100644 --- a/pkg/backup/manager.go +++ b/pkg/backup/manager.go @@ -71,12 +71,12 @@ func (bm *Manager) RunBackup(cfgPath string, clusterSpec *v1beta1.ClusterSpec, v } func (bm *Manager) discoverSteps(cfgPath string, clusterSpec *v1beta1.ClusterSpec, vars constant.CfgVars, action string, restoredConfigPath string) { - if clusterSpec.Storage.Type == v1beta1.EtcdStorageType { + if clusterSpec.Storage.Type == v1beta1.EtcdStorageType && !clusterSpec.Storage.Etcd.IsExternalClusterUsed() { bm.Add(newEtcdStep(bm.tmpDir, vars.CertRootDir, vars.EtcdCertDir, clusterSpec.Storage.Etcd.PeerAddress, vars.EtcdDataDir)) } else if clusterSpec.Storage.Type == v1beta1.KineStorageType && strings.HasPrefix(clusterSpec.Storage.Kine.DataSource, "sqlite://") { bm.Add(newSqliteStep(bm.tmpDir, clusterSpec.Storage.Kine.DataSource, vars.DataDir)) } else { - logrus.Warnf("only etcd and sqlite %s is supported. Other storage backends must be backed-up/restored manually.", action) + logrus.Warnf("only internal etcd and sqlite %s are supported. Other storage backends must be backed-up/restored manually.", action) } bm.dataDir = vars.DataDir for _, path := range []string{ diff --git a/pkg/component/controller/apiserver.go b/pkg/component/controller/apiserver.go index 6d4738d9811b..8075c28da0b7 100644 --- a/pkg/component/controller/apiserver.go +++ b/pkg/component/controller/apiserver.go @@ -163,19 +163,13 @@ func (a *APIServer) Run(_ context.Context) error { UID: a.uid, GID: a.gid, } - switch a.ClusterConfig.Spec.Storage.Type { - case v1beta1.KineStorageType: - a.supervisor.Args = append(a.supervisor.Args, - fmt.Sprintf("--etcd-servers=unix://%s", a.K0sVars.KineSocketPath)) // kine endpoint - case v1beta1.EtcdStorageType: - a.supervisor.Args = append(a.supervisor.Args, - "--etcd-servers=https://127.0.0.1:2379", - fmt.Sprintf("--etcd-cafile=%s", path.Join(a.K0sVars.CertRootDir, "etcd/ca.crt")), - fmt.Sprintf("--etcd-certfile=%s", path.Join(a.K0sVars.CertRootDir, "apiserver-etcd-client.crt")), - fmt.Sprintf("--etcd-keyfile=%s", path.Join(a.K0sVars.CertRootDir, "apiserver-etcd-client.key"))) - default: - return fmt.Errorf("invalid storage type: %s", a.ClusterConfig.Spec.Storage.Type) + + etcdArgs, err := getEtcdArgs(a.ClusterConfig.Spec.Storage, a.K0sVars) + if err != nil { + return err } + a.supervisor.Args = append(a.supervisor.Args, etcdArgs...) + return a.supervisor.Supervise() } @@ -246,3 +240,27 @@ func (a *APIServer) Healthy() error { } return nil } + +func getEtcdArgs(storage *v1beta1.StorageSpec, k0sVars constant.CfgVars) ([]string, error) { + var args []string + + switch storage.Type { + case v1beta1.KineStorageType: + args = append(args, fmt.Sprintf("--etcd-servers=unix://%s", k0sVars.KineSocketPath)) // kine endpoint + case v1beta1.EtcdStorageType: + args = append(args, fmt.Sprintf("--etcd-servers=%s", storage.Etcd.GetEndpointsAsString())) + if storage.Etcd.IsTLSEnabled() { + args = append(args, + fmt.Sprintf("--etcd-cafile=%s", storage.Etcd.GetCaFilePath(k0sVars.EtcdCertDir)), + fmt.Sprintf("--etcd-certfile=%s", storage.Etcd.GetCertFilePath(k0sVars.CertRootDir)), + fmt.Sprintf("--etcd-keyfile=%s", storage.Etcd.GetKeyFilePath(k0sVars.CertRootDir))) + } + if storage.Etcd.IsExternalClusterUsed() { + args = append(args, fmt.Sprintf("--etcd-prefix=%s", storage.Etcd.ExternalCluster.EtcdPrefix)) + } + default: + return nil, fmt.Errorf("invalid storage type: %s", storage.Type) + } + + return args, nil +} diff --git a/pkg/component/controller/apiserver_test.go b/pkg/component/controller/apiserver_test.go new file mode 100644 index 000000000000..6281a302aef3 --- /dev/null +++ b/pkg/component/controller/apiserver_test.go @@ -0,0 +1,87 @@ +package controller + +import ( + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/constant" + "github.com/stretchr/testify/suite" + "testing" +) + +type apiServerSuite struct { + suite.Suite +} + +func TestApiServerSuite(t *testing.T) { + apiServerSuite := &apiServerSuite{} + + suite.Run(t, apiServerSuite) +} + +func (a *apiServerSuite) TestGetEtcdArgs() { + k0sVars := constant.CfgVars{ + CertRootDir: "/var/lib/k0s/pki", + EtcdCertDir: "/var/lib/k0s/pki/etcd", + } + + a.T().Run("internal etcd cluster", func(t *testing.T) { + storageSpec := &v1beta1.StorageSpec{ + Etcd: &v1beta1.EtcdConfig{ + PeerAddress: "192.168.68.104", + }, + Type: "etcd", + } + + result, err := getEtcdArgs(storageSpec, k0sVars) + + a.Nil(err) + a.Len(result, 4) + a.Contains(result[0], "--etcd-servers=https://127.0.0.1:2379") + a.Contains(result[1], "--etcd-cafile=/var/lib/k0s/pki/etcd/ca.crt") + a.Contains(result[2], "--etcd-certfile=/var/lib/k0s/pki/apiserver-etcd-client.crt") + a.Contains(result[3], "--etcd-keyfile=/var/lib/k0s/pki/apiserver-etcd-client.key") + }) + + a.T().Run("external etcd cluster with TLS", func(t *testing.T) { + storageSpec := &v1beta1.StorageSpec{ + Etcd: &v1beta1.EtcdConfig{ + ExternalCluster: &v1beta1.ExternalCluster{ + Endpoints: []string{"https://192.168.10.10:2379", "https://192.168.10.11:2379"}, + EtcdPrefix: "k0s-tenant-1", + CaFile: "/etc/pki/CA/ca.crt", + ClientCertFile: "/etc/pki/tls/certs/etcd-client.crt", + ClientKeyFile: "/etc/pki/tls/private/etcd-client.key", + }, + }, + Type: "etcd", + } + + result, err := getEtcdArgs(storageSpec, k0sVars) + + a.Nil(err) + a.Len(result, 5) + a.Contains(result[0], "--etcd-servers=https://192.168.10.10:2379,https://192.168.10.11:2379") + a.Contains(result[1], "--etcd-cafile=/etc/pki/CA/ca.crt") + a.Contains(result[2], "--etcd-certfile=/etc/pki/tls/certs/etcd-client.crt") + a.Contains(result[3], "--etcd-keyfile=/etc/pki/tls/private/etcd-client.key") + a.Contains(result[4], "--etcd-prefix=k0s-tenant-1") + }) + + a.T().Run("external etcd cluster without TLS", func(t *testing.T) { + storageSpec := &v1beta1.StorageSpec{ + Etcd: &v1beta1.EtcdConfig{ + ExternalCluster: &v1beta1.ExternalCluster{ + Endpoints: []string{"http://192.168.10.10:2379", "http://192.168.10.11:2379"}, + EtcdPrefix: "k0s-tenant-1", + }, + }, + Type: "etcd", + } + + result, err := getEtcdArgs(storageSpec, k0sVars) + + a.Nil(err) + a.Len(result, 2) + a.Contains(result[0], "--etcd-servers=http://192.168.10.10:2379,http://192.168.10.11:2379") + a.Contains(result[1], "--etcd-prefix=k0s-tenant-1") + }) +} diff --git a/pkg/component/controller/etcd.go b/pkg/component/controller/etcd.go index 7473ab493336..b14c39b40c1f 100644 --- a/pkg/component/controller/etcd.go +++ b/pkg/component/controller/etcd.go @@ -124,8 +124,12 @@ func (e *Etcd) syncEtcdConfig(peerURL, etcdCaCert, etcdCaCertKey string) ([]stri return etcdResponse.InitialCluster, nil } -// Run runs etcd +// Run runs etcd if external cluster is not configured func (e *Etcd) Run(ctx context.Context) error { + if e.Config.IsExternalClusterUsed() { + return nil + } + etcdCaCert := filepath.Join(e.K0sVars.EtcdCertDir, "ca.crt") etcdCaCertKey := filepath.Join(e.K0sVars.EtcdCertDir, "ca.key") etcdServerCert := filepath.Join(e.K0sVars.EtcdCertDir, "server.crt") @@ -282,7 +286,7 @@ func (e *Etcd) Healthy() error { logrus.WithField("component", "etcd").Debug("checking etcd endpoint for health") ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - err := etcd.CheckEtcdReady(ctx, e.K0sVars.CertRootDir, e.K0sVars.EtcdCertDir) + err := etcd.CheckEtcdReady(ctx, e.K0sVars.CertRootDir, e.K0sVars.EtcdCertDir, e.Config) return err } diff --git a/pkg/config/config.go b/pkg/config/config.go index 6e1779aef42e..d0d9caa19002 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -200,7 +200,8 @@ func GetNodeConfig(cfgPath string, k0sVars constant.CfgVars) (*v1beta1.ClusterCo var etcdConfig *v1beta1.EtcdConfig if cfg.Spec.Storage.Type == v1beta1.EtcdStorageType { etcdConfig = &v1beta1.EtcdConfig{ - PeerAddress: cfg.Spec.Storage.Etcd.PeerAddress, + ExternalCluster: cfg.Spec.Storage.Etcd.ExternalCluster, + PeerAddress: cfg.Spec.Storage.Etcd.PeerAddress, } nodeConfig.Spec.Storage.Etcd = etcdConfig } diff --git a/pkg/etcd/client.go b/pkg/etcd/client.go index 82158f70249c..93beb72d96b1 100644 --- a/pkg/etcd/client.go +++ b/pkg/etcd/client.go @@ -17,9 +17,9 @@ package etcd import ( "context" + "crypto/tls" "fmt" - "path/filepath" - + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" @@ -33,21 +33,26 @@ type Client struct { } // NewClient creates new Client -func NewClient(certDir string, etcdCertDir string) (*Client, error) { +func NewClient(certDir, etcdCertDir string, etcdConf *v1beta1.EtcdConfig) (*Client, error) { client := &Client{} - client.tlsInfo = transport.TLSInfo{ - CertFile: filepath.Join(certDir, "apiserver-etcd-client.crt"), - KeyFile: filepath.Join(certDir, "apiserver-etcd-client.key"), - TrustedCAFile: filepath.Join(etcdCertDir, "ca.crt"), - } - tlsConfig, err := client.tlsInfo.ClientConfig() - if err != nil { - return nil, err + var tlsConfig *tls.Config + if etcdConf.IsTLSEnabled() { + client.tlsInfo = transport.TLSInfo{ + CertFile: etcdConf.GetCertFilePath(certDir), + KeyFile: etcdConf.GetKeyFilePath(certDir), + TrustedCAFile: etcdConf.GetCaFilePath(etcdCertDir), + } + + var err error + tlsConfig, err = client.tlsInfo.ClientConfig() + if err != nil { + return nil, err + } } cfg := clientv3.Config{ - Endpoints: []string{"https://127.0.0.1:2379"}, + Endpoints: etcdConf.GetEndpoints(), TLS: tlsConfig, } cli, _ := clientv3.New(cfg) diff --git a/pkg/etcd/health.go b/pkg/etcd/health.go index 19c2b6981feb..3bf0b597230b 100644 --- a/pkg/etcd/health.go +++ b/pkg/etcd/health.go @@ -2,13 +2,14 @@ package etcd import ( "context" + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" "github.com/sirupsen/logrus" ) // CheckEtcdReady returns true if etcd responds to the metrics endpoint with a status code of 200 -func CheckEtcdReady(ctx context.Context, certDir string, etcdCertDir string) error { - c, err := NewClient(certDir, etcdCertDir) +func CheckEtcdReady(ctx context.Context, certDir string, etcdCertDir string, etcdConf *v1beta1.EtcdConfig) error { + c, err := NewClient(certDir, etcdCertDir, etcdConf) if err != nil { logrus.Errorf("failed to initialize etcd client: %v", err) return err diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index dfd3f2c56802..8229f9d9766e 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -360,6 +360,33 @@ spec: etcd: description: EtcdConfig defines etcd related config options properties: + externalCluster: + description: ExternalCluster defines external etcd cluster + related config options + properties: + caFile: + description: CaFile is the host path to a file with CA + certificate + type: string + clientCertFile: + description: ClientCertFile is the host path to a file + with TLS certificate for etcd client + type: string + clientKeyFile: + description: ClientKeyFile is the host path to a file + with TLS key for etcd client + type: string + endpoints: + description: Endpoints of external etcd cluster used to + connect by k0s + items: + type: string + type: array + etcdPrefix: + description: EtcdPrefix is a prefix to prepend to all + resource paths in etcd + type: string + type: object peerAddress: description: Node address used for etcd cluster peering type: string