[Release-1.35] Allow k3s secrets-encrypt enable on existing clusters

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -131,19 +131,21 @@ func Status(app *cli.Context) error {
 	}
 
 	var tabBuffer bytes.Buffer
-	w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0)
-	fmt.Fprintf(w, "\n")
-	fmt.Fprintf(w, "Active\tKey Type\tName\n")
-	fmt.Fprintf(w, "------\t--------\t----\n")
-	if status.ActiveKey != "" {
-		ak := strings.Split(status.ActiveKey, " ")
-		fmt.Fprintf(w, " *\t%s\t%s\n", ak[0], ak[1])
-	}
-	for _, k := range status.InactiveKeys {
-		ik := strings.Split(k, " ")
-		fmt.Fprintf(w, "\t%s\t%s\n", ik[0], ik[1])
-	}
-	w.Flush()
+	if status.ActiveKey != "" || len(status.InactiveKeys) > 0 {
+		w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0)
+		fmt.Fprint(w, "\n")
+		fmt.Fprint(w, "Active\tKey Type\tName\n")
+		fmt.Fprint(w, "------\t--------\t----\n")
+		if status.ActiveKey != "" {
+			ak := strings.Split(status.ActiveKey, " ")
+			fmt.Fprintf(w, " *\t%s\t%s\n", ak[0], ak[1])
+		}
+		for _, k := range status.InactiveKeys {
+			ik := strings.Split(k, " ")
+			fmt.Fprintf(w, "\t%s\t%s\n", ik[0], ik[1])
+		}
+		w.Flush()
+	}
 	fmt.Println(statusOutput + tabBuffer.String())
 	return nil
 }

pkg/daemons/control/deps/deps.go

@@ -774,7 +774,7 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error {
 	case secretsencrypt.SecretBoxProvider:
 		keyName = "secretboxkey"
 	default:
-		return fmt.Errorf("unsupported secrets-encryption-key-type %s", controlConfig.EncryptProvider)
+		return fmt.Errorf("unsupported secrets-encryption-provider %s", controlConfig.EncryptProvider)
 	}
 	if s, err := os.Stat(runtime.EncryptionConfig); err == nil && s.Size() > 0 {
 		// On upgrade from older versions, the encryption hash may not exist, create it
@@ -801,7 +801,8 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error {
 		},
 	}
 	var provider []apiserverconfigv1.ProviderConfiguration
-	if controlConfig.EncryptProvider == secretsencrypt.AESCBCProvider {
+	switch controlConfig.EncryptProvider {
+	case secretsencrypt.AESCBCProvider:
 		provider = []apiserverconfigv1.ProviderConfiguration{
 			{
 				AESCBC: &apiserverconfigv1.AESConfiguration{
@@ -812,7 +813,7 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error {
 				Identity: &apiserverconfigv1.IdentityConfiguration{},
 			},
 		}
-	} else if controlConfig.EncryptProvider == secretsencrypt.SecretBoxProvider {
+	case secretsencrypt.SecretBoxProvider:
 		provider = []apiserverconfigv1.ProviderConfiguration{
 			{
 				Secretbox: &apiserverconfigv1.SecretboxConfiguration{

pkg/secretsencrypt/config.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
@@ -175,6 +176,39 @@ func WriteEncryptionConfig(runtime *config.ControlRuntime, keys *EncryptionKeys,
 	return util.AtomicWrite(runtime.EncryptionConfig, jsonfile, 0600)
 }
 
+// WriteIdentityConfig creates an identity-only configuration for clusters that
+// previously had no encryption config, effectively disabling encryption, but
+// preparing a node for future reencryption.
+func WriteIdentityConfig(control *config.Control) error {
+	providers := []apiserverconfigv1.ProviderConfiguration{
+		{
+			Identity: &apiserverconfigv1.IdentityConfiguration{},
+		},
+	}
+
+	encConfig := apiserverconfigv1.EncryptionConfiguration{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "EncryptionConfiguration",
+			APIVersion: "apiserver.config.k8s.io/v1",
+		},
+		Resources: []apiserverconfigv1.ResourceConfiguration{
+			{
+				Resources: []string{"secrets"},
+				Providers: providers,
+			},
+		},
+	}
+	jsonfile, err := json.Marshal(encConfig)
+	if err != nil {
+		return err
+	}
+	if control.Runtime.EncryptionConfig == "" {
+		control.Runtime.EncryptionConfig = filepath.Join(control.DataDir, "cred", "encryption-config.json")
+	}
+	logrus.Info("Enabling secrets encryption with identity provider, restart with secrets-encryption")
+	return util.AtomicWrite(control.Runtime.EncryptionConfig, jsonfile, 0600)
+}
+
 func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) {
 	curEncryptionByte, err := os.ReadFile(runtime.EncryptionConfig)
 	if err != nil {

pkg/server/handlers/secrets-encrypt.go

@@ -88,6 +88,8 @@ func encryptionStatus(control *config.Control) (EncryptionState, error) {
 	}
 	if providers[len(providers)-1].Identity != nil && (providers[0].AESCBC != nil || providers[0].Secretbox != nil) {
 		state.Enable = ptr.To(true)
+	} else if control.EncryptSecrets && providers[0].Identity != nil && len(providers) == 1 {
+		state.Enable = ptr.To(false)
 	} else if !control.EncryptSecrets || providers[0].Identity != nil && (providers[1].AESCBC != nil || providers[1].Secretbox != nil) {
 		state.Enable = ptr.To(false)
 	}
@@ -137,7 +139,13 @@ func encryptionStatus(control *config.Control) (EncryptionState, error) {
 
 func encryptionEnable(ctx context.Context, control *config.Control, enable bool) error {
 	providers, err := secretsencrypt.GetEncryptionProviders(control.Runtime)
-	if err != nil {
+	// Enable secrets encryption with an identity provider on a cluster that does not have any encryption config
+	if err != nil && os.IsNotExist(err) && enable {
+		if err := secretsencrypt.WriteIdentityConfig(control); err != nil {
+			return err
+		}
+		return cluster.Save(ctx, control, true)
+	} else if err != nil {
 		return err
 	}
 	if len(providers) > 3 {
@@ -390,6 +398,7 @@ func reencryptAndRemoveKey(ctx context.Context, control *config.Control, skip bo
 
 	// Remove old key. If there is only one of that key type, the cluster just
 	// migrated between key types. Check for the other key type and remove that.
+	// If that key type type doesn't exist, we are switching from the identity provider, so no key is removed.
 	curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime)
 	if err != nil {
 		return err
@@ -400,6 +409,8 @@ func reencryptAndRemoveKey(ctx context.Context, control *config.Control, skip bo
 		if len(curKeys.AESCBCKeys) == 1 && len(curKeys.SBKeys) > 0 {
 			logrus.Infoln("Removing secretbox key: ", curKeys.SBKeys[len(curKeys.SBKeys)-1])
 			curKeys.SBKeys = curKeys.SBKeys[:len(curKeys.SBKeys)-1]
+		} else if len(curKeys.AESCBCKeys) == 1 && curKeys.Identity {
+			logrus.Infoln("No keys to remove, switched from identity provider")
 		} else {
 			logrus.Infoln("Removing aescbc key: ", curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1])
 			curKeys.AESCBCKeys = curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1]
@@ -408,6 +419,8 @@ func reencryptAndRemoveKey(ctx context.Context, control *config.Control, skip bo
 		if len(curKeys.SBKeys) == 1 && len(curKeys.AESCBCKeys) > 0 {
 			logrus.Infoln("Removing aescbc key: ", curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1])
 			curKeys.AESCBCKeys = curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1]
+		} else if len(curKeys.SBKeys) == 1 && curKeys.Identity {
+			logrus.Infoln("No keys to remove, switched from identity provider")
 		} else {
 			logrus.Infoln("Removing secretbox key: ", curKeys.SBKeys[len(curKeys.SBKeys)-1])
 			curKeys.SBKeys = curKeys.SBKeys[:len(curKeys.SBKeys)-1]

tests/docker/hardened/hardened_test.go

@@ -38,7 +38,7 @@ kubelet-arg:
   - 'event-qps=0'
   - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
 kube-apiserver-arg:
-  - 'admission-control-config-file=/tmp/cluster-level-pss.yaml'
+  - 'admission-control-config-file=/home/cluster-level-pss.yaml'
   - 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log'
   - 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml'
   - 'audit-log-maxage=30'
@@ -54,7 +54,7 @@ kubelet-arg:
 			Expect(config.ProvisionServers(1)).To(Succeed())
 
 			for _, server := range config.Servers {
-				cmd := "docker cp ./cluster-level-pss.yaml " + server.Name + ":/tmp/cluster-level-pss.yaml"
+				cmd := "docker cp ./cluster-level-pss.yaml " + server.Name + ":/home/cluster-level-pss.yaml"
 				Expect(tests.RunCommand(cmd)).Error().NotTo(HaveOccurred())
 
 				cmd = "mkdir -p /var/lib/rancher/k3s/server/logs"

tests/docker/secretsencryption/secretsencryption_test.go

@@ -28,7 +28,6 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 			var err error
 			tc, err = docker.NewTestConfig("rancher/systemd-node")
 			Expect(err).NotTo(HaveOccurred())
-			tc.ServerYaml = `secrets-encryption: true`
 			Expect(tc.ProvisionServers(*serverCount)).To(Succeed())
 			Eventually(func() error {
 				return tests.CheckDefaultDeployments(tc.KubeconfigFile)
@@ -38,23 +37,48 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 			}, "40s", "5s").Should(Succeed())
 		})
 	})
-	Context("Secrets Keys are rotated:", func() {
+	Context("Secrets are added without encryption:", func() {
 		It("Deploys several secrets", func() {
 			_, err := tc.DeployWorkload("secrets.yaml")
 			Expect(err).NotTo(HaveOccurred(), "Secrets not deployed")
 		})
-
-		It("Verifies encryption start stage", func() {
+		It("Verifies encryption disabled", func() {
 			cmd := "k3s secrets-encrypt status"
 			for _, node := range tc.Servers {
 				res, err := node.RunCmdOnNode(cmd)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-				Expect(res).Should(ContainSubstring("Current Rotation Stage: start"))
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				Expect(res).Should(ContainSubstring("Encryption Status: Disabled, no configuration file found"))
 			}
 		})
+	})
+	Context("Secrets encryption is enabled on the cluster:", func() {
+		It("Enable secrets-encryption", func() {
+			cmd := "k3s secrets-encrypt enable"
+			Expect(tc.Servers[0].RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
+			cmd = "echo 'secrets-encryption: true\n' >> /etc/rancher/k3s/config.yaml"
+			for _, node := range tc.Servers {
+				Expect(node.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
+			}
+		})
+
+		It("Restarts K3s servers", func() {
+			Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
+		})
 
+		It("Verifies encryption start stage", func() {
+			cmd := "k3s secrets-encrypt status"
+			for _, node := range tc.Servers {
+				Eventually(func(g Gomega) {
+					res, err := node.RunCmdOnNode(cmd)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: start"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "120s", "5s").Should(Succeed())
+			}
+		})
+	})
+	Context("Secrets Keys are rotated:", func() {
 		It("Rotates the Secrets-Encryption Keys", func() {
 			cmd := "k3s secrets-encrypt rotate-keys"
 			res, err := tc.Servers[0].RunCmdOnNode(cmd)
@@ -70,7 +94,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 					} else {
 						g.Expect(res).Should(ContainSubstring("Current Rotation Stage: start"))
 					}
-				}, "420s", "10s").Should(Succeed())
+				}, "240s", "10s").Should(Succeed())
 			}
 		})
 
@@ -87,10 +111,9 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
 					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
 					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
-				}, "420s", "2s").Should(Succeed())
+				}, "240s", "2s").Should(Succeed())
 			}
 		})
-
 	})
 
 	Context("Disabling Secrets-Encryption", func() {
@@ -113,7 +136,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 					} else {
 						g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
 					}
-				}, "420s", "2s").Should(Succeed())
+				}, "240s", "2s").Should(Succeed())
 			}
 		})
 
@@ -126,7 +149,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 			for _, node := range tc.Servers {
 				Eventually(func(g Gomega) {
 					g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Disabled"))
-				}, "420s", "2s").Should(Succeed())
+				}, "240s", "2s").Should(Succeed())
 			}
 		})
 
@@ -152,7 +175,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 					} else {
 						g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
 					}
-				}, "420s", "2s").Should(Succeed())
+				}, "240s", "2s").Should(Succeed())
 			}
 		})
 
@@ -165,7 +188,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 			for _, node := range tc.Servers {
 				Eventually(func(g Gomega) {
 					g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Enabled"))
-				}, "420s", "2s").Should(Succeed())
+				}, "240s", "2s").Should(Succeed())
 			}
 		})
 	})
@@ -195,7 +218,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 					} else {
 						g.Expect(res).Should(ContainSubstring("AES-CBC"))
 					}
-				}, "420s", "10s").Should(Succeed())
+				}, "240s", "10s").Should(Succeed())
 			}
 		})
 
@@ -211,7 +234,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 					g.Expect(err).NotTo(HaveOccurred())
 					g.Expect(res).Should(ContainSubstring("XSalsa20-POLY1305"))
 					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
-				}, "420s", "2s").Should(Succeed())
+				}, "240s", "2s").Should(Succeed())
 			}
 		})
 	})
@@ -225,7 +248,11 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", docker.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
+		log_length := 10
+		if *ci {
+			log_length = 1000
+		}
+		AddReportEntry("journald-logs", docker.TailJournalLogs(log_length, append(tc.Servers, tc.Agents...)))
 	}
 	if *ci || (tc != nil && !failed) {
 		Expect(tc.Cleanup()).To(Succeed())

tests/docker/test-helpers.go

@@ -179,7 +179,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 				"-v", "/var/run/docker.sock:/var/run/docker.sock",
 				"-v", "/var/lib/docker:/var/lib/docker",
 				"--mount", "type=bind,source=$(pwd)/../../../dist/artifacts/k3s,target=/usr/local/bin/k3s",
-				fmt.Sprintf("%s:v0.0.5", config.K3sImage),
+				fmt.Sprintf("%s:v0.0.7-rc1", config.K3sImage),
 				"/usr/lib/systemd/systemd --unit=noop.target --show-status=true"}, " ")
 			if out, err := tests.RunCommand(dRun); err != nil {
 				return fmt.Errorf("failed to start systemd container: %s: %v", out, err)
@@ -344,7 +344,7 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
 					"-v", "/var/run/docker.sock:/var/run/docker.sock",
 					"-v", "/var/lib/docker:/var/lib/docker",
 					"--mount", "type=bind,source=$(pwd)/../../../dist/artifacts/k3s,target=/usr/local/bin/k3s",
-					fmt.Sprintf("%s:v0.0.5", config.K3sImage),
+					fmt.Sprintf("%s:v0.0.7-rc1", config.K3sImage),
 					"/usr/lib/systemd/systemd --unit=noop.target --show-status=true"}, " ")
 				if out, err := tests.RunCommand(dRun); err != nil {
 					return fmt.Errorf("failed to start systemd container: %s: %v", out, err)

tests/docker/upgrade/upgrade_test.go

@@ -226,7 +226,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
 					g.Expect(docker.VerifyValidVersion(server, "kubectl")).To(Succeed())
 					g.Expect(docker.VerifyValidVersion(server, "ctr")).To(Succeed())
 					g.Expect(docker.VerifyValidVersion(server, "crictl")).To(Succeed())
-				}).Should(Succeed())
+				}, "10s", "2s").Should(Succeed())
 
 				out, err := server.RunCmdOnNode("k3s --version")
 				Expect(err).NotTo(HaveOccurred())