Secrets-encryption rotation

.github/workflows/integration.yaml

@@ -56,17 +56,8 @@ jobs:
     - name: Run Integration Tests
       run: | 
         chmod +x ./dist/artifacts/k3s
-        go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Integration
-        go tool cover -func coverage.out
-        # these tests do not relate to coverage and must be run separately
-        go test ./tests/integration/... -run Integration 
+        go test ./pkg/... ./tests/integration/... -run Integration
     - name: On Failure, Launch Debug Session
       if: ${{ failure() }}
       uses: mxschmitt/action-tmate@v3
-      timeout-minutes: 5
-    - name: Upload Results To Codecov
-      uses: codecov/codecov-action@v1
-      with:
-        files: ./coverage.out
-        flags: inttests # optional
-        verbose: true # optional (default = false)
+      timeout-minutes: 5

cmd/encrypt/main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
+	"github.com/rancher/k3s/pkg/configfilearg"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := cmds.NewApp()
+	app.Commands = []cli.Command{
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
+	}
+
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
+		logrus.Fatal(err)
+	}
+}

cmd/k3s/main.go

@@ -35,6 +35,7 @@ func main() {
 	}
 
 	etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)
+	secretsencryptCommand := internalCLIAction(version.Program+"-"+cmds.SecretsEncryptCommand, dataDir, os.Args)
 	certCommand := internalCLIAction(version.Program+"-"+cmds.CertCommand, dataDir, os.Args)
 
 	// Handle subcommand invocation (k3s server, k3s crictl, etc)
@@ -53,6 +54,15 @@ func main() {
 				etcdsnapshotCommand,
 				etcdsnapshotCommand),
 		),
+		cmds.NewSecretsEncryptCommand(secretsencryptCommand,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				certCommand),

cmd/server/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/ctr"
 	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/cli/kubectl"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
 	"github.com/rancher/k3s/pkg/cli/server"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/rancher/k3s/pkg/containerd"
@@ -53,6 +54,15 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				cert.Run),

main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/crictl"
 	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/cli/kubectl"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
 	"github.com/rancher/k3s/pkg/cli/server"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/sirupsen/logrus"
@@ -37,6 +38,15 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				cert.Run),

pkg/cli/cmds/agent.go

@@ -56,9 +56,15 @@ type AgentShared struct {
 }
 
 var (
-	appName     = filepath.Base(os.Args[0])
-	AgentConfig Agent
-	NodeIPFlag  = cli.StringSliceFlag{
+	appName        = filepath.Base(os.Args[0])
+	AgentConfig    Agent
+	AgentTokenFlag = cli.StringFlag{
+		Name:        "token,t",
+		Usage:       "(cluster) Token to use for authentication",
+		EnvVar:      version.ProgramUpper + "_TOKEN",
+		Destination: &AgentConfig.Token,
+	}
+	NodeIPFlag = cli.StringSliceFlag{
 		Name:  "node-ip,i",
 		Usage: "(agent/networking) IPv4/IPv6 addresses to advertise for node",
 		Value: &AgentConfig.NodeIP,
@@ -217,12 +223,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			VModule,
 			LogFile,
 			AlsoLogToStderr,
-			cli.StringFlag{
-				Name:        "token,t",
-				Usage:       "(cluster) Token to use for authentication",
-				EnvVar:      version.ProgramUpper + "_TOKEN",
-				Destination: &AgentConfig.Token,
-			},
+			AgentTokenFlag,
 			cli.StringFlag{
 				Name:        "token-file",
 				Usage:       "(cluster) Token file to use for authentication",

pkg/cli/cmds/etcd_snapshot.go

@@ -20,11 +20,7 @@ var EtcdSnapshotFlags = []cli.Flag{
 		EnvVar:      version.ProgramUpper + "_NODE_NAME",
 		Destination: &AgentConfig.NodeName,
 	},
-	cli.StringFlag{
-		Name:        "data-dir,d",
-		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
-		Destination: &ServerConfig.DataDir,
-	},
+	DataDirFlag,
 	&cli.StringFlag{
 		Name:        "dir,etcd-snapshot-dir",
 		Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",

pkg/cli/cmds/secrets_encrypt.go

@@ -0,0 +1,94 @@
+package cmds
+
+import (
+	"github.com/urfave/cli"
+)
+
+const SecretsEncryptCommand = "secrets-encrypt"
+
+var EncryptFlags = []cli.Flag{
+	DataDirFlag,
+	ServerToken,
+}
+
+func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {
+	return cli.Command{
+		Name:            SecretsEncryptCommand,
+		Usage:           "Control secrets encryption and keys rotation",
+		SkipFlagParsing: false,
+		SkipArgReorder:  true,
+		Action:          action,
+		Subcommands:     subcommands,
+	}
+}
+
+func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command {
+	return []cli.Command{
+		{
+			Name:            "status",
+			Usage:           "Print current status of secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          status,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "enable",
+			Usage:           "Enable secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          enable,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "disable",
+			Usage:           "Disable secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          disable,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "prepare",
+			Usage:           "Prepare for encryption keys rotation",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          prepare,
+			Flags: append(EncryptFlags, &cli.BoolFlag{
+				Name:        "f,force",
+				Usage:       "Force preparation.",
+				Destination: &ServerConfig.EncryptForce,
+			}),
+		},
+		{
+			Name:            "rotate",
+			Usage:           "Rotate secrets encryption keys",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          rotate,
+			Flags: append(EncryptFlags, &cli.BoolFlag{
+				Name:        "f,force",
+				Usage:       "Force key rotation.",
+				Destination: &ServerConfig.EncryptForce,
+			}),
+		},
+		{
+			Name:            "reencrypt",
+			Usage:           "Reencrypt all data with new encryption key",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          reencrypt,
+			Flags: append(EncryptFlags,
+				&cli.BoolFlag{
+					Name:        "f,force",
+					Usage:       "Force secrets reencryption.",
+					Destination: &ServerConfig.EncryptForce,
+				},
+				&cli.BoolFlag{
+					Name:        "skip",
+					Usage:       "Skip removing old key",
+					Destination: &ServerConfig.EncryptSkip,
+				}),
+		},
+	}
+}

pkg/cli/cmds/server.go

@@ -74,6 +74,8 @@ type Server struct {
 	ClusterReset             bool
 	ClusterResetRestorePath  string
 	EncryptSecrets           bool
+	EncryptForce             bool
+	EncryptSkip              bool
 	SystemDefaultRegistry    string
 	StartupHooks             []StartupHook
 	EtcdSnapshotName         string
@@ -97,7 +99,18 @@ type Server struct {
 
 var (
 	ServerConfig Server
-	ClusterCIDR  = cli.StringSliceFlag{
+	DataDirFlag  = cli.StringFlag{
+		Name:        "data-dir,d",
+		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
+		Destination: &ServerConfig.DataDir,
+	}
+	ServerToken = cli.StringFlag{
+		Name:        "token,t",
+		Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
+		Destination: &ServerConfig.Token,
+		EnvVar:      version.ProgramUpper + "_TOKEN",
+	}
+	ClusterCIDR = cli.StringSliceFlag{
 		Name:  "cluster-cidr",
 		Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)",
 		Value: &ServerConfig.ClusterCIDR,
@@ -179,11 +192,7 @@ var ServerFlags = []cli.Flag{
 		Usage: "(listener) Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert",
 		Value: &ServerConfig.TLSSan,
 	},
-	cli.StringFlag{
-		Name:        "data-dir,d",
-		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
-		Destination: &ServerConfig.DataDir,
-	},
+	DataDirFlag,
 	ClusterCIDR,
 	ServiceCIDR,
 	ServiceNodePortRange,
@@ -195,12 +204,7 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.FlannelBackend,
 		Value:       "vxlan",
 	},
-	cli.StringFlag{
-		Name:        "token,t",
-		Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
-		Destination: &ServerConfig.Token,
-		EnvVar:      version.ProgramUpper + "_TOKEN",
-	},
+	ServerToken,
 	cli.StringFlag{
 		Name:        "token-file",
 		Usage:       "(cluster) File containing the cluster-secret/token",