diff --git a/go.mod b/go.mod index 4eeccbd071..fe75ddd028 100644 --- a/go.mod +++ b/go.mod @@ -136,3 +136,7 @@ require ( ) replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 + +// TODO: Remove this replace once library-go PRs #2085 and #2086 are merged +// Pulls from gangwgr:kms-test which includes KMS encryption mode and test scenarios +replace github.com/openshift/library-go => github.com/gangwgr/library-go v0.0.0-20260203130836-0f1824cf5c74 diff --git a/go.sum b/go.sum index 573c1a2ee8..6e27ce99bf 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gangwgr/library-go v0.0.0-20260203130836-0f1824cf5c74 h1:8q2HtwDJvFgwPvjkuaKPN7TEAwfdBKmd1cZIZprvCyc= +github.com/gangwgr/library-go v0.0.0-20260203130836-0f1824cf5c74/go.mod h1:DCRz1EgdayEmr9b6KXKDL+DWBN0rGHu/VYADeHzPoOk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -165,8 +167,6 @@ github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee h1:+S github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE= github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13 h1:6rd4zSo2UaWQcAPZfHK9yzKVqH0BnMv1hqMzqXZyTds= github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13/go.mod h1:YvOmPmV7wcJxpfhTDuFqqs2Xpb3M3ovsM6Qs/i2ptq4= -github.com/openshift/library-go v0.0.0-20260129122340-60005ae435eb h1:RCm3Kw8gPmalqT4a+O61YtVmj2nfEMIZZUSqfukNrM0= -github.com/openshift/library-go v0.0.0-20260129122340-60005ae435eb/go.mod h1:DCRz1EgdayEmr9b6KXKDL+DWBN0rGHu/VYADeHzPoOk= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 h1:PMTgifBcBRLJJiM+LgSzPDTk9/Rx4qS09OUrfpY6GBQ= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= diff --git a/test/e2e-encryption-kms/encryption_kms_test.go b/test/e2e-encryption-kms/encryption_kms_test.go index cb8ca759e3..2caf16b817 100644 --- a/test/e2e-encryption-kms/encryption_kms_test.go +++ b/test/e2e-encryption-kms/encryption_kms_test.go @@ -1,20 +1,47 @@ package e2e_encryption_kms import ( + "context" + "fmt" "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient" + operatorencryption "github.com/openshift/cluster-kube-apiserver-operator/test/library/encryption" + library "github.com/openshift/library-go/test/library/encryption" + librarykms "github.com/openshift/library-go/test/library/encryption/kms" ) // TestKMSEncryptionOnOff tests KMS encryption on/off cycle. // This test: // 1. Deploys the mock KMS plugin -// 2. Enables KMS encryption -// 3. Verifies secrets are encrypted -// 4. Disables encryption (Identity) -// 5. Verifies secrets are not encrypted -// 6. Re-enables KMS encryption -// 7. Cleans up -// -// TODO: Implement full KMS encryption test once the CI job is validated. +// 2. Creates a test secret (SecretOfLife) +// 3. Enables KMS encryption +// 4. Verifies secret is encrypted +// 5. Disables encryption (Identity) +// 6. Verifies secret is NOT encrypted +// 7. Re-enables KMS encryption +// 8. Verifies secret is encrypted again +// 9. Disables encryption (Identity) again +// 10. Verifies secret is NOT encrypted again +// 11. Cleans up the KMS plugin func TestKMSEncryptionOnOff(t *testing.T) { - t.Log("KMS encryption on/off test placeholder - CI job validation") + t.Cleanup(librarykms.DeployUpstreamMockKMSPlugin(context.Background(), t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage)) + library.TestEncryptionTurnOnAndOff(t, library.OnOffScenario{ + BasicScenario: library.BasicScenario{ + Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, + EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace), + EncryptionConfigSecretNamespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + OperatorNamespace: operatorclient.OperatorNamespace, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertSecretsAndConfigMaps, + }, + CreateResourceFunc: operatorencryption.CreateAndStoreSecretOfLife, + AssertResourceEncryptedFunc: operatorencryption.AssertSecretOfLifeEncrypted, + AssertResourceNotEncryptedFunc: operatorencryption.AssertSecretOfLifeNotEncrypted, + ResourceFunc: operatorencryption.SecretOfLife, + ResourceName: "SecretOfLife", + EncryptionProvider: configv1.EncryptionTypeKMS, + }) } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go index c999f140f0..2de95a6da4 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go @@ -27,6 +27,7 @@ import ( "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/encryption/crypto" + "github.com/openshift/library-go/pkg/operator/encryption/kms" "github.com/openshift/library-go/pkg/operator/encryption/secrets" "github.com/openshift/library-go/pkg/operator/encryption/state" "github.com/openshift/library-go/pkg/operator/encryption/statemachine" @@ -159,7 +160,7 @@ func (c *keyController) sync(ctx context.Context, syncCtx factory.SyncContext) ( } func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext factory.SyncContext, encryptedGRs []schema.GroupResource) error { - currentMode, externalReason, err := c.getCurrentModeAndExternalReason(ctx) + currentMode, externalReason, kmsConfig, err := c.getCurrentModeAndExternalReason(ctx) if err != nil { return err } @@ -191,7 +192,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact var commonReason *string for gr, grKeys := range desiredEncryptionState { - latestKeyID, internalReason, needed := needsNewKey(grKeys, currentMode, externalReason, encryptedGRs) + latestKeyID, internalReason, needed := needsNewKey(grKeys, currentMode, externalReason, encryptedGRs, kmsConfig) if !needed { continue } @@ -218,7 +219,8 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact sort.Sort(sort.StringSlice(reasons)) internalReason := strings.Join(reasons, ", ") - keySecret, err := c.generateKeySecret(newKeyID, currentMode, internalReason, externalReason) + + keySecret, err := c.generateKeySecret(newKeyID, currentMode, internalReason, externalReason, kmsConfig) if err != nil { return fmt.Errorf("failed to create key: %v", err) } @@ -255,8 +257,8 @@ func (c *keyController) validateExistingSecret(ctx context.Context, keySecret *c return nil // we made this key earlier } -func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, internalReason, externalReason string) (*corev1.Secret, error) { - bs := crypto.ModeToNewKeyFunc[currentMode]() +func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, internalReason, externalReason string, kmsConfig []byte) (*corev1.Secret, error) { + bs := crypto.ModeToNewKeyFunc[currentMode](kmsConfig) ks := state.KeyState{ Key: apiserverv1.Key{ Name: fmt.Sprintf("%d", keyID), @@ -265,40 +267,51 @@ func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, Mode: currentMode, InternalReason: internalReason, ExternalReason: externalReason, + KMSConfig: kmsConfig, } return secrets.FromKeyState(c.instanceName, ks) } -func (c *keyController) getCurrentModeAndExternalReason(ctx context.Context) (state.Mode, string, error) { +func (c *keyController) getCurrentModeAndExternalReason(ctx context.Context) (state.Mode, string, []byte, error) { apiServer, err := c.apiServerClient.Get(ctx, "cluster", metav1.GetOptions{}) if err != nil { - return "", "", err + return "", "", nil, err } operatorSpec, _, _, err := c.operatorClient.GetOperatorState() if err != nil { - return "", "", err + return "", "", nil, err } encryptionConfig, err := structuredUnsupportedConfigFrom(operatorSpec.UnsupportedConfigOverrides.Raw, c.unsupportedConfigPrefix) if err != nil { - return "", "", err + return "", "", nil, err } reason := encryptionConfig.Encryption.Reason switch currentMode := state.Mode(apiServer.Spec.Encryption.Type); currentMode { case state.AESCBC, state.AESGCM, state.Identity: // secretbox is disabled for now - return currentMode, reason, nil + return currentMode, reason, nil, nil + case state.KMS: + // KMS object is used to track any configurational changes to detect KMS to KMS migration. + // This object is generated from the configurable fields in APIServer config to store; + // * Endpoint is used in EncryptionConfiguration to be passed to apiservers. Currently, it is statically set to DefaultEndpoint + // * Other plugin specific configurations that are considered to trigger KMS migration + kmsConfig, err := kms.NewKMS(kms.DefaultEndpoint).ToBytes() + if err != nil { + return "", "", nil, err + } + return currentMode, reason, kmsConfig, nil case "": // unspecified means use the default (which can change over time) - return state.DefaultMode, reason, nil + return state.DefaultMode, reason, nil, nil default: - return "", "", fmt.Errorf("unknown encryption mode configured: %s", currentMode) + return "", "", nil, fmt.Errorf("unknown encryption mode configured: %s", currentMode) } } // needsNewKey checks whether a new key must be created for the given resource. If true, it also returns the latest // used key ID and a reason string. -func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, externalReason string, encryptedGRs []schema.GroupResource) (uint64, string, bool) { +func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, externalReason string, encryptedGRs []schema.GroupResource, kmsConfig []byte) (uint64, string, bool) { // we always need to have some encryption keys unless we are turned off if len(grKeys.ReadKeys) == 0 { return 0, "key-does-not-exist", currentMode != state.Identity @@ -341,6 +354,21 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern return 0, "", false } + if currentMode == state.KMS { + // We are here because Encryption Mode is not changed + secret := crypto.ModeToNewKeyFunc[state.KMS](kmsConfig) + if latestKey.Key.Secret != base64.StdEncoding.EncodeToString(secret) { + // If the current secret differs from the latestKey secret, that means KMS config is updated. + // Therefore, we need to initiate migration. + return latestKeyID, "kms-config-changed", true + } + // For KMS mode, we don't do time-based rotation. Therefore, we shortcut here + // KMS keys are rotated externally by the KMS system. + // Moreover, we don't trigger new key when external reason is changed. + // Because it would lead to duplicate providers which is not allowed. + return 0, "", false + } + // if the most recent secret has a different external reason than the current reason, we need to generate a new key if latestKey.ExternalReason != externalReason && len(externalReason) != 0 { return latestKeyID, "external-reason-changed", true diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/crypto/keys.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/crypto/keys.go index a623d30f79..dc664719c0 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/crypto/keys.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/crypto/keys.go @@ -2,20 +2,22 @@ package crypto import ( "crypto/rand" + "crypto/sha256" "github.com/openshift/library-go/pkg/operator/encryption/state" ) var ( - ModeToNewKeyFunc = map[state.Mode]func() []byte{ + ModeToNewKeyFunc = map[state.Mode]func(externalKey []byte) []byte{ state.AESCBC: NewAES256Key, state.AESGCM: NewAES256Key, state.SecretBox: NewAES256Key, // secretbox requires a 32 byte key so we can reuse the same function here state.Identity: NewIdentityKey, + state.KMS: NewKMSKey, } ) -func NewAES256Key() []byte { +func NewAES256Key(_ []byte) []byte { b := make([]byte, 32) // AES-256 == 32 byte key if _, err := rand.Read(b); err != nil { panic(err) // rand should never fail @@ -23,6 +25,13 @@ func NewAES256Key() []byte { return b } -func NewIdentityKey() []byte { +func NewIdentityKey(_ []byte) []byte { return make([]byte, 16) // the key is not used to perform encryption but must be a valid AES key } + +func NewKMSKey(externalKey []byte) []byte { + // Used as fixed-length identifier in EncryptionConfiguration provider names. + // Example: kms-secrets-1-a7K9mJ2xH4nQ8pL5vR3wTg== + hash := sha256.Sum256(externalKey) + return hash[:16] +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionconfig/config.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionconfig/config.go index 3082aa653f..25208a170e 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionconfig/config.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionconfig/config.go @@ -4,7 +4,9 @@ import ( "encoding/base64" "sort" + "github.com/openshift/library-go/pkg/operator/encryption/kms" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" "k8s.io/klog/v2" @@ -15,7 +17,7 @@ import ( ) var ( - emptyStaticIdentityKey = base64.StdEncoding.EncodeToString(crypto.NewIdentityKey()) + emptyStaticIdentityKey = base64.StdEncoding.EncodeToString(crypto.NewIdentityKey(nil)) ) // FromEncryptionState converts state to config. @@ -25,7 +27,7 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes for gr, grKeys := range encryptionState { resourceConfigs = append(resourceConfigs, apiserverconfigv1.ResourceConfiguration{ Resources: []string{gr.String()}, // we are forced to lose data here because this API is broken - Providers: stateToProviders(grKeys), + Providers: stateToProviders(gr.Resource, grKeys), }) } @@ -106,6 +108,22 @@ func ToEncryptionState(encryptionConfig *apiserverconfigv1.EncryptionConfigurati Mode: s, } + case provider.KMS != nil: + s := state.KMS + keyID, kmsKey, err := kms.FromProviderName(provider.KMS.Name) + if err != nil { + klog.Warningf("skipping invalid KMS provider %s: %v", provider.KMS.Name, err) + continue + } + + ks = state.KeyState{ + // We must generate exact Key with correct Name and Secret to find our matching backed Secret + Key: apiserverconfigv1.Key{ + Name: keyID, + Secret: kmsKey, + }, + Mode: s, + } default: klog.Infof("skipping invalid provider index %d for resource %s", i, resourceConfig.Resources[0]) continue // should never happen @@ -139,7 +157,7 @@ func ToEncryptionState(encryptionConfig *apiserverconfigv1.EncryptionConfigurati // it primarily handles the conversion of KeyState to the appropriate provider config. // the identity mode is transformed into a custom aesgcm provider that simply exists to // curry the associated null key secret through the encryption state machine. -func stateToProviders(desired state.GroupResourceState) []apiserverconfigv1.ProviderConfiguration { +func stateToProviders(resource string, desired state.GroupResourceState) []apiserverconfigv1.ProviderConfiguration { allKeys := desired.ReadKeys providers := make([]apiserverconfigv1.ProviderConfiguration, 0, len(allKeys)+1) // one extra for identity @@ -192,6 +210,25 @@ func stateToProviders(desired state.GroupResourceState) []apiserverconfigv1.Prov Keys: []apiserverconfigv1.Key{key.Key}, }, }) + case state.KMS: + kmsConfig, err := kms.FromBytes(key.KMSConfig) + if err != nil { + klog.Warningf("skipping provider as kms config is invalid: %v", err) + continue + } + provider := apiserverconfigv1.ProviderConfiguration{ + // This generated KMSConfiguration can be convertible to KeyState, + // since provider name contains the key id and secret. + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: kms.ToProviderName(resource, key.Key), + Endpoint: kmsConfig.Endpoint, + Timeout: &metav1.Duration{ + Duration: kms.DefaultTimeout, + }, + }, + } + providers = append(providers, provider) default: // this should never happen because our input should always be valid klog.Infof("skipping key %s as it has invalid mode %s", key.Key.Name, key.Mode) diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/kms.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/kms.go new file mode 100644 index 0000000000..8c261690a5 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/kms.go @@ -0,0 +1,68 @@ +package kms + +import ( + "encoding/json" + "fmt" + "regexp" + "time" + + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" +) + +const ( + DefaultEndpoint = "unix:///var/run/kmsplugin/kms.sock" + DefaultTimeout = 10 * time.Second +) + +// providerNameRegex matches KMS provider names in format: kms-{resource}-{keyID}-{keySecret} +// Example: "kms-secrets-1-XUFAKrxLKna5cZnZEQH8Ug==" +var providerNameRegex = regexp.MustCompile(`^kms-(.+)-(\d+)-([A-Za-z0-9+/=]+)$`) + +// KMS represents the configuration for an external Key Management Service provider. +// It contains the endpoint information needed to communicate with the KMS plugin. +// The configuration is serialized to JSON and stored in Kubernetes secrets. +type KMS struct { + Endpoint string `json:"endpoint"` +} + +// NewKMS creates a new KMS configuration with the specified endpoint. +func NewKMS(endpoint string) *KMS { + return &KMS{ + Endpoint: endpoint, + } +} + +// ToBytes serializes the KMS configuration to JSON. +func (k *KMS) ToBytes() ([]byte, error) { + jsonBytes, err := json.Marshal(k) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal KMS config: %w", err) + } + return jsonBytes, nil +} + +// FromBytes deserializes KMS configuration from JSON. +func FromBytes(data []byte) (*KMS, error) { + kms := &KMS{} + if err := json.Unmarshal(data, kms); err != nil { + return nil, fmt.Errorf("failed to unmarshal KMS config: %w", err) + } + return kms, nil +} + +// ToProviderName converts resource name, key ID, and KMS secret to KMS provider name format: kms-{resourceName}-{keyID}-{keySecret} +// Example: "kms-secrets-1-XUFAKrxLKna5cZnZEQH8Ug==" +func ToProviderName(resourceName string, key apiserverconfigv1.Key) string { + return fmt.Sprintf("kms-%s-%s-%s", resourceName, key.Name, key.Secret) +} + +// FromProviderName extracts the key ID and KMS Secret from a KMS provider name. +// Expected format: kms-{resourceName}-{keyID}-{keySecret} +func FromProviderName(providerName string) (keyID string, kmsKey string, err error) { + matches := providerNameRegex.FindStringSubmatch(providerName) + if len(matches) != 4 { + return "", "", fmt.Errorf("provider name %q has invalid format, expected kms-{resource}-{keyID}-{checksumBase64}", providerName) + } + + return matches[2], matches[3], nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go index 4e54317c7d..7c9a664179 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go @@ -58,9 +58,13 @@ func ToKeyState(s *corev1.Secret) (state.KeyState, error) { key.ExternalReason = v } + if v, ok := s.Annotations[EncryptionSecretKMSConfig]; ok && len(v) > 0 { + key.KMSConfig = []byte(v) + } + keyMode := state.Mode(s.Annotations[encryptionSecretMode]) switch keyMode { - case state.AESCBC, state.AESGCM, state.SecretBox, state.Identity: + case state.AESCBC, state.AESGCM, state.SecretBox, state.Identity, state.KMS: key.Mode = keyMode default: return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid mode: %s", s.Namespace, s.Name, keyMode) @@ -113,6 +117,10 @@ func FromKeyState(component string, ks state.KeyState) (*corev1.Secret, error) { s.Annotations[EncryptionSecretMigratedResources] = string(bs) } + if len(ks.KMSConfig) > 0 { + s.Annotations[EncryptionSecretKMSConfig] = string(ks.KMSConfig) + } + return s, nil } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go index 7161e4a124..443c7975e1 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go @@ -49,6 +49,9 @@ const ( // by the encryption controllers. Its sole purpose is to prevent the accidental // deletion of secrets by enforcing a two phase delete. EncryptionSecretFinalizer = "encryption.apiserver.operator.openshift.io/deletion-protection" + + // EncryptionSecretKMSConfig is the annotation that stores the encoded KMS configuration. + EncryptionSecretKMSConfig = "encryption.apiserver.operator.openshift.io/kms-config" ) // MigratedGroupResources is the data structured stored in the diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go index 460c21bfa2..844bd17ed7 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go @@ -40,6 +40,8 @@ type KeyState struct { InternalReason string // the user via unsupportConfigOverrides.encryption.reason triggered this key. ExternalReason string + // Encoded KMSConfig that stores the KMS related fields + KMSConfig []byte } type MigrationState struct { @@ -60,6 +62,7 @@ const ( AESGCM Mode = "aesgcm" SecretBox Mode = "secretbox" // available from the first release, see defaultMode below Identity Mode = "identity" // available from the first release, see defaultMode below + KMS Mode = "KMS" // only supports KMS v2 // Changing this value requires caution to not break downgrades. // Specifically, if some new Mode is released in version X, that new Mode cannot diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/assertion.go b/vendor/github.com/openshift/library-go/test/library/encryption/assertion.go index f4493e2df1..02261356ba 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/assertion.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/assertion.go @@ -34,6 +34,7 @@ const ( aesCBCTransformerPrefixV1 = "k8s:enc:aescbc:v1:" aesGCMTransformerPrefixV1 = "k8s:enc:aesgcm:v1:" secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:" + kmsTransformerPrefixV2 = "k8s:enc:kms:v2:" ) func init() { @@ -163,6 +164,8 @@ func encryptionModeFromEtcdValue(data []byte) (string, bool) { return "aesgcm" case hasPrefixAndTrailingData(data, []byte(secretboxTransformerPrefixV1)): // Secretbox has this prefix return "secretbox" + case hasPrefixAndTrailingData(data, []byte(kmsTransformerPrefixV2)): // KMS v2 has this prefix + return "KMS" case hasPrefixAndTrailingData(data, []byte(jsonEncodingPrefix)): // unencrypted json data has this prefix return "identity-json" case hasPrefixAndTrailingData(data, protoEncodingPrefix): // unencrypted protobuf data has this prefix diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_configmap.yaml b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_configmap.yaml new file mode 100644 index 0000000000..d9c3e9a0f6 --- /dev/null +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_configmap.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8s-mock-kms-plugin + namespace: {{ .Namespace }} +data: + softhsm-config.json: | + { + "Path": "/usr/lib/softhsm/libsofthsm2.so", + "TokenLabel": "kms-test", + "Pin": "1234" + } + # pre-generated SoftHSM token with AES-256 key. + # run ../k8s-mock-plugin-key-gen/generate.sh to regenerate. + softhsm-tokens.tar.gz.b64: | + H4sIAAAAAAAAA9PTZ6A5MDAwMDQ3NwfRIIBOg9mGpoYmQFVmpsZmQHFzcyMzBgVT2juNgaG0uCSx + SEGBoSg/vwSfOkLyQxTo6SclWSalGaYk6SanJVvqmhmaGemaGxumAlkG5gbmJmZmhpaplCUSUASb + Y8Y77vg3NDExH41/ugAi4z89NS+1KLEkMz+PDDtAEWyGL/4N0ePfzNTElEHBgOq+xQJGePxDAdNA + O2AUDAwgMv8D62+jNEsjS91EQ8tkXVOTlFRdy9REE13TxCQjU2PjRBMjixS9nPzkbGx2EMz/6OW/ + kZGhseFo/qcHoGb85ydlpSZjCSQC8W9oZGqGHv8mZqPlPz2BMhofVh+wQGlGGP0fTQFMgoEZjebI + zi3WLUkthgWaIJq8wtmYw16NO3743+n8IbrBQ654o8P1RS3cTE2rvqxes3f2681QdW3oFk1At9A9 + /CyaEpjj5KHiTGhaYEbC+HDfwb0L9SYjK7oAG7oAO7oAB7oAF7oAD7q1Ajjchx5mMPFEKAPmLwWo + eBK6wcnoVqegC6Sia0lDNVugASpegK6zEF2gCE2ACeYtiNkODEww/8CCGSYuhF0cFtYMrAyjgEaA + yPK/JD87NQ9nBU8AkFz/G5qZmBmMlv/0ACTFP64KngAgFP+mGPFvbmBgPBr/dATgOqqBIdgTyofX + 0rBaXAEHgOrzQtMnAEpCyCkIqs4bKg+tXVh0oeI+aPo9hLuLzFeazTs0z0uff0K8jyJbufak7QIT + v1jacNd/lLvW0Bink9F64W7jVjf1Y/o3JrhXvTgRdIo7qjH6Q94aa7Wis/x+aYIib+56Qc33RTd/ + mrLoLKtVa1bK/Mt6/MqI48T9mDkzbzYfleKNm8ytfO3yvnVnmdZzNTxQ5330z2/9zpt+B+dP+bn+ + UWq1aNfvZvMIhu2djYvqNbukYbXlKBgFo2AUDDkAAM1LQHIAGgAA \ No newline at end of file diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_daemonset.yaml b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_daemonset.yaml new file mode 100644 index 0000000000..1b670fbc1c --- /dev/null +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_daemonset.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: k8s-mock-kms-plugin + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: k8s-mock-kms-plugin + template: + metadata: + labels: + app: k8s-mock-kms-plugin + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + serviceAccountName: k8s-mock-kms-plugin + tolerations: + - operator: Exists + initContainers: + - name: init-softhsm + image: {{ .Image }} + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - /bin/sh + - -c + args: + - | + set -e + set -x + + # if token exists, skip initialization + if [ $(ls -1 /var/lib/softhsm/tokens 2>/dev/null | wc -l) -ge 1 ]; then + echo "Skipping initialization of softhsm" + exit 0 + fi + + mkdir -p /var/lib/softhsm/tokens + cd /var/lib/softhsm/tokens + + # extract tokens from the configmap + # see ../k8s-mock-plugin-key-gen/README.md for details. + cat /etc/softhsm-tokens.tar.gz.b64 | base64 -d | tar xzf - + volumeMounts: + - mountPath: /var/lib/softhsm/tokens + name: softhsm-tokens + - mountPath: /etc/softhsm-tokens.tar.gz.b64 + name: softhsm-config + subPath: softhsm-tokens.tar.gz.b64 + containers: + - name: kms-plugin + image: {{ .Image }} + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - /bin/sh + - -c + args: + - | + # remove the socket to prevent "bind: address already in use" + # not sure this is the best way + rm -f /var/run/kmsplugin/kms.sock + exec /usr/local/bin/mock-kms-plugin -listen-addr=unix:///var/run/kmsplugin/kms.sock -config-file-path=/etc/softhsm-config.json + volumeMounts: + - name: socket + mountPath: /var/run/kmsplugin + - name: softhsm-config + mountPath: /etc/softhsm-config.json + subPath: softhsm-config.json + - name: softhsm-tokens + mountPath: /var/lib/softhsm/tokens + volumes: + - name: socket + hostPath: + path: /var/run/kmsplugin + type: DirectoryOrCreate + - name: softhsm-tokens + hostPath: + path: /var/lib/softhsm/tokens + type: DirectoryOrCreate + - name: softhsm-config + configMap: + name: k8s-mock-kms-plugin diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_namespace.yaml b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_namespace.yaml new file mode 100644 index 0000000000..4141b1d701 --- /dev/null +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Namespace }} + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged \ No newline at end of file diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_rolebinding.yaml b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_rolebinding.yaml new file mode 100644 index 0000000000..9c42e8b826 --- /dev/null +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_rolebinding.yaml @@ -0,0 +1,16 @@ +# RoleBinding to grant the k8s-mock-kms-plugin ServiceAccount access to the +# privileged SCC. This is required because the KMS plugin needs privileged +# access to create the Unix socket on the host filesystem. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: k8s-mock-kms-plugin + namespace: {{ .Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:privileged +subjects: + - kind: ServiceAccount + name: k8s-mock-kms-plugin + namespace: {{ .Namespace }} diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_serviceaccount.yaml b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_serviceaccount.yaml new file mode 100644 index 0000000000..5eebaf6ad3 --- /dev/null +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/assets/k8s_mock_kms_plugin_serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8s-mock-kms-plugin + namespace: {{ .Namespace }} diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/k8s_mock_kms_plugin_deployer.go b/vendor/github.com/openshift/library-go/test/library/encryption/kms/k8s_mock_kms_plugin_deployer.go new file mode 100644 index 0000000000..223702951f --- /dev/null +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/k8s_mock_kms_plugin_deployer.go @@ -0,0 +1,244 @@ +package kms + +import ( + "bytes" + "context" + "embed" + "fmt" + "io" + "os" + "path/filepath" + "testing" + "text/template" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/clock" + + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" + "github.com/openshift/library-go/pkg/operator/resource/resourceread" +) + +//go:embed assets +var assetsFS embed.FS + +const ( + // WellKnownUpstreamMockKMSPluginNamespace is the default namespace where the KMS plugin runs. + WellKnownUpstreamMockKMSPluginNamespace = "k8s-mock-plugin" + + // WellKnownUpstreamMockKMSPluginImage is the pre-built mock KMS plugin image. + WellKnownUpstreamMockKMSPluginImage = "quay.io/openshifttest/mock-kms-plugin@sha256:998e1d48eba257f589ab86c30abd5043f662213e9aeff253e1c308301879d48a" + + // defaultPollTimeout the default poll timeout used by the deployer + defaultPollTimeout = 2 * time.Minute +) + +var manifestFilesToApplyDirectly = []string{ + "k8s_mock_kms_plugin_namespace.yaml", + "k8s_mock_kms_plugin_serviceaccount.yaml", + "k8s_mock_kms_plugin_rolebinding.yaml", + "k8s_mock_kms_plugin_configmap.yaml", +} + +var daemonSetManifestFile = "k8s_mock_kms_plugin_daemonset.yaml" + +// yamlTemplateData holds the template variables for YAML manifests. +// Fields must be exported (uppercase) for Go templates to access them. +type yamlTemplateData struct { + Namespace string + Image string +} + +// DeployUpstreamMockKMSPlugin deploys the upstream mock KMS v2 plugin using embedded YAML assets. +// It returns a cleanup function that removes the entire namespace where the DaemonSet was deployed. +func DeployUpstreamMockKMSPlugin(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace, image string) func() { + t.Helper() + + // TEMPORARILY DISABLED FOR TESTING - namespace deletion before deploy + // if err := destroyNamespaceIfNotExists(ctx, t, kubeClient, namespace); err != nil { + // t.Fatalf("Failed to cleanup existing namespace %q: %v", namespace, err) + // } + + t.Logf("Deploying upstream mock KMS v2 plugin in namespace %q using image %s", namespace, image) + daemonSetName, err := applyUpstreamMockKMSPluginManifests(ctx, t, kubeClient, namespace, image) + if err != nil { + t.Fatalf("Failed to apply manifests: %v", err) + } + if err := waitForDaemonSetReady(ctx, t, kubeClient, namespace, daemonSetName); err != nil { + t.Fatalf("DaemonSet not ready: %v", err) + } + t.Logf("Upstream mock KMS v2 plugin deployed successfully!") + + return func() { + // Before destroying the namespace, collect the logs of the pods in namespace + collectPodLogs(ctx, t, kubeClient, namespace) + + // TEMPORARILY DISABLED FOR TESTING - namespace deletion in cleanup + // if err := destroyNamespaceIfNotExists(ctx, t, kubeClient, namespace); err != nil { + // t.Errorf("Failed to cleanup namespace %q: %v", namespace, err) + // } + } +} + +// collectPodLogs collects logs from all pods in the namespace and writes them to ARTIFACT_DIR. +func collectPodLogs(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace string) { + t.Helper() + + artifactDir := os.Getenv("ARTIFACT_DIR") + if artifactDir == "" { + t.Log("artifact directory not set. Skipping collection of pod logs...") + return + } + + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Logf("Failed to list pods: %v", err) + return + } + if len(pods.Items) == 0 { + t.Logf("No pods found in %s", namespace) + return + } + + for _, pod := range pods.Items { + for _, container := range pod.Spec.Containers { + func() { + logFileName := filepath.Join(artifactDir, fmt.Sprintf("%s_%s_%s_%s.log", namespace, t.Name(), pod.Name, container.Name)) + + logOpts := &corev1.PodLogOptions{Container: container.Name} + logs, err := kubeClient.CoreV1().Pods(namespace).GetLogs(pod.Name, logOpts).Stream(ctx) + if err != nil { + t.Logf("Pod %s logs can not be captured err: %v", pod.Name, err) + return + } + defer logs.Close() + + logFile, err := os.Create(logFileName) + if err != nil { + t.Logf("creating log file %s failed: %v", logFileName, err) + return + } + defer logFile.Close() + + _, err = io.Copy(logFile, logs) + if err != nil { + t.Logf("failed to copying logs: %v", err) + } + }() + } + } +} + +// applyUpstreamMockKMSPluginManifests applies all the KMS plugin manifests. +// Returns the DaemonSet name on success. +func applyUpstreamMockKMSPluginManifests(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace, image string) (string, error) { + t.Helper() + + data := yamlTemplateData{ + Namespace: namespace, + Image: image, + } + + recorder := events.NewInMemoryRecorder("k8s-mock-kms-plugin-deployer", clock.RealClock{}) + assetFunc := wrapAssetWithTemplateDataFunc(data) + + clientHolder := resourceapply.NewKubeClientHolder(kubeClient) + results := resourceapply.ApplyDirectly(ctx, clientHolder, recorder, resourceapply.NewResourceCache(), assetFunc, manifestFilesToApplyDirectly...) + + for _, result := range results { + if result.Error != nil { + return "", result.Error + } + t.Logf("Applied %s (changed=%v)", result.File, result.Changed) + } + + rawDaemonSet, err := assetFunc(daemonSetManifestFile) + if err != nil { + return "", err + } + + daemonSet := resourceread.ReadDaemonSetV1OrDie(rawDaemonSet) + _, _, err = resourceapply.ApplyDaemonSet(ctx, kubeClient.AppsV1(), recorder, daemonSet, -1) + if err != nil { + return "", err + } + t.Logf("Applied DaemonSet %s/%s", namespace, daemonSet.Name) + + return daemonSet.Name, nil +} + +// waitForDaemonSetReady waits for the KMS plugin DaemonSet to be ready. +func waitForDaemonSetReady(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace, daemonSetName string) error { + t.Helper() + + t.Logf("Waiting for DaemonSet %s/%s to be ready...", namespace, daemonSetName) + + return wait.PollUntilContextTimeout(ctx, time.Second, defaultPollTimeout, true, func(ctx context.Context) (bool, error) { + ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, daemonSetName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + t.Logf("DaemonSet %s/%s status: desired=%d, ready=%d, available=%d", + namespace, daemonSetName, ds.Status.DesiredNumberScheduled, ds.Status.NumberReady, ds.Status.NumberAvailable) + + // for simplicity just ensure at least one pod is scheduled before checking readiness + if ds.Status.DesiredNumberScheduled == 0 { + return false, nil + } + return ds.Status.NumberReady == ds.Status.DesiredNumberScheduled, nil + }) +} + +// destroyNamespaceIfNotExists removes the namespace and waits for deletion. +func destroyNamespaceIfNotExists(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace string) error { + t.Helper() + + t.Logf("Deleting namespace %q", namespace) + err := kubeClient.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + return wait.PollUntilContextTimeout(ctx, time.Second, defaultPollTimeout, true, func(ctx context.Context) (bool, error) { + _, err := kubeClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + t.Logf("Namespace %q deleted", namespace) + return true, nil + } + return false, nil + }) +} + +// wrapAssetWithTemplateDataFunc returns an AssetFunc that templates the YAML with the given data. +func wrapAssetWithTemplateDataFunc(data yamlTemplateData) resourceapply.AssetFunc { + return func(name string) ([]byte, error) { + content, err := assetsFS.ReadFile(filepath.Join("assets", name)) + if err != nil { + return nil, err + } + + tmpl, err := template.New(name).Parse(string(content)) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return nil, err + } + + return buf.Bytes(), nil + } +} diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go b/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go index 99d60bfe49..79a9ff96ef 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go @@ -50,12 +50,21 @@ func TestEncryptionTypeAESGCM(t *testing.T, scenario BasicScenario) { AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) } +func TestEncryptionTypeKMS(t *testing.T, scenario BasicScenario) { + e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) + clientSet := SetAndWaitForEncryptionType(e, configv1.EncryptionTypeKMS, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + scenario.AssertFunc(e, clientSet, configv1.EncryptionTypeKMS, scenario.Namespace, scenario.LabelSelector) + AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) +} + func TestEncryptionType(t *testing.T, scenario BasicScenario, provider configv1.EncryptionType) { switch provider { case configv1.EncryptionTypeAESCBC: TestEncryptionTypeAESCBC(t, scenario) case configv1.EncryptionTypeAESGCM: TestEncryptionTypeAESGCM(t, scenario) + case configv1.EncryptionTypeKMS: + TestEncryptionTypeKMS(t, scenario) case configv1.EncryptionTypeIdentity, "": TestEncryptionTypeIdentity(t, scenario) default: diff --git a/vendor/modules.txt b/vendor/modules.txt index 39f77609a4..00c0629614 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -406,7 +406,7 @@ github.com/openshift/client-go/security/informers/externalversions/internalinter github.com/openshift/client-go/security/informers/externalversions/security github.com/openshift/client-go/security/informers/externalversions/security/v1 github.com/openshift/client-go/security/listers/security/v1 -# github.com/openshift/library-go v0.0.0-20260129122340-60005ae435eb +# github.com/openshift/library-go v0.0.0-20260129122340-60005ae435eb => github.com/gangwgr/library-go v0.0.0-20260203130836-0f1824cf5c74 ## explicit; go 1.24.0 github.com/openshift/library-go/pkg/apiserver/jsonpatch github.com/openshift/library-go/pkg/assets @@ -496,6 +496,7 @@ github.com/openshift/library-go/pkg/serviceability github.com/openshift/library-go/test/library github.com/openshift/library-go/test/library/apiserver github.com/openshift/library-go/test/library/encryption +github.com/openshift/library-go/test/library/encryption/kms # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors @@ -1688,3 +1689,4 @@ sigs.k8s.io/structured-merge-diff/v6/value ## explicit; go 1.22 sigs.k8s.io/yaml # github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 +# github.com/openshift/library-go => github.com/gangwgr/library-go v0.0.0-20260203130836-0f1824cf5c74