From 8eaf235d1e96035f62b5e3e55141053968e77106 Mon Sep 17 00:00:00 2001 From: Yury Nikitenko Date: Thu, 6 Aug 2026 17:43:25 +0300 Subject: [PATCH 1/3] ARO-28729 No preflight validation that Key Vault is accessible for etcd KMS encryption --- backend/go.mod | 1 + backend/go.sum | 2 + backend/pkg/app/backend.go | 7 + .../pkg/azure/client/fpa_client_builder.go | 11 + backend/pkg/azure/client/keyvault_client.go | 30 +++ .../azure/client/mock_fpa_client_builder.go | 39 ++++ .../pkg/azure/client/mock_keyvault_client.go | 81 +++++++ ...uster_keyvault_accessibility_validation.go | 118 +++++++++++ ..._keyvault_accessibility_validation_test.go | 200 ++++++++++++++++++ test-integration/go.mod | 1 + test-integration/go.sum | 2 + 11 files changed, 492 insertions(+) create mode 100644 backend/pkg/azure/client/keyvault_client.go create mode 100644 backend/pkg/azure/client/mock_keyvault_client.go create mode 100644 backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go create mode 100644 backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go diff --git a/backend/go.mod b/backend/go.mod index 766fbb5102d..3bdf06d0a2e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -9,6 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.5.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.3.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 diff --git a/backend/go.sum b/backend/go.sum index 651fd02823d..ed2b96e93a9 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -18,6 +18,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFG github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 h1:1kpY4qe+BGAH2ykv4baVSqyx+AY5VjXeJ15SldlU6hs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1/go.mod h1:nT6cWpWdUt+g81yuKmjeYPUtI73Ak3yQIT4PVVsCEEQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 h1:akP6VpxJGgQRpDR1P462piz/8OhYLRCreDj48AyNabc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0/go.mod h1:8wzvopPfyZYPaQUoKW87Zfdul7jmJMDfp/k7YY3oJyA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index 6d6d1d10107..657cd7b3630 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -826,6 +826,12 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, serviceProviderClusterLister, backendInformers, ) + azureClusterKeyVaultAccessibilityValidationController := clustervalidation.NewClusterValidationController( + validationutils.NewAzureClusterKeyVaultAccessibilityValidation(b.options.FPAClientBuilder), + b.options.ResourcesDBClient, + serviceProviderClusterLister, + backendInformers, + ) azureVMSizeSupportsEphemeralOSDiskValidationController := nodepoolvalidation.NewNodePoolValidationController( validationutils.NewAzureVMSizeSupportsEphemeralOSDiskValidation(virtualMachineResourceSKUsCachedReaderController), b.options.ResourcesDBClient, @@ -1080,6 +1086,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, go azureRPRegistrationValidationController.Run(ctx, 20) go azureClusterResourceGroupExistenceValidationController.Run(ctx, 20) go azureClusterManagedIdentitiesExistenceValidationController.Run(ctx, 20) + go azureClusterKeyVaultAccessibilityValidationController.Run(ctx, 20) go azureVMSizeSupportsEphemeralOSDiskValidationController.Run(ctx, 20) go azureNodePoolVMQuotaValidationController.Run(ctx, 20) go controlPlaneIdentitiesPermissionsValidationController.Run(ctx, 20) diff --git a/backend/pkg/azure/client/fpa_client_builder.go b/backend/pkg/azure/client/fpa_client_builder.go index e33f897eca6..7deb5dad376 100644 --- a/backend/pkg/azure/client/fpa_client_builder.go +++ b/backend/pkg/azure/client/fpa_client_builder.go @@ -19,6 +19,7 @@ package client import ( azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/Azure/ARO-HCP/internal/fpa" @@ -47,6 +48,7 @@ type FirstPartyApplicationClientBuilder interface { ResourceProvidersClient(tenantID string, subscriptionID string) (ResourceProvidersClient, error) ResourceSKUsClient(tenantID string, subscriptionID string) (ResourceSKUsClient, error) UsageClient(tenantID string, subscriptionID string) (UsageClient, error) + KeyVaultVaultsClient(tenantID string, subscriptionID string) (KeyVaultVaultsClient, error) } type firstPartyApplicationClientBuilder struct { @@ -103,6 +105,15 @@ func (b *firstPartyApplicationClientBuilder) UsageClient(tenantID string, subscr return armcompute.NewUsageClient(subscriptionID, creds, b.options) } +func (b *firstPartyApplicationClientBuilder) KeyVaultVaultsClient(tenantID string, subscriptionID string) (KeyVaultVaultsClient, error) { + creds, err := b.fpaTokenCredRetriever.RetrieveCredential(tenantID) + if err != nil { + return nil, err + } + + return armkeyvault.NewVaultsClient(subscriptionID, creds, b.options) +} + func (b *firstPartyApplicationClientBuilder) BuilderType() FirstPartyApplicationClientBuilderType { return FirstPartyApplicationClientBuilderTypeValue } diff --git a/backend/pkg/azure/client/keyvault_client.go b/backend/pkg/azure/client/keyvault_client.go new file mode 100644 index 00000000000..26a9ffda019 --- /dev/null +++ b/backend/pkg/azure/client/keyvault_client.go @@ -0,0 +1,30 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +//go:generate $MOCKGEN -typed -source=keyvault_client.go -destination=mock_keyvault_client.go -package client KeyVaultVaultsClient + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" +) + +type KeyVaultVaultsClient interface { + Get(ctx context.Context, resourceGroupName string, vaultName string, + options *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error) +} + +var _ KeyVaultVaultsClient = (*armkeyvault.VaultsClient)(nil) diff --git a/backend/pkg/azure/client/mock_fpa_client_builder.go b/backend/pkg/azure/client/mock_fpa_client_builder.go index 1ce150e9fb7..ab6dd06a574 100644 --- a/backend/pkg/azure/client/mock_fpa_client_builder.go +++ b/backend/pkg/azure/client/mock_fpa_client_builder.go @@ -77,6 +77,45 @@ func (c *MockFirstPartyApplicationClientBuilderBuilderTypeCall) DoAndReturn(f fu return c } +// KeyVaultVaultsClient mocks base method. +func (m *MockFirstPartyApplicationClientBuilder) KeyVaultVaultsClient(tenantID, subscriptionID string) (KeyVaultVaultsClient, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KeyVaultVaultsClient", tenantID, subscriptionID) + ret0, _ := ret[0].(KeyVaultVaultsClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// KeyVaultVaultsClient indicates an expected call of KeyVaultVaultsClient. +func (mr *MockFirstPartyApplicationClientBuilderMockRecorder) KeyVaultVaultsClient(tenantID, subscriptionID any) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyVaultVaultsClient", reflect.TypeOf((*MockFirstPartyApplicationClientBuilder)(nil).KeyVaultVaultsClient), tenantID, subscriptionID) + return &MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall{Call: call} +} + +// MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall wrap *gomock.Call +type MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall) Return(arg0 KeyVaultVaultsClient, arg1 error) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall) Do(f func(string, string) (KeyVaultVaultsClient, error)) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall) DoAndReturn(f func(string, string) (KeyVaultVaultsClient, error)) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // ResourceGroupsClient mocks base method. func (m *MockFirstPartyApplicationClientBuilder) ResourceGroupsClient(tenantID, subscriptionID string) (ResourceGroupsClient, error) { m.ctrl.T.Helper() diff --git a/backend/pkg/azure/client/mock_keyvault_client.go b/backend/pkg/azure/client/mock_keyvault_client.go new file mode 100644 index 00000000000..ace0b55dca4 --- /dev/null +++ b/backend/pkg/azure/client/mock_keyvault_client.go @@ -0,0 +1,81 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: keyvault_client.go +// +// Generated by this command: +// +// mockgen-v0.6.0 -typed -source=keyvault_client.go -destination=mock_keyvault_client.go -package client KeyVaultVaultsClient +// + +// Package client is a generated GoMock package. +package client + +import ( + context "context" + reflect "reflect" + + armkeyvault "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" + gomock "go.uber.org/mock/gomock" +) + +// MockKeyVaultVaultsClient is a mock of KeyVaultVaultsClient interface. +type MockKeyVaultVaultsClient struct { + ctrl *gomock.Controller + recorder *MockKeyVaultVaultsClientMockRecorder + isgomock struct{} +} + +// MockKeyVaultVaultsClientMockRecorder is the mock recorder for MockKeyVaultVaultsClient. +type MockKeyVaultVaultsClientMockRecorder struct { + mock *MockKeyVaultVaultsClient +} + +// NewMockKeyVaultVaultsClient creates a new mock instance. +func NewMockKeyVaultVaultsClient(ctrl *gomock.Controller) *MockKeyVaultVaultsClient { + mock := &MockKeyVaultVaultsClient{ctrl: ctrl} + mock.recorder = &MockKeyVaultVaultsClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockKeyVaultVaultsClient) EXPECT() *MockKeyVaultVaultsClientMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockKeyVaultVaultsClient) Get(ctx context.Context, resourceGroupName, vaultName string, options *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, vaultName, options) + ret0, _ := ret[0].(armkeyvault.VaultsClientGetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockKeyVaultVaultsClientMockRecorder) Get(ctx, resourceGroupName, vaultName, options any) *MockKeyVaultVaultsClientGetCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKeyVaultVaultsClient)(nil).Get), ctx, resourceGroupName, vaultName, options) + return &MockKeyVaultVaultsClientGetCall{Call: call} +} + +// MockKeyVaultVaultsClientGetCall wrap *gomock.Call +type MockKeyVaultVaultsClientGetCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockKeyVaultVaultsClientGetCall) Return(arg0 armkeyvault.VaultsClientGetResponse, arg1 error) *MockKeyVaultVaultsClientGetCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockKeyVaultVaultsClientGetCall) Do(f func(context.Context, string, string, *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error)) *MockKeyVaultVaultsClientGetCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockKeyVaultVaultsClientGetCall) DoAndReturn(f func(context.Context, string, string, *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error)) *MockKeyVaultVaultsClientGetCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go new file mode 100644 index 00000000000..82319859c82 --- /dev/null +++ b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go @@ -0,0 +1,118 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validationutils + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" + + azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/utils" +) + +// AzureClusterKeyVaultAccessibilityValidation validates that the Azure Key Vault +// used for etcd KMS encryption exists and is network-accessible. +type AzureClusterKeyVaultAccessibilityValidation struct { + azureFPAClientBuilder azureclient.FirstPartyApplicationClientBuilder +} + +func NewAzureClusterKeyVaultAccessibilityValidation( + azureFPAClientBuilder azureclient.FirstPartyApplicationClientBuilder, +) *AzureClusterKeyVaultAccessibilityValidation { + return &AzureClusterKeyVaultAccessibilityValidation{ + azureFPAClientBuilder: azureFPAClientBuilder, + } +} + +func (a *AzureClusterKeyVaultAccessibilityValidation) Name() string { + return "AzureClusterKeyVaultAccessibilityValidation" +} + +func (a *AzureClusterKeyVaultAccessibilityValidation) Validate( + ctx context.Context, clusterSubscription *arm.Subscription, cluster *api.HCPOpenShiftCluster, +) error { + kmsProfile := customerManagedKmsProfile(cluster) + if kmsProfile == nil { + return nil + } + + vaultsClient, err := a.azureFPAClientBuilder.KeyVaultVaultsClient( + *clusterSubscription.Properties.TenantId, + cluster.ID.SubscriptionID, + ) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get key vault client: %w", err)) + } + + resp, err := vaultsClient.Get(ctx, cluster.ID.ResourceGroupName, kmsProfile.ActiveKey.VaultName, nil) + if err != nil { + return utils.TrackError(fmt.Errorf( + "key vault %q is not accessible in resource group %q: %w", + kmsProfile.ActiveKey.VaultName, cluster.ID.ResourceGroupName, err)) + } + + if err := validateKeyVaultNetworkAccess(resp.Vault, kmsProfile); err != nil { + return utils.TrackError(err) + } + + return nil +} + +func validateKeyVaultNetworkAccess(vault armkeyvault.Vault, kmsProfile *api.KmsEncryptionProfile) error { + if vault.Properties == nil { + return fmt.Errorf("key vault %q has no properties", *vault.Name) + } + + publicAccess := vault.Properties.PublicNetworkAccess + isPublicDisabled := publicAccess != nil && strings.EqualFold(*publicAccess, "Disabled") + + if !isPublicDisabled { + return nil + } + + if kmsProfile.Visibility == api.KeyVaultVisibilityPublic { + return fmt.Errorf( + "key vault %q has public network access disabled, but the cluster is configured with key vault visibility %q. "+ + "Either enable public network access on the key vault or configure the cluster with private key vault visibility "+ + "and ensure a private endpoint and DNS are properly configured", + kmsProfile.ActiveKey.VaultName, api.KeyVaultVisibilityPublic) + } + + hasPrivateEndpoints := len(vault.Properties.PrivateEndpointConnections) > 0 + if !hasPrivateEndpoints { + return fmt.Errorf( + "key vault %q has public network access disabled but no private endpoint connections are configured. "+ + "Configure a private endpoint and DNS entries so the service can reach the key vault", + kmsProfile.ActiveKey.VaultName) + } + + return nil +} + +func customerManagedKmsProfile(cluster *api.HCPOpenShiftCluster) *api.KmsEncryptionProfile { + etcd := cluster.CustomerProperties.Etcd + if etcd.DataEncryption.KeyManagementMode != api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { + return nil + } + if etcd.DataEncryption.CustomerManaged == nil { + return nil + } + return etcd.DataEncryption.CustomerManaged.Kms +} diff --git a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go new file mode 100644 index 00000000000..cd1faf0dc77 --- /dev/null +++ b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go @@ -0,0 +1,200 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validationutils + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "k8s.io/utils/ptr" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" + + azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" +) + +const ( + testKeyVaultName = "test-kv" + testKeyName = "etcd-data-kms-encryption-key" + testKeyVersion = "abc123" +) + +func newTestCluster(t *testing.T, keyManagementMode api.EtcdDataEncryptionKeyManagementModeType, visibility api.KeyVaultVisibility) *api.HCPOpenShiftCluster { + t.Helper() + resourceID := api.Must(azcorearm.ParseResourceID( + "/subscriptions/" + testSubscriptionID + + "/resourceGroups/" + testResourceGroup + + "/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/" + testClusterName)) + + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: resourceID, + Name: testClusterName, + }, + Location: "eastus", + }, + } + + if keyManagementMode == api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { + cluster.CustomerProperties.Etcd = api.EtcdProfile{ + DataEncryption: api.EtcdDataEncryptionProfile{ + KeyManagementMode: api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, + CustomerManaged: &api.CustomerManagedEncryptionProfile{ + EncryptionType: api.CustomerManagedEncryptionTypeKMS, + Kms: &api.KmsEncryptionProfile{ + Visibility: visibility, + ActiveKey: api.KmsKey{ + Name: testKeyName, + VaultName: testKeyVaultName, + Version: testKeyVersion, + }, + }, + }, + }, + } + } + + return cluster +} + +func makeVaultResponse(publicNetworkAccess string, privateEndpointCount int) armkeyvault.VaultsClientGetResponse { + var peConnections []*armkeyvault.PrivateEndpointConnectionItem + for i := range privateEndpointCount { + peConnections = append(peConnections, &armkeyvault.PrivateEndpointConnectionItem{ + ID: ptr.To("pe-" + string(rune('0'+i))), + }) + } + + return armkeyvault.VaultsClientGetResponse{ + Vault: armkeyvault.Vault{ + Name: ptr.To(testKeyVaultName), + Properties: &armkeyvault.VaultProperties{ + PublicNetworkAccess: ptr.To(publicNetworkAccess), + PrivateEndpointConnections: peConnections, + }, + }, + } +} + +func TestAzureClusterKeyVaultAccessibilityValidation_Name(t *testing.T) { + validation := NewAzureClusterKeyVaultAccessibilityValidation(nil) + assert.Equal(t, "AzureClusterKeyVaultAccessibilityValidation", validation.Name()) +} + +func TestAzureClusterKeyVaultAccessibilityValidation_Validate(t *testing.T) { + ctx := context.Background() + subscription := newTestSubscription() + + tests := []struct { + name string + cluster *api.HCPOpenShiftCluster + setupMock func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) + wantErr string + wantNoAzCall bool + }{ + { + name: "skips validation when not using customer-managed encryption", + cluster: newTestCluster(t, "", api.KeyVaultVisibilityPublic), + wantNoAzCall: true, + }, + { + name: "succeeds when vault is publicly accessible", + cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), + setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { + builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) + vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). + Return(makeVaultResponse("Enabled", 0), nil) + }, + }, + { + name: "succeeds when vault is private with private endpoints", + cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPrivate), + setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { + builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) + vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). + Return(makeVaultResponse("Disabled", 1), nil) + }, + }, + { + name: "fails when vault has public access disabled but cluster configured for public visibility", + cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), + setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { + builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) + vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). + Return(makeVaultResponse("Disabled", 0), nil) + }, + wantErr: `key vault "test-kv" has public network access disabled, but the cluster is configured with key vault visibility "Public"`, + }, + { + name: "fails when vault is private with no private endpoints", + cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPrivate), + setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { + builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) + vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). + Return(makeVaultResponse("Disabled", 0), nil) + }, + wantErr: `key vault "test-kv" has public network access disabled but no private endpoint connections are configured`, + }, + { + name: "fails when vault does not exist", + cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), + setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { + builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) + vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). + Return(armkeyvault.VaultsClientGetResponse{}, errors.New("vault not found")) + }, + wantErr: `key vault "test-kv" is not accessible in resource group "test-rg"`, + }, + { + name: "fails when client builder returns error", + cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), + setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { + builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(nil, errors.New("credential error")) + }, + wantErr: "failed to get key vault client", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockBuilder := azureclient.NewMockFirstPartyApplicationClientBuilder(ctrl) + mockVaultsClient := azureclient.NewMockKeyVaultVaultsClient(ctrl) + + if tt.setupMock != nil { + tt.setupMock(mockBuilder, mockVaultsClient) + } + + validation := NewAzureClusterKeyVaultAccessibilityValidation(mockBuilder) + err := validation.Validate(ctx, subscription, tt.cluster) + + if tt.wantErr == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + assert.ErrorContains(t, err, tt.wantErr) + } + }) + } +} diff --git a/test-integration/go.mod b/test-integration/go.mod index beff315537e..9b832fd45a4 100644 --- a/test-integration/go.mod +++ b/test-integration/go.mod @@ -33,6 +33,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0 // indirect diff --git a/test-integration/go.sum b/test-integration/go.sum index bf92547a79e..9d76eb72746 100644 --- a/test-integration/go.sum +++ b/test-integration/go.sum @@ -25,6 +25,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFG github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 h1:1kpY4qe+BGAH2ykv4baVSqyx+AY5VjXeJ15SldlU6hs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1/go.mod h1:nT6cWpWdUt+g81yuKmjeYPUtI73Ak3yQIT4PVVsCEEQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 h1:akP6VpxJGgQRpDR1P462piz/8OhYLRCreDj48AyNabc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0/go.mod h1:8wzvopPfyZYPaQUoKW87Zfdul7jmJMDfp/k7YY3oJyA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= From 63fac025ab9450b6f8e8b8328049221ca0e7e06c Mon Sep 17 00:00:00 2001 From: Yury Nikitenko Date: Wed, 12 Aug 2026 11:18:52 +0300 Subject: [PATCH 2/3] ARO-28729 keyvault validation changes after review --- backend/go.mod | 2 +- backend/go.sum | 4 +- backend/pkg/app/backend.go | 2 +- .../pkg/azure/client/fpa_client_builder.go | 11 - backend/pkg/azure/client/keyvault_client.go | 15 +- .../azure/client/mock_fpa_client_builder.go | 39 ---- .../pkg/azure/client/mock_keyvault_client.go | 52 ++--- .../azure/client/mock_smi_client_builder.go | 39 ++++ .../pkg/azure/client/smi_client_builder.go | 58 ++++-- ...uster_keyvault_accessibility_validation.go | 118 +++++------ ..._keyvault_accessibility_validation_test.go | 196 +++++++++--------- test-integration/go.mod | 2 +- test-integration/go.sum | 4 +- test/go.mod | 2 +- test/go.sum | 3 +- tooling/secret-sync/go.mod | 2 +- tooling/secret-sync/go.sum | 3 +- tooling/templatize/go.mod | 2 +- tooling/templatize/go.sum | 3 +- 19 files changed, 278 insertions(+), 279 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 3bdf06d0a2e..efc59b49933 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -9,10 +9,10 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.5.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.3.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 github.com/Azure/checkaccess-v2-go-sdk v0.0.4-0.20260707173829-4b27d1696d22 diff --git a/backend/go.sum b/backend/go.sum index ed2b96e93a9..4fe916c61e1 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -18,8 +18,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFG github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 h1:1kpY4qe+BGAH2ykv4baVSqyx+AY5VjXeJ15SldlU6hs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1/go.mod h1:nT6cWpWdUt+g81yuKmjeYPUtI73Ak3yQIT4PVVsCEEQ= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 h1:akP6VpxJGgQRpDR1P462piz/8OhYLRCreDj48AyNabc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0/go.mod h1:8wzvopPfyZYPaQUoKW87Zfdul7jmJMDfp/k7YY3oJyA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= @@ -34,6 +32,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1/go.mod h1:8h8yhzh9o+0HeSIhUxYny+rEQajScrfIpNktvgYG3Q8= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index 657cd7b3630..708a8252eb7 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -827,7 +827,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, backendInformers, ) azureClusterKeyVaultAccessibilityValidationController := clustervalidation.NewClusterValidationController( - validationutils.NewAzureClusterKeyVaultAccessibilityValidation(b.options.FPAClientBuilder), + validationutils.NewAzureClusterKeyVaultAccessibilityValidation(b.options.SMIClientBuilder), b.options.ResourcesDBClient, serviceProviderClusterLister, backendInformers, diff --git a/backend/pkg/azure/client/fpa_client_builder.go b/backend/pkg/azure/client/fpa_client_builder.go index 7deb5dad376..e33f897eca6 100644 --- a/backend/pkg/azure/client/fpa_client_builder.go +++ b/backend/pkg/azure/client/fpa_client_builder.go @@ -19,7 +19,6 @@ package client import ( azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/Azure/ARO-HCP/internal/fpa" @@ -48,7 +47,6 @@ type FirstPartyApplicationClientBuilder interface { ResourceProvidersClient(tenantID string, subscriptionID string) (ResourceProvidersClient, error) ResourceSKUsClient(tenantID string, subscriptionID string) (ResourceSKUsClient, error) UsageClient(tenantID string, subscriptionID string) (UsageClient, error) - KeyVaultVaultsClient(tenantID string, subscriptionID string) (KeyVaultVaultsClient, error) } type firstPartyApplicationClientBuilder struct { @@ -105,15 +103,6 @@ func (b *firstPartyApplicationClientBuilder) UsageClient(tenantID string, subscr return armcompute.NewUsageClient(subscriptionID, creds, b.options) } -func (b *firstPartyApplicationClientBuilder) KeyVaultVaultsClient(tenantID string, subscriptionID string) (KeyVaultVaultsClient, error) { - creds, err := b.fpaTokenCredRetriever.RetrieveCredential(tenantID) - if err != nil { - return nil, err - } - - return armkeyvault.NewVaultsClient(subscriptionID, creds, b.options) -} - func (b *firstPartyApplicationClientBuilder) BuilderType() FirstPartyApplicationClientBuilderType { return FirstPartyApplicationClientBuilderTypeValue } diff --git a/backend/pkg/azure/client/keyvault_client.go b/backend/pkg/azure/client/keyvault_client.go index 26a9ffda019..aecc6a6ac52 100644 --- a/backend/pkg/azure/client/keyvault_client.go +++ b/backend/pkg/azure/client/keyvault_client.go @@ -14,17 +14,20 @@ package client -//go:generate $MOCKGEN -typed -source=keyvault_client.go -destination=mock_keyvault_client.go -package client KeyVaultVaultsClient +//go:generate $MOCKGEN -typed -source=keyvault_client.go -destination=mock_keyvault_client.go -package client KeyVaultKeysClient import ( "context" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" ) -type KeyVaultVaultsClient interface { - Get(ctx context.Context, resourceGroupName string, vaultName string, - options *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error) +// KeyVaultKeysClient is the Key Vault Keys data plane client, addressed +// directly by vault DNS name rather than by ARM resource ID. This lets +// callers reach a customer-owned Key Vault without needing to know which +// resource group or subscription it lives in. +type KeyVaultKeysClient interface { + GetKey(ctx context.Context, name string, version string, options *azkeys.GetKeyOptions) (azkeys.GetKeyResponse, error) } -var _ KeyVaultVaultsClient = (*armkeyvault.VaultsClient)(nil) +var _ KeyVaultKeysClient = (*azkeys.Client)(nil) diff --git a/backend/pkg/azure/client/mock_fpa_client_builder.go b/backend/pkg/azure/client/mock_fpa_client_builder.go index ab6dd06a574..1ce150e9fb7 100644 --- a/backend/pkg/azure/client/mock_fpa_client_builder.go +++ b/backend/pkg/azure/client/mock_fpa_client_builder.go @@ -77,45 +77,6 @@ func (c *MockFirstPartyApplicationClientBuilderBuilderTypeCall) DoAndReturn(f fu return c } -// KeyVaultVaultsClient mocks base method. -func (m *MockFirstPartyApplicationClientBuilder) KeyVaultVaultsClient(tenantID, subscriptionID string) (KeyVaultVaultsClient, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "KeyVaultVaultsClient", tenantID, subscriptionID) - ret0, _ := ret[0].(KeyVaultVaultsClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// KeyVaultVaultsClient indicates an expected call of KeyVaultVaultsClient. -func (mr *MockFirstPartyApplicationClientBuilderMockRecorder) KeyVaultVaultsClient(tenantID, subscriptionID any) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyVaultVaultsClient", reflect.TypeOf((*MockFirstPartyApplicationClientBuilder)(nil).KeyVaultVaultsClient), tenantID, subscriptionID) - return &MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall{Call: call} -} - -// MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall wrap *gomock.Call -type MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall) Return(arg0 KeyVaultVaultsClient, arg1 error) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall) Do(f func(string, string) (KeyVaultVaultsClient, error)) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall) DoAndReturn(f func(string, string) (KeyVaultVaultsClient, error)) *MockFirstPartyApplicationClientBuilderKeyVaultVaultsClientCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // ResourceGroupsClient mocks base method. func (m *MockFirstPartyApplicationClientBuilder) ResourceGroupsClient(tenantID, subscriptionID string) (ResourceGroupsClient, error) { m.ctrl.T.Helper() diff --git a/backend/pkg/azure/client/mock_keyvault_client.go b/backend/pkg/azure/client/mock_keyvault_client.go index ace0b55dca4..619d7fd1a88 100644 --- a/backend/pkg/azure/client/mock_keyvault_client.go +++ b/backend/pkg/azure/client/mock_keyvault_client.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen-v0.6.0 -typed -source=keyvault_client.go -destination=mock_keyvault_client.go -package client KeyVaultVaultsClient +// mockgen-v0.6.0 -typed -source=keyvault_client.go -destination=mock_keyvault_client.go -package client KeyVaultKeysClient // // Package client is a generated GoMock package. @@ -13,69 +13,69 @@ import ( context "context" reflect "reflect" - armkeyvault "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" + azkeys "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" gomock "go.uber.org/mock/gomock" ) -// MockKeyVaultVaultsClient is a mock of KeyVaultVaultsClient interface. -type MockKeyVaultVaultsClient struct { +// MockKeyVaultKeysClient is a mock of KeyVaultKeysClient interface. +type MockKeyVaultKeysClient struct { ctrl *gomock.Controller - recorder *MockKeyVaultVaultsClientMockRecorder + recorder *MockKeyVaultKeysClientMockRecorder isgomock struct{} } -// MockKeyVaultVaultsClientMockRecorder is the mock recorder for MockKeyVaultVaultsClient. -type MockKeyVaultVaultsClientMockRecorder struct { - mock *MockKeyVaultVaultsClient +// MockKeyVaultKeysClientMockRecorder is the mock recorder for MockKeyVaultKeysClient. +type MockKeyVaultKeysClientMockRecorder struct { + mock *MockKeyVaultKeysClient } -// NewMockKeyVaultVaultsClient creates a new mock instance. -func NewMockKeyVaultVaultsClient(ctrl *gomock.Controller) *MockKeyVaultVaultsClient { - mock := &MockKeyVaultVaultsClient{ctrl: ctrl} - mock.recorder = &MockKeyVaultVaultsClientMockRecorder{mock} +// NewMockKeyVaultKeysClient creates a new mock instance. +func NewMockKeyVaultKeysClient(ctrl *gomock.Controller) *MockKeyVaultKeysClient { + mock := &MockKeyVaultKeysClient{ctrl: ctrl} + mock.recorder = &MockKeyVaultKeysClientMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockKeyVaultVaultsClient) EXPECT() *MockKeyVaultVaultsClientMockRecorder { +func (m *MockKeyVaultKeysClient) EXPECT() *MockKeyVaultKeysClientMockRecorder { return m.recorder } -// Get mocks base method. -func (m *MockKeyVaultVaultsClient) Get(ctx context.Context, resourceGroupName, vaultName string, options *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error) { +// GetKey mocks base method. +func (m *MockKeyVaultKeysClient) GetKey(ctx context.Context, name, version string, options *azkeys.GetKeyOptions) (azkeys.GetKeyResponse, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, vaultName, options) - ret0, _ := ret[0].(armkeyvault.VaultsClientGetResponse) + ret := m.ctrl.Call(m, "GetKey", ctx, name, version, options) + ret0, _ := ret[0].(azkeys.GetKeyResponse) ret1, _ := ret[1].(error) return ret0, ret1 } -// Get indicates an expected call of Get. -func (mr *MockKeyVaultVaultsClientMockRecorder) Get(ctx, resourceGroupName, vaultName, options any) *MockKeyVaultVaultsClientGetCall { +// GetKey indicates an expected call of GetKey. +func (mr *MockKeyVaultKeysClientMockRecorder) GetKey(ctx, name, version, options any) *MockKeyVaultKeysClientGetKeyCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKeyVaultVaultsClient)(nil).Get), ctx, resourceGroupName, vaultName, options) - return &MockKeyVaultVaultsClientGetCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKey", reflect.TypeOf((*MockKeyVaultKeysClient)(nil).GetKey), ctx, name, version, options) + return &MockKeyVaultKeysClientGetKeyCall{Call: call} } -// MockKeyVaultVaultsClientGetCall wrap *gomock.Call -type MockKeyVaultVaultsClientGetCall struct { +// MockKeyVaultKeysClientGetKeyCall wrap *gomock.Call +type MockKeyVaultKeysClientGetKeyCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockKeyVaultVaultsClientGetCall) Return(arg0 armkeyvault.VaultsClientGetResponse, arg1 error) *MockKeyVaultVaultsClientGetCall { +func (c *MockKeyVaultKeysClientGetKeyCall) Return(arg0 azkeys.GetKeyResponse, arg1 error) *MockKeyVaultKeysClientGetKeyCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockKeyVaultVaultsClientGetCall) Do(f func(context.Context, string, string, *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error)) *MockKeyVaultVaultsClientGetCall { +func (c *MockKeyVaultKeysClientGetKeyCall) Do(f func(context.Context, string, string, *azkeys.GetKeyOptions) (azkeys.GetKeyResponse, error)) *MockKeyVaultKeysClientGetKeyCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockKeyVaultVaultsClientGetCall) DoAndReturn(f func(context.Context, string, string, *armkeyvault.VaultsClientGetOptions) (armkeyvault.VaultsClientGetResponse, error)) *MockKeyVaultVaultsClientGetCall { +func (c *MockKeyVaultKeysClientGetKeyCall) DoAndReturn(f func(context.Context, string, string, *azkeys.GetKeyOptions) (azkeys.GetKeyResponse, error)) *MockKeyVaultKeysClientGetKeyCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/backend/pkg/azure/client/mock_smi_client_builder.go b/backend/pkg/azure/client/mock_smi_client_builder.go index dd41fed1a29..fdf1d1bb0b5 100644 --- a/backend/pkg/azure/client/mock_smi_client_builder.go +++ b/backend/pkg/azure/client/mock_smi_client_builder.go @@ -79,6 +79,45 @@ func (c *MockServiceManagedIdentityClientBuilderBuilderTypeCall) DoAndReturn(f f return c } +// KeyVaultKeysClient mocks base method. +func (m *MockServiceManagedIdentityClientBuilder) KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, identityResourceID *arm.ResourceID, vaultName string) (KeyVaultKeysClient, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KeyVaultKeysClient", ctx, clusterIdentityURL, identityResourceID, vaultName) + ret0, _ := ret[0].(KeyVaultKeysClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// KeyVaultKeysClient indicates an expected call of KeyVaultKeysClient. +func (mr *MockServiceManagedIdentityClientBuilderMockRecorder) KeyVaultKeysClient(ctx, clusterIdentityURL, identityResourceID, vaultName any) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyVaultKeysClient", reflect.TypeOf((*MockServiceManagedIdentityClientBuilder)(nil).KeyVaultKeysClient), ctx, clusterIdentityURL, identityResourceID, vaultName) + return &MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall{Call: call} +} + +// MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall wrap *gomock.Call +type MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall) Return(arg0 KeyVaultKeysClient, arg1 error) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall) Do(f func(context.Context, string, *arm.ResourceID, string) (KeyVaultKeysClient, error)) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall) DoAndReturn(f func(context.Context, string, *arm.ResourceID, string) (KeyVaultKeysClient, error)) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // SubnetsClient mocks base method. func (m *MockServiceManagedIdentityClientBuilder) SubnetsClient(ctx context.Context, clusterIdentityURL string, smiResourceID *arm.ResourceID, subscriptionID string) (SubnetsClient, error) { m.ctrl.T.Helper() diff --git a/backend/pkg/azure/client/smi_client_builder.go b/backend/pkg/azure/client/smi_client_builder.go index 01095d0758c..cbc9ef00ad7 100644 --- a/backend/pkg/azure/client/smi_client_builder.go +++ b/backend/pkg/azure/client/smi_client_builder.go @@ -24,6 +24,7 @@ import ( azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" "github.com/Azure/msi-dataplane/pkg/dataplane" "github.com/Azure/ARO-HCP/internal/utils" @@ -50,6 +51,12 @@ type ServiceManagedIdentityClientBuilder interface { UserAssignedIdentitiesClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (UserAssignedIdentitiesClient, error) // SubnetsClient returns a new Subnet client. SubnetsClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (SubnetsClient, error) + // KeyVaultKeysClient returns a new Key Vault Keys data plane client, + // authenticated as the given customer-granted managed identity (e.g. a + // cluster operator identity such as "kms") rather than as the SMI. This + // is the only credential path that has RBAC access to a customer-owned + // Key Vault; the FPA and SMI do not. + KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, identityResourceID *azcorearm.ResourceID, vaultName string) (KeyVaultKeysClient, error) } type serviceManagedIdentityClientBuilder struct { @@ -63,42 +70,57 @@ func (b *serviceManagedIdentityClientBuilder) BuilderType() ServiceManagedIdenti return ServiceManagedIdentityClientBuilderTypeValue } -// credentialsForServiceManagedIdentity acquires an azcore.TokenCredential for the cluster's -// Service Managed Identity by going through the Managed Identities Data Plane. -func (b *serviceManagedIdentityClientBuilder) credentialsForServiceManagedIdentity(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID) (azcore.TokenCredential, error) { - miDataplaneClient, err := b.fpaMIdataplaneClientBuilder.ManagedIdentitiesDataplane(clusterIdentityURL) +func (b *serviceManagedIdentityClientBuilder) UserAssignedIdentitiesClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (UserAssignedIdentitiesClient, error) { + creds, err := b.credentialForIdentity(ctx, clusterIdentityURL, smiResourceID) if err != nil { return nil, err } - dataplaneRequest := dataplane.UserAssignedIdentitiesRequest{ - IdentityIDs: []string{smiResourceID.String()}, - } - resp, err := miDataplaneClient.GetUserAssignedIdentitiesCredentials(ctx, dataplaneRequest) + return armmsi.NewUserAssignedIdentitiesClient(subscriptionID, creds, b.azCoreARMClientOptions) +} + +func (b *serviceManagedIdentityClientBuilder) SubnetsClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (SubnetsClient, error) { + creds, err := b.credentialForIdentity(ctx, clusterIdentityURL, smiResourceID) if err != nil { return nil, err } - if len(resp.ExplicitIdentities) == 0 { - return nil, utils.TrackError(fmt.Errorf("managed identities data plane returned no credentials for the cluster's service managed identity '%s'", smiResourceID.String())) + return armnetwork.NewSubnetsClient(subscriptionID, creds, b.azCoreARMClientOptions) +} + +func (b *serviceManagedIdentityClientBuilder) KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, identityResourceID *azcorearm.ResourceID, vaultName string) (KeyVaultKeysClient, error) { + creds, err := b.credentialForIdentity(ctx, clusterIdentityURL, identityResourceID) + if err != nil { + return nil, err } - return dataplane.GetCredential(b.azCoreARMClientOptions.ClientOptions, resp.ExplicitIdentities[0]) + vaultURL := fmt.Sprintf("https://%s.vault.azure.net/", vaultName) + return azkeys.NewClient(vaultURL, creds, nil) } -func (b *serviceManagedIdentityClientBuilder) UserAssignedIdentitiesClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (UserAssignedIdentitiesClient, error) { - creds, err := b.credentialsForServiceManagedIdentity(ctx, clusterIdentityURL, smiResourceID) +// credentialForIdentity obtains an azcore.TokenCredential for the given +// customer-granted managed identity, via the Managed Identities Data Plane +// Service, using the cluster's identity URL. It works for any identity the +// cluster has been granted (the SMI, or a cluster operator identity like +// "kms") -- the data plane request is keyed by resource ID alone. +func (b *serviceManagedIdentityClientBuilder) credentialForIdentity(ctx context.Context, clusterIdentityURL string, identityResourceID *azcorearm.ResourceID) (azcore.TokenCredential, error) { + miDataplaneClient, err := b.fpaMIdataplaneClientBuilder.ManagedIdentitiesDataplane(clusterIdentityURL) if err != nil { return nil, err } - return armmsi.NewUserAssignedIdentitiesClient(subscriptionID, creds, b.azCoreARMClientOptions) -} -func (b *serviceManagedIdentityClientBuilder) SubnetsClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (SubnetsClient, error) { - creds, err := b.credentialsForServiceManagedIdentity(ctx, clusterIdentityURL, smiResourceID) + dataplaneRequest := dataplane.UserAssignedIdentitiesRequest{ + IdentityIDs: []string{identityResourceID.String()}, + } + resp, err := miDataplaneClient.GetUserAssignedIdentitiesCredentials(ctx, dataplaneRequest) if err != nil { return nil, err } - return armnetwork.NewSubnetsClient(subscriptionID, creds, b.azCoreARMClientOptions) + if len(resp.ExplicitIdentities) == 0 { + return nil, + utils.TrackError(fmt.Errorf("managed identities data plane returned no credentials for identity '%s'", identityResourceID.String())) + } + + return dataplane.GetCredential(b.azCoreARMClientOptions.ClientOptions, resp.ExplicitIdentities[0]) } func NewServiceManagedIdentityClientBuilder(fpaMIdataplaneClientBuilder FPAMIDataplaneClientBuilder, options *azcorearm.ClientOptions) ServiceManagedIdentityClientBuilder { diff --git a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go index 82319859c82..0bfdbfb87a1 100644 --- a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go +++ b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go @@ -17,27 +17,27 @@ package validationutils import ( "context" "fmt" - "strings" - - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" - "github.com/Azure/ARO-HCP/internal/api" - "github.com/Azure/ARO-HCP/internal/api/arm" - "github.com/Azure/ARO-HCP/internal/utils" + "github.com/Azure/ARO-HCP/internal/api/coreapi" + "github.com/Azure/ARO-HCP/internal/api/metadataapi" + internalazure "github.com/Azure/ARO-HCP/internal/azure" ) // AzureClusterKeyVaultAccessibilityValidation validates that the Azure Key Vault -// used for etcd KMS encryption exists and is network-accessible. +// used for customer-managed etcd KMS encryption is reachable by the cluster's +// KMS operator managed identity -- the same identity, over the same data +// plane path, that etcd itself uses to perform the real encrypt/decrypt +// operations. type AzureClusterKeyVaultAccessibilityValidation struct { - azureFPAClientBuilder azureclient.FirstPartyApplicationClientBuilder + smiClientBuilder azureclient.ServiceManagedIdentityClientBuilder } func NewAzureClusterKeyVaultAccessibilityValidation( - azureFPAClientBuilder azureclient.FirstPartyApplicationClientBuilder, + smiClientBuilder azureclient.ServiceManagedIdentityClientBuilder, ) *AzureClusterKeyVaultAccessibilityValidation { return &AzureClusterKeyVaultAccessibilityValidation{ - azureFPAClientBuilder: azureFPAClientBuilder, + smiClientBuilder: smiClientBuilder, } } @@ -45,74 +45,64 @@ func (a *AzureClusterKeyVaultAccessibilityValidation) Name() string { return "AzureClusterKeyVaultAccessibilityValidation" } -func (a *AzureClusterKeyVaultAccessibilityValidation) Validate( - ctx context.Context, clusterSubscription *arm.Subscription, cluster *api.HCPOpenShiftCluster, -) error { - kmsProfile := customerManagedKmsProfile(cluster) - if kmsProfile == nil { - return nil +func (a *AzureClusterKeyVaultAccessibilityValidation) Validate(ctx context.Context, _ *coreapi.Subscription, cluster *coreapi.HCPOpenShiftCluster) ValidationResult { + if cluster.CustomerProperties.Etcd.DataEncryption.KeyManagementMode != metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { + return PassedValidation(coreapi.ControllerConditionReasonAsExpected, "As expected", "Cluster does not use customer-managed KMS encryption.") } - vaultsClient, err := a.azureFPAClientBuilder.KeyVaultVaultsClient( - *clusterSubscription.Properties.TenantId, - cluster.ID.SubscriptionID, - ) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to get key vault client: %w", err)) + // API validation (DiscriminatedUnion) guarantees CustomerManaged != nil + // when KeyManagementMode == CustomerManaged. Guard against data corruption. + cm := cluster.CustomerProperties.Etcd.DataEncryption.CustomerManaged + if cm == nil { + return UnknownValidation( + "InternalError", + "Unable to verify Key Vault accessibility.", + "customer-managed key management mode is set but customer-managed encryption profile is nil", + ControllerReportingPolicyTypeError, + ) } - resp, err := vaultsClient.Get(ctx, cluster.ID.ResourceGroupName, kmsProfile.ActiveKey.VaultName, nil) - if err != nil { - return utils.TrackError(fmt.Errorf( - "key vault %q is not accessible in resource group %q: %w", - kmsProfile.ActiveKey.VaultName, cluster.ID.ResourceGroupName, err)) + if cm.EncryptionType != metadataapi.CustomerManagedEncryptionTypeKMS { + return PassedValidation(coreapi.ControllerConditionReasonAsExpected, "As expected", "Cluster does not use KMS encryption.") } - if err := validateKeyVaultNetworkAccess(resp.Vault, kmsProfile); err != nil { - return utils.TrackError(err) + // API validation (DiscriminatedUnion) guarantees Kms != nil + // when EncryptionType == KMS. Guard against data corruption. + if cm.Kms == nil { + return UnknownValidation( + "InternalError", + "Unable to verify Key Vault accessibility.", + "KMS encryption type is set but KMS encryption profile is nil", + ControllerReportingPolicyTypeError, + ) } - return nil -} + kmsProfile := cm.Kms -func validateKeyVaultNetworkAccess(vault armkeyvault.Vault, kmsProfile *api.KmsEncryptionProfile) error { - if vault.Properties == nil { - return fmt.Errorf("key vault %q has no properties", *vault.Name) + kmsIdentityResourceID := cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators[string(internalazure.ClusterOperatorIdentifierKMS)] + if kmsIdentityResourceID == nil { + internalAndUserMsg := fmt.Sprintf("cluster has no %q operator identity configured", internalazure.ClusterOperatorIdentifierKMS) + return FailedValidation("KmsIdentityNotConfigured", internalAndUserMsg, internalAndUserMsg) } - publicAccess := vault.Properties.PublicNetworkAccess - isPublicDisabled := publicAccess != nil && strings.EqualFold(*publicAccess, "Disabled") - - if !isPublicDisabled { - return nil - } + clusterIdentityURL := cluster.ServiceProviderProperties.ManagedIdentitiesDataPlaneIdentityURL - if kmsProfile.Visibility == api.KeyVaultVisibilityPublic { - return fmt.Errorf( - "key vault %q has public network access disabled, but the cluster is configured with key vault visibility %q. "+ - "Either enable public network access on the key vault or configure the cluster with private key vault visibility "+ - "and ensure a private endpoint and DNS are properly configured", - kmsProfile.ActiveKey.VaultName, api.KeyVaultVisibilityPublic) + keysClient, err := a.smiClientBuilder.KeyVaultKeysClient(ctx, clusterIdentityURL, kmsIdentityResourceID, kmsProfile.ActiveKey.VaultName) + if err != nil { + return UnknownValidation( + "InternalError", + "Unable to verify Key Vault accessibility.", + fmt.Sprintf("failed to get key vault client: %s", err), + ControllerReportingPolicyTypeError, + ) } - hasPrivateEndpoints := len(vault.Properties.PrivateEndpointConnections) > 0 - if !hasPrivateEndpoints { - return fmt.Errorf( - "key vault %q has public network access disabled but no private endpoint connections are configured. "+ - "Configure a private endpoint and DNS entries so the service can reach the key vault", - kmsProfile.ActiveKey.VaultName) + if _, err := keysClient.GetKey(ctx, kmsProfile.ActiveKey.Name, kmsProfile.ActiveKey.Version, nil); err != nil { + internalAndUserMsg := fmt.Sprintf( + "key vault %q is not accessible to the cluster's KMS operator identity: %s", + kmsProfile.ActiveKey.VaultName, err) + return FailedValidation("KeyVaultNotAccessible", internalAndUserMsg, internalAndUserMsg) } - return nil -} - -func customerManagedKmsProfile(cluster *api.HCPOpenShiftCluster) *api.KmsEncryptionProfile { - etcd := cluster.CustomerProperties.Etcd - if etcd.DataEncryption.KeyManagementMode != api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { - return nil - } - if etcd.DataEncryption.CustomerManaged == nil { - return nil - } - return etcd.DataEncryption.CustomerManaged.Kms + return PassedValidation(coreapi.ControllerConditionReasonAsExpected, "As expected", "Key Vault is accessible to the cluster's KMS operator identity.") } diff --git a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go index cd1faf0dc77..4d1c44c1fba 100644 --- a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go +++ b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go @@ -23,48 +23,57 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "k8s.io/utils/ptr" - azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" - "github.com/Azure/ARO-HCP/internal/api" - "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/api/coreapi" + "github.com/Azure/ARO-HCP/internal/api/metadataapi" + internalazure "github.com/Azure/ARO-HCP/internal/azure" ) const ( - testKeyVaultName = "test-kv" - testKeyName = "etcd-data-kms-encryption-key" - testKeyVersion = "abc123" + testKeyVaultName = "test-kv" + testKeyName = "etcd-data-kms-encryption-key" + testKeyVersion = "abc123" + testClusterIdentityURL = "https://identity.example.com/cluster" + testKmsIdentityResourceID = "/subscriptions/" + testSubscriptionID + "/resourceGroups/" + testResourceGroup + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity" ) -func newTestCluster(t *testing.T, keyManagementMode api.EtcdDataEncryptionKeyManagementModeType, visibility api.KeyVaultVisibility) *api.HCPOpenShiftCluster { +func newTestCluster(t *testing.T, keyManagementMode metadataapi.EtcdDataEncryptionKeyManagementModeType, configureKmsIdentity bool) *coreapi.HCPOpenShiftCluster { t.Helper() - resourceID := api.Must(azcorearm.ParseResourceID( + resourceID := metadataapi.Must(azcorearm.ParseResourceID( "/subscriptions/" + testSubscriptionID + "/resourceGroups/" + testResourceGroup + "/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/" + testClusterName)) - cluster := &api.HCPOpenShiftCluster{ - TrackedResource: arm.TrackedResource{ - Resource: arm.Resource{ + cluster := &coreapi.HCPOpenShiftCluster{ + TrackedResource: coreapi.TrackedResource{ + Resource: coreapi.Resource{ ID: resourceID, Name: testClusterName, }, Location: "eastus", }, } + cluster.ServiceProviderProperties.ManagedIdentitiesDataPlaneIdentityURL = testClusterIdentityURL - if keyManagementMode == api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { - cluster.CustomerProperties.Etcd = api.EtcdProfile{ - DataEncryption: api.EtcdDataEncryptionProfile{ - KeyManagementMode: api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, - CustomerManaged: &api.CustomerManagedEncryptionProfile{ - EncryptionType: api.CustomerManagedEncryptionTypeKMS, - Kms: &api.KmsEncryptionProfile{ - Visibility: visibility, - ActiveKey: api.KmsKey{ + if configureKmsIdentity { + kmsResourceID := metadataapi.Must(azcorearm.ParseResourceID(testKmsIdentityResourceID)) + cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators = map[string]*azcorearm.ResourceID{ + string(internalazure.ClusterOperatorIdentifierKMS): kmsResourceID, + } + } + + if keyManagementMode == metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { + cluster.CustomerProperties.Etcd = coreapi.EtcdProfile{ + DataEncryption: coreapi.EtcdDataEncryptionProfile{ + KeyManagementMode: metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, + CustomerManaged: &coreapi.CustomerManagedEncryptionProfile{ + EncryptionType: metadataapi.CustomerManagedEncryptionTypeKMS, + Kms: &coreapi.KmsEncryptionProfile{ + Visibility: metadataapi.KeyVaultVisibilityPublic, + ActiveKey: coreapi.KmsKey{ Name: testKeyName, VaultName: testKeyVaultName, Version: testKeyVersion, @@ -78,25 +87,6 @@ func newTestCluster(t *testing.T, keyManagementMode api.EtcdDataEncryptionKeyMan return cluster } -func makeVaultResponse(publicNetworkAccess string, privateEndpointCount int) armkeyvault.VaultsClientGetResponse { - var peConnections []*armkeyvault.PrivateEndpointConnectionItem - for i := range privateEndpointCount { - peConnections = append(peConnections, &armkeyvault.PrivateEndpointConnectionItem{ - ID: ptr.To("pe-" + string(rune('0'+i))), - }) - } - - return armkeyvault.VaultsClientGetResponse{ - Vault: armkeyvault.Vault{ - Name: ptr.To(testKeyVaultName), - Properties: &armkeyvault.VaultProperties{ - PublicNetworkAccess: ptr.To(publicNetworkAccess), - PrivateEndpointConnections: peConnections, - }, - }, - } -} - func TestAzureClusterKeyVaultAccessibilityValidation_Name(t *testing.T) { validation := NewAzureClusterKeyVaultAccessibilityValidation(nil) assert.Equal(t, "AzureClusterKeyVaultAccessibilityValidation", validation.Name()) @@ -107,93 +97,101 @@ func TestAzureClusterKeyVaultAccessibilityValidation_Validate(t *testing.T) { subscription := newTestSubscription() tests := []struct { - name string - cluster *api.HCPOpenShiftCluster - setupMock func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) - wantErr string - wantNoAzCall bool + name string + cluster *coreapi.HCPOpenShiftCluster + setupMock func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) + wantOutcome OutcomeType + wantReason string }{ { - name: "skips validation when not using customer-managed encryption", - cluster: newTestCluster(t, "", api.KeyVaultVisibilityPublic), - wantNoAzCall: true, + name: "skips validation when not using customer-managed encryption", + cluster: newTestCluster(t, "", false), + wantOutcome: OutcomeTypePassed, }, { - name: "succeeds when vault is publicly accessible", - cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), - setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { - builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) - vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). - Return(makeVaultResponse("Enabled", 0), nil) - }, + name: "returns unknown when customer-managed mode is set but profile is nil", + cluster: func() *coreapi.HCPOpenShiftCluster { + c := newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true) + c.CustomerProperties.Etcd.DataEncryption.CustomerManaged = nil + return c + }(), + wantOutcome: OutcomeTypeUnknown, + wantReason: "InternalError", }, { - name: "succeeds when vault is private with private endpoints", - cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPrivate), - setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { - builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) - vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). - Return(makeVaultResponse("Disabled", 1), nil) - }, + name: "skips validation for non-KMS encryption types", + cluster: func() *coreapi.HCPOpenShiftCluster { + c := newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true) + c.CustomerProperties.Etcd.DataEncryption.CustomerManaged.EncryptionType = "FutureType" + c.CustomerProperties.Etcd.DataEncryption.CustomerManaged.Kms = nil + return c + }(), + wantOutcome: OutcomeTypePassed, }, { - name: "fails when vault has public access disabled but cluster configured for public visibility", - cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), - setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { - builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) - vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). - Return(makeVaultResponse("Disabled", 0), nil) - }, - wantErr: `key vault "test-kv" has public network access disabled, but the cluster is configured with key vault visibility "Public"`, + name: "returns unknown when KMS encryption type is set but Kms profile is nil", + cluster: func() *coreapi.HCPOpenShiftCluster { + c := newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true) + c.CustomerProperties.Etcd.DataEncryption.CustomerManaged.Kms = nil + return c + }(), + wantOutcome: OutcomeTypeUnknown, + wantReason: "InternalError", }, { - name: "fails when vault is private with no private endpoints", - cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPrivate), - setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { - builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) - vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). - Return(makeVaultResponse("Disabled", 0), nil) + name: "succeeds when the key vault is accessible to the KMS operator identity", + cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), + setupMock: func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(keysClient, nil) + keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, nil) }, - wantErr: `key vault "test-kv" has public network access disabled but no private endpoint connections are configured`, + wantOutcome: OutcomeTypePassed, + }, + { + name: "fails when the cluster has no KMS operator identity configured", + cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, false), + wantOutcome: OutcomeTypeFailed, + wantReason: "KmsIdentityNotConfigured", }, { - name: "fails when vault does not exist", - cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), - setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { - builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(vaultsClient, nil) - vaultsClient.EXPECT().Get(gomock.Any(), testResourceGroup, testKeyVaultName, nil). - Return(armkeyvault.VaultsClientGetResponse{}, errors.New("vault not found")) + name: "reports unknown when the client builder returns an error", + cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), + setupMock: func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(nil, errors.New("credential error")) }, - wantErr: `key vault "test-kv" is not accessible in resource group "test-rg"`, + wantOutcome: OutcomeTypeUnknown, + wantReason: "InternalError", }, { - name: "fails when client builder returns error", - cluster: newTestCluster(t, api.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, api.KeyVaultVisibilityPublic), - setupMock: func(builder *azureclient.MockFirstPartyApplicationClientBuilder, vaultsClient *azureclient.MockKeyVaultVaultsClient) { - builder.EXPECT().KeyVaultVaultsClient(testTenantID, testSubscriptionID).Return(nil, errors.New("credential error")) + name: "fails when the key vault is not accessible to the KMS operator identity", + cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), + setupMock: func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(keysClient, nil) + keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, errors.New("access denied")) }, - wantErr: "failed to get key vault client", + wantOutcome: OutcomeTypeFailed, + wantReason: "KeyVaultNotAccessible", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) - mockBuilder := azureclient.NewMockFirstPartyApplicationClientBuilder(ctrl) - mockVaultsClient := azureclient.NewMockKeyVaultVaultsClient(ctrl) + mockBuilder := azureclient.NewMockServiceManagedIdentityClientBuilder(ctrl) + mockKeysClient := azureclient.NewMockKeyVaultKeysClient(ctrl) if tt.setupMock != nil { - tt.setupMock(mockBuilder, mockVaultsClient) + tt.setupMock(mockBuilder, mockKeysClient) } validation := NewAzureClusterKeyVaultAccessibilityValidation(mockBuilder) - err := validation.Validate(ctx, subscription, tt.cluster) + result := validation.Validate(ctx, subscription, tt.cluster) + + require.NoError(t, result.Validate(), "result should be well-formed") + assert.Equal(t, tt.wantOutcome, result.Outcome.Type) - if tt.wantErr == "" { - require.NoError(t, err) - } else { - require.Error(t, err) - assert.ErrorContains(t, err, tt.wantErr) + if tt.wantReason != "" { + assert.Equal(t, tt.wantReason, result.Reason()) } }) } diff --git a/test-integration/go.mod b/test-integration/go.mod index 9b832fd45a4..0610e862853 100644 --- a/test-integration/go.mod +++ b/test-integration/go.mod @@ -33,11 +33,11 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect diff --git a/test-integration/go.sum b/test-integration/go.sum index 9d76eb72746..29dcd6e0d83 100644 --- a/test-integration/go.sum +++ b/test-integration/go.sum @@ -25,8 +25,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFG github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 h1:1kpY4qe+BGAH2ykv4baVSqyx+AY5VjXeJ15SldlU6hs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1/go.mod h1:nT6cWpWdUt+g81yuKmjeYPUtI73Ak3yQIT4PVVsCEEQ= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 h1:akP6VpxJGgQRpDR1P462piz/8OhYLRCreDj48AyNabc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0/go.mod h1:8wzvopPfyZYPaQUoKW87Zfdul7jmJMDfp/k7YY3oJyA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= @@ -41,6 +39,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1/go.mod h1:8h8yhzh9o+0HeSIhUxYny+rEQajScrfIpNktvgYG3Q8= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= diff --git a/test/go.mod b/test/go.mod index adcbf65df60..dba64d8634a 100644 --- a/test/go.mod +++ b/test/go.mod @@ -31,7 +31,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 github.com/blang/semver/v4 v4.0.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dusted-go/logging v1.3.0 diff --git a/test/go.sum b/test/go.sum index 8121f402b80..32330002b15 100644 --- a/test/go.sum +++ b/test/go.sum @@ -152,8 +152,7 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0 h1:15MiSirdxWpIKFEHwfwfMWYieETCUnPTERPIlnytQeo= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0/go.mod h1:0AAf/fHAqhIsvD6XGKlCsDO8u+cCiYBBuro+lWS6os4= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 h1:E4MgwLBGeVB5f2MdcIVD3ELVAWpr+WD6MUe1i+tM/PA= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= diff --git a/tooling/secret-sync/go.mod b/tooling/secret-sync/go.mod index 25b626d341a..801abb97f77 100644 --- a/tooling/secret-sync/go.mod +++ b/tooling/secret-sync/go.mod @@ -6,7 +6,7 @@ require ( github.com/Azure/ARO-Tools/tools/secret-sync v0.0.0-20260807054659-4a463f5d83c7 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 github.com/dusted-go/logging v1.3.0 github.com/go-logr/logr v1.4.3 github.com/golang-jwt/jwt/v5 v5.3.1 diff --git a/tooling/secret-sync/go.sum b/tooling/secret-sync/go.sum index 3ec0175db49..dd9d87831ab 100644 --- a/tooling/secret-sync/go.sum +++ b/tooling/secret-sync/go.sum @@ -16,8 +16,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDH github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 h1:E4MgwLBGeVB5f2MdcIVD3ELVAWpr+WD6MUe1i+tM/PA= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= diff --git a/tooling/templatize/go.mod b/tooling/templatize/go.mod index 7c3cbd31483..04d04a2e69a 100644 --- a/tooling/templatize/go.mod +++ b/tooling/templatize/go.mod @@ -66,7 +66,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armlocks v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect diff --git a/tooling/templatize/go.sum b/tooling/templatize/go.sum index 88e9c17962b..33e751f6ac5 100644 --- a/tooling/templatize/go.sum +++ b/tooling/templatize/go.sum @@ -138,8 +138,7 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0 h1:15MiSirdxWpIKFEHwfwfMWYieETCUnPTERPIlnytQeo= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0/go.mod h1:0AAf/fHAqhIsvD6XGKlCsDO8u+cCiYBBuro+lWS6os4= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 h1:E4MgwLBGeVB5f2MdcIVD3ELVAWpr+WD6MUe1i+tM/PA= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= From e8f516c9d68de9af6db8616b38dd58451a7b3ced Mon Sep 17 00:00:00 2001 From: Yury Nikitenko Date: Thu, 13 Aug 2026 15:27:12 +0300 Subject: [PATCH 3/3] ARO-28729 keyvault validation changes review comments fixed --- backend/cmd/root.go | 14 ++- backend/pkg/app/backend.go | 3 +- backend/pkg/app/fpa_wiring.go | 7 ++ backend/pkg/azure/client/credential_helper.go | 58 +++++++++ .../mock_operator_identity_client_builder.go | 119 ++++++++++++++++++ .../azure/client/mock_smi_client_builder.go | 39 ------ .../operator_identity_client_builder.go | 79 ++++++++++++ .../pkg/azure/client/smi_client_builder.go | 52 +------- ...uster_keyvault_accessibility_validation.go | 104 ++++++++++----- ..._keyvault_accessibility_validation_test.go | 80 +++++++----- .../apitesting/coreapitesting/testhelpers.go | 17 ++- internal/ocm/convert_test.go | 20 ++- internal/validation/validate_cluster.go | 30 +++++ .../validate_cluster_comprehensive_test.go | 68 ++++++++-- test/go.sum | 1 + tooling/secret-sync/go.sum | 1 + tooling/templatize/go.sum | 1 + 17 files changed, 514 insertions(+), 179 deletions(-) create mode 100644 backend/pkg/azure/client/credential_helper.go create mode 100644 backend/pkg/azure/client/mock_operator_identity_client_builder.go create mode 100644 backend/pkg/azure/client/operator_identity_client_builder.go diff --git a/backend/cmd/root.go b/backend/cmd/root.go index dcc989507da..7ce7b5f16d4 100644 --- a/backend/cmd/root.go +++ b/backend/cmd/root.go @@ -415,6 +415,7 @@ func (f *BackendRootCmdFlags) ToBackendOptions(ctx context.Context, cmd *cobra.C } smiClientBuilder := app.NewServiceManagedIdentityClientBuilder(fpaMIDataplaneClientBuilder, azureConfig) + clusterOperatorIdentityClientBuilder := app.NewClusterOperatorIdentityClientBuilder(fpaMIDataplaneClientBuilder, azureConfig) azCoreClientOptions := *azureConfig.CloudEnvironment.AZCoreClientOptions() miDataplaneBasedIdentityAccessTokenRetrieverBuilder := azureclient.NewMIDataplaneBasedIdentityAccessTokenRetrieverBuilder( @@ -489,12 +490,13 @@ func (f *BackendRootCmdFlags) ToBackendOptions(ctx context.Context, cmd *cobra.C BackupConfig: backupConfig, FPAMIDataplaneClientBuilder: fpaMIDataplaneClientBuilder, MIDataplaneBasedIdentityAccessTokenRetrieverBuilder: miDataplaneBasedIdentityAccessTokenRetrieverBuilder, - SMIClientBuilder: smiClientBuilder, - CheckAccessV2ClientBuilder: checkAccessV2ClientBuilder, - ClusterScopedIdentitiesConfig: clusterScopedIdentitiesConfig, - CloudEnvironment: azureConfig.CloudEnvironment, - MetricsRegisterer: legacyregistry.Registerer(), - MetricsGatherer: legacyregistry.DefaultGatherer, + SMIClientBuilder: smiClientBuilder, + ClusterOperatorIdentityClientBuilder: clusterOperatorIdentityClientBuilder, + CheckAccessV2ClientBuilder: checkAccessV2ClientBuilder, + ClusterScopedIdentitiesConfig: clusterScopedIdentitiesConfig, + CloudEnvironment: azureConfig.CloudEnvironment, + MetricsRegisterer: legacyregistry.Registerer(), + MetricsGatherer: legacyregistry.DefaultGatherer, } return backendOptions, nil diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index 708a8252eb7..78286cb6cf7 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -119,6 +119,7 @@ type BackendOptions struct { MIDataplaneBasedIdentityAccessTokenRetrieverBuilder azureclient.MIDataplaneBasedIdentityAccessTokenRetrieverBuilder BackupConfig *clusterbackups.BackupConfig SMIClientBuilder azureclient.ServiceManagedIdentityClientBuilder + ClusterOperatorIdentityClientBuilder azureclient.ClusterOperatorIdentityClientBuilder CheckAccessV2ClientBuilder azureclient.CheckAccessV2ClientBuilder ClusterScopedIdentitiesConfig *internalazure.ClusterScopedIdentitiesConfig CloudEnvironment *azureconfig.AzureCloudEnvironment @@ -827,7 +828,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, backendInformers, ) azureClusterKeyVaultAccessibilityValidationController := clustervalidation.NewClusterValidationController( - validationutils.NewAzureClusterKeyVaultAccessibilityValidation(b.options.SMIClientBuilder), + validationutils.NewAzureClusterKeyVaultAccessibilityValidation(b.options.ClusterOperatorIdentityClientBuilder), b.options.ResourcesDBClient, serviceProviderClusterLister, backendInformers, diff --git a/backend/pkg/app/fpa_wiring.go b/backend/pkg/app/fpa_wiring.go index 1870db41293..870ec115562 100644 --- a/backend/pkg/app/fpa_wiring.go +++ b/backend/pkg/app/fpa_wiring.go @@ -72,3 +72,10 @@ func NewServiceManagedIdentityClientBuilder(fpaMIdataplaneClientBuilder azurecli azureConfig.CloudEnvironment.ARMClientOptions(), ) } + +func NewClusterOperatorIdentityClientBuilder(fpaMIdataplaneClientBuilder azureclient.FPAMIDataplaneClientBuilder, azureConfig *azureconfig.AzureConfig) azureclient.ClusterOperatorIdentityClientBuilder { + return azureclient.NewClusterOperatorIdentityClientBuilder( + fpaMIdataplaneClientBuilder, + azureConfig.CloudEnvironment.ARMClientOptions(), + ) +} diff --git a/backend/pkg/azure/client/credential_helper.go b/backend/pkg/azure/client/credential_helper.go new file mode 100644 index 00000000000..41d5893ec44 --- /dev/null +++ b/backend/pkg/azure/client/credential_helper.go @@ -0,0 +1,58 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/msi-dataplane/pkg/dataplane" + + "github.com/Azure/ARO-HCP/internal/utils" +) + +// credentialForIdentity obtains an azcore.TokenCredential for the given +// customer-granted managed identity, via the Managed Identities Data Plane +// Service, using the cluster's identity URL. It works for any identity the +// cluster has been granted (the SMI, or a cluster operator identity like +// "kms") -- the data plane request is keyed by resource ID alone. +func credentialForIdentity( + ctx context.Context, + fpaMIdataplaneClientBuilder FPAMIDataplaneClientBuilder, + azCoreClientOptions azcore.ClientOptions, + clusterIdentityURL string, + identityResourceID *azcorearm.ResourceID, +) (azcore.TokenCredential, error) { + miDataplaneClient, err := fpaMIdataplaneClientBuilder.ManagedIdentitiesDataplane(clusterIdentityURL) + if err != nil { + return nil, err + } + + dataplaneRequest := dataplane.UserAssignedIdentitiesRequest{ + IdentityIDs: []string{identityResourceID.String()}, + } + resp, err := miDataplaneClient.GetUserAssignedIdentitiesCredentials(ctx, dataplaneRequest) + if err != nil { + return nil, err + } + if len(resp.ExplicitIdentities) == 0 { + return nil, + utils.TrackError(fmt.Errorf("managed identities data plane returned no credentials for identity '%s'", identityResourceID.String())) + } + + return dataplane.GetCredential(azCoreClientOptions, resp.ExplicitIdentities[0]) +} diff --git a/backend/pkg/azure/client/mock_operator_identity_client_builder.go b/backend/pkg/azure/client/mock_operator_identity_client_builder.go new file mode 100644 index 00000000000..ea1b3d71591 --- /dev/null +++ b/backend/pkg/azure/client/mock_operator_identity_client_builder.go @@ -0,0 +1,119 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: operator_identity_client_builder.go +// +// Generated by this command: +// +// mockgen-v0.6.0 -typed -source=operator_identity_client_builder.go -destination=mock_operator_identity_client_builder.go -package client ClusterOperatorIdentityClientBuilder +// + +// Package client is a generated GoMock package. +package client + +import ( + context "context" + reflect "reflect" + + arm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + gomock "go.uber.org/mock/gomock" +) + +// MockClusterOperatorIdentityClientBuilder is a mock of ClusterOperatorIdentityClientBuilder interface. +type MockClusterOperatorIdentityClientBuilder struct { + ctrl *gomock.Controller + recorder *MockClusterOperatorIdentityClientBuilderMockRecorder + isgomock struct{} +} + +// MockClusterOperatorIdentityClientBuilderMockRecorder is the mock recorder for MockClusterOperatorIdentityClientBuilder. +type MockClusterOperatorIdentityClientBuilderMockRecorder struct { + mock *MockClusterOperatorIdentityClientBuilder +} + +// NewMockClusterOperatorIdentityClientBuilder creates a new mock instance. +func NewMockClusterOperatorIdentityClientBuilder(ctrl *gomock.Controller) *MockClusterOperatorIdentityClientBuilder { + mock := &MockClusterOperatorIdentityClientBuilder{ctrl: ctrl} + mock.recorder = &MockClusterOperatorIdentityClientBuilderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClusterOperatorIdentityClientBuilder) EXPECT() *MockClusterOperatorIdentityClientBuilderMockRecorder { + return m.recorder +} + +// BuilderType mocks base method. +func (m *MockClusterOperatorIdentityClientBuilder) BuilderType() ClusterOperatorIdentityClientBuilderType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BuilderType") + ret0, _ := ret[0].(ClusterOperatorIdentityClientBuilderType) + return ret0 +} + +// BuilderType indicates an expected call of BuilderType. +func (mr *MockClusterOperatorIdentityClientBuilderMockRecorder) BuilderType() *MockClusterOperatorIdentityClientBuilderBuilderTypeCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuilderType", reflect.TypeOf((*MockClusterOperatorIdentityClientBuilder)(nil).BuilderType)) + return &MockClusterOperatorIdentityClientBuilderBuilderTypeCall{Call: call} +} + +// MockClusterOperatorIdentityClientBuilderBuilderTypeCall wrap *gomock.Call +type MockClusterOperatorIdentityClientBuilderBuilderTypeCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockClusterOperatorIdentityClientBuilderBuilderTypeCall) Return(arg0 ClusterOperatorIdentityClientBuilderType) *MockClusterOperatorIdentityClientBuilderBuilderTypeCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockClusterOperatorIdentityClientBuilderBuilderTypeCall) Do(f func() ClusterOperatorIdentityClientBuilderType) *MockClusterOperatorIdentityClientBuilderBuilderTypeCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockClusterOperatorIdentityClientBuilderBuilderTypeCall) DoAndReturn(f func() ClusterOperatorIdentityClientBuilderType) *MockClusterOperatorIdentityClientBuilderBuilderTypeCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// KeyVaultKeysClient mocks base method. +func (m *MockClusterOperatorIdentityClientBuilder) KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, operatorIdentityResourceID *arm.ResourceID, vaultName string) (KeyVaultKeysClient, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KeyVaultKeysClient", ctx, clusterIdentityURL, operatorIdentityResourceID, vaultName) + ret0, _ := ret[0].(KeyVaultKeysClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// KeyVaultKeysClient indicates an expected call of KeyVaultKeysClient. +func (mr *MockClusterOperatorIdentityClientBuilderMockRecorder) KeyVaultKeysClient(ctx, clusterIdentityURL, operatorIdentityResourceID, vaultName any) *MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyVaultKeysClient", reflect.TypeOf((*MockClusterOperatorIdentityClientBuilder)(nil).KeyVaultKeysClient), ctx, clusterIdentityURL, operatorIdentityResourceID, vaultName) + return &MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall{Call: call} +} + +// MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall wrap *gomock.Call +type MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall) Return(arg0 KeyVaultKeysClient, arg1 error) *MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall) Do(f func(context.Context, string, *arm.ResourceID, string) (KeyVaultKeysClient, error)) *MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall) DoAndReturn(f func(context.Context, string, *arm.ResourceID, string) (KeyVaultKeysClient, error)) *MockClusterOperatorIdentityClientBuilderKeyVaultKeysClientCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/backend/pkg/azure/client/mock_smi_client_builder.go b/backend/pkg/azure/client/mock_smi_client_builder.go index fdf1d1bb0b5..dd41fed1a29 100644 --- a/backend/pkg/azure/client/mock_smi_client_builder.go +++ b/backend/pkg/azure/client/mock_smi_client_builder.go @@ -79,45 +79,6 @@ func (c *MockServiceManagedIdentityClientBuilderBuilderTypeCall) DoAndReturn(f f return c } -// KeyVaultKeysClient mocks base method. -func (m *MockServiceManagedIdentityClientBuilder) KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, identityResourceID *arm.ResourceID, vaultName string) (KeyVaultKeysClient, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "KeyVaultKeysClient", ctx, clusterIdentityURL, identityResourceID, vaultName) - ret0, _ := ret[0].(KeyVaultKeysClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// KeyVaultKeysClient indicates an expected call of KeyVaultKeysClient. -func (mr *MockServiceManagedIdentityClientBuilderMockRecorder) KeyVaultKeysClient(ctx, clusterIdentityURL, identityResourceID, vaultName any) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyVaultKeysClient", reflect.TypeOf((*MockServiceManagedIdentityClientBuilder)(nil).KeyVaultKeysClient), ctx, clusterIdentityURL, identityResourceID, vaultName) - return &MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall{Call: call} -} - -// MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall wrap *gomock.Call -type MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall) Return(arg0 KeyVaultKeysClient, arg1 error) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall) Do(f func(context.Context, string, *arm.ResourceID, string) (KeyVaultKeysClient, error)) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall) DoAndReturn(f func(context.Context, string, *arm.ResourceID, string) (KeyVaultKeysClient, error)) *MockServiceManagedIdentityClientBuilderKeyVaultKeysClientCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // SubnetsClient mocks base method. func (m *MockServiceManagedIdentityClientBuilder) SubnetsClient(ctx context.Context, clusterIdentityURL string, smiResourceID *arm.ResourceID, subscriptionID string) (SubnetsClient, error) { m.ctrl.T.Helper() diff --git a/backend/pkg/azure/client/operator_identity_client_builder.go b/backend/pkg/azure/client/operator_identity_client_builder.go new file mode 100644 index 00000000000..2e54fbbd049 --- /dev/null +++ b/backend/pkg/azure/client/operator_identity_client_builder.go @@ -0,0 +1,79 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +//go:generate $MOCKGEN -typed -source=operator_identity_client_builder.go -destination=mock_operator_identity_client_builder.go -package client ClusterOperatorIdentityClientBuilder + +import ( + "context" + "fmt" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" +) + +// ClusterOperatorIdentityClientBuilderType is a type that represents the type of the +// ClusterOperatorIdentityClientBuilder interface. It is used to ensure that +// that interface is incompatible with other client builder interfaces that +// might have the same set of methods. +type ClusterOperatorIdentityClientBuilderType string + +const ( + // ClusterOperatorIdentityClientBuilderTypeValue is the value of the + // ClusterOperatorIdentityClientBuilderType type that represents the + // cluster operator identity client builder. + ClusterOperatorIdentityClientBuilderTypeValue ClusterOperatorIdentityClientBuilderType = "ClusterOperatorIdentity" +) + +// ClusterOperatorIdentityClientBuilder offers the ability to create Azure clients +// authenticating as a cluster's operator identity (e.g. "kms", "cloud-controller", +// etc.). These are customer-granted managed identities that operators use to +// access customer resources. +type ClusterOperatorIdentityClientBuilder interface { + BuilderType() ClusterOperatorIdentityClientBuilderType + // KeyVaultKeysClient returns a new Key Vault Keys data plane client, + // authenticated as the given cluster operator identity. This is the + // credential path that has RBAC access to a customer-owned Key Vault; + // the FPA and SMI do not have such access. + KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, operatorIdentityResourceID *azcorearm.ResourceID, vaultName string) (KeyVaultKeysClient, error) +} + +type clusterOperatorIdentityClientBuilder struct { + fpaMIdataplaneClientBuilder FPAMIDataplaneClientBuilder + azCoreARMClientOptions *azcorearm.ClientOptions +} + +var _ ClusterOperatorIdentityClientBuilder = (*clusterOperatorIdentityClientBuilder)(nil) + +func (b *clusterOperatorIdentityClientBuilder) BuilderType() ClusterOperatorIdentityClientBuilderType { + return ClusterOperatorIdentityClientBuilderTypeValue +} + +func (b *clusterOperatorIdentityClientBuilder) KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, operatorIdentityResourceID *azcorearm.ResourceID, vaultName string) (KeyVaultKeysClient, error) { + creds, err := credentialForIdentity(ctx, b.fpaMIdataplaneClientBuilder, b.azCoreARMClientOptions.ClientOptions, clusterIdentityURL, operatorIdentityResourceID) + if err != nil { + return nil, err + } + + vaultURL := fmt.Sprintf("https://%s.vault.azure.net/", vaultName) + return azkeys.NewClient(vaultURL, creds, nil) +} + +func NewClusterOperatorIdentityClientBuilder(fpaMIdataplaneClientBuilder FPAMIDataplaneClientBuilder, options *azcorearm.ClientOptions) ClusterOperatorIdentityClientBuilder { + return &clusterOperatorIdentityClientBuilder{ + fpaMIdataplaneClientBuilder: fpaMIdataplaneClientBuilder, + azCoreARMClientOptions: options, + } +} diff --git a/backend/pkg/azure/client/smi_client_builder.go b/backend/pkg/azure/client/smi_client_builder.go index cbc9ef00ad7..5d42eb7ffda 100644 --- a/backend/pkg/azure/client/smi_client_builder.go +++ b/backend/pkg/azure/client/smi_client_builder.go @@ -18,16 +18,10 @@ package client import ( "context" - "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" - "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" - "github.com/Azure/msi-dataplane/pkg/dataplane" - - "github.com/Azure/ARO-HCP/internal/utils" ) // ServiceManagedIdentityClientBuilderType is a type that represents the type of the @@ -51,12 +45,6 @@ type ServiceManagedIdentityClientBuilder interface { UserAssignedIdentitiesClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (UserAssignedIdentitiesClient, error) // SubnetsClient returns a new Subnet client. SubnetsClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (SubnetsClient, error) - // KeyVaultKeysClient returns a new Key Vault Keys data plane client, - // authenticated as the given customer-granted managed identity (e.g. a - // cluster operator identity such as "kms") rather than as the SMI. This - // is the only credential path that has RBAC access to a customer-owned - // Key Vault; the FPA and SMI do not. - KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, identityResourceID *azcorearm.ResourceID, vaultName string) (KeyVaultKeysClient, error) } type serviceManagedIdentityClientBuilder struct { @@ -71,7 +59,7 @@ func (b *serviceManagedIdentityClientBuilder) BuilderType() ServiceManagedIdenti } func (b *serviceManagedIdentityClientBuilder) UserAssignedIdentitiesClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (UserAssignedIdentitiesClient, error) { - creds, err := b.credentialForIdentity(ctx, clusterIdentityURL, smiResourceID) + creds, err := credentialForIdentity(ctx, b.fpaMIdataplaneClientBuilder, b.azCoreARMClientOptions.ClientOptions, clusterIdentityURL, smiResourceID) if err != nil { return nil, err } @@ -80,49 +68,13 @@ func (b *serviceManagedIdentityClientBuilder) UserAssignedIdentitiesClient(ctx c } func (b *serviceManagedIdentityClientBuilder) SubnetsClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (SubnetsClient, error) { - creds, err := b.credentialForIdentity(ctx, clusterIdentityURL, smiResourceID) + creds, err := credentialForIdentity(ctx, b.fpaMIdataplaneClientBuilder, b.azCoreARMClientOptions.ClientOptions, clusterIdentityURL, smiResourceID) if err != nil { return nil, err } return armnetwork.NewSubnetsClient(subscriptionID, creds, b.azCoreARMClientOptions) } -func (b *serviceManagedIdentityClientBuilder) KeyVaultKeysClient(ctx context.Context, clusterIdentityURL string, identityResourceID *azcorearm.ResourceID, vaultName string) (KeyVaultKeysClient, error) { - creds, err := b.credentialForIdentity(ctx, clusterIdentityURL, identityResourceID) - if err != nil { - return nil, err - } - - vaultURL := fmt.Sprintf("https://%s.vault.azure.net/", vaultName) - return azkeys.NewClient(vaultURL, creds, nil) -} - -// credentialForIdentity obtains an azcore.TokenCredential for the given -// customer-granted managed identity, via the Managed Identities Data Plane -// Service, using the cluster's identity URL. It works for any identity the -// cluster has been granted (the SMI, or a cluster operator identity like -// "kms") -- the data plane request is keyed by resource ID alone. -func (b *serviceManagedIdentityClientBuilder) credentialForIdentity(ctx context.Context, clusterIdentityURL string, identityResourceID *azcorearm.ResourceID) (azcore.TokenCredential, error) { - miDataplaneClient, err := b.fpaMIdataplaneClientBuilder.ManagedIdentitiesDataplane(clusterIdentityURL) - if err != nil { - return nil, err - } - - dataplaneRequest := dataplane.UserAssignedIdentitiesRequest{ - IdentityIDs: []string{identityResourceID.String()}, - } - resp, err := miDataplaneClient.GetUserAssignedIdentitiesCredentials(ctx, dataplaneRequest) - if err != nil { - return nil, err - } - if len(resp.ExplicitIdentities) == 0 { - return nil, - utils.TrackError(fmt.Errorf("managed identities data plane returned no credentials for identity '%s'", identityResourceID.String())) - } - - return dataplane.GetCredential(b.azCoreARMClientOptions.ClientOptions, resp.ExplicitIdentities[0]) -} - func NewServiceManagedIdentityClientBuilder(fpaMIdataplaneClientBuilder FPAMIDataplaneClientBuilder, options *azcorearm.ClientOptions) ServiceManagedIdentityClientBuilder { return &serviceManagedIdentityClientBuilder{ fpaMIdataplaneClientBuilder: fpaMIdataplaneClientBuilder, diff --git a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go index 0bfdbfb87a1..a805bfc7eb5 100644 --- a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go +++ b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation.go @@ -16,7 +16,11 @@ package validationutils import ( "context" + "errors" "fmt" + "net/http" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" "github.com/Azure/ARO-HCP/internal/api/coreapi" @@ -30,14 +34,14 @@ import ( // plane path, that etcd itself uses to perform the real encrypt/decrypt // operations. type AzureClusterKeyVaultAccessibilityValidation struct { - smiClientBuilder azureclient.ServiceManagedIdentityClientBuilder + operatorIdentityClientBuilder azureclient.ClusterOperatorIdentityClientBuilder } func NewAzureClusterKeyVaultAccessibilityValidation( - smiClientBuilder azureclient.ServiceManagedIdentityClientBuilder, + operatorIdentityClientBuilder azureclient.ClusterOperatorIdentityClientBuilder, ) *AzureClusterKeyVaultAccessibilityValidation { return &AzureClusterKeyVaultAccessibilityValidation{ - smiClientBuilder: smiClientBuilder, + operatorIdentityClientBuilder: operatorIdentityClientBuilder, } } @@ -47,47 +51,38 @@ func (a *AzureClusterKeyVaultAccessibilityValidation) Name() string { func (a *AzureClusterKeyVaultAccessibilityValidation) Validate(ctx context.Context, _ *coreapi.Subscription, cluster *coreapi.HCPOpenShiftCluster) ValidationResult { if cluster.CustomerProperties.Etcd.DataEncryption.KeyManagementMode != metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { - return PassedValidation(coreapi.ControllerConditionReasonAsExpected, "As expected", "Cluster does not use customer-managed KMS encryption.") + return SkippedValidation("NotApplicable", "Cluster does not use customer-managed KMS encryption.", "Cluster does not use customer-managed KMS encryption.") } - // API validation (DiscriminatedUnion) guarantees CustomerManaged != nil - // when KeyManagementMode == CustomerManaged. Guard against data corruption. + // Frontend API validation (DiscriminatedUnion) guarantees CustomerManaged != nil + // when KeyManagementMode == CustomerManaged, and Kms != nil when + // EncryptionType == KMS. This validation runs on already-validated stored + // cluster data, so we trust those invariants (consistent with the other + // backend cluster validations). cm := cluster.CustomerProperties.Etcd.DataEncryption.CustomerManaged - if cm == nil { - return UnknownValidation( - "InternalError", - "Unable to verify Key Vault accessibility.", - "customer-managed key management mode is set but customer-managed encryption profile is nil", - ControllerReportingPolicyTypeError, - ) - } - if cm.EncryptionType != metadataapi.CustomerManagedEncryptionTypeKMS { - return PassedValidation(coreapi.ControllerConditionReasonAsExpected, "As expected", "Cluster does not use KMS encryption.") + return SkippedValidation("NotApplicable", "Cluster does not use KMS encryption.", "Cluster does not use KMS encryption.") } - // API validation (DiscriminatedUnion) guarantees Kms != nil - // when EncryptionType == KMS. Guard against data corruption. - if cm.Kms == nil { + kmsProfile := cm.Kms + + // The frontend RP guarantees (via validateKmsIdentityRequirement) that the + // "kms" operator identity is configured whenever customer-managed KMS + // encryption is selected. If it is missing here it's an internal + // inconsistency rather than a user error, so report Unknown, not Failed. + kmsIdentityResourceID := cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators[string(internalazure.ClusterOperatorIdentifierKMS)] + if kmsIdentityResourceID == nil { return UnknownValidation( "InternalError", "Unable to verify Key Vault accessibility.", - "KMS encryption type is set but KMS encryption profile is nil", + fmt.Sprintf("cluster has no %q operator identity configured, but the RP should guarantee it for customer-managed KMS encryption", internalazure.ClusterOperatorIdentifierKMS), ControllerReportingPolicyTypeError, ) } - kmsProfile := cm.Kms - - kmsIdentityResourceID := cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators[string(internalazure.ClusterOperatorIdentifierKMS)] - if kmsIdentityResourceID == nil { - internalAndUserMsg := fmt.Sprintf("cluster has no %q operator identity configured", internalazure.ClusterOperatorIdentifierKMS) - return FailedValidation("KmsIdentityNotConfigured", internalAndUserMsg, internalAndUserMsg) - } - clusterIdentityURL := cluster.ServiceProviderProperties.ManagedIdentitiesDataPlaneIdentityURL - keysClient, err := a.smiClientBuilder.KeyVaultKeysClient(ctx, clusterIdentityURL, kmsIdentityResourceID, kmsProfile.ActiveKey.VaultName) + keysClient, err := a.operatorIdentityClientBuilder.KeyVaultKeysClient(ctx, clusterIdentityURL, kmsIdentityResourceID, kmsProfile.ActiveKey.VaultName) if err != nil { return UnknownValidation( "InternalError", @@ -98,11 +93,52 @@ func (a *AzureClusterKeyVaultAccessibilityValidation) Validate(ctx context.Conte } if _, err := keysClient.GetKey(ctx, kmsProfile.ActiveKey.Name, kmsProfile.ActiveKey.Version, nil); err != nil { - internalAndUserMsg := fmt.Sprintf( - "key vault %q is not accessible to the cluster's KMS operator identity: %s", - kmsProfile.ActiveKey.VaultName, err) - return FailedValidation("KeyVaultNotAccessible", internalAndUserMsg, internalAndUserMsg) + return a.classifyGetKeyError(err, kmsProfile.ActiveKey.VaultName, kmsProfile.ActiveKey.Name) + } + + return PassedValidation(coreapi.ControllerConditionReasonAsExpected, "As expected", "KMS key is accessible to the cluster's KMS operator identity.") +} + +// classifyGetKeyError examines the error from GetKey and returns an appropriate +// ValidationResult based on the error type: +// - 401/403: Permission denied - the KMS identity lacks access to the Key Vault +// - 404: Key not found - the specified key does not exist in the vault +// - Other errors: Unknown - could be transient (network, service outage) or permanent +func (a *AzureClusterKeyVaultAccessibilityValidation) classifyGetKeyError(err error, vaultName, keyName string) ValidationResult { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + switch respErr.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + userMsg := fmt.Sprintf( + "The KMS operator identity does not have access to Key Vault %q. "+ + "Ensure the identity has the 'Key Vault Crypto User' role on the vault.", + vaultName) + internalMsg := fmt.Sprintf( + "KMS identity lacks permission to access key vault %q: %s", + vaultName, err) + return FailedValidation("KeyVaultAccessDenied", userMsg, internalMsg) + + case http.StatusNotFound: + userMsg := fmt.Sprintf( + "KMS key %q was not found in Key Vault %q. "+ + "Ensure the key exists and the name is correct.", + keyName, vaultName) + internalMsg := fmt.Sprintf( + "KMS key %q not found in vault %q: %s", + keyName, vaultName, err) + return FailedValidation("KmsKeyNotFound", userMsg, internalMsg) + } } - return PassedValidation(coreapi.ControllerConditionReasonAsExpected, "As expected", "Key Vault is accessible to the cluster's KMS operator identity.") + // For all other errors (network issues, 5xx, timeouts, etc.), we cannot + // determine if this is a permanent or transient issue. + internalMsg := fmt.Sprintf( + "failed to verify KMS key accessibility in vault %q: %s", + vaultName, err) + return UnknownValidation( + "KeyVaultAccessibilityUnknown", + "Unable to verify Key Vault accessibility. This may be a transient issue.", + internalMsg, + ControllerReportingPolicyTypeLogOnly, + ) } diff --git a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go index 4d1c44c1fba..222418a7c05 100644 --- a/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go +++ b/backend/pkg/utils/validationutils/azure_cluster_keyvault_accessibility_validation_test.go @@ -17,12 +17,14 @@ package validationutils import ( "context" "errors" + "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" @@ -99,24 +101,15 @@ func TestAzureClusterKeyVaultAccessibilityValidation_Validate(t *testing.T) { tests := []struct { name string cluster *coreapi.HCPOpenShiftCluster - setupMock func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) + setupMock func(builder *azureclient.MockClusterOperatorIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) wantOutcome OutcomeType wantReason string }{ { name: "skips validation when not using customer-managed encryption", cluster: newTestCluster(t, "", false), - wantOutcome: OutcomeTypePassed, - }, - { - name: "returns unknown when customer-managed mode is set but profile is nil", - cluster: func() *coreapi.HCPOpenShiftCluster { - c := newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true) - c.CustomerProperties.Etcd.DataEncryption.CustomerManaged = nil - return c - }(), - wantOutcome: OutcomeTypeUnknown, - wantReason: "InternalError", + wantOutcome: OutcomeTypeSkipped, + wantReason: "NotApplicable", }, { name: "skips validation for non-KMS encryption types", @@ -126,58 +119,79 @@ func TestAzureClusterKeyVaultAccessibilityValidation_Validate(t *testing.T) { c.CustomerProperties.Etcd.DataEncryption.CustomerManaged.Kms = nil return c }(), - wantOutcome: OutcomeTypePassed, - }, - { - name: "returns unknown when KMS encryption type is set but Kms profile is nil", - cluster: func() *coreapi.HCPOpenShiftCluster { - c := newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true) - c.CustomerProperties.Etcd.DataEncryption.CustomerManaged.Kms = nil - return c - }(), - wantOutcome: OutcomeTypeUnknown, - wantReason: "InternalError", + wantOutcome: OutcomeTypeSkipped, + wantReason: "NotApplicable", }, { name: "succeeds when the key vault is accessible to the KMS operator identity", cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), - setupMock: func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + setupMock: func(builder *azureclient.MockClusterOperatorIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(keysClient, nil) keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, nil) }, wantOutcome: OutcomeTypePassed, }, { - name: "fails when the cluster has no KMS operator identity configured", + name: "reports unknown when the cluster has no KMS operator identity configured (RP should prevent this)", cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, false), - wantOutcome: OutcomeTypeFailed, - wantReason: "KmsIdentityNotConfigured", + wantOutcome: OutcomeTypeUnknown, + wantReason: "InternalError", }, { name: "reports unknown when the client builder returns an error", cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), - setupMock: func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + setupMock: func(builder *azureclient.MockClusterOperatorIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(nil, errors.New("credential error")) }, wantOutcome: OutcomeTypeUnknown, wantReason: "InternalError", }, { - name: "fails when the key vault is not accessible to the KMS operator identity", + name: "fails with access denied when GetKey returns 403 Forbidden", + cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), + setupMock: func(builder *azureclient.MockClusterOperatorIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(keysClient, nil) + keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, &azcore.ResponseError{StatusCode: http.StatusForbidden}) + }, + wantOutcome: OutcomeTypeFailed, + wantReason: "KeyVaultAccessDenied", + }, + { + name: "fails with access denied when GetKey returns 401 Unauthorized", + cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), + setupMock: func(builder *azureclient.MockClusterOperatorIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(keysClient, nil) + keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, &azcore.ResponseError{StatusCode: http.StatusUnauthorized}) + }, + wantOutcome: OutcomeTypeFailed, + wantReason: "KeyVaultAccessDenied", + }, + { + name: "fails with key not found when GetKey returns 404 Not Found", cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), - setupMock: func(builder *azureclient.MockServiceManagedIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + setupMock: func(builder *azureclient.MockClusterOperatorIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(keysClient, nil) - keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, errors.New("access denied")) + keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, &azcore.ResponseError{StatusCode: http.StatusNotFound}) }, wantOutcome: OutcomeTypeFailed, - wantReason: "KeyVaultNotAccessible", + wantReason: "KmsKeyNotFound", + }, + { + name: "returns unknown for other errors (network, 5xx, etc.)", + cluster: newTestCluster(t, metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged, true), + setupMock: func(builder *azureclient.MockClusterOperatorIdentityClientBuilder, keysClient *azureclient.MockKeyVaultKeysClient) { + builder.EXPECT().KeyVaultKeysClient(gomock.Any(), testClusterIdentityURL, gomock.Any(), testKeyVaultName).Return(keysClient, nil) + keysClient.EXPECT().GetKey(gomock.Any(), testKeyName, testKeyVersion, nil).Return(azkeys.GetKeyResponse{}, errors.New("connection timeout")) + }, + wantOutcome: OutcomeTypeUnknown, + wantReason: "KeyVaultAccessibilityUnknown", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) - mockBuilder := azureclient.NewMockServiceManagedIdentityClientBuilder(ctrl) + mockBuilder := azureclient.NewMockClusterOperatorIdentityClientBuilder(ctrl) mockKeysClient := azureclient.NewMockKeyVaultKeysClient(ctrl) if tt.setupMock != nil { diff --git a/internal/apitesting/coreapitesting/testhelpers.go b/internal/apitesting/coreapitesting/testhelpers.go index bee8bec611d..e965a1c809d 100644 --- a/internal/apitesting/coreapitesting/testhelpers.go +++ b/internal/apitesting/coreapitesting/testhelpers.go @@ -30,6 +30,7 @@ import ( "github.com/Azure/ARO-HCP/internal/api/coreapi" "github.com/Azure/ARO-HCP/internal/api/metadataapi" + internalazure "github.com/Azure/ARO-HCP/internal/azure" "github.com/Azure/ARO-HCP/internal/azureapi/v20240610preview/generated" ) @@ -42,6 +43,7 @@ const ( TestSubscriptionID = "11111111-1111-1111-1111-111111111111" TestAltSubscriptionID = "22222222-2222-2222-2222-222222222222" TestResourceGroupName = "testResourceGroup" + TestIdentityResourceGroupName = "testIdentityResourceGroup" TestClusterName = "testCluster" TestNodePoolName = "testNodePool" TestExternalAuthName = "testExtAuth" @@ -61,6 +63,7 @@ const ( var ( TestSubscriptionResourceID = path.Join("/subscriptions", TestSubscriptionID) TestResourceGroupResourceID = path.Join(TestSubscriptionResourceID, "resourceGroups", TestResourceGroupName) + TestIdentityResourceGroupResourceID = path.Join(TestSubscriptionResourceID, "resourceGroups", TestIdentityResourceGroupName) TestClusterResourceID = path.Join(TestResourceGroupResourceID, "providers", coreapi.ProviderNamespace, coreapi.ClusterResourceTypeName, TestClusterName) TestNodePoolResourceID = path.Join(TestClusterResourceID, coreapi.NodePoolResourceTypeName, TestNodePoolName) TestExternalAuthResourceID = path.Join(TestClusterResourceID, coreapi.ExternalAuthResourceTypeName, TestExternalAuthName) @@ -77,7 +80,7 @@ func NewTestLogger() *slog.Logger { } func NewTestUserAssignedIdentity(name string) *azcorearm.ResourceID { - return metadataapi.Must(azcorearm.ParseResourceID(path.Join(TestResourceGroupResourceID, "providers", "Microsoft.ManagedIdentity", "userAssignedIdentities", name))) + return metadataapi.Must(azcorearm.ParseResourceID(path.Join(TestIdentityResourceGroupResourceID, "providers", "Microsoft.ManagedIdentity", "userAssignedIdentities", name))) } func MinimumValidClusterTestCase() *coreapi.HCPOpenShiftCluster { @@ -114,6 +117,18 @@ func MinimumValidClusterTestCase() *coreapi.HCPOpenShiftCluster { }, }, } + // Add KMS operator identity (required when using KMS encryption) + kmsIdentity := NewTestUserAssignedIdentity("KmsOperatorIdentity") + resource.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators = map[string]*azcorearm.ResourceID{ + string(internalazure.ClusterOperatorIdentifierKMS): kmsIdentity, + } + // Add the identity to the resource's UserAssignedIdentities (required for validation) + resource.Identity = &coreapi.ManagedServiceIdentity{ + Type: coreapi.ManagedServiceIdentityTypeUserAssigned, + UserAssignedIdentities: map[string]*coreapi.UserAssignedIdentity{ + kmsIdentity.String(): {}, + }, + } // Add required systemData fields createdAt := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) resource.SystemData = &coreapi.SystemData{ diff --git a/internal/ocm/convert_test.go b/internal/ocm/convert_test.go index 96005ce9b42..daf5e2ef307 100644 --- a/internal/ocm/convert_test.go +++ b/internal/ocm/convert_test.go @@ -260,7 +260,10 @@ func ocmClusterDefaults(azureLocation string) *arohcpv1alpha1.ClusterBuilder { OutboundType(csOutboundType)). OperatorsAuthentication(arohcpv1alpha1.NewAzureOperatorsAuthentication(). ManagedIdentities(arohcpv1alpha1.NewAzureOperatorsAuthenticationManagedIdentities(). - ControlPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder)). + ControlPlaneOperatorsManagedIdentities(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder{ + "kms": arohcpv1alpha1.NewAzureControlPlaneManagedIdentity(). + ResourceID(coreapitesting.NewTestUserAssignedIdentity("KmsOperatorIdentity").String()), + }). DataPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureDataPlaneManagedIdentityBuilder)). ManagedIdentitiesDataPlaneIdentityUrl(coreapitesting.TestManagedIdentitiesDataPlaneIdentityURL))). ResourceGroupName(strings.ToLower(coreapitesting.TestResourceGroupName)). @@ -729,7 +732,10 @@ func getBaseCSClusterBuilder(updating bool) *arohcpv1alpha1.ClusterBuilder { OutboundType(csOutboundType)). OperatorsAuthentication(arohcpv1alpha1.NewAzureOperatorsAuthentication(). ManagedIdentities(arohcpv1alpha1.NewAzureOperatorsAuthenticationManagedIdentities(). - ControlPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder)). + ControlPlaneOperatorsManagedIdentities(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder{ + "kms": arohcpv1alpha1.NewAzureControlPlaneManagedIdentity(). + ResourceID(coreapitesting.NewTestUserAssignedIdentity("KmsOperatorIdentity").String()), + }). DataPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureDataPlaneManagedIdentityBuilder)). ManagedIdentitiesDataPlaneIdentityUrl(coreapitesting.TestManagedIdentitiesDataPlaneIdentityURL))). ResourceGroupName(strings.ToLower(coreapitesting.TestResourceGroupName)). @@ -1173,7 +1179,10 @@ func TestBuildCSCluster(t *testing.T) { OutboundType(csOutboundType)). OperatorsAuthentication(arohcpv1alpha1.NewAzureOperatorsAuthentication(). ManagedIdentities(arohcpv1alpha1.NewAzureOperatorsAuthenticationManagedIdentities(). - ControlPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder)). + ControlPlaneOperatorsManagedIdentities(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder{ + "kms": arohcpv1alpha1.NewAzureControlPlaneManagedIdentity(). + ResourceID(coreapitesting.NewTestUserAssignedIdentity("KmsOperatorIdentity").String()), + }). DataPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureDataPlaneManagedIdentityBuilder)). ManagedIdentitiesDataPlaneIdentityUrl(coreapitesting.TestManagedIdentitiesDataPlaneIdentityURL))). ResourceGroupName(strings.ToLower(coreapitesting.TestResourceGroupName)). @@ -1243,7 +1252,10 @@ func TestBuildCSCluster(t *testing.T) { OutboundType(csOutboundType)). OperatorsAuthentication(arohcpv1alpha1.NewAzureOperatorsAuthentication(). ManagedIdentities(arohcpv1alpha1.NewAzureOperatorsAuthenticationManagedIdentities(). - ControlPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder)). + ControlPlaneOperatorsManagedIdentities(map[string]*arohcpv1alpha1.AzureControlPlaneManagedIdentityBuilder{ + "kms": arohcpv1alpha1.NewAzureControlPlaneManagedIdentity(). + ResourceID(coreapitesting.NewTestUserAssignedIdentity("KmsOperatorIdentity").String()), + }). DataPlaneOperatorsManagedIdentities(make(map[string]*arohcpv1alpha1.AzureDataPlaneManagedIdentityBuilder)). ManagedIdentitiesDataPlaneIdentityUrl(coreapitesting.TestManagedIdentitiesDataPlaneIdentityURL))). ResourceGroupName(strings.ToLower(coreapitesting.TestResourceGroupName)). diff --git a/internal/validation/validate_cluster.go b/internal/validation/validate_cluster.go index ddb1348ad09..306c45fa103 100644 --- a/internal/validation/validate_cluster.go +++ b/internal/validation/validate_cluster.go @@ -33,6 +33,7 @@ import ( "github.com/Azure/ARO-HCP/internal/api/coreapi" "github.com/Azure/ARO-HCP/internal/api/metadataapi" + internalazure "github.com/Azure/ARO-HCP/internal/azure" ) const ( @@ -91,6 +92,9 @@ func ValidateCluster(ctx context.Context, op operation.Operation, newCluster, ol // there are pieces of clusterProperties that are dependent upon values in .identity errs = append(errs, validateOperatorAuthenticationAgainstIdentities(ctx, op, newCluster, oldCluster)...) + // KMS encryption requires the KMS operator identity to be configured + errs = append(errs, validateKmsIdentityRequirement(ctx, op, newCluster)...) + RewriteValidationFieldPaths(errs, validationPathMapper) return errs @@ -166,6 +170,32 @@ func validateOperatorAuthenticationAgainstIdentities(ctx context.Context, op ope return errs } +// validateKmsIdentityRequirement validates that when customer-managed KMS encryption +// is configured, the KMS operator identity must be present in ControlPlaneOperators. +func validateKmsIdentityRequirement(_ context.Context, _ operation.Operation, newCluster *coreapi.HCPOpenShiftCluster) field.ErrorList { + // Check if customer-managed KMS encryption is configured + if newCluster.CustomerProperties.Etcd.DataEncryption.KeyManagementMode != metadataapi.EtcdDataEncryptionKeyManagementModeTypeCustomerManaged { + return nil + } + + cm := newCluster.CustomerProperties.Etcd.DataEncryption.CustomerManaged + if cm == nil || cm.EncryptionType != metadataapi.CustomerManagedEncryptionTypeKMS { + return nil + } + + // KMS encryption is configured - verify the KMS operator identity is present + kmsOperatorName := string(internalazure.ClusterOperatorIdentifierKMS) + kmsIdentity := newCluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators[kmsOperatorName] + if kmsIdentity == nil { + fldPath := field.NewPath("customerProperties", "platform", "operatorsAuthentication", "userAssignedIdentities", "controlPlaneOperators").Key(kmsOperatorName) + return field.ErrorList{ + field.Required(fldPath, "KMS operator identity is required when customer-managed KMS encryption is configured"), + } + } + + return nil +} + func validateResourceIDsAgainstClusterID(ctx context.Context, op operation.Operation, newCluster, _ *coreapi.HCPOpenShiftCluster) field.ErrorList { if newCluster.ID == nil { return nil diff --git a/internal/validation/validate_cluster_comprehensive_test.go b/internal/validation/validate_cluster_comprehensive_test.go index 88ec7efcfe9..4d3c60c3b50 100644 --- a/internal/validation/validate_cluster_comprehensive_test.go +++ b/internal/validation/validate_cluster_comprehensive_test.go @@ -501,6 +501,21 @@ func TestValidateClusterCreate(t *testing.T) { return c }(), }, + { + name: "missing kms operator identity when kms encryption configured - create", + cluster: func() *coreapi.HCPOpenShiftCluster { + c := createValidCluster() + // Remove the kms identity from ControlPlaneOperators + delete(c.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators, "kms") + // Also remove it from Identity.UserAssignedIdentities to avoid "identity assigned but not used" error + kmsIdentityID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity" + delete(c.Identity.UserAssignedIdentities, kmsIdentityID) + return c + }(), + expectErrors: []utils.ExpectedError{ + {Message: "Required value", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, + }, + }, { name: "invalid cluster image registry state - create", cluster: func() *coreapi.HCPOpenShiftCluster { @@ -525,7 +540,9 @@ func TestValidateClusterCreate(t *testing.T) { {Message: "Required value", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators"}, {Message: "must be in the same Azure subscription", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[]"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[]"}, - {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities"}, + {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity]"}, + {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -541,7 +558,9 @@ func TestValidateClusterCreate(t *testing.T) { {Message: "resource ID must reference an instance of type", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, {Message: "must be in the same Azure subscription", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, - {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities"}, + {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity]"}, + {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -559,6 +578,7 @@ func TestValidateClusterCreate(t *testing.T) { {Message: "Required value", FieldPath: "identity.type"}, {Message: "Unsupported value", FieldPath: "identity.state"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities"}, }, }, @@ -577,6 +597,7 @@ func TestValidateClusterCreate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "Unsupported value", FieldPath: "identity.state"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities"}, }, }, @@ -596,6 +617,7 @@ func TestValidateClusterCreate(t *testing.T) { {Message: "resource ID must reference an instance of type", FieldPath: "identity.userAssignedIdentities"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -632,6 +654,7 @@ func TestValidateClusterCreate(t *testing.T) { }(), expectErrors: []utils.ExpectedError{ {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/unused-identity]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -650,6 +673,7 @@ func TestValidateClusterCreate(t *testing.T) { }(), expectErrors: []utils.ExpectedError{ {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -674,6 +698,7 @@ func TestValidateClusterCreate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "must be unique within the cluster", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators"}, {Message: "identity is used multiple times", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/shared-identity]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -713,6 +738,7 @@ func TestValidateClusterCreate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "must be unique within the cluster", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.serviceManagedIdentity"}, {Message: "identity is used multiple times", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/shared-identity]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -736,6 +762,7 @@ func TestValidateClusterCreate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "cannot use identity assigned to this resource by .identities.userAssignedIdentities", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.dataPlaneOperators[dataplane-operator]"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dataplane-identity]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -743,13 +770,17 @@ func TestValidateClusterCreate(t *testing.T) { cluster: func() *coreapi.HCPOpenShiftCluster { c := createValidCluster() identityID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-identity" + kmsIdentityID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity" c.Identity = &coreapi.ManagedServiceIdentity{ Type: coreapi.ManagedServiceIdentityTypeUserAssigned, UserAssignedIdentities: map[string]*coreapi.UserAssignedIdentity{ - identityID: {}, + identityID: {}, + kmsIdentityID: {}, }, } - c.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators = map[string]*azcorearm.ResourceID{} + c.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators = map[string]*azcorearm.ResourceID{ + "kms": metadataapi.Must(azcorearm.ParseResourceID(kmsIdentityID)), + } c.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ServiceManagedIdentity = metadataapi.Must(azcorearm.ParseResourceID(identityID)) return c }(), @@ -761,15 +792,18 @@ func TestValidateClusterCreate(t *testing.T) { c := createValidCluster() lowerCaseID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourcegroups/some-resource-group/providers/microsoft.managedidentity/userassignedidentities/test-identity" upperCaseID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity" + kmsIdentityID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity" c.Identity = &coreapi.ManagedServiceIdentity{ Type: coreapi.ManagedServiceIdentityTypeUserAssigned, UserAssignedIdentities: map[string]*coreapi.UserAssignedIdentity{ - lowerCaseID: {}, + lowerCaseID: {}, + kmsIdentityID: {}, }, } // Reference with different casing should work c.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators = map[string]*azcorearm.ResourceID{ "test-operator": metadataapi.Must(azcorearm.ParseResourceID(upperCaseID)), + "kms": metadataapi.Must(azcorearm.ParseResourceID(kmsIdentityID)), } return c }(), @@ -790,6 +824,7 @@ func TestValidateClusterCreate(t *testing.T) { {Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.vnetIntegrationSubnetId"}, {Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.managedResourceGroup"}, {Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -895,7 +930,9 @@ func TestValidateClusterCreate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "must be in the same Azure subscription", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, - {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities"}, + {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity]"}, + {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -1477,6 +1514,7 @@ func TestValidateClusterUpdate(t *testing.T) { {Message: "field is immutable", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators"}, {Message: "must be in the same Azure subscription", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, {Message: "must be in the same Azure subscription", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator-2]"}, + {Message: "KMS operator identity is required when customer-managed KMS encryption is configured", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, }, }, { @@ -2122,6 +2160,7 @@ func TestValidateClusterUpdate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "field is immutable", FieldPath: "identity.principalId"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity]"}, }, }, @@ -2152,6 +2191,7 @@ func TestValidateClusterUpdate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "field is immutable", FieldPath: "identity.tenantId"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity]"}, }, }, @@ -2184,6 +2224,7 @@ func TestValidateClusterUpdate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "field is immutable", FieldPath: "identity.userAssignedIdentities[/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity].clientId"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity]"}, }, }, @@ -2216,6 +2257,7 @@ func TestValidateClusterUpdate(t *testing.T) { expectErrors: []utils.ExpectedError{ {Message: "field is immutable", FieldPath: "identity.userAssignedIdentities[/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity].principalId"}, {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"}, + {Message: "identity is not assigned to this resource", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[kms]"}, {Message: "identity is assigned to this resource but not used", FieldPath: "identity.userAssignedIdentities[/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity]"}, }, }, @@ -2575,17 +2617,21 @@ func createValidCluster() *coreapi.HCPOpenShiftCluster { // Set ClusterUID for valid testing (generated by admission on CREATE; immutable thereafter) cluster.ServiceProviderProperties.ClusterUID = "00000000-0000-0000-0000-000000000000" - // Set up user assigned identities for valid testing with matching subscription and location - identityID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity" + // Set up user assigned identities for valid testing with matching subscription and location. + // Include kms identity since the default cluster uses CustomerManaged KMS encryption. + testOperatorIdentityID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity" + kmsIdentityID := "/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/some-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms-identity" cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators = map[string]*azcorearm.ResourceID{ - "test-operator": metadataapi.Must(azcorearm.ParseResourceID(identityID)), + "test-operator": metadataapi.Must(azcorearm.ParseResourceID(testOperatorIdentityID)), + "kms": metadataapi.Must(azcorearm.ParseResourceID(kmsIdentityID)), } - // Add the identity to the cluster's identity section so it's properly assigned + // Add the identities to the cluster's identity section so they're properly assigned cluster.Identity = &coreapi.ManagedServiceIdentity{ Type: coreapi.ManagedServiceIdentityTypeUserAssigned, UserAssignedIdentities: map[string]*coreapi.UserAssignedIdentity{ - identityID: {}, + testOperatorIdentityID: {}, + kmsIdentityID: {}, }, } diff --git a/test/go.sum b/test/go.sum index 32330002b15..39b284d8831 100644 --- a/test/go.sum +++ b/test/go.sum @@ -153,6 +153,7 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0 h1:15MiSirdxWpIKFEHwfwfMWYieETCUnPTERPIlnytQeo= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0/go.mod h1:0AAf/fHAqhIsvD6XGKlCsDO8u+cCiYBBuro+lWS6os4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= diff --git a/tooling/secret-sync/go.sum b/tooling/secret-sync/go.sum index dd9d87831ab..c175fb63203 100644 --- a/tooling/secret-sync/go.sum +++ b/tooling/secret-sync/go.sum @@ -17,6 +17,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlM github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= diff --git a/tooling/templatize/go.sum b/tooling/templatize/go.sum index 33e751f6ac5..2530ee72344 100644 --- a/tooling/templatize/go.sum +++ b/tooling/templatize/go.sum @@ -139,6 +139,7 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0 h1:15MiSirdxWpIKFEHwfwfMWYieETCUnPTERPIlnytQeo= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0/go.mod h1:0AAf/fHAqhIsvD6XGKlCsDO8u+cCiYBBuro+lWS6os4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4=