AGENT-1449: Add single-phase IRI registry credential rotation

pkg/apihelpers/apihelpers.go

@@ -31,6 +31,16 @@ var (
 					},
 				},
 			},
+			{
+				// The Distribution registry re-reads htpasswd on mtime change,
+				// so credential rotation does not require a service restart.
+				Path: "/etc/iri-registry/auth/htpasswd",
+				Actions: []opv1.NodeDisruptionPolicyStatusAction{
+					{
+						Type: opv1.NoneStatusAction,
+					},
+				},
+			},
 			{
 				Path: constants.GPGNoRebootPath,
 				Actions: []opv1.NodeDisruptionPolicyStatusAction{

pkg/controller/common/iri_secret_merger.go

@@ -42,9 +42,14 @@ func NewIRISecretMerger(
 	iriLister mcfglistersv1alpha1.InternalReleaseImageLister,
 	fgHandler FeatureGatesHandler,
 ) *IRISecretMerger {
+	// Evaluate the feature gate once at construction time. Using a live check
+	// inside the closure would race with the informer factory startup: the gate
+	// could be off when New() runs (nil listers) but on by the time Merge() is
+	// called, causing a nil-lister dereference.
+	iriEnabled := fgHandler.Enabled(features.FeatureGateNoRegistryClusterInstall)
 	return &IRISecretMerger{
 		resolve: func() (string, string, error) {
-			if !fgHandler.Enabled(features.FeatureGateNoRegistryClusterInstall) {
+			if !iriEnabled {
 				return "", "", errIRIDisabled
 			}
 			_, err := iriLister.Get(InternalReleaseImageInstanceName)

pkg/controller/internalreleaseimage/internalreleaseimage_bootstrap_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestRunInternalReleaseImageBootstrap(t *testing.T) {
-	configs, err := RunInternalReleaseImageBootstrap(&mcfgv1alpha1.InternalReleaseImage{}, iriCertSecret().obj, iriRegistryCredentialsSecret().obj, cconfig().withDNS("example.com").obj)
+	configs, err := RunInternalReleaseImageBootstrap(&mcfgv1alpha1.InternalReleaseImage{}, iriCertSecret().obj, iriAuthSecret().obj, cconfig().obj)
 	assert.NoError(t, err)
 	assert.Len(t, configs, 2)
 	verifyInternalReleaseMasterMachineConfig(t, configs[0])

pkg/controller/internalreleaseimage/internalreleaseimage_controller.go

@@ -55,6 +55,7 @@ var (
 // Controller defines the InternalReleaseImage controller.
 type Controller struct {
 	client        mcfgclientset.Interface
+	kubeClient    clientset.Interface
 	eventRecorder record.EventRecorder
 
 	syncHandler                 func(mcp string) error
@@ -94,6 +95,7 @@ func New(
 
 	ctrl := &Controller{
 		client:        mcfgClient,
+		kubeClient:    kubeClient,
 		eventRecorder: ctrlcommon.NamespacedEventRecorder(eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "machineconfigcontroller-internalreleaseimagecontroller"})),
 		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
 			workqueue.DefaultTypedControllerRateLimiter[string](),
@@ -281,8 +283,9 @@ func (ctrl *Controller) processMachineConfigEvent(obj interface{}, logMsg string
 
 func (ctrl *Controller) addSecret(obj interface{}, _ bool) {
 	secret := obj.(*corev1.Secret)
-	if secret.Name != ctrlcommon.InternalReleaseImageTLSSecretName &&
-		secret.Name != ctrlcommon.InternalReleaseImageAuthSecretName {
+	if secret.Namespace != ctrlcommon.MCONamespace ||
+		(secret.Name != ctrlcommon.InternalReleaseImageTLSSecretName &&
+			secret.Name != ctrlcommon.InternalReleaseImageAuthSecretName) {
 		return
 	}
 	klog.V(4).Infof("Secret %s added, re-queuing IRI sync", secret.Name)
@@ -292,8 +295,9 @@ func (ctrl *Controller) addSecret(obj interface{}, _ bool) {
 func (ctrl *Controller) updateSecret(_, cur interface{}) {
 	secret := cur.(*corev1.Secret)
 
-	if secret.Name != ctrlcommon.InternalReleaseImageTLSSecretName &&
-		secret.Name != ctrlcommon.InternalReleaseImageAuthSecretName {
+	if secret.Namespace != ctrlcommon.MCONamespace ||
+		(secret.Name != ctrlcommon.InternalReleaseImageTLSSecretName &&
+			secret.Name != ctrlcommon.InternalReleaseImageAuthSecretName) {
 		return
 	}
 
@@ -374,6 +378,14 @@ func (ctrl *Controller) syncInternalReleaseImage(key string) (syncErr error) {
 		return fmt.Errorf("could not get Secret %s: %w", ctrlcommon.InternalReleaseImageAuthSecretName, err)
 	}
 
+	// Ensure the htpasswd field is in sync with the password field. If the
+	// password was rotated, this generates a new bcrypt hash and updates the
+	// secret before re-rendering the MachineConfig.
+	iriRegistryCredentialsSecret, err = reconcileHtpasswd(ctrl.kubeClient, iriRegistryCredentialsSecret)
+	if err != nil {
+		return fmt.Errorf("failed to reconcile IRI registry htpasswd: %w", err)
+	}
+
 	for _, role := range SupportedRoles {
 		r := NewRendererByRole(role, iri, iriSecret, iriRegistryCredentialsSecret, cconfig)
 
@@ -531,7 +543,6 @@ func (ctrl *Controller) addFinalizerToInternalReleaseImage(iri *mcfgv1alpha1.Int
 	_, err := ctrl.client.MachineconfigurationV1alpha1().InternalReleaseImages().Update(context.TODO(), iri, metav1.UpdateOptions{})
 	return err
 }
-
 func (ctrl *Controller) cascadeDelete(iri *mcfgv1alpha1.InternalReleaseImage) error {
 	mcName := iri.GetFinalizers()[0]
 	err := ctrl.client.MachineconfigurationV1().MachineConfigs().Delete(context.TODO(), mcName, metav1.DeleteOptions{})

pkg/controller/internalreleaseimage/internalreleaseimage_controller_test.go

@@ -43,7 +43,7 @@ func TestInternalReleaseImageCreate(t *testing.T) {
 		},
 		{
 			name:           "add finalizer if not present",
-			initialObjects: objs(iri(), clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriRegistryCredentialsSecret(), pullSecret()),
+			initialObjects: objs(iri(), clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriAuthSecret(), pullSecret()),
 			verify: func(t *testing.T, actualIRI *mcfgv1alpha1.InternalReleaseImage, actualMasterMC *mcfgv1.MachineConfig, actualWorkerMC *mcfgv1.MachineConfig) {
 				assert.Len(t, actualIRI.Finalizers, 2)
 				assert.Contains(t, actualIRI.Finalizers, masterName())
@@ -54,7 +54,7 @@ func TestInternalReleaseImageCreate(t *testing.T) {
 			name: "update status if not set",
 			initialObjects: objs(
 				iri().finalizer(masterName(), workerName()),
-				clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriRegistryCredentialsSecret(), pullSecret()),
+				clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriAuthSecret(), pullSecret()),
 			verify: func(t *testing.T, actualIRI *mcfgv1alpha1.InternalReleaseImage, actualMasterMC *mcfgv1.MachineConfig, actualWorkerMC *mcfgv1.MachineConfig) {
 				assert.Len(t, actualIRI.Status.Releases, 1)
 				assert.Equal(t, actualIRI.Status.Releases[0].Name, "ocp-release-bundle-4.21.5-x86_64")
@@ -66,7 +66,7 @@ func TestInternalReleaseImageCreate(t *testing.T) {
 		},
 		{
 			name:           "generate iri machine-config if not present",
-			initialObjects: objs(iri(), clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriRegistryCredentialsSecret(), pullSecret()),
+			initialObjects: objs(iri(), clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriAuthSecret(), pullSecret()),
 			verify: func(t *testing.T, actualIRI *mcfgv1alpha1.InternalReleaseImage, actualMasterMC *mcfgv1.MachineConfig, actualWorkerMC *mcfgv1.MachineConfig) {
 				verifyInternalReleaseMasterMachineConfig(t, actualMasterMC)
 				verifyInternalReleaseWorkerMachineConfig(t, actualWorkerMC)
@@ -76,7 +76,7 @@ func TestInternalReleaseImageCreate(t *testing.T) {
 			name: "avoid machine-config drifting",
 			initialObjects: objs(
 				iri().finalizer(masterName(), workerName()),
-				clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriRegistryCredentialsSecret(), pullSecret(),
+				clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriAuthSecret(), pullSecret(),
 				machineconfigmaster().ignition("some garbage"),
 				machineconfigworker().ignition("other garbage")),
 			verify: func(t *testing.T, actualIRI *mcfgv1alpha1.InternalReleaseImage, actualMasterMC *mcfgv1.MachineConfig, actualWorkerMC *mcfgv1.MachineConfig) {
@@ -88,7 +88,7 @@ func TestInternalReleaseImageCreate(t *testing.T) {
 			name: "refresh machine-config on controllerConfig update",
 			initialObjects: objs(
 				iri().finalizer(masterName(), workerName()),
-				clusterVersion(), cconfig().dockerRegistryImage("a-new-docker-registry-image-pullspec").withDNS("example.com"), iriCertSecret(), iriRegistryCredentialsSecret(), pullSecret(),
+				clusterVersion(), cconfig().dockerRegistryImage("a-new-docker-registry-image-pullspec").withDNS("example.com"), iriCertSecret(), iriAuthSecret(), pullSecret(),
 				machineconfigmaster(), machineconfigworker()),
 			verify: func(t *testing.T, actualIRI *mcfgv1alpha1.InternalReleaseImage, actualMasterMC *mcfgv1.MachineConfig, actualWorkerMC *mcfgv1.MachineConfig) {
 				verifyInternalReleaseMasterMachineConfig(t, actualMasterMC)
@@ -125,7 +125,7 @@ func TestInternalReleaseImageCreate(t *testing.T) {
 			name: "status condition Degraded=False on successful sync",
 			initialObjects: objs(
 				iri().finalizer(masterName(), workerName()),
-				clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriRegistryCredentialsSecret(), pullSecret(),
+				clusterVersion(), cconfig().withDNS("example.com"), iriCertSecret(), iriAuthSecret(), pullSecret(),
 				machineconfigmaster(), machineconfigworker()),
 			verify: func(t *testing.T, actualIRI *mcfgv1alpha1.InternalReleaseImage, actualMasterMC *mcfgv1.MachineConfig, actualWorkerMC *mcfgv1.MachineConfig) {
 				assert.NotNil(t, actualIRI)
@@ -233,6 +233,91 @@ func TestInternalReleaseImageStatusOnError(t *testing.T) {
 	}
 }
 
+func TestReconcileHtpasswd(t *testing.T) {
+	cases := []struct {
+		name             string
+		password         string
+		existingHtpasswd string
+		expectUpdate     bool
+	}{
+		{
+			name:             "htpasswd already matches password, no update",
+			password:         "mypassword",
+			existingHtpasswd: mustGenerateHtpasswd(t, "mypassword"),
+			expectUpdate:     false,
+		},
+		{
+			name:             "htpasswd missing, generates new",
+			password:         "mypassword",
+			existingHtpasswd: "",
+			expectUpdate:     true,
+		},
+		{
+			name:             "password changed, regenerates htpasswd",
+			password:         "newpassword",
+			existingHtpasswd: mustGenerateHtpasswd(t, "oldpassword"),
+			expectUpdate:     true,
+		},
+	}
+
+	t.Run("empty password returns error", func(t *testing.T) {
+		secret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      ctrlcommon.InternalReleaseImageAuthSecretName,
+				Namespace: ctrlcommon.MCONamespace,
+			},
+			Data: map[string][]byte{
+				"password": []byte(""),
+			},
+		}
+		f := newFixture(t, []runtime.Object{secret})
+		_, err := reconcileHtpasswd(f.k8sClient, secret)
+		assert.Error(t, err)
+	})
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			secret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      ctrlcommon.InternalReleaseImageAuthSecretName,
+					Namespace: ctrlcommon.MCONamespace,
+				},
+				Data: map[string][]byte{
+					"password": []byte(tc.password),
+					"htpasswd": []byte(tc.existingHtpasswd),
+				},
+			}
+
+			f := newFixture(t, []runtime.Object{secret})
+			result, err := reconcileHtpasswd(f.k8sClient, secret)
+			assert.NoError(t, err)
+
+			if tc.expectUpdate {
+				// Verify the returned secret has a valid htpasswd
+				assert.True(t, HtpasswdMatchesPassword(string(result.Data["htpasswd"]), ctrlcommon.IRIRegistryUsername, tc.password),
+					"updated htpasswd should match the password")
+
+				// Verify the secret was updated in the API
+				updated, err := f.k8sClient.CoreV1().Secrets(ctrlcommon.MCONamespace).Get(
+					context.TODO(), ctrlcommon.InternalReleaseImageAuthSecretName, metav1.GetOptions{})
+				assert.NoError(t, err)
+				assert.True(t, HtpasswdMatchesPassword(string(updated.Data["htpasswd"]), ctrlcommon.IRIRegistryUsername, tc.password),
+					"secret in API should have updated htpasswd")
+			} else {
+				// Verify the htpasswd was not changed
+				assert.Equal(t, tc.existingHtpasswd, string(result.Data["htpasswd"]),
+					"htpasswd should not change when already matching")
+			}
+		})
+	}
+}
+
+func mustGenerateHtpasswd(t *testing.T, password string) string {
+	t.Helper()
+	entry, err := generateHtpasswdEntry(ctrlcommon.IRIRegistryUsername, password)
+	assert.NoError(t, err)
+	return entry
+}
 // The fixture used to setup and run the controller.
 type fixture struct {
 	t *testing.T

pkg/controller/internalreleaseimage/internalreleaseimage_helpers_test.go

@@ -35,7 +35,7 @@ func verifyInternalReleaseMasterMachineConfig(t *testing.T, mc *mcfgv1.MachineCo
 	verifyIgnitionFile(t, &ignCfg, "/etc/pki/ca-trust/source/anchors/iri-root-ca.crt", "iri-root-ca-data")
 	verifyIgnitionFile(t, &ignCfg, "/etc/iri-registry/certs/tls.key", "iri-tls-key")
 	verifyIgnitionFile(t, &ignCfg, "/etc/iri-registry/certs/tls.crt", "iri-tls-crt")
-	verifyIgnitionFile(t, &ignCfg, "/etc/iri-registry/auth/htpasswd", "openshift:$2y$05$testhash")
+	verifyIgnitionFileMatches(t, &ignCfg, "/etc/iri-registry/auth/htpasswd", ctrlcommon.IRIRegistryUsername, "testpassword")
 	verifyIgnitionFileContains(t, &ignCfg, "/usr/local/bin/load-registry-image.sh", "docker-registry-image-pullspec")
 	assert.Contains(t, *ignCfg.Systemd.Units[0].Contents, `REGISTRY_STORAGE_MAINTENANCE_READONLY={"enabled":true}`)
 	assert.NotContains(t, *ignCfg.Systemd.Units[0].Contents, "REGISTRY_STORAGE_MAINTENANCE_READONLY_ENABLED")
@@ -66,6 +66,15 @@ func verifyIgnitionFileContains(t *testing.T, ignCfg *ign3types.Config, path str
 	assert.Contains(t, string(data), expectedContent, path)
 }
 
+// verifyIgnitionFileMatches verifies that the ignition file at path contains a
+// valid htpasswd entry matching the given username and password.
+func verifyIgnitionFileMatches(t *testing.T, ignCfg *ign3types.Config, path, username, password string) {
+	t.Helper()
+	data, err := ctrlcommon.GetIgnitionFileDataByPath(ignCfg, path)
+	assert.NoError(t, err)
+	assert.True(t, HtpasswdMatchesPassword(string(data), username, password),
+		"htpasswd at %s should match %s:<password>", path, username)
+}
 
 // objs is an helper func to improve the test readability.
 func objs(builders ...objBuilder) func() []runtime.Object {
@@ -247,16 +256,23 @@ func pullSecret() *secretBuilder {
 	}
 }
 
-func iriRegistryCredentialsSecret() *secretBuilder {
+// iriAuthSecret returns an auth secret with a testpassword and pre-generated
+// bcrypt htpasswd, suitable for both bootstrap and controller tests.
+// The controller's reconcileHtpasswd will verify the htpasswd matches and leave it unchanged.
+func iriAuthSecret() *secretBuilder {
+	htpasswd, err := generateHtpasswdEntry(ctrlcommon.IRIRegistryUsername, "testpassword")
+	if err != nil {
+		panic(err)
+	}
 	return &secretBuilder{
 		obj: &corev1.Secret{
 			ObjectMeta: v1.ObjectMeta{
 				Namespace: ctrlcommon.MCONamespace,
 				Name:      ctrlcommon.InternalReleaseImageAuthSecretName,
 			},
 			Data: map[string][]byte{
-				"htpasswd": []byte("openshift:$2y$05$testhash"),
 				"password": []byte("testpassword"),
+				"htpasswd": []byte(htpasswd),
 			},
 		},
 	}

pkg/controller/internalreleaseimage/internalreleaseimage_registry_auth.go

@@ -0,0 +1,72 @@
+package internalreleaseimage
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
+	"golang.org/x/crypto/bcrypt"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+)
+
+// generateHtpasswdEntry generates an htpasswd-formatted line for the given username
+// and password using bcrypt hashing.
+func generateHtpasswdEntry(username, password string) (string, error) {
+	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	if err != nil {
+		return "", fmt.Errorf("failed to generate bcrypt hash: %w", err)
+	}
+	return fmt.Sprintf("%s:%s", username, string(hash)), nil
+}
+
+// HtpasswdMatchesPassword reports whether the given htpasswd line matches
+// the provided username and password.
+func HtpasswdMatchesPassword(htpasswd, username, password string) bool {
+	prefix := username + ":"
+	if !strings.HasPrefix(htpasswd, prefix) {
+		return false
+	}
+	hash := []byte(strings.TrimPrefix(htpasswd, prefix))
+	return bcrypt.CompareHashAndPassword(hash, []byte(password)) == nil
+}
+
+// reconcileHtpasswd ensures the htpasswd field in the IRI auth secret is in
+// sync with the password field. If the password has changed (or htpasswd is
+// missing), it generates a new bcrypt hash and updates the secret. This is the
+// trigger for single-phase credential rotation: the updated htpasswd causes the
+// MachineConfig to be re-rendered, which MCDs roll out to nodes. Brief registry
+// downtime during the rollout is accepted.
+func reconcileHtpasswd(kubeClient clientset.Interface, authSecret *corev1.Secret) (*corev1.Secret, error) {
+	password := string(authSecret.Data["password"])
+	if password == "" {
+		return nil, fmt.Errorf("IRI auth secret %s/%s missing or empty \"password\" field", authSecret.Namespace, authSecret.Name)
+	}
+	htpasswd := string(authSecret.Data["htpasswd"])
+
+	if HtpasswdMatchesPassword(htpasswd, ctrlcommon.IRIRegistryUsername, password) {
+		return authSecret, nil
+	}
+
+	klog.V(4).Infof("IRI auth secret htpasswd is out of sync with password, regenerating")
+
+	newHtpasswd, err := generateHtpasswdEntry(ctrlcommon.IRIRegistryUsername, password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate htpasswd: %w", err)
+	}
+
+	updated := authSecret.DeepCopy()
+	updated.Data["htpasswd"] = []byte(newHtpasswd)
+
+	result, err := kubeClient.CoreV1().Secrets(authSecret.Namespace).Update(
+		context.TODO(), updated, metav1.UpdateOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to update IRI auth secret: %w", err)
+	}
+
+	klog.Infof("Regenerated IRI auth secret htpasswd for credential rotation (secret %s/%s)", authSecret.Namespace, authSecret.Name)
+	return result, nil
+}