OCPBUGS-113993: [release-4.20] fix router component ordering to prevent missing HAProxy backends

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -239,7 +239,6 @@ func (r *HostedControlPlaneReconciler) registerComponents(hcp *hyperv1.HostedCon
 		kcmv2.NewComponent(),
 		schedulerv2.NewComponent(),
 		oapiv2.NewComponent(),
-		routerv2.NewComponent(),
 		oauthapiv2.NewComponent(),
 		autoscalerv2.NewComponent(),
 		cvov2.NewComponent(r.EnableCVOManagementClusterMetricsAccess),
@@ -266,6 +265,7 @@ func (r *HostedControlPlaneReconciler) registerComponents(hcp *hyperv1.HostedCon
 		konnectivityv2.NewComponent(),
 		ignitionserverv2.NewComponent(r.ReleaseProvider, r.DefaultIngressDomain),
 		ignitionproxyv2.NewComponent(r.DefaultIngressDomain),
+		routerv2.NewComponent(),
 	)
 	r.components = append(r.components,
 		olmv2.NewComponents(r.ManagementClusterCapabilities.Has(capabilities.CapabilityImageStream))...,

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go

@@ -22,6 +22,7 @@ import (
 	ignitionproxyv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/ignitionserver_proxy"
 	kasv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/kas"
 	oapiv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/oapi"
+	routerv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/router"
 	"github.com/openshift/hypershift/support/api"
 	"github.com/openshift/hypershift/support/azureutil"
 	"github.com/openshift/hypershift/support/capabilities"
@@ -2162,3 +2163,62 @@ func TestRemoveHCPIngressFromRoutes(t *testing.T) {
 		})
 	}
 }
+
+func TestRouterComponentComesAfterRouteCreatingComponents(t *testing.T) {
+	t.Parallel()
+
+	reconciler := &HostedControlPlaneReconciler{
+		ReleaseProvider:               &fakereleaseprovider.FakeReleaseProvider{},
+		ManagementClusterCapabilities: &fakecapabilities.FakeSupportAllCapabilities{},
+	}
+
+	hcp := &hyperv1.HostedControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-hcp",
+			Namespace: "test-ns",
+		},
+		Spec: hyperv1.HostedControlPlaneSpec{
+			Platform: hyperv1.PlatformSpec{
+				Type: hyperv1.AWSPlatform,
+			},
+			Etcd: hyperv1.EtcdSpec{
+				ManagementType: hyperv1.Managed,
+			},
+			Services: []hyperv1.ServicePublishingStrategyMapping{
+				{
+					Service: hyperv1.Ignition,
+					ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
+						Type: hyperv1.Route,
+					},
+				},
+			},
+		},
+	}
+
+	reconciler.registerComponents(hcp)
+
+	positions := make(map[string]int, len(reconciler.components))
+	for i, c := range reconciler.components {
+		positions[c.Name()] = i
+	}
+
+	routerPos, ok := positions[routerv2.ComponentName]
+	if !ok {
+		t.Fatal("router component not found in registered components")
+	}
+
+	routeCreatingComponents := []string{
+		ignitionserverv2.ComponentName,
+	}
+	for _, name := range routeCreatingComponents {
+		pos, ok := positions[name]
+		if !ok {
+			t.Fatalf("route-creating component %q not found in registered components", name)
+		}
+		if routerPos < pos {
+			t.Errorf("router component (position %d) must be registered after %s (position %d) "+
+				"so that the HAProxy config includes %s's route on the first reconcile pass",
+				routerPos, name, pos, name)
+		}
+	}
+}

control-plane-operator/controllers/hostedcontrolplane/testdata/router/AROSwift/zz_fixture_TestControlPlaneComponents_router_controlplanecomponent.yaml

@@ -9,14 +9,13 @@ spec: {}
 status:
   conditions:
   - lastTransitionTime: null
-    message: router Deployment Available condition not found
+    message: deployments.apps "router" not found
     reason: NotFound
     status: "False"
     type: Available
   - lastTransitionTime: null
-    message: 'Waiting for deployment router rollout to finish: 0 out of 3 new replicas
-      have been updated'
-    reason: WaitingForRolloutComplete
+    message: 'Waiting for Dependencies: ignition-server'
+    reason: WaitingForDependencies
     status: "False"
     type: RolloutComplete
   resources:

control-plane-operator/controllers/hostedcontrolplane/testdata/router/IBMCloud/zz_fixture_TestControlPlaneComponents_router_controlplanecomponent.yaml

@@ -9,14 +9,13 @@ spec: {}
 status:
   conditions:
   - lastTransitionTime: null
-    message: router Deployment Available condition not found
+    message: deployments.apps "router" not found
     reason: NotFound
     status: "False"
     type: Available
   - lastTransitionTime: null
-    message: 'Waiting for deployment router rollout to finish: 0 out of 3 new replicas
-      have been updated'
-    reason: WaitingForRolloutComplete
+    message: 'Waiting for Dependencies: ignition-server'
+    reason: WaitingForDependencies
     status: "False"
     type: RolloutComplete
   resources:

control-plane-operator/controllers/hostedcontrolplane/testdata/router/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_router_controlplanecomponent.yaml

@@ -9,14 +9,13 @@ spec: {}
 status:
   conditions:
   - lastTransitionTime: null
-    message: router Deployment Available condition not found
+    message: deployments.apps "router" not found
     reason: NotFound
     status: "False"
     type: Available
   - lastTransitionTime: null
-    message: 'Waiting for deployment router rollout to finish: 0 out of 3 new replicas
-      have been updated'
-    reason: WaitingForRolloutComplete
+    message: 'Waiting for Dependencies: ignition-server'
+    reason: WaitingForDependencies
     status: "False"
     type: RolloutComplete
   resources:

control-plane-operator/controllers/hostedcontrolplane/testdata/router/zz_fixture_TestControlPlaneComponents_router_controlplanecomponent.yaml

@@ -9,14 +9,13 @@ spec: {}
 status:
   conditions:
   - lastTransitionTime: null
-    message: router Deployment Available condition not found
+    message: deployments.apps "router" not found
     reason: NotFound
     status: "False"
     type: Available
   - lastTransitionTime: null
-    message: 'Waiting for deployment router rollout to finish: 0 out of 3 new replicas
-      have been updated'
-    reason: WaitingForRolloutComplete
+    message: 'Waiting for Dependencies: ignition-server'
+    reason: WaitingForDependencies
     status: "False"
     type: RolloutComplete
   resources:

control-plane-operator/controllers/hostedcontrolplane/v2/router/component.go

@@ -1,8 +1,19 @@
 package router
 
 import (
+	"fmt"
+	"strings"
+
+	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	ignitionserverv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/ignitionserver"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/router/util"
+	"github.com/openshift/hypershift/support/azureutil"
 	component "github.com/openshift/hypershift/support/controlplane-component"
+	supportutil "github.com/openshift/hypershift/support/util"
+
+	routev1 "github.com/openshift/api/route/v1"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -31,9 +42,7 @@ func (k *router) NeedsManagementKASAccess() bool {
 
 func NewComponent() component.ControlPlaneComponent {
 	return component.NewDeploymentComponent(ComponentName, &router{}).
-		WithPredicate(func(cpContext component.WorkloadContext) (bool, error) {
-			return util.UseHCPRouter(cpContext.HCP), nil
-		}).
+		WithPredicate(routerPredicate).
 		WithAdaptFunction(adaptDeployment).
 		WithManifestAdapter(
 			"config.yaml",
@@ -43,5 +52,72 @@ func NewComponent() component.ControlPlaneComponent {
 			"pdb.yaml",
 			component.AdaptPodDisruptionBudget(),
 		).
+		WithDependencies(ignitionserverv2.ComponentName).
 		Build()
 }
+
+func routerPredicate(cpContext component.WorkloadContext) (bool, error) {
+	if !util.UseHCPRouter(cpContext.HCP) {
+		return false, nil
+	}
+	if azureutil.IsAroHCP() {
+		if err := ensureHCPRouterRoutesExist(cpContext); err != nil {
+			return false, err
+		}
+	}
+	return true, nil
+}
+
+// TODO: introduce live reloading like in shared proxy so the router config
+// is updated when routes change after the initial reconcile.
+
+func ensureHCPRouterRoutesExist(cpContext component.WorkloadContext) error {
+	expected := aroExpectedHCPRouterRouteNames(cpContext.HCP)
+	if len(expected) == 0 {
+		return nil
+	}
+
+	routeList := &routev1.RouteList{}
+	if err := cpContext.Client.List(cpContext, routeList, client.InNamespace(cpContext.HCP.Namespace)); err != nil {
+		return fmt.Errorf("failed to list routes: %w", err)
+	}
+
+	routesByName := make(map[string]routev1.Route, len(routeList.Items))
+	for _, route := range routeList.Items {
+		routesByName[route.Name] = route
+	}
+
+	var missing []string
+	for _, name := range expected {
+		route, ok := routesByName[name]
+		if !ok || !hcpRouterRouteReady(&route) {
+			missing = append(missing, name)
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("waiting for HCP router routes: %s", strings.Join(missing, ", "))
+	}
+	return nil
+}
+
+var aroBaseHCPRouterRouteNames = []string{
+	"kube-apiserver-internal",
+	"konnectivity-server",
+	"ignition-server",
+}
+
+func aroExpectedHCPRouterRouteNames(hcp *hyperv1.HostedControlPlane) []string {
+	if !azureutil.IsAroHCP() {
+		return nil
+	}
+
+	names := append([]string(nil), aroBaseHCPRouterRouteNames...)
+	if supportutil.HCPOAuthEnabled(hcp) {
+		names = append(names, "oauth-internal")
+	}
+	return names
+}
+
+func hcpRouterRouteReady(route *routev1.Route) bool {
+	return route.Spec.Host != ""
+}