OCPBUGS-23466: Let router use svc ips

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -164,7 +164,6 @@ type HostedControlPlaneReconciler struct {
 	ec2Client                     ec2iface.EC2API
 	awsSession                    *session.Session
 	reconcileInfrastructureStatus func(ctx context.Context, hcp *hyperv1.HostedControlPlane) (InfrastructureStatus, error)
-	NameServerIP                  string
 }
 
 func (r *HostedControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager, createOrUpdate upsert.CreateOrUpdateFN) error {
@@ -3422,10 +3421,26 @@ func (r *HostedControlPlaneReconciler) reconcileRouter(ctx context.Context, hcp
 	}
 
 	// reconcile the router's configuration
+	svcsNameToIP := make(map[string]string)
 	if util.IsPrivateHCP(hcp) || exposeKASThroughRouter {
+		for _, route := range routeList.Items {
+			svc := &corev1.Service{
+				TypeMeta: metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      route.Spec.To.Name,
+					Namespace: hcp.Namespace,
+				},
+			}
+			if err := r.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil {
+				return err
+			}
+
+			svcsNameToIP[route.Spec.To.Name] = svc.Spec.ClusterIP
+		}
+
 		routerConfig := manifests.RouterConfigurationConfigMap(hcp.Namespace)
 		if _, err := createOrUpdate(ctx, r.Client, routerConfig, func() error {
-			return ingress.ReconcileRouterConfiguration(config.OwnerRefFrom(hcp), routerConfig, routeList, r.NameServerIP)
+			return ingress.ReconcileRouterConfiguration(config.OwnerRefFrom(hcp), routerConfig, routeList, svcsNameToIP)
 		}); err != nil {
 			return fmt.Errorf("failed to reconcile router configuration: %w", err)
 		}

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go

@@ -1055,7 +1055,7 @@ func TestReconcileRouter(t *testing.T) {
 	ingress.ReconcileRouterConfiguration(config.OwnerRefFrom(&hyperv1.HostedControlPlane{ObjectMeta: metav1.ObjectMeta{
 		Name:      "hcp",
 		Namespace: namespace,
-	}}), routerCfg, &routev1.RouteList{}, "172.30.0.10")
+	}}), routerCfg, &routev1.RouteList{}, map[string]string{})
 
 	testCases := []struct {
 		name                         string
@@ -1276,9 +1276,8 @@ func TestReconcileRouter(t *testing.T) {
 			c := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(append(tc.existingObjects, hcp)...).Build()
 
 			r := HostedControlPlaneReconciler{
-				Client:       c,
-				Log:          ctrl.LoggerFrom(ctx),
-				NameServerIP: "172.30.0.10",
+				Client: c,
+				Log:    ctrl.LoggerFrom(ctx),
 			}
 
 			releaseInfo := &releaseinfo.ReleaseImage{ImageStream: &imagev1.ImageStream{}}

control-plane-operator/controllers/hostedcontrolplane/ingress/router.go

@@ -72,24 +72,20 @@ func (r byRouteName) Len() int           { return len(r) }
 func (r byRouteName) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
 func (r byRouteName) Less(i, j int) bool { return r[i].Name < r[j].Name }
 
-func generateRouterConfig(namespace string, routeList *routev1.RouteList, nameServerIP string) (string, error) {
+func generateRouterConfig(routeList *routev1.RouteList, svcsNameToIP map[string]string) (string, error) {
 	type backendDesc struct {
-		Name               string
-		HostName           string
-		DestinationService string
-		DestinationPort    int32
+		Name                 string
+		HostName             string
+		DestinationServiceIP string
+		DestinationPort      int32
 	}
 	type templateParams struct {
-		HasKubeAPI   bool
-		Namespace    string
-		KASSVCPort   int32
-		Backends     []backendDesc
-		NameServerIP string
-	}
-	p := templateParams{
-		Namespace:    namespace,
-		NameServerIP: nameServerIP,
+		HasKubeAPI              bool
+		KASSVCPort              int32
+		KASDestinationServiceIP string
+		Backends                []backendDesc
 	}
+	p := templateParams{}
 	sort.Sort(byRouteName(routeList.Items))
 	for _, route := range routeList.Items {
 		if _, hasHCPLabel := route.Labels[util.HCPRouteLabel]; !hasHCPLabel {
@@ -102,21 +98,22 @@ func generateRouterConfig(namespace string, routeList *routev1.RouteList, nameSe
 			manifests.KubeAPIServerExternalPublicRoute("").Name,
 			manifests.KubeAPIServerExternalPrivateRoute("").Name:
 			p.HasKubeAPI = true
+			p.KASDestinationServiceIP = svcsNameToIP["kube-apiserver"]
 			continue
 		case ignitionserver.Route("").Name:
-			p.Backends = append(p.Backends, backendDesc{Name: "ignition", HostName: route.Spec.Host, DestinationService: route.Spec.To.Name, DestinationPort: 443})
+			p.Backends = append(p.Backends, backendDesc{Name: "ignition", HostName: route.Spec.Host, DestinationServiceIP: svcsNameToIP[route.Spec.To.Name], DestinationPort: 443})
 		case manifests.KonnectivityServerRoute("").Name:
-			p.Backends = append(p.Backends, backendDesc{Name: "konnectivity", HostName: route.Spec.Host, DestinationService: route.Spec.To.Name, DestinationPort: 8091})
+			p.Backends = append(p.Backends, backendDesc{Name: "konnectivity", HostName: route.Spec.Host, DestinationServiceIP: svcsNameToIP[route.Spec.To.Name], DestinationPort: 8091})
 		case manifests.OauthServerExternalPrivateRoute("").Name:
-			p.Backends = append(p.Backends, backendDesc{Name: "oauth_private", HostName: route.Spec.Host, DestinationService: route.Spec.To.Name, DestinationPort: 6443})
+			p.Backends = append(p.Backends, backendDesc{Name: "oauth_private", HostName: route.Spec.Host, DestinationServiceIP: svcsNameToIP[route.Spec.To.Name], DestinationPort: 6443})
 		case manifests.OauthServerExternalPublicRoute("").Name:
-			p.Backends = append(p.Backends, backendDesc{Name: "oauth", HostName: route.Spec.Host, DestinationService: route.Spec.To.Name, DestinationPort: 6443})
+			p.Backends = append(p.Backends, backendDesc{Name: "oauth", HostName: route.Spec.Host, DestinationServiceIP: svcsNameToIP[route.Spec.To.Name], DestinationPort: 6443})
 		case manifests.OauthServerInternalRoute("").Name:
-			p.Backends = append(p.Backends, backendDesc{Name: "oauth_internal", HostName: route.Spec.Host, DestinationService: route.Spec.To.Name, DestinationPort: 6443})
+			p.Backends = append(p.Backends, backendDesc{Name: "oauth_internal", HostName: route.Spec.Host, DestinationServiceIP: svcsNameToIP[route.Spec.To.Name], DestinationPort: 6443})
 		case manifests.OVNKubeSBDBRoute("").Name:
-			p.Backends = append(p.Backends, backendDesc{Name: "ovnkube_sbdb", HostName: route.Spec.Host, DestinationService: route.Spec.To.Name, DestinationPort: route.Spec.Port.TargetPort.IntVal})
+			p.Backends = append(p.Backends, backendDesc{Name: "ovnkube_sbdb", HostName: route.Spec.Host, DestinationServiceIP: svcsNameToIP[route.Spec.To.Name], DestinationPort: route.Spec.Port.TargetPort.IntVal})
 		case manifests.MetricsForwarderRoute("").Name:
-			p.Backends = append(p.Backends, backendDesc{Name: "metrics_forwarder", HostName: route.Spec.Host, DestinationService: route.Spec.To.Name, DestinationPort: route.Spec.Port.TargetPort.IntVal})
+			p.Backends = append(p.Backends, backendDesc{Name: "metrics_forwarder", HostName: route.Spec.Host, DestinationServiceIP: svcsNameToIP[route.Spec.To.Name], DestinationPort: route.Spec.Port.TargetPort.IntVal})
 		}
 	}
 	if p.HasKubeAPI {
@@ -129,13 +126,13 @@ func generateRouterConfig(namespace string, routeList *routev1.RouteList, nameSe
 	return out.String(), nil
 }
 
-func ReconcileRouterConfiguration(ownerRef config.OwnerRef, cm *corev1.ConfigMap, routeList *routev1.RouteList, nameServerIP string) error {
+func ReconcileRouterConfiguration(ownerRef config.OwnerRef, cm *corev1.ConfigMap, routeList *routev1.RouteList, svcsNameToIP map[string]string) error {
 	ownerRef.ApplyTo(cm)
 
 	if cm.Data == nil {
 		cm.Data = map[string]string{}
 	}
-	routerConfig, err := generateRouterConfig(cm.Namespace, routeList, nameServerIP)
+	routerConfig, err := generateRouterConfig(routeList, svcsNameToIP)
 	if err != nil {
 		return err
 	}

control-plane-operator/controllers/hostedcontrolplane/ingress/router_config.template

@@ -27,21 +27,16 @@ frontend main
   {{- end }}
   {{- if .HasKubeAPI }}
   default_backend kube_api{{- end }}
-  
+
 listen health_check_http_url
   bind :::9444 v4v6
   mode http
   monitor-uri /haproxy_ready
-
-resolvers coredns
-  nameserver ns1 {{ .NameServerIP }}:53
-
-{{ $ns := .Namespace }}
-{{- range .Backends }}
+{{range .Backends }}
 backend {{ .Name }}
-  server {{ .Name }} {{ .DestinationService }}.{{ $ns }}.svc.cluster.local:{{.DestinationPort}} check resolvers coredns init-addr last,libc,none
+  server {{ .Name }} {{ .DestinationServiceIP }}:{{.DestinationPort}}
 {{ end -}}
 {{- if .HasKubeAPI }}
 backend kube_api
-  server kube_api kube-apiserver.{{ $ns }}.svc.cluster.local:{{ .KASSVCPort }} check resolvers coredns init-addr last,libc,none
+  server kube_api {{ .KASDestinationServiceIP }}:{{ .KASSVCPort }}
 {{ end -}}

control-plane-operator/controllers/hostedcontrolplane/ingress/router_test.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"fmt"
 	"testing"
 
 	routev1 "github.com/openshift/api/route/v1"
@@ -61,7 +62,14 @@ func TestGenerateRouterConfig(t *testing.T) {
 		Items: []routev1.Route{*ignition, *konnectivity, *oauthInternal, *oauthExternalPrivate, *oauthExternalPublic, *ovnKube, *metricsForwarder, *kasPublic, *kasPrivate},
 	}
 
-	cfg, err := generateRouterConfig(testNS, routeList, "172.30.0.10")
+	svcsNameToIP := make(map[string]string)
+	i := 0
+	for _, r := range routeList.Items {
+		svcsNameToIP[r.Spec.To.Name] = fmt.Sprintf("0.0.0.%v", i)
+		i++
+	}
+
+	cfg, err := generateRouterConfig(routeList, svcsNameToIP)
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
 	}

control-plane-operator/controllers/hostedcontrolplane/ingress/testdata/zz_fixture_TestGenerateRouterConfig.yaml

@@ -34,36 +34,32 @@ frontend main
   use_backend oauth_private if is_oauth_private
   use_backend ovnkube_sbdb if is_ovnkube_sbdb
   default_backend kube_api
-  
+
 listen health_check_http_url
   bind :::9444 v4v6
   mode http
   monitor-uri /haproxy_ready
 
-resolvers coredns
-  nameserver ns1 172.30.0.10:53
-
-
 backend ignition
-  server ignition ignition-server-proxy.test-ns.svc.cluster.local:443 check resolvers coredns init-addr last,libc,none
+  server ignition 0.0.0.0:443
 
 backend konnectivity
-  server konnectivity konnectivity-server.test-ns.svc.cluster.local:8091 check resolvers coredns init-addr last,libc,none
+  server konnectivity 0.0.0.1:8091
 
 backend metrics_forwarder
-  server metrics_forwarder metrics-forwarder.test-ns.svc.cluster.local:4000 check resolvers coredns init-addr last,libc,none
+  server metrics_forwarder 0.0.0.6:4000
 
 backend oauth
-  server oauth openshift-oauth.test-ns.svc.cluster.local:6443 check resolvers coredns init-addr last,libc,none
+  server oauth 0.0.0.4:6443
 
 backend oauth_internal
-  server oauth_internal openshift-oauth.test-ns.svc.cluster.local:6443 check resolvers coredns init-addr last,libc,none
+  server oauth_internal 0.0.0.4:6443
 
 backend oauth_private
-  server oauth_private openshift-oauth.test-ns.svc.cluster.local:6443 check resolvers coredns init-addr last,libc,none
+  server oauth_private 0.0.0.4:6443
 
 backend ovnkube_sbdb
-  server ovnkube_sbdb ovnkube-master-external.test-ns.svc.cluster.local:3000 check resolvers coredns init-addr last,libc,none
+  server ovnkube_sbdb 0.0.0.5:3000
 
 backend kube_api
-  server kube_api kube-apiserver.test-ns.svc.cluster.local:6443 check resolvers coredns init-addr last,libc,none
+  server kube_api 0.0.0.8:6443

control-plane-operator/main.go

@@ -388,12 +388,6 @@ func NewStartCommand() *cobra.Command {
 		}
 		setupLog.Info("Using metrics set", "set", metricsSet.String())
 
-		nameServerIP, err := util.NameServerIP()
-		if err != nil {
-			setupLog.Error(err, "cannot determine nameserver IP")
-			os.Exit(1)
-		}
-
 		if err := (&hostedcontrolplane.HostedControlPlaneReconciler{
 			Client:                        mgr.GetClient(),
 			ManagementClusterCapabilities: mgmtClusterCaps,
@@ -402,7 +396,6 @@ func NewStartCommand() *cobra.Command {
 			OperateOnReleaseImage:         os.Getenv("OPERATE_ON_RELEASE_IMAGE"),
 			DefaultIngressDomain:          defaultIngressDomain,
 			MetricsSet:                    metricsSet,
-			NameServerIP:                  nameServerIP,
 		}).SetupWithManager(mgr, upsert.New(enableCIDebugOutput).CreateOrUpdate); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "hosted-control-plane")
 			os.Exit(1)