diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 8130f1f8ea1..1b1bf02e5b5 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -2707,7 +2707,13 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftControllerManager(ctx c func (r *HostedControlPlaneReconciler) reconcileOpenShiftRouteControllerManager(ctx context.Context, hcp *hyperv1.HostedControlPlane, observedConfig *globalconfig.ObservedConfig, releaseImageProvider *imageprovider.ReleaseImageProvider, createOrUpdate upsert.CreateOrUpdateFN) error { p := routecm.NewOpenShiftRouteControllerManagerParams(hcp, observedConfig, releaseImageProvider, r.SetDefaultSecurityContext) - config := manifests.OpenShiftControllerManagerConfig(hcp.Namespace) + config := manifests.OpenShiftRouteControllerManagerConfig(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, config, func() error { + return routecm.ReconcileOpenShiftRouteControllerManagerConfig(config, p.OwnerRef, p.MinTLSVersion(), p.CipherSuites(), p.Network) + }); err != nil { + return fmt.Errorf("failed to reconcile openshift route controller manager config: %w", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(config), config); err != nil { return fmt.Errorf("failed to get openshift controller manager config: %w", err) } diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/openshift_cm.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/openshift_cm.go index 4564b30e21a..a930d9a4f7c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/openshift_cm.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/openshift_cm.go @@ -16,6 +16,15 @@ func OpenShiftControllerManagerConfig(ns string) *corev1.ConfigMap { } } +func OpenShiftRouteControllerManagerConfig(ns string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "openshift-route-controller-manager-config", + Namespace: ns, + }, + } +} + func OpenShiftControllerManagerDeployment(ns string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/control-plane-operator/controllers/hostedcontrolplane/ocm/config.go b/control-plane-operator/controllers/hostedcontrolplane/ocm/config.go index 8635f307471..9d2711a2804 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/ocm/config.go +++ b/control-plane-operator/controllers/hostedcontrolplane/ocm/config.go @@ -23,6 +23,8 @@ const ( ) func ReconcileOpenShiftControllerManagerConfig(cm *corev1.ConfigMap, ownerRef config.OwnerRef, deployerImage, dockerBuilderImage, minTLSVersion string, cipherSuites []string, imageConfig *configv1.Image, buildConfig *configv1.Build, networkConfig *configv1.NetworkSpec) error { + ownerRef.ApplyTo(cm) + if cm.Data == nil { cm.Data = map[string]string{} } diff --git a/control-plane-operator/controllers/hostedcontrolplane/ocm/config_test.go b/control-plane-operator/controllers/hostedcontrolplane/ocm/config_test.go new file mode 100644 index 00000000000..d0de2742fa7 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/ocm/config_test.go @@ -0,0 +1,81 @@ +package ocm + +import ( + v1 "github.com/openshift/api/config/v1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + config2 "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/globalconfig" + corev1 "k8s.io/api/core/v1" + "testing" + + hyperv1 "github.com/openshift/hypershift/api/v1beta1" + "github.com/openshift/hypershift/support/api" + "github.com/openshift/hypershift/support/testutil" + "github.com/openshift/hypershift/support/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestReconcileOpenShiftControllerManagerConfig(t *testing.T) { + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-namespace", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + ReleaseImage: "quay.io/ocp-dev/test-release-image:latest", + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + }, + IssuerURL: "https://www.example.com", + }, + } + images := map[string]string{ + "openshift-controller-manager": "quay.io/test/openshift-controller-manager", + "docker-builder": "quay.io/test/docker-builder", + "deployer": "quay.io/test/deployer", + } + imageProvider := imageprovider.NewFromImages(images) + + imageConfig := &v1.Image{ + Status: v1.ImageStatus{ + InternalRegistryHostname: "image-registry.openshift-image-registry.svc:5000", + }, + } + + buildConfig := &v1.Build{ + Spec: v1.BuildSpec{ + BuildDefaults: v1.BuildDefaults{ + Env: []corev1.EnvVar{ + { + Name: "TEST_VAR", + Value: "TEST_VALUE", + }, + }, + }, + }, + } + + networkConfig := &v1.NetworkSpec{ + ExternalIP: &v1.ExternalIPConfig{ + AutoAssignCIDRs: []string{"99.1.0.0/24"}, + }, + } + + observedConfig := &globalconfig.ObservedConfig{ + Build: buildConfig, + Image: imageConfig, + } + + params := NewOpenShiftControllerManagerParams(hcp, observedConfig, imageProvider, true) + configMap := manifests.OpenShiftControllerManagerConfig(hcp.Namespace) + + if err := ReconcileOpenShiftControllerManagerConfig(configMap, config2.OwnerRefFrom(hcp), params.DeployerImage, params.DockerBuilderImage, params.MinTLSVersion(), params.CipherSuites(), imageConfig, buildConfig, networkConfig); err != nil { + t.Fatalf("unexpected error: %v", err) + } + configMapYaml, err := util.SerializeResource(configMap, api.Scheme) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + testutil.CompareWithFixture(t, configMapYaml) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/ocm/testdata/zz_fixture_TestReconcileOpenShiftControllerManagerConfig.yaml b/control-plane-operator/controllers/hostedcontrolplane/ocm/testdata/zz_fixture_TestReconcileOpenShiftControllerManagerConfig.yaml new file mode 100644 index 00000000000..b04a13d35d1 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/ocm/testdata/zz_fixture_TestReconcileOpenShiftControllerManagerConfig.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +data: + config.yaml: | + apiVersion: openshiftcontrolplane.config.openshift.io/v1 + build: + additionalTrustedCA: "" + buildDefaults: + env: + - name: TEST_VAR + value: TEST_VALUE + resources: {} + buildOverrides: null + imageTemplateFormat: + format: quay.io/test/docker-builder + latest: false + controllers: null + deployer: + imageTemplateFormat: + format: quay.io/test/deployer + latest: false + dockerPullSecret: + internalRegistryHostname: image-registry.openshift-image-registry.svc:5000 + registryURLs: null + featureGates: null + imageImport: + disableScheduledImport: false + maxScheduledImageImportsPerMinute: 0 + scheduledImageImportMinimumIntervalSeconds: 0 + ingress: + ingressIPNetworkCIDR: 99.1.0.0/24 + kind: OpenShiftControllerManagerConfig + kubeClientConfig: + connectionOverrides: + acceptContentTypes: "" + burst: 0 + contentType: "" + qps: 0 + kubeConfig: /etc/kubernetes/secrets/svc-kubeconfig/kubeconfig + leaderElection: + leaseDuration: 0s + renewDeadline: 0s + retryPeriod: 0s + network: + clusterNetworks: null + networkPluginName: "" + serviceNetworkCIDR: "" + vxlanPort: 0 + resourceQuota: + concurrentSyncs: 0 + minResyncPeriod: 0s + syncPeriod: 0s + securityAllocator: + mcsAllocatorRange: "" + mcsLabelsPerProject: 0 + uidAllocatorRange: "" + serviceAccount: + managedNames: null + serviceServingCert: + signer: null + servingInfo: + bindAddress: 0.0.0.0:8443 + bindNetwork: "" + certFile: /etc/kubernetes/certs/tls.crt + cipherSuites: + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 + clientCA: /etc/kubernetes/client-ca/ca.crt + keyFile: /etc/kubernetes/certs/tls.key + maxRequestsInFlight: 0 + minTLSVersion: VersionTLS12 + requestTimeoutSeconds: 0 +kind: ConfigMap +metadata: + creationTimestamp: null + name: openshift-controller-manager-config + namespace: test-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: test + uid: "" diff --git a/control-plane-operator/controllers/hostedcontrolplane/routecm/config.go b/control-plane-operator/controllers/hostedcontrolplane/routecm/config.go new file mode 100644 index 00000000000..b464ce03f89 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/routecm/config.go @@ -0,0 +1,81 @@ +package routecm + +import ( + "fmt" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" + "path" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1 "github.com/openshift/api/config/v1" + openshiftcpv1 "github.com/openshift/api/openshiftcontrolplane/v1" + + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" + "github.com/openshift/hypershift/support/api" + "github.com/openshift/hypershift/support/certs" + "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/util" +) + +const ( + configKey = "config.yaml" +) + +func ReconcileOpenShiftRouteControllerManagerConfig(cm *corev1.ConfigMap, ownerRef config.OwnerRef, minTLSVersion string, cipherSuites []string, networkConfig *configv1.NetworkSpec) error { + ownerRef.ApplyTo(cm) + + if cm.Data == nil { + cm.Data = map[string]string{} + } + config := &openshiftcpv1.OpenShiftControllerManagerConfig{} + if configStr, exists := cm.Data[configKey]; exists && len(configStr) > 0 { + err := util.DeserializeResource(configStr, config, api.Scheme) + if err != nil { + return fmt.Errorf("unable to decode existing openshift route controller manager configuration: %w", err) + } + } + if err := reconcileConfig(config, minTLSVersion, cipherSuites, networkConfig); err != nil { + return err + } + configStr, err := util.SerializeResource(config, api.Scheme) + if err != nil { + return fmt.Errorf("failed to serialize openshift route controller manager configuration: %w", err) + } + cm.Data[configKey] = configStr + return nil +} + +func reconcileConfig(cfg *openshiftcpv1.OpenShiftControllerManagerConfig, minTLSVersion string, cipherSuites []string, networkConfig *configv1.NetworkSpec) error { + cpath := func(volume, file string) string { + dir := volumeMounts.Path(routeOCMContainerMain().Name, volume) + return path.Join(dir, file) + } + cfg.TypeMeta = metav1.TypeMeta{ + Kind: "OpenShiftControllerManagerConfig", + APIVersion: openshiftcpv1.GroupVersion.String(), + } + + // network config + if networkConfig != nil && networkConfig.ExternalIP != nil && len(networkConfig.ExternalIP.AutoAssignCIDRs) > 0 { + cfg.Ingress.IngressIPNetworkCIDR = networkConfig.ExternalIP.AutoAssignCIDRs[0] + } else { + cfg.Ingress.IngressIPNetworkCIDR = "" + } + + cfg.LeaderElection.Name = "openshift-route-controllers" + cfg.KubeClientConfig.KubeConfig = cpath(routeOCMVolumeKubeconfig().Name, kas.KubeconfigKey) + cfg.ServingInfo = &configv1.HTTPServingInfo{ + ServingInfo: configv1.ServingInfo{ + BindAddress: fmt.Sprintf("0.0.0.0:%d", servingPort), + CertInfo: configv1.CertInfo{ + CertFile: cpath(routeOCMVolumeServingCert().Name, corev1.TLSCertKey), + KeyFile: cpath(routeOCMVolumeServingCert().Name, corev1.TLSPrivateKeyKey), + }, + ClientCA: cpath(common.VolumeTotalClientCA().Name, certs.CASignerCertMapKey), + MinTLSVersion: minTLSVersion, + CipherSuites: cipherSuites, + }, + } + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/routecm/config_test.go b/control-plane-operator/controllers/hostedcontrolplane/routecm/config_test.go new file mode 100644 index 00000000000..44549797c63 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/routecm/config_test.go @@ -0,0 +1,53 @@ +package routecm + +import ( + v1 "github.com/openshift/api/config/v1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + config2 "github.com/openshift/hypershift/support/config" + "testing" + + hyperv1 "github.com/openshift/hypershift/api/v1beta1" + "github.com/openshift/hypershift/support/api" + "github.com/openshift/hypershift/support/testutil" + "github.com/openshift/hypershift/support/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestReconcileOpenShiftRouteControllerManagerConfig(t *testing.T) { + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-namespace", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + ReleaseImage: "quay.io/ocp-dev/test-release-image:latest", + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + }, + IssuerURL: "https://www.example.com", + }, + } + images := map[string]string{ + "route-controller-manager": "quay.io/test/route-controller-manager", + } + imageProvider := imageprovider.NewFromImages(images) + + params := NewOpenShiftRouteControllerManagerParams(hcp, nil, imageProvider, true) + configMap := manifests.OpenShiftRouteControllerManagerConfig(hcp.Namespace) + + networkConfig := &v1.NetworkSpec{ + ExternalIP: &v1.ExternalIPConfig{ + AutoAssignCIDRs: []string{"99.1.0.0/24"}, + }, + } + + if err := ReconcileOpenShiftRouteControllerManagerConfig(configMap, config2.OwnerRefFrom(hcp), params.MinTLSVersion(), params.CipherSuites(), networkConfig); err != nil { + t.Fatalf("unexpected error: %v", err) + } + configMapYaml, err := util.SerializeResource(configMap, api.Scheme) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + testutil.CompareWithFixture(t, configMapYaml) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/routecm/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/routecm/deployment.go index 7edc311da51..45740b1ad6a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/routecm/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/routecm/deployment.go @@ -12,6 +12,7 @@ import ( "k8s.io/utils/pointer" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/util" @@ -21,7 +22,6 @@ const ( configHashAnnotation = "openshift-route-controller-manager.hypershift.openshift.io/config-hash" servingPort int32 = 8443 - configKey = "config.yaml" ) var ( @@ -99,6 +99,9 @@ func buildRouteOCMContainerMain(image string) func(*corev1.Container) { "start", "--config", path.Join(volumeMounts.Path(c.Name, routeOCMVolumeConfig().Name), configKey), + "--kubeconfig", + path.Join(volumeMounts.Path(c.Name, routeOCMVolumeKubeconfig().Name), kas.KubeconfigKey), + "--namespace=openshift-route-controller-manager", } c.VolumeMounts = volumeMounts.ContainerMounts(c.Name) c.Ports = []corev1.ContainerPort{ @@ -108,6 +111,20 @@ func buildRouteOCMContainerMain(image string) func(*corev1.Container) { Protocol: corev1.ProtocolTCP, }, } + c.Env = []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "POD_NAMESPACE", + Value: "openshift-route-controller-manager", + }, + } } } @@ -119,7 +136,7 @@ func routeOCMVolumeConfig() *corev1.Volume { func buildRouteOCMVolumeConfig(v *corev1.Volume) { v.ConfigMap = &corev1.ConfigMapVolumeSource{} - v.ConfigMap.Name = manifests.OpenShiftControllerManagerConfig("").Name + v.ConfigMap.Name = manifests.OpenShiftRouteControllerManagerConfig("").Name } func routeOCMVolumeKubeconfig() *corev1.Volume { diff --git a/control-plane-operator/controllers/hostedcontrolplane/routecm/params.go b/control-plane-operator/controllers/hostedcontrolplane/routecm/params.go index 9386beeec03..cf83201fa24 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/routecm/params.go +++ b/control-plane-operator/controllers/hostedcontrolplane/routecm/params.go @@ -14,6 +14,7 @@ import ( type OpenShiftRouteControllerManagerParams struct { OpenShiftControllerManagerImage string APIServer *configv1.APIServerSpec + Network *configv1.NetworkSpec DeploymentConfig config.DeploymentConfig config.OwnerRef @@ -25,6 +26,7 @@ func NewOpenShiftRouteControllerManagerParams(hcp *hyperv1.HostedControlPlane, o } if hcp.Spec.Configuration != nil { params.APIServer = hcp.Spec.Configuration.APIServer + params.Network = hcp.Spec.Configuration.Network } params.DeploymentConfig = config.DeploymentConfig{ @@ -50,3 +52,17 @@ func NewOpenShiftRouteControllerManagerParams(hcp *hyperv1.HostedControlPlane, o params.OwnerRef = config.OwnerRefFrom(hcp) return params } + +func (p *OpenShiftRouteControllerManagerParams) MinTLSVersion() string { + if p.APIServer != nil { + return config.MinTLSVersion(p.APIServer.TLSSecurityProfile) + } + return config.MinTLSVersion(nil) +} + +func (p *OpenShiftRouteControllerManagerParams) CipherSuites() []string { + if p.APIServer != nil { + return config.CipherSuites(p.APIServer.TLSSecurityProfile) + } + return config.CipherSuites(nil) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/routecm/testdata/zz_fixture_TestReconcileOpenShiftRouteControllerManagerConfig.yaml b/control-plane-operator/controllers/hostedcontrolplane/routecm/testdata/zz_fixture_TestReconcileOpenShiftRouteControllerManagerConfig.yaml new file mode 100644 index 00000000000..73a75036ad5 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/routecm/testdata/zz_fixture_TestReconcileOpenShiftRouteControllerManagerConfig.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +data: + config.yaml: | + apiVersion: openshiftcontrolplane.config.openshift.io/v1 + build: + additionalTrustedCA: "" + buildDefaults: null + buildOverrides: null + imageTemplateFormat: + format: "" + latest: false + controllers: null + deployer: + imageTemplateFormat: + format: "" + latest: false + dockerPullSecret: + internalRegistryHostname: "" + registryURLs: null + featureGates: null + imageImport: + disableScheduledImport: false + maxScheduledImageImportsPerMinute: 0 + scheduledImageImportMinimumIntervalSeconds: 0 + ingress: + ingressIPNetworkCIDR: 99.1.0.0/24 + kind: OpenShiftControllerManagerConfig + kubeClientConfig: + connectionOverrides: + acceptContentTypes: "" + burst: 0 + contentType: "" + qps: 0 + kubeConfig: /etc/kubernetes/secrets/svc-kubeconfig/kubeconfig + leaderElection: + leaseDuration: 0s + name: openshift-route-controllers + renewDeadline: 0s + retryPeriod: 0s + network: + clusterNetworks: null + networkPluginName: "" + serviceNetworkCIDR: "" + vxlanPort: 0 + resourceQuota: + concurrentSyncs: 0 + minResyncPeriod: 0s + syncPeriod: 0s + securityAllocator: + mcsAllocatorRange: "" + mcsLabelsPerProject: 0 + uidAllocatorRange: "" + serviceAccount: + managedNames: null + serviceServingCert: + signer: null + servingInfo: + bindAddress: 0.0.0.0:8443 + bindNetwork: "" + certFile: /etc/kubernetes/certs/tls.crt + cipherSuites: + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 + clientCA: /etc/kubernetes/client-ca/ca.crt + keyFile: /etc/kubernetes/certs/tls.key + maxRequestsInFlight: 0 + minTLSVersion: VersionTLS12 + requestTimeoutSeconds: 0 +kind: ConfigMap +metadata: + creationTimestamp: null + name: openshift-route-controller-manager-config + namespace: test-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: test + uid: ""