diff --git a/control-plane/consul/resource_client.go b/control-plane/consul/resource_client.go index e476f4e737..17dc4a9229 100644 --- a/control-plane/consul/resource_client.go +++ b/control-plane/consul/resource_client.go @@ -1,29 +1,16 @@ package consul import ( - "context" "fmt" "github.com/hashicorp/consul-server-connection-manager/discovery" "github.com/hashicorp/consul/proto-public/pbresource" - "github.com/hashicorp/go-hclog" ) // NewResourceServiceClient creates a pbresource.ResourceServiceClient for creating V2 Consul resources. -// It is initialized with a consul-server-connection-manager discovery config to continuously find Consul +// It is initialized with a consul-server-connection-manager Watcher to continuously find Consul // server addresses. -// The caller should make sure to Stop() the returned `watcher` (preferably with a `defer`) to clean up the gRPC -// connection and the discovery client. -// The caller can also set `config.ServerWatchDisabled=false` to prevent subscribing to Consul server address -// changes, as is the case with single-shot operations. -func NewResourceServiceClient(ctx context.Context, config discovery.Config, logger hclog.Logger, hack int) (pbresource.ResourceServiceClient, *discovery.Watcher, error) { - - watcher, err := discovery.NewWatcher(ctx, config, logger.Named("consul-server-connection-manager")) - if err != nil { - return nil, nil, fmt.Errorf("unable to create Consul server watcher: %w", err) - } - - go watcher.Run() +func NewResourceServiceClient(watcher *discovery.Watcher) (pbresource.ResourceServiceClient, error) { // We recycle the GRPC connection from the discovery client because it // should have all the necessary dial options, including the resolver that @@ -31,10 +18,9 @@ func NewResourceServiceClient(ctx context.Context, config discovery.Config, logg // would need to be duplicated state, err := watcher.State() if err != nil { - watcher.Stop() - return nil, nil, fmt.Errorf("unable to get connection manager state: %w", err) + return nil, fmt.Errorf("unable to get connection manager state: %w", err) } resourceClient := pbresource.NewResourceServiceClient(state.GRPCConn) - return resourceClient, watcher, nil + return resourceClient, nil } diff --git a/control-plane/consul/resource_client_test.go b/control-plane/consul/resource_client_test.go index 7dd9ef0666..29507304b0 100644 --- a/control-plane/consul/resource_client_test.go +++ b/control-plane/consul/resource_client_test.go @@ -36,12 +36,18 @@ func Test_NewResourceServiceClient(t *testing.T) { opts := hclog.LoggerOptions{Name: "resource-service-client"} logger := hclog.New(&opts) - client, watcher, err := NewResourceServiceClient(context.Background(), discoverConfig, logger, serverConfig.Ports.GRPCTLS) + + watcher, err := discovery.NewWatcher(context.Background(), discoverConfig, logger) require.NoError(t, err) - require.NotNil(t, client) require.NotNil(t, watcher) defer watcher.Stop() + go watcher.Run() + + client, err := NewResourceServiceClient(watcher) + require.NoError(t, err) + require.NotNil(t, client) + require.NotNil(t, watcher) req := createWriteRequest(t, "foo") res, err := client.Write(context.Background(), req) diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index c6ad8109e9..4434d52d98 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -27,22 +27,11 @@ import ( "k8s.io/client-go/rest" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - ctrlRuntimeWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - gatewaycommon "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" - gatewaycontrollers "github.com/hashicorp/consul-k8s/control-plane/api-gateway/controllers" - apicommon "github.com/hashicorp/consul-k8s/control-plane/api/common" "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" - "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/endpoints" - "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/peering" - "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" - "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" - "github.com/hashicorp/consul-k8s/control-plane/connect-inject/webhook" - "github.com/hashicorp/consul-k8s/control-plane/controllers" - mutatingwebhookconfiguration "github.com/hashicorp/consul-k8s/control-plane/helper/mutating-webhook-configuration" "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" ) @@ -143,6 +132,18 @@ type Command struct { clientset kubernetes.Interface + // sidecarProxy* are resource limits that are parsed and validated from other flags + // these are individual members because there are override annotations + sidecarProxyCPULimit resource.Quantity + sidecarProxyCPURequest resource.Quantity + sidecarProxyMemoryLimit resource.Quantity + sidecarProxyMemoryRequest resource.Quantity + + // static resources requirements for connect-init + initContainerResources corev1.ResourceRequirements + + caCertPem []byte + once sync.Once help string } @@ -278,67 +279,13 @@ func (c *Command) Run(args []string) int { return 1 } - // Proxy resources. - var sidecarProxyCPULimit, sidecarProxyCPURequest, sidecarProxyMemoryLimit, sidecarProxyMemoryRequest resource.Quantity - var err error - if c.flagDefaultSidecarProxyCPURequest != "" { - sidecarProxyCPURequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPURequest) - if err != nil { - c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-cpu-request is invalid: %s", err)) - return 1 - } - } - - if c.flagDefaultSidecarProxyCPULimit != "" { - sidecarProxyCPULimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPULimit) - if err != nil { - c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-cpu-limit is invalid: %s", err)) - return 1 - } - } - if sidecarProxyCPULimit.Value() != 0 && sidecarProxyCPURequest.Cmp(sidecarProxyCPULimit) > 0 { - c.UI.Error(fmt.Sprintf( - "request must be <= limit: -default-sidecar-proxy-cpu-request value of %q is greater than the -default-sidecar-proxy-cpu-limit value of %q", - c.flagDefaultSidecarProxyCPURequest, c.flagDefaultSidecarProxyCPULimit)) - return 1 - } - - if c.flagDefaultSidecarProxyMemoryRequest != "" { - sidecarProxyMemoryRequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryRequest) - if err != nil { - c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-memory-request is invalid: %s", err)) - return 1 - } - } - if c.flagDefaultSidecarProxyMemoryLimit != "" { - sidecarProxyMemoryLimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryLimit) - if err != nil { - c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-memory-limit is invalid: %s", err)) - return 1 - } - } - if sidecarProxyMemoryLimit.Value() != 0 && sidecarProxyMemoryRequest.Cmp(sidecarProxyMemoryLimit) > 0 { - c.UI.Error(fmt.Sprintf( - "request must be <= limit: -default-sidecar-proxy-memory-request value of %q is greater than the -default-sidecar-proxy-memory-limit value of %q", - c.flagDefaultSidecarProxyMemoryRequest, c.flagDefaultSidecarProxyMemoryLimit)) - return 1 - } - - // Validate ports in metrics flags. - err = common.ValidateUnprivilegedPort("-default-merged-metrics-port", c.flagDefaultMergedMetricsPort) - if err != nil { - c.UI.Error(err.Error()) - return 1 - } - err = common.ValidateUnprivilegedPort("-default-prometheus-scrape-port", c.flagDefaultPrometheusScrapePort) - if err != nil { + if err := c.parseAndValidateSidecarProxyFlags(); err != nil { c.UI.Error(err.Error()) return 1 } // Validate resource request/limit flags and parse into corev1.ResourceRequirements - initResources, err := c.parseAndValidateResourceFlags() - if err != nil { + if err := c.parseAndValidateResourceFlags(); err != nil { c.UI.Error(err.Error()) return 1 } @@ -357,10 +304,6 @@ func (c *Command) Run(args []string) int { } } - // Convert allow/deny lists to sets. - allowK8sNamespaces := flags.ToSet(c.flagAllowK8sNamespacesList) - denyK8sNamespaces := flags.ToSet(c.flagDenyK8sNamespacesList) - zapLogger, err := common.ZapLogger(c.flagLogLevel, c.flagLogJSON) if err != nil { c.UI.Error(fmt.Sprintf("Error setting up logging: %s", err.Error())) @@ -387,13 +330,9 @@ func (c *Command) Run(args []string) int { return 1 } - // Create Consul API config object. - consulConfig := c.consul.ConsulClientConfig() - - var caCertPem []byte if c.consul.CACertFile != "" { var err error - caCertPem, err = os.ReadFile(c.consul.CACertFile) + c.caCertPem, err = os.ReadFile(c.consul.CACertFile) if err != nil { c.UI.Error(fmt.Sprintf("error reading Consul's CA cert file %q", c.consul.CACertFile)) return 1 @@ -410,14 +349,14 @@ func (c *Command) Run(args []string) int { c.UI.Error(fmt.Sprintf("unable to create config for consul-server-connection-manager: %s", err)) return 1 } - watcher, err := discovery.NewWatcher(ctx, serverConnMgrCfg, hcLog) + + watcher, err := discovery.NewWatcher(ctx, serverConnMgrCfg, hcLog.Named("consul-server-connection-manager")) if err != nil { c.UI.Error(fmt.Sprintf("unable to create Consul server watcher: %s", err)) return 1 } - - go watcher.Run() defer watcher.Stop() + go watcher.Run() // This is a blocking command that is run in order to ensure we only start the // connect-inject controllers only after we have access to the Consul server. @@ -442,435 +381,17 @@ func (c *Command) Run(args []string) int { return 1 } - lifecycleConfig := lifecycle.Config{ - DefaultEnableProxyLifecycle: c.flagDefaultEnableSidecarProxyLifecycle, - DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, - DefaultShutdownGracePeriodSeconds: c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, - DefaultGracefulPort: c.flagDefaultSidecarProxyLifecycleGracefulPort, - DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, - } - - metricsConfig := metrics.Config{ - DefaultEnableMetrics: c.flagDefaultEnableMetrics, - EnableGatewayMetrics: c.flagEnableGatewayMetrics, - DefaultEnableMetricsMerging: c.flagDefaultEnableMetricsMerging, - DefaultMergedMetricsPort: c.flagDefaultMergedMetricsPort, - DefaultPrometheusScrapePort: c.flagDefaultPrometheusScrapePort, - DefaultPrometheusScrapePath: c.flagDefaultPrometheusScrapePath, - } - - if err = (&endpoints.Controller{ - Client: mgr.GetClient(), - ConsulClientConfig: consulConfig, - ConsulServerConnMgr: watcher, - AllowK8sNamespacesSet: allowK8sNamespaces, - DenyK8sNamespacesSet: denyK8sNamespaces, - MetricsConfig: metricsConfig, - EnableConsulPartitions: c.flagEnablePartitions, - EnableConsulNamespaces: c.flagEnableNamespaces, - ConsulDestinationNamespace: c.flagConsulDestinationNamespace, - EnableNSMirroring: c.flagEnableK8SNSMirroring, - NSMirroringPrefix: c.flagK8SNSMirroringPrefix, - CrossNSACLPolicy: c.flagCrossNamespaceACLPolicy, - EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, - EnableWANFederation: c.flagEnableFederation, - TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, - AuthMethod: c.flagACLAuthMethod, - NodeMeta: c.flagNodeMeta, - Log: ctrl.Log.WithName("controller").WithName("endpoints"), - Scheme: mgr.GetScheme(), - ReleaseName: c.flagReleaseName, - ReleaseNamespace: c.flagReleaseNamespace, - EnableAutoEncrypt: c.flagEnableAutoEncrypt, - EnableTelemetryCollector: c.flagEnableTelemetryCollector, - Context: ctx, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", endpoints.Controller{}) - return 1 - } - - // API Gateway Controllers - if err := gatewaycontrollers.RegisterFieldIndexes(ctx, mgr); err != nil { - setupLog.Error(err, "unable to register field indexes") - return 1 - } - - if err = (&gatewaycontrollers.GatewayClassConfigController{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName("gateways"), - }).SetupWithManager(ctx, mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", gatewaycontrollers.GatewayClassConfigController{}) - return 1 - } - - if err := (&gatewaycontrollers.GatewayClassController{ - ControllerName: gatewaycommon.GatewayClassControllerName, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("GatewayClass"), - }).SetupWithManager(ctx, mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") - return 1 + //Right now we exclusively start controllers for V1 or V2. + //In the future we might add a flag to pick and choose from both. + if c.flagResourceAPIs { + err = c.configureV2Controllers(ctx, mgr, watcher) + } else { + err = c.configureV1Controllers(ctx, mgr, watcher) } - - cache, err := gatewaycontrollers.SetupGatewayControllerWithManager(ctx, mgr, gatewaycontrollers.GatewayControllerConfig{ - HelmConfig: gatewaycommon.HelmConfig{ - ConsulConfig: gatewaycommon.ConsulConfig{ - Address: c.consul.Addresses, - GRPCPort: consulConfig.GRPCPort, - HTTPPort: consulConfig.HTTPPort, - APITimeout: consulConfig.APITimeout, - }, - ImageDataplane: c.flagConsulDataplaneImage, - ImageConsulK8S: c.flagConsulK8sImage, - ConsulDestinationNamespace: c.flagConsulDestinationNamespace, - NamespaceMirroringPrefix: c.flagK8SNSMirroringPrefix, - EnableNamespaces: c.flagEnableNamespaces, - PeeringEnabled: c.flagEnablePeering, - EnableOpenShift: c.flagEnableOpenShift, - EnableNamespaceMirroring: c.flagEnableK8SNSMirroring, - AuthMethod: c.consul.ConsulLogin.AuthMethod, - LogLevel: c.flagLogLevel, - LogJSON: c.flagLogJSON, - TLSEnabled: c.consul.UseTLS, - ConsulTLSServerName: c.consul.TLSServerName, - ConsulPartition: c.consul.Partition, - ConsulCACert: string(caCertPem), - }, - AllowK8sNamespacesSet: allowK8sNamespaces, - DenyK8sNamespacesSet: denyK8sNamespaces, - ConsulClientConfig: consulConfig, - ConsulServerConnMgr: watcher, - NamespacesEnabled: c.flagEnableNamespaces, - CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy, - Partition: c.consul.Partition, - Datacenter: c.consul.Datacenter, - }) - if err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Gateway") - return 1 - } - - go cache.Run(ctx) - - // wait for the cache to fill - setupLog.Info("waiting for Consul cache sync") - cache.WaitSynced(ctx) - setupLog.Info("Consul cache synced") - - configEntryReconciler := &controllers.ConfigEntryController{ - ConsulClientConfig: c.consul.ConsulClientConfig(), - ConsulServerConnMgr: watcher, - DatacenterName: c.consul.Datacenter, - EnableConsulNamespaces: c.flagEnableNamespaces, - ConsulDestinationNamespace: c.flagConsulDestinationNamespace, - EnableNSMirroring: c.flagEnableK8SNSMirroring, - NSMirroringPrefix: c.flagK8SNSMirroringPrefix, - CrossNSACLPolicy: c.flagCrossNamespaceACLPolicy, - } - if err = (&controllers.ServiceDefaultsController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceDefaults), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceDefaults) - return 1 - } - if err = (&controllers.ServiceResolverController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceResolver), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceResolver) - return 1 - } - if err = (&controllers.ProxyDefaultsController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ProxyDefaults), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ProxyDefaults) - return 1 - } - if err = (&controllers.MeshController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.Mesh), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.Mesh) - return 1 - } - if err = (&controllers.ExportedServicesController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ExportedServices), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ExportedServices) - return 1 - } - if err = (&controllers.ServiceRouterController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceRouter), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceRouter) + setupLog.Error(err, fmt.Sprintf("could not configure controllers: %s", err.Error())) return 1 } - if err = (&controllers.ServiceSplitterController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceSplitter), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceSplitter) - return 1 - } - if err = (&controllers.ServiceIntentionsController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceIntentions), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceIntentions) - return 1 - } - if err = (&controllers.IngressGatewayController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.IngressGateway), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.IngressGateway) - return 1 - } - if err = (&controllers.TerminatingGatewayController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.TerminatingGateway), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.TerminatingGateway) - return 1 - } - if err = (&controllers.SamenessGroupController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.SamenessGroup), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.SamenessGroup) - return 1 - } - if err = (&controllers.JWTProviderController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.JWTProvider), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.JWTProvider) - return 1 - } - if err = (&controllers.ControlPlaneRequestLimitController{ - ConfigEntryController: configEntryReconciler, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controller").WithName(apicommon.ControlPlaneRequestLimit), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", apicommon.ControlPlaneRequestLimit) - return 1 - } - - if err = mgr.AddReadyzCheck("ready", webhook.ReadinessCheck{CertDir: c.flagCertDir}.Ready); err != nil { - setupLog.Error(err, "unable to create readiness check", "controller", endpoints.Controller{}) - return 1 - } - - if c.flagEnablePeering { - if err = (&peering.AcceptorController{ - Client: mgr.GetClient(), - ConsulClientConfig: consulConfig, - ConsulServerConnMgr: watcher, - ExposeServersServiceName: c.flagResourcePrefix + "-expose-servers", - ReleaseNamespace: c.flagReleaseNamespace, - Log: ctrl.Log.WithName("controller").WithName("peering-acceptor"), - Scheme: mgr.GetScheme(), - Context: ctx, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "peering-acceptor") - return 1 - } - if err = (&peering.PeeringDialerController{ - Client: mgr.GetClient(), - ConsulClientConfig: consulConfig, - ConsulServerConnMgr: watcher, - Log: ctrl.Log.WithName("controller").WithName("peering-dialer"), - Scheme: mgr.GetScheme(), - Context: ctx, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "peering-dialer") - return 1 - } - - mgr.GetWebhookServer().Register("/mutate-v1alpha1-peeringacceptors", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.PeeringAcceptorWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName("peering-acceptor"), - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-peeringdialers", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.PeeringDialerWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName("peering-dialer"), - }}) - } - - mgr.GetWebhookServer().CertDir = c.flagCertDir - - mgr.GetWebhookServer().Register("/mutate", - &ctrlRuntimeWebhook.Admission{Handler: &webhook.MeshWebhook{ - Clientset: c.clientset, - ReleaseNamespace: c.flagReleaseNamespace, - ConsulConfig: consulConfig, - ConsulServerConnMgr: watcher, - ImageConsul: c.flagConsulImage, - ImageConsulDataplane: c.flagConsulDataplaneImage, - EnvoyExtraArgs: c.flagEnvoyExtraArgs, - ImageConsulK8S: c.flagConsulK8sImage, - RequireAnnotation: !c.flagDefaultInject, - AuthMethod: c.flagACLAuthMethod, - ConsulCACert: string(caCertPem), - TLSEnabled: c.consul.UseTLS, - ConsulAddress: c.consul.Addresses, - SkipServerWatch: c.consul.SkipServerWatch, - ConsulTLSServerName: c.consul.TLSServerName, - DefaultProxyCPURequest: sidecarProxyCPURequest, - DefaultProxyCPULimit: sidecarProxyCPULimit, - DefaultProxyMemoryRequest: sidecarProxyMemoryRequest, - DefaultProxyMemoryLimit: sidecarProxyMemoryLimit, - DefaultEnvoyProxyConcurrency: c.flagDefaultEnvoyProxyConcurrency, - LifecycleConfig: lifecycleConfig, - MetricsConfig: metricsConfig, - InitContainerResources: initResources, - ConsulPartition: c.consul.Partition, - AllowK8sNamespacesSet: allowK8sNamespaces, - DenyK8sNamespacesSet: denyK8sNamespaces, - EnableNamespaces: c.flagEnableNamespaces, - ConsulDestinationNamespace: c.flagConsulDestinationNamespace, - EnableK8SNSMirroring: c.flagEnableK8SNSMirroring, - K8SNSMirroringPrefix: c.flagK8SNSMirroringPrefix, - CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy, - EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, - EnableCNI: c.flagEnableCNI, - TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, - EnableConsulDNS: c.flagEnableConsulDNS, - EnableOpenShift: c.flagEnableOpenShift, - Log: ctrl.Log.WithName("handler").WithName("connect"), - LogLevel: c.flagLogLevel, - LogJSON: c.flagLogJSON, - }}) - - consulMeta := apicommon.ConsulMeta{ - PartitionsEnabled: c.flagEnablePartitions, - Partition: c.consul.Partition, - NamespacesEnabled: c.flagEnableNamespaces, - DestinationNamespace: c.flagConsulDestinationNamespace, - Mirroring: c.flagEnableK8SNSMirroring, - Prefix: c.flagK8SNSMirroringPrefix, - } - - // Note: The path here should be identical to the one on the kubebuilder - // annotation in each webhook file. - mgr.GetWebhookServer().Register("/mutate-v1alpha1-servicedefaults", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceDefaultsWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceDefaults), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-serviceresolver", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceResolverWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceResolver), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-proxydefaults", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ProxyDefaultsWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ProxyDefaults), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-mesh", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.MeshWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.Mesh), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-exportedservices", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ExportedServicesWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ExportedServices), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-servicerouter", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceRouterWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceRouter), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-servicesplitter", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceSplitterWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceSplitter), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-serviceintentions", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceIntentionsWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceIntentions), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-ingressgateway", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.IngressGatewayWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.IngressGateway), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-terminatinggateway", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.TerminatingGatewayWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.TerminatingGateway), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-samenessgroup", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.SamenessGroupWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.SamenessGroup), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-jwtprovider", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.JWTProviderWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.JWTProvider), - ConsulMeta: consulMeta, - }}) - mgr.GetWebhookServer().Register("/mutate-v1alpha1-controlplanerequestlimits", - &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ControlPlaneRequestLimitWebhook{ - Client: mgr.GetClient(), - Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ControlPlaneRequestLimit), - ConsulMeta: consulMeta, - }}) - - if c.flagEnableWebhookCAUpdate { - err = c.updateWebhookCABundle(ctx) - if err != nil { - setupLog.Error(err, "problem getting CA Cert") - return 1 - } - } if err = mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") @@ -880,20 +401,6 @@ func (c *Command) Run(args []string) int { return 0 } -func (c *Command) updateWebhookCABundle(ctx context.Context) error { - webhookConfigName := fmt.Sprintf("%s-connect-injector", c.flagResourcePrefix) - caPath := fmt.Sprintf("%s/%s", c.flagCertDir, WebhookCAFilename) - caCert, err := os.ReadFile(caPath) - if err != nil { - return err - } - err = mutatingwebhookconfiguration.UpdateWithCABundle(ctx, c.clientset, webhookConfigName, caCert) - if err != nil { - return err - } - return nil -} - func (c *Command) validateFlags() error { if c.flagConsulK8sImage == "" { return errors.New("-consul-k8s-image must be set") @@ -917,48 +424,95 @@ func (c *Command) validateFlags() error { return errors.New("-default-envoy-proxy-concurrency must be >= 0 if set") } + // Validate ports in metrics flags. + err := common.ValidateUnprivilegedPort("-default-merged-metrics-port", c.flagDefaultMergedMetricsPort) + if err != nil { + return err + } + err = common.ValidateUnprivilegedPort("-default-prometheus-scrape-port", c.flagDefaultPrometheusScrapePort) + if err != nil { + return err + } + + return nil +} + +func (c *Command) parseAndValidateSidecarProxyFlags() error { + var err error + + if c.flagDefaultSidecarProxyCPURequest != "" { + c.sidecarProxyCPURequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPURequest) + if err != nil { + return fmt.Errorf("-default-sidecar-proxy-cpu-request is invalid: %w", err) + } + } + + if c.flagDefaultSidecarProxyCPULimit != "" { + c.sidecarProxyCPULimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPULimit) + if err != nil { + return fmt.Errorf("-default-sidecar-proxy-cpu-limit is invalid: %w", err) + } + } + if c.sidecarProxyCPULimit.Value() != 0 && c.sidecarProxyCPURequest.Cmp(c.sidecarProxyMemoryLimit) > 0 { + return fmt.Errorf("request must be <= limit: -default-sidecar-proxy-cpu-request value of %q is greater than the -default-sidecar-proxy-cpu-limit value of %q", + c.flagDefaultSidecarProxyCPURequest, c.flagDefaultSidecarProxyCPULimit) + } + + if c.flagDefaultSidecarProxyMemoryRequest != "" { + c.sidecarProxyMemoryRequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryRequest) + if err != nil { + return fmt.Errorf("-default-sidecar-proxy-memory-request is invalid: %w", err) + } + } + if c.flagDefaultSidecarProxyMemoryLimit != "" { + c.sidecarProxyMemoryLimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryLimit) + if err != nil { + return fmt.Errorf("-default-sidecar-proxy-memory-limit is invalid: %w", err) + } + } + if c.sidecarProxyMemoryLimit.Value() != 0 && c.sidecarProxyMemoryRequest.Cmp(c.sidecarProxyMemoryLimit) > 0 { + return fmt.Errorf("request must be <= limit: -default-sidecar-proxy-memory-request value of %q is greater than the -default-sidecar-proxy-memory-limit value of %q", + c.flagDefaultSidecarProxyMemoryRequest, c.flagDefaultSidecarProxyMemoryLimit) + } + return nil } -func (c *Command) parseAndValidateResourceFlags() (corev1.ResourceRequirements, error) { +func (c *Command) parseAndValidateResourceFlags() error { // Init container var initContainerCPULimit, initContainerCPURequest, initContainerMemoryLimit, initContainerMemoryRequest resource.Quantity // Parse and validate the initContainer resources. initContainerCPURequest, err := resource.ParseQuantity(c.flagInitContainerCPURequest) if err != nil { - return corev1.ResourceRequirements{}, - fmt.Errorf("-init-container-cpu-request '%s' is invalid: %s", c.flagInitContainerCPURequest, err) + return fmt.Errorf("-init-container-cpu-request '%s' is invalid: %s", c.flagInitContainerCPURequest, err) } initContainerCPULimit, err = resource.ParseQuantity(c.flagInitContainerCPULimit) if err != nil { - return corev1.ResourceRequirements{}, - fmt.Errorf("-init-container-cpu-limit '%s' is invalid: %s", c.flagInitContainerCPULimit, err) + return fmt.Errorf("-init-container-cpu-limit '%s' is invalid: %s", c.flagInitContainerCPULimit, err) } if initContainerCPULimit.Value() != 0 && initContainerCPURequest.Cmp(initContainerCPULimit) > 0 { - return corev1.ResourceRequirements{}, fmt.Errorf( + return fmt.Errorf( "request must be <= limit: -init-container-cpu-request value of %q is greater than the -init-container-cpu-limit value of %q", c.flagInitContainerCPURequest, c.flagInitContainerCPULimit) } initContainerMemoryRequest, err = resource.ParseQuantity(c.flagInitContainerMemoryRequest) if err != nil { - return corev1.ResourceRequirements{}, - fmt.Errorf("-init-container-memory-request '%s' is invalid: %s", c.flagInitContainerMemoryRequest, err) + return fmt.Errorf("-init-container-memory-request '%s' is invalid: %s", c.flagInitContainerMemoryRequest, err) } initContainerMemoryLimit, err = resource.ParseQuantity(c.flagInitContainerMemoryLimit) if err != nil { - return corev1.ResourceRequirements{}, - fmt.Errorf("-init-container-memory-limit '%s' is invalid: %s", c.flagInitContainerMemoryLimit, err) + return fmt.Errorf("-init-container-memory-limit '%s' is invalid: %s", c.flagInitContainerMemoryLimit, err) } if initContainerMemoryLimit.Value() != 0 && initContainerMemoryRequest.Cmp(initContainerMemoryLimit) > 0 { - return corev1.ResourceRequirements{}, fmt.Errorf( + return fmt.Errorf( "request must be <= limit: -init-container-memory-request value of %q is greater than the -init-container-memory-limit value of %q", c.flagInitContainerMemoryRequest, c.flagInitContainerMemoryLimit) } // Put into corev1.ResourceRequirements form - initResources := corev1.ResourceRequirements{ + c.initContainerResources = corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: initContainerCPURequest, corev1.ResourceMemory: initContainerMemoryRequest, @@ -969,7 +523,7 @@ func (c *Command) parseAndValidateResourceFlags() (corev1.ResourceRequirements, }, } - return initResources, nil + return nil } func (c *Command) Synopsis() string { return synopsis } diff --git a/control-plane/subcommand/inject-connect/v1controllers.go b/control-plane/subcommand/inject-connect/v1controllers.go new file mode 100644 index 0000000000..57ac1691c3 --- /dev/null +++ b/control-plane/subcommand/inject-connect/v1controllers.go @@ -0,0 +1,484 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package connectinject + +import ( + "context" + "fmt" + "os" + + "github.com/hashicorp/consul-server-connection-manager/discovery" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + ctrlRuntimeWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" + + gatewaycommon "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" + gatewaycontrollers "github.com/hashicorp/consul-k8s/control-plane/api-gateway/controllers" + apicommon "github.com/hashicorp/consul-k8s/control-plane/api/common" + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/endpoints" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/peering" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/webhook" + "github.com/hashicorp/consul-k8s/control-plane/controllers" + mutatingwebhookconfiguration "github.com/hashicorp/consul-k8s/control-plane/helper/mutating-webhook-configuration" + "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" +) + +func (c *Command) configureV1Controllers(ctx context.Context, mgr manager.Manager, watcher *discovery.Watcher) error { + + // Create Consul API config object. + consulConfig := c.consul.ConsulClientConfig() + + // Convert allow/deny lists to sets. + allowK8sNamespaces := flags.ToSet(c.flagAllowK8sNamespacesList) + denyK8sNamespaces := flags.ToSet(c.flagDenyK8sNamespacesList) + + lifecycleConfig := lifecycle.Config{ + DefaultEnableProxyLifecycle: c.flagDefaultEnableSidecarProxyLifecycle, + DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, + DefaultShutdownGracePeriodSeconds: c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, + DefaultGracefulPort: c.flagDefaultSidecarProxyLifecycleGracefulPort, + DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, + } + + metricsConfig := metrics.Config{ + DefaultEnableMetrics: c.flagDefaultEnableMetrics, + EnableGatewayMetrics: c.flagEnableGatewayMetrics, + DefaultEnableMetricsMerging: c.flagDefaultEnableMetricsMerging, + DefaultMergedMetricsPort: c.flagDefaultMergedMetricsPort, + DefaultPrometheusScrapePort: c.flagDefaultPrometheusScrapePort, + DefaultPrometheusScrapePath: c.flagDefaultPrometheusScrapePath, + } + + if err := (&endpoints.Controller{ + Client: mgr.GetClient(), + ConsulClientConfig: consulConfig, + ConsulServerConnMgr: watcher, + AllowK8sNamespacesSet: allowK8sNamespaces, + DenyK8sNamespacesSet: denyK8sNamespaces, + MetricsConfig: metricsConfig, + EnableConsulPartitions: c.flagEnablePartitions, + EnableConsulNamespaces: c.flagEnableNamespaces, + ConsulDestinationNamespace: c.flagConsulDestinationNamespace, + EnableNSMirroring: c.flagEnableK8SNSMirroring, + NSMirroringPrefix: c.flagK8SNSMirroringPrefix, + CrossNSACLPolicy: c.flagCrossNamespaceACLPolicy, + EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, + EnableWANFederation: c.flagEnableFederation, + TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, + AuthMethod: c.flagACLAuthMethod, + NodeMeta: c.flagNodeMeta, + Log: ctrl.Log.WithName("controller").WithName("endpoints"), + Scheme: mgr.GetScheme(), + ReleaseName: c.flagReleaseName, + ReleaseNamespace: c.flagReleaseNamespace, + EnableAutoEncrypt: c.flagEnableAutoEncrypt, + EnableTelemetryCollector: c.flagEnableTelemetryCollector, + Context: ctx, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", endpoints.Controller{}) + return err + } + + // API Gateway Controllers + if err := gatewaycontrollers.RegisterFieldIndexes(ctx, mgr); err != nil { + setupLog.Error(err, "unable to register field indexes") + return err + } + + if err := (&gatewaycontrollers.GatewayClassConfigController{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName("gateways"), + }).SetupWithManager(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", gatewaycontrollers.GatewayClassConfigController{}) + return err + } + + if err := (&gatewaycontrollers.GatewayClassController{ + ControllerName: gatewaycommon.GatewayClassControllerName, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("GatewayClass"), + }).SetupWithManager(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") + return err + } + + cache, err := gatewaycontrollers.SetupGatewayControllerWithManager(ctx, mgr, gatewaycontrollers.GatewayControllerConfig{ + HelmConfig: gatewaycommon.HelmConfig{ + ConsulConfig: gatewaycommon.ConsulConfig{ + Address: c.consul.Addresses, + GRPCPort: consulConfig.GRPCPort, + HTTPPort: consulConfig.HTTPPort, + APITimeout: consulConfig.APITimeout, + }, + ImageDataplane: c.flagConsulDataplaneImage, + ImageConsulK8S: c.flagConsulK8sImage, + ConsulDestinationNamespace: c.flagConsulDestinationNamespace, + NamespaceMirroringPrefix: c.flagK8SNSMirroringPrefix, + EnableNamespaces: c.flagEnableNamespaces, + PeeringEnabled: c.flagEnablePeering, + EnableOpenShift: c.flagEnableOpenShift, + EnableNamespaceMirroring: c.flagEnableK8SNSMirroring, + AuthMethod: c.consul.ConsulLogin.AuthMethod, + LogLevel: c.flagLogLevel, + LogJSON: c.flagLogJSON, + TLSEnabled: c.consul.UseTLS, + ConsulTLSServerName: c.consul.TLSServerName, + ConsulPartition: c.consul.Partition, + ConsulCACert: string(c.caCertPem), + }, + AllowK8sNamespacesSet: allowK8sNamespaces, + DenyK8sNamespacesSet: denyK8sNamespaces, + ConsulClientConfig: consulConfig, + ConsulServerConnMgr: watcher, + NamespacesEnabled: c.flagEnableNamespaces, + CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy, + Partition: c.consul.Partition, + Datacenter: c.consul.Datacenter, + }) + + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Gateway") + return err + } + + go cache.Run(ctx) + + // wait for the cache to fill + setupLog.Info("waiting for Consul cache sync") + cache.WaitSynced(ctx) + setupLog.Info("Consul cache synced") + + configEntryReconciler := &controllers.ConfigEntryController{ + ConsulClientConfig: consulConfig, + ConsulServerConnMgr: watcher, + DatacenterName: c.consul.Datacenter, + EnableConsulNamespaces: c.flagEnableNamespaces, + ConsulDestinationNamespace: c.flagConsulDestinationNamespace, + EnableNSMirroring: c.flagEnableK8SNSMirroring, + NSMirroringPrefix: c.flagK8SNSMirroringPrefix, + CrossNSACLPolicy: c.flagCrossNamespaceACLPolicy, + } + if err := (&controllers.ServiceDefaultsController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceDefaults), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceDefaults) + return err + } + if err := (&controllers.ServiceResolverController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceResolver), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceResolver) + return err + } + if err := (&controllers.ProxyDefaultsController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ProxyDefaults), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ProxyDefaults) + return err + } + if err := (&controllers.MeshController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.Mesh), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.Mesh) + return err + } + if err := (&controllers.ExportedServicesController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ExportedServices), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ExportedServices) + return err + } + if err := (&controllers.ServiceRouterController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceRouter), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceRouter) + return err + } + if err := (&controllers.ServiceSplitterController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceSplitter), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceSplitter) + return err + } + if err := (&controllers.ServiceIntentionsController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ServiceIntentions), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ServiceIntentions) + return err + } + if err := (&controllers.IngressGatewayController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.IngressGateway), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.IngressGateway) + return err + } + if err := (&controllers.TerminatingGatewayController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.TerminatingGateway), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.TerminatingGateway) + return err + } + if err := (&controllers.SamenessGroupController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.SamenessGroup), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.SamenessGroup) + return err + } + if err := (&controllers.JWTProviderController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.JWTProvider), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.JWTProvider) + return err + } + if err := (&controllers.ControlPlaneRequestLimitController{ + ConfigEntryController: configEntryReconciler, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controller").WithName(apicommon.ControlPlaneRequestLimit), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", apicommon.ControlPlaneRequestLimit) + return err + } + + if err := mgr.AddReadyzCheck("ready", webhook.ReadinessCheck{CertDir: c.flagCertDir}.Ready); err != nil { + setupLog.Error(err, "unable to create readiness check", "controller", endpoints.Controller{}) + return err + } + + if c.flagEnablePeering { + if err := (&peering.AcceptorController{ + Client: mgr.GetClient(), + ConsulClientConfig: consulConfig, + ConsulServerConnMgr: watcher, + ExposeServersServiceName: c.flagResourcePrefix + "-expose-servers", + ReleaseNamespace: c.flagReleaseNamespace, + Log: ctrl.Log.WithName("controller").WithName("peering-acceptor"), + Scheme: mgr.GetScheme(), + Context: ctx, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "peering-acceptor") + return err + } + if err := (&peering.PeeringDialerController{ + Client: mgr.GetClient(), + ConsulClientConfig: consulConfig, + ConsulServerConnMgr: watcher, + Log: ctrl.Log.WithName("controller").WithName("peering-dialer"), + Scheme: mgr.GetScheme(), + Context: ctx, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "peering-dialer") + return err + } + + mgr.GetWebhookServer().Register("/mutate-v1alpha1-peeringacceptors", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.PeeringAcceptorWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName("peering-acceptor"), + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-peeringdialers", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.PeeringDialerWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName("peering-dialer"), + }}) + } + + mgr.GetWebhookServer().CertDir = c.flagCertDir + + mgr.GetWebhookServer().Register("/mutate", + &ctrlRuntimeWebhook.Admission{Handler: &webhook.MeshWebhook{ + Clientset: c.clientset, + ReleaseNamespace: c.flagReleaseNamespace, + ConsulConfig: consulConfig, + ConsulServerConnMgr: watcher, + ImageConsul: c.flagConsulImage, + ImageConsulDataplane: c.flagConsulDataplaneImage, + EnvoyExtraArgs: c.flagEnvoyExtraArgs, + ImageConsulK8S: c.flagConsulK8sImage, + RequireAnnotation: !c.flagDefaultInject, + AuthMethod: c.flagACLAuthMethod, + ConsulCACert: string(c.caCertPem), + TLSEnabled: c.consul.UseTLS, + ConsulAddress: c.consul.Addresses, + SkipServerWatch: c.consul.SkipServerWatch, + ConsulTLSServerName: c.consul.TLSServerName, + DefaultProxyCPURequest: c.sidecarProxyCPURequest, + DefaultProxyCPULimit: c.sidecarProxyCPULimit, + DefaultProxyMemoryRequest: c.sidecarProxyMemoryRequest, + DefaultProxyMemoryLimit: c.sidecarProxyMemoryLimit, + DefaultEnvoyProxyConcurrency: c.flagDefaultEnvoyProxyConcurrency, + LifecycleConfig: lifecycleConfig, + MetricsConfig: metricsConfig, + InitContainerResources: c.initContainerResources, + ConsulPartition: c.consul.Partition, + AllowK8sNamespacesSet: allowK8sNamespaces, + DenyK8sNamespacesSet: denyK8sNamespaces, + EnableNamespaces: c.flagEnableNamespaces, + ConsulDestinationNamespace: c.flagConsulDestinationNamespace, + EnableK8SNSMirroring: c.flagEnableK8SNSMirroring, + K8SNSMirroringPrefix: c.flagK8SNSMirroringPrefix, + CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy, + EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, + EnableCNI: c.flagEnableCNI, + TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, + EnableConsulDNS: c.flagEnableConsulDNS, + EnableOpenShift: c.flagEnableOpenShift, + Log: ctrl.Log.WithName("handler").WithName("connect"), + LogLevel: c.flagLogLevel, + LogJSON: c.flagLogJSON, + }}) + + consulMeta := apicommon.ConsulMeta{ + PartitionsEnabled: c.flagEnablePartitions, + Partition: c.consul.Partition, + NamespacesEnabled: c.flagEnableNamespaces, + DestinationNamespace: c.flagConsulDestinationNamespace, + Mirroring: c.flagEnableK8SNSMirroring, + Prefix: c.flagK8SNSMirroringPrefix, + } + + // Note: The path here should be identical to the one on the kubebuilder + // annotation in each webhook file. + mgr.GetWebhookServer().Register("/mutate-v1alpha1-servicedefaults", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceDefaultsWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceDefaults), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-serviceresolver", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceResolverWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceResolver), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-proxydefaults", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ProxyDefaultsWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ProxyDefaults), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-mesh", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.MeshWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.Mesh), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-exportedservices", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ExportedServicesWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ExportedServices), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-servicerouter", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceRouterWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceRouter), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-servicesplitter", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceSplitterWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceSplitter), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-serviceintentions", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ServiceIntentionsWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ServiceIntentions), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-ingressgateway", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.IngressGatewayWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.IngressGateway), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-terminatinggateway", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.TerminatingGatewayWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.TerminatingGateway), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-samenessgroup", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.SamenessGroupWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.SamenessGroup), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-jwtprovider", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.JWTProviderWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.JWTProvider), + ConsulMeta: consulMeta, + }}) + mgr.GetWebhookServer().Register("/mutate-v1alpha1-controlplanerequestlimits", + &ctrlRuntimeWebhook.Admission{Handler: &v1alpha1.ControlPlaneRequestLimitWebhook{ + Client: mgr.GetClient(), + Logger: ctrl.Log.WithName("webhooks").WithName(apicommon.ControlPlaneRequestLimit), + ConsulMeta: consulMeta, + }}) + + if c.flagEnableWebhookCAUpdate { + err = c.updateWebhookCABundle(ctx) + if err != nil { + setupLog.Error(err, "problem getting CA Cert") + return err + } + } + + return nil +} + +func (c *Command) updateWebhookCABundle(ctx context.Context) error { + webhookConfigName := fmt.Sprintf("%s-connect-injector", c.flagResourcePrefix) + caPath := fmt.Sprintf("%s/%s", c.flagCertDir, WebhookCAFilename) + caCert, err := os.ReadFile(caPath) + if err != nil { + return err + } + err = mutatingwebhookconfiguration.UpdateWithCABundle(ctx, c.clientset, webhookConfigName, caCert) + if err != nil { + return err + } + return nil +} diff --git a/control-plane/subcommand/inject-connect/v2controllers.go b/control-plane/subcommand/inject-connect/v2controllers.go new file mode 100644 index 0000000000..d851f71cb8 --- /dev/null +++ b/control-plane/subcommand/inject-connect/v2controllers.go @@ -0,0 +1,94 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package connectinject + +import ( + "context" + + "github.com/hashicorp/consul-server-connection-manager/discovery" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +func (c *Command) configureV2Controllers(ctx context.Context, mgr manager.Manager, watcher *discovery.Watcher) error { + + //resourceClient, err := consul.NewResourceServiceClient(watcher) + //if err != nil { + // return fmt.Errorf("unable to create Consul resource service client: %w", err) + //} + + //// Create Consul API config object. + //consulConfig := c.consul.ConsulClientConfig() + // + ////Convert allow/deny lists to sets. + //allowK8sNamespaces := flags.ToSet(c.flagAllowK8sNamespacesList) + //denyK8sNamespaces := flags.ToSet(c.flagDenyK8sNamespacesList) + + //lifecycleConfig := lifecycle.Config{ + // DefaultEnableProxyLifecycle: c.flagDefaultEnableSidecarProxyLifecycle, + // DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, + // DefaultShutdownGracePeriodSeconds: c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, + // DefaultGracefulPort: c.flagDefaultSidecarProxyLifecycleGracefulPort, + // DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, + //} + + //metricsConfig := metrics.Config{ + // DefaultEnableMetrics: c.flagDefaultEnableMetrics, + // EnableGatewayMetrics: c.flagEnableGatewayMetrics, + // DefaultEnableMetricsMerging: c.flagDefaultEnableMetricsMerging, + // DefaultMergedMetricsPort: c.flagDefaultMergedMetricsPort, + // DefaultPrometheusScrapePort: c.flagDefaultPrometheusScrapePort, + // DefaultPrometheusScrapePath: c.flagDefaultPrometheusScrapePath, + //} + + // TODO(dans): Pods Controller + //if err := (&pod.Controller{ + // Client: mgr.GetClient(), + // ConsulClientConfig: consulConfig, + // ConsulServerConnMgr: watcher, + // ConsulResourceServiceClient: client, + // AllowK8sNamespacesSet: allowK8sNamespaces, + // DenyK8sNamespacesSet: denyK8sNamespaces, + // MetricsConfig: metricsConfig, + // EnableConsulPartitions: c.flagEnablePartitions, + // EnableConsulNamespaces: c.flagEnableNamespaces, + // ConsulDestinationNamespace: c.flagConsulDestinationNamespace, + // EnableNSMirroring: c.flagEnableK8SNSMirroring, + // NSMirroringPrefix: c.flagK8SNSMirroringPrefix, + // EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, + // TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, + // AuthMethod: c.flagACLAuthMethod, + // NodeMeta: c.flagNodeMeta, + // Log: ctrl.Log.WithName("controller").WithName("pods"), + // Scheme: mgr.GetScheme(), + // EnableTelemetryCollector: c.flagEnableTelemetryCollector, + // Context: ctx, + //}).SetupWithManager(mgr); err != nil { + // setupLog.Error(err, "unable to create controller", "controller", pod.Controller{}) + // return err + //} + + // TODO: V2 Endpoints Controller + + // TODO: Nodes Controller + + // TODO: Serviceaccounts Controller + + // TODO: V2 Config Controller(s) + + // // Metadata for webhooks + //consulMeta := apicommon.ConsulMeta{ + // PartitionsEnabled: c.flagEnablePartitions, + // Partition: c.consul.Partition, + // NamespacesEnabled: c.flagEnableNamespaces, + // DestinationNamespace: c.flagConsulDestinationNamespace, + // Mirroring: c.flagEnableK8SNSMirroring, + // Prefix: c.flagK8SNSMirroringPrefix, + //} + + // TODO: register webhooks + + // TODO: Update Webhook CA Bundle + + return nil +}