AutoOps ECK integration

cmd/manager/main.go

@@ -44,6 +44,7 @@ import (
 	agentv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/agent/v1alpha1"
 	apmv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/apm/v1"
 	apmv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/apm/v1beta1"
+	autoopsv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/autoops/v1alpha1"
 	beatv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/beat/v1beta1"
 	esv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/elasticsearch/v1"
 	esv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/elasticsearch/v1beta1"
@@ -59,11 +60,13 @@ import (
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/apmserver"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/association"
 	associationctl "github.com/elastic/cloud-on-k8s/v3/pkg/controller/association/controller"
+	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/autoops"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/autoscaling"
 	esavalidation "github.com/elastic/cloud-on-k8s/v3/pkg/controller/autoscaling/elasticsearch/validation"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/beat"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/certificates"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container"
+	commonesclient "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/esclient"
 	commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/password"
@@ -758,7 +761,7 @@ func startOperator(ctx context.Context) error {
 
 	disableTelemetry := viper.GetBool(operator.DisableTelemetryFlag)
 	telemetryInterval := viper.GetDuration(operator.TelemetryIntervalFlag)
-	go asyncTasks(ctx, mgr, cfg, managedNamespaces, operatorNamespace, operatorInfo, disableTelemetry, telemetryInterval, tracer)
+	go asyncTasks(ctx, mgr, cfg, managedNamespaces, operatorNamespace, operatorInfo, disableTelemetry, telemetryInterval, tracer, dialer)
 
 	log.Info("Starting the manager", "uuid", operatorInfo.OperatorUUID,
 		"namespace", operatorNamespace, "version", operatorInfo.BuildInfo.Version,
@@ -817,6 +820,7 @@ func asyncTasks(
 	disableTelemetry bool,
 	telemetryInterval time.Duration,
 	tracer *apm.Tracer,
+	dialer net.Dialer,
 ) {
 	<-mgr.Elected() // wait for this operator instance to be elected
 
@@ -843,13 +847,18 @@ func asyncTasks(
 	// Garbage collect orphaned secrets leftover from deleted resources while the operator was not running
 	// - association user secrets
 	gcCtx := tracing.NewContextTransaction(ctx, tracer, tracing.RunOnceTxType, "garbage-collection", nil)
+	gcCtx = logconf.AddToContext(gcCtx, logf.Log.WithName("garbage-collection"))
 	err := garbageCollectUsers(gcCtx, cfg, managedNamespaces)
 	if err != nil {
 		log.Error(err, "exiting due to unrecoverable error")
 		os.Exit(1)
 	}
 	// - soft-owned secrets
 	garbageCollectSoftOwnedSecrets(gcCtx, mgr.GetClient())
+	// - autoops orphaned resources (API key secrets without owner references)
+	if err := garbageCollectAutoOpsResources(gcCtx, mgr.GetClient(), dialer); err != nil {
+		log.Error(err, "AutoOps garbage collection failed, will be attempted again at next operator restart")
+	}
 	tracing.EndContextTransaction(gcCtx)
 }
 
@@ -932,6 +941,9 @@ func registerControllers(mgr manager.Manager, params operator.Parameters, access
 		name         string
 		registerFunc func(manager.Manager, rbac.AccessReviewer, operator.Parameters) error
 	}{
+		// AutoOps isn't technically an association controller, but it's closely related and
+		// it's Add function signature is the same as the association controllers.
+		{name: "AutoOpsAgentPolicy", registerFunc: autoops.Add},
 		{name: "RemoteCA", registerFunc: remotecluster.Add},
 		{name: "APM-ES", registerFunc: associationctl.AddApmES},
 		{name: "APM-KB", registerFunc: associationctl.AddApmKibana},
@@ -999,6 +1011,7 @@ func garbageCollectSoftOwnedSecrets(ctx context.Context, k8sClient k8s.Client) {
 		emsv1alpha1.Kind:      &emsv1alpha1.ElasticMapsServer{},
 		eprv1alpha1.Kind:      &eprv1alpha1.PackageRegistry{},
 		policyv1alpha1.Kind:   &policyv1alpha1.StackConfigPolicy{},
+		autoopsv1alpha1.Kind:  &autoopsv1alpha1.AutoOpsAgentPolicy{},
 		logstashv1alpha1.Kind: &logstashv1alpha1.Logstash{},
 	}); err != nil {
 		log.Error(err, "Orphan secrets garbage collection failed, will be attempted again at next operator restart.")
@@ -1007,6 +1020,17 @@ func garbageCollectSoftOwnedSecrets(ctx context.Context, k8sClient k8s.Client) {
 	log.Info("Orphan secrets garbage collection complete")
 }
 
+func garbageCollectAutoOpsResources(ctx context.Context, k8sClient k8s.Client, dialer net.Dialer) error {
+	span, ctx := apm.StartSpan(ctx, "gc_autoops_resources", tracing.SpanTypeApp)
+	defer span.End()
+
+	gc := autoops.NewGarbageCollector(k8sClient, commonesclient.NewClient, dialer)
+	if err := gc.DoGarbageCollection(ctx); err != nil {
+		return fmt.Errorf("AutoOps garbage collection failed: %w", err)
+	}
+	return nil
+}
+
 func setupWebhook(
 	ctx context.Context,
 	mgr manager.Manager,
@@ -1044,6 +1068,7 @@ func setupWebhook(
 		&emsv1alpha1.ElasticMapsServer{},
 		&eprv1alpha1.PackageRegistry{},
 		&policyv1alpha1.StackConfigPolicy{},
+		&autoopsv1alpha1.AutoOpsAgentPolicy{},
 	}
 	for _, obj := range webhookObjects {
 		commonwebhook.SetupValidatingWebhookWithConfig(&commonwebhook.Config{

config/crds/v1/all-crds.yaml

@@ -2355,6 +2355,182 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.19.0
+  name: autoopsagentpolicies.autoops.k8s.elastic.co
+spec:
+  group: autoops.k8s.elastic.co
+  names:
+    categories:
+    - elastic
+    kind: AutoOpsAgentPolicy
+    listKind: AutoOpsAgentPolicyList
+    plural: autoopsagentpolicies
+    shortNames:
+    - aop
+    singular: autoopsagentpolicy
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Ready resources
+      jsonPath: .status.ready
+      name: Ready
+      type: string
+    - jsonPath: .status.phase
+      name: Phase
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: AutoOpsAgentPolicy represents an Elastic AutoOps Policy resource
+          in a Kubernetes cluster.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              autoOpsRef:
+                description: AutoOpsRef defines a reference to a secret containing
+                  connection details for AutoOps via Cloud Connect.
+                properties:
+                  secretName:
+                    description: |-
+                      SecretName references a Secret containing connection details for external AutoOps.
+                      Required when connecting via Cloud Connect. The secret must contain:
+                      - `cloud-connected-mode-api-key`: Cloud Connected Mode API key
+                      - `autoops-otel-url`: AutoOps OpenTelemetry endpoint URL
+                      - `autoops-token`: AutoOps authentication token
+                      - `cloud-connected-mode-api-url`: (optional) Cloud Connected Mode API URL
+                      This field cannot be used in combination with `name`.
+                    type: string
+                type: object
+              image:
+                description: Image is the AutoOps Agent Docker image to deploy.
+                type: string
+              podTemplate:
+                description: PodTemplate provides customisation options (labels, annotations,
+                  affinity rules, resource requests, and so on) for the Agent pods
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
+              resourceSelector:
+                description: |-
+                  ResourceSelector is a label selector for the resources to be configured.
+                  Any Elasticsearch instances that match the selector will be configured to send data to AutoOps.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              revisionHistoryLimit:
+                description: RevisionHistoryLimit is the number of revisions to retain
+                  to allow rollback in the underlying Deployment.
+                format: int32
+                type: integer
+              serviceAccountName:
+                description: |-
+                  ServiceAccountName is used to check access to Elasticsearch resources in different namespaces.
+                  Can only be used if ECK is enforcing RBAC on references (--enforce-rbac-on-refs flag).
+                  The service account must have "get" permission on elasticsearch.k8s.elastic.co/elasticsearches
+                  in the target namespaces.
+                type: string
+              version:
+                description: Version of the AutoOpsAgentPolicy.
+                type: string
+            required:
+            - version
+            type: object
+          status:
+            properties:
+              errors:
+                description: Errors is the number of resources that are in an error
+                  state.
+                type: integer
+              observedGeneration:
+                description: ObservedGeneration is the most recent generation observed
+                  for this AutoOpsAgentPolicy.
+                format: int64
+                type: integer
+              phase:
+                description: Phase is the phase of the AutoOpsAgentPolicy.
+                type: string
+              ready:
+                description: Ready is the number of resources that are in a ready
+                  state.
+                type: integer
+              resources:
+                description: Resources is the number of resources that match the ResourceSelector.
+                type: integer
+            required:
+            - errors
+            - ready
+            - resources
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.19.0

config/crds/v1/patches/autoopsagentpolicy-patches.yaml

@@ -0,0 +1,8 @@
+# Using `kubectl apply` stores the complete CRD file as an annotation,
+# which may be too big for the annotations size limit.
+# One way to mitigate this problem is to remove the (huge) podTemplate properties from the CRD.
+# It also avoids the problem of having any k8s-version specific field in the Pod schema,
+# that would maybe not match the user's k8s version.
+- op: remove
+  path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/podTemplate/properties
+

config/crds/v1/patches/kustomization.yaml

@@ -70,13 +70,20 @@ patches:
       kind: CustomResourceDefinition
       name: elasticmapsservers.maps.k8s.elastic.co
     path: maps-patches.yaml
-    # custom patches for Logstash
+  # custom patches for Logstash
   - target:
       group: apiextensions.k8s.io
       version: v1
       kind: CustomResourceDefinition
       name: logstashes.logstash.k8s.elastic.co
     path: logstash-patches.yaml
+  # custom patches for AutoOpsAgentPolicy
+  - target:
+      group: apiextensions.k8s.io
+      version: v1
+      kind: CustomResourceDefinition
+      name: autoopsagentpolicies.autoops.k8s.elastic.co
+    path: autoopsagentpolicy-patches.yaml
   # custom patches for PackageRegistry
   - target:
       group: apiextensions.k8s.io