Stack monitoring for Logstash

cmd/manager/main.go

@@ -879,6 +879,7 @@ func registerControllers(mgr manager.Manager, params operator.Parameters, access
 		{name: "ES-MONITORING", registerFunc: associationctl.AddEsMonitoring},
 		{name: "KB-MONITORING", registerFunc: associationctl.AddKbMonitoring},
 		{name: "BEAT-MONITORING", registerFunc: associationctl.AddBeatMonitoring},
+		{name: "LOGSTASH-MONITORING", registerFunc: associationctl.AddLogstashMonitoring},
 	}
 
 	for _, c := range assocControllers {
@@ -917,6 +918,7 @@ func garbageCollectUsers(ctx context.Context, cfg *rest.Config, managedNamespace
 		For(&beatv1beta1.BeatList{}, associationctl.BeatAssociationLabelNamespace, associationctl.BeatAssociationLabelName).
 		For(&agentv1alpha1.AgentList{}, associationctl.AgentAssociationLabelNamespace, associationctl.AgentAssociationLabelName).
 		For(&emsv1alpha1.ElasticMapsServerList{}, associationctl.MapsESAssociationLabelNamespace, associationctl.MapsESAssociationLabelName).
+		For(&logstashv1alpha1.LogstashList{}, associationctl.LogstashAssociationLabelNamespace, associationctl.LogstashAssociationLabelName).
 		DoGarbageCollection(ctx)
 	if err != nil {
 		return fmt.Errorf("user garbage collector failed: %w", err)

config/crds/v1/all-crds.yaml

@@ -9121,6 +9121,108 @@ spec:
                 description: Image is the Logstash Docker image to deploy. Version
                   and Type have to match the Logstash in the image.
                 type: string
+              monitoring:
+                description: Monitoring enables you to collect and ship log and monitoring
+                  data of this Logstash. Metricbeat and Filebeat are deployed in the
+                  same Pod as sidecars and each one sends data to one or two different
+                  Elasticsearch monitoring clusters running in the same Kubernetes
+                  cluster.
+                properties:
+                  logs:
+                    description: Logs holds references to Elasticsearch clusters which
+                      receive log data from an associated resource.
+                    properties:
+                      elasticsearchRefs:
+                        description: ElasticsearchRefs is a reference to a list of
+                          monitoring Elasticsearch clusters running in the same Kubernetes
+                          cluster. Due to existing limitations, only a single Elasticsearch
+                          cluster is currently supported.
+                        items:
+                          description: ObjectSelector defines a reference to a Kubernetes
+                            object which can be an Elastic resource managed by the
+                            operator or a Secret describing an external Elastic resource
+                            not managed by the operator.
+                          properties:
+                            name:
+                              description: Name of an existing Kubernetes object corresponding
+                                to an Elastic resource managed by ECK.
+                              type: string
+                            namespace:
+                              description: Namespace of the Kubernetes object. If
+                                empty, defaults to the current namespace.
+                              type: string
+                            secretName:
+                              description: 'SecretName is the name of an existing
+                                Kubernetes secret that contains connection information
+                                for associating an Elastic resource not managed by
+                                the operator. The referenced secret must contain the
+                                following: - `url`: the URL to reach the Elastic resource
+                                - `username`: the username of the user to be authenticated
+                                to the Elastic resource - `password`: the password
+                                of the user to be authenticated to the Elastic resource
+                                - `ca.crt`: the CA certificate in PEM format (optional).
+                                This field cannot be used in combination with the
+                                other fields name, namespace or serviceName.'
+                              type: string
+                            serviceName:
+                              description: ServiceName is the name of an existing
+                                Kubernetes service which is used to make requests
+                                to the referenced object. It has to be in the same
+                                namespace as the referenced resource. If left empty,
+                                the default HTTP service of the referenced resource
+                                is used.
+                              type: string
+                          type: object
+                        type: array
+                    type: object
+                  metrics:
+                    description: Metrics holds references to Elasticsearch clusters
+                      which receive monitoring data from this resource.
+                    properties:
+                      elasticsearchRefs:
+                        description: ElasticsearchRefs is a reference to a list of
+                          monitoring Elasticsearch clusters running in the same Kubernetes
+                          cluster. Due to existing limitations, only a single Elasticsearch
+                          cluster is currently supported.
+                        items:
+                          description: ObjectSelector defines a reference to a Kubernetes
+                            object which can be an Elastic resource managed by the
+                            operator or a Secret describing an external Elastic resource
+                            not managed by the operator.
+                          properties:
+                            name:
+                              description: Name of an existing Kubernetes object corresponding
+                                to an Elastic resource managed by ECK.
+                              type: string
+                            namespace:
+                              description: Namespace of the Kubernetes object. If
+                                empty, defaults to the current namespace.
+                              type: string
+                            secretName:
+                              description: 'SecretName is the name of an existing
+                                Kubernetes secret that contains connection information
+                                for associating an Elastic resource not managed by
+                                the operator. The referenced secret must contain the
+                                following: - `url`: the URL to reach the Elastic resource
+                                - `username`: the username of the user to be authenticated
+                                to the Elastic resource - `password`: the password
+                                of the user to be authenticated to the Elastic resource
+                                - `ca.crt`: the CA certificate in PEM format (optional).
+                                This field cannot be used in combination with the
+                                other fields name, namespace or serviceName.'
+                              type: string
+                            serviceName:
+                              description: ServiceName is the name of an existing
+                                Kubernetes service which is used to make requests
+                                to the referenced object. It has to be in the same
+                                namespace as the referenced resource. If left empty,
+                                the default HTTP service of the referenced resource
+                                is used.
+                              type: string
+                          type: object
+                        type: array
+                    type: object
+                type: object
               podTemplate:
                 description: PodTemplate provides customisation options for the Logstash
                   pods.
@@ -9629,6 +9731,13 @@ spec:
               expectedNodes:
                 format: int32
                 type: integer
+              monitoringAssociationStatus:
+                additionalProperties:
+                  description: AssociationStatus is the status of an association resource.
+                  type: string
+                description: MonitoringAssociationStatus is the status of any auto-linking
+                  to monitoring Elasticsearch clusters.
+                type: object
               observedGeneration:
                 description: ObservedGeneration is the most recent generation observed
                   for this Logstash instance. It corresponds to the metadata generation,

config/crds/v1/bases/logstash.k8s.elastic.co_logstashes.yaml

@@ -77,6 +77,108 @@ spec:
                 description: Image is the Logstash Docker image to deploy. Version
                   and Type have to match the Logstash in the image.
                 type: string
+              monitoring:
+                description: Monitoring enables you to collect and ship log and monitoring
+                  data of this Logstash. Metricbeat and Filebeat are deployed in the
+                  same Pod as sidecars and each one sends data to one or two different
+                  Elasticsearch monitoring clusters running in the same Kubernetes
+                  cluster.
+                properties:
+                  logs:
+                    description: Logs holds references to Elasticsearch clusters which
+                      receive log data from an associated resource.
+                    properties:
+                      elasticsearchRefs:
+                        description: ElasticsearchRefs is a reference to a list of
+                          monitoring Elasticsearch clusters running in the same Kubernetes
+                          cluster. Due to existing limitations, only a single Elasticsearch
+                          cluster is currently supported.
+                        items:
+                          description: ObjectSelector defines a reference to a Kubernetes
+                            object which can be an Elastic resource managed by the
+                            operator or a Secret describing an external Elastic resource
+                            not managed by the operator.
+                          properties:
+                            name:
+                              description: Name of an existing Kubernetes object corresponding
+                                to an Elastic resource managed by ECK.
+                              type: string
+                            namespace:
+                              description: Namespace of the Kubernetes object. If
+                                empty, defaults to the current namespace.
+                              type: string
+                            secretName:
+                              description: 'SecretName is the name of an existing
+                                Kubernetes secret that contains connection information
+                                for associating an Elastic resource not managed by
+                                the operator. The referenced secret must contain the
+                                following: - `url`: the URL to reach the Elastic resource
+                                - `username`: the username of the user to be authenticated
+                                to the Elastic resource - `password`: the password
+                                of the user to be authenticated to the Elastic resource
+                                - `ca.crt`: the CA certificate in PEM format (optional).
+                                This field cannot be used in combination with the
+                                other fields name, namespace or serviceName.'
+                              type: string
+                            serviceName:
+                              description: ServiceName is the name of an existing
+                                Kubernetes service which is used to make requests
+                                to the referenced object. It has to be in the same
+                                namespace as the referenced resource. If left empty,
+                                the default HTTP service of the referenced resource
+                                is used.
+                              type: string
+                          type: object
+                        type: array
+                    type: object
+                  metrics:
+                    description: Metrics holds references to Elasticsearch clusters
+                      which receive monitoring data from this resource.
+                    properties:
+                      elasticsearchRefs:
+                        description: ElasticsearchRefs is a reference to a list of
+                          monitoring Elasticsearch clusters running in the same Kubernetes
+                          cluster. Due to existing limitations, only a single Elasticsearch
+                          cluster is currently supported.
+                        items:
+                          description: ObjectSelector defines a reference to a Kubernetes
+                            object which can be an Elastic resource managed by the
+                            operator or a Secret describing an external Elastic resource
+                            not managed by the operator.
+                          properties:
+                            name:
+                              description: Name of an existing Kubernetes object corresponding
+                                to an Elastic resource managed by ECK.
+                              type: string
+                            namespace:
+                              description: Namespace of the Kubernetes object. If
+                                empty, defaults to the current namespace.
+                              type: string
+                            secretName:
+                              description: 'SecretName is the name of an existing
+                                Kubernetes secret that contains connection information
+                                for associating an Elastic resource not managed by
+                                the operator. The referenced secret must contain the
+                                following: - `url`: the URL to reach the Elastic resource
+                                - `username`: the username of the user to be authenticated
+                                to the Elastic resource - `password`: the password
+                                of the user to be authenticated to the Elastic resource
+                                - `ca.crt`: the CA certificate in PEM format (optional).
+                                This field cannot be used in combination with the
+                                other fields name, namespace or serviceName.'
+                              type: string
+                            serviceName:
+                              description: ServiceName is the name of an existing
+                                Kubernetes service which is used to make requests
+                                to the referenced object. It has to be in the same
+                                namespace as the referenced resource. If left empty,
+                                the default HTTP service of the referenced resource
+                                is used.
+                              type: string
+                          type: object
+                        type: array
+                    type: object
+                type: object
               podTemplate:
                 description: PodTemplate provides customisation options for the Logstash
                   pods.
@@ -7992,6 +8094,13 @@ spec:
               expectedNodes:
                 format: int32
                 type: integer
+              monitoringAssociationStatus:
+                additionalProperties:
+                  description: AssociationStatus is the status of an association resource.
+                  type: string
+                description: MonitoringAssociationStatus is the status of any auto-linking
+                  to monitoring Elasticsearch clusters.
+                type: object
               observedGeneration:
                 description: ObservedGeneration is the most recent generation observed
                   for this Logstash instance. It corresponds to the metadata generation,

config/samples/logstash/logstash_stackmonitor.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: monitoring
+spec:
+  version: 8.6.1
+  nodeSets:
+    - name: default
+      count: 1
+      config:
+        node.store.allow_mmap: false
+---
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash-sample
+spec:
+  count: 1
+  version: 8.7.0
+  config:
+    log.level: info
+    api.http.host: "0.0.0.0"
+    queue.type: memory
+  podTemplate:
+    spec:
+      containers:
+        - name: logstash
+  monitoring:
+    metrics:
+      elasticsearchRefs:
+        - name: monitoring
+    logs:
+      elasticsearchRefs:
+        - name: monitoring
+---
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana-sample
+spec:
+  version: 8.6.1
+  elasticsearchRef:
+    name: monitoring
+  count: 1
+---