Cherry-pick #14330 to 7.x: [Metricbeat] Add Kafka JMX metricsets

CHANGELOG.next.asciidoc

@@ -479,6 +479,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221]
 - Add support for Application ELB and Network ELB. {pull}14123[14123] {issue}13538[13538] {issue}13539[13539]
 - Release aws ebs metricset as GA. {pull}14312[14312] {issue}14060[14060]
+- Add `connection.state` field for RabbitMQ module. {pull}13981[13981]
+- Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
+- Add Kafka JMX metricsets. {pull}14330[14330]
 - Add metrics to envoyproxy server metricset and support for envoy proxy 1.12. {pull}14416[14416] {issue}13642[13642]
 
 *Packetbeat*

metricbeat/docker-compose.yml

@@ -129,7 +129,7 @@ services:
 
   haproxy:
     image: docker.elastic.co/observability-ci/beats-integration-haproxy:${HAPROXY_VERSION:-1.8}-1
-    build: 
+    build:
       context: ./module/haproxy/_meta
       args:
         HAPROXY_VERSION: ${HAPROXY_VERSION:-1.8}
@@ -155,13 +155,16 @@ services:
       - 8778
 
   kafka:
-    image: docker.elastic.co/observability-ci/beats-integration-kafka:${KAFKA_VERSION:-2.1.1}-1
+    image: docker.elastic.co/observability-ci/beats-integration-kafka:${KAFKA_VERSION:-2.1.1}-2
     build:
       context: ./module/kafka/_meta
       args:
         KAFKA_VERSION: ${KAFKA_VERSION:-2.1.1}
     ports:
       - 9092
+      - 8779
+      - 8775
+      - 8774
 
   kibana:
     image: docker.elastic.co/observability-ci/beats-integration-kibana:${KIBANA_VERSION:-7.4.0}-1
@@ -262,7 +265,7 @@ services:
 
   nginx:
     image: docker.elastic.co/observability-ci/beats-integration-nginx:${NGINX_VERSION:-1.9}-1
-    build: 
+    build:
       context: ./module/nginx/_meta
       args:
         NGINX_VERSION: ${NGINX_VERSION:-1.9}
@@ -271,7 +274,7 @@ services:
 
   phpfpm:
     image: docker.elastic.co/observability-ci/beats-integration-phpfpm:${PHPFPM_VERSION:-7.1}-1
-    build: 
+    build:
       context: ./module/php_fpm/_meta
       args:
         PHPFPM_VERSION: ${PHPFPM_VERSION:-7.1}

metricbeat/docs/fields.asciidoc

@@ -14965,6 +14965,198 @@ type: keyword
 
 --
 
+[float]
+=== broker
+
+Broker metrics from Kafka Broker JMX
+
+
+*`kafka.broker.mbean`*::
++
+--
+Mbean that this event is related to
+
+type: keyword
+
+--
+
+*`kafka.broker.request.channel.queue.size`*::
++
+--
+The size of the request queue
+
+type: long
+
+--
+
+*`kafka.broker.request.produce.failed_per_second`*::
++
+--
+The rate of failed produce requests per second
+
+type: float
+
+--
+
+*`kafka.broker.request.fetch.failed_per_second`*::
++
+--
+The rate of client fetch request failures per second
+
+type: float
+
+--
+
+*`kafka.broker.replication.leader_elections`*::
++
+--
+The leader election rate
+
+type: float
+
+--
+
+*`kafka.broker.replication.unclean_leader_elections`*::
++
+--
+The unclean leader election rate
+
+type: float
+
+--
+
+*`kafka.broker.session.zookeeper.disconnect`*::
++
+--
+The ZooKeeper closed sessions per second
+
+type: float
+
+--
+
+*`kafka.broker.session.zookeeper.expire`*::
++
+--
+The ZooKeeper expired sessions per second
+
+type: float
+
+--
+
+*`kafka.broker.session.zookeeper.readonly`*::
++
+--
+The ZooKeeper readonly sessions per second
+
+type: float
+
+--
+
+*`kafka.broker.session.zookeeper.sync`*::
++
+--
+The ZooKeeper client connections per second
+
+type: float
+
+--
+
+*`kafka.broker.log.flush_rate`*::
++
+--
+The log flush rate
+
+type: float
+
+--
+
+*`kafka.broker.topic.net.bytes_in`*::
++
+--
+The incoming byte rate
+
+type: float
+
+--
+
+*`kafka.broker.topic.net.bytes_out`*::
++
+--
+The outgoing byte rate
+
+type: float
+
+--
+
+*`kafka.broker.topic.net.bytes_rejected`*::
++
+--
+The rejected byte rate
+
+type: float
+
+--
+
+*`kafka.broker.topic.messages_in`*::
++
+--
+The incoming message rate
+
+type: float
+
+--
+
+[float]
+=== consumer
+
+Consumer metrics from Kafka Consumer JMX
+
+
+*`kafka.consumer.mbean`*::
++
+--
+Mbean that this event is related to
+
+type: keyword
+
+--
+
+*`kafka.consumer.fetch_rate`*::
++
+--
+The minimum rate at which the consumer sends fetch requests to a broker
+
+type: float
+
+--
+
+*`kafka.consumer.bytes_consumed`*::
++
+--
+The average number of bytes consumed for a specific topic per second
+
+type: float
+
+--
+
+*`kafka.consumer.records_consumed`*::
++
+--
+The average number of records consumed per second for a specific topic
+
+type: float
+
+--
+
+*`kafka.consumer.bytes_in`*::
++
+--
+The rate of bytes coming in to the consumer
+
+type: float
+
+--
+
 [float]
 === consumergroup
 
@@ -15260,6 +15452,138 @@ type: keyword
 
 --
 
+[float]
+=== producer
+
+Producer metrics from Kafka Producer JMX
+
+
+*`kafka.producer.mbean`*::
++
+--
+Mbean that this event is related to
+
+type: keyword
+
+--
+
+*`kafka.producer.available_buffer_bytes`*::
++
+--
+The total amount of buffer memory
+
+type: float
+
+--
+
+*`kafka.producer.batch_size_avg`*::
++
+--
+The average number of bytes sent
+
+type: float
+
+--
+
+*`kafka.producer.batch_size_max`*::
++
+--
+The maximum number of bytes sent
+
+type: long
+
+--
+
+*`kafka.producer.record_send_rate`*::
++
+--
+The average number of records sent per second
+
+type: float
+
+--
+
+*`kafka.producer.record_retry_rate`*::
++
+--
+The average number of retried record sends per second
+
+type: float
+
+--
+
+*`kafka.producer.record_error_rate`*::
++
+--
+The average number of retried record sends per second
+
+type: float
+
+--
+
+*`kafka.producer.records_per_request`*::
++
+--
+The average number of records sent per second
+
+type: float
+
+--
+
+*`kafka.producer.record_size_avg`*::
++
+--
+The average record size
+
+type: float
+
+--
+
+*`kafka.producer.record_size_max`*::
++
+--
+The maximum record size
+
+type: long
+
+--
+
+*`kafka.producer.request_rate`*::
++
+--
+The number of producer requests per second
+
+type: float
+
+--
+
+*`kafka.producer.response_rate`*::
++
+--
+The number of producer responses per second
+
+type: float
+
+--
+
+*`kafka.producer.io_wait`*::
++
+--
+The producer I/O wait time
+
+type: float
+
+--
+
+*`kafka.producer.bytes_out`*::
++
+--
+The rate of bytes going out for the producer
+
+type: float
+
+--
+
 [[exported-fields-kibana]]
 == Kibana fields