Jolokia Module with dynamic JMX Metricset

CHANGELOG.asciidoc

@@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v5.1.2...v5.2.0[View commits]
 - Experimental Prometheus module. {pull}3202[3202]
 - Add system socket module that reports all TCP sockets. {pull}3246[3246]
 - Kafka consumer groups metricset. {pull}3240[3240]
+- Add jolokia module with dynamic jmx metricset. {pull}3570[3570]
 
 *Winlogbeat*
 

libbeat/scripts/generate_index_pattern.py

@@ -19,6 +19,10 @@
 
 def fields_to_json(section, path, output):
 
+    # Need in case there are no fields
+    if section["fields"] is None:
+        section["fields"] = {}
+
     for field in section["fields"]:
         if path == "":
             newpath = field["name"]

libbeat/scripts/generate_template.py

@@ -135,6 +135,10 @@ def dedot(group):
     fields = []
     dedotted = {}
 
+    # Need in case there are no fields
+    if group["fields"] is None:
+        group["fields"] = {}
+
     for field in group["fields"]:
         if "." in field["name"]:
             # dedot

libbeat/tests/system/beat/beat.py

@@ -390,6 +390,10 @@ def load_fields(self, fields_doc="../../_meta/fields.generated.yml"):
         def extract_fields(doc_list, name):
             fields = []
             dictfields = []
+
+            if doc_list is None:
+                return fields, dictfields
+
             for field in doc_list:
 
                 # Chain together names

metricbeat/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       - ceph
       - couchbase
       - haproxy
+      - jolokia
       - kafka
       - mongodb
       - mysql
@@ -33,6 +34,7 @@ services:
       - ${PWD}/module/ceph/_meta/env
       - ${PWD}/module/couchbase/_meta/env
       - ${PWD}/module/haproxy/_meta/env
+      - ${PWD}/module/jolokia/_meta/env
       - ${PWD}/module/kafka/_meta/env
       - ${PWD}/module/mongodb/_meta/env
       - ${PWD}/module/mysql/_meta/env
@@ -56,6 +58,9 @@ services:
   haproxy:
     build: ${PWD}/module/haproxy/_meta
 
+  jolokia:
+    build: ${PWD}/module/jolokia/_meta
+
   kafka:
     build: ${PWD}/module/kafka/_meta
 

metricbeat/docs/fields.asciidoc

@@ -20,6 +20,7 @@ grouped in the following categories:
 * <<exported-fields-couchbase>>
 * <<exported-fields-docker>>
 * <<exported-fields-haproxy>>
+* <<exported-fields-jolokia>>
 * <<exported-fields-kafka>>
 * <<exported-fields-mongodb>>
 * <<exported-fields-mysql>>
@@ -2804,6 +2805,20 @@ type: integer
 The average queue time in ms over the last 1024 requests.
 
 
+[[exported-fields-jolokia]]
+== Jolokia Fields
+
+[]beta
+Jolokia Module
+
+
+
+[float]
+== jolokia Fields
+
+jolokia contains metrics exposed via jolokia agent
+
+
 [[exported-fields-kafka]]
 == kafka Fields
 

metricbeat/docs/modules/jolokia.asciidoc

@@ -0,0 +1,43 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-jolokia]]
+== Jolokia Module
+
+beta[]
+
+This is the Jolokia Module.
+
+
+
+[float]
+=== Example Configuration
+
+The Jolokia module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+#- module: jolokia
+#  metricsets: ["jmx"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost"]
+#  namespace: "metrics"
+#  path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+#  jmx.mapping:
+#  jmx.application:
+#  jmx.instance:
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-jolokia-jmx,jmx>>
+
+include::jolokia/jmx.asciidoc[]
+

metricbeat/docs/modules/jolokia/jmx.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-jolokia-jmx]]
+include::../../../module/jolokia/jmx/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-jolokia,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/jolokia/jmx/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -7,6 +7,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-couchbase,Couchbase>>
   * <<metricbeat-module-docker,Docker>>
   * <<metricbeat-module-haproxy,HAProxy>>
+  * <<metricbeat-module-jolokia,Jolokia>>
   * <<metricbeat-module-kafka,kafka>>
   * <<metricbeat-module-mongodb,MongoDB>>
   * <<metricbeat-module-mysql,MySQL>>
@@ -26,6 +27,7 @@ include::modules/ceph.asciidoc[]
 include::modules/couchbase.asciidoc[]
 include::modules/docker.asciidoc[]
 include::modules/haproxy.asciidoc[]
+include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]
 include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]

metricbeat/helper/http.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 
@@ -15,6 +16,8 @@ type HTTP struct {
 	base    mb.BaseMetricSet
 	client  *http.Client // HTTP client that is reused across requests.
 	headers map[string]string
+	method  string
+	body    []byte
 }
 
 // NewHTTP creates new http helper
@@ -23,14 +26,23 @@ func NewHTTP(base mb.BaseMetricSet) *HTTP {
 		base:    base,
 		client:  &http.Client{Timeout: base.Module().Config().Timeout},
 		headers: map[string]string{},
+		method:  "GET",
+		body:    nil,
 	}
 }
 
 // FetchResponse fetches a response for the http metricset.
 // It's important that resp.Body has to be closed if this method is used. Before using this method
 // check if one of the other Fetch* methods could be used as they ensure that the Body is properly closed.
 func (h *HTTP) FetchResponse() (*http.Response, error) {
-	req, err := http.NewRequest("GET", h.base.HostData().SanitizedURI, nil)
+
+	// Create a fresh reader every time
+	var reader io.Reader
+	if h.body != nil {
+		reader = bytes.NewReader(h.body)
+	}
+
+	req, err := http.NewRequest(h.method, h.base.HostData().SanitizedURI, reader)
 	if h.base.HostData().User != "" || h.base.HostData().Password != "" {
 		req.SetBasicAuth(h.base.HostData().User, h.base.HostData().Password)
 	}
@@ -51,6 +63,14 @@ func (h *HTTP) SetHeader(key, value string) {
 	h.headers[key] = value
 }
 
+func (h *HTTP) SetMethod(method string) {
+	h.method = method
+}
+
+func (h *HTTP) SetBody(body []byte) {
+	h.body = body
+}
+
 // FetchContent makes an HTTP request to the configured url and returns the body content.
 func (h *HTTP) FetchContent() ([]byte, error) {
 	resp, err := h.FetchResponse()

metricbeat/include/list.go

@@ -31,6 +31,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/haproxy"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/info"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/stat"
+	_ "github.com/elastic/beats/metricbeat/module/jolokia"
+	_ "github.com/elastic/beats/metricbeat/module/jolokia/jmx"
 	_ "github.com/elastic/beats/metricbeat/module/kafka"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/partition"

metricbeat/metricbeat.full.yml

@@ -128,6 +128,18 @@ metricbeat.modules:
   #period: 10s
   #hosts: ["tcp://127.0.0.1:14567"]
 
+#------------------------------- Jolokia Module ------------------------------
+#- module: jolokia
+#  metricsets: ["jmx"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost"]
+#  namespace: "metrics"
+#  path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+#  jmx.mapping:
+#  jmx.application:
+#  jmx.instance:
+
 #-------------------------------- kafka Module -------------------------------
 #- module: kafka
   #metricsets: ["partition"]

metricbeat/module/jolokia/_meta/Dockerfile

@@ -0,0 +1,10 @@
+# Tomcat is started to fetch Jolokia metrics from it
+FROM jolokia/java-jolokia:7
+ENV TOMCAT_VERSION 7.0.55
+ENV TC apache-tomcat-${TOMCAT_VERSION}
+
+EXPOSE 8778
+RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz
+RUN tar xzf ${TC}.tar.gz -C /opt
+
+CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run

metricbeat/module/jolokia/_meta/config.yml

@@ -0,0 +1,10 @@
+#- module: jolokia
+#  metricsets: ["jmx"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost"]
+#  namespace: "metrics"
+#  path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+#  jmx.mapping:
+#  jmx.application:
+#  jmx.instance:

metricbeat/module/jolokia/_meta/docs.asciidoc

@@ -0,0 +1,6 @@
+== Jolokia Module
+
+beta[]
+
+This is the Jolokia Module.
+

metricbeat/module/jolokia/_meta/env

@@ -0,0 +1,2 @@
+JOLOKIA_HOST=jolokia
+JOLOKIA_PORT=8778

metricbeat/module/jolokia/_meta/fields.yml

@@ -0,0 +1,13 @@
+- key: jolokia
+  title: "Jolokia"
+  description: >
+    []beta
+
+    Jolokia Module
+  short_config: false
+  fields:
+    - name: jolokia
+      type: group
+      description: >
+          jolokia contains metrics exposed via jolokia agent
+      fields:

metricbeat/module/jolokia/doc.go

@@ -0,0 +1,4 @@
+/*
+Package jolokia is a Metricbeat module that contains MetricSets.
+*/
+package jolokia

metricbeat/module/jolokia/jmx/_meta/data.json

@@ -0,0 +1,34 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "jolokia": {
+        "testnamespace": {
+            "memory": {
+                "heap_usage": {
+                    "committed": 1.09051904e+08,
+                    "init": 3.2753408e+07,
+                    "max": 6.20756992e+08,
+                    "used": 5.8796168e+07
+                },
+                "non_heap_usage": {
+                    "committed": 3.244032e+07,
+                    "init": 2.4576e+07,
+                    "max": 2.24395264e+08,
+                    "used": 1.7975176e+07
+                }
+            },
+            "uptime": 6.1802139e+07
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:8778",
+        "module": "jolokia",
+        "name": "jmx",
+        "namespace": "testnamespace",
+        "rtt": 115
+    },
+    "type": "metricsets"
+}

metricbeat/module/jolokia/jmx/_meta/docs.asciidoc

@@ -0,0 +1,49 @@
+=== jolokia jmx MetricSet
+
+This is the jmx metricset of the module jolokia.
+
+[float]
+=== Features and configuration
+Tested with Jolokia 1.3.4.
+
+Metrics to be collected from each Jolokia instance are defined in the mapping section with an MBean ObjectName and
+an array of Attributes to be requested with Elastic field names under which the return values should be saved.
+
+For example: to get the "Uptime" attribute from the "java.lang:type=Runtime" MBean and map it to something like
+"uptime" (actually "jolokia.jmx.uptime", the prexif is added by beats framework) you have to configure following
+mapping:
+
+```
+- module: jolokia
+  metricsets: ["jmx"]
+  hosts: ["localhost:8778"]
+  namespace: "metrics"
+  jmx.mappings:
+    - mbean: 'java.lang:type=Runtime'
+      attributes:
+        - attr: Uptime
+          field: uptime
+```
+
+In case the underlying attribute is an object (e.g. see HeapMemoryUsage attribute in java.lang:type=Memory) it`s
+structure will be published to Elastic "as is".
+
+It is possible to configure nested metric aliases by using dots in the mapping name (e.g. gc.cms_collection_time). For examples please refer to the
+/jolokia/jmx/test/config.yml.
+
+All metrics from a single mapping will be POSTed to the defined host/port and sent to Elastic as a single event.
+To make it possible to differentiate between metrics from multiple similar applications running on the same host,
+please configure multiple modules.
+
+It is required to set a namespace in the general module config section.
+
+[float]
+=== Limitations
+No authentication against Jolokia is supported yet. No wildcards in Jolokia requests supported yet.
+All Jolokia requests have canonicalNaming set to false (details see here: https://jolokia.org/reference/html/protocol.html).
+
+
+[float]
+=== Exposed fields, Dashboards, Indexes, etc.
+Since this is a very general module that can be tailored for any application that exposes it's metrics over Jolokia, it
+comes with no exposed fields description, dashboards or index patterns.