New module http for Metricbeat with json metricset (#4156)

CHANGELOG.asciidoc

@@ -151,6 +151,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add dropwizard module {pull}4022[4022]
 - Adding query APIs for metricsets and modules from metricbeat registry {pull}4102[4102]
 - Fixing nil pointer on prometheus collector when http response is nil {pull}4119[4119]
+- Add http module with json metricset. {pull}4092[4092]
 
 *Packetbeat*
 - Add `fields` and `fields_under_root` to packetbeat protocols configurations. {pull}3518[3518]

metricbeat/docs/fields.asciidoc

@@ -23,6 +23,7 @@ grouped in the following categories:
 * <<exported-fields-elasticsearch>>
 * <<exported-fields-golang>>
 * <<exported-fields-haproxy>>
+* <<exported-fields-http>>
 * <<exported-fields-jolokia>>
 * <<exported-fields-kafka>>
 * <<exported-fields-kibana>>
@@ -3444,6 +3445,96 @@ type: integer
 The average queue time in ms over the last 1024 requests.
 
 
+[[exported-fields-http]]
+== http Fields
+
+http Module
+
+
+
+[float]
+== http Fields
+
+
+
+
+[float]
+== request Fields
+
+HTTP request information
+
+
+
+[float]
+=== http.request.header
+
+type: nested
+
+The HTTP headers sent
+
+
+[float]
+=== http.request.method
+
+type: keyword
+
+The HTTP method used
+
+
+[float]
+=== http.request.body
+
+type: keyword
+
+The HTTP payload sent
+
+
+[float]
+== response Fields
+
+HTTP response information
+
+
+
+[float]
+=== http.response.header
+
+type: nested
+
+The HTTP headers received
+
+
+[float]
+=== http.response.status_code
+
+type: keyword
+
+The HTTP status code
+
+
+[float]
+=== http.response.body
+
+type: keyword
+
+The HTTP payload received
+
+
+[float]
+== json Fields
+
+json metricset
+
+
+
+[float]
+=== http.json.body
+
+type: keyword
+
+The HTTP payload received
+
+
 [[exported-fields-jolokia]]
 == Jolokia Fields
 

metricbeat/docs/modules/http.asciidoc

@@ -0,0 +1,41 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-http]]
+== http Module
+
+Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call arbitrary HTTP endpoints for which not a dedicated metricbeat is available.
+Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel.
+
+Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch.
+This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored.
+
+
+[float]
+=== Example Configuration
+
+The http module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: http
+  metricsets: ["json"]
+  enabled: false
+  period: 10s
+  hosts: ["httpbin.org"]
+  namespace: "http_json_namespace"
+  path: "/headers"
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-http-json,json>>
+
+include::http/json.asciidoc[]
+

metricbeat/docs/modules/http/json.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-http-json]]
+include::../../../module/http/json/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-http,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/http/json/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-elasticsearch,elasticsearch>>
   * <<metricbeat-module-golang,golang>>
   * <<metricbeat-module-haproxy,HAProxy>>
+  * <<metricbeat-module-http,http>>
   * <<metricbeat-module-jolokia,Jolokia>>
   * <<metricbeat-module-kafka,kafka>>
   * <<metricbeat-module-kibana,kibana>>
@@ -37,6 +38,7 @@ include::modules/dropwizard.asciidoc[]
 include::modules/elasticsearch.asciidoc[]
 include::modules/golang.asciidoc[]
 include::modules/haproxy.asciidoc[]
+include::modules/http.asciidoc[]
 include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]
 include::modules/kibana.asciidoc[]

metricbeat/include/list.go

@@ -39,6 +39,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/haproxy"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/info"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/stat"
+	_ "github.com/elastic/beats/metricbeat/module/http"
+	_ "github.com/elastic/beats/metricbeat/module/http/json"
 	_ "github.com/elastic/beats/metricbeat/module/jolokia"
 	_ "github.com/elastic/beats/metricbeat/module/jolokia/jmx"
 	_ "github.com/elastic/beats/metricbeat/module/kafka"

metricbeat/metricbeat.full.yml

@@ -160,6 +160,15 @@ metricbeat.modules:
   period: 10s
   hosts: ["tcp://127.0.0.1:14567"]
 
+#-------------------------------- http Module --------------------------------
+- module: http
+  metricsets: ["json"]
+  enabled: false
+  period: 10s
+  hosts: ["httpbin.org"]
+  namespace: "http_json_namespace"
+  path: "/headers"
+
 #------------------------------- Jolokia Module ------------------------------
 - module: jolokia
   metricsets: ["jmx"]

metricbeat/module/http/_meta/Dockerfile

@@ -0,0 +1,11 @@
+# Tomcat is started to fetch Jolokia metrics from it
+FROM jolokia/java-jolokia:7
+ENV TOMCAT_VERSION 7.0.55
+ENV TC apache-tomcat-${TOMCAT_VERSION}
+
+HEALTHCHECK CMD curl -f curl localhost:8778/jolokia/
+EXPOSE 8778
+RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz
+RUN tar xzf ${TC}.tar.gz -C /opt
+
+CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run

metricbeat/module/http/_meta/config.yml

@@ -0,0 +1,7 @@
+- module: http
+  metricsets: ["json"]
+  enabled: false
+  period: 10s
+  hosts: ["httpbin.org"]
+  namespace: "http_json_namespace"
+  path: "/headers"

metricbeat/module/http/_meta/docs.asciidoc

@@ -0,0 +1,7 @@
+== http Module
+
+Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call arbitrary HTTP endpoints for which not a dedicated metricbeat is available.
+Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel.
+
+Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch.
+This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored.

metricbeat/module/http/_meta/env

@@ -0,0 +1,2 @@
+JOLOKIA_HOST=jolokia
+JOLOKIA_PORT=8778

metricbeat/module/http/_meta/fields.yml

@@ -0,0 +1,43 @@
+- key: http
+  title: "http"
+  description: >
+    http Module
+  fields:
+    - name: http
+      type: group
+      description: >
+      fields:
+        - name: request
+          type: group
+          description: >
+              HTTP request information
+          fields:
+            - name: header
+              type: nested
+              description: >
+                The HTTP headers sent
+            - name: method
+              type: keyword
+              description: >
+                The HTTP method used
+            - name: body
+              type: keyword
+              description: >
+                The HTTP payload sent
+        - name: response
+          type: group
+          description: >
+              HTTP response information
+          fields:
+            - name: header
+              type: nested
+              description: >
+                The HTTP headers received
+            - name: status_code
+              type: keyword
+              description: >
+                The HTTP status code
+            - name: body
+              type: keyword
+              description: >
+                The HTTP payload received

metricbeat/module/http/doc.go

@@ -0,0 +1,4 @@
+/*
+Package http is a Metricbeat module that contains MetricSets.
+*/
+package http

metricbeat/module/http/json/_meta/data.json

@@ -0,0 +1,19 @@
+{
+    "@timestamp":"2016-05-23T08:05:34.853Z",
+    "beat":{
+        "hostname":"beathost",
+        "name":"beathost"
+    },
+    "metricset":{
+        "host":"localhost",
+        "module":"http",
+        "name":"json",
+        "rtt":44269
+    },
+    "http":{
+        "json":{
+            "example": "json"
+        }
+    },
+    "type":"metricsets"
+}