Http module improvements (#4170)

This is a follow up PR for #4156

* Add environment with small golang web service
* Remove not needed body fields
* Add defaults to configs
* Add system tests
* Update config with all config options

metricbeat/docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - ${PWD}/module/dropwizard/_meta/env
       - ${PWD}/module/elasticsearch/_meta/env
       - ${PWD}/module/haproxy/_meta/env
+      - ${PWD}/module/http/_meta/env
       - ${PWD}/module/jolokia/_meta/env
       - ${PWD}/module/kafka/_meta/env
       - ${PWD}/module/kibana/_meta/env
@@ -46,6 +47,7 @@ services:
       dropwizard:     { condition: service_healthy }
       elasticsearch:  { condition: service_healthy }
       haproxy:        { condition: service_healthy }
+      http:           { condition: service_healthy }
       jolokia:        { condition: service_healthy }
       kafka:          { condition: service_healthy }
       kibana:         { condition: service_healthy }
@@ -80,6 +82,9 @@ services:
   haproxy:
     build: ${PWD}/module/haproxy/_meta
 
+  http:
+    build: ${PWD}/module/http/_meta
+
   jolokia:
     build: ${PWD}/module/jolokia/_meta
 

metricbeat/docs/fields.asciidoc

@@ -3446,7 +3446,7 @@ The average queue time in ms over the last 1024 requests.
 
 
 [[exported-fields-http]]
-== http Fields
+== HTTP Fields
 
 http Module
 
@@ -3526,15 +3526,6 @@ The HTTP payload received
 json metricset
 
 
-
-[float]
-=== http.json.body
-
-type: keyword
-
-The HTTP payload received
-
-
 [[exported-fields-jolokia]]
 == Jolokia Fields
 

metricbeat/docs/modules/http.asciidoc

@@ -5,8 +5,9 @@ This file is generated! See scripts/docs_collector.py
 [[metricbeat-module-http]]
 == http Module
 
-Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call arbitrary HTTP endpoints for which not a dedicated metricbeat is available.
-Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel.
+Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) module used to call arbitrary HTTP endpoints for which a dedicated metricbeat module is not available.
+
+Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. It is recommended to install a metricbeat instance on each host from which data should be fetched.
 
 Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch.
 This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored.
@@ -15,7 +16,7 @@ This is often necessary in security restricted network setups, where Logstash is
 [float]
 === Example Configuration
 
-The http module supports the standard configuration options that are described
+The HTTP module supports the standard configuration options that are described
 in <<configuration-metricbeat>>. Here is an example configuration:
 
 [source,yaml]
@@ -25,9 +26,13 @@ metricbeat.modules:
   metricsets: ["json"]
   enabled: false
   period: 10s
-  hosts: ["httpbin.org"]
-  namespace: "http_json_namespace"
-  path: "/headers"
+  hosts: ["localhost:80"]
+  namespace: "json_namespace"
+  path: "/"
+  #body: ""
+  #method: "GET"
+  #request.enabled: false
+  #response.enabled: false
 ----
 
 [float]

metricbeat/docs/modules_list.asciidoc

@@ -10,7 +10,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-elasticsearch,elasticsearch>>
   * <<metricbeat-module-golang,golang>>
   * <<metricbeat-module-haproxy,HAProxy>>
-  * <<metricbeat-module-http,http>>
+  * <<metricbeat-module-http,HTTP>>
   * <<metricbeat-module-jolokia,Jolokia>>
   * <<metricbeat-module-kafka,kafka>>
   * <<metricbeat-module-kibana,kibana>>

metricbeat/metricbeat.full.yml

@@ -177,14 +177,18 @@ metricbeat.modules:
   period: 10s
   hosts: ["tcp://127.0.0.1:14567"]
 
-#-------------------------------- http Module --------------------------------
+#-------------------------------- HTTP Module --------------------------------
 - module: http
   metricsets: ["json"]
   enabled: false
   period: 10s
-  hosts: ["httpbin.org"]
-  namespace: "http_json_namespace"
-  path: "/headers"
+  hosts: ["localhost:80"]
+  namespace: "json_namespace"
+  path: "/"
+  #body: ""
+  #method: "GET"
+  #request.enabled: false
+  #response.enabled: false
 
 #------------------------------- Jolokia Module ------------------------------
 - module: jolokia

metricbeat/module/http/_meta/Dockerfile

@@ -1,11 +1,10 @@
 # Tomcat is started to fetch Jolokia metrics from it
-FROM jolokia/java-jolokia:7
-ENV TOMCAT_VERSION 7.0.55
-ENV TC apache-tomcat-${TOMCAT_VERSION}
+FROM golang:1.8.1
 
-HEALTHCHECK CMD curl -f curl localhost:8778/jolokia/
-EXPOSE 8778
-RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz
-RUN tar xzf ${TC}.tar.gz -C /opt
+COPY test/main.go main.go
 
-CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run
+EXPOSE 8080
+
+HEALTHCHECK CMD curl -f curl localhost:8080/
+
+CMD go run main.go

metricbeat/module/http/_meta/config.yml

@@ -2,6 +2,10 @@
   metricsets: ["json"]
   enabled: false
   period: 10s
-  hosts: ["httpbin.org"]
-  namespace: "http_json_namespace"
-  path: "/headers"
+  hosts: ["localhost:80"]
+  namespace: "json_namespace"
+  path: "/"
+  #body: ""
+  #method: "GET"
+  #request.enabled: false
+  #response.enabled: false

metricbeat/module/http/_meta/docs.asciidoc

@@ -1,7 +1,8 @@
 == http Module
 
-Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call arbitrary HTTP endpoints for which not a dedicated metricbeat is available.
-Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel.
+Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) module used to call arbitrary HTTP endpoints for which a dedicated metricbeat module is not available.
+
+Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. It is recommended to install a metricbeat instance on each host from which data should be fetched.
 
 Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch.
 This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored.

metricbeat/module/http/_meta/env

@@ -1,2 +1,2 @@
-JOLOKIA_HOST=jolokia
-JOLOKIA_PORT=8778
+HTTP_HOST=http
+HTTP_PORT=8080

metricbeat/module/http/_meta/fields.yml

@@ -1,5 +1,5 @@
 - key: http
-  title: "http"
+  title: "HTTP"
   description: >
     http Module
   fields:

metricbeat/module/http/_meta/test/main.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+)
+
+func main() {
+	http.HandleFunc("/", serve)
+	err := http.ListenAndServe(":8080", nil)
+	if err != nil {
+		log.Fatal("ListenAndServe: ", err)
+	}
+}
+
+func serve(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, `{"hello":"world"}`)
+}

metricbeat/module/http/json/_meta/docs.asciidoc

@@ -7,7 +7,8 @@ This is the json metricset of the module http.
 
 The JSON structure returned by the HTTP endpoint will be added to the provided `namespace` field as shown in the following example:
 
-```json
+[source,json]
+----
 {
   "@timestamp": "2017-05-01T13:00:24.745Z",
   "beat": {
@@ -31,16 +32,18 @@ The JSON structure returned by the HTTP endpoint will be added to the provided `
   },
   "type": "metricsets"
 }
-```
+----
 
 Here the response from `date.jsontest.com` is returned in the configured `http_json_namespace` namespace:
-```json
+
+[source,json]
+----
 {
       "date": "05-01-2017",
       "milliseconds_since_epoch": 1493643625474.000000,
       "time": "01:00:25 PM"
 }
-```
+----
 
 It is required to set a namespace in the general module config section.
 
@@ -53,7 +56,9 @@ With this configuration enabled additional information about the request are inc
 * Body/Payload
 
 Example:
-```json
+
+[source,json]
+----
 {
   "@timestamp": "2017-05-01T13:00:24.745Z",
   "beat": {
@@ -84,7 +89,7 @@ Example:
   },
   "type": "metricsets"
 }
-```
+----
 
 [float]
 ==== response.enabled
@@ -94,7 +99,9 @@ With this configuration enabled additional information about the response are in
 * HTTP Status Code
 
 Example:
-```json
+
+[source,json]
+----
 {
   "@timestamp": "2017-05-01T13:00:24.745Z",
   "beat": {
@@ -129,7 +136,7 @@ Example:
   },
   "type": "metricsets"
 }
-```
+----
 
 
 [float]

metricbeat/module/http/json/_meta/fields.yml

@@ -3,7 +3,3 @@
   description: >
     json metricset
   fields:
-    - name: body
-      type: keyword
-      description: >
-        The HTTP payload received

metricbeat/module/http/json/json.go

@@ -64,7 +64,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		Body            string `config:"body"`
 		RequestEnabled  bool   `config:"request.enabled"`
 		ResponseEnabled bool   `config:"response.enabled"`
-	}{}
+	}{
+		Method:          "GET",
+		Body:            "",
+		RequestEnabled:  false,
+		ResponseEnabled: false,
+	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err

metricbeat/module/http/json/json_integration_test.go

@@ -33,13 +33,13 @@ func getConfig() map[string]interface{} {
 		"module":     "http",
 		"metricsets": []string{"json"},
 		"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
-		"path":       "/jolokia/?ignoreErrors=true&canonicalNaming=false",
+		"path":       "/",
 		"namespace":  "testnamespace",
 	}
 }
 
 func getEnvHost() string {
-	host := os.Getenv("JOLOKIA_HOST")
+	host := os.Getenv("HTTP_HOST")
 
 	if len(host) == 0 {
 		host = "127.0.0.1"
@@ -48,10 +48,10 @@ func getEnvHost() string {
 }
 
 func getEnvPort() string {
-	port := os.Getenv("JOLOKIA_PORT")
+	port := os.Getenv("HTTP_PORT")
 
 	if len(port) == 0 {
-		port = "8778"
+		port = "8080"
 	}
 	return port
 }

metricbeat/tests/system/test_http.py

@@ -0,0 +1,46 @@
+import os
+import metricbeat
+import unittest
+from nose.plugins.attrib import attr
+
+HTTP_FIELDS = metricbeat.COMMON_FIELDS + ["http"]
+
+
+class Test(metricbeat.BaseTest):
+
+    @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
+    def test_json(self):
+        """
+        http json metricset test
+        """
+        self.render_config_template(modules=[{
+            "name": "http",
+            "metricsets": ["json"],
+            "hosts": self.get_hosts(),
+            "period": "5s",
+            "namespace": "test",
+        }])
+        proc = self.start_beat()
+        self.wait_until(lambda: self.output_lines() > 0)
+        proc.check_kill_and_wait()
+
+        # Ensure no errors or warnings exist in the log.
+        log = self.get_log()
+        self.assertNotRegexpMatches(log.replace("WARN The http json metricset is in beta.", ""), "ERR|WARN")
+
+        output = self.read_output_json()
+        self.assertEqual(len(output), 1)
+        evt = output[0]
+
+        assert evt["http"]["test"]["hello"] == "world"
+
+        # Delete dynamic namespace part for fields comparison
+        del evt["http"]["test"]
+
+        self.assertItemsEqual(self.de_dot(HTTP_FIELDS), evt.keys(), evt)
+
+        self.assert_fields_are_documented(evt)
+
+    def get_hosts(self):
+        return ["http://" + os.getenv('HTTP_HOST', 'localhost') + ':' +
+                os.getenv('HTTP_PORT', '8080')]