Httpmetricbeat

metricbeat/docs/fields.asciidoc

@@ -23,6 +23,7 @@ grouped in the following categories:
 * <<exported-fields-elasticsearch>>
 * <<exported-fields-golang>>
 * <<exported-fields-haproxy>>
+* <<exported-fields-http>>
 * <<exported-fields-jolokia>>
 * <<exported-fields-kafka>>
 * <<exported-fields-kibana>>
@@ -3443,6 +3444,88 @@ type: integer
 The average queue time in ms over the last 1024 requests.
 
 
+[[exported-fields-http]]
+== http Fields
+
+http Module
+
+
+
+[float]
+== http Fields
+
+
+
+
+[float]
+== json Fields
+
+json metricset
+
+
+
+[float]
+== request Fields
+
+HTTP request information
+
+
+
+[float]
+=== http.json.request.header
+
+type: nested
+
+The HTTP headers sent
+
+
+[float]
+=== http.json.request.method
+
+type: keyword
+
+The HTTP method used
+
+
+[float]
+=== http.json.request.body
+
+type: keyword
+
+The HTTP payload sent
+
+
+[float]
+== response Fields
+
+HTTP response information
+
+
+
+[float]
+=== http.json.response.header
+
+type: nested
+
+The HTTP headers received
+
+
+[float]
+=== http.json.response.status_code
+
+type: keyword
+
+The HTTP status code
+
+
+[float]
+=== http.json.response.body
+
+type: keyword
+
+The HTTP payload received
+
+
 [[exported-fields-jolokia]]
 == Jolokia Fields
 

metricbeat/docs/modules/http.asciidoc

@@ -0,0 +1,37 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-http]]
+== http Module
+
+This is the http Module.
+
+
+
+[float]
+=== Example Configuration
+
+The http module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: http
+  metricsets: ["json"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost"]
+
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-http-json,json>>
+
+include::http/json.asciidoc[]
+

metricbeat/docs/modules/http/json.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-http-json]]
+include::../../../module/http/json/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-http,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/http/json/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-elasticsearch,elasticsearch>>
   * <<metricbeat-module-golang,golang>>
   * <<metricbeat-module-haproxy,HAProxy>>
+  * <<metricbeat-module-http,http>>
   * <<metricbeat-module-jolokia,Jolokia>>
   * <<metricbeat-module-kafka,kafka>>
   * <<metricbeat-module-kibana,kibana>>
@@ -37,6 +38,7 @@ include::modules/dropwizard.asciidoc[]
 include::modules/elasticsearch.asciidoc[]
 include::modules/golang.asciidoc[]
 include::modules/haproxy.asciidoc[]
+include::modules/http.asciidoc[]
 include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]
 include::modules/kibana.asciidoc[]

metricbeat/include/list.go

@@ -39,6 +39,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/haproxy"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/info"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/stat"
+	_ "github.com/elastic/beats/metricbeat/module/http"
+	_ "github.com/elastic/beats/metricbeat/module/http/json"
 	_ "github.com/elastic/beats/metricbeat/module/jolokia"
 	_ "github.com/elastic/beats/metricbeat/module/jolokia/jmx"
 	_ "github.com/elastic/beats/metricbeat/module/kafka"

metricbeat/metricbeat.full.yml

@@ -160,6 +160,14 @@ metricbeat.modules:
   period: 10s
   hosts: ["tcp://127.0.0.1:14567"]
 
+#-------------------------------- http Module --------------------------------
+- module: http
+  metricsets: ["json"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost"]
+
+
 #------------------------------- Jolokia Module ------------------------------
 - module: jolokia
   metricsets: ["jmx"]

metricbeat/module/http/_meta/config.yml

@@ -0,0 +1,6 @@
+- module: http
+  metricsets: ["json"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost"]
+

metricbeat/module/http/_meta/docs.asciidoc

@@ -0,0 +1,4 @@
+== http Module
+
+This is the http Module.
+

metricbeat/module/http/_meta/fields.yml

@@ -0,0 +1,9 @@
+- key: http
+  title: "http"
+  description: >
+    http Module
+  fields:
+    - name: http
+      type: group
+      description: >
+      fields:

metricbeat/module/http/doc.go

@@ -0,0 +1,4 @@
+/*
+Package http is a Metricbeat module that contains MetricSets.
+*/
+package http

metricbeat/module/http/json/_meta/data.json

@@ -0,0 +1,19 @@
+{
+    "@timestamp":"2016-05-23T08:05:34.853Z",
+    "beat":{
+        "hostname":"beathost",
+        "name":"beathost"
+    },
+    "metricset":{
+        "host":"localhost",
+        "module":"http",
+        "name":"json",
+        "rtt":44269
+    },
+    "http":{
+        "json":{
+            "example": "json"
+        }
+    },
+    "type":"metricsets"
+}

metricbeat/module/http/json/_meta/docs.asciidoc

@@ -0,0 +1,3 @@
+=== http json MetricSet
+
+This is the json metricset of the module http.

metricbeat/module/http/json/_meta/fields.yml

@@ -0,0 +1,39 @@
+- name: json
+  type: group
+  description: >
+    json metricset
+  fields:
+    - name: request
+      type: group
+      description: >
+          HTTP request information
+      fields:
+        - name: header
+          type: nested
+          description: >
+            The HTTP headers sent
+        - name: method
+          type: keyword
+          description: >
+            The HTTP method used
+        - name: body
+          type: keyword
+          description: >
+            The HTTP payload sent
+    - name: response
+      type: group
+      description: >
+          HTTP response information
+      fields:
+        - name: header
+          type: nested
+          description: >
+            The HTTP headers received
+        - name: status_code
+          type: keyword
+          description: >
+            The HTTP status code
+        - name: body
+          type: keyword
+          description: >
+            The HTTP payload received

metricbeat/module/http/json/json.go

@@ -0,0 +1,108 @@
+package json
+
+import (
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/metricbeat/helper"
+	"github.com/elastic/beats/metricbeat/mb"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// init registers the MetricSet with the central registry.
+// The New method will be called after the setup of the module and before starting to fetch data
+func init() {
+	if err := mb.Registry.AddMetricSet("http", "json", New); err != nil {
+		panic(err)
+	}
+}
+
+// MetricSet type defines all fields of the MetricSet
+// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with
+// additional entries. These variables can be used to persist data or configuration between
+// multiple fetch calls.
+type MetricSet struct {
+	mb.BaseMetricSet
+	http    *helper.HTTP
+	headers map[string]string
+	method  string
+	body    string
+}
+
+// New create a new instance of the MetricSet
+// Part of new is also setting up the configuration by processing additional
+// configuration entries if needed.
+func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+
+	config := struct {
+		Method  string            `config:"method"`
+		Body    string            `config:"body"`
+		Headers map[string]string `config:"headers"`
+	}{}
+
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
+
+	http := helper.NewHTTP(base)
+	http.SetMethod(config.Method)
+	http.SetBody([]byte(config.Body))
+	for key, value := range config.Headers {
+		http.SetHeader(key, value)
+	}
+
+	return &MetricSet{
+		BaseMetricSet: base,
+		http:          http,
+		headers:       config.Headers,
+		method:        config.Method,
+		body:          config.Body,
+	}, nil
+}
+
+// Fetch methods implements the data gathering and data conversion to the right format
+// It returns the event which is then forward to the output. In case of an error, a
+// descriptive error must be returned.
+func (m *MetricSet) Fetch() (common.MapStr, error) {
+
+	response, err := m.http.FetchResponse()
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	responseBody, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	event := common.MapStr{}
+
+	event["request"] = common.MapStr{
+		"headers": m.headers,
+		"method":  m.method,
+		"body":    m.body,
+	}
+
+	event["response"] = common.MapStr{
+		"status_code": response.StatusCode,
+		"headers":     m.getHeaders(response.Header),
+		"body":        responseBody,
+	}
+
+	return event, nil
+}
+
+func (m *MetricSet) getHeaders(header http.Header) map[string]string {
+
+	headers := make(map[string]string)
+	for k, v := range header {
+		value := ""
+		for _, h := range v {
+			value += h + " ,"
+		}
+		value = strings.TrimRight(value, " ,")
+		headers[k] = value
+	}
+	return headers
+}