From 557706ef57f6f09132a4fb045d2ab9af912b6e27 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 17:12:43 -0700 Subject: [PATCH 01/37] Renaming anchor for beats common fields --- .backportrc.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .backportrc.json diff --git a/.backportrc.json b/.backportrc.json new file mode 100644 index 000000000000..fd3e906db9f0 --- /dev/null +++ b/.backportrc.json @@ -0,0 +1,5 @@ +{ + "upstream": "elastic/beats", + "branches": ["7.x", { "name": "7.2", "checked": true }, "7.1", "6.8" ], + "labels": ["backport"] +} From b843fa0f2dbcbd9dd965cc84197d7582a5d5a2e5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 17:25:38 -0700 Subject: [PATCH 02/37] Removing extraneous file accidentally checked in --- .backportrc.json | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .backportrc.json diff --git a/.backportrc.json b/.backportrc.json deleted file mode 100644 index fd3e906db9f0..000000000000 --- a/.backportrc.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "upstream": "elastic/beats", - "branches": ["7.x", { "name": "7.2", "checked": true }, "7.1", "6.8" ], - "labels": ["backport"] -} From 20f22119eb74fc62818613da0a5cde0ce58f3e33 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 13 May 2019 15:42:35 -0700 Subject: [PATCH 03/37] Adding beats module with stats metricset --- metricbeat/docs/fields.asciidoc | 250 +++++++++++++++++- metricbeat/include/list.go | 2 + metricbeat/metricbeat.reference.yml | 12 + .../module/beats/_meta/config-xpack.yml | 9 + .../module/beats/_meta/config.reference.yml | 10 + metricbeat/module/beats/_meta/config.yml | 6 + metricbeat/module/beats/_meta/fields.yml | 20 ++ metricbeat/module/beats/beats.go | 64 +++++ metricbeat/module/beats/config.go | 30 +++ metricbeat/module/beats/docs.asciidoc | 9 + metricbeat/module/beats/fields.go | 36 +++ metricbeat/module/beats/metricset.go | 58 ++++ .../module/beats/stats/_meta/fields.yml | 95 +++++++ metricbeat/module/beats/stats/data.go | 88 ++++++ metricbeat/module/beats/stats/stats.go | 86 ++++++ metricbeat/modules.d/beats-xpack.yml.disabled | 12 + metricbeat/modules.d/beats.yml.disabled | 9 + 17 files changed, 784 insertions(+), 12 deletions(-) create mode 100644 metricbeat/module/beats/_meta/config-xpack.yml create mode 100644 metricbeat/module/beats/_meta/config.reference.yml create mode 100644 metricbeat/module/beats/_meta/config.yml create mode 100644 metricbeat/module/beats/_meta/fields.yml create mode 100644 metricbeat/module/beats/beats.go create mode 100644 metricbeat/module/beats/config.go create mode 100644 metricbeat/module/beats/docs.asciidoc create mode 100644 metricbeat/module/beats/fields.go create mode 100644 metricbeat/module/beats/metricset.go create mode 100644 metricbeat/module/beats/stats/_meta/fields.yml create mode 100644 metricbeat/module/beats/stats/data.go create mode 100644 metricbeat/module/beats/stats/stats.go create mode 100644 metricbeat/modules.d/beats-xpack.yml.disabled create mode 100644 metricbeat/modules.d/beats.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e60444157e98..18680235bbe0 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1568,6 +1569,231 @@ type: keyword Time series instance id +-- + +[[exported-fields-beats]] +== Beats fields + +Beats module + + + +[float] +== beats fields + + + + +*`beats.id`*:: ++ +-- +type: keyword + +Beat ID. + + +-- + +*`beats.type`*:: ++ +-- +type: keyword + +Beat type. + + +-- + +[float] +== stats fields + +Beat stats + + + +*`beats.stats.uptime.ms`*:: ++ +-- +type: long + +Beat uptime + + +-- + +*`beats.stats.runtime.goroutines`*:: ++ +-- +type: long + +Number of goroutines running in Beat + + +-- + +[float] +== libbeat fields + +Fields common to all Beats + + + +[float] +== output fields + +Output stats + + + +*`beats.stats.libbeat.output.type`*:: ++ +-- +type: keyword + +Type of output + + +-- + +[float] +== events fields + +Event counters + + + +*`beats.stats.libbeat.output.events.acked`*:: ++ +-- +type: long + +Number of events acknowledged + + +-- + +*`beats.stats.libbeat.output.events.active`*:: ++ +-- +type: long + +Number of active events + + +-- + +*`beats.stats.libbeat.output.events.batches`*:: ++ +-- +type: long + +Number of event batches + + +-- + +*`beats.stats.libbeat.output.events.dropped`*:: ++ +-- +type: long + +Number of events dropped + + +-- + +*`beats.stats.libbeat.output.events.duplicates`*:: ++ +-- +type: long + +Number of events duplicated + + +-- + +*`beats.stats.libbeat.output.events.failed`*:: ++ +-- +type: long + +Number of events failed + + +-- + +*`beats.stats.libbeat.output.events.toomany`*:: ++ +-- +type: long + +Number of too many events + + +-- + +*`beats.stats.libbeat.output.events.total`*:: ++ +-- +type: long + +Total number of events + + +-- + +[float] +== read fields + +Read stats + + + +*`beats.stats.libbeat.output.read.bytes`*:: ++ +-- +type: long + +Number of bytes read + + +-- + +*`beats.stats.libbeat.output.read.errors`*:: ++ +-- +type: long + +Number of read errors + + +-- + +[float] +== write fields + +Write stats + + + +*`beats.stats.libbeat.output.write.bytes`*:: ++ +-- +type: long + +Number of bytes written + + +-- + +*`beats.stats.libbeat.output.write.errors`*:: ++ +-- +type: long + +Number of write errors + + -- [[exported-fields-ceph]] @@ -19673,7 +19899,7 @@ The number of requests to read a row based on a key. -- type: long -The number of requests to read the last key in an index. +The number of requests to read the last key in an index. -- @@ -19703,7 +19929,7 @@ The number of requests to read the previous row in key order. -- type: long -The number of requests to read a row based on a fixed position. +The number of requests to read a row based on a fixed position. -- @@ -19713,7 +19939,7 @@ The number of requests to read a row based on a fixed position. -- type: long -The number of requests to read the next row in the data file. +The number of requests to read the next row in the data file. -- @@ -19811,7 +20037,7 @@ The progress of an operation to warm up the InnoDB buffer pool by reading in a s -- type: long -The total number of bytes in the InnoDB buffer pool containing data. +The total number of bytes in the InnoDB buffer pool containing data. -- @@ -22698,7 +22924,7 @@ type: long format: bytes -Used memory by the Lua engine. +Used memory by the Lua engine. -- @@ -22710,7 +22936,7 @@ type: long format: bytes -The size in bytes of the dataset +The size in bytes of the dataset -- @@ -22955,7 +23181,7 @@ type: long format: bytes -The size in bytes of copy-on-write allocations during the last RBD save operation +The size in bytes of copy-on-write allocations during the last RBD save operation -- @@ -23084,7 +23310,7 @@ type: long format: bytes -AOF current file size +AOF current file size -- @@ -23188,7 +23414,7 @@ Total size in bytes of the replication backlog buffer -- type: long -The master offset of the replication backlog buffer +The master offset of the replication backlog buffer -- @@ -26237,7 +26463,7 @@ The raid level of the device -- type: keyword -Current sync action, if the RAID array is redundant +Current sync action, if the RAID array is redundant -- @@ -26522,7 +26748,7 @@ type: integer format: bytes -Memory used by TCP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/tcp_mem. Only available on Linux. +Memory used by TCP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/tcp_mem. Only available on Linux. -- @@ -26608,7 +26834,7 @@ type: integer format: bytes -Memory used by UDP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/udp_mem. Only available on Linux. +Memory used by UDP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/udp_mem. Only available on Linux. -- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 74c4fae51f0b..6f79d09c3ea5 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -30,6 +30,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/aerospike/namespace" _ "github.com/elastic/beats/metricbeat/module/apache" _ "github.com/elastic/beats/metricbeat/module/apache/status" + _ "github.com/elastic/beats/metricbeat/module/beats" + _ "github.com/elastic/beats/metricbeat/module/beats/stats" _ "github.com/elastic/beats/metricbeat/module/ceph" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_health" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 372e34f4cb58..a2845fa6b8d2 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -156,6 +156,18 @@ metricbeat.modules: # Password of hosts. Empty by default #password: password +#-------------------------------- Beats Module ------------------------------- +- module: beats + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false + #-------------------------------- Ceph Module -------------------------------- - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] diff --git a/metricbeat/module/beats/_meta/config-xpack.yml b/metricbeat/module/beats/_meta/config-xpack.yml new file mode 100644 index 000000000000..8b7ec8b55e01 --- /dev/null +++ b/metricbeat/module/beats/_meta/config-xpack.yml @@ -0,0 +1,9 @@ +- module: beats + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #username: "user" + #password: "secret" + xpack.enabled: true + diff --git a/metricbeat/module/beats/_meta/config.reference.yml b/metricbeat/module/beats/_meta/config.reference.yml new file mode 100644 index 000000000000..8d723ba37c7e --- /dev/null +++ b/metricbeat/module/beats/_meta/config.reference.yml @@ -0,0 +1,10 @@ +- module: beats + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false diff --git a/metricbeat/module/beats/_meta/config.yml b/metricbeat/module/beats/_meta/config.yml new file mode 100644 index 000000000000..7ce8653a26b3 --- /dev/null +++ b/metricbeat/module/beats/_meta/config.yml @@ -0,0 +1,6 @@ +- module: beats + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + diff --git a/metricbeat/module/beats/_meta/fields.yml b/metricbeat/module/beats/_meta/fields.yml new file mode 100644 index 000000000000..c718c5e5375d --- /dev/null +++ b/metricbeat/module/beats/_meta/fields.yml @@ -0,0 +1,20 @@ +- key: beats + title: "Beats" + description: > + Beats module + release: ga + settings: ["ssl", "http"] + short_config: false + fields: + - name: beats + type: group + description: > + fields: + - name: id + type: keyword + description: > + Beat ID. + - name: type + type: keyword + description: > + Beat type. diff --git a/metricbeat/module/beats/beats.go b/metricbeat/module/beats/beats.go new file mode 100644 index 000000000000..01c961b976e9 --- /dev/null +++ b/metricbeat/module/beats/beats.go @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beats + +import ( + "encoding/json" + "net/url" + + "github.com/elastic/beats/metricbeat/helper" +) + +// ModuleName is the name of this module. +const ModuleName = "beats" + +// Info construct contains the data from the Beats / endpoint +type Info struct { + UUID string `json:"uuid"` + Beat string `json:"beat"` +} + +// GetInfo returns the data for the Beats / endpoint. +func GetInfo(m *MetricSet) (*Info, error) { + content, err := fetchPath(m.HTTP, "/", "") + if err != nil { + return nil, err + } + + info := &Info{} + err = json.Unmarshal(content, &info) + if err != nil { + return nil, err + } + + return info, nil +} + +func fetchPath(http *helper.HTTP, path string, query string) ([]byte, error) { + currentURI := http.GetURI() + defer http.SetURI(currentURI) + + // Parses the uri to replace the path + u, _ := url.Parse(currentURI) + u.Path = path + u.RawQuery = query + + // Http helper includes the HostData with username and password + http.SetURI(u.String()) + return http.FetchContent() +} diff --git a/metricbeat/module/beats/config.go b/metricbeat/module/beats/config.go new file mode 100644 index 000000000000..3cf0a6b554ac --- /dev/null +++ b/metricbeat/module/beats/config.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beats + +// Config defines the structure for the Beats module configuration options +type Config struct { + XPackEnabled bool `config:"xpack.enabled"` +} + +// DefaultConfig returns the default configuration for the Beats module +func DefaultConfig() Config { + return Config{ + XPackEnabled: false, + } +} diff --git a/metricbeat/module/beats/docs.asciidoc b/metricbeat/module/beats/docs.asciidoc new file mode 100644 index 000000000000..b56f5f6291bd --- /dev/null +++ b/metricbeat/module/beats/docs.asciidoc @@ -0,0 +1,9 @@ +The Beats module contains a minimal set of metrics to enable monitoring of any Beat or other software based on libbeat across +multiple versions. To monitor more Beats metrics, use our {stack-ov}/xpack-monitoring.html[monitoring] feature. + +The default metricset is `stats`. + +[float] +=== Compability + +The Beats module is tested with Metricbeat 7.2 and is expected to work with all 7.x versions. diff --git a/metricbeat/module/beats/fields.go b/metricbeat/module/beats/fields.go new file mode 100644 index 000000000000..6cf95e2440d6 --- /dev/null +++ b/metricbeat/module/beats/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package beats + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "beats", asset.ModuleFieldsPri, AssetBeats); err != nil { + panic(err) + } +} + +// AssetBeats returns asset data. +// This is the base64 encoded gzipped contents of ../metricbeat/module/beats. +func AssetBeats() string { + return "eJzcls+OmzAQxu95ihHn7j4Ahx6qtlIvrVSt1ENVVQ4MxIrxWPZ4I96+MpRA+GOSKrtU61Nih+/78Q3D5AGOWKewR8FuB8CSFaaQfAjfkx1Aji6z0rAkncL7HQBAcwYV5V7hDsCiQuEwhVLsABwyS126FH4mzqnkHSQHZpP8CmcHsvw7I13IMoVCKBeuLySq3KWN9ANoUWGPExbXJohb8ubvzgzTpcxQSubnrU7riPWJ7HB/VrFd4W7hy8fHiXDQuoN0uG4q7rgPYD6Ea7THKpel6tY4uSGHNywrfKzcxWnHo0iXo4MI0hmrFZ31s143hiVZ8iw13sv4q6/2aIEK6KWDm5a6BKkbsFkiJff78dlSPa7g+NyEDRlVFWlgAqFU21Gjn85VZchFno0fY8XRrsAL61ujPHl41sBguTemeNM+uREyrKfaYKjoYhQ9ED6jnr0bWE3sBqBPwQUy8prRLrnF8hsii+yISwFBrBP+ARwuOqQNKwBoOinMywhHT8vyeanmL4vbWsdLPBgtgrPD5LXyOqQN4ipBR5pbMmbbR2AN4YzqjZKZ4C1zdT3FOnAhpNo22hWC85uUqBK63oSUiSCYX9taTCzUK5I+BT/Qo2Sjc8CiWMr8blPgO4p8cXrCDRNgX2/VUI1zLKzBZLWWFmfdy1IGvph/h3iykuN/R+5Q9R/B5E2UPcTFqP/nyjcVHQPEPvwJAAD//wysTmY=" +} diff --git a/metricbeat/module/beats/metricset.go b/metricbeat/module/beats/metricset.go new file mode 100644 index 000000000000..e7f7c1deac88 --- /dev/null +++ b/metricbeat/module/beats/metricset.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beats + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" +) + +// MetricSet can be used to build other metricsets within the Beats module. +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP + XPackEnabled bool +} + +// NewMetricSet creates a metricset that can be used to build other metricsets +// within the Beats module. +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + config := DefaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + fmt.Println("uri: ", http.GetURI()) + + ms := &MetricSet{ + base, + http, + config.XPackEnabled, + } + + fmt.Println("uri 2: ", http.GetURI()) + + return ms, nil +} diff --git a/metricbeat/module/beats/stats/_meta/fields.yml b/metricbeat/module/beats/stats/_meta/fields.yml new file mode 100644 index 000000000000..043b39a6df0f --- /dev/null +++ b/metricbeat/module/beats/stats/_meta/fields.yml @@ -0,0 +1,95 @@ +- name: stats + type: group + description: > + Beat stats + release: ga + fields: + - name: uptime.ms + type: long + description: > + Beat uptime + - name: runtime.goroutines + type: long + description: > + Number of goroutines running in Beat + - name: libbeat + type: group + description: > + Fields common to all Beats + fields: + - name: output + type: group + description: > + Output stats + fields: + - name: type + type: keyword + description: > + Type of output + - name: events + type: group + description: > + Event counters + fields: + - name: acked + type: long + description: > + Number of events acknowledged + - name: active + type: long + description: > + Number of active events + - name: batches + type: long + description: > + Number of event batches + - name: dropped + type: long + description: > + Number of events dropped + - name: duplicates + type: long + description: > + Number of events duplicated + - name: failed + type: long + description: > + Number of events failed + - name: toomany + type: long + description: > + Number of too many events + - name: total + type: long + description: > + Total number of events + - name: read + type: group + description: > + Read stats + fields: + - name: bytes + type: long + description: > + Number of bytes read + - name: errors + type: long + description: > + Number of read errors + - name: write + type: group + description: > + Write stats + fields: + - name: bytes + type: long + description: > + Number of bytes written + - name: errors + type: long + description: > + Number of write errors + + + + diff --git a/metricbeat/module/beats/stats/data.go b/metricbeat/module/beats/stats/data.go new file mode 100644 index 000000000000..c200f725f694 --- /dev/null +++ b/metricbeat/module/beats/stats/data.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "encoding/json" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/beats" +) + +var ( + schema = s.Schema{ + "uptime": c.Dict("beat.info.uptime", s.Schema{ + "ms": c.Int("ms"), + }), + "runtime": c.Dict("beat.runtime", s.Schema{ + "goroutines": c.Int("goroutines"), + }), + "libbeat": c.Dict("libbeat", s.Schema{ + "output": c.Dict("output", s.Schema{ + "type": c.Str("type"), + "events": c.Dict("events", s.Schema{ + "acked": c.Int("acked"), + "active": c.Int("active"), + "batches": c.Int("batches"), + "dropped": c.Int("dropped"), + "duplicates": c.Int("duplicates"), + "failed": c.Int("failed"), + "toomany": c.Int("toomany"), + "total": c.Int("total"), + }), + "read": c.Dict("read", s.Schema{ + "bytes": c.Int("bytes"), + "errors": c.Int("errors"), + }), + "write": c.Dict("write", s.Schema{ + "bytes": c.Int("bytes"), + "errors": c.Int("errors"), + }), + }), + }), + } +) + +func eventMapping(r mb.ReporterV2, info beats.Info, content []byte) error { + var event mb.Event + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", beats.ModuleName) + + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("id", info.UUID) + event.ModuleFields.Put("type", info.Beat) + + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return errors.Wrap(err, "failure parsing Beats Stats API response") + } + + event.MetricSetFields, err = schema.Apply(data) + if err != nil { + return errors.Wrap(err, "failure to apply stats schema") + } + + r.Event(event) + return nil +} diff --git a/metricbeat/module/beats/stats/stats.go b/metricbeat/module/beats/stats/stats.go new file mode 100644 index 000000000000..20f8d6181ff9 --- /dev/null +++ b/metricbeat/module/beats/stats/stats.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/beats" +) + +func init() { + mb.Registry.MustAddMetricSet(beats.ModuleName, "stats", New, + mb.WithHostParser(hostParser), + ) +} + +const ( + statsPath = "stats" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: statsPath, + }.Build() +) + +// MetricSet defines all fields of the MetricSet +type MetricSet struct { + *beats.MetricSet +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := beats.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +func (m *MetricSet) Fetch(r mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + fmt.Println("here") + return err + } + + info, err := beats.GetInfo(m.MetricSet) + if err != nil { + return err + } + + if m.MetricSet.XPackEnabled { + // err = eventMappingXPack(r, m, *info, content) + // if err != nil { + // // Since this is an x-pack code path, we log the error but don't + // // return it. Otherwise it would get reported into `metricbeat-*` + // // indices. + // m.Logger().Error(err) + // return nil + // } + } else { + return eventMapping(r, *info, content) + } + + return nil +} diff --git a/metricbeat/modules.d/beats-xpack.yml.disabled b/metricbeat/modules.d/beats-xpack.yml.disabled new file mode 100644 index 000000000000..4426a9ffacfb --- /dev/null +++ b/metricbeat/modules.d/beats-xpack.yml.disabled @@ -0,0 +1,12 @@ +# Module: beats +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beats.html + +- module: beats + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #username: "user" + #password: "secret" + xpack.enabled: true + diff --git a/metricbeat/modules.d/beats.yml.disabled b/metricbeat/modules.d/beats.yml.disabled new file mode 100644 index 000000000000..bc3ff7aaa15d --- /dev/null +++ b/metricbeat/modules.d/beats.yml.disabled @@ -0,0 +1,9 @@ +# Module: beats +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beats.html + +- module: beats + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + From 14e760b189ef111e3b1adb7bdf400c27bb8e88ac Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 20 May 2019 07:48:52 -0700 Subject: [PATCH 04/37] Rename argument for clarity --- metricbeat/module/beats/beats.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/beats/beats.go b/metricbeat/module/beats/beats.go index 01c961b976e9..760accf6399c 100644 --- a/metricbeat/module/beats/beats.go +++ b/metricbeat/module/beats/beats.go @@ -49,9 +49,9 @@ func GetInfo(m *MetricSet) (*Info, error) { return info, nil } -func fetchPath(http *helper.HTTP, path string, query string) ([]byte, error) { - currentURI := http.GetURI() - defer http.SetURI(currentURI) +func fetchPath(httpHelper *helper.HTTP, path string, query string) ([]byte, error) { + currentURI := httpHelper.GetURI() + defer httpHelper.SetURI(currentURI) // Parses the uri to replace the path u, _ := url.Parse(currentURI) @@ -59,6 +59,6 @@ func fetchPath(http *helper.HTTP, path string, query string) ([]byte, error) { u.RawQuery = query // Http helper includes the HostData with username and password - http.SetURI(u.String()) - return http.FetchContent() + httpHelper.SetURI(u.String()) + return httpHelper.FetchContent() } From 38c4b4d8d857bcc8b66a0691b2d2d05026ef0555 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 20 May 2019 11:38:07 -0700 Subject: [PATCH 05/37] Adding tests --- metricbeat/docker-compose.yml | 6 ++ metricbeat/module/beats/_meta/Dockerfile | 6 ++ metricbeat/module/beats/_meta/env | 2 + metricbeat/module/beats/_meta/healthcheck.sh | 6 ++ .../module/beats/beats_integration_test.go | 64 +++++++++++++++++++ metricbeat/module/beats/testing.go | 49 ++++++++++++++ metricbeat/tests/system/test_beats.py | 32 ++++++++++ 7 files changed, 165 insertions(+) create mode 100644 metricbeat/module/beats/_meta/Dockerfile create mode 100644 metricbeat/module/beats/_meta/env create mode 100644 metricbeat/module/beats/_meta/healthcheck.sh create mode 100644 metricbeat/module/beats/beats_integration_test.go create mode 100644 metricbeat/module/beats/testing.go create mode 100644 metricbeat/tests/system/test_beats.py diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 939b356c15b5..24c8de4327b3 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -14,6 +14,7 @@ services: env_file: - ./module/aerospike/_meta/env - ./module/apache/_meta/env + - ./module/beats/_meta/env - ./module/ceph/_meta/env - ./module/consul/_meta/env - ./module/couchbase/_meta/env @@ -63,6 +64,11 @@ services: ports: - 80 + beats: + build: ./module/beats/_meta + ports: + - 5066 + ceph: build: ./module/ceph/_meta ports: diff --git a/metricbeat/module/beats/_meta/Dockerfile b/metricbeat/module/beats/_meta/Dockerfile new file mode 100644 index 000000000000..44b84c00b907 --- /dev/null +++ b/metricbeat/module/beats/_meta/Dockerfile @@ -0,0 +1,6 @@ +FROM docker.elastic.co/beats/metricbeat:7.0.0 + +COPY healthcheck.sh / +HEALTHCHECK --interval=1s --retries=300 CMD sh /healthcheck.sh + +ENTRYPOINT [ "metricbeat", "-E", "http.enabled=true", "-E", "http.host=0.0.0.0" ] diff --git a/metricbeat/module/beats/_meta/env b/metricbeat/module/beats/_meta/env new file mode 100644 index 000000000000..4cbbdbf548c0 --- /dev/null +++ b/metricbeat/module/beats/_meta/env @@ -0,0 +1,2 @@ +METRICBEAT_HOST=metricbeat +METRICBEAT_PORT=5066 diff --git a/metricbeat/module/beats/_meta/healthcheck.sh b/metricbeat/module/beats/_meta/healthcheck.sh new file mode 100644 index 000000000000..ff9d08e49784 --- /dev/null +++ b/metricbeat/module/beats/_meta/healthcheck.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Check that all endpoints are available +curl -f http://localhost:5066 || exit 1 +curl -f http://localhost:5066/stats || exit 1 +curl -f http://localhost:5066/state || exit 1 diff --git a/metricbeat/module/beats/beats_integration_test.go b/metricbeat/module/beats/beats_integration_test.go new file mode 100644 index 000000000000..b78c9077f5f2 --- /dev/null +++ b/metricbeat/module/beats/beats_integration_test.go @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package beats_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/beats/node" +) + +var metricSets = []string{ + "stats", +} + +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "metricbeat") + + for _, metricSet := range metricSets { + f := mbtest.NewReportingMetricSetV2Error(t, beats.GetConfig(metricSet)) + events, errs := mbtest.ReportingFetchV2Error(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("beats", metricSet).Fields.StringToPrint()) + } +} + +func TestData(t *testing.T) { + compose.EnsureUp(t, "metricbeat") + + for _, metricSet := range metricSets { + config := beats.GetConfig(metricSet) + f := mbtest.NewReportingMetricSetV2Error(t, config) + err := mbtest.WriteEventsReporterV2Error(f, t, metricSet) + if err != nil { + t.Fatal("write", err) + } + } +} diff --git a/metricbeat/module/beats/testing.go b/metricbeat/module/beats/testing.go new file mode 100644 index 000000000000..8e41a4e3de6d --- /dev/null +++ b/metricbeat/module/beats/testing.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beats + +import "os" + +// GetEnvHost for Metricbeat +func GetEnvHost() string { + host := os.Getenv("METRICBEAT_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +// GetEnvPort for Metricbeat +func GetEnvPort() string { + port := os.Getenv("METRICBEAT_HOST") + + if len(port) == 0 { + port = "5066" + } + return port +} + +// GetConfig for Metricbeat +func GetConfig(metricset string) map[string]interface{} { + return map[string]interface{}{ + "module": ModuleName, + "metricsets": []string{metricset}, + "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, + } +} diff --git a/metricbeat/tests/system/test_beats.py b/metricbeat/tests/system/test_beats.py new file mode 100644 index 000000000000..8ef94c6ba2d0 --- /dev/null +++ b/metricbeat/tests/system/test_beats.py @@ -0,0 +1,32 @@ +import os +import metricbeat +import unittest +import time + + +class Test(metricbeat.BaseTest): + + COMPOSE_SERVICES = ['metricbeat'] + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_stats(self): + """ + beats stats metricset test + """ + self.render_config_template(modules=[{ + "name": "beats", + "metricsets": ["stats"], + "hosts": self.get_hosts(), + "period": "1s", + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print(evt) + + self.assert_fields_are_documented(evt) From a4c1d1fe9371aace617b509902111b75501313f5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 15:20:21 -0700 Subject: [PATCH 06/37] Adding tests + sample data --- metricbeat/module/beats/_meta/env | 4 +- .../module/beats/beats_integration_test.go | 36 +++- metricbeat/module/beats/stats/_meta/data.json | 51 ++++++ .../beats/stats/_meta/test/stats.800.json | 155 ++++++++++++++++++ metricbeat/module/beats/stats/data_test.go | 55 +++++++ 5 files changed, 295 insertions(+), 6 deletions(-) create mode 100644 metricbeat/module/beats/stats/_meta/data.json create mode 100644 metricbeat/module/beats/stats/_meta/test/stats.800.json create mode 100644 metricbeat/module/beats/stats/data_test.go diff --git a/metricbeat/module/beats/_meta/env b/metricbeat/module/beats/_meta/env index 4cbbdbf548c0..5e4a55fde9cb 100644 --- a/metricbeat/module/beats/_meta/env +++ b/metricbeat/module/beats/_meta/env @@ -1,2 +1,2 @@ -METRICBEAT_HOST=metricbeat -METRICBEAT_PORT=5066 +BEATS_HOST=metricbeat +BEATS_PORT=5066 diff --git a/metricbeat/module/beats/beats_integration_test.go b/metricbeat/module/beats/beats_integration_test.go index b78c9077f5f2..593f7231194c 100644 --- a/metricbeat/module/beats/beats_integration_test.go +++ b/metricbeat/module/beats/beats_integration_test.go @@ -20,13 +20,15 @@ package beats_test import ( + "os" "testing" "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - _ "github.com/elastic/beats/metricbeat/module/beats/node" + "github.com/elastic/beats/metricbeat/module/beats" + _ "github.com/elastic/beats/metricbeat/module/beats/stats" ) var metricSets = []string{ @@ -34,7 +36,7 @@ var metricSets = []string{ } func TestFetch(t *testing.T) { - compose.EnsureUp(t, "metricbeat") + compose.EnsureUp(t, "beats") for _, metricSet := range metricSets { f := mbtest.NewReportingMetricSetV2Error(t, beats.GetConfig(metricSet)) @@ -51,10 +53,10 @@ func TestFetch(t *testing.T) { } func TestData(t *testing.T) { - compose.EnsureUp(t, "metricbeat") + compose.EnsureUp(t, "beats") for _, metricSet := range metricSets { - config := beats.GetConfig(metricSet) + config := getConfig(metricSet) f := mbtest.NewReportingMetricSetV2Error(t, config) err := mbtest.WriteEventsReporterV2Error(f, t, metricSet) if err != nil { @@ -62,3 +64,29 @@ func TestData(t *testing.T) { } } } + +func getConfig(metricset string) map[string]interface{} { + return map[string]interface{}{ + "module": beats.ModuleName, + "metricsets": []string{metricset}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + } +} + +func getEnvHost() string { + host := os.Getenv("BEATS_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func getEnvPort() string { + port := os.Getenv("BEATS_PORT") + + if len(port) == 0 { + port = "5066" + } + return port +} diff --git a/metricbeat/module/beats/stats/_meta/data.json b/metricbeat/module/beats/stats/_meta/data.json new file mode 100644 index 000000000000..43ddf69bea21 --- /dev/null +++ b/metricbeat/module/beats/stats/_meta/data.json @@ -0,0 +1,51 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beats": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "stats": { + "libbeat": { + "output": { + "events": { + "acked": 0, + "active": 0, + "batches": 0, + "dropped": 0, + "duplicates": 0, + "failed": 0, + "toomany": 0, + "total": 0 + }, + "read": { + "bytes": 0, + "errors": 0 + }, + "type": "elasticsearch", + "write": { + "bytes": 0, + "errors": 0 + } + } + }, + "runtime": { + "goroutines": 39 + }, + "uptime": { + "ms": 207906 + } + }, + "type": "metricbeat" + }, + "event": { + "dataset": "beats.stats", + "duration": 115000, + "module": "beats" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:5066", + "name": "beats", + "type": "beats" + } +} \ No newline at end of file diff --git a/metricbeat/module/beats/stats/_meta/test/stats.800.json b/metricbeat/module/beats/stats/_meta/test/stats.800.json new file mode 100644 index 000000000000..ced95cd76e3f --- /dev/null +++ b/metricbeat/module/beats/stats/_meta/test/stats.800.json @@ -0,0 +1,155 @@ +{ + "beat": { + "cpu": { + "system": { + "ticks": 95, + "time": { + "ms": 95 + } + }, + "total": { + "ticks": 458, + "time": { + "ms": 458 + }, + "value": 458 + }, + "user": { + "ticks": 363, + "time": { + "ms": 363 + } + } + }, + "info": { + "ephemeral_id": "f32e9a62-56c2-4cde-87be-de5869b2d7b7", + "uptime": { + "ms": 21557 + } + }, + "memstats": { + "gc_next": 10723216, + "memory_alloc": 6145008, + "memory_total": 360839144, + "rss": 44339200 + }, + "runtime": { + "goroutines": 40 + } + }, + "libbeat": { + "config": { + "module": { + "running": 0, + "starts": 0, + "stops": 0 + }, + "reloads": 1 + }, + "output": { + "events": { + "acked": 0, + "active": 0, + "batches": 0, + "dropped": 0, + "duplicates": 0, + "failed": 0, + "toomany": 0, + "total": 0 + }, + "read": { + "bytes": 0, + "errors": 0 + }, + "type": "elasticsearch", + "write": { + "bytes": 0, + "errors": 0 + } + }, + "pipeline": { + "clients": 3, + "events": { + "active": 37, + "dropped": 0, + "failed": 0, + "filtered": 1, + "published": 37, + "retry": 69, + "total": 38 + }, + "queue": { + "acked": 0 + } + } + }, + "metricbeat": { + "system": { + "cpu": { + "events": 2, + "failures": 0, + "success": 2 + }, + "filesystem": { + "events": 7, + "failures": 0, + "success": 7 + }, + "fsstat": { + "events": 1, + "failures": 0, + "success": 1 + }, + "load": { + "events": 3, + "failures": 0, + "success": 3 + }, + "memory": { + "events": 3, + "failures": 0, + "success": 3 + }, + "network": { + "events": 0, + "failures": 0, + "success": 0 + }, + "process": { + "events": 17, + "failures": 0, + "success": 17 + }, + "process_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "socket_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "uptime": { + "events": 1, + "failures": 0, + "success": 1 + } + } + }, + "system": { + "cpu": { + "cores": 8 + }, + "load": { + "1": 1.6753, + "15": 1.8076, + "5": 1.8257, + "norm": { + "1": 0.2094, + "15": 0.226, + "5": 0.2282 + } + } + } +} diff --git a/metricbeat/module/beats/stats/data_test.go b/metricbeat/module/beats/stats/data_test.go new file mode 100644 index 000000000000..7dc073d01ecf --- /dev/null +++ b/metricbeat/module/beats/stats/data_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package stats + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/elastic/beats/metricbeat/module/beats" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapping(t *testing.T) { + + files, err := filepath.Glob("./_meta/test/stats.*.json") + assert.NoError(t, err) + + info := beats.Info{ + UUID: "1234", + Beat: "helloworld", + } + + for _, f := range files { + input, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = eventMapping(reporter, info, input) + + assert.NoError(t, err, f) + assert.True(t, len(reporter.GetEvents()) >= 1, f) + assert.Equal(t, 0, len(reporter.GetErrors()), f) + } +} From 3fb7030cd617a16ede45e222caf6596eb62ba358 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 15:23:31 -0700 Subject: [PATCH 07/37] Removing debugging statements --- metricbeat/module/beats/metricset.go | 6 ------ metricbeat/module/beats/stats/stats.go | 3 --- 2 files changed, 9 deletions(-) diff --git a/metricbeat/module/beats/metricset.go b/metricbeat/module/beats/metricset.go index e7f7c1deac88..2548968deba5 100644 --- a/metricbeat/module/beats/metricset.go +++ b/metricbeat/module/beats/metricset.go @@ -18,8 +18,6 @@ package beats import ( - "fmt" - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" ) @@ -44,15 +42,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, err } - fmt.Println("uri: ", http.GetURI()) - ms := &MetricSet{ base, http, config.XPackEnabled, } - fmt.Println("uri 2: ", http.GetURI()) - return ms, nil } diff --git a/metricbeat/module/beats/stats/stats.go b/metricbeat/module/beats/stats/stats.go index 20f8d6181ff9..931979d09fa8 100644 --- a/metricbeat/module/beats/stats/stats.go +++ b/metricbeat/module/beats/stats/stats.go @@ -18,8 +18,6 @@ package stats import ( - "fmt" - "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/beats" @@ -60,7 +58,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(r mb.ReporterV2) error { content, err := m.HTTP.FetchContent() if err != nil { - fmt.Println("here") return err } From ae9b43a1b186a72bf235b74582bb12893ee9a983 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 15:23:39 -0700 Subject: [PATCH 08/37] Regenerating data.json file --- metricbeat/module/beats/stats/_meta/data.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/beats/stats/_meta/data.json b/metricbeat/module/beats/stats/_meta/data.json index 43ddf69bea21..d0e48a2c8f79 100644 --- a/metricbeat/module/beats/stats/_meta/data.json +++ b/metricbeat/module/beats/stats/_meta/data.json @@ -27,10 +27,10 @@ } }, "runtime": { - "goroutines": 39 + "goroutines": 64 }, "uptime": { - "ms": 207906 + "ms": 1631 } }, "type": "metricbeat" From 564e5d7f28794e7507eef578702c260a9a6d3946 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 16:54:33 -0700 Subject: [PATCH 09/37] Result of running make update --- x-pack/metricbeat/metricbeat.reference.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 56e377a89dea..df2487c618b7 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -205,6 +205,18 @@ metricbeat.modules: # - us-east-1 # - us-east-2 +#-------------------------------- Beats Module -------------------------------- +- module: beats + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false + #--------------------------------- Ceph Module --------------------------------- - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] From 80f6f87756b5b0de62009415215fdc1f90df0625 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 17:44:15 -0700 Subject: [PATCH 10/37] Trying a different host name --- metricbeat/module/beats/_meta/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beats/_meta/Dockerfile b/metricbeat/module/beats/_meta/Dockerfile index 44b84c00b907..6c959bf542b2 100644 --- a/metricbeat/module/beats/_meta/Dockerfile +++ b/metricbeat/module/beats/_meta/Dockerfile @@ -3,4 +3,4 @@ FROM docker.elastic.co/beats/metricbeat:7.0.0 COPY healthcheck.sh / HEALTHCHECK --interval=1s --retries=300 CMD sh /healthcheck.sh -ENTRYPOINT [ "metricbeat", "-E", "http.enabled=true", "-E", "http.host=0.0.0.0" ] +ENTRYPOINT [ "metricbeat", "-E", "http.enabled=true", "-E", "http.host=127.0.0.1" ] From 98106b848ba82c0e19b00807e3ee6c204809bb64 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 19:46:48 -0700 Subject: [PATCH 11/37] Stubbing out xpack code path --- metricbeat/module/beats/stats/data_xpack.go | 57 +++++++++++++++++++++ metricbeat/module/beats/stats/stats.go | 16 +++--- 2 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 metricbeat/module/beats/stats/data_xpack.go diff --git a/metricbeat/module/beats/stats/data_xpack.go b/metricbeat/module/beats/stats/data_xpack.go new file mode 100644 index 000000000000..c5ed62c30f35 --- /dev/null +++ b/metricbeat/module/beats/stats/data_xpack.go @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "time" + + "github.com/elastic/beats/metricbeat/helper/elastic" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/beats" +) + +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beats.Info, content []byte) error { + now := time.Now() + clusterUUID := "TODO: Get from GET /state API response" + + fields := common.MapStr{ + "metrics": "TODO: parse / construct from content", + "beat": "TODO: parse / construct from info", + "timestamp": now, + } + + var event mb.Event + event.RootFields = common.MapStr{ + "cluster_uuid": clusterUUID, + "timestamp": now, + "interval_ms": m.calculateIntervalMs(), + "type": "beats_stats", + "beats_stats": fields, + } + + event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + + r.Event(event) + return nil +} + +func (m *MetricSet) calculateIntervalMs() int64 { + return m.Module().Config().Period.Nanoseconds() / 1000 / 1000 +} diff --git a/metricbeat/module/beats/stats/stats.go b/metricbeat/module/beats/stats/stats.go index 931979d09fa8..c3205247763e 100644 --- a/metricbeat/module/beats/stats/stats.go +++ b/metricbeat/module/beats/stats/stats.go @@ -67,14 +67,14 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } if m.MetricSet.XPackEnabled { - // err = eventMappingXPack(r, m, *info, content) - // if err != nil { - // // Since this is an x-pack code path, we log the error but don't - // // return it. Otherwise it would get reported into `metricbeat-*` - // // indices. - // m.Logger().Error(err) - // return nil - // } + err = eventMappingXPack(r, m, *info, content) + if err != nil { + // Since this is an x-pack code path, we log the error but don't + // return it. Otherwise it would get reported into `metricbeat-*` + // indices. + m.Logger().Error(err) + return nil + } } else { return eventMapping(r, *info, content) } From a819586ef35e4a9b4938d23e79eb2310c000e4f3 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 19:47:18 -0700 Subject: [PATCH 12/37] Fixing service name in system test --- metricbeat/tests/system/test_beats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/test_beats.py b/metricbeat/tests/system/test_beats.py index 8ef94c6ba2d0..6c9653d7c57a 100644 --- a/metricbeat/tests/system/test_beats.py +++ b/metricbeat/tests/system/test_beats.py @@ -6,7 +6,7 @@ class Test(metricbeat.BaseTest): - COMPOSE_SERVICES = ['metricbeat'] + COMPOSE_SERVICES = ['beats'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_stats(self): From 3cb31395673886b05ef5d24fcf83e8e75ff9f7e9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 20:13:12 -0700 Subject: [PATCH 13/37] Fleshing out TODOs --- metricbeat/module/beats/beats.go | 34 +++++++++++++++++-- metricbeat/module/beats/stats/data_xpack.go | 36 +++++++++++++++++++-- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/beats/beats.go b/metricbeat/module/beats/beats.go index 760accf6399c..24227b0e2d52 100644 --- a/metricbeat/module/beats/beats.go +++ b/metricbeat/module/beats/beats.go @@ -27,10 +27,22 @@ import ( // ModuleName is the name of this module. const ModuleName = "beats" -// Info construct contains the data from the Beats / endpoint +// Info construct contains the relevant data from the Beats / endpoint type Info struct { - UUID string `json:"uuid"` - Beat string `json:"beat"` + UUID string `json:"uuid"` + Beat string `json:"beat"` + Name string `json:"name"` + Hostname string `json:"hostname"` + Version string `json:"version"` +} + +// State construct contains the relevant data from the Beats /state endpoint +type State struct { + Outputs struct { + Elasticsearch struct { + ClusterUUID string `json:"cluster_uuid"` + } `json:"elasticsearch"` + } `json:"outputs"` } // GetInfo returns the data for the Beats / endpoint. @@ -49,6 +61,22 @@ func GetInfo(m *MetricSet) (*Info, error) { return info, nil } +// GetState returns the data for the Beats /state endpoint. +func GetState(m *MetricSet) (*State, error) { + content, err := fetchPath(m.HTTP, "/state", "") + if err != nil { + return nil, err + } + + info := &State{} + err = json.Unmarshal(content, &info) + if err != nil { + return nil, err + } + + return info, nil +} + func fetchPath(httpHelper *helper.HTTP, path string, query string) ([]byte, error) { currentURI := httpHelper.GetURI() defer httpHelper.SetURI(currentURI) diff --git a/metricbeat/module/beats/stats/data_xpack.go b/metricbeat/module/beats/stats/data_xpack.go index c5ed62c30f35..4e3fd70e8b24 100644 --- a/metricbeat/module/beats/stats/data_xpack.go +++ b/metricbeat/module/beats/stats/data_xpack.go @@ -18,8 +18,11 @@ package stats import ( + "encoding/json" "time" + "github.com/pkg/errors" + "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/elastic/beats/libbeat/common" @@ -29,11 +32,29 @@ import ( func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beats.Info, content []byte) error { now := time.Now() - clusterUUID := "TODO: Get from GET /state API response" + clusterUUID, err := m.getClusterUUID() + if err != nil { + return errors.Wrap(err, "could not determine cluster UUID") + } + + // Massage info into beat + beat := common.MapStr{ + "name": info.Name, + "host": info.Hostname, + "type": info.Beat, + "uuid": info.UUID, + "version": info.Version, + } + + var metrics map[string]interface{} + err = json.Unmarshal(content, &metrics) + if err != nil { + return errors.Wrap(err, "failure parsing Beats Stats API response") + } fields := common.MapStr{ - "metrics": "TODO: parse / construct from content", - "beat": "TODO: parse / construct from info", + "metrics": metrics, + "beat": beat, "timestamp": now, } @@ -55,3 +76,12 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beats.Info, content [ func (m *MetricSet) calculateIntervalMs() int64 { return m.Module().Config().Period.Nanoseconds() / 1000 / 1000 } + +func (m *MetricSet) getClusterUUID() (string, error) { + state, err := beats.GetState(m.MetricSet) + if err != nil { + return "", errors.Wrap(err, "could not get state information") + } + + return state.Outputs.Elasticsearch.ClusterUUID, nil +} From ad75bb699cf4506578d71172eb511440a433f4e5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 17 Jun 2019 20:16:51 -0700 Subject: [PATCH 14/37] Fixing index name --- metricbeat/module/beats/stats/data_xpack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beats/stats/data_xpack.go b/metricbeat/module/beats/stats/data_xpack.go index 4e3fd70e8b24..3c94050af813 100644 --- a/metricbeat/module/beats/stats/data_xpack.go +++ b/metricbeat/module/beats/stats/data_xpack.go @@ -67,7 +67,7 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beats.Info, content [ "beats_stats": fields, } - event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Beats) r.Event(event) return nil From 9843f38004b1954c703195f944bc923592657b62 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 06:21:38 -0700 Subject: [PATCH 15/37] Renaming module name from beats to beat --- metricbeat/docs/fields.asciidoc | 42 +++++++++---------- metricbeat/include/list.go | 4 +- metricbeat/metricbeat.reference.yml | 2 +- .../module/{beats => beat}/_meta/Dockerfile | 0 .../{beats => beat}/_meta/config-xpack.yml | 0 .../_meta/config.reference.yml | 0 .../module/{beats => beat}/_meta/config.yml | 0 metricbeat/module/beat/_meta/env | 2 + .../module/{beats => beat}/_meta/fields.yml | 8 ++-- .../{beats => beat}/_meta/healthcheck.sh | 0 .../module/{beats/beats.go => beat/beat.go} | 12 +++--- .../beat_integration_test.go} | 20 ++++----- metricbeat/module/{beats => beat}/config.go | 6 +-- metricbeat/module/beat/docs.asciidoc | 9 ++++ metricbeat/module/{beats => beat}/fields.go | 12 +++--- .../module/{beats => beat}/metricset.go | 6 +-- .../{beats => beat}/stats/_meta/data.json | 0 .../{beats => beat}/stats/_meta/fields.yml | 0 .../stats/_meta/test/stats.800.json | 0 .../module/{beats => beat}/stats/data.go | 8 ++-- .../module/{beats => beat}/stats/data_test.go | 4 +- .../{beats => beat}/stats/data_xpack.go | 8 ++-- .../module/{beats => beat}/stats/stats.go | 10 ++--- metricbeat/module/{beats => beat}/testing.go | 2 +- metricbeat/module/beats/_meta/env | 2 - metricbeat/module/beats/docs.asciidoc | 9 ---- ...k.yml.disabled => beat-xpack.yml.disabled} | 4 +- .../{beats.yml.disabled => beat.yml.disabled} | 4 +- 28 files changed, 87 insertions(+), 87 deletions(-) rename metricbeat/module/{beats => beat}/_meta/Dockerfile (100%) rename metricbeat/module/{beats => beat}/_meta/config-xpack.yml (100%) rename metricbeat/module/{beats => beat}/_meta/config.reference.yml (100%) rename metricbeat/module/{beats => beat}/_meta/config.yml (100%) create mode 100644 metricbeat/module/beat/_meta/env rename metricbeat/module/{beats => beat}/_meta/fields.yml (83%) rename metricbeat/module/{beats => beat}/_meta/healthcheck.sh (100%) rename metricbeat/module/{beats/beats.go => beat/beat.go} (87%) rename metricbeat/module/{beats/beats_integration_test.go => beat/beat_integration_test.go} (81%) rename metricbeat/module/{beats => beat}/config.go (85%) create mode 100644 metricbeat/module/beat/docs.asciidoc rename metricbeat/module/{beats => beat}/fields.go (52%) rename metricbeat/module/{beats => beat}/metricset.go (96%) rename metricbeat/module/{beats => beat}/stats/_meta/data.json (100%) rename metricbeat/module/{beats => beat}/stats/_meta/fields.yml (100%) rename metricbeat/module/{beats => beat}/stats/_meta/test/stats.800.json (100%) rename metricbeat/module/{beats => beat}/stats/data.go (90%) rename metricbeat/module/{beats => beat}/stats/data_test.go (95%) rename metricbeat/module/{beats => beat}/stats/data_xpack.go (89%) rename metricbeat/module/{beats => beat}/stats/stats.go (90%) rename metricbeat/module/{beats => beat}/testing.go (98%) delete mode 100644 metricbeat/module/beats/_meta/env delete mode 100644 metricbeat/module/beats/docs.asciidoc rename metricbeat/modules.d/{beats-xpack.yml.disabled => beat-xpack.yml.disabled} (83%) rename metricbeat/modules.d/{beats.yml.disabled => beat.yml.disabled} (77%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 18680235bbe0..3ac69a7d1f3f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1571,20 +1571,20 @@ Time series instance id -- -[[exported-fields-beats]] -== Beats fields +[[exported-fields-beat]] +== Beat fields -Beats module +Beat module [float] -== beats fields +== beat fields -*`beats.id`*:: +*`beat.id`*:: + -- type: keyword @@ -1594,7 +1594,7 @@ Beat ID. -- -*`beats.type`*:: +*`beat.type`*:: + -- type: keyword @@ -1611,7 +1611,7 @@ Beat stats -*`beats.stats.uptime.ms`*:: +*`beat.stats.uptime.ms`*:: + -- type: long @@ -1621,7 +1621,7 @@ Beat uptime -- -*`beats.stats.runtime.goroutines`*:: +*`beat.stats.runtime.goroutines`*:: + -- type: long @@ -1645,7 +1645,7 @@ Output stats -*`beats.stats.libbeat.output.type`*:: +*`beat.stats.libbeat.output.type`*:: + -- type: keyword @@ -1662,7 +1662,7 @@ Event counters -*`beats.stats.libbeat.output.events.acked`*:: +*`beat.stats.libbeat.output.events.acked`*:: + -- type: long @@ -1672,7 +1672,7 @@ Number of events acknowledged -- -*`beats.stats.libbeat.output.events.active`*:: +*`beat.stats.libbeat.output.events.active`*:: + -- type: long @@ -1682,7 +1682,7 @@ Number of active events -- -*`beats.stats.libbeat.output.events.batches`*:: +*`beat.stats.libbeat.output.events.batches`*:: + -- type: long @@ -1692,7 +1692,7 @@ Number of event batches -- -*`beats.stats.libbeat.output.events.dropped`*:: +*`beat.stats.libbeat.output.events.dropped`*:: + -- type: long @@ -1702,7 +1702,7 @@ Number of events dropped -- -*`beats.stats.libbeat.output.events.duplicates`*:: +*`beat.stats.libbeat.output.events.duplicates`*:: + -- type: long @@ -1712,7 +1712,7 @@ Number of events duplicated -- -*`beats.stats.libbeat.output.events.failed`*:: +*`beat.stats.libbeat.output.events.failed`*:: + -- type: long @@ -1722,7 +1722,7 @@ Number of events failed -- -*`beats.stats.libbeat.output.events.toomany`*:: +*`beat.stats.libbeat.output.events.toomany`*:: + -- type: long @@ -1732,7 +1732,7 @@ Number of too many events -- -*`beats.stats.libbeat.output.events.total`*:: +*`beat.stats.libbeat.output.events.total`*:: + -- type: long @@ -1749,7 +1749,7 @@ Read stats -*`beats.stats.libbeat.output.read.bytes`*:: +*`beat.stats.libbeat.output.read.bytes`*:: + -- type: long @@ -1759,7 +1759,7 @@ Number of bytes read -- -*`beats.stats.libbeat.output.read.errors`*:: +*`beat.stats.libbeat.output.read.errors`*:: + -- type: long @@ -1776,7 +1776,7 @@ Write stats -*`beats.stats.libbeat.output.write.bytes`*:: +*`beat.stats.libbeat.output.write.bytes`*:: + -- type: long @@ -1786,7 +1786,7 @@ Number of bytes written -- -*`beats.stats.libbeat.output.write.errors`*:: +*`beat.stats.libbeat.output.write.errors`*:: + -- type: long diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 6f79d09c3ea5..84e7f21c6dab 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -30,8 +30,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/aerospike/namespace" _ "github.com/elastic/beats/metricbeat/module/apache" _ "github.com/elastic/beats/metricbeat/module/apache/status" - _ "github.com/elastic/beats/metricbeat/module/beats" - _ "github.com/elastic/beats/metricbeat/module/beats/stats" + _ "github.com/elastic/beats/metricbeat/module/beat" + _ "github.com/elastic/beats/metricbeat/module/beat/stats" _ "github.com/elastic/beats/metricbeat/module/ceph" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_health" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index a2845fa6b8d2..bebdeb832774 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -156,7 +156,7 @@ metricbeat.modules: # Password of hosts. Empty by default #password: password -#-------------------------------- Beats Module ------------------------------- +#-------------------------------- Beat Module -------------------------------- - module: beats metricsets: - stats diff --git a/metricbeat/module/beats/_meta/Dockerfile b/metricbeat/module/beat/_meta/Dockerfile similarity index 100% rename from metricbeat/module/beats/_meta/Dockerfile rename to metricbeat/module/beat/_meta/Dockerfile diff --git a/metricbeat/module/beats/_meta/config-xpack.yml b/metricbeat/module/beat/_meta/config-xpack.yml similarity index 100% rename from metricbeat/module/beats/_meta/config-xpack.yml rename to metricbeat/module/beat/_meta/config-xpack.yml diff --git a/metricbeat/module/beats/_meta/config.reference.yml b/metricbeat/module/beat/_meta/config.reference.yml similarity index 100% rename from metricbeat/module/beats/_meta/config.reference.yml rename to metricbeat/module/beat/_meta/config.reference.yml diff --git a/metricbeat/module/beats/_meta/config.yml b/metricbeat/module/beat/_meta/config.yml similarity index 100% rename from metricbeat/module/beats/_meta/config.yml rename to metricbeat/module/beat/_meta/config.yml diff --git a/metricbeat/module/beat/_meta/env b/metricbeat/module/beat/_meta/env new file mode 100644 index 000000000000..3e4c68f72f2e --- /dev/null +++ b/metricbeat/module/beat/_meta/env @@ -0,0 +1,2 @@ +BEAT_HOST=metricbeat +BEAT_PORT=5066 diff --git a/metricbeat/module/beats/_meta/fields.yml b/metricbeat/module/beat/_meta/fields.yml similarity index 83% rename from metricbeat/module/beats/_meta/fields.yml rename to metricbeat/module/beat/_meta/fields.yml index c718c5e5375d..d8140c26e141 100644 --- a/metricbeat/module/beats/_meta/fields.yml +++ b/metricbeat/module/beat/_meta/fields.yml @@ -1,12 +1,12 @@ -- key: beats - title: "Beats" +- key: beat + title: "Beat" description: > - Beats module + Beat module release: ga settings: ["ssl", "http"] short_config: false fields: - - name: beats + - name: beat type: group description: > fields: diff --git a/metricbeat/module/beats/_meta/healthcheck.sh b/metricbeat/module/beat/_meta/healthcheck.sh similarity index 100% rename from metricbeat/module/beats/_meta/healthcheck.sh rename to metricbeat/module/beat/_meta/healthcheck.sh diff --git a/metricbeat/module/beats/beats.go b/metricbeat/module/beat/beat.go similarity index 87% rename from metricbeat/module/beats/beats.go rename to metricbeat/module/beat/beat.go index 24227b0e2d52..59af78bdc63f 100644 --- a/metricbeat/module/beats/beats.go +++ b/metricbeat/module/beat/beat.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package beats +package beat import ( "encoding/json" @@ -25,9 +25,9 @@ import ( ) // ModuleName is the name of this module. -const ModuleName = "beats" +const ModuleName = "beat" -// Info construct contains the relevant data from the Beats / endpoint +// Info construct contains the relevant data from the Beat's / endpoint type Info struct { UUID string `json:"uuid"` Beat string `json:"beat"` @@ -36,7 +36,7 @@ type Info struct { Version string `json:"version"` } -// State construct contains the relevant data from the Beats /state endpoint +// State construct contains the relevant data from the Beat's /state endpoint type State struct { Outputs struct { Elasticsearch struct { @@ -45,7 +45,7 @@ type State struct { } `json:"outputs"` } -// GetInfo returns the data for the Beats / endpoint. +// GetInfo returns the data for the Beat's / endpoint. func GetInfo(m *MetricSet) (*Info, error) { content, err := fetchPath(m.HTTP, "/", "") if err != nil { @@ -61,7 +61,7 @@ func GetInfo(m *MetricSet) (*Info, error) { return info, nil } -// GetState returns the data for the Beats /state endpoint. +// GetState returns the data for the Beat's /state endpoint. func GetState(m *MetricSet) (*State, error) { content, err := fetchPath(m.HTTP, "/state", "") if err != nil { diff --git a/metricbeat/module/beats/beats_integration_test.go b/metricbeat/module/beat/beat_integration_test.go similarity index 81% rename from metricbeat/module/beats/beats_integration_test.go rename to metricbeat/module/beat/beat_integration_test.go index 593f7231194c..32a031b144a6 100644 --- a/metricbeat/module/beats/beats_integration_test.go +++ b/metricbeat/module/beat/beat_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package beats_test +package beat_test import ( "os" @@ -27,8 +27,8 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/beats" - _ "github.com/elastic/beats/metricbeat/module/beats/stats" + "github.com/elastic/beats/metricbeat/module/beat" + _ "github.com/elastic/beats/metricbeat/module/beat/stats" ) var metricSets = []string{ @@ -36,10 +36,10 @@ var metricSets = []string{ } func TestFetch(t *testing.T) { - compose.EnsureUp(t, "beats") + compose.EnsureUp(t, "beat") for _, metricSet := range metricSets { - f := mbtest.NewReportingMetricSetV2Error(t, beats.GetConfig(metricSet)) + f := mbtest.NewReportingMetricSetV2Error(t, beat.GetConfig(metricSet)) events, errs := mbtest.ReportingFetchV2Error(f) assert.Empty(t, errs) @@ -48,12 +48,12 @@ func TestFetch(t *testing.T) { } t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), - events[0].BeatEvent("beats", metricSet).Fields.StringToPrint()) + events[0].BeatEvent("beat", metricSet).Fields.StringToPrint()) } } func TestData(t *testing.T) { - compose.EnsureUp(t, "beats") + compose.EnsureUp(t, "beat") for _, metricSet := range metricSets { config := getConfig(metricSet) @@ -67,14 +67,14 @@ func TestData(t *testing.T) { func getConfig(metricset string) map[string]interface{} { return map[string]interface{}{ - "module": beats.ModuleName, + "module": beat.ModuleName, "metricsets": []string{metricset}, "hosts": []string{getEnvHost() + ":" + getEnvPort()}, } } func getEnvHost() string { - host := os.Getenv("BEATS_HOST") + host := os.Getenv("BEAT_HOST") if len(host) == 0 { host = "127.0.0.1" @@ -83,7 +83,7 @@ func getEnvHost() string { } func getEnvPort() string { - port := os.Getenv("BEATS_PORT") + port := os.Getenv("BEAT_PORT") if len(port) == 0 { port = "5066" diff --git a/metricbeat/module/beats/config.go b/metricbeat/module/beat/config.go similarity index 85% rename from metricbeat/module/beats/config.go rename to metricbeat/module/beat/config.go index 3cf0a6b554ac..83ea1879729a 100644 --- a/metricbeat/module/beats/config.go +++ b/metricbeat/module/beat/config.go @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -package beats +package beat -// Config defines the structure for the Beats module configuration options +// Config defines the structure for the Beat module configuration options type Config struct { XPackEnabled bool `config:"xpack.enabled"` } -// DefaultConfig returns the default configuration for the Beats module +// DefaultConfig returns the default configuration for the Beat module func DefaultConfig() Config { return Config{ XPackEnabled: false, diff --git a/metricbeat/module/beat/docs.asciidoc b/metricbeat/module/beat/docs.asciidoc new file mode 100644 index 000000000000..c3e066704765 --- /dev/null +++ b/metricbeat/module/beat/docs.asciidoc @@ -0,0 +1,9 @@ +The Beat module contains a minimal set of metrics to enable monitoring of any Beat or other software based on libbeat across +multiple versions. To monitor more Beat metrics, use our {stack-ov}/xpack-monitoring.html[monitoring] feature. + +The default metricset is `stats`. + +[float] +=== Compability + +The Beat module is tested with Metricbeat 7.2 and is expected to work with all 7.x versions. diff --git a/metricbeat/module/beats/fields.go b/metricbeat/module/beat/fields.go similarity index 52% rename from metricbeat/module/beats/fields.go rename to metricbeat/module/beat/fields.go index 6cf95e2440d6..ec8d2423860e 100644 --- a/metricbeat/module/beats/fields.go +++ b/metricbeat/module/beat/fields.go @@ -17,20 +17,20 @@ // Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. -package beats +package beat import ( "github.com/elastic/beats/libbeat/asset" ) func init() { - if err := asset.SetFields("metricbeat", "beats", asset.ModuleFieldsPri, AssetBeats); err != nil { + if err := asset.SetFields("metricbeat", "beat", asset.ModuleFieldsPri, AssetBeat); err != nil { panic(err) } } -// AssetBeats returns asset data. -// This is the base64 encoded gzipped contents of ../metricbeat/module/beats. -func AssetBeats() string { - return "eJzcls+OmzAQxu95ihHn7j4Ahx6qtlIvrVSt1ENVVQ4MxIrxWPZ4I96+MpRA+GOSKrtU61Nih+/78Q3D5AGOWKewR8FuB8CSFaaQfAjfkx1Aji6z0rAkncL7HQBAcwYV5V7hDsCiQuEwhVLsABwyS126FH4mzqnkHSQHZpP8CmcHsvw7I13IMoVCKBeuLySq3KWN9ANoUWGPExbXJohb8ubvzgzTpcxQSubnrU7riPWJ7HB/VrFd4W7hy8fHiXDQuoN0uG4q7rgPYD6Ea7THKpel6tY4uSGHNywrfKzcxWnHo0iXo4MI0hmrFZ31s143hiVZ8iw13sv4q6/2aIEK6KWDm5a6BKkbsFkiJff78dlSPa7g+NyEDRlVFWlgAqFU21Gjn85VZchFno0fY8XRrsAL61ujPHl41sBguTemeNM+uREyrKfaYKjoYhQ9ED6jnr0bWE3sBqBPwQUy8prRLrnF8hsii+yISwFBrBP+ARwuOqQNKwBoOinMywhHT8vyeanmL4vbWsdLPBgtgrPD5LXyOqQN4ipBR5pbMmbbR2AN4YzqjZKZ4C1zdT3FOnAhpNo22hWC85uUqBK63oSUiSCYX9taTCzUK5I+BT/Qo2Sjc8CiWMr8blPgO4p8cXrCDRNgX2/VUI1zLKzBZLWWFmfdy1IGvph/h3iykuN/R+5Q9R/B5E2UPcTFqP/nyjcVHQPEPvwJAAD//wysTmY=" +// AssetBeat returns asset data. +// This is the base64 encoded gzipped contents of ../metricbeat/module/beat. +func AssetBeat() string { + return "eJzcls+K2zAQxu95isHn7j6ADz2UttBLC2Whh1KKYk8cEVkjpNEGv32RUjuO/8hJya7L6hQk/H0/fePx5AEO2OSwRcEbAJasMIfsAwrONgAlusJKw5J0Du83AADhCGoqvcINgEWFwmEOldgAOGSWunI5/MycU9k7yPbMJvsVzvZk+XdBeierHHZCufD8TqIqXR6VH0CLGjuWsLgxQduSN393JoguVfpKsuy2Wq0DNkey/f1JxdOKd/3y8XEkHLTuIB2eG4s7FuxG6v0QrtEeqlxWql3D5Poc3rCs8bF2F6ctjyJdDQ4SSB3WSXTSz3odDSuy5FlqvJfxV19v0QLt4Cwd3LTUFUgdwSaJlNxuh2dz9biC43MMGwqqa9LABEKpaD686FRV+lzk2fghVhrtCrywvkXl0cuzBAbzvTHGG/fJjZBhPTUGQ0VnozgD4TPqydvAYmI3AH0KLlCQ14x2zi2VXx9ZFAecCwhSnfAP4HDRIaewAoCmo8KySnCcaVk+z9X8ZXFP1ukS9yaL4GI/+qy8DmlEXCRoSUtLxqz7CiwhdKjeKFkIXjNXd6ZYBt4JqdaNdoGg+5IS1UI3q5AyEQTza1uLiYV6RdKn4Ad6kGxyDlgUc5nfbQp8R1HOTk+4YQJsm7UaKjqnwupNVmtpdta9LGXgS/m3iEcrOf135A5V/xFM3kTZQ1yM+n+ufKzoECD1408AAAD//2l5TJo=" } diff --git a/metricbeat/module/beats/metricset.go b/metricbeat/module/beat/metricset.go similarity index 96% rename from metricbeat/module/beats/metricset.go rename to metricbeat/module/beat/metricset.go index 2548968deba5..ba4957b47c68 100644 --- a/metricbeat/module/beats/metricset.go +++ b/metricbeat/module/beat/metricset.go @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -package beats +package beat import ( "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" ) -// MetricSet can be used to build other metricsets within the Beats module. +// MetricSet can be used to build other metricsets within the Beat module. type MetricSet struct { mb.BaseMetricSet *helper.HTTP @@ -30,7 +30,7 @@ type MetricSet struct { } // NewMetricSet creates a metricset that can be used to build other metricsets -// within the Beats module. +// within the Beat module. func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { config := DefaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { diff --git a/metricbeat/module/beats/stats/_meta/data.json b/metricbeat/module/beat/stats/_meta/data.json similarity index 100% rename from metricbeat/module/beats/stats/_meta/data.json rename to metricbeat/module/beat/stats/_meta/data.json diff --git a/metricbeat/module/beats/stats/_meta/fields.yml b/metricbeat/module/beat/stats/_meta/fields.yml similarity index 100% rename from metricbeat/module/beats/stats/_meta/fields.yml rename to metricbeat/module/beat/stats/_meta/fields.yml diff --git a/metricbeat/module/beats/stats/_meta/test/stats.800.json b/metricbeat/module/beat/stats/_meta/test/stats.800.json similarity index 100% rename from metricbeat/module/beats/stats/_meta/test/stats.800.json rename to metricbeat/module/beat/stats/_meta/test/stats.800.json diff --git a/metricbeat/module/beats/stats/data.go b/metricbeat/module/beat/stats/data.go similarity index 90% rename from metricbeat/module/beats/stats/data.go rename to metricbeat/module/beat/stats/data.go index c200f725f694..aaa8bd686005 100644 --- a/metricbeat/module/beats/stats/data.go +++ b/metricbeat/module/beat/stats/data.go @@ -26,7 +26,7 @@ import ( s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/beats" + "github.com/elastic/beats/metricbeat/module/beat" ) var ( @@ -63,10 +63,10 @@ var ( } ) -func eventMapping(r mb.ReporterV2, info beats.Info, content []byte) error { +func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error { var event mb.Event event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", beats.ModuleName) + event.RootFields.Put("service.name", beat.ModuleName) event.ModuleFields = common.MapStr{} event.ModuleFields.Put("id", info.UUID) @@ -75,7 +75,7 @@ func eventMapping(r mb.ReporterV2, info beats.Info, content []byte) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { - return errors.Wrap(err, "failure parsing Beats Stats API response") + return errors.Wrap(err, "failure parsing Beat's Stats API response") } event.MetricSetFields, err = schema.Apply(data) diff --git a/metricbeat/module/beats/stats/data_test.go b/metricbeat/module/beat/stats/data_test.go similarity index 95% rename from metricbeat/module/beats/stats/data_test.go rename to metricbeat/module/beat/stats/data_test.go index 7dc073d01ecf..dd73d5fff06c 100644 --- a/metricbeat/module/beats/stats/data_test.go +++ b/metricbeat/module/beat/stats/data_test.go @@ -24,7 +24,7 @@ import ( "path/filepath" "testing" - "github.com/elastic/beats/metricbeat/module/beats" + "github.com/elastic/beats/metricbeat/module/beat" mbtest "github.com/elastic/beats/metricbeat/mb/testing" @@ -36,7 +36,7 @@ func TestEventMapping(t *testing.T) { files, err := filepath.Glob("./_meta/test/stats.*.json") assert.NoError(t, err) - info := beats.Info{ + info := beat.Info{ UUID: "1234", Beat: "helloworld", } diff --git a/metricbeat/module/beats/stats/data_xpack.go b/metricbeat/module/beat/stats/data_xpack.go similarity index 89% rename from metricbeat/module/beats/stats/data_xpack.go rename to metricbeat/module/beat/stats/data_xpack.go index 3c94050af813..2c75d7c6d438 100644 --- a/metricbeat/module/beats/stats/data_xpack.go +++ b/metricbeat/module/beat/stats/data_xpack.go @@ -27,10 +27,10 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/beats" + "github.com/elastic/beats/metricbeat/module/beat" ) -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beats.Info, content []byte) error { +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beat.Info, content []byte) error { now := time.Now() clusterUUID, err := m.getClusterUUID() if err != nil { @@ -49,7 +49,7 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beats.Info, content [ var metrics map[string]interface{} err = json.Unmarshal(content, &metrics) if err != nil { - return errors.Wrap(err, "failure parsing Beats Stats API response") + return errors.Wrap(err, "failure parsing Beat's Stats API response") } fields := common.MapStr{ @@ -78,7 +78,7 @@ func (m *MetricSet) calculateIntervalMs() int64 { } func (m *MetricSet) getClusterUUID() (string, error) { - state, err := beats.GetState(m.MetricSet) + state, err := beat.GetState(m.MetricSet) if err != nil { return "", errors.Wrap(err, "could not get state information") } diff --git a/metricbeat/module/beats/stats/stats.go b/metricbeat/module/beat/stats/stats.go similarity index 90% rename from metricbeat/module/beats/stats/stats.go rename to metricbeat/module/beat/stats/stats.go index c3205247763e..fcab548c5a37 100644 --- a/metricbeat/module/beats/stats/stats.go +++ b/metricbeat/module/beat/stats/stats.go @@ -20,11 +20,11 @@ package stats import ( "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" - "github.com/elastic/beats/metricbeat/module/beats" + "github.com/elastic/beats/metricbeat/module/beat" ) func init() { - mb.Registry.MustAddMetricSet(beats.ModuleName, "stats", New, + mb.Registry.MustAddMetricSet(beat.ModuleName, "stats", New, mb.WithHostParser(hostParser), ) } @@ -42,12 +42,12 @@ var ( // MetricSet defines all fields of the MetricSet type MetricSet struct { - *beats.MetricSet + *beat.MetricSet } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - ms, err := beats.NewMetricSet(base) + ms, err := beat.NewMetricSet(base) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - info, err := beats.GetInfo(m.MetricSet) + info, err := beat.GetInfo(m.MetricSet) if err != nil { return err } diff --git a/metricbeat/module/beats/testing.go b/metricbeat/module/beat/testing.go similarity index 98% rename from metricbeat/module/beats/testing.go rename to metricbeat/module/beat/testing.go index 8e41a4e3de6d..9c3e49c13f29 100644 --- a/metricbeat/module/beats/testing.go +++ b/metricbeat/module/beat/testing.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package beats +package beat import "os" diff --git a/metricbeat/module/beats/_meta/env b/metricbeat/module/beats/_meta/env deleted file mode 100644 index 5e4a55fde9cb..000000000000 --- a/metricbeat/module/beats/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -BEATS_HOST=metricbeat -BEATS_PORT=5066 diff --git a/metricbeat/module/beats/docs.asciidoc b/metricbeat/module/beats/docs.asciidoc deleted file mode 100644 index b56f5f6291bd..000000000000 --- a/metricbeat/module/beats/docs.asciidoc +++ /dev/null @@ -1,9 +0,0 @@ -The Beats module contains a minimal set of metrics to enable monitoring of any Beat or other software based on libbeat across -multiple versions. To monitor more Beats metrics, use our {stack-ov}/xpack-monitoring.html[monitoring] feature. - -The default metricset is `stats`. - -[float] -=== Compability - -The Beats module is tested with Metricbeat 7.2 and is expected to work with all 7.x versions. diff --git a/metricbeat/modules.d/beats-xpack.yml.disabled b/metricbeat/modules.d/beat-xpack.yml.disabled similarity index 83% rename from metricbeat/modules.d/beats-xpack.yml.disabled rename to metricbeat/modules.d/beat-xpack.yml.disabled index 4426a9ffacfb..91e29819f400 100644 --- a/metricbeat/modules.d/beats-xpack.yml.disabled +++ b/metricbeat/modules.d/beat-xpack.yml.disabled @@ -1,5 +1,5 @@ -# Module: beats -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beats.html +# Module: beat +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beat.html - module: beats metricsets: diff --git a/metricbeat/modules.d/beats.yml.disabled b/metricbeat/modules.d/beat.yml.disabled similarity index 77% rename from metricbeat/modules.d/beats.yml.disabled rename to metricbeat/modules.d/beat.yml.disabled index bc3ff7aaa15d..4f56f97f608e 100644 --- a/metricbeat/modules.d/beats.yml.disabled +++ b/metricbeat/modules.d/beat.yml.disabled @@ -1,5 +1,5 @@ -# Module: beats -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beats.html +# Module: beat +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beat.html - module: beats metricsets: From 3e9ff212543a51fea4bc47e680e9edb087e0a4f2 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 06:23:35 -0700 Subject: [PATCH 16/37] Regenerating data.json file --- metricbeat/module/beat/stats/_meta/data.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/beat/stats/_meta/data.json b/metricbeat/module/beat/stats/_meta/data.json index d0e48a2c8f79..318fc80d7be5 100644 --- a/metricbeat/module/beat/stats/_meta/data.json +++ b/metricbeat/module/beat/stats/_meta/data.json @@ -1,6 +1,6 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beats": { + "beat": { "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", "stats": { "libbeat": { @@ -27,25 +27,25 @@ } }, "runtime": { - "goroutines": 64 + "goroutines": 39 }, "uptime": { - "ms": 1631 + "ms": 12019 } }, "type": "metricbeat" }, "event": { - "dataset": "beats.stats", + "dataset": "beat.stats", "duration": 115000, - "module": "beats" + "module": "beat" }, "metricset": { "name": "stats" }, "service": { "address": "127.0.0.1:5066", - "name": "beats", - "type": "beats" + "name": "beat", + "type": "beat" } } \ No newline at end of file From b202ae50c389a06befc9f73b24accfdf29db9fa3 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 06:59:17 -0700 Subject: [PATCH 17/37] Update metricbeat/docker-compose.yml Co-Authored-By: Nicolas Ruflin --- metricbeat/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 24c8de4327b3..1c313c70a27c 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -14,7 +14,7 @@ services: env_file: - ./module/aerospike/_meta/env - ./module/apache/_meta/env - - ./module/beats/_meta/env + - ./module/beat/_meta/env - ./module/ceph/_meta/env - ./module/consul/_meta/env - ./module/couchbase/_meta/env From 7360ebefff4a7e9cfd0d7cdb94e18be76520c742 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 06:59:31 -0700 Subject: [PATCH 18/37] Update metricbeat/module/beat/_meta/config.reference.yml Co-Authored-By: Nicolas Ruflin --- metricbeat/module/beat/_meta/config.reference.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/_meta/config.reference.yml b/metricbeat/module/beat/_meta/config.reference.yml index 8d723ba37c7e..a82db61430a5 100644 --- a/metricbeat/module/beat/_meta/config.reference.yml +++ b/metricbeat/module/beat/_meta/config.reference.yml @@ -1,4 +1,4 @@ -- module: beats +- module: beat metricsets: - stats period: 10s From c9c50d4d0bd0a1cb2b736a792c8de90e916686b9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 06:59:39 -0700 Subject: [PATCH 19/37] Update metricbeat/metricbeat.reference.yml Co-Authored-By: Nicolas Ruflin --- metricbeat/metricbeat.reference.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index bebdeb832774..4514890d3b34 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -157,7 +157,7 @@ metricbeat.modules: #password: password #-------------------------------- Beat Module -------------------------------- -- module: beats +- module: beat metricsets: - stats period: 10s From 0d1944d251d42db9413f4d82f16e944c9873e67b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 06:59:56 -0700 Subject: [PATCH 20/37] Update metricbeat/docker-compose.yml Co-Authored-By: Nicolas Ruflin --- metricbeat/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 1c313c70a27c..8c8475903e06 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -65,7 +65,7 @@ services: - 80 beats: - build: ./module/beats/_meta + build: ./module/beat/_meta ports: - 5066 From c5c954ba1b6c4f7c3d5e339234cddbe50761c656 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:00:22 -0700 Subject: [PATCH 21/37] Update metricbeat/module/beat/_meta/config-xpack.yml Co-Authored-By: Nicolas Ruflin --- metricbeat/module/beat/_meta/config-xpack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/_meta/config-xpack.yml b/metricbeat/module/beat/_meta/config-xpack.yml index 8b7ec8b55e01..011e01b04ada 100644 --- a/metricbeat/module/beat/_meta/config-xpack.yml +++ b/metricbeat/module/beat/_meta/config-xpack.yml @@ -1,4 +1,4 @@ -- module: beats +- module: beat metricsets: - stats period: 10s From 3189e4c11de548269017eebe5b3dd6248086d4c2 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:00:30 -0700 Subject: [PATCH 22/37] Update metricbeat/module/beat/_meta/config.yml Co-Authored-By: Nicolas Ruflin --- metricbeat/module/beat/_meta/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/_meta/config.yml b/metricbeat/module/beat/_meta/config.yml index 7ce8653a26b3..1b7457a89898 100644 --- a/metricbeat/module/beat/_meta/config.yml +++ b/metricbeat/module/beat/_meta/config.yml @@ -1,4 +1,4 @@ -- module: beats +- module: beat metricsets: - stats period: 10s From a9d2fec24bc538f56c97b557874871949c64fa5c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:00:56 -0700 Subject: [PATCH 23/37] Update metricbeat/docker-compose.yml --- metricbeat/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 8c8475903e06..1291f8a9f95e 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -64,7 +64,7 @@ services: ports: - 80 - beats: + beat: build: ./module/beat/_meta ports: - 5066 From 56ebe7cc894e8a93400e007c67e6137e4f24cfe1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:01:37 -0700 Subject: [PATCH 24/37] Update metricbeat/modules.d/beat-xpack.yml.disabled --- metricbeat/modules.d/beat-xpack.yml.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/modules.d/beat-xpack.yml.disabled b/metricbeat/modules.d/beat-xpack.yml.disabled index 91e29819f400..e941a5257876 100644 --- a/metricbeat/modules.d/beat-xpack.yml.disabled +++ b/metricbeat/modules.d/beat-xpack.yml.disabled @@ -1,7 +1,7 @@ # Module: beat # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beat.html -- module: beats +- module: beat metricsets: - stats period: 10s From aa56839e83886b638eb42bf95690a19c501f97f6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:02:14 -0700 Subject: [PATCH 25/37] Update x-pack/metricbeat/metricbeat.reference.yml --- x-pack/metricbeat/metricbeat.reference.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index df2487c618b7..0fc70dd75567 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -206,7 +206,7 @@ metricbeat.modules: # - us-east-2 #-------------------------------- Beats Module -------------------------------- -- module: beats +- module: beat metricsets: - stats period: 10s From 3198cd37a90536522297d15eee8df29426f8e82a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:02:30 -0700 Subject: [PATCH 26/37] Update metricbeat/modules.d/beat.yml.disabled --- metricbeat/modules.d/beat.yml.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/modules.d/beat.yml.disabled b/metricbeat/modules.d/beat.yml.disabled index 4f56f97f608e..ed31130210a4 100644 --- a/metricbeat/modules.d/beat.yml.disabled +++ b/metricbeat/modules.d/beat.yml.disabled @@ -1,7 +1,7 @@ # Module: beat # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beat.html -- module: beats +- module: beat metricsets: - stats period: 10s From 1626576341e03a72bbf00879cce50b72ba8906e6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:04:21 -0700 Subject: [PATCH 27/37] Renaming system test --- metricbeat/tests/system/{test_beats.py => test_beat.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename metricbeat/tests/system/{test_beats.py => test_beat.py} (88%) diff --git a/metricbeat/tests/system/test_beats.py b/metricbeat/tests/system/test_beat.py similarity index 88% rename from metricbeat/tests/system/test_beats.py rename to metricbeat/tests/system/test_beat.py index 6c9653d7c57a..e8b486bfbf64 100644 --- a/metricbeat/tests/system/test_beats.py +++ b/metricbeat/tests/system/test_beat.py @@ -6,15 +6,15 @@ class Test(metricbeat.BaseTest): - COMPOSE_SERVICES = ['beats'] + COMPOSE_SERVICES = ['beat'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_stats(self): """ - beats stats metricset test + beat stats metricset test """ self.render_config_template(modules=[{ - "name": "beats", + "name": "beat", "metricsets": ["stats"], "hosts": self.get_hosts(), "period": "1s", From d09091bf1b62f65ad083aa3e7fe02c7a4d218c77 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:04:31 -0700 Subject: [PATCH 28/37] Regenerating files --- x-pack/metricbeat/metricbeat.reference.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 0fc70dd75567..c6932be644c6 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -205,7 +205,7 @@ metricbeat.modules: # - us-east-1 # - us-east-2 -#-------------------------------- Beats Module -------------------------------- +#--------------------------------- Beat Module --------------------------------- - module: beat metricsets: - stats From 82fcefbd8b1d3c6abe87e08718ddfeb00f412cba Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 07:56:02 -0700 Subject: [PATCH 29/37] Trying to use the new testdata framework --- .../beat/stats/_meta/testdata/config.yml | 2 + .../beat/stats/_meta/testdata/stats.800.json | 155 ++++++++++++++++++ .../testdata/stats.800.json-expected.json | 19 +++ metricbeat/module/beat/stats/data_test.go | 55 ------- 4 files changed, 176 insertions(+), 55 deletions(-) create mode 100644 metricbeat/module/beat/stats/_meta/testdata/config.yml create mode 100644 metricbeat/module/beat/stats/_meta/testdata/stats.800.json create mode 100644 metricbeat/module/beat/stats/_meta/testdata/stats.800.json-expected.json delete mode 100644 metricbeat/module/beat/stats/data_test.go diff --git a/metricbeat/module/beat/stats/_meta/testdata/config.yml b/metricbeat/module/beat/stats/_meta/testdata/config.yml new file mode 100644 index 000000000000..9f0b7e9baa98 --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/testdata/config.yml @@ -0,0 +1,2 @@ +type: http +url: "/stats" diff --git a/metricbeat/module/beat/stats/_meta/testdata/stats.800.json b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json new file mode 100644 index 000000000000..ced95cd76e3f --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json @@ -0,0 +1,155 @@ +{ + "beat": { + "cpu": { + "system": { + "ticks": 95, + "time": { + "ms": 95 + } + }, + "total": { + "ticks": 458, + "time": { + "ms": 458 + }, + "value": 458 + }, + "user": { + "ticks": 363, + "time": { + "ms": 363 + } + } + }, + "info": { + "ephemeral_id": "f32e9a62-56c2-4cde-87be-de5869b2d7b7", + "uptime": { + "ms": 21557 + } + }, + "memstats": { + "gc_next": 10723216, + "memory_alloc": 6145008, + "memory_total": 360839144, + "rss": 44339200 + }, + "runtime": { + "goroutines": 40 + } + }, + "libbeat": { + "config": { + "module": { + "running": 0, + "starts": 0, + "stops": 0 + }, + "reloads": 1 + }, + "output": { + "events": { + "acked": 0, + "active": 0, + "batches": 0, + "dropped": 0, + "duplicates": 0, + "failed": 0, + "toomany": 0, + "total": 0 + }, + "read": { + "bytes": 0, + "errors": 0 + }, + "type": "elasticsearch", + "write": { + "bytes": 0, + "errors": 0 + } + }, + "pipeline": { + "clients": 3, + "events": { + "active": 37, + "dropped": 0, + "failed": 0, + "filtered": 1, + "published": 37, + "retry": 69, + "total": 38 + }, + "queue": { + "acked": 0 + } + } + }, + "metricbeat": { + "system": { + "cpu": { + "events": 2, + "failures": 0, + "success": 2 + }, + "filesystem": { + "events": 7, + "failures": 0, + "success": 7 + }, + "fsstat": { + "events": 1, + "failures": 0, + "success": 1 + }, + "load": { + "events": 3, + "failures": 0, + "success": 3 + }, + "memory": { + "events": 3, + "failures": 0, + "success": 3 + }, + "network": { + "events": 0, + "failures": 0, + "success": 0 + }, + "process": { + "events": 17, + "failures": 0, + "success": 17 + }, + "process_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "socket_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "uptime": { + "events": 1, + "failures": 0, + "success": 1 + } + } + }, + "system": { + "cpu": { + "cores": 8 + }, + "load": { + "1": 1.6753, + "15": 1.8076, + "5": 1.8257, + "norm": { + "1": 0.2094, + "15": 0.226, + "5": 0.2282 + } + } + } +} diff --git a/metricbeat/module/beat/stats/_meta/testdata/stats.800.json-expected.json b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json-expected.json new file mode 100644 index 000000000000..1f43330729b8 --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json-expected.json @@ -0,0 +1,19 @@ +[ + { + "error": { + "message": "HTTP error 404 in : 404 Not Found" + }, + "event": { + "dataset": "beat.stats", + "duration": 115000, + "module": "beat" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "beat" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/beat/stats/data_test.go b/metricbeat/module/beat/stats/data_test.go deleted file mode 100644 index dd73d5fff06c..000000000000 --- a/metricbeat/module/beat/stats/data_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package stats - -import ( - "io/ioutil" - "path/filepath" - "testing" - - "github.com/elastic/beats/metricbeat/module/beat" - - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" -) - -func TestEventMapping(t *testing.T) { - - files, err := filepath.Glob("./_meta/test/stats.*.json") - assert.NoError(t, err) - - info := beat.Info{ - UUID: "1234", - Beat: "helloworld", - } - - for _, f := range files { - input, err := ioutil.ReadFile(f) - assert.NoError(t, err) - - reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, info, input) - - assert.NoError(t, err, f) - assert.True(t, len(reporter.GetEvents()) >= 1, f) - assert.Equal(t, 0, len(reporter.GetErrors()), f) - } -} From 06faf3292510d8e9391e3b4a2711311121cb9a06 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 10:45:36 -0700 Subject: [PATCH 30/37] Renaming service to avoid naming collision --- metricbeat/docker-compose.yml | 2 +- metricbeat/module/beat/beat_integration_test.go | 4 ++-- metricbeat/tests/system/test_beat.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 1291f8a9f95e..c61291b387b6 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -64,7 +64,7 @@ services: ports: - 80 - beat: + metricbeat: build: ./module/beat/_meta ports: - 5066 diff --git a/metricbeat/module/beat/beat_integration_test.go b/metricbeat/module/beat/beat_integration_test.go index 32a031b144a6..404b9245be6b 100644 --- a/metricbeat/module/beat/beat_integration_test.go +++ b/metricbeat/module/beat/beat_integration_test.go @@ -36,7 +36,7 @@ var metricSets = []string{ } func TestFetch(t *testing.T) { - compose.EnsureUp(t, "beat") + compose.EnsureUp(t, "metricbeat") for _, metricSet := range metricSets { f := mbtest.NewReportingMetricSetV2Error(t, beat.GetConfig(metricSet)) @@ -53,7 +53,7 @@ func TestFetch(t *testing.T) { } func TestData(t *testing.T) { - compose.EnsureUp(t, "beat") + compose.EnsureUp(t, "metricbeat") for _, metricSet := range metricSets { config := getConfig(metricSet) diff --git a/metricbeat/tests/system/test_beat.py b/metricbeat/tests/system/test_beat.py index e8b486bfbf64..42ccb9dd8e04 100644 --- a/metricbeat/tests/system/test_beat.py +++ b/metricbeat/tests/system/test_beat.py @@ -6,7 +6,7 @@ class Test(metricbeat.BaseTest): - COMPOSE_SERVICES = ['beat'] + COMPOSE_SERVICES = ['metricbeat'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_stats(self): From 71a8cb01a1f59c92954f1558c343970286c54e94 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 11:26:53 -0700 Subject: [PATCH 31/37] Adding back data_test.go --- metricbeat/module/beat/stats/data_test.go | 55 +++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 metricbeat/module/beat/stats/data_test.go diff --git a/metricbeat/module/beat/stats/data_test.go b/metricbeat/module/beat/stats/data_test.go new file mode 100644 index 000000000000..dd73d5fff06c --- /dev/null +++ b/metricbeat/module/beat/stats/data_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package stats + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/elastic/beats/metricbeat/module/beat" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapping(t *testing.T) { + + files, err := filepath.Glob("./_meta/test/stats.*.json") + assert.NoError(t, err) + + info := beat.Info{ + UUID: "1234", + Beat: "helloworld", + } + + for _, f := range files { + input, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = eventMapping(reporter, info, input) + + assert.NoError(t, err, f) + assert.True(t, len(reporter.GetEvents()) >= 1, f) + assert.Equal(t, 0, len(reporter.GetErrors()), f) + } +} From ea3756494d712b4255c9432ef6daaf66a2bbc258 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 12:25:13 -0700 Subject: [PATCH 32/37] Fixing HTTP server hostname --- metricbeat/module/beat/_meta/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/_meta/Dockerfile b/metricbeat/module/beat/_meta/Dockerfile index 6c959bf542b2..44b84c00b907 100644 --- a/metricbeat/module/beat/_meta/Dockerfile +++ b/metricbeat/module/beat/_meta/Dockerfile @@ -3,4 +3,4 @@ FROM docker.elastic.co/beats/metricbeat:7.0.0 COPY healthcheck.sh / HEALTHCHECK --interval=1s --retries=300 CMD sh /healthcheck.sh -ENTRYPOINT [ "metricbeat", "-E", "http.enabled=true", "-E", "http.host=127.0.0.1" ] +ENTRYPOINT [ "metricbeat", "-E", "http.enabled=true", "-E", "http.host=0.0.0.0" ] From 9f8f2d0a294439b875e78238474dbe9bf0019239 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 13:15:08 -0700 Subject: [PATCH 33/37] Trying localhost --- metricbeat/module/beat/beat_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/beat_integration_test.go b/metricbeat/module/beat/beat_integration_test.go index 404b9245be6b..d5bb0b3c63da 100644 --- a/metricbeat/module/beat/beat_integration_test.go +++ b/metricbeat/module/beat/beat_integration_test.go @@ -77,7 +77,7 @@ func getEnvHost() string { host := os.Getenv("BEAT_HOST") if len(host) == 0 { - host = "127.0.0.1" + host = "localhost" } return host } From 48a696d2098274df260be33297e9afe3dbc12643 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 14:31:24 -0700 Subject: [PATCH 34/37] Fixing inconsistencies --- .../module/beat/beat_integration_test.go | 30 +------------------ metricbeat/module/beat/testing.go | 4 +-- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/metricbeat/module/beat/beat_integration_test.go b/metricbeat/module/beat/beat_integration_test.go index d5bb0b3c63da..ec91196fdd68 100644 --- a/metricbeat/module/beat/beat_integration_test.go +++ b/metricbeat/module/beat/beat_integration_test.go @@ -20,7 +20,6 @@ package beat_test import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -56,37 +55,10 @@ func TestData(t *testing.T) { compose.EnsureUp(t, "metricbeat") for _, metricSet := range metricSets { - config := getConfig(metricSet) - f := mbtest.NewReportingMetricSetV2Error(t, config) + f := mbtest.NewReportingMetricSetV2Error(t, beat.GetConfig(metricSet)) err := mbtest.WriteEventsReporterV2Error(f, t, metricSet) if err != nil { t.Fatal("write", err) } } } - -func getConfig(metricset string) map[string]interface{} { - return map[string]interface{}{ - "module": beat.ModuleName, - "metricsets": []string{metricset}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, - } -} - -func getEnvHost() string { - host := os.Getenv("BEAT_HOST") - - if len(host) == 0 { - host = "localhost" - } - return host -} - -func getEnvPort() string { - port := os.Getenv("BEAT_PORT") - - if len(port) == 0 { - port = "5066" - } - return port -} diff --git a/metricbeat/module/beat/testing.go b/metricbeat/module/beat/testing.go index 9c3e49c13f29..5c03b7449ed2 100644 --- a/metricbeat/module/beat/testing.go +++ b/metricbeat/module/beat/testing.go @@ -21,7 +21,7 @@ import "os" // GetEnvHost for Metricbeat func GetEnvHost() string { - host := os.Getenv("METRICBEAT_HOST") + host := os.Getenv("BEAT_HOST") if len(host) == 0 { host = "127.0.0.1" @@ -31,7 +31,7 @@ func GetEnvHost() string { // GetEnvPort for Metricbeat func GetEnvPort() string { - port := os.Getenv("METRICBEAT_HOST") + port := os.Getenv("BEAT_HOST") if len(port) == 0 { port = "5066" From 90417739b28ea758550eb83d8c38ac90bc90da0c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 15:14:56 -0700 Subject: [PATCH 35/37] Fixing typo --- metricbeat/module/beat/testing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/testing.go b/metricbeat/module/beat/testing.go index 5c03b7449ed2..5a282228d05a 100644 --- a/metricbeat/module/beat/testing.go +++ b/metricbeat/module/beat/testing.go @@ -31,7 +31,7 @@ func GetEnvHost() string { // GetEnvPort for Metricbeat func GetEnvPort() string { - port := os.Getenv("BEAT_HOST") + port := os.Getenv("BEAT_PORT") if len(port) == 0 { port = "5066" From d7c128b002ac187d02f68a8e4ec530601203c710 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 15:54:53 -0700 Subject: [PATCH 36/37] Make runtime optional as it is absent in older versions of Beats --- metricbeat/module/beat/stats/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/stats/data.go b/metricbeat/module/beat/stats/data.go index aaa8bd686005..8c868e1d5b29 100644 --- a/metricbeat/module/beat/stats/data.go +++ b/metricbeat/module/beat/stats/data.go @@ -36,7 +36,7 @@ var ( }), "runtime": c.Dict("beat.runtime", s.Schema{ "goroutines": c.Int("goroutines"), - }), + }, c.DictOptional), "libbeat": c.Dict("libbeat", s.Schema{ "output": c.Dict("output", s.Schema{ "type": c.Str("type"), From 6e4b903462fdedc1bb175bfb2109df3c1e61f355 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 19 Jun 2019 18:08:36 -0700 Subject: [PATCH 37/37] Updates from make update --- metricbeat/docs/fields.asciidoc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3ac69a7d1f3f..30842e04c89f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19899,7 +19899,7 @@ The number of requests to read a row based on a key. -- type: long -The number of requests to read the last key in an index. +The number of requests to read the last key in an index. -- @@ -19929,7 +19929,7 @@ The number of requests to read the previous row in key order. -- type: long -The number of requests to read a row based on a fixed position. +The number of requests to read a row based on a fixed position. -- @@ -19939,7 +19939,7 @@ The number of requests to read a row based on a fixed position. -- type: long -The number of requests to read the next row in the data file. +The number of requests to read the next row in the data file. -- @@ -20037,7 +20037,7 @@ The progress of an operation to warm up the InnoDB buffer pool by reading in a s -- type: long -The total number of bytes in the InnoDB buffer pool containing data. +The total number of bytes in the InnoDB buffer pool containing data. -- @@ -22924,7 +22924,7 @@ type: long format: bytes -Used memory by the Lua engine. +Used memory by the Lua engine. -- @@ -22936,7 +22936,7 @@ type: long format: bytes -The size in bytes of the dataset +The size in bytes of the dataset -- @@ -23181,7 +23181,7 @@ type: long format: bytes -The size in bytes of copy-on-write allocations during the last RBD save operation +The size in bytes of copy-on-write allocations during the last RBD save operation -- @@ -23310,7 +23310,7 @@ type: long format: bytes -AOF current file size +AOF current file size -- @@ -23414,7 +23414,7 @@ Total size in bytes of the replication backlog buffer -- type: long -The master offset of the replication backlog buffer +The master offset of the replication backlog buffer -- @@ -26463,7 +26463,7 @@ The raid level of the device -- type: keyword -Current sync action, if the RAID array is redundant +Current sync action, if the RAID array is redundant -- @@ -26748,7 +26748,7 @@ type: integer format: bytes -Memory used by TCP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/tcp_mem. Only available on Linux. +Memory used by TCP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/tcp_mem. Only available on Linux. -- @@ -26834,7 +26834,7 @@ type: integer format: bytes -Memory used by UDP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/udp_mem. Only available on Linux. +Memory used by UDP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/udp_mem. Only available on Linux. --