diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 939b356c15b5..c61291b387b6 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -14,6 +14,7 @@ services: env_file: - ./module/aerospike/_meta/env - ./module/apache/_meta/env + - ./module/beat/_meta/env - ./module/ceph/_meta/env - ./module/consul/_meta/env - ./module/couchbase/_meta/env @@ -63,6 +64,11 @@ services: ports: - 80 + metricbeat: + build: ./module/beat/_meta + ports: + - 5066 + ceph: build: ./module/ceph/_meta ports: diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e60444157e98..30842e04c89f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1568,6 +1569,231 @@ type: keyword Time series instance id +-- + +[[exported-fields-beat]] +== Beat fields + +Beat module + + + +[float] +== beat fields + + + + +*`beat.id`*:: ++ +-- +type: keyword + +Beat ID. + + +-- + +*`beat.type`*:: ++ +-- +type: keyword + +Beat type. + + +-- + +[float] +== stats fields + +Beat stats + + + +*`beat.stats.uptime.ms`*:: ++ +-- +type: long + +Beat uptime + + +-- + +*`beat.stats.runtime.goroutines`*:: ++ +-- +type: long + +Number of goroutines running in Beat + + +-- + +[float] +== libbeat fields + +Fields common to all Beats + + + +[float] +== output fields + +Output stats + + + +*`beat.stats.libbeat.output.type`*:: ++ +-- +type: keyword + +Type of output + + +-- + +[float] +== events fields + +Event counters + + + +*`beat.stats.libbeat.output.events.acked`*:: ++ +-- +type: long + +Number of events acknowledged + + +-- + +*`beat.stats.libbeat.output.events.active`*:: ++ +-- +type: long + +Number of active events + + +-- + +*`beat.stats.libbeat.output.events.batches`*:: ++ +-- +type: long + +Number of event batches + + +-- + +*`beat.stats.libbeat.output.events.dropped`*:: ++ +-- +type: long + +Number of events dropped + + +-- + +*`beat.stats.libbeat.output.events.duplicates`*:: ++ +-- +type: long + +Number of events duplicated + + +-- + +*`beat.stats.libbeat.output.events.failed`*:: ++ +-- +type: long + +Number of events failed + + +-- + +*`beat.stats.libbeat.output.events.toomany`*:: ++ +-- +type: long + +Number of too many events + + +-- + +*`beat.stats.libbeat.output.events.total`*:: ++ +-- +type: long + +Total number of events + + +-- + +[float] +== read fields + +Read stats + + + +*`beat.stats.libbeat.output.read.bytes`*:: ++ +-- +type: long + +Number of bytes read + + +-- + +*`beat.stats.libbeat.output.read.errors`*:: ++ +-- +type: long + +Number of read errors + + +-- + +[float] +== write fields + +Write stats + + + +*`beat.stats.libbeat.output.write.bytes`*:: ++ +-- +type: long + +Number of bytes written + + +-- + +*`beat.stats.libbeat.output.write.errors`*:: ++ +-- +type: long + +Number of write errors + + -- [[exported-fields-ceph]] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 74c4fae51f0b..84e7f21c6dab 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -30,6 +30,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/aerospike/namespace" _ "github.com/elastic/beats/metricbeat/module/apache" _ "github.com/elastic/beats/metricbeat/module/apache/status" + _ "github.com/elastic/beats/metricbeat/module/beat" + _ "github.com/elastic/beats/metricbeat/module/beat/stats" _ "github.com/elastic/beats/metricbeat/module/ceph" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_health" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 372e34f4cb58..4514890d3b34 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -156,6 +156,18 @@ metricbeat.modules: # Password of hosts. Empty by default #password: password +#-------------------------------- Beat Module -------------------------------- +- module: beat + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false + #-------------------------------- Ceph Module -------------------------------- - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] diff --git a/metricbeat/module/beat/_meta/Dockerfile b/metricbeat/module/beat/_meta/Dockerfile new file mode 100644 index 000000000000..44b84c00b907 --- /dev/null +++ b/metricbeat/module/beat/_meta/Dockerfile @@ -0,0 +1,6 @@ +FROM docker.elastic.co/beats/metricbeat:7.0.0 + +COPY healthcheck.sh / +HEALTHCHECK --interval=1s --retries=300 CMD sh /healthcheck.sh + +ENTRYPOINT [ "metricbeat", "-E", "http.enabled=true", "-E", "http.host=0.0.0.0" ] diff --git a/metricbeat/module/beat/_meta/config-xpack.yml b/metricbeat/module/beat/_meta/config-xpack.yml new file mode 100644 index 000000000000..011e01b04ada --- /dev/null +++ b/metricbeat/module/beat/_meta/config-xpack.yml @@ -0,0 +1,9 @@ +- module: beat + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #username: "user" + #password: "secret" + xpack.enabled: true + diff --git a/metricbeat/module/beat/_meta/config.reference.yml b/metricbeat/module/beat/_meta/config.reference.yml new file mode 100644 index 000000000000..a82db61430a5 --- /dev/null +++ b/metricbeat/module/beat/_meta/config.reference.yml @@ -0,0 +1,10 @@ +- module: beat + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false diff --git a/metricbeat/module/beat/_meta/config.yml b/metricbeat/module/beat/_meta/config.yml new file mode 100644 index 000000000000..1b7457a89898 --- /dev/null +++ b/metricbeat/module/beat/_meta/config.yml @@ -0,0 +1,6 @@ +- module: beat + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + diff --git a/metricbeat/module/beat/_meta/env b/metricbeat/module/beat/_meta/env new file mode 100644 index 000000000000..3e4c68f72f2e --- /dev/null +++ b/metricbeat/module/beat/_meta/env @@ -0,0 +1,2 @@ +BEAT_HOST=metricbeat +BEAT_PORT=5066 diff --git a/metricbeat/module/beat/_meta/fields.yml b/metricbeat/module/beat/_meta/fields.yml new file mode 100644 index 000000000000..d8140c26e141 --- /dev/null +++ b/metricbeat/module/beat/_meta/fields.yml @@ -0,0 +1,20 @@ +- key: beat + title: "Beat" + description: > + Beat module + release: ga + settings: ["ssl", "http"] + short_config: false + fields: + - name: beat + type: group + description: > + fields: + - name: id + type: keyword + description: > + Beat ID. + - name: type + type: keyword + description: > + Beat type. diff --git a/metricbeat/module/beat/_meta/healthcheck.sh b/metricbeat/module/beat/_meta/healthcheck.sh new file mode 100644 index 000000000000..ff9d08e49784 --- /dev/null +++ b/metricbeat/module/beat/_meta/healthcheck.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Check that all endpoints are available +curl -f http://localhost:5066 || exit 1 +curl -f http://localhost:5066/stats || exit 1 +curl -f http://localhost:5066/state || exit 1 diff --git a/metricbeat/module/beat/beat.go b/metricbeat/module/beat/beat.go new file mode 100644 index 000000000000..59af78bdc63f --- /dev/null +++ b/metricbeat/module/beat/beat.go @@ -0,0 +1,92 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beat + +import ( + "encoding/json" + "net/url" + + "github.com/elastic/beats/metricbeat/helper" +) + +// ModuleName is the name of this module. +const ModuleName = "beat" + +// Info construct contains the relevant data from the Beat's / endpoint +type Info struct { + UUID string `json:"uuid"` + Beat string `json:"beat"` + Name string `json:"name"` + Hostname string `json:"hostname"` + Version string `json:"version"` +} + +// State construct contains the relevant data from the Beat's /state endpoint +type State struct { + Outputs struct { + Elasticsearch struct { + ClusterUUID string `json:"cluster_uuid"` + } `json:"elasticsearch"` + } `json:"outputs"` +} + +// GetInfo returns the data for the Beat's / endpoint. +func GetInfo(m *MetricSet) (*Info, error) { + content, err := fetchPath(m.HTTP, "/", "") + if err != nil { + return nil, err + } + + info := &Info{} + err = json.Unmarshal(content, &info) + if err != nil { + return nil, err + } + + return info, nil +} + +// GetState returns the data for the Beat's /state endpoint. +func GetState(m *MetricSet) (*State, error) { + content, err := fetchPath(m.HTTP, "/state", "") + if err != nil { + return nil, err + } + + info := &State{} + err = json.Unmarshal(content, &info) + if err != nil { + return nil, err + } + + return info, nil +} + +func fetchPath(httpHelper *helper.HTTP, path string, query string) ([]byte, error) { + currentURI := httpHelper.GetURI() + defer httpHelper.SetURI(currentURI) + + // Parses the uri to replace the path + u, _ := url.Parse(currentURI) + u.Path = path + u.RawQuery = query + + // Http helper includes the HostData with username and password + httpHelper.SetURI(u.String()) + return httpHelper.FetchContent() +} diff --git a/metricbeat/module/beat/beat_integration_test.go b/metricbeat/module/beat/beat_integration_test.go new file mode 100644 index 000000000000..ec91196fdd68 --- /dev/null +++ b/metricbeat/module/beat/beat_integration_test.go @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package beat_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/beat" + _ "github.com/elastic/beats/metricbeat/module/beat/stats" +) + +var metricSets = []string{ + "stats", +} + +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "metricbeat") + + for _, metricSet := range metricSets { + f := mbtest.NewReportingMetricSetV2Error(t, beat.GetConfig(metricSet)) + events, errs := mbtest.ReportingFetchV2Error(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("beat", metricSet).Fields.StringToPrint()) + } +} + +func TestData(t *testing.T) { + compose.EnsureUp(t, "metricbeat") + + for _, metricSet := range metricSets { + f := mbtest.NewReportingMetricSetV2Error(t, beat.GetConfig(metricSet)) + err := mbtest.WriteEventsReporterV2Error(f, t, metricSet) + if err != nil { + t.Fatal("write", err) + } + } +} diff --git a/metricbeat/module/beat/config.go b/metricbeat/module/beat/config.go new file mode 100644 index 000000000000..83ea1879729a --- /dev/null +++ b/metricbeat/module/beat/config.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beat + +// Config defines the structure for the Beat module configuration options +type Config struct { + XPackEnabled bool `config:"xpack.enabled"` +} + +// DefaultConfig returns the default configuration for the Beat module +func DefaultConfig() Config { + return Config{ + XPackEnabled: false, + } +} diff --git a/metricbeat/module/beat/docs.asciidoc b/metricbeat/module/beat/docs.asciidoc new file mode 100644 index 000000000000..c3e066704765 --- /dev/null +++ b/metricbeat/module/beat/docs.asciidoc @@ -0,0 +1,9 @@ +The Beat module contains a minimal set of metrics to enable monitoring of any Beat or other software based on libbeat across +multiple versions. To monitor more Beat metrics, use our {stack-ov}/xpack-monitoring.html[monitoring] feature. + +The default metricset is `stats`. + +[float] +=== Compability + +The Beat module is tested with Metricbeat 7.2 and is expected to work with all 7.x versions. diff --git a/metricbeat/module/beat/fields.go b/metricbeat/module/beat/fields.go new file mode 100644 index 000000000000..ec8d2423860e --- /dev/null +++ b/metricbeat/module/beat/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package beat + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "beat", asset.ModuleFieldsPri, AssetBeat); err != nil { + panic(err) + } +} + +// AssetBeat returns asset data. +// This is the base64 encoded gzipped contents of ../metricbeat/module/beat. +func AssetBeat() string { + return "eJzcls+K2zAQxu95isHn7j6ADz2UttBLC2Whh1KKYk8cEVkjpNEGv32RUjuO/8hJya7L6hQk/H0/fePx5AEO2OSwRcEbAJasMIfsAwrONgAlusJKw5J0Du83AADhCGoqvcINgEWFwmEOldgAOGSWunI5/MycU9k7yPbMJvsVzvZk+XdBeierHHZCufD8TqIqXR6VH0CLGjuWsLgxQduSN393JoguVfpKsuy2Wq0DNkey/f1JxdOKd/3y8XEkHLTuIB2eG4s7FuxG6v0QrtEeqlxWql3D5Poc3rCs8bF2F6ctjyJdDQ4SSB3WSXTSz3odDSuy5FlqvJfxV19v0QLt4Cwd3LTUFUgdwSaJlNxuh2dz9biC43MMGwqqa9LABEKpaD686FRV+lzk2fghVhrtCrywvkXl0cuzBAbzvTHGG/fJjZBhPTUGQ0VnozgD4TPqydvAYmI3AH0KLlCQ14x2zi2VXx9ZFAecCwhSnfAP4HDRIaewAoCmo8KySnCcaVk+z9X8ZXFP1ukS9yaL4GI/+qy8DmlEXCRoSUtLxqz7CiwhdKjeKFkIXjNXd6ZYBt4JqdaNdoGg+5IS1UI3q5AyEQTza1uLiYV6RdKn4Ad6kGxyDlgUc5nfbQp8R1HOTk+4YQJsm7UaKjqnwupNVmtpdta9LGXgS/m3iEcrOf135A5V/xFM3kTZQ1yM+n+ufKzoECD1408AAAD//2l5TJo=" +} diff --git a/metricbeat/module/beat/metricset.go b/metricbeat/module/beat/metricset.go new file mode 100644 index 000000000000..ba4957b47c68 --- /dev/null +++ b/metricbeat/module/beat/metricset.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beat + +import ( + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" +) + +// MetricSet can be used to build other metricsets within the Beat module. +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP + XPackEnabled bool +} + +// NewMetricSet creates a metricset that can be used to build other metricsets +// within the Beat module. +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + config := DefaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + ms := &MetricSet{ + base, + http, + config.XPackEnabled, + } + + return ms, nil +} diff --git a/metricbeat/module/beat/stats/_meta/data.json b/metricbeat/module/beat/stats/_meta/data.json new file mode 100644 index 000000000000..318fc80d7be5 --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/data.json @@ -0,0 +1,51 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beat": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "stats": { + "libbeat": { + "output": { + "events": { + "acked": 0, + "active": 0, + "batches": 0, + "dropped": 0, + "duplicates": 0, + "failed": 0, + "toomany": 0, + "total": 0 + }, + "read": { + "bytes": 0, + "errors": 0 + }, + "type": "elasticsearch", + "write": { + "bytes": 0, + "errors": 0 + } + } + }, + "runtime": { + "goroutines": 39 + }, + "uptime": { + "ms": 12019 + } + }, + "type": "metricbeat" + }, + "event": { + "dataset": "beat.stats", + "duration": 115000, + "module": "beat" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:5066", + "name": "beat", + "type": "beat" + } +} \ No newline at end of file diff --git a/metricbeat/module/beat/stats/_meta/fields.yml b/metricbeat/module/beat/stats/_meta/fields.yml new file mode 100644 index 000000000000..043b39a6df0f --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/fields.yml @@ -0,0 +1,95 @@ +- name: stats + type: group + description: > + Beat stats + release: ga + fields: + - name: uptime.ms + type: long + description: > + Beat uptime + - name: runtime.goroutines + type: long + description: > + Number of goroutines running in Beat + - name: libbeat + type: group + description: > + Fields common to all Beats + fields: + - name: output + type: group + description: > + Output stats + fields: + - name: type + type: keyword + description: > + Type of output + - name: events + type: group + description: > + Event counters + fields: + - name: acked + type: long + description: > + Number of events acknowledged + - name: active + type: long + description: > + Number of active events + - name: batches + type: long + description: > + Number of event batches + - name: dropped + type: long + description: > + Number of events dropped + - name: duplicates + type: long + description: > + Number of events duplicated + - name: failed + type: long + description: > + Number of events failed + - name: toomany + type: long + description: > + Number of too many events + - name: total + type: long + description: > + Total number of events + - name: read + type: group + description: > + Read stats + fields: + - name: bytes + type: long + description: > + Number of bytes read + - name: errors + type: long + description: > + Number of read errors + - name: write + type: group + description: > + Write stats + fields: + - name: bytes + type: long + description: > + Number of bytes written + - name: errors + type: long + description: > + Number of write errors + + + + diff --git a/metricbeat/module/beat/stats/_meta/test/stats.800.json b/metricbeat/module/beat/stats/_meta/test/stats.800.json new file mode 100644 index 000000000000..ced95cd76e3f --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/test/stats.800.json @@ -0,0 +1,155 @@ +{ + "beat": { + "cpu": { + "system": { + "ticks": 95, + "time": { + "ms": 95 + } + }, + "total": { + "ticks": 458, + "time": { + "ms": 458 + }, + "value": 458 + }, + "user": { + "ticks": 363, + "time": { + "ms": 363 + } + } + }, + "info": { + "ephemeral_id": "f32e9a62-56c2-4cde-87be-de5869b2d7b7", + "uptime": { + "ms": 21557 + } + }, + "memstats": { + "gc_next": 10723216, + "memory_alloc": 6145008, + "memory_total": 360839144, + "rss": 44339200 + }, + "runtime": { + "goroutines": 40 + } + }, + "libbeat": { + "config": { + "module": { + "running": 0, + "starts": 0, + "stops": 0 + }, + "reloads": 1 + }, + "output": { + "events": { + "acked": 0, + "active": 0, + "batches": 0, + "dropped": 0, + "duplicates": 0, + "failed": 0, + "toomany": 0, + "total": 0 + }, + "read": { + "bytes": 0, + "errors": 0 + }, + "type": "elasticsearch", + "write": { + "bytes": 0, + "errors": 0 + } + }, + "pipeline": { + "clients": 3, + "events": { + "active": 37, + "dropped": 0, + "failed": 0, + "filtered": 1, + "published": 37, + "retry": 69, + "total": 38 + }, + "queue": { + "acked": 0 + } + } + }, + "metricbeat": { + "system": { + "cpu": { + "events": 2, + "failures": 0, + "success": 2 + }, + "filesystem": { + "events": 7, + "failures": 0, + "success": 7 + }, + "fsstat": { + "events": 1, + "failures": 0, + "success": 1 + }, + "load": { + "events": 3, + "failures": 0, + "success": 3 + }, + "memory": { + "events": 3, + "failures": 0, + "success": 3 + }, + "network": { + "events": 0, + "failures": 0, + "success": 0 + }, + "process": { + "events": 17, + "failures": 0, + "success": 17 + }, + "process_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "socket_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "uptime": { + "events": 1, + "failures": 0, + "success": 1 + } + } + }, + "system": { + "cpu": { + "cores": 8 + }, + "load": { + "1": 1.6753, + "15": 1.8076, + "5": 1.8257, + "norm": { + "1": 0.2094, + "15": 0.226, + "5": 0.2282 + } + } + } +} diff --git a/metricbeat/module/beat/stats/_meta/testdata/config.yml b/metricbeat/module/beat/stats/_meta/testdata/config.yml new file mode 100644 index 000000000000..9f0b7e9baa98 --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/testdata/config.yml @@ -0,0 +1,2 @@ +type: http +url: "/stats" diff --git a/metricbeat/module/beat/stats/_meta/testdata/stats.800.json b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json new file mode 100644 index 000000000000..ced95cd76e3f --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json @@ -0,0 +1,155 @@ +{ + "beat": { + "cpu": { + "system": { + "ticks": 95, + "time": { + "ms": 95 + } + }, + "total": { + "ticks": 458, + "time": { + "ms": 458 + }, + "value": 458 + }, + "user": { + "ticks": 363, + "time": { + "ms": 363 + } + } + }, + "info": { + "ephemeral_id": "f32e9a62-56c2-4cde-87be-de5869b2d7b7", + "uptime": { + "ms": 21557 + } + }, + "memstats": { + "gc_next": 10723216, + "memory_alloc": 6145008, + "memory_total": 360839144, + "rss": 44339200 + }, + "runtime": { + "goroutines": 40 + } + }, + "libbeat": { + "config": { + "module": { + "running": 0, + "starts": 0, + "stops": 0 + }, + "reloads": 1 + }, + "output": { + "events": { + "acked": 0, + "active": 0, + "batches": 0, + "dropped": 0, + "duplicates": 0, + "failed": 0, + "toomany": 0, + "total": 0 + }, + "read": { + "bytes": 0, + "errors": 0 + }, + "type": "elasticsearch", + "write": { + "bytes": 0, + "errors": 0 + } + }, + "pipeline": { + "clients": 3, + "events": { + "active": 37, + "dropped": 0, + "failed": 0, + "filtered": 1, + "published": 37, + "retry": 69, + "total": 38 + }, + "queue": { + "acked": 0 + } + } + }, + "metricbeat": { + "system": { + "cpu": { + "events": 2, + "failures": 0, + "success": 2 + }, + "filesystem": { + "events": 7, + "failures": 0, + "success": 7 + }, + "fsstat": { + "events": 1, + "failures": 0, + "success": 1 + }, + "load": { + "events": 3, + "failures": 0, + "success": 3 + }, + "memory": { + "events": 3, + "failures": 0, + "success": 3 + }, + "network": { + "events": 0, + "failures": 0, + "success": 0 + }, + "process": { + "events": 17, + "failures": 0, + "success": 17 + }, + "process_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "socket_summary": { + "events": 2, + "failures": 0, + "success": 2 + }, + "uptime": { + "events": 1, + "failures": 0, + "success": 1 + } + } + }, + "system": { + "cpu": { + "cores": 8 + }, + "load": { + "1": 1.6753, + "15": 1.8076, + "5": 1.8257, + "norm": { + "1": 0.2094, + "15": 0.226, + "5": 0.2282 + } + } + } +} diff --git a/metricbeat/module/beat/stats/_meta/testdata/stats.800.json-expected.json b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json-expected.json new file mode 100644 index 000000000000..1f43330729b8 --- /dev/null +++ b/metricbeat/module/beat/stats/_meta/testdata/stats.800.json-expected.json @@ -0,0 +1,19 @@ +[ + { + "error": { + "message": "HTTP error 404 in : 404 Not Found" + }, + "event": { + "dataset": "beat.stats", + "duration": 115000, + "module": "beat" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "beat" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/beat/stats/data.go b/metricbeat/module/beat/stats/data.go new file mode 100644 index 000000000000..8c868e1d5b29 --- /dev/null +++ b/metricbeat/module/beat/stats/data.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "encoding/json" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/beat" +) + +var ( + schema = s.Schema{ + "uptime": c.Dict("beat.info.uptime", s.Schema{ + "ms": c.Int("ms"), + }), + "runtime": c.Dict("beat.runtime", s.Schema{ + "goroutines": c.Int("goroutines"), + }, c.DictOptional), + "libbeat": c.Dict("libbeat", s.Schema{ + "output": c.Dict("output", s.Schema{ + "type": c.Str("type"), + "events": c.Dict("events", s.Schema{ + "acked": c.Int("acked"), + "active": c.Int("active"), + "batches": c.Int("batches"), + "dropped": c.Int("dropped"), + "duplicates": c.Int("duplicates"), + "failed": c.Int("failed"), + "toomany": c.Int("toomany"), + "total": c.Int("total"), + }), + "read": c.Dict("read", s.Schema{ + "bytes": c.Int("bytes"), + "errors": c.Int("errors"), + }), + "write": c.Dict("write", s.Schema{ + "bytes": c.Int("bytes"), + "errors": c.Int("errors"), + }), + }), + }), + } +) + +func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error { + var event mb.Event + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", beat.ModuleName) + + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("id", info.UUID) + event.ModuleFields.Put("type", info.Beat) + + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return errors.Wrap(err, "failure parsing Beat's Stats API response") + } + + event.MetricSetFields, err = schema.Apply(data) + if err != nil { + return errors.Wrap(err, "failure to apply stats schema") + } + + r.Event(event) + return nil +} diff --git a/metricbeat/module/beat/stats/data_test.go b/metricbeat/module/beat/stats/data_test.go new file mode 100644 index 000000000000..dd73d5fff06c --- /dev/null +++ b/metricbeat/module/beat/stats/data_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package stats + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/elastic/beats/metricbeat/module/beat" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapping(t *testing.T) { + + files, err := filepath.Glob("./_meta/test/stats.*.json") + assert.NoError(t, err) + + info := beat.Info{ + UUID: "1234", + Beat: "helloworld", + } + + for _, f := range files { + input, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = eventMapping(reporter, info, input) + + assert.NoError(t, err, f) + assert.True(t, len(reporter.GetEvents()) >= 1, f) + assert.Equal(t, 0, len(reporter.GetErrors()), f) + } +} diff --git a/metricbeat/module/beat/stats/data_xpack.go b/metricbeat/module/beat/stats/data_xpack.go new file mode 100644 index 000000000000..2c75d7c6d438 --- /dev/null +++ b/metricbeat/module/beat/stats/data_xpack.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "encoding/json" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/metricbeat/helper/elastic" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/beat" +) + +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beat.Info, content []byte) error { + now := time.Now() + clusterUUID, err := m.getClusterUUID() + if err != nil { + return errors.Wrap(err, "could not determine cluster UUID") + } + + // Massage info into beat + beat := common.MapStr{ + "name": info.Name, + "host": info.Hostname, + "type": info.Beat, + "uuid": info.UUID, + "version": info.Version, + } + + var metrics map[string]interface{} + err = json.Unmarshal(content, &metrics) + if err != nil { + return errors.Wrap(err, "failure parsing Beat's Stats API response") + } + + fields := common.MapStr{ + "metrics": metrics, + "beat": beat, + "timestamp": now, + } + + var event mb.Event + event.RootFields = common.MapStr{ + "cluster_uuid": clusterUUID, + "timestamp": now, + "interval_ms": m.calculateIntervalMs(), + "type": "beats_stats", + "beats_stats": fields, + } + + event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Beats) + + r.Event(event) + return nil +} + +func (m *MetricSet) calculateIntervalMs() int64 { + return m.Module().Config().Period.Nanoseconds() / 1000 / 1000 +} + +func (m *MetricSet) getClusterUUID() (string, error) { + state, err := beat.GetState(m.MetricSet) + if err != nil { + return "", errors.Wrap(err, "could not get state information") + } + + return state.Outputs.Elasticsearch.ClusterUUID, nil +} diff --git a/metricbeat/module/beat/stats/stats.go b/metricbeat/module/beat/stats/stats.go new file mode 100644 index 000000000000..fcab548c5a37 --- /dev/null +++ b/metricbeat/module/beat/stats/stats.go @@ -0,0 +1,83 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/beat" +) + +func init() { + mb.Registry.MustAddMetricSet(beat.ModuleName, "stats", New, + mb.WithHostParser(hostParser), + ) +} + +const ( + statsPath = "stats" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: statsPath, + }.Build() +) + +// MetricSet defines all fields of the MetricSet +type MetricSet struct { + *beat.MetricSet +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := beat.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +func (m *MetricSet) Fetch(r mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + info, err := beat.GetInfo(m.MetricSet) + if err != nil { + return err + } + + if m.MetricSet.XPackEnabled { + err = eventMappingXPack(r, m, *info, content) + if err != nil { + // Since this is an x-pack code path, we log the error but don't + // return it. Otherwise it would get reported into `metricbeat-*` + // indices. + m.Logger().Error(err) + return nil + } + } else { + return eventMapping(r, *info, content) + } + + return nil +} diff --git a/metricbeat/module/beat/testing.go b/metricbeat/module/beat/testing.go new file mode 100644 index 000000000000..5a282228d05a --- /dev/null +++ b/metricbeat/module/beat/testing.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beat + +import "os" + +// GetEnvHost for Metricbeat +func GetEnvHost() string { + host := os.Getenv("BEAT_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +// GetEnvPort for Metricbeat +func GetEnvPort() string { + port := os.Getenv("BEAT_PORT") + + if len(port) == 0 { + port = "5066" + } + return port +} + +// GetConfig for Metricbeat +func GetConfig(metricset string) map[string]interface{} { + return map[string]interface{}{ + "module": ModuleName, + "metricsets": []string{metricset}, + "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, + } +} diff --git a/metricbeat/modules.d/beat-xpack.yml.disabled b/metricbeat/modules.d/beat-xpack.yml.disabled new file mode 100644 index 000000000000..e941a5257876 --- /dev/null +++ b/metricbeat/modules.d/beat-xpack.yml.disabled @@ -0,0 +1,12 @@ +# Module: beat +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beat.html + +- module: beat + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #username: "user" + #password: "secret" + xpack.enabled: true + diff --git a/metricbeat/modules.d/beat.yml.disabled b/metricbeat/modules.d/beat.yml.disabled new file mode 100644 index 000000000000..ed31130210a4 --- /dev/null +++ b/metricbeat/modules.d/beat.yml.disabled @@ -0,0 +1,9 @@ +# Module: beat +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-beat.html + +- module: beat + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + diff --git a/metricbeat/tests/system/test_beat.py b/metricbeat/tests/system/test_beat.py new file mode 100644 index 000000000000..42ccb9dd8e04 --- /dev/null +++ b/metricbeat/tests/system/test_beat.py @@ -0,0 +1,32 @@ +import os +import metricbeat +import unittest +import time + + +class Test(metricbeat.BaseTest): + + COMPOSE_SERVICES = ['metricbeat'] + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_stats(self): + """ + beat stats metricset test + """ + self.render_config_template(modules=[{ + "name": "beat", + "metricsets": ["stats"], + "hosts": self.get_hosts(), + "period": "1s", + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print(evt) + + self.assert_fields_are_documented(evt) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 56e377a89dea..c6932be644c6 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -205,6 +205,18 @@ metricbeat.modules: # - us-east-1 # - us-east-2 +#--------------------------------- Beat Module --------------------------------- +- module: beat + metricsets: + - stats + period: 10s + hosts: ["http://localhost:5066"] + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false + #--------------------------------- Ceph Module --------------------------------- - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"]