From bae50e7991e94c722944f401ac287232fe134e23 Mon Sep 17 00:00:00 2001 From: Audrius Karabanovas Date: Fri, 7 Sep 2018 14:42:57 +0300 Subject: [PATCH] Initial --- .gitignore | 4 + .promu.yml | 23 +++ .travis.yml | 20 +++ Dockerfile | 14 ++ Gopkg.lock | 110 +++++++++++++ Gopkg.toml | 11 ++ LICENSE | 21 +++ Makefile | 77 +++++++++ VERSION | 1 + collector/beat.go | 182 +++++++++++++++++++++ collector/filebeat.go | 155 ++++++++++++++++++ collector/libbeat.go | 344 ++++++++++++++++++++++++++++++++++++++++ collector/main.go | 148 +++++++++++++++++ collector/metricbeat.go | 222 ++++++++++++++++++++++++++ collector/registrar.go | 85 ++++++++++ collector/structs.go | 30 ++++ collector/system.go | 119 ++++++++++++++ main.go | 158 ++++++++++++++++++ readme.md | 57 +++++++ 19 files changed, 1781 insertions(+) create mode 100644 .gitignore create mode 100644 .promu.yml create mode 100644 .travis.yml create mode 100644 Dockerfile create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 VERSION create mode 100644 collector/beat.go create mode 100644 collector/filebeat.go create mode 100644 collector/libbeat.go create mode 100644 collector/main.go create mode 100644 collector/metricbeat.go create mode 100644 collector/registrar.go create mode 100644 collector/structs.go create mode 100644 collector/system.go create mode 100644 main.go create mode 100644 readme.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5ba8269 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +vendor +.build +.tarballs +beat-exporter \ No newline at end of file diff --git a/.promu.yml b/.promu.yml new file mode 100644 index 0000000..a0b0a64 --- /dev/null +++ b/.promu.yml @@ -0,0 +1,23 @@ +verbose: true +repository: + path: github.com/trustpilot/beat-exporter +build: + flags: -a -tags 'netgo static_build' + ldflags: | + -s + -X {{repoPath}}/vendor/github.com/prometheus/common/version.Version={{.Version}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.Revision={{.Revision}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.Branch={{.Branch}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.BuildUser={{user}}@{{host}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.BuildDate={{date "20060102-15:04:05"}} +tarball: + files: + - LICENSE +crossbuild: + platforms: + - linux/amd64 + - linux/386 + - darwin/amd64 + - darwin/386 + - windows/amd64 + - windows/386 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..86c8bcc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +dist: trusty +sudo: required +language: go + +go: + - 1.10.x + +go_import_path: github.com/trustpilot/beat-exporter + +services: + - docker + +before_script: + - make dependencies + +script: + - make travis + +branches: + only: master \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c130a8c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM quay.io/prometheus/golang-builder as builder + +ADD . /go/src/github.com/trustpilot/beat-exporter +WORKDIR /go/src/github.com/trustpilot/beat-exporter + +RUN make + +FROM quay.io/prometheus/busybox:latest +MAINTAINER Audrius Karabanovas + +COPY --from=builder /go/src/github.com/trustpilot/beat-exporter/beat-exporter /bin/beat-exporter + +EXPOSE 9479 +ENTRYPOINT [ "/bin/beat-exporter" ] \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..a4cbfeb --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,110 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + digest = "1:c0bec5f9b98d0bc872ff5e834fac186b807b656683bd29cb82fb207a1513fabb" + name = "github.com/beorn7/perks" + packages = ["quantile"] + pruneopts = "" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" + name = "github.com/golang/protobuf" + packages = ["proto"] + pruneopts = "" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" + +[[projects]] + digest = "1:63722a4b1e1717be7b98fc686e0b30d5e7f734b9e93d7dee86293b6deab7ea28" + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + pruneopts = "" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + digest = "1:4142d94383572e74b42352273652c62afec5b23f325222ed09198f46009022d1" + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/promhttp", + ] + pruneopts = "" + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" + +[[projects]] + branch = "master" + digest = "1:185cf55b1f44a1bf243558901c3f06efa5c64ba62cfdcbb1bf7bbe8c3fb68561" + name = "github.com/prometheus/client_model" + packages = ["go"] + pruneopts = "" + revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" + +[[projects]] + branch = "master" + digest = "1:f477ef7b65d94fb17574fc6548cef0c99a69c1634ea3b6da248b63a61ebe0498" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + "version", + ] + pruneopts = "" + revision = "c7de2306084e37d54b8be01f3541a8464345e9a5" + +[[projects]] + branch = "master" + digest = "1:e04aaa0e8f8da0ed3d6c0700bd77eda52a47f38510063209d72d62f0ef807d5e" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs", + ] + pruneopts = "" + revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" + +[[projects]] + digest = "1:3fcbf733a8d810a21265a7f2fe08a3353db2407da052b233f8b204b5afc03d9b" + name = "github.com/sirupsen/logrus" + packages = ["."] + pruneopts = "" + revision = "3e01752db0189b9157070a0e1668a620f9a85da2" + version = "v1.0.6" + +[[projects]] + branch = "master" + digest = "1:793a79198b755828dec284c6f1325e24e09186f1b7ba818b65c7c35104ed86eb" + name = "golang.org/x/crypto" + packages = ["ssh/terminal"] + pruneopts = "" + revision = "614d502a4dac94afa3a6ce146bd1736da82514c6" + +[[projects]] + branch = "master" + digest = "1:7a5f7a1206de6b90f67cb465e489eac3298e95afa7262813b542df4fab38952f" + name = "golang.org/x/sys" + packages = [ + "unix", + "windows", + ] + pruneopts = "" + revision = "4910a1d54f876d7b22162a85f4d066d3ee649450" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/prometheus/common/version", + "github.com/sirupsen/logrus", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..fcb0951 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,11 @@ +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "0.8.0" + +[[constraint]] + branch = "master" + name = "github.com/prometheus/common" + +[[constraint]] + name = "github.com/sirupsen/logrus" + version = "1.0.6" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6550a90 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Trustpilot + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2122c2b --- /dev/null +++ b/Makefile @@ -0,0 +1,77 @@ +# Copyright 2016 The Prometheus Authors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO := GO15VENDOREXPERIMENT=1 go +PROMU := $(GOPATH)/bin/promu +pkgs = $(shell $(GO) list ./... | grep -v /vendor/) + +PREFIX ?= $(shell pwd) +BIN_DIR ?= $(shell pwd) +DOCKER_IMAGE_NAME ?= beat-exporter +DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) + + +all: format dependencies build test + +style: + @echo ">> checking code style" + @! gofmt -d $(shell find . -path ./vendor -prune -o -name '*.go' -print) | grep '^' + +test: + @echo ">> running tests" + @$(GO) test -short $(pkgs) + +format: + @echo ">> formatting code" + @$(GO) fmt $(pkgs) + +vet: + @echo ">> vetting code" + @$(GO) vet $(pkgs) + +dependencies: + @echo ">> installing dep" + @$(GO) get -u github.com/golang/dep/cmd/dep + @echo ">> running dep ensure" + @dep ensure + +build: promu + @echo ">> building binaries" + @$(PROMU) build --prefix $(PREFIX) + +crossbuild: promu + @echo ">> cross-building binaries" + @$(PROMU) crossbuild + +tarball: promu + @echo ">> building release tarball" + @$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR) + +tarballs: promu + @echo ">> building release tarballs" + @$(PROMU) crossbuild tarballs $(BIN_DIR) + +travis: dependencies format build + @$(PROMU) info + +docker: + @echo ">> building docker image" + @docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" . + +promu: + @echo ">> installing promu" + @GOOS=$(shell uname -s | tr A-Z a-z) \ + GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(shell uname -m))) \ + $(GO) get -u github.com/prometheus/promu + +.PHONY: all style format build test vet tarball docker promu travis dependencies \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6c6aa7c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file diff --git a/collector/beat.go b/collector/beat.go new file mode 100644 index 0000000..9cc846b --- /dev/null +++ b/collector/beat.go @@ -0,0 +1,182 @@ +package collector + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +//CPUTimings json structure +type CPUTimings struct { + Ticks float64 `json:"ticks"` + Time struct { + MS float64 `json:"ms"` + } `json:"time"` + Value float64 `json:"value"` +} + +//BeatStats json structure +type BeatStats struct { + CPU struct { + Sytem CPUTimings `json:"system"` + Total CPUTimings `json:"total"` + User CPUTimings `json:"user"` + } `json:"cpu"` + BeatUptime struct { + Uptime struct { + MS float64 `json:"ms"` + } `json:"uptime"` + + EmphemeralID string `json:"emphemeral_id"` + } `json:"info"` + + Memstats struct { + GCNext float64 `json:"gc_next"` + MemoryAlloc float64 `json:"memory_alloc"` + MemoryTotal float64 `json:"memory_total"` + RSS float64 `json:"rss"` + } `json:"memstats"` +} + +type beatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewBeatCollector constructor +func NewBeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &beatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu_time", "miliseconds"), + "beat.cpu.time", + nil, prometheus.Labels{"mode": "system"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Sytem.Time.MS }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu_time", "miliseconds"), + "beat.cpu.time", + nil, prometheus.Labels{"mode": "user"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.User.Time.MS }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu_time", "miliseconds"), + "beat.cpu.time", + nil, prometheus.Labels{"mode": "total"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Total.Time.MS }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu", "ticks"), + "beat.cpu.ticks", + nil, prometheus.Labels{"mode": "system"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Sytem.Ticks }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu", "ticks"), + "beat.cpu.ticks", + nil, prometheus.Labels{"mode": "user"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.User.Ticks }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu", "ticks"), + "beat.cpu.ticks", + nil, prometheus.Labels{"mode": "total"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Total.Ticks }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "uptime", "seconds"), + "beat.info.uptime.ms", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return (time.Duration(stats.Beat.BeatUptime.Uptime.MS) * time.Millisecond).Seconds() + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "gc_next"), + "beat.memstats.gc_next", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.GCNext + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "memory_alloc"), + "beat.memstats.memory_alloc", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.MemoryAlloc + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "memory_total"), + "beat.memstats.memory_total", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.MemoryTotal + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "rss"), + "beat.memstats.rss", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.RSS + }, + valType: prometheus.GaugeValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *beatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *beatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/filebeat.go b/collector/filebeat.go new file mode 100644 index 0000000..7c58fa0 --- /dev/null +++ b/collector/filebeat.go @@ -0,0 +1,155 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//Filebeat json structure +type Filebeat struct { + Events struct { + Active float64 `json:"active"` + Added float64 `json:"added"` + Done float64 `json:"done"` + } `json:"events"` + + Harvester struct { + Closed float64 `json:"closed"` + OpenFiles float64 `json:"open_files"` + Running float64 `json:"running"` + Skipped float64 `json:"skipped"` + Started float64 `json:"started"` + } `json:"harvester"` + + Prospector struct { + Log struct { + Files struct { + Renamed float64 `json:"renamed"` + Truncated float64 `json:"truncated"` + } `json:"files"` + } `json:"log"` + } `json:"prospector"` +} + +type filebeatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewFilebeatCollector constructor +func NewFilebeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &filebeatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "events"), + "filebeat.events", + nil, prometheus.Labels{"event": "active"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Events.Active }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "events"), + "filebeat.events", + nil, prometheus.Labels{"event": "added"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Events.Added }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "events"), + "filebeat.events", + nil, prometheus.Labels{"event": "done"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Events.Done }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "closed"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Closed }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "open_files"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.OpenFiles }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "running"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Running }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "skipped"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Skipped }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "started"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Started }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "prospector_log"), + "filebeat.prospector_log", + nil, prometheus.Labels{"files": "renamed"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Prospector.Log.Files.Renamed }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "prospector_log"), + "filebeat.prospector_log", + nil, prometheus.Labels{"files": "truncated"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Prospector.Log.Files.Truncated }, + valType: prometheus.UntypedValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *filebeatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *filebeatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/libbeat.go b/collector/libbeat.go new file mode 100644 index 0000000..f732047 --- /dev/null +++ b/collector/libbeat.go @@ -0,0 +1,344 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//LibBeat json structure +type LibBeat struct { + Config struct { + Module struct { + Running float64 `json:"running"` + Starts float64 `json:"starts"` + Stops float64 `json:"stops"` + } `json:"module"` + Reloads float64 `json:"reloads"` + } `json:"config"` + Output LibBeatOutput `json:"output"` + Pipeline LibBeatPipeline `json:"pipeline"` +} + +//LibBeatEvents json structure +type LibBeatEvents struct { + Acked float64 `json:"acked"` + Active float64 `json:"active"` + Batches float64 `json:"batches"` + Dropped float64 `json:"dropped"` + Duplicates float64 `json:"duplicates"` + Failed float64 `json:"failed"` + Filtered float64 `json:"filtered"` + Published float64 `json:"published"` + Retry float64 `json:"retry"` +} + +//LibBeatOutputBytesErrors json structure +type LibBeatOutputBytesErrors struct { + Bytes float64 `json:"bytes"` + Errors float64 `json:"errors"` +} + +//LibBeatOutput json structure +type LibBeatOutput struct { + Events LibBeatEvents `json:"events"` + Read LibBeatOutputBytesErrors `json:"read"` + Write LibBeatOutputBytesErrors `json:"write"` + Type string `json:"type"` +} + +//LibBeatPipeline json structure +type LibBeatPipeline struct { + Clients float64 `json:"clients"` + Events LibBeatEvents `json:"events"` + Queue struct { + Acked float64 `json:"acked"` + } `json:"queue"` +} + +type libbeatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +var libbeatOutputType *prometheus.Desc + +// NewLibBeatCollector constructor +func NewLibBeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &libbeatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat_config", "reloads"), + "libbeat.config.reloads", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Reloads + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "config"), + "libbeat.config.module", + nil, prometheus.Labels{"module": "running"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Module.Running + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "config"), + "libbeat.config.module", + nil, prometheus.Labels{"module": "starts"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Module.Starts + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "config"), + "libbeat.config.module", + nil, prometheus.Labels{"module": "stops"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Module.Stops + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_read_bytes"), + "libbeat.output.read.bytes", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Read.Bytes + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_read_errors"), + "libbeat.output.read.errors", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Read.Errors + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_write_bytes"), + "libbeat.output.write.bytes", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Write.Bytes + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_write_errors"), + "libbeat.output.write.errors", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Write.Errors + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "acked"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Acked + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "active"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Active + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "batches"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Batches + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "dropped"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Dropped + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "duplicates"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Duplicates + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "failed"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Failed + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_clients"), + "libbeat.pipeline.clients", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Clients + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_queue"), + "libbeat.pipeline.queue", + nil, prometheus.Labels{"type": "acked"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Queue.Acked + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "active"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Active + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "dropped"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Dropped + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "failed"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Failed + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "filtered"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Filtered + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "published"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Published + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "retry"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Retry + }, + valType: prometheus.UntypedValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *libbeatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + + libbeatOutputType = prometheus.NewDesc( + prometheus.BuildFQName(c.beatInfo.Beat, "libbeat", "output"), + "libbeat.output.type", + []string{"type"}, nil, + ) + + ch <- libbeatOutputType + +} + +// Collect returns the current state of all metrics of the collector. +func (c *libbeatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + + // output.type with dynamic label + ch <- prometheus.MustNewConstMetric(libbeatOutputType, prometheus.CounterValue, float64(1), c.stats.LibBeat.Output.Type) + +} diff --git a/collector/main.go b/collector/main.go new file mode 100644 index 0000000..fb59f74 --- /dev/null +++ b/collector/main.go @@ -0,0 +1,148 @@ +package collector + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "regexp" + + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" +) + +type mainCollector struct { + Collectors map[string]prometheus.Collector + Stats *Stats + client *http.Client + beatURL *url.URL + name string + beatInfo *BeatInfo + targetDesc *prometheus.Desc + targetUp *prometheus.Desc + metrics exportedMetrics +} + +// HackfixRegex regex to replace JSON part +var HackfixRegex = regexp.MustCompile("\"time\":(\\d+)") // replaces time:123 to time.ms:123, only filebeat has different naming of time metric + +// NewMainCollector constructor +func NewMainCollector(client *http.Client, url *url.URL, name string, beatInfo *BeatInfo) prometheus.Collector { + instance := fmt.Sprintf("%s:%s", url.Hostname(), url.Port()) + beat := &mainCollector{ + Collectors: make(map[string]prometheus.Collector), + Stats: &Stats{}, + client: client, + beatURL: url, + name: name, + targetDesc: prometheus.NewDesc( + prometheus.BuildFQName(name, "target", "info"), + "target information", + nil, + prometheus.Labels{"version": beatInfo.Version, "beat": beatInfo.Beat, "uri": instance}), + targetUp: prometheus.NewDesc( + prometheus.BuildFQName("", beatInfo.Beat, "up"), + "Target up", + nil, + nil), + + beatInfo: beatInfo, + metrics: exportedMetrics{}, + } + + beat.Collectors["system"] = NewSystemCollector(beatInfo, beat.Stats) + beat.Collectors["beat"] = NewBeatCollector(beatInfo, beat.Stats) + beat.Collectors["libbeat"] = NewLibBeatCollector(beatInfo, beat.Stats) + beat.Collectors["registrar"] = NewRegistrarCollector(beatInfo, beat.Stats) + beat.Collectors["filebeat"] = NewFilebeatCollector(beatInfo, beat.Stats) + beat.Collectors["metricbeat"] = NewMetricbeatCollector(beatInfo, beat.Stats) + + return beat +} + +// Describe returns all descriptions of the collector. +func (b *mainCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- b.targetDesc + ch <- b.targetUp + + for _, metric := range b.metrics { + ch <- metric.desc + } + + // standard collectors for all types of beats + b.Collectors["system"].Describe(ch) + b.Collectors["beat"].Describe(ch) + b.Collectors["libbeat"].Describe(ch) + + // Customized collectors per beat type + switch b.beatInfo.Beat { + case "filebeat": + b.Collectors["filebeat"].Describe(ch) + b.Collectors["registrar"].Describe(ch) + case "metricbeat": + b.Collectors["metricbeat"].Describe(ch) + } + +} + +// Collect returns the current state of all metrics of the collector. +func (b *mainCollector) Collect(ch chan<- prometheus.Metric) { + + err := b.fetchStatsEndpoint() + if err != nil { + ch <- prometheus.MustNewConstMetric(b.targetUp, prometheus.GaugeValue, float64(0)) // set target down + log.Errorf("Failed getting /stats endpoint of target") + return + } + + ch <- prometheus.MustNewConstMetric(b.targetDesc, prometheus.GaugeValue, float64(1)) + ch <- prometheus.MustNewConstMetric(b.targetUp, prometheus.GaugeValue, float64(1)) // target up + + for _, i := range b.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(b.Stats)) + } + + // standard collectors for all types of beats + b.Collectors["system"].Collect(ch) + b.Collectors["beat"].Collect(ch) + b.Collectors["libbeat"].Collect(ch) + + // Customized collectors per beat type + switch b.beatInfo.Beat { + case "filebeat": + b.Collectors["filebeat"].Collect(ch) + b.Collectors["registrar"].Collect(ch) + case "metricbeat": + b.Collectors["metricbeat"].Collect(ch) + } + +} + +func (b *mainCollector) fetchStatsEndpoint() error { + + response, err := b.client.Get(b.beatURL.String() + "/stats") + if err != nil { + log.Errorf("Could not fetch stats endpoint of target: %v", b.beatURL.String()) + return err + } + + defer response.Body.Close() + + bodyBytes, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Error("Can't read body of response") + return err + } + + // @TODO remove this when filebeat stats endpoint output matches all other beats output + bodyBytes = HackfixRegex.ReplaceAll(bodyBytes, []byte("\"time\":{\"ms\":$1}")) + + err = json.Unmarshal(bodyBytes, &b.Stats) + if err != nil { + log.Error("Could not parse JSON response for target") + return err + } + + return nil +} diff --git a/collector/metricbeat.go b/collector/metricbeat.go new file mode 100644 index 0000000..674b5b5 --- /dev/null +++ b/collector/metricbeat.go @@ -0,0 +1,222 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//MetricbeatEvent json structure +type MetricbeatEvent struct { + Failures float64 `json:"failures"` + Success float64 `json:"success"` +} + +//Metricbeat json structure +type Metricbeat struct { + System struct { + CPU MetricbeatEvent `json:"cpu"` + Filesystem MetricbeatEvent `json:"filesystem"` + Fsstat MetricbeatEvent `json:"fsstat"` + Load MetricbeatEvent `json:"load"` + Memory MetricbeatEvent `json:"memory"` + Network MetricbeatEvent `json:"network"` + Process MetricbeatEvent `json:"process"` + ProcessSummary MetricbeatEvent `json:"process_summary"` + Uptime MetricbeatEvent `json:"uptime"` + } `json:"system"` +} + +type metricbeatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewMetricbeatCollector constructor +func NewMetricbeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &metricbeatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "cpu"), + "system.cpu", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.CPU.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "cpu"), + "system.cpu", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.CPU.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "filesystem"), + "system.filesystem", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Filesystem.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "filesystem"), + "system.filesystem", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Filesystem.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "fsstat"), + "system.fsstat", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Fsstat.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "fsstat"), + "system.fsstat", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Fsstat.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "load"), + "system.load", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Load.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "load"), + "system.load", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Load.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "memory"), + "system.memory", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Memory.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "memory"), + "system.memory", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Memory.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "network"), + "system.network", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Network.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "network"), + "system.network", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Network.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process"), + "system.process", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Process.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process"), + "system.process", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Process.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process_summary"), + "system.process_summary", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.ProcessSummary.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process_summary"), + "system.process_summary", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.ProcessSummary.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "uptime"), + "system.uptime", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Uptime.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "uptime"), + "system.uptime", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Uptime.Failures }, + valType: prometheus.CounterValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *metricbeatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *metricbeatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/registrar.go b/collector/registrar.go new file mode 100644 index 0000000..83100ee --- /dev/null +++ b/collector/registrar.go @@ -0,0 +1,85 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//Registrar json structure +type Registrar struct { + Writes float64 `json:"writes"` + States struct { + Cleanup float64 `json:"cleanup"` + Current float64 `json:"current"` + Update float64 `json:"update"` + } `json:"states"` +} + +type registrarCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewRegistrarCollector constructor +func NewRegistrarCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return ®istrarCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "writes"), + "registrar.writes", + nil, nil, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.Writes }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "states"), + "registrar.states", + nil, prometheus.Labels{"state": "cleanup"}, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.States.Cleanup }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "states"), + "registrar.states", + nil, prometheus.Labels{"state": "current"}, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.States.Current }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "states"), + "registrar.states", + nil, prometheus.Labels{"state": "update"}, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.States.Update }, + valType: prometheus.GaugeValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *registrarCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *registrarCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/structs.go b/collector/structs.go new file mode 100644 index 0000000..8bc69a4 --- /dev/null +++ b/collector/structs.go @@ -0,0 +1,30 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//BeatInfo beat info json structure +type BeatInfo struct { + Beat string `json:"beat"` + Hostname string `json:"hostname"` + Name string `json:"name"` + UUID string `json:"uuid"` + Version string `json:"version"` +} + +//Stats stats endpoint json structure +type Stats struct { + System System `json:"system"` + Beat BeatStats `json:"beat"` + LibBeat LibBeat `json:"libbeat"` + Registrar Registrar `json:"registrar"` + Filebeat Filebeat `json:"filebeat"` + Metricbeat Metricbeat `json:"metricbeat"` +} + +type exportedMetrics []struct { + desc *prometheus.Desc + eval func(stats *Stats) float64 + valType prometheus.ValueType +} diff --git a/collector/system.go b/collector/system.go new file mode 100644 index 0000000..4afddb1 --- /dev/null +++ b/collector/system.go @@ -0,0 +1,119 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//CPUStats json structure +type CPUStats struct { + M1 float64 `json:"1"` + M5 float64 `json:"5"` + M15 float64 `json:"15"` +} + +//System json structure +type System struct { + CPU struct { + Cores int64 `json:"cores"` + } `json:"cpu"` + Load struct { + CPUStats + Norm CPUStats `json:"norm"` + } `json:"load"` +} +type systemCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewSystemCollector constructor +func NewSystemCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &systemCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_cpu", "cores"), + "cpu cores", + nil, nil, + ), + eval: func(stats *Stats) float64 { return float64(stats.System.CPU.Cores) }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system", "load"), + "system load 1m", + nil, prometheus.Labels{"period": "1"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.M1 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system", "load"), + "system load 1m", + nil, prometheus.Labels{"period": "5"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.M5 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system", "load"), + "system load 1m", + nil, prometheus.Labels{"period": "15"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.M15 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_load", "norm"), + "system load 1m", + nil, prometheus.Labels{"period": "1"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.Norm.M1 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_load", "norm"), + "system load 1m", + nil, prometheus.Labels{"period": "5"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.Norm.M5 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_load", "norm"), + "system load 1m", + nil, prometheus.Labels{"period": "15"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.Norm.M15 }, + valType: prometheus.GaugeValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *systemCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *systemCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..cf1f2a1 --- /dev/null +++ b/main.go @@ -0,0 +1,158 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/version" + "github.com/trustpilot/beat-exporter/collector" +) + +func main() { + var ( + Name = "beat_exporter" + listenAddress = flag.String("web.listen-address", ":9479", "Address to listen on for web interface and telemetry.") + metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") + beatURI = flag.String("beat.uri", "http://localhost:5066", "HTTP API address of beat.") + beatTimeout = flag.Duration("beat.timeout", 10*time.Second, "Timeout for trying to get stats from beat.") + showVersion = flag.Bool("version", false, "Show version and exit") + ) + flag.Parse() + + if *showVersion { + fmt.Print(version.Print(Name)) + os.Exit(0) + } + + log.SetLevel(log.DebugLevel) + + log.SetFormatter(&log.JSONFormatter{ + FieldMap: log.FieldMap{ + log.FieldKeyMsg: "message", + }, + }) + + beatURL, err := url.Parse(*beatURI) + + if err != nil { + log.Fatalf("failed to parse beat.uri, error: %v", err) + } + + httpClient := &http.Client{ + Timeout: *beatTimeout, + } + + log.Info("Exploring target for beat type") + + var beatInfo *collector.BeatInfo + + for { + beatInfo, err = loadBeatType(httpClient, *beatURL) + if err != nil { + log.Errorf("Could not load beat type, with error: %v, retrying in 5s", err) + time.Sleep(5 * time.Second) + } else { + break + } + } + + // version metric + registry := prometheus.NewRegistry() + versionMetric := version.NewCollector(Name) + mainCollector := collector.NewMainCollector(httpClient, beatURL, Name, beatInfo) + registry.MustRegister(versionMetric) + registry.MustRegister(mainCollector) + + http.Handle(*metricsPath, promhttp.HandlerFor( + registry, + promhttp.HandlerOpts{ + ErrorLog: log.New(), + DisableCompression: false, + ErrorHandling: promhttp.ContinueOnError}), + ) + + http.HandleFunc("/", IndexHandler(*metricsPath)) + + log.WithFields(log.Fields{ + "addr": *listenAddress, + }).Infof("Starting exporter with configured type: %s", beatInfo.Beat) + + if err := http.ListenAndServe(*listenAddress, nil); err != nil { + + log.WithFields(log.Fields{ + "err": err, + }).Errorf("http server quit with error: %v", err) + + } +} + +// IndexHandler returns a http handler with the correct metricsPath +func IndexHandler(metricsPath string) http.HandlerFunc { + indexHTML := ` + + + Beat Exporter + + +

Beat Exporter

+

+ Metrics +

+ + +` + index := []byte(fmt.Sprintf(strings.TrimSpace(indexHTML), metricsPath)) + + return func(w http.ResponseWriter, r *http.Request) { + w.Write(index) + } +} + +func loadBeatType(client *http.Client, url url.URL) (*collector.BeatInfo, error) { + beatInfo := &collector.BeatInfo{} + + response, err := client.Get(url.String()) + if err != nil { + return beatInfo, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + log.Errorf("Beat URL: %q status code: %d", url.String(), response.StatusCode) + return beatInfo, err + } + + bodyBytes, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Error("Can't read body of response") + return beatInfo, err + } + + err = json.Unmarshal(bodyBytes, &beatInfo) + if err != nil { + log.Error("Could not parse JSON response for target") + return beatInfo, err + } + + log.WithFields( + log.Fields{ + "beat": beatInfo.Beat, + "version": beatInfo.Version, + "name": beatInfo.Name, + "hostname": beatInfo.Hostname, + "uuid": beatInfo.UUID, + }).Info("Target beat configuration loaded successfully!") + + return beatInfo, nil +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..c0b71e3 --- /dev/null +++ b/readme.md @@ -0,0 +1,57 @@ +beat-exporter for Prometheus += +[![Build Status](https://travis-ci.com/trustpilot/beat-exporter.svg?token=iiryqu5r7QUR7uXsnd8L&branch=master)](https://travis-ci.com/trustpilot/beat-exporter) + +Exposes (file|metric)beat statistics from beats statistics endpoint to prometheus format, automaticly configuring collectors for apporiate beat type. + +Current coverage +- + + * filebeat + * metricbeat + * packetbeat - _partial_ + * auditbeat - _partial_ + +Setup +- + +Edit your *beat configuration and add following: + +``` +http: + enabled: true + host: localhost + port: 5066 +``` + +This will expose `(file|metrics|*)beat` http endpoint at given port. + +Run beat-exporter: +``` +$ ./beat-exporter +``` + +beat-exported default port for prometheus is: `9479` + +Point your Prometheus to `0.0.0.0:9479/metrics` + +Configuration reference +- +``` +$ ./beat-exporter -help +Usage of ./beat-exporter: + -beat.timeout duration + Timeout for trying to get stats from beat. (default 10s) + -beat.uri string + HTTP API address of beat. (default "http://localhost:5066") + -version + Show version and exit + -web.listen-address string + Address to listen on for web interface and telemetry. (default ":9479") + -web.telemetry-path string + Path under which to expose metrics. (default "/metrics") +``` + +Contribution +- +Please use pull requests, issues \ No newline at end of file