diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5ba8269 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +vendor +.build +.tarballs +beat-exporter \ No newline at end of file diff --git a/.promu.yml b/.promu.yml new file mode 100644 index 0000000..a0b0a64 --- /dev/null +++ b/.promu.yml @@ -0,0 +1,23 @@ +verbose: true +repository: + path: github.com/trustpilot/beat-exporter +build: + flags: -a -tags 'netgo static_build' + ldflags: | + -s + -X {{repoPath}}/vendor/github.com/prometheus/common/version.Version={{.Version}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.Revision={{.Revision}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.Branch={{.Branch}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.BuildUser={{user}}@{{host}} + -X {{repoPath}}/vendor/github.com/prometheus/common/version.BuildDate={{date "20060102-15:04:05"}} +tarball: + files: + - LICENSE +crossbuild: + platforms: + - linux/amd64 + - linux/386 + - darwin/amd64 + - darwin/386 + - windows/amd64 + - windows/386 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..86c8bcc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +dist: trusty +sudo: required +language: go + +go: + - 1.10.x + +go_import_path: github.com/trustpilot/beat-exporter + +services: + - docker + +before_script: + - make dependencies + +script: + - make travis + +branches: + only: master \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c130a8c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM quay.io/prometheus/golang-builder as builder + +ADD . /go/src/github.com/trustpilot/beat-exporter +WORKDIR /go/src/github.com/trustpilot/beat-exporter + +RUN make + +FROM quay.io/prometheus/busybox:latest +MAINTAINER Audrius Karabanovas + +COPY --from=builder /go/src/github.com/trustpilot/beat-exporter/beat-exporter /bin/beat-exporter + +EXPOSE 9479 +ENTRYPOINT [ "/bin/beat-exporter" ] \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..a4cbfeb --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,110 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + digest = "1:c0bec5f9b98d0bc872ff5e834fac186b807b656683bd29cb82fb207a1513fabb" + name = "github.com/beorn7/perks" + packages = ["quantile"] + pruneopts = "" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" + name = "github.com/golang/protobuf" + packages = ["proto"] + pruneopts = "" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" + +[[projects]] + digest = "1:63722a4b1e1717be7b98fc686e0b30d5e7f734b9e93d7dee86293b6deab7ea28" + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + pruneopts = "" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + digest = "1:4142d94383572e74b42352273652c62afec5b23f325222ed09198f46009022d1" + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/promhttp", + ] + pruneopts = "" + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" + +[[projects]] + branch = "master" + digest = "1:185cf55b1f44a1bf243558901c3f06efa5c64ba62cfdcbb1bf7bbe8c3fb68561" + name = "github.com/prometheus/client_model" + packages = ["go"] + pruneopts = "" + revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" + +[[projects]] + branch = "master" + digest = "1:f477ef7b65d94fb17574fc6548cef0c99a69c1634ea3b6da248b63a61ebe0498" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + "version", + ] + pruneopts = "" + revision = "c7de2306084e37d54b8be01f3541a8464345e9a5" + +[[projects]] + branch = "master" + digest = "1:e04aaa0e8f8da0ed3d6c0700bd77eda52a47f38510063209d72d62f0ef807d5e" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs", + ] + pruneopts = "" + revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" + +[[projects]] + digest = "1:3fcbf733a8d810a21265a7f2fe08a3353db2407da052b233f8b204b5afc03d9b" + name = "github.com/sirupsen/logrus" + packages = ["."] + pruneopts = "" + revision = "3e01752db0189b9157070a0e1668a620f9a85da2" + version = "v1.0.6" + +[[projects]] + branch = "master" + digest = "1:793a79198b755828dec284c6f1325e24e09186f1b7ba818b65c7c35104ed86eb" + name = "golang.org/x/crypto" + packages = ["ssh/terminal"] + pruneopts = "" + revision = "614d502a4dac94afa3a6ce146bd1736da82514c6" + +[[projects]] + branch = "master" + digest = "1:7a5f7a1206de6b90f67cb465e489eac3298e95afa7262813b542df4fab38952f" + name = "golang.org/x/sys" + packages = [ + "unix", + "windows", + ] + pruneopts = "" + revision = "4910a1d54f876d7b22162a85f4d066d3ee649450" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/prometheus/common/version", + "github.com/sirupsen/logrus", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..fcb0951 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,11 @@ +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "0.8.0" + +[[constraint]] + branch = "master" + name = "github.com/prometheus/common" + +[[constraint]] + name = "github.com/sirupsen/logrus" + version = "1.0.6" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6550a90 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Trustpilot + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2122c2b --- /dev/null +++ b/Makefile @@ -0,0 +1,77 @@ +# Copyright 2016 The Prometheus Authors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO := GO15VENDOREXPERIMENT=1 go +PROMU := $(GOPATH)/bin/promu +pkgs = $(shell $(GO) list ./... | grep -v /vendor/) + +PREFIX ?= $(shell pwd) +BIN_DIR ?= $(shell pwd) +DOCKER_IMAGE_NAME ?= beat-exporter +DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) + + +all: format dependencies build test + +style: + @echo ">> checking code style" + @! gofmt -d $(shell find . -path ./vendor -prune -o -name '*.go' -print) | grep '^' + +test: + @echo ">> running tests" + @$(GO) test -short $(pkgs) + +format: + @echo ">> formatting code" + @$(GO) fmt $(pkgs) + +vet: + @echo ">> vetting code" + @$(GO) vet $(pkgs) + +dependencies: + @echo ">> installing dep" + @$(GO) get -u github.com/golang/dep/cmd/dep + @echo ">> running dep ensure" + @dep ensure + +build: promu + @echo ">> building binaries" + @$(PROMU) build --prefix $(PREFIX) + +crossbuild: promu + @echo ">> cross-building binaries" + @$(PROMU) crossbuild + +tarball: promu + @echo ">> building release tarball" + @$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR) + +tarballs: promu + @echo ">> building release tarballs" + @$(PROMU) crossbuild tarballs $(BIN_DIR) + +travis: dependencies format build + @$(PROMU) info + +docker: + @echo ">> building docker image" + @docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" . + +promu: + @echo ">> installing promu" + @GOOS=$(shell uname -s | tr A-Z a-z) \ + GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(shell uname -m))) \ + $(GO) get -u github.com/prometheus/promu + +.PHONY: all style format build test vet tarball docker promu travis dependencies \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6c6aa7c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file diff --git a/collector/beat.go b/collector/beat.go new file mode 100644 index 0000000..9cc846b --- /dev/null +++ b/collector/beat.go @@ -0,0 +1,182 @@ +package collector + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +//CPUTimings json structure +type CPUTimings struct { + Ticks float64 `json:"ticks"` + Time struct { + MS float64 `json:"ms"` + } `json:"time"` + Value float64 `json:"value"` +} + +//BeatStats json structure +type BeatStats struct { + CPU struct { + Sytem CPUTimings `json:"system"` + Total CPUTimings `json:"total"` + User CPUTimings `json:"user"` + } `json:"cpu"` + BeatUptime struct { + Uptime struct { + MS float64 `json:"ms"` + } `json:"uptime"` + + EmphemeralID string `json:"emphemeral_id"` + } `json:"info"` + + Memstats struct { + GCNext float64 `json:"gc_next"` + MemoryAlloc float64 `json:"memory_alloc"` + MemoryTotal float64 `json:"memory_total"` + RSS float64 `json:"rss"` + } `json:"memstats"` +} + +type beatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewBeatCollector constructor +func NewBeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &beatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu_time", "miliseconds"), + "beat.cpu.time", + nil, prometheus.Labels{"mode": "system"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Sytem.Time.MS }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu_time", "miliseconds"), + "beat.cpu.time", + nil, prometheus.Labels{"mode": "user"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.User.Time.MS }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu_time", "miliseconds"), + "beat.cpu.time", + nil, prometheus.Labels{"mode": "total"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Total.Time.MS }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu", "ticks"), + "beat.cpu.ticks", + nil, prometheus.Labels{"mode": "system"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Sytem.Ticks }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu", "ticks"), + "beat.cpu.ticks", + nil, prometheus.Labels{"mode": "user"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.User.Ticks }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "cpu", "ticks"), + "beat.cpu.ticks", + nil, prometheus.Labels{"mode": "total"}, + ), + eval: func(stats *Stats) float64 { return stats.Beat.CPU.Total.Ticks }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "uptime", "seconds"), + "beat.info.uptime.ms", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return (time.Duration(stats.Beat.BeatUptime.Uptime.MS) * time.Millisecond).Seconds() + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "gc_next"), + "beat.memstats.gc_next", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.GCNext + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "memory_alloc"), + "beat.memstats.memory_alloc", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.MemoryAlloc + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "memory_total"), + "beat.memstats.memory_total", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.MemoryTotal + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "memstats", "rss"), + "beat.memstats.rss", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.Beat.Memstats.RSS + }, + valType: prometheus.GaugeValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *beatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *beatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/filebeat.go b/collector/filebeat.go new file mode 100644 index 0000000..7c58fa0 --- /dev/null +++ b/collector/filebeat.go @@ -0,0 +1,155 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//Filebeat json structure +type Filebeat struct { + Events struct { + Active float64 `json:"active"` + Added float64 `json:"added"` + Done float64 `json:"done"` + } `json:"events"` + + Harvester struct { + Closed float64 `json:"closed"` + OpenFiles float64 `json:"open_files"` + Running float64 `json:"running"` + Skipped float64 `json:"skipped"` + Started float64 `json:"started"` + } `json:"harvester"` + + Prospector struct { + Log struct { + Files struct { + Renamed float64 `json:"renamed"` + Truncated float64 `json:"truncated"` + } `json:"files"` + } `json:"log"` + } `json:"prospector"` +} + +type filebeatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewFilebeatCollector constructor +func NewFilebeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &filebeatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "events"), + "filebeat.events", + nil, prometheus.Labels{"event": "active"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Events.Active }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "events"), + "filebeat.events", + nil, prometheus.Labels{"event": "added"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Events.Added }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "events"), + "filebeat.events", + nil, prometheus.Labels{"event": "done"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Events.Done }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "closed"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Closed }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "open_files"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.OpenFiles }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "running"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Running }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "skipped"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Skipped }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "harvester"), + "filebeat.harvester", + nil, prometheus.Labels{"harvester": "started"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Harvester.Started }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "prospector_log"), + "filebeat.prospector_log", + nil, prometheus.Labels{"files": "renamed"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Prospector.Log.Files.Renamed }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "filebeat", "prospector_log"), + "filebeat.prospector_log", + nil, prometheus.Labels{"files": "truncated"}, + ), + eval: func(stats *Stats) float64 { return stats.Filebeat.Prospector.Log.Files.Truncated }, + valType: prometheus.UntypedValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *filebeatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *filebeatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/libbeat.go b/collector/libbeat.go new file mode 100644 index 0000000..f732047 --- /dev/null +++ b/collector/libbeat.go @@ -0,0 +1,344 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//LibBeat json structure +type LibBeat struct { + Config struct { + Module struct { + Running float64 `json:"running"` + Starts float64 `json:"starts"` + Stops float64 `json:"stops"` + } `json:"module"` + Reloads float64 `json:"reloads"` + } `json:"config"` + Output LibBeatOutput `json:"output"` + Pipeline LibBeatPipeline `json:"pipeline"` +} + +//LibBeatEvents json structure +type LibBeatEvents struct { + Acked float64 `json:"acked"` + Active float64 `json:"active"` + Batches float64 `json:"batches"` + Dropped float64 `json:"dropped"` + Duplicates float64 `json:"duplicates"` + Failed float64 `json:"failed"` + Filtered float64 `json:"filtered"` + Published float64 `json:"published"` + Retry float64 `json:"retry"` +} + +//LibBeatOutputBytesErrors json structure +type LibBeatOutputBytesErrors struct { + Bytes float64 `json:"bytes"` + Errors float64 `json:"errors"` +} + +//LibBeatOutput json structure +type LibBeatOutput struct { + Events LibBeatEvents `json:"events"` + Read LibBeatOutputBytesErrors `json:"read"` + Write LibBeatOutputBytesErrors `json:"write"` + Type string `json:"type"` +} + +//LibBeatPipeline json structure +type LibBeatPipeline struct { + Clients float64 `json:"clients"` + Events LibBeatEvents `json:"events"` + Queue struct { + Acked float64 `json:"acked"` + } `json:"queue"` +} + +type libbeatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +var libbeatOutputType *prometheus.Desc + +// NewLibBeatCollector constructor +func NewLibBeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &libbeatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat_config", "reloads"), + "libbeat.config.reloads", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Reloads + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "config"), + "libbeat.config.module", + nil, prometheus.Labels{"module": "running"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Module.Running + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "config"), + "libbeat.config.module", + nil, prometheus.Labels{"module": "starts"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Module.Starts + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "config"), + "libbeat.config.module", + nil, prometheus.Labels{"module": "stops"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Config.Module.Stops + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_read_bytes"), + "libbeat.output.read.bytes", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Read.Bytes + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_read_errors"), + "libbeat.output.read.errors", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Read.Errors + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_write_bytes"), + "libbeat.output.write.bytes", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Write.Bytes + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_write_errors"), + "libbeat.output.write.errors", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Write.Errors + }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "acked"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Acked + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "active"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Active + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "batches"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Batches + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "dropped"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Dropped + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "duplicates"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Duplicates + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "output_events"), + "libbeat.output.events", + nil, prometheus.Labels{"type": "failed"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Output.Events.Failed + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_clients"), + "libbeat.pipeline.clients", + nil, nil, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Clients + }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_queue"), + "libbeat.pipeline.queue", + nil, prometheus.Labels{"type": "acked"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Queue.Acked + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "active"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Active + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "dropped"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Dropped + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "failed"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Failed + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "filtered"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Filtered + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "published"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Published + }, + valType: prometheus.UntypedValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "libbeat", "pipeline_events"), + "libbeat.pipeline.events", + nil, prometheus.Labels{"type": "retry"}, + ), + eval: func(stats *Stats) float64 { + return stats.LibBeat.Pipeline.Events.Retry + }, + valType: prometheus.UntypedValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *libbeatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + + libbeatOutputType = prometheus.NewDesc( + prometheus.BuildFQName(c.beatInfo.Beat, "libbeat", "output"), + "libbeat.output.type", + []string{"type"}, nil, + ) + + ch <- libbeatOutputType + +} + +// Collect returns the current state of all metrics of the collector. +func (c *libbeatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + + // output.type with dynamic label + ch <- prometheus.MustNewConstMetric(libbeatOutputType, prometheus.CounterValue, float64(1), c.stats.LibBeat.Output.Type) + +} diff --git a/collector/main.go b/collector/main.go new file mode 100644 index 0000000..fb59f74 --- /dev/null +++ b/collector/main.go @@ -0,0 +1,148 @@ +package collector + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "regexp" + + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" +) + +type mainCollector struct { + Collectors map[string]prometheus.Collector + Stats *Stats + client *http.Client + beatURL *url.URL + name string + beatInfo *BeatInfo + targetDesc *prometheus.Desc + targetUp *prometheus.Desc + metrics exportedMetrics +} + +// HackfixRegex regex to replace JSON part +var HackfixRegex = regexp.MustCompile("\"time\":(\\d+)") // replaces time:123 to time.ms:123, only filebeat has different naming of time metric + +// NewMainCollector constructor +func NewMainCollector(client *http.Client, url *url.URL, name string, beatInfo *BeatInfo) prometheus.Collector { + instance := fmt.Sprintf("%s:%s", url.Hostname(), url.Port()) + beat := &mainCollector{ + Collectors: make(map[string]prometheus.Collector), + Stats: &Stats{}, + client: client, + beatURL: url, + name: name, + targetDesc: prometheus.NewDesc( + prometheus.BuildFQName(name, "target", "info"), + "target information", + nil, + prometheus.Labels{"version": beatInfo.Version, "beat": beatInfo.Beat, "uri": instance}), + targetUp: prometheus.NewDesc( + prometheus.BuildFQName("", beatInfo.Beat, "up"), + "Target up", + nil, + nil), + + beatInfo: beatInfo, + metrics: exportedMetrics{}, + } + + beat.Collectors["system"] = NewSystemCollector(beatInfo, beat.Stats) + beat.Collectors["beat"] = NewBeatCollector(beatInfo, beat.Stats) + beat.Collectors["libbeat"] = NewLibBeatCollector(beatInfo, beat.Stats) + beat.Collectors["registrar"] = NewRegistrarCollector(beatInfo, beat.Stats) + beat.Collectors["filebeat"] = NewFilebeatCollector(beatInfo, beat.Stats) + beat.Collectors["metricbeat"] = NewMetricbeatCollector(beatInfo, beat.Stats) + + return beat +} + +// Describe returns all descriptions of the collector. +func (b *mainCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- b.targetDesc + ch <- b.targetUp + + for _, metric := range b.metrics { + ch <- metric.desc + } + + // standard collectors for all types of beats + b.Collectors["system"].Describe(ch) + b.Collectors["beat"].Describe(ch) + b.Collectors["libbeat"].Describe(ch) + + // Customized collectors per beat type + switch b.beatInfo.Beat { + case "filebeat": + b.Collectors["filebeat"].Describe(ch) + b.Collectors["registrar"].Describe(ch) + case "metricbeat": + b.Collectors["metricbeat"].Describe(ch) + } + +} + +// Collect returns the current state of all metrics of the collector. +func (b *mainCollector) Collect(ch chan<- prometheus.Metric) { + + err := b.fetchStatsEndpoint() + if err != nil { + ch <- prometheus.MustNewConstMetric(b.targetUp, prometheus.GaugeValue, float64(0)) // set target down + log.Errorf("Failed getting /stats endpoint of target") + return + } + + ch <- prometheus.MustNewConstMetric(b.targetDesc, prometheus.GaugeValue, float64(1)) + ch <- prometheus.MustNewConstMetric(b.targetUp, prometheus.GaugeValue, float64(1)) // target up + + for _, i := range b.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(b.Stats)) + } + + // standard collectors for all types of beats + b.Collectors["system"].Collect(ch) + b.Collectors["beat"].Collect(ch) + b.Collectors["libbeat"].Collect(ch) + + // Customized collectors per beat type + switch b.beatInfo.Beat { + case "filebeat": + b.Collectors["filebeat"].Collect(ch) + b.Collectors["registrar"].Collect(ch) + case "metricbeat": + b.Collectors["metricbeat"].Collect(ch) + } + +} + +func (b *mainCollector) fetchStatsEndpoint() error { + + response, err := b.client.Get(b.beatURL.String() + "/stats") + if err != nil { + log.Errorf("Could not fetch stats endpoint of target: %v", b.beatURL.String()) + return err + } + + defer response.Body.Close() + + bodyBytes, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Error("Can't read body of response") + return err + } + + // @TODO remove this when filebeat stats endpoint output matches all other beats output + bodyBytes = HackfixRegex.ReplaceAll(bodyBytes, []byte("\"time\":{\"ms\":$1}")) + + err = json.Unmarshal(bodyBytes, &b.Stats) + if err != nil { + log.Error("Could not parse JSON response for target") + return err + } + + return nil +} diff --git a/collector/metricbeat.go b/collector/metricbeat.go new file mode 100644 index 0000000..674b5b5 --- /dev/null +++ b/collector/metricbeat.go @@ -0,0 +1,222 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//MetricbeatEvent json structure +type MetricbeatEvent struct { + Failures float64 `json:"failures"` + Success float64 `json:"success"` +} + +//Metricbeat json structure +type Metricbeat struct { + System struct { + CPU MetricbeatEvent `json:"cpu"` + Filesystem MetricbeatEvent `json:"filesystem"` + Fsstat MetricbeatEvent `json:"fsstat"` + Load MetricbeatEvent `json:"load"` + Memory MetricbeatEvent `json:"memory"` + Network MetricbeatEvent `json:"network"` + Process MetricbeatEvent `json:"process"` + ProcessSummary MetricbeatEvent `json:"process_summary"` + Uptime MetricbeatEvent `json:"uptime"` + } `json:"system"` +} + +type metricbeatCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewMetricbeatCollector constructor +func NewMetricbeatCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &metricbeatCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "cpu"), + "system.cpu", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.CPU.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "cpu"), + "system.cpu", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.CPU.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "filesystem"), + "system.filesystem", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Filesystem.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "filesystem"), + "system.filesystem", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Filesystem.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "fsstat"), + "system.fsstat", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Fsstat.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "fsstat"), + "system.fsstat", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Fsstat.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "load"), + "system.load", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Load.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "load"), + "system.load", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Load.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "memory"), + "system.memory", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Memory.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "memory"), + "system.memory", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Memory.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "network"), + "system.network", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Network.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "network"), + "system.network", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Network.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process"), + "system.process", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Process.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process"), + "system.process", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Process.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process_summary"), + "system.process_summary", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.ProcessSummary.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "process_summary"), + "system.process_summary", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.ProcessSummary.Failures }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "uptime"), + "system.uptime", + nil, prometheus.Labels{"event": "success"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Uptime.Success }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "metricbeat_system", "uptime"), + "system.uptime", + nil, prometheus.Labels{"event": "failures"}, + ), + eval: func(stats *Stats) float64 { return stats.Metricbeat.System.Uptime.Failures }, + valType: prometheus.CounterValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *metricbeatCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *metricbeatCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/registrar.go b/collector/registrar.go new file mode 100644 index 0000000..83100ee --- /dev/null +++ b/collector/registrar.go @@ -0,0 +1,85 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//Registrar json structure +type Registrar struct { + Writes float64 `json:"writes"` + States struct { + Cleanup float64 `json:"cleanup"` + Current float64 `json:"current"` + Update float64 `json:"update"` + } `json:"states"` +} + +type registrarCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewRegistrarCollector constructor +func NewRegistrarCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return ®istrarCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "writes"), + "registrar.writes", + nil, nil, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.Writes }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "states"), + "registrar.states", + nil, prometheus.Labels{"state": "cleanup"}, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.States.Cleanup }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "states"), + "registrar.states", + nil, prometheus.Labels{"state": "current"}, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.States.Current }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "registrar", "states"), + "registrar.states", + nil, prometheus.Labels{"state": "update"}, + ), + eval: func(stats *Stats) float64 { return stats.Registrar.States.Update }, + valType: prometheus.GaugeValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *registrarCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *registrarCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/collector/structs.go b/collector/structs.go new file mode 100644 index 0000000..8bc69a4 --- /dev/null +++ b/collector/structs.go @@ -0,0 +1,30 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//BeatInfo beat info json structure +type BeatInfo struct { + Beat string `json:"beat"` + Hostname string `json:"hostname"` + Name string `json:"name"` + UUID string `json:"uuid"` + Version string `json:"version"` +} + +//Stats stats endpoint json structure +type Stats struct { + System System `json:"system"` + Beat BeatStats `json:"beat"` + LibBeat LibBeat `json:"libbeat"` + Registrar Registrar `json:"registrar"` + Filebeat Filebeat `json:"filebeat"` + Metricbeat Metricbeat `json:"metricbeat"` +} + +type exportedMetrics []struct { + desc *prometheus.Desc + eval func(stats *Stats) float64 + valType prometheus.ValueType +} diff --git a/collector/system.go b/collector/system.go new file mode 100644 index 0000000..4afddb1 --- /dev/null +++ b/collector/system.go @@ -0,0 +1,119 @@ +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +//CPUStats json structure +type CPUStats struct { + M1 float64 `json:"1"` + M5 float64 `json:"5"` + M15 float64 `json:"15"` +} + +//System json structure +type System struct { + CPU struct { + Cores int64 `json:"cores"` + } `json:"cpu"` + Load struct { + CPUStats + Norm CPUStats `json:"norm"` + } `json:"load"` +} +type systemCollector struct { + beatInfo *BeatInfo + stats *Stats + metrics exportedMetrics +} + +// NewSystemCollector constructor +func NewSystemCollector(beatInfo *BeatInfo, stats *Stats) prometheus.Collector { + return &systemCollector{ + beatInfo: beatInfo, + stats: stats, + metrics: exportedMetrics{ + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_cpu", "cores"), + "cpu cores", + nil, nil, + ), + eval: func(stats *Stats) float64 { return float64(stats.System.CPU.Cores) }, + valType: prometheus.CounterValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system", "load"), + "system load 1m", + nil, prometheus.Labels{"period": "1"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.M1 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system", "load"), + "system load 1m", + nil, prometheus.Labels{"period": "5"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.M5 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system", "load"), + "system load 1m", + nil, prometheus.Labels{"period": "15"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.M15 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_load", "norm"), + "system load 1m", + nil, prometheus.Labels{"period": "1"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.Norm.M1 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_load", "norm"), + "system load 1m", + nil, prometheus.Labels{"period": "5"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.Norm.M5 }, + valType: prometheus.GaugeValue, + }, + { + desc: prometheus.NewDesc( + prometheus.BuildFQName(beatInfo.Beat, "system_load", "norm"), + "system load 1m", + nil, prometheus.Labels{"period": "15"}, + ), + eval: func(stats *Stats) float64 { return stats.System.Load.Norm.M15 }, + valType: prometheus.GaugeValue, + }, + }, + } +} + +// Describe returns all descriptions of the collector. +func (c *systemCollector) Describe(ch chan<- *prometheus.Desc) { + + for _, metric := range c.metrics { + ch <- metric.desc + } + +} + +// Collect returns the current state of all metrics of the collector. +func (c *systemCollector) Collect(ch chan<- prometheus.Metric) { + + for _, i := range c.metrics { + ch <- prometheus.MustNewConstMetric(i.desc, i.valType, i.eval(c.stats)) + } + +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..cf1f2a1 --- /dev/null +++ b/main.go @@ -0,0 +1,158 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/version" + "github.com/trustpilot/beat-exporter/collector" +) + +func main() { + var ( + Name = "beat_exporter" + listenAddress = flag.String("web.listen-address", ":9479", "Address to listen on for web interface and telemetry.") + metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") + beatURI = flag.String("beat.uri", "http://localhost:5066", "HTTP API address of beat.") + beatTimeout = flag.Duration("beat.timeout", 10*time.Second, "Timeout for trying to get stats from beat.") + showVersion = flag.Bool("version", false, "Show version and exit") + ) + flag.Parse() + + if *showVersion { + fmt.Print(version.Print(Name)) + os.Exit(0) + } + + log.SetLevel(log.DebugLevel) + + log.SetFormatter(&log.JSONFormatter{ + FieldMap: log.FieldMap{ + log.FieldKeyMsg: "message", + }, + }) + + beatURL, err := url.Parse(*beatURI) + + if err != nil { + log.Fatalf("failed to parse beat.uri, error: %v", err) + } + + httpClient := &http.Client{ + Timeout: *beatTimeout, + } + + log.Info("Exploring target for beat type") + + var beatInfo *collector.BeatInfo + + for { + beatInfo, err = loadBeatType(httpClient, *beatURL) + if err != nil { + log.Errorf("Could not load beat type, with error: %v, retrying in 5s", err) + time.Sleep(5 * time.Second) + } else { + break + } + } + + // version metric + registry := prometheus.NewRegistry() + versionMetric := version.NewCollector(Name) + mainCollector := collector.NewMainCollector(httpClient, beatURL, Name, beatInfo) + registry.MustRegister(versionMetric) + registry.MustRegister(mainCollector) + + http.Handle(*metricsPath, promhttp.HandlerFor( + registry, + promhttp.HandlerOpts{ + ErrorLog: log.New(), + DisableCompression: false, + ErrorHandling: promhttp.ContinueOnError}), + ) + + http.HandleFunc("/", IndexHandler(*metricsPath)) + + log.WithFields(log.Fields{ + "addr": *listenAddress, + }).Infof("Starting exporter with configured type: %s", beatInfo.Beat) + + if err := http.ListenAndServe(*listenAddress, nil); err != nil { + + log.WithFields(log.Fields{ + "err": err, + }).Errorf("http server quit with error: %v", err) + + } +} + +// IndexHandler returns a http handler with the correct metricsPath +func IndexHandler(metricsPath string) http.HandlerFunc { + indexHTML := ` + + + Beat Exporter + + +

Beat Exporter

+

+ Metrics +

+ + +` + index := []byte(fmt.Sprintf(strings.TrimSpace(indexHTML), metricsPath)) + + return func(w http.ResponseWriter, r *http.Request) { + w.Write(index) + } +} + +func loadBeatType(client *http.Client, url url.URL) (*collector.BeatInfo, error) { + beatInfo := &collector.BeatInfo{} + + response, err := client.Get(url.String()) + if err != nil { + return beatInfo, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + log.Errorf("Beat URL: %q status code: %d", url.String(), response.StatusCode) + return beatInfo, err + } + + bodyBytes, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Error("Can't read body of response") + return beatInfo, err + } + + err = json.Unmarshal(bodyBytes, &beatInfo) + if err != nil { + log.Error("Could not parse JSON response for target") + return beatInfo, err + } + + log.WithFields( + log.Fields{ + "beat": beatInfo.Beat, + "version": beatInfo.Version, + "name": beatInfo.Name, + "hostname": beatInfo.Hostname, + "uuid": beatInfo.UUID, + }).Info("Target beat configuration loaded successfully!") + + return beatInfo, nil +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..c0b71e3 --- /dev/null +++ b/readme.md @@ -0,0 +1,57 @@ +beat-exporter for Prometheus += +[![Build Status](https://travis-ci.com/trustpilot/beat-exporter.svg?token=iiryqu5r7QUR7uXsnd8L&branch=master)](https://travis-ci.com/trustpilot/beat-exporter) + +Exposes (file|metric)beat statistics from beats statistics endpoint to prometheus format, automaticly configuring collectors for apporiate beat type. + +Current coverage +- + + * filebeat + * metricbeat + * packetbeat - _partial_ + * auditbeat - _partial_ + +Setup +- + +Edit your *beat configuration and add following: + +``` +http: + enabled: true + host: localhost + port: 5066 +``` + +This will expose `(file|metrics|*)beat` http endpoint at given port. + +Run beat-exporter: +``` +$ ./beat-exporter +``` + +beat-exported default port for prometheus is: `9479` + +Point your Prometheus to `0.0.0.0:9479/metrics` + +Configuration reference +- +``` +$ ./beat-exporter -help +Usage of ./beat-exporter: + -beat.timeout duration + Timeout for trying to get stats from beat. (default 10s) + -beat.uri string + HTTP API address of beat. (default "http://localhost:5066") + -version + Show version and exit + -web.listen-address string + Address to listen on for web interface and telemetry. (default ":9479") + -web.telemetry-path string + Path under which to expose metrics. (default "/metrics") +``` + +Contribution +- +Please use pull requests, issues \ No newline at end of file