diff --git a/CHANGELOG.md b/CHANGELOG.md index a2815a3454fb..6f7926ac221e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Main (unreleased) +- [FEATURE] Added [Github exporter](https://github.com/infinityworks/github-exporter) integration. (@rgeyer) + - [FEATURE] Add TLS config options for tempo `remote_write`s. (@mapno) - [FEATURE] Add support for OTLP HTTP trace exporting. (@mapno) diff --git a/docs/configuration/integrations/_index.md b/docs/configuration/integrations/_index.md index 9adcd034dc21..8e56df4f68a7 100644 --- a/docs/configuration/integrations/_index.md +++ b/docs/configuration/integrations/_index.md @@ -83,6 +83,9 @@ windows_exporter: # Controls the kafka_exporter integration kafka_exporter: +# Controls the github_exporter integration +github_exporter: + # Automatically collect metrics from enabled integrations. If disabled, # integrations will be run but not scraped and thus not remote_written. Metrics # for integrations will be exposed at /integrations//metrics diff --git a/docs/configuration/integrations/github-exporter-config.md b/docs/configuration/integrations/github-exporter-config.md new file mode 100644 index 000000000000..88075498296e --- /dev/null +++ b/docs/configuration/integrations/github-exporter-config.md @@ -0,0 +1,72 @@ ++++ +title = "github_exporter_config" ++++ + +# github_exporter_config + +The `github_exporter_config` block configures the `github_exporter` integration, +which is an embedded version of +[`github_exporter`](https://github.com/infinityworks/github-exporter). This allows for the collection of metrics from the github api. + +Full reference of options: + +```yaml + # Enables the github_exporter integration, allowing the Agent to automatically + # collect metrics for the specified github objects. + [enabled: | default = false] + + # Automatically collect metrics from this integration. If disabled, + # the github_exporter integration will be run but not scraped and thus not + # remote-written. Metrics for the integration will be exposed at + # /integrations/github_exporter/metrics and can be scraped by an external + # process. + [scrape_integration: | default = ] + + # How often should the metrics be collected? Defaults to + # prometheus.global.scrape_interval. + [scrape_interval: | default = ] + + # The timeout before considering the scrape a failure. Defaults to + # prometheus.global.scrape_timeout. + [scrape_timeout: | default = ] + + # Allows for relabeling labels on the target. + relabel_configs: + [- ... ] + + # Relabel metrics coming from the integration, allowing to drop series + # from the integration that you don't care about. + metric_relabel_configs: + [ - ... ] + + # How frequent to truncate the WAL for this integration. + [wal_truncate_frequency: | default = "60m"] + + # + # Exporter-specific configuration options + # + + # The full URI of the github API. + [api_url: | default = "https://api.github.com"] + + # A list of github repositories for which to collect metrics. + repositories: + [ - ] + + # A list of github organizations for which to collect metrics. + organizations: + [ - ] + + # A list of github users for which to collect metrics. + users: + [ - ] + + # A github authentication token that allows the API to be queried more often. + # Optional, but recommended. + [api_token: ] + + # A path to a file containing a github authentication token that allows the + # API to be queried more often. If supplied, this supercedes `api_token` + # Optional, but recommended. + [api_token_file: ] +``` diff --git a/example/docker-compose/agent/config/agent-local.yaml b/example/docker-compose/agent/config/agent-local.yaml index 8996b5f4e453..43e8006aa6c1 100644 --- a/example/docker-compose/agent/config/agent-local.yaml +++ b/example/docker-compose/agent/config/agent-local.yaml @@ -72,4 +72,7 @@ integrations: kafka_exporter: enabled: true kafka_uris: [localhost:9093] - + github_exporter: + enabled: true + repositories: + - grafana/agent diff --git a/go.mod b/go.mod index 2389a141955f..6e75fb565329 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/grafana/loki v1.6.2-0.20210429132126-d88f3996eaa2 github.com/hashicorp/consul/api v1.8.1 github.com/hashicorp/go-getter v1.5.3 + github.com/infinityworks/github-exporter v0.0.0-20201016091012-831b72461034 github.com/jsternberg/zap-logfmt v1.2.0 github.com/justwatchcom/elasticsearch_exporter v1.1.0 github.com/klauspost/compress v1.13.1 // indirect @@ -99,11 +100,11 @@ replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20200521142311-984 // TODO(rfratto): remove forks when changes are merged upstream replace ( github.com/google/dnsmasq_exporter => github.com/grafana/dnsmasq_exporter v0.2.1-0.20201029182940-e5169b835a23 + github.com/infinityworks/github-exporter => github.com/rgeyer/github-exporter v0.0.0-20210722215637-d0cec2ee0dc8 github.com/ncabatoff/process-exporter => github.com/grafana/process-exporter v0.7.3-0.20210106202358-831154072e2a github.com/prometheus-community/postgres_exporter => github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520 github.com/prometheus-community/windows_exporter => github.com/grafana/windows_exporter v0.15.1-0.20210325142439-9e8f66d53433 github.com/prometheus/mysqld_exporter => github.com/grafana/mysqld_exporter v0.12.2-0.20201015182516-5ac885b2d38a - ) // Required for redis_exporter, which is incompatible with v2.0.0+incompatible. diff --git a/go.sum b/go.sum index 9b3bd53e8d5d..4fadba5e2f62 100644 --- a/go.sum +++ b/go.sum @@ -1021,6 +1021,8 @@ github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU= +github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/go-syslog/v2 v2.0.1/go.mod h1:hjvie1UTaD5E1fTnDmxaCw8RRDrT4Ve+XHr5O2dKSCo= github.com/influxdata/go-syslog/v3 v3.0.1-0.20201128200927-a1889d947b48 h1:0WbZ+ZVg74wbyQoRx1TD4D1Xoz8MsXJSTwdP9F7RMeQ= @@ -1546,6 +1548,8 @@ github.com/rfratto/go-yaml v0.0.0-20200521142311-984fc90c8a04 h1:1HAVwfHi7d7nGq7 github.com/rfratto/go-yaml v0.0.0-20200521142311-984fc90c8a04/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= github.com/rfratto/weaveworks-common v0.0.0-20210326192855-c95210d58ba7 h1:pi0dqcmXlW/cEqOG3OH0yy3BLHwmshd0w+0JLmK4dX4= github.com/rfratto/weaveworks-common v0.0.0-20210326192855-c95210d58ba7/go.mod h1:ykzWac1LtVfOxdCK+jD754at1Ws9dKCwFeUzkFBffPs= +github.com/rgeyer/github-exporter v0.0.0-20210722215637-d0cec2ee0dc8 h1:wNuNGrFzFmZlhrtz1Q8EiK1Ob6yWli8lX7D2AGmSGzE= +github.com/rgeyer/github-exporter v0.0.0-20210722215637-d0cec2ee0dc8/go.mod h1:6XoOvFDTfk3aqGaOLHLxoWiZNx4zHobApOhKc3oHF/g= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1665,6 +1669,7 @@ github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/y github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/steinfletcher/apitest v1.3.8/go.mod h1:LOVbGzWvWCiiVE4PZByfhRnA5L00l5uZQEx403xQ4K8= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/streadway/amqp v0.0.0-20180528204448-e5adc2ada8b8/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -1715,6 +1720,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/tonistiigi/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:Q5IRRDY+cjIaiOjTAnXN5LKQV5MPqVx5ofQn85Jy5Yw= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.20.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= diff --git a/pkg/integrations/github_exporter/github_exporter.go b/pkg/integrations/github_exporter/github_exporter.go new file mode 100644 index 000000000000..33b8399e4bfc --- /dev/null +++ b/pkg/integrations/github_exporter/github_exporter.go @@ -0,0 +1,100 @@ +package github_exporter //nolint:golint + +import ( + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/grafana/agent/pkg/integrations" + "github.com/grafana/agent/pkg/integrations/config" + gh_config "github.com/infinityworks/github-exporter/config" + "github.com/infinityworks/github-exporter/exporter" +) + +// DefaultConfig holds the default settings for the github_exporter integration +var DefaultConfig Config = Config{ + APIURL: "https://api.github.com", +} + +// Config controls github_exporter +type Config struct { + Common config.Common `yaml:",inline"` + + // URL for the github API + APIURL string `yaml:"api_url,omitempty"` + + // A list of github repositories for which to collect metrics. + Repositories []string `yaml:"repositories,omitempty"` + + // A list of github organizations for which to collect metrics. + Organizations []string `yaml:"organizations,omitempty"` + + // A list of github users for which to collect metrics. + Users []string `yaml:"users,omitempty"` + + // A github authentication token that allows the API to be queried more often. + APIToken string `yaml:"api_token,omitempty"` + + // A path to a file containing a github authentication token that allows the API to be queried more often. If supplied, this supercedes `api_token` + APITokenFile string `yaml:"api_token_file,omitempty"` +} + +// UnmarshalYAML implements yaml.Unmarshaler for Config +func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultConfig + + type plain Config + return unmarshal((*plain)(c)) +} + +// Name returns the name of the integration that this config represents. +func (c *Config) Name() string { + return "github_exporter" +} + +// CommonConfig returns the common settings shared across all configs for +// integrations. +func (c *Config) CommonConfig() config.Common { + return c.Common +} + +// NewIntegration creates a new github_exporter +func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, error) { + return New(logger, c) +} + +func init() { + integrations.RegisterIntegration(&Config{}) +} + +// New creates a new github_exporter integration. +func New(logger log.Logger, c *Config) (integrations.Integration, error) { + + conf := gh_config.Config{} + err := conf.SetAPIURL(c.APIURL) + if err != nil { + level.Error(logger).Log("msg", "api url is invalid", "err", err) + return nil, err + } + conf.SetRepositories(c.Repositories) + conf.SetOrganisations(c.Organizations) + conf.SetUsers(c.Users) + if c.APIToken != "" { + conf.SetAPIToken(c.APIToken) + } + if c.APITokenFile != "" { + err = conf.SetAPITokenFromFile(c.APITokenFile) + if err != nil { + level.Error(logger).Log("msg", "unable to load Github API token from file", "err", err) + return nil, err + } + } + + ghExporter := exporter.Exporter{ + APIMetrics: exporter.AddMetrics(), + Config: conf, + } + + return integrations.NewCollectorIntegration( + c.Name(), + integrations.WithCollectors(&ghExporter), + ), nil +} diff --git a/pkg/integrations/install/install.go b/pkg/integrations/install/install.go index a9a270578530..c2dd2c6238ce 100644 --- a/pkg/integrations/install/install.go +++ b/pkg/integrations/install/install.go @@ -6,6 +6,7 @@ import ( _ "github.com/grafana/agent/pkg/integrations/consul_exporter" // register consul_exporter _ "github.com/grafana/agent/pkg/integrations/dnsmasq_exporter" // register dnsmasq_exporter _ "github.com/grafana/agent/pkg/integrations/elasticsearch_exporter" // register elasticsearch_exporter + _ "github.com/grafana/agent/pkg/integrations/github_exporter" // register github_exporter _ "github.com/grafana/agent/pkg/integrations/kafka_exporter" // register kafka_exporter _ "github.com/grafana/agent/pkg/integrations/memcached_exporter" // register memcached_exporter _ "github.com/grafana/agent/pkg/integrations/mysqld_exporter" // register mysqld_exporter