add consul_exporter integration

.github/depcheck.yml

@@ -15,3 +15,4 @@ github_repos:
   - github.com/prometheus/mysqld_exporter v0.12.1
   - github.com/prometheus/prometheus v2.21.0
   - github.com/wrouesnel/postgres_exporter v0.8.0
+  - github.com/prometheus/consul_exporter v0.7.1

CHANGELOG.md

@@ -13,6 +13,9 @@ this platform. FreeBSD builds will return in a future release.
 - [FEATURE] New integration:
   [statsd_exporter](https://github.com/prometheus/statsd_exporter) (@rfratto)
 
+- [FEATURE] New integration:
+  [consul_exporter](https://github.com/prometheus/consul_exporter) (@rfratto)
+
 - [ENHANCEMENT] `min_wal_time` and `max_wal_time` have been added to the
   instance config settings, guaranteeing that data in the WAL will exist for at
   least `min_wal_time` and will not exist for longer than `max_wal_time`. This

cmd/agent/agent-integrations-config.yaml

@@ -38,6 +38,8 @@ integrations:
     timeout: 10s
   statsd_exporter:
     enabled: true
+  consul_exporter:
+    enabled: true
   prometheus_remote_write:
     - url: http://localhost:9009/api/prom/push
 

docs/configuration-reference.md

@@ -1928,6 +1928,9 @@ postgres_exporter: <postgres_exporter_config>
 # Controls the statsd_exporter integration
 statsd_exporter: <statsd_exporter_config>
 
+# Controls the consul_exporter integration
+consul_exporter: <consul_exporter_config>
+
 # Automatically collect metrics from enabled integrations. If disabled,
 # integrations will be run but not scraped and thus not remote_written. Metrics
 # for integrations will be exposed at /integrations/<integration_key>/metrics
@@ -3023,3 +3026,94 @@ Full reference of options:
   # Parse SignalFX style tags.
   [parse_signalfx_tags: <bool> | default = true]
 ```
+
+### consul_exporter_config
+
+The `consul_exporter_config` block configures the `consul_exporter`
+integration, which is an embedded version of
+[`consul_exporter`](https://github.com/prometheus/consul_exporter). This allows
+for the collection of consul metrics and exposing them as Prometheus metrics.
+
+Full reference of options:
+
+```yaml
+  # Enables the consul_exporter integration, allowing the Agent to automatically
+  # collect system metrics from the configured consul server address
+  [enabled: <boolean> | default = false]
+
+  # Automatically collect metrics from this integration. If disabled,
+  # the consul_exporter integration will be run but not scraped and thus not
+  # remote-written. Metrics for the integration will be exposed at
+  # /integrations/consul_exporter/metrics and can be scraped by an external
+  # process.
+  [scrape_integration: <boolean> | default = <integrations_config.scrape_integrations>]
+
+  # How often should the metrics be collected? Defaults to
+  # prometheus.global.scrape_interval.
+  [scrape_interval: <duration> | default = <global_config.scrape_interval>]
+
+  # The timeout before considering the scrape a failure. Defaults to
+  # prometheus.global.scrape_timeout.
+  [scrape_timeout: <duration> | default = <global_config.scrape_timeout>]
+
+  # Allows for relabeling labels on the target.
+  relabel_configs:
+    [- <relabel_config> ... ]
+
+  # Relabel metrics coming from the integration, allowing to drop series
+  # from the integration that you don't care about.
+  metric_relabel_configs:
+    [ - <relabel_config> ... ]
+
+  # Monitor the exporter itself and include those metrics in the results.
+  [include_exporter_metrics: <bool> | default = false]
+
+  #
+  # Exporter-specific configuration options
+  #
+
+  # Prefix from which to expose key/value pairs.
+  [kv_prefix: <string> | default = ""]
+
+  # Regex that determines which keys to expose.
+  [kv_filter: <string> | default = ".*"]
+
+  # Generate a health summary for each service instance. Needs n+1 queries to
+  # collect all information.
+  [generate_health_summary: <bool> | default = true]
+
+  # HTTP API address of a Consul server or agent. Prefix with https:// to
+  # connect using HTTPS.
+  [server: <string> | default = "http://localhost:8500"]
+
+  # Disable TLS host verification.
+  [insecure_skip_verify: <bool> | default = false]
+
+  # File path to a PEM-encoded certificate authority used to validate the
+  # authenticity of a server certificate.
+  [ca_file: <string> | default = ""]
+
+  # File path to a PEM-encoded certificate used with the private key to verify
+  # the exporter's authenticity.
+  [cert_file: <string> | default = ""]
+
+  # File path to a PEM-encoded private key used with the certificate to verify
+  # the exporter's authenticity.
+  [key_file: <string> | default = ""]
+
+  # When provided, this overrides the hostname for the TLS certificate. It can
+  # be used to ensure that the certificate name matches the hostname we declare.
+  [server_name: <string> | default = ""]
+
+  # Timeout on HTTP requests to the Consul API.
+  [timeout: <duration> | default = "500ms"]
+
+  # Limit the maximum number of concurrent requests to consul. 0 means no limit.
+  [concurrent_request_limit: <int> | default = 0]
+
+  # Allows any Consul server (non-leader) to service a read. 
+  [allow_stale: <bool> | default = true]
+
+  # Forces the read to be fully consistent.
+  [require_consistent: <bool> | default = false]
+```

example/docker-compose/docker-compose.integrations.yaml

@@ -90,3 +90,8 @@ services:
     image: memcached
     ports:
       - "11211:11211"
+
+  consul:
+    image: consul
+    ports:
+      - "8500:8500"

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/google/dnsmasq_exporter v0.0.0-00010101000000-000000000000
 	github.com/gorilla/mux v1.8.0
 	github.com/grafana/loki v1.6.2-0.20201026154740-6978ee5d7387
+	github.com/hashicorp/consul/api v1.7.0
 	github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d // indirect
 	github.com/jsternberg/zap-logfmt v1.2.0
 	github.com/miekg/dns v1.1.31
@@ -24,6 +25,7 @@ require (
 	github.com/prometheus/client_golang v1.8.0
 	github.com/prometheus/common v0.15.0
 	github.com/prometheus/memcached_exporter v0.7.1-0.20201204153924-4290508e6dcf
+	github.com/prometheus/consul_exporter v0.0.0-00010101000000-000000000000
 	github.com/prometheus/mysqld_exporter v0.0.0-00010101000000-000000000000
 	github.com/prometheus/node_exporter v1.0.1
 	github.com/prometheus/procfs v0.2.0
@@ -72,6 +74,7 @@ replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20200521142311-984
 replace (
 	github.com/google/dnsmasq_exporter => github.com/grafana/dnsmasq_exporter v0.2.1-0.20201029182940-e5169b835a23
 	github.com/ncabatoff/process-exporter => github.com/grafana/process-exporter v0.7.3-0.20200902205007-6343dc1182cf
+	github.com/prometheus/consul_exporter => github.com/grafana/consul_exporter v0.7.2-0.20201202203923-5390bb9ecffc
 	github.com/prometheus/mysqld_exporter => github.com/grafana/mysqld_exporter v0.12.2-0.20201015182516-5ac885b2d38a
 	github.com/wrouesnel/postgres_exporter => github.com/grafana/postgres_exporter v0.8.1-0.20201106170118-5eedee00c1db
 )

go.sum

@@ -958,6 +958,8 @@ github.com/gostaticanalysis/analysisutil v0.0.3 h1:iwp+5/UAyzQSFgQ4uR2sni99sJ8Eo
 github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gotestyourself/gotestyourself v1.3.0/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
 github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
+github.com/grafana/consul_exporter v0.7.2-0.20201202203923-5390bb9ecffc h1:jMbe0Qwt3AZ88usjD/NM3E9n9j6BlgnTG45S7SQ/V64=
+github.com/grafana/consul_exporter v0.7.2-0.20201202203923-5390bb9ecffc/go.mod h1:OwZlAa0i3Erc/Rokn9HM3zk3KKfZGM/TeHscq/kdj5o=
 github.com/grafana/dnsmasq_exporter v0.2.1-0.20201029175341-d5c660a4eb09 h1:APAFkjGkNdNMq6vPkxJ1dNOXsMInppMY7Y4IUO2JZo0=
 github.com/grafana/dnsmasq_exporter v0.2.1-0.20201029175341-d5c660a4eb09/go.mod h1:VkXXMzcoJSroCo6+avVowsPU78ikVuFVYGSCyzrCtXU=
 github.com/grafana/dnsmasq_exporter v0.2.1-0.20201029182940-e5169b835a23 h1:unz+d8qr+wZl1V1GiA/tY/I8BW/ZomsMpC+8RbhqkzE=

pkg/integrations/consul_exporter/consul_exporter.go

@@ -0,0 +1,84 @@
+// Package consul_exporter embeds https://github.com/prometheus/consul_exporter
+package consul_exporter //nolint:golint
+
+import (
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/grafana/agent/pkg/integrations/common"
+	"github.com/grafana/agent/pkg/integrations/config"
+	consul_api "github.com/hashicorp/consul/api"
+	"github.com/prometheus/consul_exporter/pkg/exporter"
+)
+
+var DefaultConfig = Config{
+	Server:        "http://localhost:8500",
+	Timeout:       500 * time.Millisecond,
+	AllowStale:    true,
+	KVFilter:      ".*",
+	HealthSummary: true,
+}
+
+// Config controls the consul_exporter integration.
+type Config struct {
+	// Enabled enables the integration.
+	Enabled bool `yaml:"enabled"`
+
+	CommonConfig config.Common `yaml:",inline"`
+
+	Server             string        `yaml:"server"`
+	CAFile             string        `yaml:"ca_file"`
+	CertFile           string        `yaml:"cert_file"`
+	KeyFile            string        `yaml:"key_file"`
+	ServerName         string        `yaml:"server_name"`
+	Timeout            time.Duration `yaml:"timeout"`
+	InsecureSkipVerify bool          `yaml:"insecure_skip_verify"`
+	RequestLimit       int           `yaml:"concurrent_request_limit"`
+	AllowStale         bool          `yaml:"allow_stale"`
+	RequireConsistent  bool          `yaml:"require_consistent"`
+
+	KVPrefix      string `yaml:"kv_prefix"`
+	KVFilter      string `yaml:"kv_filter"`
+	HealthSummary bool   `yaml:"generate_health_summary"`
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler for Config.
+func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultConfig
+
+	type plain Config
+	return unmarshal((*plain)(c))
+}
+
+// New creates a new consul_exporter integration. The integration scrapes
+// metrics from a consul process.
+func New(log log.Logger, c Config) (common.Integration, error) {
+	var (
+		consulOpts = exporter.ConsulOpts{
+			CAFile:       c.CAFile,
+			CertFile:     c.CertFile,
+			Insecure:     c.InsecureSkipVerify,
+			KeyFile:      c.KeyFile,
+			RequestLimit: c.RequestLimit,
+			ServerName:   c.ServerName,
+			Timeout:      c.Timeout,
+			URI:          c.Server,
+		}
+		queryOptions = consul_api.QueryOptions{
+			AllowStale:        c.AllowStale,
+			RequireConsistent: c.RequireConsistent,
+		}
+	)
+
+	e, err := exporter.New(consulOpts, queryOptions, c.KVPrefix, c.KVFilter, c.HealthSummary, log)
+	if err != nil {
+		return nil, err
+	}
+
+	return common.NewCollectorIntegration(
+		"consul_exporter",
+		c.CommonConfig,
+		e,
+		false,
+	), nil
+}

pkg/integrations/manager.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/grafana/agent/pkg/integrations/agent"
 	"github.com/grafana/agent/pkg/integrations/common"
+	"github.com/grafana/agent/pkg/integrations/consul_exporter"
 	"github.com/grafana/agent/pkg/integrations/dnsmasq_exporter"
 	"github.com/grafana/agent/pkg/integrations/memcached_exporter"
 	"github.com/grafana/agent/pkg/integrations/mysqld_exporter"
@@ -68,6 +69,7 @@ type Config struct {
 	MemcachedExporter memcached_exporter.Config `yaml:"memcached_exporter"`
 	PostgresExporter  postgres_exporter.Config  `yaml:"postgres_exporter"`
 	StatsdExporter    statsd_exporter.Config    `yaml:"statsd_exporter"`
+	ConsulExporter    consul_exporter.Config    `yaml:"consul_exporter"`
 
 	// Extra labels to add for all integration samples
 	Labels model.LabelSet `yaml:"labels"`
@@ -206,6 +208,14 @@ func NewManager(c Config, logger log.Logger, im instance.Manager) (*Manager, err
 		}
 		integrations = append(integrations, i)
 	}
+	if c.ConsulExporter.Enabled {
+		l := log.With(logger, "integration", "consul_exporter")
+		i, err := consul_exporter.New(l, c.ConsulExporter)
+		if err != nil {
+			return nil, err
+		}
+		integrations = append(integrations, i)
+	}
 
 	return newManager(c, logger, im, integrations)
 }