diff --git a/example/docker-compose/agent/config/agent-integrations-next.yaml b/example/docker-compose/agent/config/agent-integrations-next.yaml new file mode 100644 index 000000000000..78428dd40246 --- /dev/null +++ b/example/docker-compose/agent/config/agent-integrations-next.yaml @@ -0,0 +1,23 @@ +# agent-integrations-next.yaml is intended to be used with +# docker-compose.integrations-next.yaml. It demonstrates a config with +# --enable-features=integrations-next and --config.expand-env, but does not +# test absolutely all integrations. +server: + log_level: info + http_listen_port: 12345 + +metrics: + wal_directory: /tmp/agent + global: + scrape_interval: 1m + remote_write: + - url: http://${REMOTE_WRITE:-localhost:9009}/api/prom/push + configs: + - name: default + +integrations: + agent: {} + node_exporter: {} + redis_exporter_configs: + - redis_addr: ${REDIS_1:-localhost:6379} + - redis_addr: ${REDIS_2:-localhost:6380} diff --git a/example/docker-compose/docker-compose.integrations-next.yaml b/example/docker-compose/docker-compose.integrations-next.yaml new file mode 100644 index 000000000000..5a0a00be6f10 --- /dev/null +++ b/example/docker-compose/docker-compose.integrations-next.yaml @@ -0,0 +1,90 @@ +# The docker-compose.integrations-next.yaml runs an agent environment with +# -enable-features=integrations-next being used. +version: "2" +services: + agent: + image: grafana/agent:latest + volumes: + - ./agent/config:/etc/agent-config + - /var/log:/var/log + entrypoint: + - /bin/agent + - -config.file=/etc/agent-config/agent-integrations-next.yaml + - -prometheus.wal-directory=/tmp/agent/wal + - -enable-features=integrations-next + - -config.expand-env + - -config.enable-read-api + environment: + REMOTE_WRITE: cortex:9009 + REDIS_1: redis_1:6379 + REDIS_2: redis_2:6379 + HOSTNAME: agent + ports: + - "12345:12345" + depends_on: + - cortex + - loki + - tempo + - redis_1 + - redis_2 + + # + # Frontend + # + + grafana: + image: grafana/grafana:8.3.3 + entrypoint: + - /usr/share/grafana/bin/grafana-server + - --homepath=/usr/share/grafana + - --config=/etc/grafana-config/grafana.ini + volumes: + - ./grafana/config:/etc/grafana-config + - ./grafana/datasources:/etc/grafana/provisioning/datasources + - ./grafana/dashboards-provisioning:/etc/grafana/provisioning/dashboards + - ./grafana/dashboards:/var/lib/grafana/dashboards + ports: + - "3000:3000" + + # + # Backend for remote_write components + # + + cortex: + image: cortexproject/cortex:v1.8.1 + volumes: + - /tmp/cortex:/tmp/cortex + - ./cortex/config:/etc/cortex-config + entrypoint: + - /bin/cortex + - -config.file=/etc/cortex-config/cortex.yaml + ports: + - "9009:9009" + + loki: + image: grafana/loki:1.5.0 + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + + tempo: + image: grafana/tempo:df7225ae + command: + - "-storage.trace.backend=local" # tell tempo where to permanently put traces + - "-storage.trace.local.path=/tmp/tempo/traces" + - "-storage.trace.wal.path=/tmp/tempo/wal" # tell tempo where to store the wal + - "-auth.enabled=false" # disables the requirement for the X-Scope-OrgID header + - "-server.http-listen-port=3200" + ports: + - "3200:3200" + + # + # Integrations + # + + redis_1: + image: redis:6 + ports: ["6379:6379"] + redis_2: + image: redis:6 + ports: ["6380:6379"] diff --git a/pkg/integrations/consul_exporter/consul_exporter.go b/pkg/integrations/consul_exporter/consul_exporter.go index b8f59406508d..82acbb40441a 100644 --- a/pkg/integrations/consul_exporter/consul_exporter.go +++ b/pkg/integrations/consul_exporter/consul_exporter.go @@ -8,6 +8,8 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" consul_api "github.com/hashicorp/consul/api" "github.com/prometheus/consul_exporter/pkg/exporter" ) @@ -68,6 +70,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new consul_exporter integration. The integration scrapes diff --git a/pkg/integrations/dnsmasq_exporter/dnsmasq_exporter.go b/pkg/integrations/dnsmasq_exporter/dnsmasq_exporter.go index 6df60bd5c7ca..93dd2ee1f837 100644 --- a/pkg/integrations/dnsmasq_exporter/dnsmasq_exporter.go +++ b/pkg/integrations/dnsmasq_exporter/dnsmasq_exporter.go @@ -5,6 +5,8 @@ import ( "github.com/go-kit/log" "github.com/google/dnsmasq_exporter/collector" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/miekg/dns" ) @@ -48,6 +50,7 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new dnsmasq_exporter integration. The integration scrapes metrics diff --git a/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go b/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go index 8e404dde8df6..6ffbb75d9cf0 100644 --- a/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go +++ b/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go @@ -12,6 +12,8 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus-community/elasticsearch_exporter/collector" @@ -88,6 +90,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new elasticsearch_exporter diff --git a/pkg/integrations/github_exporter/github_exporter.go b/pkg/integrations/github_exporter/github_exporter.go index ba016f42fa9a..b9997eb4ee07 100644 --- a/pkg/integrations/github_exporter/github_exporter.go +++ b/pkg/integrations/github_exporter/github_exporter.go @@ -7,6 +7,8 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" gh_config "github.com/infinityworks/github-exporter/config" "github.com/infinityworks/github-exporter/exporter" config_util "github.com/prometheus/common/config" @@ -67,6 +69,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new github_exporter integration. diff --git a/pkg/integrations/install/shims.go b/pkg/integrations/install/shims.go index afd8099df33f..af8ac573844e 100644 --- a/pkg/integrations/install/shims.go +++ b/pkg/integrations/install/shims.go @@ -3,7 +3,6 @@ package install import ( v1 "github.com/grafana/agent/pkg/integrations" v2 "github.com/grafana/agent/pkg/integrations/v2" - "github.com/grafana/agent/pkg/integrations/v2/common" metricsutils "github.com/grafana/agent/pkg/integrations/v2/metricsutils" ) @@ -25,9 +24,7 @@ func init() { } } if !found { - v2.RegisterLegacy(v1Integration, v2.TypeSingleton, func(cfg v1.Config, common common.MetricsConfig) v2.UpgradedConfig { - return metricsutils.CreateShim(cfg, common) - }) + v2.RegisterLegacy(v1Integration, v2.TypeSingleton, metricsutils.CreateShim) } } } diff --git a/pkg/integrations/kafka_exporter/kafka_exporter.go b/pkg/integrations/kafka_exporter/kafka_exporter.go index 53b5610421ce..2db14ec309df 100644 --- a/pkg/integrations/kafka_exporter/kafka_exporter.go +++ b/pkg/integrations/kafka_exporter/kafka_exporter.go @@ -9,6 +9,8 @@ import ( kafka_exporter "github.com/davidmparrott/kafka_exporter/v2/exporter" "github.com/go-kit/log" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" ) // DefaultConfig holds the default settings for the kafka_lag_exporter @@ -121,6 +123,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new kafka_exporter integration. diff --git a/pkg/integrations/memcached_exporter/memcached_exporter.go b/pkg/integrations/memcached_exporter/memcached_exporter.go index 727b835de3d6..c117528450bf 100644 --- a/pkg/integrations/memcached_exporter/memcached_exporter.go +++ b/pkg/integrations/memcached_exporter/memcached_exporter.go @@ -6,6 +6,8 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/prometheus/memcached_exporter/pkg/exporter" ) @@ -49,6 +51,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new memcached_exporter integration. The integration scrapes metrics diff --git a/pkg/integrations/mongodb_exporter/mongodb_exporter.go b/pkg/integrations/mongodb_exporter/mongodb_exporter.go index 16d8459c903a..e666d54c2b34 100644 --- a/pkg/integrations/mongodb_exporter/mongodb_exporter.go +++ b/pkg/integrations/mongodb_exporter/mongodb_exporter.go @@ -6,6 +6,8 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/percona/mongodb_exporter/exporter" config_util "github.com/prometheus/common/config" ) @@ -44,6 +46,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new mongodb_exporter integration. diff --git a/pkg/integrations/mysqld_exporter/mysqld-exporter.go b/pkg/integrations/mysqld_exporter/mysqld-exporter.go index 25565b631f28..618f0d602e3e 100644 --- a/pkg/integrations/mysqld_exporter/mysqld-exporter.go +++ b/pkg/integrations/mysqld_exporter/mysqld-exporter.go @@ -12,6 +12,8 @@ import ( "github.com/go-kit/log/level" "github.com/go-sql-driver/mysql" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/prometheus/mysqld_exporter/collector" ) @@ -103,6 +105,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new mysqld_exporter integration. The integration scrapes diff --git a/pkg/integrations/postgres_exporter/postgres_exporter.go b/pkg/integrations/postgres_exporter/postgres_exporter.go index f5f2838892e6..af65f3455364 100644 --- a/pkg/integrations/postgres_exporter/postgres_exporter.go +++ b/pkg/integrations/postgres_exporter/postgres_exporter.go @@ -10,6 +10,8 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/lib/pq" "github.com/prometheus-community/postgres_exporter/exporter" ) @@ -120,6 +122,7 @@ func (c *Config) getDataSourceNames() ([]string, error) { func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new postgres_exporter integration. The integration scrapes diff --git a/pkg/integrations/redis_exporter/redis_exporter.go b/pkg/integrations/redis_exporter/redis_exporter.go index 21c818916bbf..2a71d35c1e44 100644 --- a/pkg/integrations/redis_exporter/redis_exporter.go +++ b/pkg/integrations/redis_exporter/redis_exporter.go @@ -8,6 +8,8 @@ import ( "time" "github.com/grafana/agent/pkg/integrations" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -116,6 +118,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // New creates a new redis_exporter integration. The integration queries diff --git a/pkg/integrations/statsd_exporter/statsd_exporter.go b/pkg/integrations/statsd_exporter/statsd_exporter.go index 265b7c79a12a..a5fdcc8ac67d 100644 --- a/pkg/integrations/statsd_exporter/statsd_exporter.go +++ b/pkg/integrations/statsd_exporter/statsd_exporter.go @@ -10,14 +10,12 @@ import ( "strconv" "time" - "github.com/prometheus/statsd_exporter/pkg/mappercache/randomreplacement" - - "github.com/prometheus/statsd_exporter/pkg/mappercache/lru" - "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/agent/pkg/integrations" "github.com/grafana/agent/pkg/integrations/config" + integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" + "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/version" @@ -27,6 +25,8 @@ import ( "github.com/prometheus/statsd_exporter/pkg/line" "github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/mapper" + "github.com/prometheus/statsd_exporter/pkg/mappercache/lru" + "github.com/prometheus/statsd_exporter/pkg/mappercache/randomreplacement" "gopkg.in/yaml.v2" ) @@ -94,6 +94,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) func init() { integrations.RegisterIntegration(&Config{}) + integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim) } // Exporter defines the statsd_exporter integration. diff --git a/pkg/integrations/v2/agent/agent.go b/pkg/integrations/v2/agent/agent.go index 047d6cca6655..e151998572f4 100644 --- a/pkg/integrations/v2/agent/agent.go +++ b/pkg/integrations/v2/agent/agent.go @@ -22,6 +22,9 @@ func (c *Config) Name() string { return "agent" } // ApplyDefaults applies runtime-specific defaults to c. func (c *Config) ApplyDefaults(globals integrations.Globals) error { c.Common.ApplyDefaults(globals.SubsystemOpts.Metrics.Autoscrape) + if id, err := c.Identifier(globals); err == nil { + c.Common.InstanceKey = &id + } return nil } diff --git a/pkg/integrations/v2/metricsutils/versionshim.go b/pkg/integrations/v2/metricsutils/versionshim.go index a547d6d249b2..4d64d5b5b5df 100644 --- a/pkg/integrations/v2/metricsutils/versionshim.go +++ b/pkg/integrations/v2/metricsutils/versionshim.go @@ -38,6 +38,9 @@ func (s *configShim) Name() string { return s.orig.Name() } func (s *configShim) ApplyDefaults(g v2.Globals) error { s.common.ApplyDefaults(g.SubsystemOpts.Metrics.Autoscrape) + if id, err := s.Identifier(g); err == nil { + s.common.InstanceKey = &id + } return nil } @@ -50,6 +53,9 @@ func (s *configShim) ConfigEquals(c v2.Config) bool { } func (s *configShim) Identifier(g v2.Globals) (string, error) { + if s.common.InstanceKey != nil { + return *s.common.InstanceKey, nil + } return s.orig.InstanceKey(g.AgentIdentifier) } diff --git a/pkg/integrations/v2/register_test.go b/pkg/integrations/v2/register_test.go index 0c8121eb69b8..d09baeb73185 100644 --- a/pkg/integrations/v2/register_test.go +++ b/pkg/integrations/v2/register_test.go @@ -6,7 +6,6 @@ import ( "github.com/go-kit/log" v1 "github.com/grafana/agent/pkg/integrations" - "github.com/grafana/agent/pkg/integrations/github_exporter" "github.com/grafana/agent/pkg/integrations/v2/common" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" @@ -80,15 +79,15 @@ test_configs: func TestIntegrationRegistration_Legacy(t *testing.T) { setRegistered(t, nil) - RegisterLegacy(&github_exporter.Config{}, TypeSingleton, func(in v1.Config, mc common.MetricsConfig) UpgradedConfig { + RegisterLegacy(&legacyConfig{}, TypeSingleton, func(in v1.Config, mc common.MetricsConfig) UpgradedConfig { return &legacyShim{Data: in, Common: mc} }) var cfgToParse = ` name: John Doe duration: 500ms -github_exporter: - api_url: nowhere` +legacy: + text: hello` var fullCfg testFullConfig err := yaml.UnmarshalStrict([]byte(cfgToParse), &fullCfg) @@ -98,12 +97,51 @@ github_exporter: require.IsType(t, &legacyShim{}, fullCfg.Configs[0]) shim := fullCfg.Configs[0].(*legacyShim) - require.IsType(t, &github_exporter.Config{}, shim.Data) + require.IsType(t, &legacyConfig{}, shim.Data) - v1Config := shim.Data.(*github_exporter.Config) - require.Equal(t, "nowhere", v1Config.APIURL) + v1Config := shim.Data.(*legacyConfig) + require.Equal(t, "hello", v1Config.Text) } +func TestIntegrationRegistration_Legacy_Multiplex(t *testing.T) { + setRegistered(t, nil) + + RegisterLegacy(&legacyConfig{}, TypeMultiplex, func(in v1.Config, mc common.MetricsConfig) UpgradedConfig { + return &legacyShim{Data: in, Common: mc} + }) + + var cfgToParse = ` +name: John Doe +duration: 500ms +legacy_configs: + - text: hello + - text: world` + + var fullCfg testFullConfig + err := yaml.UnmarshalStrict([]byte(cfgToParse), &fullCfg) + require.NoError(t, err) + + require.Len(t, fullCfg.Configs, 2) + require.IsType(t, &legacyShim{}, fullCfg.Configs[0]) + require.IsType(t, &legacyShim{}, fullCfg.Configs[1]) + + shim := fullCfg.Configs[0].(*legacyShim) + require.IsType(t, &legacyConfig{}, shim.Data) + require.Equal(t, "hello", shim.Data.(*legacyConfig).Text) + + shim = fullCfg.Configs[1].(*legacyShim) + require.IsType(t, &legacyConfig{}, shim.Data) + require.Equal(t, "world", shim.Data.(*legacyConfig).Text) +} + +type legacyConfig struct { + Text string `yaml:"text"` +} + +func (lc *legacyConfig) Name() string { return "legacy" } +func (lc *legacyConfig) InstanceKey(agentKey string) (string, error) { return agentKey, nil } +func (lc *legacyConfig) NewIntegration(l log.Logger) (v1.Integration, error) { return nil, nil } + type legacyShim struct { Data v1.Config Common common.MetricsConfig