[dev.multiple-integrations] Multiple instances of integrations

example/docker-compose/agent/config/agent-integrations-next.yaml

@@ -0,0 +1,23 @@
+# agent-integrations-next.yaml is intended to be used with
+# docker-compose.integrations-next.yaml. It demonstrates a config with
+# --enable-features=integrations-next and --config.expand-env, but does not
+# test absolutely all integrations.
+server:
+  log_level: info
+  http_listen_port: 12345
+
+metrics:
+  wal_directory: /tmp/agent
+  global:
+    scrape_interval: 1m
+    remote_write:
+      - url: http://${REMOTE_WRITE:-localhost:9009}/api/prom/push
+  configs:
+    - name: default
+
+integrations:
+  agent: {}
+  node_exporter: {}
+  redis_exporter_configs:
+  - redis_addr: ${REDIS_1:-localhost:6379}
+  - redis_addr: ${REDIS_2:-localhost:6380}

example/docker-compose/docker-compose.integrations-next.yaml

@@ -0,0 +1,90 @@
+# The docker-compose.integrations-next.yaml runs an agent environment with
+# -enable-features=integrations-next being used.
+version: "2"
+services:
+  agent:
+    image: grafana/agent:latest
+    volumes:
+      - ./agent/config:/etc/agent-config
+      - /var/log:/var/log
+    entrypoint:
+      - /bin/agent
+      - -config.file=/etc/agent-config/agent-integrations-next.yaml
+      - -prometheus.wal-directory=/tmp/agent/wal
+      - -enable-features=integrations-next
+      - -config.expand-env
+      - -config.enable-read-api
+    environment:
+      REMOTE_WRITE: cortex:9009
+      REDIS_1: redis_1:6379
+      REDIS_2: redis_2:6379
+      HOSTNAME: agent
+    ports:
+      - "12345:12345"
+    depends_on:
+      - cortex
+      - loki
+      - tempo
+      - redis_1
+      - redis_2
+
+  #
+  # Frontend
+  #
+
+  grafana:
+    image: grafana/grafana:8.3.3
+    entrypoint:
+      - /usr/share/grafana/bin/grafana-server
+      - --homepath=/usr/share/grafana
+      - --config=/etc/grafana-config/grafana.ini
+    volumes:
+      - ./grafana/config:/etc/grafana-config
+      - ./grafana/datasources:/etc/grafana/provisioning/datasources
+      - ./grafana/dashboards-provisioning:/etc/grafana/provisioning/dashboards
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+    ports:
+      - "3000:3000"
+
+  #
+  # Backend for remote_write components
+  #
+
+  cortex:
+    image: cortexproject/cortex:v1.8.1
+    volumes:
+      - /tmp/cortex:/tmp/cortex
+      - ./cortex/config:/etc/cortex-config
+    entrypoint:
+      - /bin/cortex
+      - -config.file=/etc/cortex-config/cortex.yaml
+    ports:
+      - "9009:9009"
+
+  loki:
+    image: grafana/loki:1.5.0
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+
+  tempo:
+    image: grafana/tempo:df7225ae
+    command:
+      - "-storage.trace.backend=local"                  # tell tempo where to permanently put traces
+      - "-storage.trace.local.path=/tmp/tempo/traces"
+      - "-storage.trace.wal.path=/tmp/tempo/wal"        # tell tempo where to store the wal
+      - "-auth.enabled=false"                           # disables the requirement for the X-Scope-OrgID header
+      - "-server.http-listen-port=3200"
+    ports:
+      - "3200:3200"
+
+  #
+  # Integrations
+  #
+
+  redis_1:
+    image: redis:6
+    ports: ["6379:6379"]
+  redis_2:
+    image: redis:6
+    ports: ["6380:6379"]

pkg/integrations/consul_exporter/consul_exporter.go

@@ -8,6 +8,8 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	consul_api "github.com/hashicorp/consul/api"
 	"github.com/prometheus/consul_exporter/pkg/exporter"
 )
@@ -68,6 +70,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error)
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new consul_exporter integration. The integration scrapes

pkg/integrations/dnsmasq_exporter/dnsmasq_exporter.go

@@ -5,6 +5,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/google/dnsmasq_exporter/collector"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/miekg/dns"
 )
 
@@ -48,6 +50,7 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new dnsmasq_exporter integration. The integration scrapes metrics

pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go

@@ -12,6 +12,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus-community/elasticsearch_exporter/collector"
@@ -88,6 +90,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new elasticsearch_exporter

pkg/integrations/github_exporter/github_exporter.go

@@ -7,6 +7,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	gh_config "github.com/infinityworks/github-exporter/config"
 	"github.com/infinityworks/github-exporter/exporter"
 	config_util "github.com/prometheus/common/config"
@@ -67,6 +69,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new github_exporter integration.

pkg/integrations/install/shims.go

@@ -3,7 +3,6 @@ package install
 import (
 	v1 "github.com/grafana/agent/pkg/integrations"
 	v2 "github.com/grafana/agent/pkg/integrations/v2"
-	"github.com/grafana/agent/pkg/integrations/v2/common"
 	metricsutils "github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 )
 
@@ -25,9 +24,7 @@ func init() {
 			}
 		}
 		if !found {
-			v2.RegisterLegacy(v1Integration, v2.TypeSingleton, func(cfg v1.Config, common common.MetricsConfig) v2.UpgradedConfig {
-				return metricsutils.CreateShim(cfg, common)
-			})
+			v2.RegisterLegacy(v1Integration, v2.TypeSingleton, metricsutils.CreateShim)
 		}
 	}
 }

pkg/integrations/kafka_exporter/kafka_exporter.go

@@ -9,6 +9,8 @@ import (
 	kafka_exporter "github.com/davidmparrott/kafka_exporter/v2/exporter"
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 )
 
 // DefaultConfig holds the default settings for the kafka_lag_exporter
@@ -121,6 +123,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new kafka_exporter integration.

pkg/integrations/memcached_exporter/memcached_exporter.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/prometheus/memcached_exporter/pkg/exporter"
 )
 
@@ -49,6 +51,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error)
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new memcached_exporter integration. The integration scrapes metrics

pkg/integrations/mongodb_exporter/mongodb_exporter.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/percona/mongodb_exporter/exporter"
 	config_util "github.com/prometheus/common/config"
 )
@@ -44,6 +46,7 @@ func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, er
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new mongodb_exporter integration.

pkg/integrations/mysqld_exporter/mysqld-exporter.go

@@ -12,6 +12,8 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/go-sql-driver/mysql"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/prometheus/mysqld_exporter/collector"
 )
 
@@ -103,6 +105,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error)
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new mysqld_exporter integration. The integration scrapes

pkg/integrations/postgres_exporter/postgres_exporter.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/lib/pq"
 	"github.com/prometheus-community/postgres_exporter/exporter"
 )
@@ -120,6 +122,7 @@ func (c *Config) getDataSourceNames() ([]string, error) {
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new postgres_exporter integration. The integration scrapes

pkg/integrations/redis_exporter/redis_exporter.go

@@ -8,6 +8,8 @@ import (
 	"time"
 
 	"github.com/grafana/agent/pkg/integrations"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
@@ -116,6 +118,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error)
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // New creates a new redis_exporter integration. The integration queries

pkg/integrations/statsd_exporter/statsd_exporter.go

@@ -10,14 +10,12 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/prometheus/statsd_exporter/pkg/mappercache/randomreplacement"
-
-	"github.com/prometheus/statsd_exporter/pkg/mappercache/lru"
-
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/integrations/config"
+	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
+	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/version"
@@ -27,6 +25,8 @@ import (
 	"github.com/prometheus/statsd_exporter/pkg/line"
 	"github.com/prometheus/statsd_exporter/pkg/listener"
 	"github.com/prometheus/statsd_exporter/pkg/mapper"
+	"github.com/prometheus/statsd_exporter/pkg/mappercache/lru"
+	"github.com/prometheus/statsd_exporter/pkg/mappercache/randomreplacement"
 	"gopkg.in/yaml.v2"
 )
 
@@ -94,6 +94,7 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error)
 
 func init() {
 	integrations.RegisterIntegration(&Config{})
+	integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeMultiplex, metricsutils.CreateShim)
 }
 
 // Exporter defines the statsd_exporter integration.

pkg/integrations/v2/agent/agent.go

@@ -22,6 +22,9 @@ func (c *Config) Name() string { return "agent" }
 // ApplyDefaults applies runtime-specific defaults to c.
 func (c *Config) ApplyDefaults(globals integrations.Globals) error {
 	c.Common.ApplyDefaults(globals.SubsystemOpts.Metrics.Autoscrape)
+	if id, err := c.Identifier(globals); err == nil {
+		c.Common.InstanceKey = &id
+	}
 	return nil
 }
 

pkg/integrations/v2/metricsutils/versionshim.go

@@ -38,6 +38,9 @@ func (s *configShim) Name() string { return s.orig.Name() }
 
 func (s *configShim) ApplyDefaults(g v2.Globals) error {
 	s.common.ApplyDefaults(g.SubsystemOpts.Metrics.Autoscrape)
+	if id, err := s.Identifier(g); err == nil {
+		s.common.InstanceKey = &id
+	}
 	return nil
 }
 
@@ -50,6 +53,9 @@ func (s *configShim) ConfigEquals(c v2.Config) bool {
 }
 
 func (s *configShim) Identifier(g v2.Globals) (string, error) {
+	if s.common.InstanceKey != nil {
+		return *s.common.InstanceKey, nil
+	}
 	return s.orig.InstanceKey(g.AgentIdentifier)
 }