diff --git a/cmd/agent/agent-local-config.yaml b/cmd/agent/agent-local-config.yaml index ca888bcbd2c5..d02e681c209f 100644 --- a/cmd/agent/agent-local-config.yaml +++ b/cmd/agent/agent-local-config.yaml @@ -5,14 +5,8 @@ server: metrics: global: scrape_interval: 1m - configs: - - name: test - host_filter: false - scrape_configs: - - job_name: local_scrape - static_configs: - - targets: ['127.0.0.1:12345'] - labels: - cluster: 'localhost' - remote_write: - - url: http://localhost:9009/api/prom/push + remote_write: + - url: http://localhost:9009/api/prom/push + +integrations: + agent: {} diff --git a/pkg/integrations/install/install.go b/pkg/integrations/install/install.go index 353e35fb75cc..da1b019dd830 100644 --- a/pkg/integrations/install/install.go +++ b/pkg/integrations/install/install.go @@ -2,6 +2,10 @@ package install import ( + // + // v1 integrations + // + _ "github.com/grafana/agent/pkg/integrations/agent" // register agent _ "github.com/grafana/agent/pkg/integrations/consul_exporter" // register consul_exporter _ "github.com/grafana/agent/pkg/integrations/dnsmasq_exporter" // register dnsmasq_exporter @@ -18,5 +22,9 @@ import ( _ "github.com/grafana/agent/pkg/integrations/statsd_exporter" // register statsd_exporter _ "github.com/grafana/agent/pkg/integrations/windows_exporter" // register windows_exporter + // + // v2 integrations + // + _ "github.com/grafana/agent/pkg/integrations/v2/agent" // register agent ) diff --git a/pkg/integrations/install/shims.go b/pkg/integrations/install/shims.go new file mode 100644 index 000000000000..4cc6676e00fc --- /dev/null +++ b/pkg/integrations/install/shims.go @@ -0,0 +1,32 @@ +package install + +import ( + v1 "github.com/grafana/agent/pkg/integrations" + v2 "github.com/grafana/agent/pkg/integrations/v2" + metricsutils "github.com/grafana/agent/pkg/integrations/v2/metricsutils" +) + +// Perform a migration of v1 integrations which do not yet have a v2 +// counterpart. These integrations will be registered as Singletons +// to maintain existing behavior. +// +// To migrate a v1 integration to a v2 migration with support for multiple +// instances, v1 integrations must manually migrate themselves by calling +// v2.RegisterDynamic directly. +func init() { + for _, v1Integration := range v1.RegisteredIntegrations() { + // Look to see if there's a v2 integration with the same name. + var found bool + for _, v2Integration := range v2.Registered() { + if v2Integration.Name() == v1Integration.Name() { + found = true + break + } + } + if !found { + v2.RegisterLegacy(v1Integration, v2.TypeSingleton, func(cfg v1.Config) v2.UpgradedConfig { + return metricsutils.CreateShim(cfg) + }) + } + } +} diff --git a/pkg/integrations/v2/metricsutils/metricshandler_integration.go b/pkg/integrations/v2/metricsutils/metricshandler_integration.go index 1227e59375a0..374a759f708e 100644 --- a/pkg/integrations/v2/metricsutils/metricshandler_integration.go +++ b/pkg/integrations/v2/metricsutils/metricshandler_integration.go @@ -33,9 +33,12 @@ func NewMetricsHandlerIntegration( return &metricsHandlerIntegration{ integrationName: c.Name(), instanceID: id, - common: c.MetricsConfig(), - globals: globals, - handler: h, + + common: c.MetricsConfig(), + globals: globals, + handler: h, + + targets: []handlerTarget{{MetricsPath: "metrics"}}, }, nil } @@ -45,6 +48,17 @@ type metricsHandlerIntegration struct { common CommonConfig globals integrations.Globals handler http.Handler + targets []handlerTarget + + runFunc func(ctx context.Context) error +} + +type handlerTarget struct { + // Path relative to Handler prefix where metrics are available. + MetricsPath string + // Extra labels to inject into the target. Labels here that take precedence + // over labels with the same name from the generated target group. + Labels model.LabelSet } // Static typecheck tests @@ -56,6 +70,12 @@ var ( // RunIntegration implements Integration. func (i *metricsHandlerIntegration) RunIntegration(ctx context.Context) error { + // Call our runFunc if defined (used from integrationShim), otherwise + // fallback to no-op. + if i.runFunc != nil { + return i.runFunc(ctx) + } + <-ctx.Done() return nil } @@ -71,11 +91,7 @@ func (i *metricsHandlerIntegration) Handler(prefix string) (http.Handler, error) func (i *metricsHandlerIntegration) Targets(ep integrations.Endpoint) []*targetgroup.Group { integrationNameValue := model.LabelValue("integrations/" + i.integrationName) - return []*targetgroup.Group{{ - Targets: []model.LabelSet{{ - model.AddressLabel: model.LabelValue(ep.Host), - model.MetricsPathLabel: model.LabelValue(path.Join(ep.Prefix, "metrics")), - }}, + group := &targetgroup.Group{ Labels: model.LabelSet{ model.InstanceLabel: model.LabelValue(i.instanceID), model.JobLabel: integrationNameValue, @@ -85,9 +101,18 @@ func (i *metricsHandlerIntegration) Targets(ep integrations.Endpoint) []*targetg "__meta_agent_integration_name": model.LabelValue(i.integrationName), "__meta_agent_integration_instance": model.LabelValue(i.instanceID), "__meta_agent_integration_selfscrape": model.LabelValue(boolToString(i.shouldSelfScrape())), - }.Merge(i.globals.SubsystemOpts.Labels), + }, Source: fmt.Sprintf("%s/%s", i.integrationName, i.instanceID), - }} + } + + for _, t := range i.targets { + group.Targets = append(group.Targets, model.LabelSet{ + model.AddressLabel: model.LabelValue(ep.Host), + model.MetricsPathLabel: model.LabelValue(path.Join(ep.Prefix, t.MetricsPath)), + }.Merge(t.Labels)) + } + + return []*targetgroup.Group{group} } // shouldSelfScrape returns true if the integration is self-scraping. diff --git a/pkg/integrations/v2/metricsutils/versionshim.go b/pkg/integrations/v2/metricsutils/versionshim.go new file mode 100644 index 000000000000..4042697a86a2 --- /dev/null +++ b/pkg/integrations/v2/metricsutils/versionshim.go @@ -0,0 +1,119 @@ +package metricsutils + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/go-kit/log" + "github.com/prometheus/common/model" + + v1 "github.com/grafana/agent/pkg/integrations" + v2 "github.com/grafana/agent/pkg/integrations/v2" +) + +// CreateShim creates a shim between the v1.Config and v2.Config. The resulting +// config is NOT registered. +func CreateShim(before v1.Config) (after v2.UpgradedConfig) { + return &configShim{Orig: before} +} + +type configShim struct { + Orig v1.Config +} + +var ( + _ v2.Config = (*configShim)(nil) + _ v2.UpgradedConfig = (*configShim)(nil) +) + +// LegacyConfig implements v2.UpgradedConfig. +func (s *configShim) LegacyConfig() v1.Config { return s.Orig } + +func (s *configShim) Name() string { return s.Orig.Name() } +func (s *configShim) Identifier(g v2.Globals) (string, error) { + return s.Orig.InstanceKey(g.AgentIdentifier) +} + +func (s *configShim) NewIntegration(l log.Logger, g v2.Globals) (v2.Integration, error) { + v1Integration, err := s.Orig.NewIntegration(l) + if err != nil { + return nil, err + } + + id, err := s.Identifier(g) + if err != nil { + return nil, err + } + + // Map from the original CommonConfig to the new settings. This is a 1:1 + // mapping, minus the loss of WALTruncateFrequency. + origCommon := s.Orig.CommonConfig() + newCommon := CommonConfig{ + Enabled: origCommon.Enabled, + InstanceKey: origCommon.InstanceKey, + ScrapeIntegration: origCommon.ScrapeIntegration, + ScrapeInterval: origCommon.ScrapeInterval, + ScrapeTimeout: origCommon.ScrapeTimeout, + RelabelConfigs: origCommon.RelabelConfigs, + MetricRelabelConfigs: origCommon.MetricRelabelConfigs, + } + + // Generate our handler. Original integrations didn't accept a prefix, and + // just assumed that they would be wired to /metrics somewhere. + handler, err := v1Integration.MetricsHandler() + if err != nil { + return nil, fmt.Errorf("generating http handler: %w", err) + } else if handler == nil { + handler = http.NotFoundHandler() + } + + // Generate targets. Original integrations used a static set of targets, + // so this mapping can always be generated just once. + // + // Targets are generated from the result of ScrapeConfigs(), which returns a + // tuple of job name and relative metrics path. + // + // Job names were prefixed at the subsystem level with integrations/, so we + // will retain that behavior here. + v1ScrapeConfigs := v1Integration.ScrapeConfigs() + targets := make([]handlerTarget, 0, len(v1ScrapeConfigs)) + for _, sc := range v1ScrapeConfigs { + targets = append(targets, handlerTarget{ + MetricsPath: sc.MetricsPath, + Labels: model.LabelSet{ + model.JobLabel: model.LabelValue("integrations/" + sc.JobName), + }, + }) + } + + // Convert he run function. Original integrations sometimes returned + // ctx.Err() on exit. This isn't recommended anymore, but we need to hide the + // error if it happens, since the error was previously ignored. + runFunc := func(ctx context.Context) error { + err := v1Integration.Run(ctx) + switch { + case err == nil: + return nil + case errors.Is(err, context.Canceled) && ctx.Err() != nil: + // Hide error that no longer happens in newer integrations. + return nil + default: + return err + } + } + + // Aggregate our converted settings into a v2 integration. + return &metricsHandlerIntegration{ + integrationName: s.Orig.Name(), + instanceID: id, + + common: newCommon, + globals: g, + handler: handler, + targets: targets, + + runFunc: runFunc, + }, nil +} diff --git a/pkg/integrations/v2/register.go b/pkg/integrations/v2/register.go index 62fb562753d4..85f56188e462 100644 --- a/pkg/integrations/v2/register.go +++ b/pkg/integrations/v2/register.go @@ -7,13 +7,19 @@ import ( "testing" "gopkg.in/yaml.v2" + + v1 "github.com/grafana/agent/pkg/integrations" ) var ( - registeredIntegrations = []Config{} + integrationNames = make(map[string]struct{}) // Cache of field names for uniqueness checking. + configFieldNames = make(map[reflect.Type]string) // Map of registered type to field name + integrationTypes = make(map[reflect.Type]Type) // Map of registered type to Type - configFieldNames = make(map[reflect.Type]string) - integrationTypes = make(map[reflect.Type]Type) + // Registered integrations. Registered integrations may be either a Config or + // a v1.Config. v1.Configs must have a corresponding upgrader for their type. + registered = []interface{}{} + upgraders = make(map[reflect.Type]UpgradeFunc) emptyStructType = reflect.TypeOf(struct{}{}) configsType = reflect.TypeOf(Configs{}) @@ -28,18 +34,48 @@ var ( // // Register panics if cfg is not a pointer. func Register(cfg Config, ty Type) { - if reflect.TypeOf(cfg).Kind() != reflect.Ptr { - panic(fmt.Sprintf("Register must be given a pointer, got %T", cfg)) + registerIntegration(cfg, cfg.Name(), ty, nil) +} + +func registerIntegration(v interface{}, name string, ty Type, upgrader UpgradeFunc) { + if reflect.TypeOf(v).Kind() != reflect.Ptr { + panic(fmt.Sprintf("Register must be given a pointer, got %T", v)) } - if _, exist := configFieldNames[reflect.TypeOf(cfg)]; exist { - panic(fmt.Sprintf("Integration %T registered twice", cfg)) + if _, exist := integrationNames[name]; exist { + panic(fmt.Sprintf("Integration %q registered twice", name)) } + integrationNames[name] = struct{}{} - registeredIntegrations = append(registeredIntegrations, cfg) + registered = append(registered, v) - configTy := reflect.TypeOf(cfg) + configTy := reflect.TypeOf(v) integrationTypes[configTy] = ty - configFieldNames[configTy] = cfg.Name() + configFieldNames[configTy] = name + upgraders[configTy] = upgrader +} + +// RegisterLegacy registers a v1.Config. upgrader will be used to upgrade it. +// upgrader will only be invoked after unmarshaling cfg from YAML, and the +// upgraded Config will be unwrapped again when marshaling back to YAML. +// +// RegisterLegacy only exists for the transition period where the v2 +// integrations subsystem is an experiment. RegisterLegacy will be removed at a +// later date. +func RegisterLegacy(cfg v1.Config, ty Type, upgrader UpgradeFunc) { + registerIntegration(cfg, cfg.Name(), ty, upgrader) +} + +// UpgradeFunc upgrades cfg to a UpgradedConfig. +type UpgradeFunc func(cfg v1.Config) UpgradedConfig + +// UpgradedConfig is a v2 Config that was constructed through a legacy +// v1.Config. It allows unwrapping to retrieve the original config for +// the purposes of marshaling or unmarshaling. +type UpgradedConfig interface { + Config + + // LegacyConfig returns the old v1.Config. + LegacyConfig() v1.Config } // Type determines a specific type of integration. @@ -67,9 +103,11 @@ const ( // setRegistered must not be used with parallelized tests. func setRegistered(t *testing.T, cc map[Config]Type) { clear := func() { - registeredIntegrations = registeredIntegrations[:0] + integrationNames = make(map[string]struct{}) integrationTypes = make(map[reflect.Type]Type) configFieldNames = make(map[reflect.Type]string) + registered = registered[:0] + upgraders = make(map[reflect.Type]UpgradeFunc) } t.Cleanup(clear) @@ -80,18 +118,29 @@ func setRegistered(t *testing.T, cc map[Config]Type) { } } -// Registered all Configs that were passed to Register. -// Each call will generate a new set of configs. +// Registered all Configs that were passed to Register or RegisterLegacy. Each +// call will generate a new set of configs. func Registered() []Config { - res := make([]Config, 0, len(registeredIntegrations)) - for _, in := range registeredIntegrations { - res = append(res, cloneIntegration(in)) + res := make([]Config, 0, len(registered)) + for _, r := range registered { + switch v := r.(type) { + case Config: + res = append(res, cloneValue(v).(Config)) + case v1.Config: + mut, ok := upgraders[reflect.TypeOf(r)] + if !ok || mut == nil { + panic(fmt.Sprintf("Could not find transformer for legacy integration %T", r)) + } + res = append(res, mut(cloneValue(r).(v1.Config))) + default: + panic(fmt.Sprintf("unexpected type %T", r)) + } } return res } -func cloneIntegration(c Config) Config { - return reflect.New(reflect.TypeOf(c).Elem()).Interface().(Config) +func cloneValue(in interface{}) interface{} { + return reflect.New(reflect.TypeOf(in).Elem()).Interface() } // Configs is a list of integrations. Note that Configs does not implement @@ -139,19 +188,25 @@ func MarshalYAML(v interface{}) (interface{}, error) { } for _, c := range configs { - fieldName, ok := configFieldNames[reflect.TypeOf(c)] + fieldName := c.Name() + + var data interface{} = c + if wc, ok := c.(UpgradedConfig); ok { + data = wc.LegacyConfig() + } + + integrationType, ok := integrationTypes[reflect.TypeOf(data)] if !ok { - return nil, fmt.Errorf("integrations: cannot marshal unregistered Config type: %T", c) + panic(fmt.Sprintf("config not registered: %T", data)) } - integrationType := integrationTypes[reflect.TypeOf(c)] switch integrationType { case TypeSingleton: field := outVal.FieldByName("XXX_Config_" + fieldName) - field.Set(reflect.ValueOf(c)) + field.Set(reflect.ValueOf(data)) case TypeMultiplex, TypeEither: field := outVal.FieldByName("XXX_Configs_" + fieldName) - field.Set(reflect.Append(field, reflect.ValueOf(c))) + field.Set(reflect.Append(field, reflect.ValueOf(data))) } } @@ -229,11 +284,11 @@ func UnmarshalYAML(out interface{}, unmarshal func(interface{}) error) error { switch field.Kind() { case reflect.Slice: for i := 0; i < field.Len(); i++ { - val := field.Index(i).Interface().(Config) + val := getConfig(field.Index(i).Interface()) *configs = append(*configs, val) } default: - val := field.Interface().(Config) + val := getConfig(field.Interface()) *configs = append(*configs, val) } } @@ -241,6 +296,22 @@ func UnmarshalYAML(out interface{}, unmarshal func(interface{}) error) error { return nil } +// getConfig returns a v2 config from an input. +func getConfig(in interface{}) Config { + switch c := in.(type) { + case Config: + return c + case v1.Config: + mut, ok := upgraders[reflect.TypeOf(in)] + if !ok { + panic(fmt.Sprintf("unexpected type %T", c)) + } + return mut(c) + default: + panic(fmt.Sprintf("unexpected type %T", c)) + } +} + // getConfigTypeForIntegrations returns a dynamic struct type that has all of // the same fields as out including the fields for the provided integrations. func getConfigTypeForIntegrations(out reflect.Type) reflect.Type { @@ -259,22 +330,23 @@ func getConfigTypeForIntegrations(out reflect.Type) reflect.Type { } } - for _, cfg := range registeredIntegrations { + for _, reg := range registered { // Fields use a prefix that's unlikely to collide with anything else. - configTy := reflect.TypeOf(cfg) + configTy := reflect.TypeOf(reg) integrationType := integrationTypes[configTy] + fieldName := configFieldNames[configTy] if integrationType == TypeSingleton || integrationType == TypeEither { fields = append(fields, reflect.StructField{ - Name: "XXX_Config_" + cfg.Name(), - Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s,omitempty"`, cfg.Name())), + Name: "XXX_Config_" + fieldName, + Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s,omitempty"`, fieldName)), Type: configTy, }) } if integrationType == TypeMultiplex || integrationType == TypeEither { fields = append(fields, reflect.StructField{ - Name: "XXX_Configs_" + cfg.Name(), - Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s_configs,omitempty"`, cfg.Name())), + Name: "XXX_Configs_" + fieldName, + Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s_configs,omitempty"`, fieldName)), Type: reflect.SliceOf(configTy), }) } diff --git a/pkg/integrations/v2/register_test.go b/pkg/integrations/v2/register_test.go index 33afd97e31dc..a1d17b09c131 100644 --- a/pkg/integrations/v2/register_test.go +++ b/pkg/integrations/v2/register_test.go @@ -5,6 +5,8 @@ import ( "time" "github.com/go-kit/log" + v1 "github.com/grafana/agent/pkg/integrations" + agent_v1 "github.com/grafana/agent/pkg/integrations/agent" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" ) @@ -74,6 +76,43 @@ test_configs: require.Equal(t, expect, fullCfg) } +func TestIntegrationRegistration_Legacy(t *testing.T) { + setRegistered(t, nil) + + RegisterLegacy(&agent_v1.Config{}, TypeSingleton, func(in v1.Config) UpgradedConfig { + return &legacyShim{Data: in} + }) + + var cfgToParse = ` +name: John Doe +duration: 500ms +agent: + instance: foo` + + var fullCfg testFullConfig + err := yaml.UnmarshalStrict([]byte(cfgToParse), &fullCfg) + require.NoError(t, err) + + require.Len(t, fullCfg.Configs, 1) + require.IsType(t, &legacyShim{}, fullCfg.Configs[0]) + + shim := fullCfg.Configs[0].(*legacyShim) + require.IsType(t, &agent_v1.Config{}, shim.Data) + + agentConfig := shim.Data.(*agent_v1.Config) + require.Equal(t, "foo", *agentConfig.Common.InstanceKey) + require.Equal(t, true, agentConfig.Enabled) +} + +type legacyShim struct{ Data v1.Config } + +func (s *legacyShim) LegacyConfig() v1.Config { return s.Data } +func (s *legacyShim) Name() string { return s.Data.Name() } +func (s *legacyShim) Identifier(g Globals) (string, error) { return g.AgentIdentifier, nil } +func (s *legacyShim) NewIntegration(log.Logger, Globals) (Integration, error) { + return NoOpIntegration, nil +} + type testIntegrationA struct { Text string `yaml:"text"` Truth bool `yaml:"truth"`