From 60005fc7495946e775b3cd794347bc702c1d8113 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 14 Dec 2021 22:37:23 -0500 Subject: [PATCH 1/4] integrations/v2: allow shimming between v1 and v2 integrations. Shimming is done by changing how the integration registration works; a new RegisterDynamic was added that allows for creating Configs at runtime. Here be dragons; this should be removed whenever we no longer have a need for it. --- cmd/agent/agent-local-config.yaml | 16 +- pkg/integrations/install/install.go | 4 + pkg/integrations/install/shims.go | 32 ++++ .../metricshandler_integration.go | 45 ++++-- .../v2/metricsutils/versionshim.go | 119 +++++++++++++++ pkg/integrations/v2/register.go | 139 ++++++++++++++---- pkg/integrations/v2/register_test.go | 41 ++++++ 7 files changed, 345 insertions(+), 51 deletions(-) create mode 100644 pkg/integrations/install/shims.go create mode 100644 pkg/integrations/v2/metricsutils/versionshim.go diff --git a/cmd/agent/agent-local-config.yaml b/cmd/agent/agent-local-config.yaml index ca888bcbd2c5..d02e681c209f 100644 --- a/cmd/agent/agent-local-config.yaml +++ b/cmd/agent/agent-local-config.yaml @@ -5,14 +5,8 @@ server: metrics: global: scrape_interval: 1m - configs: - - name: test - host_filter: false - scrape_configs: - - job_name: local_scrape - static_configs: - - targets: ['127.0.0.1:12345'] - labels: - cluster: 'localhost' - remote_write: - - url: http://localhost:9009/api/prom/push + remote_write: + - url: http://localhost:9009/api/prom/push + +integrations: + agent: {} diff --git a/pkg/integrations/install/install.go b/pkg/integrations/install/install.go index 353e35fb75cc..dd2c1827d971 100644 --- a/pkg/integrations/install/install.go +++ b/pkg/integrations/install/install.go @@ -1,6 +1,7 @@ // Package install registers all in-source integrations for use. package install +// v1 integrations import ( _ "github.com/grafana/agent/pkg/integrations/agent" // register agent _ "github.com/grafana/agent/pkg/integrations/consul_exporter" // register consul_exporter @@ -17,6 +18,9 @@ import ( _ "github.com/grafana/agent/pkg/integrations/redis_exporter" // register redis_exporter _ "github.com/grafana/agent/pkg/integrations/statsd_exporter" // register statsd_exporter _ "github.com/grafana/agent/pkg/integrations/windows_exporter" // register windows_exporter +) +// v2 integrations +import ( _ "github.com/grafana/agent/pkg/integrations/v2/agent" // register agent ) diff --git a/pkg/integrations/install/shims.go b/pkg/integrations/install/shims.go new file mode 100644 index 000000000000..459522cd33ad --- /dev/null +++ b/pkg/integrations/install/shims.go @@ -0,0 +1,32 @@ +package install + +import ( + v1 "github.com/grafana/agent/pkg/integrations" + v2 "github.com/grafana/agent/pkg/integrations/v2" + metricsutils "github.com/grafana/agent/pkg/integrations/v2/metricsutils" +) + +// Perform a migration of v1 integrations which do not yet have a v2 +// counterpart. These integrations will be registered as Singletons +// to maintain existing behavior. +// +// To migrate a v1 integration to a v2 migration with support for multiple +// instances, v1 integrations must manually migrate themselves by calling +// v2.RegisterDynamic directly. +func init() { + for _, v1Integration := range v1.RegisteredIntegrations() { + // Look to see if there's a v2 integration with the same name. + var found bool + for _, v2Integration := range v2.Registered() { + if v2Integration.Name() == v1Integration.Name() { + found = true + break + } + } + if !found { + v2.RegisterDynamic(v1Integration, v1Integration.Name(), v2.TypeSingleton, func(in interface{}) v2.WrappedConfig { + return metricsutils.CreateShim(in.(v1.Config)) + }) + } + } +} diff --git a/pkg/integrations/v2/metricsutils/metricshandler_integration.go b/pkg/integrations/v2/metricsutils/metricshandler_integration.go index 1227e59375a0..374a759f708e 100644 --- a/pkg/integrations/v2/metricsutils/metricshandler_integration.go +++ b/pkg/integrations/v2/metricsutils/metricshandler_integration.go @@ -33,9 +33,12 @@ func NewMetricsHandlerIntegration( return &metricsHandlerIntegration{ integrationName: c.Name(), instanceID: id, - common: c.MetricsConfig(), - globals: globals, - handler: h, + + common: c.MetricsConfig(), + globals: globals, + handler: h, + + targets: []handlerTarget{{MetricsPath: "metrics"}}, }, nil } @@ -45,6 +48,17 @@ type metricsHandlerIntegration struct { common CommonConfig globals integrations.Globals handler http.Handler + targets []handlerTarget + + runFunc func(ctx context.Context) error +} + +type handlerTarget struct { + // Path relative to Handler prefix where metrics are available. + MetricsPath string + // Extra labels to inject into the target. Labels here that take precedence + // over labels with the same name from the generated target group. + Labels model.LabelSet } // Static typecheck tests @@ -56,6 +70,12 @@ var ( // RunIntegration implements Integration. func (i *metricsHandlerIntegration) RunIntegration(ctx context.Context) error { + // Call our runFunc if defined (used from integrationShim), otherwise + // fallback to no-op. + if i.runFunc != nil { + return i.runFunc(ctx) + } + <-ctx.Done() return nil } @@ -71,11 +91,7 @@ func (i *metricsHandlerIntegration) Handler(prefix string) (http.Handler, error) func (i *metricsHandlerIntegration) Targets(ep integrations.Endpoint) []*targetgroup.Group { integrationNameValue := model.LabelValue("integrations/" + i.integrationName) - return []*targetgroup.Group{{ - Targets: []model.LabelSet{{ - model.AddressLabel: model.LabelValue(ep.Host), - model.MetricsPathLabel: model.LabelValue(path.Join(ep.Prefix, "metrics")), - }}, + group := &targetgroup.Group{ Labels: model.LabelSet{ model.InstanceLabel: model.LabelValue(i.instanceID), model.JobLabel: integrationNameValue, @@ -85,9 +101,18 @@ func (i *metricsHandlerIntegration) Targets(ep integrations.Endpoint) []*targetg "__meta_agent_integration_name": model.LabelValue(i.integrationName), "__meta_agent_integration_instance": model.LabelValue(i.instanceID), "__meta_agent_integration_selfscrape": model.LabelValue(boolToString(i.shouldSelfScrape())), - }.Merge(i.globals.SubsystemOpts.Labels), + }, Source: fmt.Sprintf("%s/%s", i.integrationName, i.instanceID), - }} + } + + for _, t := range i.targets { + group.Targets = append(group.Targets, model.LabelSet{ + model.AddressLabel: model.LabelValue(ep.Host), + model.MetricsPathLabel: model.LabelValue(path.Join(ep.Prefix, t.MetricsPath)), + }.Merge(t.Labels)) + } + + return []*targetgroup.Group{group} } // shouldSelfScrape returns true if the integration is self-scraping. diff --git a/pkg/integrations/v2/metricsutils/versionshim.go b/pkg/integrations/v2/metricsutils/versionshim.go new file mode 100644 index 000000000000..b4266021e133 --- /dev/null +++ b/pkg/integrations/v2/metricsutils/versionshim.go @@ -0,0 +1,119 @@ +package metricsutils + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/go-kit/log" + "github.com/prometheus/common/model" + + v1 "github.com/grafana/agent/pkg/integrations" + v2 "github.com/grafana/agent/pkg/integrations/v2" +) + +// CreateShim creates a shim between the v1.Config and v2.Config. The resulting +// config is NOT registered. +func CreateShim(before v1.Config) (after v2.WrappedConfig) { + return &configShim{Orig: before} +} + +type configShim struct { + Orig v1.Config +} + +var ( + _ v2.Config = (*configShim)(nil) + _ v2.WrappedConfig = (*configShim)(nil) +) + +// UnwrapConfig implements v2.WrappedConfig. +func (s *configShim) UnwrapConfig() interface{} { return s.Orig } + +func (s *configShim) Name() string { return s.Orig.Name() } +func (s *configShim) Identifier(g v2.Globals) (string, error) { + return s.Orig.InstanceKey(g.AgentIdentifier) +} + +func (s *configShim) NewIntegration(l log.Logger, g v2.Globals) (v2.Integration, error) { + v1Integration, err := s.Orig.NewIntegration(l) + if err != nil { + return nil, err + } + + id, err := s.Identifier(g) + if err != nil { + return nil, err + } + + // Map from the original CommonConfig to the new settings. This is a 1:1 + // mapping, minus the loss of WALTruncateFrequency. + origCommon := s.Orig.CommonConfig() + newCommon := CommonConfig{ + Enabled: origCommon.Enabled, + InstanceKey: origCommon.InstanceKey, + ScrapeIntegration: origCommon.ScrapeIntegration, + ScrapeInterval: origCommon.ScrapeInterval, + ScrapeTimeout: origCommon.ScrapeTimeout, + RelabelConfigs: origCommon.RelabelConfigs, + MetricRelabelConfigs: origCommon.MetricRelabelConfigs, + } + + // Generate our handler. Original integrations didn't accept a prefix, and + // just assumed that they would be wired to /metrics somewhere. + handler, err := v1Integration.MetricsHandler() + if err != nil { + return nil, fmt.Errorf("generating http handler: %w", err) + } else if handler == nil { + handler = http.NotFoundHandler() + } + + // Generate targets. Original integrations used a static set of targets, + // so this mapping can always be generated just once. + // + // Targets are generated from the result of ScrapeConfigs(), which returns a + // tuple of job name and relative metrics path. + // + // Job names were prefixed at the subsystem level with integrations/, so we + // will retain that behavior here. + v1ScrapeConfigs := v1Integration.ScrapeConfigs() + targets := make([]handlerTarget, 0, len(v1ScrapeConfigs)) + for _, sc := range v1ScrapeConfigs { + targets = append(targets, handlerTarget{ + MetricsPath: sc.MetricsPath, + Labels: model.LabelSet{ + model.JobLabel: model.LabelValue("integrations/" + sc.JobName), + }, + }) + } + + // Convert he run function. Original integrations sometimes returned + // ctx.Err() on exit. This isn't recommended anymore, but we need to hide the + // error if it happens, since the error was previously ignored. + runFunc := func(ctx context.Context) error { + err := v1Integration.Run(ctx) + switch { + case err == nil: + return nil + case errors.Is(err, context.Canceled) && ctx.Err() != nil: + // Hide error that no longer happens in newer integrations. + return nil + default: + return err + } + } + + // Aggregate our converted settings into a v2 integration. + return &metricsHandlerIntegration{ + integrationName: s.Orig.Name(), + instanceID: id, + + common: newCommon, + globals: g, + handler: handler, + targets: targets, + + runFunc: runFunc, + }, nil +} diff --git a/pkg/integrations/v2/register.go b/pkg/integrations/v2/register.go index 62fb562753d4..25d25b287ff0 100644 --- a/pkg/integrations/v2/register.go +++ b/pkg/integrations/v2/register.go @@ -10,10 +10,15 @@ import ( ) var ( - registeredIntegrations = []Config{} + integrationNames = make(map[string]struct{}) // Cache of field names for uniqueness checking. + configFieldNames = make(map[reflect.Type]string) // Map of registered type to field name + integrationTypes = make(map[reflect.Type]Type) // Map of registered type to Type - configFieldNames = make(map[reflect.Type]string) - integrationTypes = make(map[reflect.Type]Type) + // Registered integrations. Registered integrations may be any type. If they + // do not implement Config, then they must have a mutator to wrap them into + // a Config. + registered = []interface{}{} + mutators = make(map[reflect.Type]WrapperFunc) emptyStructType = reflect.TypeOf(struct{}{}) configsType = reflect.TypeOf(Configs{}) @@ -28,18 +33,52 @@ var ( // // Register panics if cfg is not a pointer. func Register(cfg Config, ty Type) { - if reflect.TypeOf(cfg).Kind() != reflect.Ptr { - panic(fmt.Sprintf("Register must be given a pointer, got %T", cfg)) + // Special case: we don't need a transformer because cfg is already a Config. + RegisterDynamic(cfg, cfg.Name(), ty, nil) +} + +// RegisterDynamic registers a dynamic integration v which does not implement +// Config directly. The transform function will be invoked to create a wrapper +// implementation of Config. transform will be invoked after unmarshaling v from +// YAML, and the result will be unwrapped back to the original form at marshal +// time. +// +// RegisterDynamic is only useful in niche instances. When possible, use +// Register instead. +// +// name is the name of the integration used for unmarshaling from YAML. Must be +// unique across all integrations. +func RegisterDynamic(v interface{}, name string, ty Type, transform WrapperFunc) { + if _, isConfig := v.(Config); !isConfig && transform == nil { + panic(fmt.Sprintf("transform must not be nil; %T does not implement Config", v)) + } + if reflect.TypeOf(v).Kind() != reflect.Ptr { + panic(fmt.Sprintf("RegisterDynamic must be given a pointer, got %T", v)) } - if _, exist := configFieldNames[reflect.TypeOf(cfg)]; exist { - panic(fmt.Sprintf("Integration %T registered twice", cfg)) + if _, exist := integrationNames[name]; exist { + panic(fmt.Sprintf("Integration %q registered twice", name)) } + integrationNames[name] = struct{}{} - registeredIntegrations = append(registeredIntegrations, cfg) + registered = append(registered, v) - configTy := reflect.TypeOf(cfg) + configTy := reflect.TypeOf(v) integrationTypes[configTy] = ty - configFieldNames[configTy] = cfg.Name() + configFieldNames[configTy] = name + mutators[configTy] = transform +} + +// WrapperFunc wraps in in a WrappedConfig container. +type WrapperFunc func(in interface{}) WrappedConfig + +// WrappedConfig represents a Config which is a container for some data. This +// is used when registering integrations through RegisterDynamic. +type WrappedConfig interface { + Config + + // UnwrapConfig returns the inner data. The inner data is used for YAML + // marshaling and unmarshaling. + UnwrapConfig() interface{} } // Type determines a specific type of integration. @@ -67,9 +106,11 @@ const ( // setRegistered must not be used with parallelized tests. func setRegistered(t *testing.T, cc map[Config]Type) { clear := func() { - registeredIntegrations = registeredIntegrations[:0] + integrationNames = make(map[string]struct{}) integrationTypes = make(map[reflect.Type]Type) configFieldNames = make(map[reflect.Type]string) + registered = registered[:0] + mutators = make(map[reflect.Type]WrapperFunc) } t.Cleanup(clear) @@ -80,18 +121,31 @@ func setRegistered(t *testing.T, cc map[Config]Type) { } } -// Registered all Configs that were passed to Register. -// Each call will generate a new set of configs. +// Registered all Configs that were passed to Register or RegisterDynamic. Each +// call will generate a new set of configs. func Registered() []Config { - res := make([]Config, 0, len(registeredIntegrations)) - for _, in := range registeredIntegrations { - res = append(res, cloneIntegration(in)) + res := make([]Config, 0, len(registered)) + for _, dyn := range registered { + switch v := dyn.(type) { + case Config: + res = append(res, cloneDynamic(v).(Config)) + default: + mut, ok := mutators[reflect.TypeOf(dyn)] + if !ok || mut == nil { + panic(fmt.Sprintf("Could not find transformer for dynamic integration %T", dyn)) + } + res = append(res, mut(cloneDynamic(dyn))) + } } return res } func cloneIntegration(c Config) Config { - return reflect.New(reflect.TypeOf(c).Elem()).Interface().(Config) + return cloneDynamic(c).(Config) +} + +func cloneDynamic(in interface{}) interface{} { + return reflect.New(reflect.TypeOf(in).Elem()).Interface() } // Configs is a list of integrations. Note that Configs does not implement @@ -139,19 +193,25 @@ func MarshalYAML(v interface{}) (interface{}, error) { } for _, c := range configs { - fieldName, ok := configFieldNames[reflect.TypeOf(c)] + fieldName := c.Name() + + var data interface{} = c + if wc, ok := c.(WrappedConfig); ok { + data = wc.UnwrapConfig() + } + + integrationType, ok := integrationTypes[reflect.TypeOf(data)] if !ok { - return nil, fmt.Errorf("integrations: cannot marshal unregistered Config type: %T", c) + panic(fmt.Sprintf("config not registered: %T", data)) } - integrationType := integrationTypes[reflect.TypeOf(c)] switch integrationType { case TypeSingleton: field := outVal.FieldByName("XXX_Config_" + fieldName) - field.Set(reflect.ValueOf(c)) + field.Set(reflect.ValueOf(data)) case TypeMultiplex, TypeEither: field := outVal.FieldByName("XXX_Configs_" + fieldName) - field.Set(reflect.Append(field, reflect.ValueOf(c))) + field.Set(reflect.Append(field, reflect.ValueOf(data))) } } @@ -229,11 +289,11 @@ func UnmarshalYAML(out interface{}, unmarshal func(interface{}) error) error { switch field.Kind() { case reflect.Slice: for i := 0; i < field.Len(); i++ { - val := field.Index(i).Interface().(Config) + val := getConfig(field.Index(i).Interface()) *configs = append(*configs, val) } default: - val := field.Interface().(Config) + val := getConfig(field.Interface()) *configs = append(*configs, val) } } @@ -241,8 +301,26 @@ func UnmarshalYAML(out interface{}, unmarshal func(interface{}) error) error { return nil } +// getConfig returns a v2 config from an input. +func getConfig(in interface{}) Config { + switch c := in.(type) { + case Config: + return c + default: + mut, ok := mutators[reflect.TypeOf(in)] + if !ok { + panic(fmt.Sprintf("unexpected type %T", c)) + } + return mut(in) + } +} + // getConfigTypeForIntegrations returns a dynamic struct type that has all of // the same fields as out including the fields for the provided integrations. +// +// If marshal is true, interface{} will be used for integration types. This +// must be true when marshaling dynamic integrations, where their type will +// have changed. func getConfigTypeForIntegrations(out reflect.Type) reflect.Type { // Initial exported fields map one-to-one. var fields []reflect.StructField @@ -259,22 +337,23 @@ func getConfigTypeForIntegrations(out reflect.Type) reflect.Type { } } - for _, cfg := range registeredIntegrations { + for _, dyn := range registered { // Fields use a prefix that's unlikely to collide with anything else. - configTy := reflect.TypeOf(cfg) + configTy := reflect.TypeOf(dyn) integrationType := integrationTypes[configTy] + fieldName := configFieldNames[configTy] if integrationType == TypeSingleton || integrationType == TypeEither { fields = append(fields, reflect.StructField{ - Name: "XXX_Config_" + cfg.Name(), - Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s,omitempty"`, cfg.Name())), + Name: "XXX_Config_" + fieldName, + Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s,omitempty"`, fieldName)), Type: configTy, }) } if integrationType == TypeMultiplex || integrationType == TypeEither { fields = append(fields, reflect.StructField{ - Name: "XXX_Configs_" + cfg.Name(), - Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s_configs,omitempty"`, cfg.Name())), + Name: "XXX_Configs_" + fieldName, + Tag: reflect.StructTag(fmt.Sprintf(`yaml:"%s_configs,omitempty"`, fieldName)), Type: reflect.SliceOf(configTy), }) } diff --git a/pkg/integrations/v2/register_test.go b/pkg/integrations/v2/register_test.go index 33afd97e31dc..2426c295bbef 100644 --- a/pkg/integrations/v2/register_test.go +++ b/pkg/integrations/v2/register_test.go @@ -74,6 +74,47 @@ test_configs: require.Equal(t, expect, fullCfg) } +func TestIntegrationRegistration_Dynamic(t *testing.T) { + setRegistered(t, nil) + + type raw struct { + Name string `yaml:"name"` + } + + RegisterDynamic(&raw{}, "dyn", TypeSingleton, func(in interface{}) WrappedConfig { + return &dynamicShim{IntegrationName: "dyn", Data: in} + }) + + var cfgToParse = ` +name: John Doe +duration: 500ms +dyn: + name: Hello, world!` + + var fullCfg testFullConfig + err := yaml.UnmarshalStrict([]byte(cfgToParse), &fullCfg) + require.NoError(t, err) + + require.Len(t, fullCfg.Configs, 1) + require.IsType(t, &dynamicShim{}, fullCfg.Configs[0]) + + dynShim := fullCfg.Configs[0].(*dynamicShim) + require.IsType(t, &raw{}, dynShim.Data) + require.Equal(t, &raw{Name: "Hello, world!"}, dynShim.Data) +} + +type dynamicShim struct { + IntegrationName string + Data interface{} +} + +func (s *dynamicShim) UnwrapConfig() interface{} { return s.Data } +func (s *dynamicShim) Name() string { return s.IntegrationName } +func (s *dynamicShim) Identifier(g Globals) (string, error) { return g.AgentIdentifier, nil } +func (s *dynamicShim) NewIntegration(log.Logger, Globals) (Integration, error) { + return NoOpIntegration, nil +} + type testIntegrationA struct { Text string `yaml:"text"` Truth bool `yaml:"truth"` From ca4a46b9454cfed90fce50c3afaf29af28fe69ff Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Thu, 16 Dec 2021 13:18:13 -0500 Subject: [PATCH 2/4] fix lint --- pkg/integrations/install/install.go | 12 ++++++++---- pkg/integrations/v2/register.go | 4 ---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/integrations/install/install.go b/pkg/integrations/install/install.go index dd2c1827d971..da1b019dd830 100644 --- a/pkg/integrations/install/install.go +++ b/pkg/integrations/install/install.go @@ -1,8 +1,11 @@ // Package install registers all in-source integrations for use. package install -// v1 integrations import ( + // + // v1 integrations + // + _ "github.com/grafana/agent/pkg/integrations/agent" // register agent _ "github.com/grafana/agent/pkg/integrations/consul_exporter" // register consul_exporter _ "github.com/grafana/agent/pkg/integrations/dnsmasq_exporter" // register dnsmasq_exporter @@ -18,9 +21,10 @@ import ( _ "github.com/grafana/agent/pkg/integrations/redis_exporter" // register redis_exporter _ "github.com/grafana/agent/pkg/integrations/statsd_exporter" // register statsd_exporter _ "github.com/grafana/agent/pkg/integrations/windows_exporter" // register windows_exporter -) -// v2 integrations -import ( + // + // v2 integrations + // + _ "github.com/grafana/agent/pkg/integrations/v2/agent" // register agent ) diff --git a/pkg/integrations/v2/register.go b/pkg/integrations/v2/register.go index 25d25b287ff0..bc18a93ffe5e 100644 --- a/pkg/integrations/v2/register.go +++ b/pkg/integrations/v2/register.go @@ -140,10 +140,6 @@ func Registered() []Config { return res } -func cloneIntegration(c Config) Config { - return cloneDynamic(c).(Config) -} - func cloneDynamic(in interface{}) interface{} { return reflect.New(reflect.TypeOf(in).Elem()).Interface() } From 2ed0101fb05a85f6b39efe6d1d92681b8059fffb Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Thu, 16 Dec 2021 15:35:25 -0500 Subject: [PATCH 3/4] pkg/integrations/v2: use "RegisterLegacy" instead of a generic mechanism --- pkg/integrations/install/shims.go | 4 +- .../v2/metricsutils/versionshim.go | 10 +- pkg/integrations/v2/register.go | 101 +++++++++--------- pkg/integrations/v2/register_test.go | 40 ++++--- 4 files changed, 75 insertions(+), 80 deletions(-) diff --git a/pkg/integrations/install/shims.go b/pkg/integrations/install/shims.go index 459522cd33ad..4cc6676e00fc 100644 --- a/pkg/integrations/install/shims.go +++ b/pkg/integrations/install/shims.go @@ -24,8 +24,8 @@ func init() { } } if !found { - v2.RegisterDynamic(v1Integration, v1Integration.Name(), v2.TypeSingleton, func(in interface{}) v2.WrappedConfig { - return metricsutils.CreateShim(in.(v1.Config)) + v2.RegisterLegacy(v1Integration, v2.TypeSingleton, func(cfg v1.Config) v2.UpgradedConfig { + return metricsutils.CreateShim(cfg) }) } } diff --git a/pkg/integrations/v2/metricsutils/versionshim.go b/pkg/integrations/v2/metricsutils/versionshim.go index b4266021e133..4042697a86a2 100644 --- a/pkg/integrations/v2/metricsutils/versionshim.go +++ b/pkg/integrations/v2/metricsutils/versionshim.go @@ -15,7 +15,7 @@ import ( // CreateShim creates a shim between the v1.Config and v2.Config. The resulting // config is NOT registered. -func CreateShim(before v1.Config) (after v2.WrappedConfig) { +func CreateShim(before v1.Config) (after v2.UpgradedConfig) { return &configShim{Orig: before} } @@ -24,12 +24,12 @@ type configShim struct { } var ( - _ v2.Config = (*configShim)(nil) - _ v2.WrappedConfig = (*configShim)(nil) + _ v2.Config = (*configShim)(nil) + _ v2.UpgradedConfig = (*configShim)(nil) ) -// UnwrapConfig implements v2.WrappedConfig. -func (s *configShim) UnwrapConfig() interface{} { return s.Orig } +// LegacyConfig implements v2.UpgradedConfig. +func (s *configShim) LegacyConfig() v1.Config { return s.Orig } func (s *configShim) Name() string { return s.Orig.Name() } func (s *configShim) Identifier(g v2.Globals) (string, error) { diff --git a/pkg/integrations/v2/register.go b/pkg/integrations/v2/register.go index bc18a93ffe5e..5d7adc9dbc8c 100644 --- a/pkg/integrations/v2/register.go +++ b/pkg/integrations/v2/register.go @@ -7,6 +7,8 @@ import ( "testing" "gopkg.in/yaml.v2" + + v1 "github.com/grafana/agent/pkg/integrations" ) var ( @@ -14,11 +16,10 @@ var ( configFieldNames = make(map[reflect.Type]string) // Map of registered type to field name integrationTypes = make(map[reflect.Type]Type) // Map of registered type to Type - // Registered integrations. Registered integrations may be any type. If they - // do not implement Config, then they must have a mutator to wrap them into - // a Config. + // Registered integrations. Registered integrations may be either a Config or + // a v1.Config. v1.Configs must have a corresponding upgrader for their type. registered = []interface{}{} - mutators = make(map[reflect.Type]WrapperFunc) + upgraders = make(map[reflect.Type]UpgradeFunc) emptyStructType = reflect.TypeOf(struct{}{}) configsType = reflect.TypeOf(Configs{}) @@ -33,27 +34,12 @@ var ( // // Register panics if cfg is not a pointer. func Register(cfg Config, ty Type) { - // Special case: we don't need a transformer because cfg is already a Config. - RegisterDynamic(cfg, cfg.Name(), ty, nil) + registerIntegration(cfg, cfg.Name(), ty, nil) } -// RegisterDynamic registers a dynamic integration v which does not implement -// Config directly. The transform function will be invoked to create a wrapper -// implementation of Config. transform will be invoked after unmarshaling v from -// YAML, and the result will be unwrapped back to the original form at marshal -// time. -// -// RegisterDynamic is only useful in niche instances. When possible, use -// Register instead. -// -// name is the name of the integration used for unmarshaling from YAML. Must be -// unique across all integrations. -func RegisterDynamic(v interface{}, name string, ty Type, transform WrapperFunc) { - if _, isConfig := v.(Config); !isConfig && transform == nil { - panic(fmt.Sprintf("transform must not be nil; %T does not implement Config", v)) - } +func registerIntegration(v interface{}, name string, ty Type, upgrader UpgradeFunc) { if reflect.TypeOf(v).Kind() != reflect.Ptr { - panic(fmt.Sprintf("RegisterDynamic must be given a pointer, got %T", v)) + panic(fmt.Sprintf("Register must be given a pointer, got %T", v)) } if _, exist := integrationNames[name]; exist { panic(fmt.Sprintf("Integration %q registered twice", name)) @@ -65,20 +51,31 @@ func RegisterDynamic(v interface{}, name string, ty Type, transform WrapperFunc) configTy := reflect.TypeOf(v) integrationTypes[configTy] = ty configFieldNames[configTy] = name - mutators[configTy] = transform + upgraders[configTy] = upgrader +} + +// RegisterLegacy registers a v1.Config. upgrader will be used to upgrade it. +// upgrader will only be invoked after unmarshaling cfg from YAML, and the +// upgraded Config will be unwrapped again when marshaling back to YAML. +// +// Deprecated: RegisterLegacy only exists for the transition period where the v2 +// integrations subsystem is an experiment. RegisterLegacy will be removed at a +// later date. +func RegisterLegacy(cfg v1.Config, ty Type, upgrader UpgradeFunc) { + registerIntegration(cfg, cfg.Name(), ty, upgrader) } -// WrapperFunc wraps in in a WrappedConfig container. -type WrapperFunc func(in interface{}) WrappedConfig +// UpgradeFunc upgrades cfg to a UpgradedConfig. +type UpgradeFunc func(cfg v1.Config) UpgradedConfig -// WrappedConfig represents a Config which is a container for some data. This -// is used when registering integrations through RegisterDynamic. -type WrappedConfig interface { +// UpgradedConfig is a v2 Config that was constructed through a legacy +// v1.Config. It allows unwrapping to retrieve the original config for +// the purposes of marshaling or unmarshaling. +type UpgradedConfig interface { Config - // UnwrapConfig returns the inner data. The inner data is used for YAML - // marshaling and unmarshaling. - UnwrapConfig() interface{} + // LegacyConfig returns the old v1.Config. + LegacyConfig() v1.Config } // Type determines a specific type of integration. @@ -110,7 +107,7 @@ func setRegistered(t *testing.T, cc map[Config]Type) { integrationTypes = make(map[reflect.Type]Type) configFieldNames = make(map[reflect.Type]string) registered = registered[:0] - mutators = make(map[reflect.Type]WrapperFunc) + upgraders = make(map[reflect.Type]UpgradeFunc) } t.Cleanup(clear) @@ -121,26 +118,28 @@ func setRegistered(t *testing.T, cc map[Config]Type) { } } -// Registered all Configs that were passed to Register or RegisterDynamic. Each +// Registered all Configs that were passed to Register or RegisterLegacy. Each // call will generate a new set of configs. func Registered() []Config { res := make([]Config, 0, len(registered)) - for _, dyn := range registered { - switch v := dyn.(type) { + for _, r := range registered { + switch v := r.(type) { case Config: - res = append(res, cloneDynamic(v).(Config)) - default: - mut, ok := mutators[reflect.TypeOf(dyn)] + res = append(res, cloneValue(v).(Config)) + case v1.Config: + mut, ok := upgraders[reflect.TypeOf(r)] if !ok || mut == nil { - panic(fmt.Sprintf("Could not find transformer for dynamic integration %T", dyn)) + panic(fmt.Sprintf("Could not find transformer for legacy integration %T", r)) } - res = append(res, mut(cloneDynamic(dyn))) + res = append(res, mut(cloneValue(r).(v1.Config))) + default: + panic(fmt.Sprintf("unexpected type %T", r)) } } return res } -func cloneDynamic(in interface{}) interface{} { +func cloneValue(in interface{}) interface{} { return reflect.New(reflect.TypeOf(in).Elem()).Interface() } @@ -192,8 +191,8 @@ func MarshalYAML(v interface{}) (interface{}, error) { fieldName := c.Name() var data interface{} = c - if wc, ok := c.(WrappedConfig); ok { - data = wc.UnwrapConfig() + if wc, ok := c.(UpgradedConfig); ok { + data = wc.LegacyConfig() } integrationType, ok := integrationTypes[reflect.TypeOf(data)] @@ -302,21 +301,19 @@ func getConfig(in interface{}) Config { switch c := in.(type) { case Config: return c - default: - mut, ok := mutators[reflect.TypeOf(in)] + case v1.Config: + mut, ok := upgraders[reflect.TypeOf(in)] if !ok { panic(fmt.Sprintf("unexpected type %T", c)) } - return mut(in) + return mut(c) + default: + panic(fmt.Sprintf("unexpected type %T", c)) } } // getConfigTypeForIntegrations returns a dynamic struct type that has all of // the same fields as out including the fields for the provided integrations. -// -// If marshal is true, interface{} will be used for integration types. This -// must be true when marshaling dynamic integrations, where their type will -// have changed. func getConfigTypeForIntegrations(out reflect.Type) reflect.Type { // Initial exported fields map one-to-one. var fields []reflect.StructField @@ -333,9 +330,9 @@ func getConfigTypeForIntegrations(out reflect.Type) reflect.Type { } } - for _, dyn := range registered { + for _, reg := range registered { // Fields use a prefix that's unlikely to collide with anything else. - configTy := reflect.TypeOf(dyn) + configTy := reflect.TypeOf(reg) integrationType := integrationTypes[configTy] fieldName := configFieldNames[configTy] diff --git a/pkg/integrations/v2/register_test.go b/pkg/integrations/v2/register_test.go index 2426c295bbef..a1d17b09c131 100644 --- a/pkg/integrations/v2/register_test.go +++ b/pkg/integrations/v2/register_test.go @@ -5,6 +5,8 @@ import ( "time" "github.com/go-kit/log" + v1 "github.com/grafana/agent/pkg/integrations" + agent_v1 "github.com/grafana/agent/pkg/integrations/agent" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" ) @@ -74,44 +76,40 @@ test_configs: require.Equal(t, expect, fullCfg) } -func TestIntegrationRegistration_Dynamic(t *testing.T) { +func TestIntegrationRegistration_Legacy(t *testing.T) { setRegistered(t, nil) - type raw struct { - Name string `yaml:"name"` - } - - RegisterDynamic(&raw{}, "dyn", TypeSingleton, func(in interface{}) WrappedConfig { - return &dynamicShim{IntegrationName: "dyn", Data: in} + RegisterLegacy(&agent_v1.Config{}, TypeSingleton, func(in v1.Config) UpgradedConfig { + return &legacyShim{Data: in} }) var cfgToParse = ` name: John Doe duration: 500ms -dyn: - name: Hello, world!` +agent: + instance: foo` var fullCfg testFullConfig err := yaml.UnmarshalStrict([]byte(cfgToParse), &fullCfg) require.NoError(t, err) require.Len(t, fullCfg.Configs, 1) - require.IsType(t, &dynamicShim{}, fullCfg.Configs[0]) + require.IsType(t, &legacyShim{}, fullCfg.Configs[0]) - dynShim := fullCfg.Configs[0].(*dynamicShim) - require.IsType(t, &raw{}, dynShim.Data) - require.Equal(t, &raw{Name: "Hello, world!"}, dynShim.Data) -} + shim := fullCfg.Configs[0].(*legacyShim) + require.IsType(t, &agent_v1.Config{}, shim.Data) -type dynamicShim struct { - IntegrationName string - Data interface{} + agentConfig := shim.Data.(*agent_v1.Config) + require.Equal(t, "foo", *agentConfig.Common.InstanceKey) + require.Equal(t, true, agentConfig.Enabled) } -func (s *dynamicShim) UnwrapConfig() interface{} { return s.Data } -func (s *dynamicShim) Name() string { return s.IntegrationName } -func (s *dynamicShim) Identifier(g Globals) (string, error) { return g.AgentIdentifier, nil } -func (s *dynamicShim) NewIntegration(log.Logger, Globals) (Integration, error) { +type legacyShim struct{ Data v1.Config } + +func (s *legacyShim) LegacyConfig() v1.Config { return s.Data } +func (s *legacyShim) Name() string { return s.Data.Name() } +func (s *legacyShim) Identifier(g Globals) (string, error) { return g.AgentIdentifier, nil } +func (s *legacyShim) NewIntegration(log.Logger, Globals) (Integration, error) { return NoOpIntegration, nil } From cb2e94b87ad331e64b2432a3f15a49741c99f5dc Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Thu, 16 Dec 2021 15:36:21 -0500 Subject: [PATCH 4/4] fine, I won't add the deprecation notice if it will make the linter sad --- pkg/integrations/v2/register.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/integrations/v2/register.go b/pkg/integrations/v2/register.go index 5d7adc9dbc8c..85f56188e462 100644 --- a/pkg/integrations/v2/register.go +++ b/pkg/integrations/v2/register.go @@ -58,7 +58,7 @@ func registerIntegration(v interface{}, name string, ty Type, upgrader UpgradeFu // upgrader will only be invoked after unmarshaling cfg from YAML, and the // upgraded Config will be unwrapped again when marshaling back to YAML. // -// Deprecated: RegisterLegacy only exists for the transition period where the v2 +// RegisterLegacy only exists for the transition period where the v2 // integrations subsystem is an experiment. RegisterLegacy will be removed at a // later date. func RegisterLegacy(cfg v1.Config, ty Type, upgrader UpgradeFunc) {