From 97b3a357038dee875682c6e95d71240a6b54bbe5 Mon Sep 17 00:00:00 2001 From: Mike Dame Date: Tue, 22 Nov 2022 14:39:21 -0500 Subject: [PATCH] Add regex matching to resource attribute filters (#535) --- exporter/collector/config.go | 12 ++++++++++ exporter/collector/config_test.go | 9 +++++++ exporter/collector/logs.go | 2 +- exporter/collector/metrics.go | 2 +- exporter/collector/monitoredresource.go | 6 ++++- exporter/collector/monitoredresource_test.go | 25 ++++++++++++++++++-- 6 files changed, 51 insertions(+), 5 deletions(-) diff --git a/exporter/collector/config.go b/exporter/collector/config.go index 5fb01c7e1..56c0fae58 100644 --- a/exporter/collector/config.go +++ b/exporter/collector/config.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "regexp" "strings" "time" @@ -138,6 +139,8 @@ type ImpersonateConfig struct { type ResourceFilter struct { // Match resource keys by prefix Prefix string `mapstructure:"prefix"` + // Match resource keys by regex + Regex string `mapstructure:"regex"` } type LogConfig struct { @@ -185,6 +188,15 @@ func ValidateConfig(cfg Config) error { } seenReplacements[mapping.Replacement] = struct{}{} } + + for _, resourceFilter := range cfg.MetricConfig.ResourceFilters { + if len(resourceFilter.Regex) == 0 { + continue + } + if _, err := regexp.Compile(resourceFilter.Regex); err != nil { + return fmt.Errorf("unable to parse resource filter regex: %s", err.Error()) + } + } return nil } diff --git a/exporter/collector/config_test.go b/exporter/collector/config_test.go index fdd60af8e..1ba12bdb3 100644 --- a/exporter/collector/config_test.go +++ b/exporter/collector/config_test.go @@ -66,6 +66,15 @@ func TestValidateConfig(t *testing.T) { }, expectedErr: true, }, + { + desc: "Invalid resource filter regex", + input: Config{ + MetricConfig: MetricConfig{ + ResourceFilters: []ResourceFilter{{Regex: "*"}}, + }, + }, + expectedErr: true, + }, } { t.Run(tc.desc, func(t *testing.T) { err := ValidateConfig(tc.input) diff --git a/exporter/collector/logs.go b/exporter/collector/logs.go index c744bceb5..107bf79a4 100644 --- a/exporter/collector/logs.go +++ b/exporter/collector/logs.go @@ -211,7 +211,7 @@ func (l logMapper) createEntries(ld plog.Logs) ([]*logpb.LogEntry, error) { for i := 0; i < ld.ResourceLogs().Len(); i++ { rl := ld.ResourceLogs().At(i) mr := defaultResourceToMonitoredResource(rl.Resource()) - extraResourceLabels := resourceToLabels(rl.Resource(), false, l.cfg.LogConfig.ResourceFilters) + extraResourceLabels := resourceToLabels(rl.Resource(), false, l.cfg.LogConfig.ResourceFilters, l.obs.log) projectID := l.cfg.ProjectID // override project ID with gcp.project.id, if present if projectFromResource, found := rl.Resource().Attributes().Get(resourcemapping.ProjectIDAttributeKey); found { diff --git a/exporter/collector/metrics.go b/exporter/collector/metrics.go index 50646b93e..2ef3800ee 100644 --- a/exporter/collector/metrics.go +++ b/exporter/collector/metrics.go @@ -174,7 +174,7 @@ func (me *MetricsExporter) PushMetrics(ctx context.Context, m pmetric.Metrics) e for i := 0; i < rms.Len(); i++ { rm := rms.At(i) monitoredResource := me.cfg.MetricConfig.MapMonitoredResource(rm.Resource()) - extraResourceLabels := resourceToLabels(rm.Resource(), me.cfg.MetricConfig.ServiceResourceLabels, me.cfg.MetricConfig.ResourceFilters) + extraResourceLabels := resourceToLabels(rm.Resource(), me.cfg.MetricConfig.ServiceResourceLabels, me.cfg.MetricConfig.ResourceFilters, me.obs.log) projectID := me.cfg.ProjectID // override project ID with gcp.project.id, if present if projectFromResource, found := rm.Resource().Attributes().Get(resourcemapping.ProjectIDAttributeKey); found { diff --git a/exporter/collector/monitoredresource.go b/exporter/collector/monitoredresource.go index a0e07c16f..92bc951c6 100644 --- a/exporter/collector/monitoredresource.go +++ b/exporter/collector/monitoredresource.go @@ -15,8 +15,11 @@ package collector import ( + "regexp" "strings" + "go.uber.org/zap" + "go.opentelemetry.io/collector/pdata/pcommon" semconv "go.opentelemetry.io/collector/semconv/v1.8.0" monitoredrespb "google.golang.org/genproto/googleapis/api/monitoredres" @@ -58,6 +61,7 @@ func resourceToLabels( resource pcommon.Resource, serviceResourceLabels bool, resourceFilters []ResourceFilter, + log *zap.Logger, ) labels { attrs := pcommon.NewMap() resource.Attributes().Range(func(k string, v pcommon.Value) bool { @@ -71,7 +75,7 @@ func resourceToLabels( } // Matches one of the resource filters for _, resourceFilter := range resourceFilters { - if strings.HasPrefix(k, resourceFilter.Prefix) { + if match, _ := regexp.Match(resourceFilter.Regex, []byte(k)); strings.HasPrefix(k, resourceFilter.Prefix) && match { v.CopyTo(attrs.PutEmpty(k)) return true } diff --git a/exporter/collector/monitoredresource_test.go b/exporter/collector/monitoredresource_test.go index 05f0f066c..7648f8f21 100644 --- a/exporter/collector/monitoredresource_test.go +++ b/exporter/collector/monitoredresource_test.go @@ -456,6 +456,27 @@ func TestResourceToMetricLabels(t *testing.T) { "service_namespace": "myservicenamespace", }, }, + { + name: "Resource filter regex config + prefix", + updateMapper: func(mapper *metricMapper) { + mapper.cfg.MetricConfig.ResourceFilters = []ResourceFilter{ + {Prefix: "cloud.", Regex: ".*availability.*"}, + {Regex: "host.*"}, + } + }, + resourceLabels: map[string]string{ + "cloud.platform": "gcp_compute_engine", + "cloud.availability_zone": "us-central1", + "cloud.availability_region": "us-central", + "host.id": "abc123", + "random.fake": "woot", + }, + expectExtraLabels: labels{ + "cloud_availability_zone": "us-central1", + "cloud_availability_region": "us-central", + "host_id": "abc123", + }, + }, } for _, test := range tests { @@ -468,7 +489,7 @@ func TestResourceToMetricLabels(t *testing.T) { for k, v := range test.resourceLabels { r.Attributes().PutStr(k, v) } - extraLabels := resourceToLabels(r, mapper.cfg.MetricConfig.ServiceResourceLabels, mapper.cfg.MetricConfig.ResourceFilters) + extraLabels := resourceToLabels(r, mapper.cfg.MetricConfig.ServiceResourceLabels, mapper.cfg.MetricConfig.ResourceFilters, nil) assert.Equal(t, test.expectExtraLabels, extraLabels) }) } @@ -516,6 +537,6 @@ func TestResourceMetricsToMonitoredResourceUTF8(t *testing.T) { } mr := defaultResourceToMonitoredResource(r) assert.Equal(t, expectMr, mr) - extraLabels := resourceToLabels(r, mapper.cfg.MetricConfig.ServiceResourceLabels, mapper.cfg.MetricConfig.ResourceFilters) + extraLabels := resourceToLabels(r, mapper.cfg.MetricConfig.ServiceResourceLabels, mapper.cfg.MetricConfig.ResourceFilters, nil) assert.Equal(t, expectExtraLabels, extraLabels) }