diff --git a/docs/sources/reference/components/loki/loki.secretfilter.md b/docs/sources/reference/components/loki/loki.secretfilter.md index 0a31ca49fe4..820f920a318 100644 --- a/docs/sources/reference/components/loki/loki.secretfilter.md +++ b/docs/sources/reference/components/loki/loki.secretfilter.md @@ -49,12 +49,13 @@ You can use the following arguments with `loki.secretfilter`: | Name | Type | Description | Default | Required | | -------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- | -------- | | `forward_to` | `list(LogsReceiver)` | List of receivers to send log entries to. | | yes | -| `drop_on_timeout` | `bool` | When true, drop entries that exceed `processing_timeout` instead of forwarding them unredacted. | `false` | no | | `gitleaks_config` | `string` | Path to a custom Gitleaks TOML config file. If empty, the default Gitleaks config is used. | `""` | no | | `origin_label` | `string` | Loki label to use for the `secrets_redacted_by_origin` metric. If empty, that metric is not registered. | `""` | no | -| `processing_timeout` | `duration` | Maximum time allowed to process a single log entry. `0` disables the timeout. | `0` | no | -| `redact_percent` | `uint` | When `redact_with` is not set: percent of the secret to redact (1–100), where 100 is full redaction. | `80` | no | +| `rate` | `float` | Entry sampling rate in `[0.0, 1.0]` where `1` processes all entries. Unsampled entries are forwarded unchanged. | `1.0` | no | | `redact_with` | `string` | Template for the redaction placeholder. Use `$SECRET_NAME` and `$SECRET_HASH`, for example, `"<$SECRET_NAME:$SECRET_HASH>"` | `""` | no | +| `redact_percent` | `uint` | When `redact_with` is not set: percent of the secret to redact (1–100), where 100 is full redaction. | `80` | no | +| `drop_on_timeout` | `bool` | When true, drop entries that exceed `processing_timeout` instead of forwarding them unredacted. | `false` | no | +| `processing_timeout` | `duration` | Maximum time allowed to process a single log entry. `0` disables the timeout. | `0` | no | The `gitleaks_config` argument is the path to a custom [Gitleaks TOML config file][gitleaks-config]. The file supports the standard Gitleaks structure (rules, allowlists, and `[extend]` to extend the default config). @@ -75,6 +76,11 @@ For consistent behavior, use an external configuration file via `gitleaks_config `100` replaces the entire secret with `"REDACTED"`. When `redact_percent` is 0 or unset, 80% redaction is used. +**Sampling:** The `rate` argument controls what fraction of log entries are processed by the secret filter. +Entries that {{< param "PRODUCT_NAME" >}} does not select based on the sampling rate pass through unchanged, with no detection or redaction applied. +Use a value below `1.0`, for example, `0.1` for 10%, to reduce CPU usage when processing high-volume logs. +Monitor `loki_secretfilter_entries_bypassed_total` to observe how many entries were skipped. + **Origin metric:** The `origin_label` argument specifies which Loki label to use for the `secrets_redacted_by_origin` metric, so you can track how many secrets were redacted per source or environment. **Processing timeout:** The `processing_timeout` argument sets a maximum duration for processing each log entry. @@ -110,14 +116,16 @@ The following fields are exported and can be referenced by other components: `loki.secretfilter` exposes the following Prometheus metrics: -| Name | Type | Description | -| -------------------------------------------------- | ------- | --------------------------------------------------------------------------------------------------- | -| `loki_secretfilter_lines_dropped_total` | Counter | Total number of log lines dropped due to processing timeout, when `drop_on_timeout` is `true`. | -| `loki_secretfilter_lines_timed_out_total` | Counter | Total number of log lines that exceeded the processing timeout, whether dropped or forwarded. | -| `loki_secretfilter_processing_duration_seconds` | Summary | Time taken to process and redact logs, in seconds. | -| `loki_secretfilter_secrets_redacted_total` | Counter | Total number of secrets redacted. | -| `loki_secretfilter_secrets_redacted_by_rule_total` | Counter | Number of secrets redacted, partitioned by rule name. | -| `loki_secretfilter_secrets_redacted_by_origin` | Counter | Number of secrets redacted, partitioned by origin label, when `origin_label` is set. | +| Name | Type | Description | +| -------------------------------------------------- | ------- | ---------------------------------------------------------------------------------------------- | +| `loki_secretfilter_entries_bypassed_total` | Counter | Total number of entries forwarded without processing due to sampling. | +| `loki_secretfilter_lines_dropped_total` | Counter | Total number of log lines dropped due to processing timeout, when `drop_on_timeout` is `true`. | +| `loki_secretfilter_lines_timed_out_total` | Counter | Total number of log lines that exceeded the processing timeout, whether dropped or forwarded. | +| `loki_secretfilter_processing_duration_seconds` | Summary | Time taken to process and redact logs, in seconds. | +| `loki_secretfilter_secrets_redacted_total` | Counter | Total number of secrets redacted. | +| `loki_secretfilter_secrets_redacted_by_rule_total` | Counter | Number of secrets redacted, partitioned by rule name. | +| `loki_secretfilter_secrets_redacted_by_origin` | Counter | Number of secrets redacted, partitioned by origin label, when `origin_label` is set. | + ## Example @@ -128,6 +136,7 @@ Alternatively, you can: - Omit `redact_with` to use percentage-based redaction, which defaults to 80% redacted. - Set `redact_percent` to `100` for full redaction. - Set `gitleaks_config` to point to a custom Gitleaks TOML configuration file. +- Set `rate` to a value below `1.0` to sample entries and reduce CPU usage; entries not selected are forwarded unchanged. ```alloy local.file_match "local_logs" { diff --git a/internal/component/loki/secretfilter/extend/extend_test.go b/internal/component/loki/secretfilter/extend/extend_test.go new file mode 100644 index 00000000000..9466dfda901 --- /dev/null +++ b/internal/component/loki/secretfilter/extend/extend_test.go @@ -0,0 +1,48 @@ +package extend + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/grafana/alloy/internal/component/loki/secretfilter" +) + +// This test file is used to test the secretfilter component with a custom gitleaks config file. +// The reason why these tests are in a separate package is that the gitleaks library uses global variables +// when loading configs with [extend] useDefault = true. If we ran these +// tests in the same package as the main secretfilter tests, the extend logic does not work correctly. +// putting the extend tests in a separate package, "go test ./..." runs them in a +// different process, so they get a clean viper and gitleaks global state. + +// gitleaksConfigPath returns the path to gitleaks.toml in the extend package directory. +func gitleaksConfigPath(t *testing.T) string { + t.Helper() + _, filename, _, _ := runtime.Caller(0) + dir := filepath.Dir(filename) + return filepath.Join(dir, "gitleaks.toml") +} + +func TestSecretFiltering_WithGitleaksConfigFile(t *testing.T) { + configPath := gitleaksConfigPath(t) + if _, err := os.Stat(configPath); err != nil { + t.Skipf("gitleaks.toml not found at %s: %v", configPath, err) + } + + config := fmt.Sprintf(` + forward_to = [] + gitleaks_config = %q + `, configPath) + + // Same cases as default, plus the UUID allowlist case (generic-api-key when secret is a UUID). + cases := secretfilter.DefaultTestCases() + cases = append(cases, secretfilter.TestCase{ + Name: "generic_api_key_uuid_allowlisted", + InputLog: `{"message": "audit tokenID=550e8400-e29b-41d4-a716-446655440000 resourceID=6ba7b810-9dad-11d1-80b4-00c04fd430c8"}`, + ShouldRedact: false, + }) + + secretfilter.RunTestCases(t, config, cases) +} diff --git a/internal/component/loki/secretfilter/extend/gitleaks.toml b/internal/component/loki/secretfilter/extend/gitleaks.toml new file mode 100644 index 00000000000..023e34e45cf --- /dev/null +++ b/internal/component/loki/secretfilter/extend/gitleaks.toml @@ -0,0 +1,14 @@ +title = "gitleakstests extended config" + +[extend] +useDefault = true + +# allowlist by secret – ignore when the “secret” is a UUID. +[[rules]] +id = "generic-api-key" + [[rules.allowlists]] + description = "ignore when extracted secret is a UUID" + regexTarget = "secret" + regexes = [ + '''^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$''', + ] diff --git a/internal/component/loki/secretfilter/run_test_cases.go b/internal/component/loki/secretfilter/run_test_cases.go new file mode 100644 index 00000000000..0ff38650dab --- /dev/null +++ b/internal/component/loki/secretfilter/run_test_cases.go @@ -0,0 +1,68 @@ +// run_test_cases.go provides RunTestCases for use by secretfilter tests and the +// extend package. It lives in the secretfilter package so it can use +// secretfilter.Arguments/Exports and avoids testhelper importing secretfilter. + +package secretfilter + +import ( + "context" + "testing" + "time" + + "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/component/common/loki" + "github.com/grafana/alloy/internal/component/loki/secretfilter/testhelper" + "github.com/grafana/alloy/internal/util" + "github.com/grafana/alloy/syntax" + "github.com/grafana/loki/pkg/push" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" +) + +// TestCase is a single test case for RunTestCases (name, log line, expected redaction). +type TestCase struct { + Name string + InputLog string + ShouldRedact bool +} + +// RunTestCases runs all cases through a single component (one config load). +// It builds the component once and calls processEntry for each case. +func RunTestCases(t *testing.T, config string, cases []TestCase) { + t.Helper() + var args Arguments + require.NoError(t, syntax.Unmarshal([]byte(config), &args)) + args.ForwardTo = []loki.LogsReceiver{loki.NewLogsReceiver()} + + opts := component.Options{ + Logger: util.TestLogger(t), + OnStateChange: func(component.Exports) {}, + GetServiceData: testhelper.GetServiceData, + Registerer: prometheus.NewRegistry(), + } + c, err := New(opts, args) + require.NoError(t, err) + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + require.NotEmpty(t, tc.InputLog) + entry := loki.Entry{Labels: model.LabelSet{}, Entry: push.Entry{Timestamp: time.Now(), Line: tc.InputLog}} + got, _ := c.processEntry(context.Background(), entry) + if tc.ShouldRedact { + require.NotEqual(t, tc.InputLog, got.Line, "Expected log to be redacted but it was not") + } else { + require.Equal(t, tc.InputLog, got.Line, "Expected log to remain unchanged but it was modified") + } + }) + } +} + +// DefaultTestCases returns the standard 7 cases from testhelper as []TestCase. +func DefaultTestCases() []TestCase { + out := make([]TestCase, 0, len(testhelper.DefaultCases)) + for _, c := range testhelper.DefaultCases { + out = append(out, TestCase{Name: c.Name, InputLog: c.InputLog, ShouldRedact: c.ShouldRedact}) + } + return out +} diff --git a/internal/component/loki/secretfilter/secretfilter.go b/internal/component/loki/secretfilter/secretfilter.go index 317f5de2887..a5dd7c55f1c 100644 --- a/internal/component/loki/secretfilter/secretfilter.go +++ b/internal/component/loki/secretfilter/secretfilter.go @@ -5,6 +5,8 @@ import ( "context" "crypto/sha1" "fmt" + "math" + "math/rand" "os" "strings" "sync" @@ -15,6 +17,7 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/service/livedebugging" "github.com/grafana/alloy/internal/util" + "github.com/grafana/alloy/syntax" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/spf13/viper" @@ -43,6 +46,7 @@ type Arguments struct { RedactWith string `alloy:"redact_with,attr,optional"` // Template for redaction placeholder; $SECRET_NAME and $SECRET_HASH are replaced. When set, percentage-based redaction is not used. RedactPercent uint `alloy:"redact_percent,attr,optional"` // When redact_with is not set: percent of the secret to redact (1-100; gitleaks-style: show leading (100-N)% + "...", 100 = "REDACTED"). 0 or unset defaults to 80. GitleaksConfig string `alloy:"gitleaks_config,attr,optional"` // Path to a gitleaks TOML config file; if empty, the default gitleaks config is used + Rate float64 `alloy:"rate,attr,optional"` // Sampling rate in [0.0, 1.0]: fraction of entries to process through the secret filter; rest are forwarded unchanged. 1.0 = process all (default). ProcessingTimeout time.Duration `alloy:"processing_timeout,attr,optional"` // Maximum time allowed to process a single log entry. 0 (default) disables the timeout. DropOnTimeout bool `alloy:"drop_on_timeout,attr,optional"` // When true, entries that exceed processing_timeout are dropped instead of forwarded unredacted. Requires processing_timeout to be set. } @@ -52,16 +56,35 @@ type Exports struct { Receiver loki.LogsReceiver `alloy:"receiver,attr"` } +// defaultRate is the default sampling rate (1.0 = process all entries). +const defaultRate = 1.0 + +// defaultRedactPercent is the percentage used for gitleaks-style redaction when redact_with is not set and redact_percent is 0 or out of range. +const defaultRedactPercent uint = 80 + // DefaultArguments defines the default settings for log scraping. -var DefaultArguments = Arguments{} +var DefaultArguments = Arguments{ + Rate: defaultRate, + RedactPercent: defaultRedactPercent, +} // SetToDefault implements syntax.Defaulter. func (args *Arguments) SetToDefault() { *args = DefaultArguments } -// defaultRedactPercent is the percentage used for gitleaks-style redaction when redact_with is not set and redact_percent is 0 or out of range. -const defaultRedactPercent uint = 80 +// Validate implements syntax.Validator. +func (args *Arguments) Validate() error { + if args.Rate < 0.0 || args.Rate > 1.0 { + return fmt.Errorf("secretfilter rate must be between 0.0 and 1.0, received %f", args.Rate) + } + return nil +} + +var _ syntax.Validator = (*Arguments)(nil) + +// maxRandomNumber is the maximum value used for sampling boundary +const maxRandomNumber = ^(uint64(1) << 63) // 0x7fffffffffffffff var ( _ component.Component = (*Component)(nil) @@ -81,6 +104,10 @@ type Component struct { // redactPercent is the effective percentage (1-100) for gitleaks-style redaction when redact_with is not set. Set at build/update. redactPercent uint + // sampling state (used when 0 < Rate < 1) + samplingBoundary uint64 + samplingSource rand.Source + metrics *metrics debugDataPublisher livedebugging.DebugDataPublisher } @@ -91,8 +118,10 @@ type Component struct { // - loki_secretfilter_secrets_redacted_by_rule_total: Number of secrets redacted, partitioned by rule name. // - loki_secretfilter_secrets_redacted_by_origin: Number of secrets redacted, partitioned by origin label value (only registered when origin_label is set). // - loki_secretfilter_processing_duration_seconds: Summary of time taken to process and redact log entries. +// - loki_secretfilter_entries_bypassed_total: Total number of entries forwarded without processing due to sampling. // - loki_secretfilter_lines_timed_out_total: Total number of log lines that exceeded the processing timeout (regardless of whether they were dropped or forwarded). // - loki_secretfilter_lines_dropped_total: Total number of log lines dropped due to processing timeout (subset of lines_timed_out_total, only when drop_on_timeout is true). + type metrics struct { // Total number of secrets redacted secretsRedactedTotal prometheus.Counter @@ -106,6 +135,9 @@ type metrics struct { // Summary of time taken for redaction log processing processingDuration prometheus.Summary + // Total number of entries bypassed by sampling (forwarded unchanged) + entriesBypassedTotal prometheus.Counter + // Total number of log lines that exceeded the processing timeout, regardless of whether they were dropped or forwarded unredacted linesTimedOutTotal prometheus.Counter @@ -148,6 +180,11 @@ func newMetrics(reg prometheus.Registerer, originLabel string) *metrics { }, }) + m.entriesBypassedTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: "loki_secretfilter", + Name: "entries_bypassed_total", + Help: "Total number of entries forwarded without processing due to sampling.", + }) m.linesTimedOutTotal = prometheus.NewCounter(prometheus.CounterOpts{ Subsystem: "loki_secretfilter", Name: "lines_timed_out_total", @@ -167,6 +204,7 @@ func newMetrics(reg prometheus.Registerer, originLabel string) *metrics { m.secretsRedactedByOrigin = util.MustRegisterOrGet(reg, m.secretsRedactedByOrigin).(*prometheus.CounterVec) } m.processingDuration = util.MustRegisterOrGet(reg, m.processingDuration).(prometheus.Summary) + m.entriesBypassedTotal = util.MustRegisterOrGet(reg, m.entriesBypassedTotal).(prometheus.Counter) m.linesTimedOutTotal = util.MustRegisterOrGet(reg, m.linesTimedOutTotal).(prometheus.Counter) m.linesDroppedTotal = util.MustRegisterOrGet(reg, m.linesDroppedTotal).(prometheus.Counter) } @@ -252,21 +290,26 @@ func (c *Component) Run(ctx context.Context) error { return nil case entry := <-c.receiver.Chan(): c.mut.RLock() - // Start processing the log entry to redact secrets - newEntry, dropped := c.processEntry(ctx, entry) - if dropped { - c.mut.RUnlock() - continue - } - c.debugDataPublisher.PublishIfActive(livedebugging.NewData( - componentID, - livedebugging.LokiLog, - 1, - func() string { - return fmt.Sprintf("%s => %s", entry.Line, newEntry.Line) - }, - )) + var newEntry loki.Entry + if c.shouldProcessEntry() { + newEntry, dropped := c.processEntry(ctx, entry) + if dropped { + c.mut.RUnlock() + continue + } + c.debugDataPublisher.PublishIfActive(livedebugging.NewData( + componentID, + livedebugging.LokiLog, + 1, + func() string { + return fmt.Sprintf("%s => %s", entry.Line, newEntry.Line) + }, + )) + } else { + newEntry = entry + c.metrics.entriesBypassedTotal.Inc() + } for _, f := range c.fanout { select { @@ -281,6 +324,31 @@ func (c *Component) Run(ctx context.Context) error { } } +// shouldProcessEntry returns true if this entry should be processed through the secret filter (rate = probability of "keep" / process). +func (c *Component) shouldProcessEntry() bool { + rate := c.args.Rate + if rate >= 1.0 { + return true + } + if rate <= 0.0 { + return false + } + return c.samplingBoundary >= c.samplingRandomID()&maxRandomNumber +} + +// samplingRandomID returns a random uint64 in [1, maxRandomNumber] for sampling. +// If samplingSource is nil (e.g. rate was 0 or 1), returns maxRandomNumber so the caller does not panic. +func (c *Component) samplingRandomID() uint64 { + if c.samplingSource == nil { + return maxRandomNumber + } + val := uint64(c.samplingSource.Int63()) + for val == 0 { + val = uint64(c.samplingSource.Int63()) + } + return val +} + // processEntry scans the log entry for secrets and redacts them. Returns the // processed entry and a boolean indicating whether the entry should be dropped. // If processing_timeout is exceeded and drop_on_timeout is false (default), @@ -382,6 +450,13 @@ func (c *Component) Update(args component.Arguments) error { } else { c.redactPercent = defaultRedactPercent } + if newArgs.Rate > 0 && newArgs.Rate < 1 { + c.samplingBoundary = uint64(float64(maxRandomNumber) * math.Max(0, math.Min(newArgs.Rate, 1))) + c.samplingSource = rand.NewSource(time.Now().UnixNano()) + } else { + c.samplingBoundary = 0 + c.samplingSource = nil + } c.metrics = newMetrics(c.opts.Registerer, newArgs.OriginLabel) return nil diff --git a/internal/component/loki/secretfilter/secretfilter_test.go b/internal/component/loki/secretfilter/secretfilter_test.go index 7784d421c9c..2ed88c2ae31 100644 --- a/internal/component/loki/secretfilter/secretfilter_test.go +++ b/internal/component/loki/secretfilter/secretfilter_test.go @@ -3,10 +3,8 @@ package secretfilter import ( "context" "fmt" - "os" "path/filepath" "strings" - "sync" "testing" "time" @@ -14,8 +12,7 @@ import ( "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/component/common/loki" - "github.com/grafana/alloy/internal/runtime/componenttest" - "github.com/grafana/alloy/internal/service/livedebugging" + "github.com/grafana/alloy/internal/component/loki/secretfilter/testhelper" "github.com/grafana/alloy/internal/util" "github.com/grafana/alloy/syntax" "github.com/grafana/loki/pkg/push" @@ -26,236 +23,19 @@ import ( "github.com/stretchr/testify/require" ) -// fakeSecret represents a fake secret to be used in the tests -type fakeSecret struct { - name string - value string -} - -// testLog represents a log entry to be used in the tests -type testLog struct { - log string - secrets []fakeSecret // List of fake secrets it contains for easy redaction check -} - -// List of fake secrets to use for testing -// They are constructed so that they will match the regexes in the gitleaks configs -// Note that some string literals are concatenated to avoid being flagged as secrets -var fakeSecrets = map[string]fakeSecret{ - "grafana-api-key": { - name: "grafana-api-key", - value: "eyJr" + "IjoiT0x6NWJuNDRuZkI1NHJ6dEJrR0g3aDVuRnY0NWJuNDRuZkI1NHJ6dEJrR0g3aDVuRnY0NWJuNDRuZkI1NHJ6dEJrR0g3aDVuRnY0NWJuNDRuZkI1NHJ6dEJrR0g3aDVu", - }, - "gcp-api-key": { - name: "gcp-api-key", - value: "AIza" + "SyDaGmWKa4JsXZ-HjGw7ISLn_3namBGewQe", - }, - "stripe-key": { - name: "stripe-access-token", - value: "sk_live_" + "51HFxYz2eZvKYlo2C9kKM5nE6qO4yKn8N3bP7hXxYz2eZvKYlo2C", - }, - "npm-token": { - name: "npm-access-token", - value: "npm_" + "1A2b3C4d5E6f7G8h9I0jK1lM2nO3pQ4rS5tU", - }, - "generic-api-key": { - name: "generic-api-key", - value: "token:" + "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8Ii9Jj0Kk", - }, -} - -// List of fake log entries to use for testing -var testLogs = map[string]testLog{ - "no_secret": { - log: `{ - "message": "This is a simple log message" - }`, - secrets: []fakeSecret{}, - }, - "grafana_api_key": { - log: `{ - "message": "This is a simple log message with a secret value ` + fakeSecrets["grafana-api-key"].value + ` ! - }`, - secrets: []fakeSecret{fakeSecrets["grafana-api-key"]}, - }, - "gcp_api_key": { - log: `{ - "message": "This is a simple log message with a secret value ` + fakeSecrets["gcp-api-key"].value + ` ! - }`, - secrets: []fakeSecret{fakeSecrets["gcp-api-key"]}, - }, - "stripe_key": { - log: `{ - "message": "This is a simple log message with a secret value ` + fakeSecrets["stripe-key"].value + ` ! - }`, - secrets: []fakeSecret{fakeSecrets["stripe-key"]}, - }, - "npm_token": { - log: `{ - "message": "This is a simple log message with a secret value ` + fakeSecrets["npm-token"].value + ` ! - }`, - secrets: []fakeSecret{fakeSecrets["npm-token"]}, - }, - "generic_api_key": { - log: `{ - "message": "This is a simple log message with a secret value ` + fakeSecrets["generic-api-key"].value + ` ! - }`, - secrets: []fakeSecret{fakeSecrets["generic-api-key"]}, - }, - "multiple_secrets": { - log: `{ - "message": "This is a simple log message with a secret value ` + fakeSecrets["grafana-api-key"].value + ` and another secret value ` + fakeSecrets["gcp-api-key"].value + ` ! - }`, - secrets: []fakeSecret{fakeSecrets["grafana-api-key"], fakeSecrets["gcp-api-key"]}, - }, -} - -// Alloy configurations for testing -var testConfigs = map[string]string{ - "default": ` - forward_to = [] - `, - "with_origin": ` - forward_to = [] - origin_label = "job" - `, - "with_redact_percent_100": ` - forward_to = [] - redact_percent = 100 - `, - "with_redact_percent_80": ` - forward_to = [] - redact_percent = 80 - `, - "with_redact_with": ` - forward_to = [] - redact_with = "***REDACTED***" - `, -} - -// minimalGitleaksConfig is a valid gitleaks TOML that extends the default config -// so all default rules (including grafana-api-key, gcp-api-key, etc.) are used. -const minimalGitleaksConfig = `title = "test" -[extend] -useDefault = true -` - -// Test cases for the secret filter -var tt = []struct { - name string - config string - inputLog string - shouldRedact bool // Whether we expect any redaction to occur -}{ - { - "no_secret", - testConfigs["default"], - testLogs["no_secret"].log, - false, - }, - { - "grafana_api_key", - testConfigs["default"], - testLogs["grafana_api_key"].log, - true, - }, - { - "gcp_api_key", - testConfigs["default"], - testLogs["gcp_api_key"].log, - true, - }, - { - "stripe_key", - testConfigs["default"], - testLogs["stripe_key"].log, - true, - }, - { - "npm_token", - testConfigs["default"], - testLogs["npm_token"].log, - true, - }, - { - "generic_api_key", - testConfigs["default"], - testLogs["generic_api_key"].log, - true, - }, - { - "multiple_secrets", - testConfigs["default"], - testLogs["multiple_secrets"].log, - true, - }, -} - func TestSecretFiltering(t *testing.T) { - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - runTest(t, tc.config, tc.inputLog, tc.shouldRedact) - }) - } -} - -func runTest(t *testing.T, config string, inputLog string, shouldRedact bool) { - ch1 := loki.NewLogsReceiver() - var args Arguments - require.NoError(t, syntax.Unmarshal([]byte(config), &args)) - args.ForwardTo = []loki.LogsReceiver{ch1} - - // Making sure we're not testing with an empty log line by mistake - require.NotEmpty(t, inputLog) - - // Create component - tc, err := componenttest.NewControllerFromID(util.TestLogger(t), "loki.secretfilter") - require.NoError(t, err) - - // Run it - ctx, cancel := context.WithCancel(t.Context()) - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - err1 := tc.Run(ctx, args) - require.NoError(t, err1) - wg.Done() - }() - require.NoError(t, tc.WaitExports(time.Second)) - - // Get the input channel - input := tc.Exports().(Exports).Receiver - - // Send the log to the secret filter - entry := loki.Entry{Labels: model.LabelSet{}, Entry: push.Entry{Timestamp: time.Now(), Line: inputLog}} - input.Chan() <- entry - tc.WaitRunning(time.Second * 10) - - // Check the output - select { - case logEntry := <-ch1.Chan(): - require.WithinDuration(t, time.Now(), logEntry.Timestamp, 1*time.Second) - if shouldRedact { - // Verify that the output is different from the input (something was redacted) - require.NotEqual(t, inputLog, logEntry.Entry.Line, "Expected log to be redacted but it was not") - } else { - // Verify that the output is the same as the input (nothing was redacted) - require.Equal(t, inputLog, logEntry.Entry.Line, "Expected log to remain unchanged but it was modified") - } - case <-time.After(5 * time.Second): - require.FailNow(t, "failed waiting for log line") - } - - // Stop the component - cancel() - wg.Wait() + // One component, one config load; all default cases run through it. + RunTestCases(t, testhelper.TestConfigs["default"], DefaultTestCases()) } +// TestGitleaksConfig_InvalidPath checks that a missing config path returns an error. +// Valid custom config file loading (and [extend] useDefault) is tested in the +// extend package so it runs in a separate process and avoids gitleaks global state. func TestGitleaksConfig_InvalidPath(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, } args := Arguments{ ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, @@ -266,32 +46,90 @@ func TestGitleaksConfig_InvalidPath(t *testing.T) { require.Contains(t, err.Error(), "read gitleaks config") } -func TestGitleaksConfig_ValidFile(t *testing.T) { - dir := t.TempDir() - configPath := filepath.Join(dir, "gitleaks.toml") - require.NoError(t, os.WriteFile(configPath, []byte(minimalGitleaksConfig), 0600)) +// TestRate_ValidateInvalid checks that rate outside [0, 1] fails validation. +func TestRate_ValidateInvalid(t *testing.T) { + for _, rate := range []float64{-0.1, 1.1, 2.0} { + args := Arguments{Rate: rate} + err := args.Validate() + require.Error(t, err) + require.Contains(t, err.Error(), "between 0.0 and 1.0") + } + // Valid bounds + require.NoError(t, (&Arguments{Rate: 0}).Validate()) + require.NoError(t, (&Arguments{Rate: 1}).Validate()) + require.NoError(t, (&Arguments{Rate: 0.5}).Validate()) +} +// TestRate_ZeroBypassesAll verifies that with rate=0 all entries are forwarded unchanged and bypass counter increases. +func TestRate_ZeroBypassesAll(t *testing.T) { registry := prometheus.NewRegistry() + downstream := loki.NewLogsReceiver() + args := Arguments{ + ForwardTo: []loki.LogsReceiver{downstream}, + Rate: 0, + } opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } - args := Arguments{ - ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, - GitleaksConfig: configPath, - } c, err := New(opts, args) require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = c.Run(ctx) }() + + secret := testhelper.FakeSecrets["grafana-api-key"].Value + line := "log with secret " + secret + " end" entry := loki.Entry{ Labels: model.LabelSet{}, - Entry: push.Entry{Timestamp: time.Now(), Line: testLogs["grafana_api_key"].log}, + Entry: push.Entry{Timestamp: time.Now(), Line: line}, } - processed, _ := c.processEntry(context.Background(), entry) - require.NotEqual(t, entry.Entry.Line, processed.Entry.Line, "expected secret to be redacted when using custom config file") - require.NotContains(t, processed.Entry.Line, fakeSecrets["grafana-api-key"].value) + c.receiver.Chan() <- entry + received := <-downstream.Chan() + require.Equal(t, line, received.Line, "entry should be forwarded unchanged when rate=0") + require.Equal(t, float64(1), testutil.ToFloat64(c.metrics.entriesBypassedTotal)) +} + +// TestRate_Half approximates that with rate=0.5 about half of entries are processed and half bypassed. +func TestRate_Half(t *testing.T) { + registry := prometheus.NewRegistry() + downstream := loki.NewLogsReceiver() + args := Arguments{ + ForwardTo: []loki.LogsReceiver{downstream}, + Rate: 0.5, + } + opts := component.Options{ + Logger: util.TestLogger(t), + OnStateChange: func(e component.Exports) {}, + GetServiceData: testhelper.GetServiceData, + Registerer: registry, + } + c, err := New(opts, args) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = c.Run(ctx) }() + + n := 400 + secret := testhelper.FakeSecrets["grafana-api-key"].Value + lineWithSecret := "log " + secret + " end" + for i := 0; i < n; i++ { + entry := loki.Entry{ + Labels: model.LabelSet{}, + Entry: push.Entry{Timestamp: time.Now(), Line: lineWithSecret}, + } + c.receiver.Chan() <- entry + <-downstream.Chan() + } + + bypassed := testutil.ToFloat64(c.metrics.entriesBypassedTotal) + // With rate=0.5 we expect roughly half bypassed; allow 35–65% to avoid flakiness. + require.GreaterOrEqual(t, bypassed, float64(n)*0.35, "expected at least ~35%% of entries bypassed") + require.LessOrEqual(t, bypassed, float64(n)*0.65, "expected at most ~65%% of entries bypassed") } func TestRedactPercent_FullRedaction(t *testing.T) { @@ -299,7 +137,7 @@ func TestRedactPercent_FullRedaction(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } args := Arguments{ @@ -311,11 +149,11 @@ func TestRedactPercent_FullRedaction(t *testing.T) { entry := loki.Entry{ Labels: model.LabelSet{}, - Entry: push.Entry{Timestamp: time.Now(), Line: testLogs["grafana_api_key"].log}, + Entry: push.Entry{Timestamp: time.Now(), Line: testhelper.TestLogs["grafana_api_key"].Log}, } processed, _ := c.processEntry(context.Background(), entry) require.Contains(t, processed.Entry.Line, "REDACTED", "expected full redaction to produce REDACTED placeholder") - require.NotContains(t, processed.Entry.Line, fakeSecrets["grafana-api-key"].value) + require.NotContains(t, processed.Entry.Line, testhelper.FakeSecrets["grafana-api-key"].Value) } func TestRedactPercent_Partial(t *testing.T) { @@ -323,7 +161,7 @@ func TestRedactPercent_Partial(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } args := Arguments{ @@ -333,7 +171,7 @@ func TestRedactPercent_Partial(t *testing.T) { c, err := New(opts, args) require.NoError(t, err) - secret := fakeSecrets["grafana-api-key"].value + secret := testhelper.FakeSecrets["grafana-api-key"].Value entry := loki.Entry{ Labels: model.LabelSet{}, Entry: push.Entry{Timestamp: time.Now(), Line: "log with secret " + secret + " end"}, @@ -353,7 +191,7 @@ func TestRedactWith_CustomPlaceholder(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } args := Arguments{ @@ -365,11 +203,11 @@ func TestRedactWith_CustomPlaceholder(t *testing.T) { entry := loki.Entry{ Labels: model.LabelSet{}, - Entry: push.Entry{Timestamp: time.Now(), Line: testLogs["gcp_api_key"].log}, + Entry: push.Entry{Timestamp: time.Now(), Line: testhelper.TestLogs["gcp_api_key"].Log}, } processed, _ := c.processEntry(context.Background(), entry) require.Contains(t, processed.Entry.Line, "***REDACTED***") - require.NotContains(t, processed.Entry.Line, fakeSecrets["gcp-api-key"].value) + require.NotContains(t, processed.Entry.Line, testhelper.FakeSecrets["gcp-api-key"].Value) } // TestDefaultRedactPercent_usesEighty verifies that with no redact_with and no redact_percent, @@ -379,7 +217,7 @@ func TestDefaultRedactPercent_usesEighty(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } args := Arguments{ @@ -389,7 +227,7 @@ func TestDefaultRedactPercent_usesEighty(t *testing.T) { c, err := New(opts, args) require.NoError(t, err) - secret := fakeSecrets["grafana-api-key"].value + secret := testhelper.FakeSecrets["grafana-api-key"].Value entry := loki.Entry{ Labels: model.LabelSet{}, Entry: push.Entry{Timestamp: time.Now(), Line: "log " + secret + " end"}, @@ -399,103 +237,19 @@ func TestDefaultRedactPercent_usesEighty(t *testing.T) { require.NotContains(t, processed.Entry.Line, secret, "original secret should not appear in full") } -func TestProcessingTimeout_ForwardsUnredactedOnTimeout(t *testing.T) { - registry := prometheus.NewRegistry() - opts := component.Options{ - Logger: util.TestLogger(t), - OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, - Registerer: registry, - } - secret := fakeSecrets["grafana-api-key"].value - line := "log line with secret " + secret + " end" - c, err := New(opts, Arguments{ - ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, - ProcessingTimeout: 1 * time.Nanosecond, // guaranteed to expire before DetectString returns - }) - require.NoError(t, err) - - entry := loki.Entry{ - Labels: model.LabelSet{}, - Entry: push.Entry{Timestamp: time.Now(), Line: line}, - } - processed, dropped := c.processEntry(context.Background(), entry) - - require.False(t, dropped, "entry should not be dropped when drop_on_timeout is false") - require.Equal(t, line, processed.Entry.Line, "original unredacted line should be forwarded on timeout") - require.Equal(t, float64(1), testutil.ToFloat64(c.metrics.linesTimedOutTotal)) - require.Equal(t, float64(0), testutil.ToFloat64(c.metrics.linesDroppedTotal)) -} - -func TestProcessingTimeout_DropsOnTimeoutWhenEnabled(t *testing.T) { - registry := prometheus.NewRegistry() - opts := component.Options{ - Logger: util.TestLogger(t), - OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, - Registerer: registry, - } - secret := fakeSecrets["grafana-api-key"].value - line := "log line with secret " + secret + " end" - c, err := New(opts, Arguments{ - ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, - ProcessingTimeout: 1 * time.Nanosecond, // guaranteed to expire before DetectString returns - DropOnTimeout: true, - }) - require.NoError(t, err) - - entry := loki.Entry{ - Labels: model.LabelSet{}, - Entry: push.Entry{Timestamp: time.Now(), Line: line}, - } - _, dropped := c.processEntry(context.Background(), entry) - - require.True(t, dropped, "entry should be dropped when drop_on_timeout is true") - require.Equal(t, float64(1), testutil.ToFloat64(c.metrics.linesTimedOutTotal)) - require.Equal(t, float64(1), testutil.ToFloat64(c.metrics.linesDroppedTotal)) -} - -func TestProcessingTimeout_NoTimeoutWhenDisabled(t *testing.T) { - registry := prometheus.NewRegistry() - opts := component.Options{ - Logger: util.TestLogger(t), - OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, - Registerer: registry, - } - secret := fakeSecrets["grafana-api-key"].value - line := "log line with secret " + secret + " end" - c, err := New(opts, Arguments{ - ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, - // ProcessingTimeout left at zero: disabled - }) - require.NoError(t, err) - - entry := loki.Entry{ - Labels: model.LabelSet{}, - Entry: push.Entry{Timestamp: time.Now(), Line: line}, - } - processed, dropped := c.processEntry(context.Background(), entry) - - require.False(t, dropped) - require.NotEqual(t, line, processed.Entry.Line, "secret should be redacted when no timeout is set") - require.Equal(t, float64(0), testutil.ToFloat64(c.metrics.linesTimedOutTotal)) - require.Equal(t, float64(0), testutil.ToFloat64(c.metrics.linesDroppedTotal)) -} - func BenchmarkAllTypesNoSecret(b *testing.B) { // Run benchmarks with no secrets in the logs, with all regexes enabled - runBenchmarks(b, testConfigs["default"], 0, "") + runBenchmarks(b, testhelper.TestConfigs["default"], 0, "") } func BenchmarkAllTypesWithSecret(b *testing.B) { // Run benchmarks with secrets in the logs (20% of log entries), with all regexes enabled - runBenchmarks(b, testConfigs["default"], 20, "gcp_api_key") + runBenchmarks(b, testhelper.TestConfigs["default"], 20, "gcp_api_key") } func BenchmarkAllTypesWithLotsOfSecrets(b *testing.B) { // Run benchmarks with secrets in the logs (80% of log entries), with all regexes enabled - runBenchmarks(b, testConfigs["default"], 80, "gcp_api_key") + runBenchmarks(b, testhelper.TestConfigs["default"], 80, "gcp_api_key") } func runBenchmarks(b *testing.B, config string, percentageSecrets int, secretName string) { @@ -507,7 +261,7 @@ func runBenchmarks(b *testing.B, config string, percentageSecrets int, secretNam opts := component.Options{ Logger: &noopLogger{}, // Disable logging so that it keeps a clean benchmark output OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, } // Create component @@ -525,7 +279,7 @@ func runBenchmarks(b *testing.B, config string, percentageSecrets int, secretNam // Add fake secrets in some log entries if secretName != "" && i < nbLogs*percentageSecrets/100 { - middleStr = testLogs[secretName].log + middleStr = testhelper.TestLogs[secretName].Log } benchInputs[i] = beginningStr + middleStr + endingStr @@ -550,20 +304,20 @@ func FuzzProcessEntry(f *testing.F) { for _, line := range sampleFuzzLogLines { f.Add(line) } - for _, testLog := range testLogs { - f.Add(testLog.log) + for _, testLog := range testhelper.TestLogs { + f.Add(testLog.Log) } opts := component.Options{ Logger: util.TestLogger(f), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, } ch1 := loki.NewLogsReceiver() // Create component with default config var args Arguments - require.NoError(f, syntax.Unmarshal([]byte(testConfigs["default"]), &args)) + require.NoError(f, syntax.Unmarshal([]byte(testhelper.TestConfigs["default"]), &args)) args.ForwardTo = []loki.LogsReceiver{ch1} c, err := New(opts, args) require.NoError(f, err) @@ -574,15 +328,6 @@ func FuzzProcessEntry(f *testing.F) { }) } -func getServiceData(name string) (any, error) { - switch name { - case livedebugging.ServiceName: - return livedebugging.NewLiveDebugging(), nil - default: - return nil, fmt.Errorf("service not found %s", name) - } -} - type noopLogger struct{} func (d *noopLogger) Log(_ ...any) error { @@ -600,12 +345,12 @@ func TestMetrics(t *testing.T) { }{ { name: "No secrets", - inputLog: testLogs["no_secret"].log, + inputLog: testhelper.TestLogs["no_secret"].Log, expectedRedactedTotal: 0, }, { name: "Single Grafana API key secret", - inputLog: testLogs["grafana_api_key"].log, + inputLog: testhelper.TestLogs["grafana_api_key"].Log, expectedRedactedTotal: 1, expectedRedactedByRule: map[string]int{ "grafana-api-key": 1, @@ -613,7 +358,7 @@ func TestMetrics(t *testing.T) { }, { name: "Multiple secrets", - inputLog: testLogs["multiple_secrets"].log, + inputLog: testhelper.TestLogs["multiple_secrets"].Log, expectedRedactedTotal: 2, expectedRedactedByRule: map[string]int{ "grafana-api-key": 1, @@ -637,7 +382,7 @@ func TestMetrics(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } @@ -745,7 +490,7 @@ func TestMetricsRegistration(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, ID: "test_secretfilter", } @@ -803,7 +548,7 @@ func TestMetricsMultipleEntries(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } @@ -816,28 +561,28 @@ func TestMetricsMultipleEntries(t *testing.T) { Labels: model.LabelSet{"job": "test1"}, Entry: push.Entry{ Timestamp: time.Now(), - Line: testLogs["grafana_api_key"].log, + Line: testhelper.TestLogs["grafana_api_key"].Log, }, }, { Labels: model.LabelSet{"job": "test2"}, Entry: push.Entry{ Timestamp: time.Now(), - Line: testLogs["gcp_api_key"].log, + Line: testhelper.TestLogs["gcp_api_key"].Log, }, }, { Labels: model.LabelSet{"job": "test3"}, Entry: push.Entry{ Timestamp: time.Now(), - Line: testLogs["no_secret"].log, + Line: testhelper.TestLogs["no_secret"].Log, }, }, { Labels: model.LabelSet{"job": "test4"}, Entry: push.Entry{ Timestamp: time.Now(), - Line: testLogs["grafana_api_key"].log, + Line: testhelper.TestLogs["grafana_api_key"].Log, }, }, } @@ -892,7 +637,7 @@ func TestArgumentsUpdate(t *testing.T) { opts := component.Options{ Logger: util.TestLogger(t), OnStateChange: func(e component.Exports) {}, - GetServiceData: getServiceData, + GetServiceData: testhelper.GetServiceData, Registerer: registry, } @@ -909,7 +654,7 @@ func TestArgumentsUpdate(t *testing.T) { { description: "Initial config - should redact secrets", args: initialArgs, - inputLog: testLogs["grafana_api_key"].log, + inputLog: testhelper.TestLogs["grafana_api_key"].Log, }, { description: "Update 1 - Add origin label tracking", @@ -917,7 +662,7 @@ func TestArgumentsUpdate(t *testing.T) { ForwardTo: []loki.LogsReceiver{ch1}, OriginLabel: "job", }, - inputLog: testLogs["gcp_api_key"].log, + inputLog: testhelper.TestLogs["gcp_api_key"].Log, }, { description: "Update 2 - Change origin label", @@ -925,7 +670,7 @@ func TestArgumentsUpdate(t *testing.T) { ForwardTo: []loki.LogsReceiver{ch1}, OriginLabel: "instance", }, - inputLog: testLogs["stripe_key"].log, + inputLog: testhelper.TestLogs["stripe_key"].Log, }, } @@ -956,3 +701,87 @@ func TestArgumentsUpdate(t *testing.T) { }) } } + +func TestProcessingTimeout_ForwardsUnredactedOnTimeout(t *testing.T) { + registry := prometheus.NewRegistry() + opts := component.Options{ + Logger: util.TestLogger(t), + OnStateChange: func(e component.Exports) {}, + GetServiceData: testhelper.GetServiceData, + Registerer: registry, + } + secret := testhelper.FakeSecrets["grafana-api-key"].Value + line := "log line with secret " + secret + " end" + c, err := New(opts, Arguments{ + ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, + ProcessingTimeout: 1 * time.Nanosecond, // guaranteed to expire before DetectString returns + }) + require.NoError(t, err) + + entry := loki.Entry{ + Labels: model.LabelSet{}, + Entry: push.Entry{Timestamp: time.Now(), Line: line}, + } + processed, dropped := c.processEntry(context.Background(), entry) + + require.False(t, dropped, "entry should not be dropped when drop_on_timeout is false") + require.Equal(t, line, processed.Entry.Line, "original unredacted line should be forwarded on timeout") + require.Equal(t, float64(1), testutil.ToFloat64(c.metrics.linesTimedOutTotal)) + require.Equal(t, float64(0), testutil.ToFloat64(c.metrics.linesDroppedTotal)) +} + +func TestProcessingTimeout_DropsOnTimeoutWhenEnabled(t *testing.T) { + registry := prometheus.NewRegistry() + opts := component.Options{ + Logger: util.TestLogger(t), + OnStateChange: func(e component.Exports) {}, + GetServiceData: testhelper.GetServiceData, + Registerer: registry, + } + secret := testhelper.FakeSecrets["grafana-api-key"].Value + line := "log line with secret " + secret + " end" + c, err := New(opts, Arguments{ + ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, + ProcessingTimeout: 1 * time.Nanosecond, // guaranteed to expire before DetectString returns + DropOnTimeout: true, + }) + require.NoError(t, err) + + entry := loki.Entry{ + Labels: model.LabelSet{}, + Entry: push.Entry{Timestamp: time.Now(), Line: line}, + } + _, dropped := c.processEntry(context.Background(), entry) + + require.True(t, dropped, "entry should be dropped when drop_on_timeout is true") + require.Equal(t, float64(1), testutil.ToFloat64(c.metrics.linesTimedOutTotal)) + require.Equal(t, float64(1), testutil.ToFloat64(c.metrics.linesDroppedTotal)) +} + +func TestProcessingTimeout_NoTimeoutWhenDisabled(t *testing.T) { + registry := prometheus.NewRegistry() + opts := component.Options{ + Logger: util.TestLogger(t), + OnStateChange: func(e component.Exports) {}, + GetServiceData: testhelper.GetServiceData, + Registerer: registry, + } + secret := testhelper.FakeSecrets["grafana-api-key"].Value + line := "log line with secret " + secret + " end" + c, err := New(opts, Arguments{ + ForwardTo: []loki.LogsReceiver{loki.NewLogsReceiver()}, + // ProcessingTimeout left at zero: disabled + }) + require.NoError(t, err) + + entry := loki.Entry{ + Labels: model.LabelSet{}, + Entry: push.Entry{Timestamp: time.Now(), Line: line}, + } + processed, dropped := c.processEntry(context.Background(), entry) + + require.False(t, dropped) + require.NotEqual(t, line, processed.Entry.Line, "secret should be redacted when no timeout is set") + require.Equal(t, float64(0), testutil.ToFloat64(c.metrics.linesTimedOutTotal)) + require.Equal(t, float64(0), testutil.ToFloat64(c.metrics.linesDroppedTotal)) +} diff --git a/internal/component/loki/secretfilter/testhelper/testhelper.go b/internal/component/loki/secretfilter/testhelper/testhelper.go new file mode 100644 index 00000000000..96fec3d2b04 --- /dev/null +++ b/internal/component/loki/secretfilter/testhelper/testhelper.go @@ -0,0 +1,145 @@ +// Package testhelper provides shared test data and helpers for the secretfilter +// component tests. It is used by both the secretfilter package tests and the +// extend subpackage tests so that config-loading tests run in separate processes +// without duplicating test data or run logic. +package testhelper + +import ( + "fmt" + + "github.com/grafana/alloy/internal/service/livedebugging" +) + +// FakeSecret holds a fake secret value for tests. +type FakeSecret struct { + Name string + Value string +} + +// TestLog holds a log line and optional metadata for tests. +type TestLog struct { + Log string +} + +// FakeSecrets is the shared map of fake secrets used to build test log lines. +var FakeSecrets = map[string]FakeSecret{ + "grafana-api-key": { + Name: "grafana-api-key", + Value: "eyJr" + "IjoiT0x6NWJuNDRuZkI1NHJ6dEJrR0g3aDVuRnY0NWJuNDRuZkI1NHJ6dEJrR0g3aDVuRnY0NWJuNDRuZkI1NHJ6dEJrR0g3aDVuRnY0NWJuNDRuZkI1NHJ6dEJrR0g3aDVu", + }, + "gcp-api-key": { + Name: "gcp-api-key", + Value: "AIza" + "SyDaGmWKa4JsXZ-HjGw7ISLn_3namBGewQe", + }, + "stripe-key": { + Name: "stripe-access-token", + Value: "sk_live_" + "51HFxYz2eZvKYlo2C9kKM5nE6qO4yKn8N3bP7hXxYz2eZvKYlo2C", + }, + "npm-token": { + Name: "npm-access-token", + Value: "npm_" + "1A2b3C4d5E6f7G8h9I0jK1lM2nO3pQ4rS5tU", + }, + "generic-api-key": { + Name: "generic-api-key", + Value: "token:" + "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8Ii9Jj0Kk", + }, +} + +// TestLogs is the shared map of test log entries. Built in init so it can reference FakeSecrets. +var TestLogs map[string]TestLog + +func init() { + TestLogs = map[string]TestLog{ + "no_secret": { + Log: `{ + "message": "This is a simple log message" + }`, + }, + "grafana_api_key": { + Log: `{ + "message": "This is a simple log message with a secret value ` + FakeSecrets["grafana-api-key"].Value + ` ! + }`, + }, + "gcp_api_key": { + Log: `{ + "message": "This is a simple log message with a secret value ` + FakeSecrets["gcp-api-key"].Value + ` ! + }`, + }, + "stripe_key": { + Log: `{ + "message": "This is a simple log message with a secret value ` + FakeSecrets["stripe-key"].Value + ` ! + }`, + }, + "npm_token": { + Log: `{ + "message": "This is a simple log message with a secret value ` + FakeSecrets["npm-token"].Value + ` ! + }`, + }, + "generic_api_key": { + Log: `{ + "message": "This is a simple log message with a secret value ` + FakeSecrets["generic-api-key"].Value + ` ! + }`, + }, + "multiple_secrets": { + Log: `{ + "message": "This is a simple log message with a secret value ` + FakeSecrets["grafana-api-key"].Value + ` and another secret value ` + FakeSecrets["gcp-api-key"].Value + ` ! + }`, + }, + } +} + +// TestConfigs holds shared Alloy config snippets for tests. +var TestConfigs = map[string]string{ + "default": ` + forward_to = [] + `, + "with_origin": ` + forward_to = [] + origin_label = "job" + `, + "with_redact_percent_100": ` + forward_to = [] + redact_percent = 100 + `, + "with_redact_percent_80": ` + forward_to = [] + redact_percent = 80 + `, + "with_redact_with": ` + forward_to = [] + redact_with = "***REDACTED***" + `, +} + +// Case is a single test case: name, log line, and whether redaction is expected. +type Case struct { + Name string + InputLog string + ShouldRedact bool +} + +// DefaultCases are the 7 standard cases (no_secret through multiple_secrets) for default config. +// Set in init() so TestLogs is populated first. +var DefaultCases []Case + +func init() { + DefaultCases = []Case{ + {"no_secret", TestLogs["no_secret"].Log, false}, + {"grafana_api_key", TestLogs["grafana_api_key"].Log, true}, + {"gcp_api_key", TestLogs["gcp_api_key"].Log, true}, + {"stripe_key", TestLogs["stripe_key"].Log, true}, + {"npm_token", TestLogs["npm_token"].Log, true}, + {"generic_api_key", TestLogs["generic_api_key"].Log, true}, + {"multiple_secrets", TestLogs["multiple_secrets"].Log, true}, + } +} + +// GetServiceData returns the livedebugging service for component tests. +func GetServiceData(name string) (any, error) { + switch name { + case livedebugging.ServiceName: + return livedebugging.NewLiveDebugging(), nil + default: + return nil, fmt.Errorf("service not found %s", name) + } +}