diff --git a/.gitignore b/.gitignore index 5757ef323a3..73b94aab42f 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ node_modules .image-tag /alloy collector/otel_engine +collector/otel_engine.exe # local testing files /targets.yaml diff --git a/collector/go.mod b/collector/go.mod index 7802129bee4..00d651ef4c8 100644 --- a/collector/go.mod +++ b/collector/go.mod @@ -332,7 +332,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e // indirect github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 // indirect diff --git a/collector/go.sum b/collector/go.sum index ceb71a08949..3e9d2206c51 100644 --- a/collector/go.sum +++ b/collector/go.sum @@ -602,8 +602,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= -github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY= diff --git a/docs/sources/reference/components/local/local.file_match.md b/docs/sources/reference/components/local/local.file_match.md index 26f78fb034c..799f5b40394 100644 --- a/docs/sources/reference/components/local/local.file_match.md +++ b/docs/sources/reference/components/local/local.file_match.md @@ -57,6 +57,8 @@ The `__path__` field uses [doublestar][] style glob patterns: * `/tmp/**/*.log` matches all subdirectories of `tmp` and include any files that end in `*.log`. * `/tmp/apache/*.log` matches only files in `/tmp/apache/` that end in `*.log`. * `/tmp/**` matches all subdirectories of `tmp`, `tmp` itself, and all files. +* `/tmp/*.{log,txt,json}` matches files with `.log`, `.txt`, or `.json` extensions in `/tmp/`. +* `/var/log/{nginx,apache}/*.log` matches `.log` files in either the `nginx` or `apache` subdirectories. `local.file_match` doesn't ignore files when you set `ignore_older_than` to the default, `0s`. @@ -181,6 +183,35 @@ Replace the following: * _``_: The username to use for authentication to the Loki API. * _``_: The password to use for authentication to the Loki API. +### Match multiple patterns + +This example shows how to use the `{a,b,c}` pattern syntax to match multiple file extensions, multiple directories, and exclude multiple file types in a single configuration. + +```alloy +local.file_match "logs" { + path_targets = [ + // Match .log, .txt, and .json files from nginx, apache, or caddy directories + // Exclude compressed and backup files + { + "__path__" = "/var/log/{nginx,apache,caddy}/*.{log,txt,json}", + "__path_exclude__" = "/var/log/{nginx,apache,caddy}/*.{gz,zip,bak,old}", + "job" = "webserver", + }, + ] +} + +loki.source.file "logs" { + targets = local.file_match.logs.targets + forward_to = [loki.write.endpoint.receiver] +} + +loki.write "endpoint" { + endpoint { + url = "" + } +} +``` + ### Send Kubernetes Pod logs to Loki This example finds all the logs on Pods and monitors them. diff --git a/docs/sources/reference/components/loki/loki.source.file.md b/docs/sources/reference/components/loki/loki.source.file.md index 3cc47f5291b..d11b99819fb 100644 --- a/docs/sources/reference/components/loki/loki.source.file.md +++ b/docs/sources/reference/components/loki/loki.source.file.md @@ -142,6 +142,11 @@ Benefits of using `file_match` over `local.file_match`: When `enabled` is set to `true`, you can use glob patterns, for example, `/tmp/*.log` or `/var/log/**/*.log`, directly in the `targets` argument's `__path__` label. The component periodically scans the filesystem based on `sync_period` and automatically discovers new files, removes deleted files, and ignores files older than `ignore_older_than` if specified. +The glob patterns support the `{a,b,c}` syntax for matching multiple alternatives: + +* `/tmp/*.{log,txt,json}` matches files with `.log`, `.txt`, or `.json` extensions. +* `/var/log/{nginx,apache}/*.log` matches `.log` files in either the `nginx` or `apache` subdirectories. + ## Exported fields `loki.source.file` doesn't export any fields. @@ -262,6 +267,34 @@ loki.write "local" { } ``` +### Match multiple patterns + +This example shows how to use the `{a,b,c}` pattern syntax to match multiple file extensions, multiple directories, and exclude multiple file types in a single configuration. + +```alloy +loki.source.file "logs" { + targets = [ + // Match .log, .txt, and .json files from nginx, apache, or caddy directories + // Exclude compressed and backup files + { + __path__ = "/var/log/{nginx,apache,caddy}/*.{log,txt,json}", + __path_exclude__ = "/var/log/{nginx,apache,caddy}/*.{gz,zip,bak,old}", + "job" = "webserver", + }, + ] + forward_to = [loki.write.local.receiver] + file_match { + enabled = true + } +} + +loki.write "local" { + endpoint { + url = "loki:3100/api/v1/push" + } +} +``` + ### Decompression This example collects log entries from compressed files matching the `*.gz` pattern using the built-in `file_match` block with decompression enabled. diff --git a/extension/alloyengine/go.mod b/extension/alloyengine/go.mod index b52460629a9..fe3fd1d0fa0 100644 --- a/extension/alloyengine/go.mod +++ b/extension/alloyengine/go.mod @@ -251,7 +251,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e // indirect github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 // indirect diff --git a/extension/alloyengine/go.sum b/extension/alloyengine/go.sum index d6b61c8e5f5..6737f5dd892 100644 --- a/extension/alloyengine/go.sum +++ b/extension/alloyengine/go.sum @@ -612,8 +612,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= -github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY= diff --git a/go.mod b/go.mod index c85081acdb7..0541a9d6890 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.39.17 github.com/blang/semver/v4 v4.0.0 - github.com/bmatcuk/doublestar v1.3.4 + github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e github.com/buger/jsonparser v1.1.1 github.com/burningalchemist/sql_exporter v0.0.0-20240103092044-466b38b6abc4 @@ -524,7 +524,6 @@ require ( github.com/beevik/ntp v1.3.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/caarlos0/env/v9 v9.0.0 github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect diff --git a/go.sum b/go.sum index 4ecb83da2fb..bcd17512588 100644 --- a/go.sum +++ b/go.sum @@ -616,8 +616,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= -github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY= diff --git a/internal/component/local/file_match/file.go b/internal/component/local/file_match/file.go index e115bf5015d..61063f1a09d 100644 --- a/internal/component/local/file_match/file.go +++ b/internal/component/local/file_match/file.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util/glob" ) func init() { @@ -42,10 +43,15 @@ type Component struct { watches []watch watchDog *time.Ticker targetsChanged chan struct{} + globber glob.Globber } // New creates a new local.file_match component. func New(o component.Options, args Arguments) (*Component, error) { + return newComponent(o, args, glob.NewGlobber()) +} + +func newComponent(o component.Options, args Arguments, globber glob.Globber) (*Component, error) { c := &Component{ opts: o, mut: sync.Mutex{}, @@ -54,6 +60,7 @@ func New(o component.Options, args Arguments) (*Component, error) { watchDog: time.NewTicker(args.SyncPeriod), // Buffered channel to avoid blocking targetsChanged: make(chan struct{}, 1), + globber: globber, } if err := c.Update(args); err != nil { @@ -92,6 +99,7 @@ func (c *Component) Update(args component.Arguments) error { target: v, log: c.opts.Logger, ignoreOlderThan: c.args.IgnoreOlderThan, + globber: c.globber, }) } diff --git a/internal/component/local/file_match/file_test.go b/internal/component/local/file_match/file_test.go index 248f082e2bb..afa6cd86ff8 100644 --- a/internal/component/local/file_match/file_test.go +++ b/internal/component/local/file_match/file_test.go @@ -1,34 +1,27 @@ -//go:build !windows - -// This should run on windows but windows does not like the tight timing of file creation and deletion. package file_match import ( "context" "os" "path" - "strings" "testing" "time" - "github.com/grafana/alloy/internal/component/discovery" - - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" - "github.com/grafana/alloy/internal/component" - "github.com/grafana/alloy/internal/util" + "github.com/grafana/alloy/internal/component/discovery" + "github.com/grafana/alloy/internal/component/local/file_match/testutil" ) func TestFile(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t1") err := os.MkdirAll(dir, 0755) require.NoError(t, err) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) + c := testCreateComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) ct := t.Context() ct, ccl := context.WithTimeout(ct, 5*time.Second) defer ccl() @@ -38,7 +31,7 @@ func TestFile(t *testing.T) { ct.Done() foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 1) - require.True(t, contains(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) } func TestDirectoryFile(t *testing.T) { @@ -46,11 +39,11 @@ func TestDirectoryFile(t *testing.T) { subdir := path.Join(dir, "subdir") err := os.MkdirAll(subdir, 0755) require.NoError(t, err) - writeFile(t, subdir, "t1.txt") + testutil.WriteFile(t, subdir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponent(t, dir, []string{path.Join(dir, "**/")}, nil) + c := testCreateComponent(t, dir, []string{path.Join(dir, "**/")}, nil) ct := t.Context() ct, ccl := context.WithTimeout(ct, 5*time.Second) defer ccl() @@ -60,18 +53,18 @@ func TestDirectoryFile(t *testing.T) { ct.Done() foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 1) - require.True(t, contains(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) } func TestFileIgnoreOlder(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t1") err := os.MkdirAll(dir, 0755) require.NoError(t, err) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) + c := testCreateComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) ct := t.Context() ct, ccl := context.WithTimeout(ct, 5*time.Second) defer ccl() @@ -82,25 +75,25 @@ func TestFileIgnoreOlder(t *testing.T) { foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 1) - require.True(t, contains(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) time.Sleep(150 * time.Millisecond) - writeFile(t, dir, "t2.txt") + testutil.WriteFile(t, dir, "t2.txt") ct.Done() foundFiles = c.getWatchedFiles() require.Len(t, foundFiles, 1) - require.True(t, contains(foundFiles, "t2.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t2.txt")) } func TestAddingFile(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t2") err := os.MkdirAll(dir, 0755) require.NoError(t, err) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) + c := testCreateComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) ct := t.Context() ct, ccl := context.WithTimeout(ct, 40*time.Second) @@ -108,29 +101,29 @@ func TestAddingFile(t *testing.T) { c.args.SyncPeriod = 10 * time.Millisecond go c.Run(ct) time.Sleep(20 * time.Millisecond) - writeFile(t, dir, "t2.txt") + testutil.WriteFile(t, dir, "t2.txt") ct.Done() foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 2) - require.True(t, contains(foundFiles, "t1.txt")) - require.True(t, contains(foundFiles, "t2.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t2.txt")) } func TestAddingFileInSubDir(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t3") os.MkdirAll(dir, 0755) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponent(t, dir, []string{path.Join(dir, "**", "*.txt")}, nil) + c := testCreateComponent(t, dir, []string{path.Join(dir, "**", "*.txt")}, nil) ct := t.Context() ct, ccl := context.WithTimeout(ct, 40*time.Second) defer ccl() c.args.SyncPeriod = 10 * time.Millisecond go c.Run(ct) time.Sleep(20 * time.Millisecond) - writeFile(t, dir, "t2.txt") + testutil.WriteFile(t, dir, "t2.txt") subdir := path.Join(dir, "subdir") os.Mkdir(subdir, 0755) time.Sleep(20 * time.Millisecond) @@ -140,28 +133,28 @@ func TestAddingFileInSubDir(t *testing.T) { ct.Done() foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 3) - require.True(t, contains(foundFiles, "t1.txt")) - require.True(t, contains(foundFiles, "t2.txt")) - require.True(t, contains(foundFiles, "t3.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t2.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t3.txt")) } func TestAddingFileInAnExcludedSubDir(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t3") os.MkdirAll(dir, 0755) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) included := []string{path.Join(dir, "**", "*.txt")} excluded := []string{path.Join(dir, "subdir", "*.txt")} - c := createComponent(t, dir, included, excluded) + c := testCreateComponent(t, dir, included, excluded) ct := t.Context() ct, ccl := context.WithTimeout(ct, 40*time.Second) defer ccl() c.args.SyncPeriod = 10 * time.Millisecond go c.Run(ct) time.Sleep(20 * time.Millisecond) - writeFile(t, dir, "t2.txt") + testutil.WriteFile(t, dir, "t2.txt") subdir := path.Join(dir, "subdir") os.Mkdir(subdir, 0755) subdir2 := path.Join(dir, "subdir2") @@ -177,19 +170,19 @@ func TestAddingFileInAnExcludedSubDir(t *testing.T) { ct.Done() foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 3) - require.True(t, contains(foundFiles, "t1.txt")) - require.True(t, contains(foundFiles, "t2.txt")) - require.True(t, contains(foundFiles, "another.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t2.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "another.txt")) } func TestAddingRemovingFileInSubDir(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t3") os.MkdirAll(dir, 0755) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponent(t, dir, []string{path.Join(dir, "**", "*.txt")}, nil) + c := testCreateComponent(t, dir, []string{path.Join(dir, "**", "*.txt")}, nil) ct := t.Context() ct, ccl := context.WithTimeout(ct, 40*time.Second) @@ -197,7 +190,7 @@ func TestAddingRemovingFileInSubDir(t *testing.T) { c.args.SyncPeriod = 10 * time.Millisecond go c.Run(ct) time.Sleep(20 * time.Millisecond) - writeFile(t, dir, "t2.txt") + testutil.WriteFile(t, dir, "t2.txt") subdir := path.Join(dir, "subdir") os.Mkdir(subdir, 0755) time.Sleep(100 * time.Millisecond) @@ -206,27 +199,27 @@ func TestAddingRemovingFileInSubDir(t *testing.T) { time.Sleep(100 * time.Millisecond) foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 3) - require.True(t, contains(foundFiles, "t1.txt")) - require.True(t, contains(foundFiles, "t2.txt")) - require.True(t, contains(foundFiles, "t3.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t2.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t3.txt")) err = os.RemoveAll(subdir) require.NoError(t, err) time.Sleep(1000 * time.Millisecond) foundFiles = c.getWatchedFiles() require.Len(t, foundFiles, 2) - require.True(t, contains(foundFiles, "t1.txt")) - require.True(t, contains(foundFiles, "t2.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t2.txt")) } func TestExclude(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t3") os.MkdirAll(dir, 0755) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponent(t, dir, []string{path.Join(dir, "**", "*.txt")}, []string{path.Join(dir, "**", "*.bad")}) + c := testCreateComponent(t, dir, []string{path.Join(dir, "**", "*.txt")}, []string{path.Join(dir, "**", "*.bad")}) ct := t.Context() ct, ccl := context.WithTimeout(ct, 40*time.Second) defer ccl() @@ -235,22 +228,100 @@ func TestExclude(t *testing.T) { time.Sleep(100 * time.Millisecond) subdir := path.Join(dir, "subdir") os.Mkdir(subdir, 0755) - writeFile(t, subdir, "t3.txt") + testutil.WriteFile(t, subdir, "t3.txt") time.Sleep(100 * time.Millisecond) foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 2) - require.True(t, contains(foundFiles, "t1.txt")) - require.True(t, contains(foundFiles, "t3.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t1.txt")) + require.True(t, testutil.ContainsPath(foundFiles, "t3.txt")) +} + +// TestMultiplePatterns verifies that the {a,b,c} pattern syntax works for matching +// multiple file extensions, multiple directories, and excluding multiple file types +// in a single glob pattern. This mirrors the documentation example. +// This is a feature of doublestar v4: https://github.com/grafana/alloy/issues/4423 +func TestMultiplePatterns(t *testing.T) { + dir := path.Join(os.TempDir(), "alloy_testing", "multi_pattern") + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) + t.Cleanup(func() { + os.RemoveAll(dir) + }) + + // Create directory structure matching the documentation example: + // {nginx,apache,caddy}/*.{log,txt,json} excluding *.{gz,zip,bak,old} + for _, subdir := range []string{"nginx", "apache", "caddy", "other"} { + subdirPath := path.Join(dir, subdir) + err := os.MkdirAll(subdirPath, 0755) + require.NoError(t, err) + + // Create files with various extensions in each directory + testutil.WriteFile(t, subdirPath, "access.log") + testutil.WriteFile(t, subdirPath, "error.txt") + testutil.WriteFile(t, subdirPath, "config.json") + testutil.WriteFile(t, subdirPath, "debug.yaml") // Should not match (wrong extension) + testutil.WriteFile(t, subdirPath, "access.log.gz") // Should be excluded + testutil.WriteFile(t, subdirPath, "error.txt.zip") // Should be excluded + testutil.WriteFile(t, subdirPath, "config.json.bak") // Should be excluded + testutil.WriteFile(t, subdirPath, "old.log.old") // Should be excluded + } + + // Use pattern matching multiple directories and extensions with exclusions + // This mirrors: /var/log/{nginx,apache,caddy}/*.{log,txt,json} excluding *.{gz,zip,bak,old} + includePath := path.Join(dir, "{nginx,apache,caddy}", "*.{log,txt,json}") + excludePath := path.Join(dir, "{nginx,apache,caddy}", "*.{gz,zip,bak,old}") + + c := testCreateComponent(t, dir, []string{includePath}, []string{excludePath}) + c.args.SyncPeriod = 10 * time.Millisecond + err = c.Update(c.args) + require.NoError(t, err) + + foundFiles := c.getWatchedFiles() + + // Expected files: 3 directories × 3 extensions + expectedFiles := []string{ + path.Join("nginx", "access.log"), + path.Join("nginx", "error.txt"), + path.Join("nginx", "config.json"), + path.Join("apache", "access.log"), + path.Join("apache", "error.txt"), + path.Join("apache", "config.json"), + path.Join("caddy", "access.log"), + path.Join("caddy", "error.txt"), + path.Join("caddy", "config.json"), + } + + require.Len(t, foundFiles, len(expectedFiles), + "Expected %d files: %v", len(expectedFiles), expectedFiles) + + // Verify all expected files are matched + for _, expected := range expectedFiles { + require.True(t, testutil.ContainsPath(foundFiles, expected), + "%s should be matched", expected) + } + + // Verify files from "other" directory are NOT matched (wrong directory) + require.False(t, testutil.ContainsPath(foundFiles, path.Join("other", "access.log")), + "other/access.log should not be matched (wrong directory)") + + // Verify excluded extensions are NOT matched + require.False(t, testutil.ContainsPath(foundFiles, "access.log.gz"), ".gz files should be excluded") + require.False(t, testutil.ContainsPath(foundFiles, "error.txt.zip"), ".zip files should be excluded") + require.False(t, testutil.ContainsPath(foundFiles, "config.json.bak"), ".bak files should be excluded") + require.False(t, testutil.ContainsPath(foundFiles, "old.log.old"), ".old files should be excluded") + + // Verify wrong extensions are NOT matched + require.False(t, testutil.ContainsPath(foundFiles, "debug.yaml"), ".yaml files should not be matched") } func TestMultiLabels(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t3") os.MkdirAll(dir, 0755) - writeFile(t, dir, "t1.txt") + testutil.WriteFile(t, dir, "t1.txt") t.Cleanup(func() { os.RemoveAll(dir) }) - c := createComponentWithLabels(t, dir, []string{path.Join(dir, "**", "*.txt"), path.Join(dir, "**", "*.txt")}, nil, map[string]string{ + c := testCreateComponentWithLabels(t, dir, []string{path.Join(dir, "**", "*.txt"), path.Join(dir, "**", "*.txt")}, nil, map[string]string{ "foo": "bar", "fruit": "apple", }) @@ -265,62 +336,6 @@ func TestMultiLabels(t *testing.T) { time.Sleep(100 * time.Millisecond) foundFiles := c.getWatchedFiles() require.Len(t, foundFiles, 2) - require.True(t, contains([]discovery.Target{foundFiles[0]}, "t1.txt")) - require.True(t, contains([]discovery.Target{foundFiles[1]}, "t1.txt")) -} - -// createComponent creates a component with the given paths and labels. The paths and excluded slices are zipped together -// to create the set of targets to pass to the component. -func createComponent(t *testing.T, dir string, paths []string, excluded []string) *Component { - return createComponentWithLabels(t, dir, paths, excluded, nil) -} - -// createComponentWithLabels creates a component with the given paths and labels. The paths and excluded slices are -// zipped together to create the set of targets to pass to the component. -func createComponentWithLabels(t *testing.T, dir string, paths []string, excluded []string, labels map[string]string) *Component { - tPaths := make([]discovery.Target, 0) - for i, p := range paths { - tb := discovery.NewTargetBuilder() - tb.Set("__path__", p) - for k, v := range labels { - tb.Set(k, v) - } - if i < len(excluded) { - tb.Set("__path_exclude__", excluded[i]) - } - tPaths = append(tPaths, tb.Target()) - } - c, err := New(component.Options{ - ID: "test", - Logger: util.TestAlloyLogger(t), - DataPath: dir, - OnStateChange: func(e component.Exports) { - - }, - Registerer: prometheus.DefaultRegisterer, - Tracer: nil, - }, Arguments{ - PathTargets: tPaths, - SyncPeriod: 1 * time.Second, - }) - - require.NoError(t, err) - require.NotNil(t, c) - return c -} - -func contains(sources []discovery.Target, match string) bool { - for _, s := range sources { - p, _ := s.Get("__path__") - if strings.Contains(p, match) { - return true - } - } - return false -} - -func writeFile(t *testing.T, dir string, name string) { - err := os.WriteFile(path.Join(dir, name), []byte("asdf"), 0664) - require.NoError(t, err) - time.Sleep(20 * time.Millisecond) + require.True(t, testutil.ContainsPath([]discovery.Target{foundFiles[0]}, "t1.txt")) + require.True(t, testutil.ContainsPath([]discovery.Target{foundFiles[1]}, "t1.txt")) } diff --git a/internal/component/local/file_match/file_unix_test.go b/internal/component/local/file_match/file_unix_test.go new file mode 100644 index 00000000000..24b079b2151 --- /dev/null +++ b/internal/component/local/file_match/file_unix_test.go @@ -0,0 +1,60 @@ +//go:build !windows + +package file_match + +import ( + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/component/local/file_match/testutil" +) + +// TestCaseSensitiveGlobMatching verifies that glob patterns are case-sensitive on Unix. +// A pattern with lowercase extension should NOT match files with uppercase extension. +func TestCaseSensitiveGlobMatching(t *testing.T) { + dir := path.Join(os.TempDir(), "alloy_testing", "case_sensitive_glob") + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) + t.Cleanup(func() { + os.RemoveAll(dir) + }) + + // Create a file with lowercase extension + testutil.WriteFile(t, dir, "test.log") + + // Search with uppercase glob pattern - should NOT match on Unix (case-sensitive) + c := testCreateComponent(t, dir, []string{path.Join(dir, "*.LOG")}, nil) + c.args.SyncPeriod = 10 * time.Millisecond + err = c.Update(c.args) + require.NoError(t, err) + + foundFiles := c.getWatchedFiles() + require.Len(t, foundFiles, 0, "Unix should be case-sensitive: *.LOG should not match test.log") +} + +// TestCaseSensitiveGlobMatchingWithCorrectCase verifies the pattern matches when case is correct. +func TestCaseSensitiveGlobMatchingWithCorrectCase(t *testing.T) { + dir := path.Join(os.TempDir(), "alloy_testing", "case_sensitive_glob_correct") + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) + t.Cleanup(func() { + os.RemoveAll(dir) + }) + + // Create a file with lowercase extension + testutil.WriteFile(t, dir, "test.log") + + // Search with matching case - should find the file + c := testCreateComponent(t, dir, []string{path.Join(dir, "*.log")}, nil) + c.args.SyncPeriod = 10 * time.Millisecond + err = c.Update(c.args) + require.NoError(t, err) + + foundFiles := c.getWatchedFiles() + require.Len(t, foundFiles, 1, "Pattern with matching case should find the file") + require.True(t, testutil.PathEndsWith(foundFiles, "test.log")) +} diff --git a/internal/component/local/file_match/file_windows_test.go b/internal/component/local/file_match/file_windows_test.go new file mode 100644 index 00000000000..0ddaa25d4c0 --- /dev/null +++ b/internal/component/local/file_match/file_windows_test.go @@ -0,0 +1,61 @@ +//go:build windows + +package file_match + +import ( + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/component/local/file_match/testutil" +) + +// TestCaseInsensitiveGlobMatching verifies that glob patterns are case-insensitive on Windows. +// A pattern with lowercase extension SHOULD match files with uppercase extension. +func TestCaseInsensitiveGlobMatching(t *testing.T) { + dir := path.Join(os.TempDir(), "alloy_testing", "case_insensitive_glob") + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) + t.Cleanup(func() { + os.RemoveAll(dir) + }) + + // Create a file with uppercase extension + testutil.WriteFile(t, dir, "test.LOG") + + // Search with lowercase glob pattern - SHOULD match on Windows (case-insensitive) + c := testCreateComponent(t, dir, []string{path.Join(dir, "*.log")}, nil) + c.args.SyncPeriod = 10 * time.Millisecond + err = c.Update(c.args) + require.NoError(t, err) + + foundFiles := c.getWatchedFiles() + require.Len(t, foundFiles, 1, "Windows should be case-insensitive: *.log should match test.LOG") + require.True(t, testutil.PathEndsWith(foundFiles, "test.log")) +} + +// TestCaseInsensitiveGlobMatchingUppercasePattern verifies uppercase patterns match lowercase files. +func TestCaseInsensitiveGlobMatchingUppercasePattern(t *testing.T) { + dir := path.Join(os.TempDir(), "alloy_testing", "case_insensitive_glob_upper") + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) + t.Cleanup(func() { + os.RemoveAll(dir) + }) + + // Create a file with lowercase extension + testutil.WriteFile(t, dir, "test.log") + + // Search with uppercase glob pattern - SHOULD match on Windows (case-insensitive) + c := testCreateComponent(t, dir, []string{path.Join(dir, "*.LOG")}, nil) + c.args.SyncPeriod = 10 * time.Millisecond + err = c.Update(c.args) + require.NoError(t, err) + + foundFiles := c.getWatchedFiles() + require.Len(t, foundFiles, 1, "Windows should be case-insensitive: *.LOG should match test.log") + require.True(t, testutil.PathEndsWith(foundFiles, "test.log")) +} diff --git a/internal/component/local/file_match/helpers_test.go b/internal/component/local/file_match/helpers_test.go new file mode 100644 index 00000000000..aa82e576037 --- /dev/null +++ b/internal/component/local/file_match/helpers_test.go @@ -0,0 +1,39 @@ +package file_match + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/component/local/file_match/testutil" + "github.com/grafana/alloy/internal/util" +) + +// testCreateComponent creates a component with the given paths and excluded patterns. +func testCreateComponent(t *testing.T, dir string, paths []string, excluded []string) *Component { + return testCreateComponentWithLabels(t, dir, paths, excluded, nil) +} + +// testCreateComponentWithLabels creates a component with the given paths, excluded patterns, and labels. +func testCreateComponentWithLabels(t *testing.T, dir string, paths []string, excluded []string, labels map[string]string) *Component { + t.Helper() + tPaths := testutil.MakeTargets(paths, excluded, labels) + c, err := New(component.Options{ + ID: "test", + Logger: util.TestAlloyLogger(t), + DataPath: dir, + OnStateChange: func(e component.Exports) { + }, + Registerer: prometheus.DefaultRegisterer, + Tracer: nil, + }, Arguments{ + PathTargets: tPaths, + SyncPeriod: 1 * time.Second, + }) + require.NoError(t, err) + require.NotNil(t, c) + return c +} diff --git a/internal/component/local/file_match/testutil/path_unix.go b/internal/component/local/file_match/testutil/path_unix.go new file mode 100644 index 00000000000..7c8a4ed5cc0 --- /dev/null +++ b/internal/component/local/file_match/testutil/path_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package testutil + +import ( + "strings" + + "github.com/grafana/alloy/internal/component/discovery" +) + +// PathEndsWith checks if any target's __path__ ends with the given suffix. +// On Unix, this is case-sensitive. +func PathEndsWith(sources []discovery.Target, suffix string) bool { + for _, s := range sources { + p, _ := s.Get("__path__") + if strings.HasSuffix(p, suffix) { + return true + } + } + return false +} diff --git a/internal/component/local/file_match/testutil/path_windows.go b/internal/component/local/file_match/testutil/path_windows.go new file mode 100644 index 00000000000..e7be0e696d6 --- /dev/null +++ b/internal/component/local/file_match/testutil/path_windows.go @@ -0,0 +1,22 @@ +//go:build windows + +package testutil + +import ( + "strings" + + "github.com/grafana/alloy/internal/component/discovery" +) + +// PathEndsWith checks if any target's __path__ ends with the given suffix. +// On Windows, this is case-insensitive. +func PathEndsWith(sources []discovery.Target, suffix string) bool { + suffix = strings.ToLower(suffix) + for _, s := range sources { + p, _ := s.Get("__path__") + if strings.HasSuffix(strings.ToLower(p), suffix) { + return true + } + } + return false +} diff --git a/internal/component/local/file_match/testutil/testutil.go b/internal/component/local/file_match/testutil/testutil.go new file mode 100644 index 00000000000..92586101a1e --- /dev/null +++ b/internal/component/local/file_match/testutil/testutil.go @@ -0,0 +1,50 @@ +// Package testutil provides test utilities for the file_match component. +package testutil + +import ( + "os" + "path" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/component/discovery" +) + +// WriteFile creates a test file with the given name in the specified directory. +func WriteFile(t *testing.T, dir string, name string) { + t.Helper() + err := os.WriteFile(path.Join(dir, name), []byte("test content"), 0664) + require.NoError(t, err) + time.Sleep(20 * time.Millisecond) +} + +// ContainsPath checks if any target's __path__ contains the given substring. +func ContainsPath(sources []discovery.Target, match string) bool { + for _, s := range sources { + p, _ := s.Get("__path__") + if strings.Contains(p, match) { + return true + } + } + return false +} + +// MakeTargets creates discovery targets from paths and excluded patterns. +func MakeTargets(paths []string, excluded []string, labels map[string]string) []discovery.Target { + tPaths := make([]discovery.Target, 0) + for i, p := range paths { + tb := discovery.NewTargetBuilder() + tb.Set("__path__", p) + for k, v := range labels { + tb.Set(k, v) + } + if i < len(excluded) { + tb.Set("__path_exclude__", excluded[i]) + } + tPaths = append(tPaths, tb.Target()) + } + return tPaths +} diff --git a/internal/component/local/file_match/watch.go b/internal/component/local/file_match/watch.go index 39727b83c2b..8d96521ff18 100644 --- a/internal/component/local/file_match/watch.go +++ b/internal/component/local/file_match/watch.go @@ -5,12 +5,12 @@ import ( "path/filepath" "time" - "github.com/bmatcuk/doublestar" "github.com/go-kit/log" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/util" + "github.com/grafana/alloy/internal/util/glob" ) // watch handles a single discovery.target for file watching. @@ -18,19 +18,20 @@ type watch struct { target discovery.Target log log.Logger ignoreOlderThan time.Duration + globber glob.Globber } func (w *watch) getPaths() ([]discovery.Target, error) { allMatchingPaths := make([]discovery.Target, 0) - matches, err := doublestar.Glob(w.getPath()) + matches, err := w.globber.FilepathGlob(w.getPath()) if err != nil { return nil, err } exclude := w.getExcludePath() for _, m := range matches { if exclude != "" { - if match, _ := doublestar.PathMatch(exclude, m); match { + if match, _ := w.globber.PathMatch(filepath.FromSlash(exclude), m); match { continue } } diff --git a/internal/component/loki/source/file/resolver.go b/internal/component/loki/source/file/resolver.go index e66a8d83c89..b02f59ce37b 100644 --- a/internal/component/loki/source/file/resolver.go +++ b/internal/component/loki/source/file/resolver.go @@ -4,12 +4,12 @@ import ( "iter" "path/filepath" - "github.com/bmatcuk/doublestar" "github.com/go-kit/log" "github.com/prometheus/common/model" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util/glob" ) type resolvedTarget struct { @@ -55,7 +55,11 @@ func (s *staticResolver) Resolve(targets []discovery.Target) iter.Seq[resolvedTa var _ resolver = (*globResolver)(nil) func newGlobResolver(logger log.Logger) *globResolver { - return &globResolver{logger} + return newGlobResolverWithGlobber(logger, glob.NewGlobber()) +} + +func newGlobResolverWithGlobber(logger log.Logger, globber glob.Globber) *globResolver { + return &globResolver{logger: logger, globber: globber} } // globResolver expands discovery targets using doublestar globbing. It reads @@ -63,7 +67,8 @@ func newGlobResolver(logger log.Logger) *globResolver { // If __path_exclude__ is present, matches that satisfy the exclude pattern are // filtered out. Returned paths are normalized to absolute form. type globResolver struct { - logger log.Logger + logger log.Logger + globber glob.Globber } func (s *globResolver) Resolve(targets []discovery.Target) iter.Seq[resolvedTarget] { @@ -72,7 +77,7 @@ func (s *globResolver) Resolve(targets []discovery.Target) iter.Seq[resolvedTarg targetPath, _ := target.Get(labelPath) labels := target.NonReservedLabelSet() - matches, err := doublestar.Glob(targetPath) + matches, err := s.globber.FilepathGlob(targetPath) if err != nil { level.Error(s.logger).Log("msg", "failed to resolve target", "error", err) continue @@ -82,7 +87,7 @@ func (s *globResolver) Resolve(targets []discovery.Target) iter.Seq[resolvedTarg for _, m := range matches { if exclude != "" { - if match, _ := doublestar.PathMatch(exclude, m); match { + if match, _ := s.globber.PathMatch(filepath.FromSlash(exclude), m); match { continue } } diff --git a/internal/component/loki/source/file/resolver_test.go b/internal/component/loki/source/file/resolver_test.go index da57812fbd2..cf4d781ee22 100644 --- a/internal/component/loki/source/file/resolver_test.go +++ b/internal/component/loki/source/file/resolver_test.go @@ -3,6 +3,7 @@ package file import ( "os" "path/filepath" + "sort" "testing" "github.com/go-kit/log" @@ -80,3 +81,103 @@ func TestResolver(t *testing.T) { }) } } + +// TestGlobResolverMultiplePatterns verifies that the {a,b,c} pattern syntax works +// for matching multiple file extensions, multiple directories, and excluding multiple +// file types in a single glob pattern. This mirrors the documentation example. +func TestGlobResolverMultiplePatterns(t *testing.T) { + // Create temp directory with subdirectories matching the documentation example + dir := t.TempDir() + + // Create directory structure: {nginx,apache,caddy}/*.{log,txt,json} excluding *.{gz,zip,bak,old} + for _, subdir := range []string{"nginx", "apache", "caddy", "other"} { + subdirPath := filepath.Join(dir, subdir) + err := os.MkdirAll(subdirPath, 0755) + require.NoError(t, err) + + // Create files with various extensions in each directory + testFiles := []string{ + "access.log", + "error.txt", + "config.json", + "debug.yaml", // Should not match (wrong extension) + "access.log.gz", // Should be excluded + "error.txt.zip", // Should be excluded + "config.json.bak", // Should be excluded + "old.log.old", // Should be excluded + } + for _, f := range testFiles { + err := os.WriteFile(filepath.Join(subdirPath, f), []byte("test"), 0644) + require.NoError(t, err) + } + } + + resolver := newGlobResolver(log.NewNopLogger()) + + // Use pattern matching multiple directories and extensions with exclusions + // This mirrors: /var/log/{nginx,apache,caddy}/*.{log,txt,json} excluding *.{gz,zip,bak,old} + targets := []discovery.Target{ + discovery.NewTargetFromLabelSet(model.LabelSet{ + "__path__": model.LabelValue(filepath.Join(dir, "{nginx,apache,caddy}", "*.{log,txt,json}")), + "__path_exclude__": model.LabelValue(filepath.Join(dir, "{nginx,apache,caddy}", "*.{gz,zip,bak,old}")), + "label": "test", + }), + } + + var results []resolvedTarget + for target := range resolver.Resolve(targets) { + results = append(results, target) + } + + // Expected files: 3 directories × 3 extensions + expectedFiles := []string{ + filepath.Join("nginx", "access.log"), + filepath.Join("nginx", "error.txt"), + filepath.Join("nginx", "config.json"), + filepath.Join("apache", "access.log"), + filepath.Join("apache", "error.txt"), + filepath.Join("apache", "config.json"), + filepath.Join("caddy", "access.log"), + filepath.Join("caddy", "error.txt"), + filepath.Join("caddy", "config.json"), + } + + require.Len(t, results, len(expectedFiles), + "Expected %d files: %v", len(expectedFiles), expectedFiles) + + // Collect all matched paths for verification + var paths []string + for _, r := range results { + // Get relative path from dir for easier checking + relPath, _ := filepath.Rel(dir, r.Path) + paths = append(paths, relPath) + } + sort.Strings(paths) + + // Verify all expected files are matched + for _, expected := range expectedFiles { + require.Contains(t, paths, expected, "%s should be matched", expected) + } + + // Verify files from "other" directory are NOT matched (wrong directory) + require.NotContains(t, paths, filepath.Join("other", "access.log"), + "other/access.log should not be matched (wrong directory)") + + // Verify excluded extensions are NOT matched + for _, subdir := range []string{"nginx", "apache", "caddy"} { + require.NotContains(t, paths, filepath.Join(subdir, "access.log.gz"), + ".gz files should be excluded") + require.NotContains(t, paths, filepath.Join(subdir, "error.txt.zip"), + ".zip files should be excluded") + require.NotContains(t, paths, filepath.Join(subdir, "config.json.bak"), + ".bak files should be excluded") + require.NotContains(t, paths, filepath.Join(subdir, "old.log.old"), + ".old files should be excluded") + } + + // Verify wrong extensions are NOT matched + for _, subdir := range []string{"nginx", "apache", "caddy"} { + require.NotContains(t, paths, filepath.Join(subdir, "debug.yaml"), + ".yaml files should not be matched") + } +} diff --git a/internal/component/loki/source/file/resolver_unix_test.go b/internal/component/loki/source/file/resolver_unix_test.go new file mode 100644 index 00000000000..fa60f7ed4c7 --- /dev/null +++ b/internal/component/loki/source/file/resolver_unix_test.go @@ -0,0 +1,54 @@ +//go:build !windows + +package file + +import ( + "testing" + + "github.com/go-kit/log" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/component/discovery" +) + +// TestGlobResolverCaseSensitive verifies that glob patterns are case-sensitive on Unix. +// An uppercase pattern should NOT match lowercase files. +func TestGlobResolverCaseSensitive(t *testing.T) { + resolver := newGlobResolver(log.NewNopLogger()) + + // Use uppercase pattern - should NOT match the lowercase .log files on Unix + targets := []discovery.Target{ + discovery.NewTargetFromLabelSet(model.LabelSet{ + "__path__": "./testdata/*.LOG", + "label": "test", + }), + } + + var results []resolvedTarget + for target := range resolver.Resolve(targets) { + results = append(results, target) + } + + require.Len(t, results, 0, "Unix should be case-sensitive: *.LOG should not match *.log files") +} + +// TestGlobResolverCaseSensitiveMatch verifies that matching case works correctly. +func TestGlobResolverCaseSensitiveMatch(t *testing.T) { + resolver := newGlobResolver(log.NewNopLogger()) + + // Use lowercase pattern - should match the lowercase .log files + targets := []discovery.Target{ + discovery.NewTargetFromLabelSet(model.LabelSet{ + "__path__": "./testdata/*.log", + "label": "test", + }), + } + + var results []resolvedTarget + for target := range resolver.Resolve(targets) { + results = append(results, target) + } + + require.Len(t, results, 2, "Pattern with matching case should find the files") +} diff --git a/internal/component/loki/source/file/resolver_windows_test.go b/internal/component/loki/source/file/resolver_windows_test.go new file mode 100644 index 00000000000..086415c33dc --- /dev/null +++ b/internal/component/loki/source/file/resolver_windows_test.go @@ -0,0 +1,54 @@ +//go:build windows + +package file + +import ( + "testing" + + "github.com/go-kit/log" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/component/discovery" +) + +// TestGlobResolverCaseInsensitive verifies that glob patterns are case-insensitive on Windows. +// An uppercase pattern SHOULD match lowercase files. +func TestGlobResolverCaseInsensitive(t *testing.T) { + resolver := newGlobResolver(log.NewNopLogger()) + + // Use uppercase pattern - SHOULD match the lowercase .log files on Windows + targets := []discovery.Target{ + discovery.NewTargetFromLabelSet(model.LabelSet{ + "__path__": "./testdata/*.LOG", + "label": "test", + }), + } + + var results []resolvedTarget + for target := range resolver.Resolve(targets) { + results = append(results, target) + } + + require.Len(t, results, 2, "Windows should be case-insensitive: *.LOG should match *.log files") +} + +// TestGlobResolverCaseInsensitiveLowercase verifies that lowercase patterns also work. +func TestGlobResolverCaseInsensitiveLowercase(t *testing.T) { + resolver := newGlobResolver(log.NewNopLogger()) + + // Use lowercase pattern - should also match + targets := []discovery.Target{ + discovery.NewTargetFromLabelSet(model.LabelSet{ + "__path__": "./testdata/*.log", + "label": "test", + }), + } + + var results []resolvedTarget + for target := range resolver.Resolve(targets) { + results = append(results, target) + } + + require.Len(t, results, 2, "Lowercase pattern should also find the files") +} diff --git a/internal/util/glob/glob.go b/internal/util/glob/glob.go new file mode 100644 index 00000000000..cb2766b8baf --- /dev/null +++ b/internal/util/glob/glob.go @@ -0,0 +1,10 @@ +package glob + +// Globber provides file path globbing functionality. +// Platform-specific implementations control options like case sensitivity. +type Globber interface { + // FilepathGlob returns a list of file paths matching the pattern. + FilepathGlob(pattern string) ([]string, error) + // PathMatch returns true if the path matches the pattern. + PathMatch(pattern, path string) (bool, error) +} diff --git a/internal/util/glob/glob_default.go b/internal/util/glob/glob_default.go new file mode 100644 index 00000000000..53656b482b2 --- /dev/null +++ b/internal/util/glob/glob_default.go @@ -0,0 +1,21 @@ +//go:build !windows + +package glob + +import "github.com/bmatcuk/doublestar/v4" + +// defaultGlobber implements Globber with default options (case-sensitive). +type defaultGlobber struct{} + +// NewGlobber creates a new Globber appropriate for the current platform. +func NewGlobber() Globber { + return &defaultGlobber{} +} + +func (g *defaultGlobber) FilepathGlob(pattern string) ([]string, error) { + return doublestar.FilepathGlob(pattern) +} + +func (g *defaultGlobber) PathMatch(pattern, path string) (bool, error) { + return doublestar.PathMatch(pattern, path) +} diff --git a/internal/util/glob/glob_windows.go b/internal/util/glob/glob_windows.go new file mode 100644 index 00000000000..8e681fbbf8f --- /dev/null +++ b/internal/util/glob/glob_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +package glob + +import "github.com/bmatcuk/doublestar/v4" + +// windowsGlobber implements Globber with case-insensitive matching, +// appropriate for Windows file systems. +type windowsGlobber struct{} + +// NewGlobber creates a new Globber appropriate for the current platform. +// On Windows, this returns a case-insensitive globber. +func NewGlobber() Globber { + return &windowsGlobber{} +} + +func (g *windowsGlobber) FilepathGlob(pattern string) ([]string, error) { + return doublestar.FilepathGlob(pattern, doublestar.WithCaseInsensitive()) +} + +func (g *windowsGlobber) PathMatch(pattern, path string) (bool, error) { + return doublestar.PathMatch(pattern, path) +}