diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a39d94478c..445660a7ba4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,8 @@ Main (unreleased) - Fix Docker log corruption for multiplexed long lines. (@axd1x8a) +- Fix the promtail converter behavior to mimic promtail behavior by default and limit kubernetes discovery to the same node. (@dehaansa) + - Allow configuration of `force_attempt_http2` and default it to `true` for otelcol exporters with HTTP client configurations. (@dehaansa) v1.12.0 diff --git a/docs/sources/set-up/migrate/from-promtail.md b/docs/sources/set-up/migrate/from-promtail.md index 58d5ebcca64..263345ea8ce 100644 --- a/docs/sources/set-up/migrate/from-promtail.md +++ b/docs/sources/set-up/migrate/from-promtail.md @@ -179,7 +179,7 @@ The following list is specific to the convert command and not {{< param "PRODUCT Make sure that you use the new metric names, for example, in your alerts and dashboards queries. * The logs produced by {{< param "PRODUCT_NAME" >}} differ from those produced by Promtail. * {{< param "PRODUCT_NAME" >}} exposes the {{< param "PRODUCT_NAME" >}} [UI][], which differs from the Promtail Web UI. -* If you are converting a Promtail configuration for a Kubernetes daemonset, [modify the generated configuration][single-node-discovery] to ensure `discovery.kubernetes` only discovers Pods residing on the same node as the {{< param "PRODUCT_NAME" >}} Pod. +* If you are converting a Promtail configuration and not deploying as a Kubernetes daemonset, [modify the generated configuration][single-node-discovery] to ensure `discovery.kubernetes` discovery behaves as expected. The converter makes the same assumption as promtail that any `pod` discovery is for a daemonset deployment. [Promtail]: https://www.grafana.com/docs/loki/latest/clients/promtail/ [debugging]: #debugging diff --git a/internal/converter/internal/promtailconvert/internal/build/service_discovery.go b/internal/converter/internal/promtailconvert/internal/build/service_discovery.go index e9e81c171aa..cfc45771dae 100644 --- a/internal/converter/internal/promtailconvert/internal/build/service_discovery.go +++ b/internal/converter/internal/promtailconvert/internal/build/service_discovery.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/alloy/internal/loki/promtail/scrapeconfig" "github.com/prometheus/common/model" prom_discover "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/kubernetes" ) func (s *ScrapeConfigBuilder) AppendSDs() { @@ -86,6 +87,13 @@ func toDiscoveryConfig(cfg *scrapeconfig.Config) prom_discover.Configs { } for _, sd := range cfg.ServiceDiscoveryConfig.KubernetesSDConfigs { + // See https://github.com/grafana/loki/blob/main/clients/pkg/promtail/targets/file/filetargetmanager.go#L126 + if sd.Role == kubernetes.RolePod { + sd.Selectors = append(sd.Selectors, kubernetes.SelectorConfig{ + Role: kubernetes.RolePod, + Field: `"spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)`, + }) + } sdConfigs = append(sdConfigs, sd) } diff --git a/internal/converter/internal/promtailconvert/promtailconvert.go b/internal/converter/internal/promtailconvert/promtailconvert.go index 49859fed350..1c5779f3434 100644 --- a/internal/converter/internal/promtailconvert/promtailconvert.go +++ b/internal/converter/internal/promtailconvert/promtailconvert.go @@ -4,6 +4,7 @@ import ( "bytes" "flag" "fmt" + "strings" promtailcfg "github.com/grafana/alloy/internal/loki/promtail/config" "github.com/grafana/alloy/internal/loki/promtail/file" @@ -84,11 +85,19 @@ func Convert(in []byte, extraArgs []string) ([]byte, diag.Diagnostics) { return nil, diags } - prettyByte, newDiags := common.PrettyPrint(buf.Bytes()) + prettyByte, newDiags := common.PrettyPrint([]byte(fixFunctionUse(buf.String()))) diags.AddAll(newDiags) return prettyByte, diags } +// There is not a way to express a function call in a struct to encode back into Alloy config, so we need to fix it +func fixFunctionUse(buf string) string { + return strings.ReplaceAll(buf, + `"\"spec.nodeName=\" + coalesce(sys.env(\"HOSTNAME\"), constants.hostname)"`, + `"spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)`, + ) +} + // AppendAll analyzes the entire promtail config in memory and transforms it // into Alloy components. It then appends each argument to the file builder. func AppendAll(f *builder.File, cfg *promtailcfg.Config, labelPrefix string, diags diag.Diagnostics) diag.Diagnostics { diff --git a/internal/converter/internal/promtailconvert/promtailconvert_test.go b/internal/converter/internal/promtailconvert/promtailconvert_test.go index 3f0b3463bbe..ff5a71c85c9 100644 --- a/internal/converter/internal/promtailconvert/promtailconvert_test.go +++ b/internal/converter/internal/promtailconvert/promtailconvert_test.go @@ -11,7 +11,7 @@ import ( _ "github.com/grafana/alloy/internal/static/metrics/instance" // Imported to override default values via the init function. ) -// Set this flag to update snapshots e.g. `go test -v ./interal/converter/internal/promtailconverter/...` -fix-tests +// Set this flag to update snapshots e.g. `go test -v ./internal/converter/internal/promtailconvert/... -fix-tests` var fixTestsFlag = flag.Bool("fix-tests", false, "update the test files with the current generated content") func TestConvert(t *testing.T) { diff --git a/internal/converter/internal/promtailconvert/testdata/kubernetes.alloy b/internal/converter/internal/promtailconvert/testdata/kubernetes.alloy index 86b853d7a55..7840b89219e 100644 --- a/internal/converter/internal/promtailconvert/testdata/kubernetes.alloy +++ b/internal/converter/internal/promtailconvert/testdata/kubernetes.alloy @@ -15,11 +15,21 @@ discovery.kubernetes "fun" { server_name = "example.local" insecure_skip_verify = true } + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } discovery.kubernetes "fun_2" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } discovery.kubernetes "fun_3" { @@ -30,6 +40,11 @@ discovery.kubernetes "fun_3" { type = "Bearer" credentials_file = "/home/robin/.special_token" } + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } discovery.kubernetes "fun_4" { @@ -40,6 +55,11 @@ discovery.kubernetes "fun_4" { type = "Bearer" credentials_file = "/home/toby/.token" } + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } discovery.kubernetes "fun_5" { diff --git a/internal/converter/internal/promtailconvert/testdata/mixed_pipeline.alloy b/internal/converter/internal/promtailconvert/testdata/mixed_pipeline.alloy index f995aea84ae..c91eca97339 100644 --- a/internal/converter/internal/promtailconvert/testdata/mixed_pipeline.alloy +++ b/internal/converter/internal/promtailconvert/testdata/mixed_pipeline.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "uber_pipeline" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } discovery.consulagent "uber_pipeline" { diff --git a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_cri_empty.alloy b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_cri_empty.alloy index ea7795b4d1d..1d7c2aa5aaa 100644 --- a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_cri_empty.alloy +++ b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_cri_empty.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "example" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "example" { diff --git a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_drop.alloy b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_drop.alloy index 46b0a64d5d1..2e2da4f0964 100644 --- a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_drop.alloy +++ b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_drop.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "example" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "example" { diff --git a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_match_nested.alloy b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_match_nested.alloy index 3f4162e2deb..1ce1715bf3a 100644 --- a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_match_nested.alloy +++ b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_match_nested.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "example" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "example" { diff --git a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part1.alloy b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part1.alloy index c2149380a9a..b9c73d74f74 100644 --- a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part1.alloy +++ b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part1.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "example" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "example" { diff --git a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part2.alloy b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part2.alloy index 13588946e44..345b09150d9 100644 --- a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part2.alloy +++ b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_part2.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "example" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "example" { diff --git a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_structured_metadata.alloy b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_structured_metadata.alloy index acb229ba67f..9c7c18c8c80 100644 --- a/internal/converter/internal/promtailconvert/testdata/pipeline_stages_structured_metadata.alloy +++ b/internal/converter/internal/promtailconvert/testdata/pipeline_stages_structured_metadata.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "example" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "example" { diff --git a/internal/converter/internal/promtailconvert/testdata/sd_pipeline_example.alloy b/internal/converter/internal/promtailconvert/testdata/sd_pipeline_example.alloy index 6746c6cb6ef..58a002c0de9 100644 --- a/internal/converter/internal/promtailconvert/testdata/sd_pipeline_example.alloy +++ b/internal/converter/internal/promtailconvert/testdata/sd_pipeline_example.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "funny_one" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } discovery.kubernetes "funny_one_2" { diff --git a/internal/converter/internal/promtailconvert/testdata/static_pipeline_example.alloy b/internal/converter/internal/promtailconvert/testdata/static_pipeline_example.alloy index 2132ad4a030..8585c793a7b 100644 --- a/internal/converter/internal/promtailconvert/testdata/static_pipeline_example.alloy +++ b/internal/converter/internal/promtailconvert/testdata/static_pipeline_example.alloy @@ -1,6 +1,11 @@ discovery.kubernetes "example" { role = "pod" kubeconfig_file = "/home/toby/.kube/config" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "example" { diff --git a/internal/converter/internal/staticconvert/staticconvert.go b/internal/converter/internal/staticconvert/staticconvert.go index 290fd1ed7ca..b0f750c3926 100644 --- a/internal/converter/internal/staticconvert/staticconvert.go +++ b/internal/converter/internal/staticconvert/staticconvert.go @@ -4,6 +4,7 @@ import ( "bytes" "flag" "fmt" + "strings" prom_config "github.com/prometheus/prometheus/config" @@ -57,11 +58,19 @@ func Convert(in []byte, extraArgs []string) ([]byte, diag.Diagnostics) { return nil, diags } - prettyByte, newDiags := common.PrettyPrint(buf.Bytes()) + prettyByte, newDiags := common.PrettyPrint([]byte(fixFunctionUse(buf.String()))) diags.AddAll(newDiags) return prettyByte, diags } +// There is not a way to express a function call in a struct to encode back into Alloy config, so we need to fix it +func fixFunctionUse(buf string) string { + return strings.ReplaceAll(buf, + `"\"spec.nodeName=\" + coalesce(sys.env(\"HOSTNAME\"), constants.hostname)"`, + `"spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)`, + ) +} + // AppendAll analyzes the entire static config in memory and transforms it into // Alloy component Arguments. It then appends each argument to the file // builder. Exports from other components are correctly referenced to build the diff --git a/internal/converter/internal/staticconvert/testdata_linux/promtail_metrics.alloy b/internal/converter/internal/staticconvert/testdata_linux/promtail_metrics.alloy index cbdc3f64fc5..fdd58d7c718 100644 --- a/internal/converter/internal/staticconvert/testdata_linux/promtail_metrics.alloy +++ b/internal/converter/internal/staticconvert/testdata_linux/promtail_metrics.alloy @@ -1,5 +1,10 @@ discovery.kubernetes "logs_config_with_metrics" { role = "pod" + + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname) + } } loki.process "logs_config_with_metrics" {