From 93f117d3e2b28b2967da06f5a94c44c7613ea518 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Mon, 9 Jun 2025 17:19:08 +0200 Subject: [PATCH 1/4] chore: prom rw exporter add extra logs for the RW2 code path --- .../prometheusremotewriteexporter/exporter.go | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 68a4aa4dc007f..7a554f5de437a 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -411,6 +411,28 @@ func (prwe *prwExporter) execute(ctx context.Context, buf *buffer) error { resp.Body.Close() }() + // Per the Prometheus remote write 2.0 specification, the response should contain X-Prometheus-Remote-Write-Samples-Written header. + // If the header is missing, it suggests that the endpoint does not support RW2 or the implementation is not compliant with the specification. + // Reference: https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/#required-written-response-headers + if enableSendingRW2FeatureGate.IsEnabled() && prwe.RemoteWriteProtoMsg == config.RemoteWriteProtoMsgV2 { + samplesWritten := resp.Header.Get("X-Prometheus-Remote-Write-Samples-Written") + if samplesWritten == "" { + prwe.settings.Logger.Warn( + "X-Prometheus-Remote-Write-Samples-Written header is missing from the response, suggesting that the endpoint doesn't support RW2 and might be silently dropping data.", + zap.String("url", resp.Request.URL.String()), + ) + } else { + prwe.settings.Logger.Debug("X-Prometheus-Remote-Write-Samples-Written", zap.String("samples_written", samplesWritten)) + // If the X-Prometheus-Remote-Write-Samples-Written header is present, we can also check and log if the other headers are present. + if histogramsWritten := resp.Header.Get("X-Prometheus-Remote-Write-Histograms-Written"); histogramsWritten != "" { + prwe.settings.Logger.Debug("X-Prometheus-Remote-Write-Histograms-Written", zap.String("histograms_written", histogramsWritten)) + } + if exemplarsWritten := resp.Header.Get("X-Prometheus-Remote-Write-Exemplars-Written"); exemplarsWritten != "" { + prwe.settings.Logger.Debug("X-Prometheus-Remote-Write-Exemplars-Written", zap.String("exemplars_written", exemplarsWritten)) + } + } + } + // 2xx status code is considered a success // 5xx errors are recoverable and the exporter should retry // Reference for different behavior according to status code: From 9ab2518193c1fd3e775432be1c043fb411b56ca6 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 15 Jun 2025 12:43:22 +0200 Subject: [PATCH 2/4] chore: limit comment to 100 columns --- exporter/prometheusremotewriteexporter/exporter.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 7a554f5de437a..dfa303583e7c2 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -411,9 +411,11 @@ func (prwe *prwExporter) execute(ctx context.Context, buf *buffer) error { resp.Body.Close() }() - // Per the Prometheus remote write 2.0 specification, the response should contain X-Prometheus-Remote-Write-Samples-Written header. - // If the header is missing, it suggests that the endpoint does not support RW2 or the implementation is not compliant with the specification. - // Reference: https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/#required-written-response-headers + // Per the Prometheus remote write 2.0 specification, the response should contain + // X-Prometheus-Remote-Write-Samples-Written header. + // If the header is missing, it suggests that the endpoint does not support RW2 or the + // implementation is not compliant with the specification. Reference: + // https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/#required-written-response-headers if enableSendingRW2FeatureGate.IsEnabled() && prwe.RemoteWriteProtoMsg == config.RemoteWriteProtoMsgV2 { samplesWritten := resp.Header.Get("X-Prometheus-Remote-Write-Samples-Written") if samplesWritten == "" { From 5191cff33b8882b0ba38d8414d4c473da323c716 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 15 Jun 2025 12:44:00 +0200 Subject: [PATCH 3/4] chore: remove extra debug log lines in favour of metrics --- exporter/prometheusremotewriteexporter/exporter.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index dfa303583e7c2..8acd4fc1006a8 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -423,16 +423,6 @@ func (prwe *prwExporter) execute(ctx context.Context, buf *buffer) error { "X-Prometheus-Remote-Write-Samples-Written header is missing from the response, suggesting that the endpoint doesn't support RW2 and might be silently dropping data.", zap.String("url", resp.Request.URL.String()), ) - } else { - prwe.settings.Logger.Debug("X-Prometheus-Remote-Write-Samples-Written", zap.String("samples_written", samplesWritten)) - // If the X-Prometheus-Remote-Write-Samples-Written header is present, we can also check and log if the other headers are present. - if histogramsWritten := resp.Header.Get("X-Prometheus-Remote-Write-Histograms-Written"); histogramsWritten != "" { - prwe.settings.Logger.Debug("X-Prometheus-Remote-Write-Histograms-Written", zap.String("histograms_written", histogramsWritten)) - } - if exemplarsWritten := resp.Header.Get("X-Prometheus-Remote-Write-Exemplars-Written"); exemplarsWritten != "" { - prwe.settings.Logger.Debug("X-Prometheus-Remote-Write-Exemplars-Written", zap.String("exemplars_written", exemplarsWritten)) - } - } } // 2xx status code is considered a success From e3dc47845ae4b5e2d6bd1679fb5540858cadb48c Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 15 Jun 2025 12:47:52 +0200 Subject: [PATCH 4/4] chore: fix missing curly bracket --- exporter/prometheusremotewriteexporter/exporter.go | 1 + 1 file changed, 1 insertion(+) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 8acd4fc1006a8..a37c4690a5f21 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -423,6 +423,7 @@ func (prwe *prwExporter) execute(ctx context.Context, buf *buffer) error { "X-Prometheus-Remote-Write-Samples-Written header is missing from the response, suggesting that the endpoint doesn't support RW2 and might be silently dropping data.", zap.String("url", resp.Request.URL.String()), ) + } } // 2xx status code is considered a success