[8.19](backport #44493) [Metricbeat] [Prometheus] Add proper error handling for query dataset when the auth/ssl is misconfigured

CHANGELOG.next.asciidoc

@@ -69,6 +69,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Fix the function to determine CPU cores on windows {issue}42593[42593] {pull}43409[43409]
 - Handle permission errors while collecting data from Windows services and don't interrupt the overall collection by skipping affected services {issue}40765[40765] {pull}43665[43665]
 - Fixed a bug where `event.duration` could be missing from an event on Windows systems due to low-resolution clock. {pull}44440[44440]
+- Add check for http error codes in the Metricbeat's Prometheus query submodule {pull}44493[44493]
 
 *Osquerybeat*
 

metricbeat/module/prometheus/query/query.go

@@ -97,6 +97,12 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 			return err
 		}
 
+		if response.StatusCode > 399 {
+			m.Logger().Debugf("error received from prometheus endpoint %v: %v", url, string(body))
+			reporter.Error(fmt.Errorf("unexpected status code %d from %v", response.StatusCode, url))
+			continue
+		}
+
 		events, parseErr := parseResponse(body, pathConfig)
 		if parseErr != nil {
 			reporter.Error(fmt.Errorf("error parsing response from %v: %w", url, parseErr))

metricbeat/module/prometheus/query/query_test.go

@@ -22,6 +22,8 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"testing"
 
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
@@ -214,3 +216,42 @@ func TestQueryFetchEventContentString(t *testing.T) {
 		t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 	}
 }
+
+func TestHTTPErrorCodeHandling(t *testing.T) {
+	statusCode := 400
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(statusCode)
+		_, _ = w.Write([]byte("Client sent an HTTP request to an HTTPS server."))
+	}))
+	defer server.Close()
+
+	config := map[string]interface{}{
+		"module":     "prometheus",
+		"metricsets": []string{"query"},
+		"hosts":      []string{server.URL},
+		// queries do not have an actual role here since all http responses are mocked
+		"queries": []mapstr.M{
+			mapstr.M{
+				"name": "string",
+				"path": "/api/v1/query",
+				"params": mapstr.M{
+					"query": "some",
+				},
+			},
+		},
+	}
+	reporter := &mbtest.CapturingReporterV2{}
+
+	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
+	_ = metricSet.Fetch(reporter)
+
+	errs := reporter.GetErrors()
+	if len(errs) != 1 {
+		t.Fatalf("Expected 1 error, had %d: %v\n", len(errs), errs)
+	}
+
+	if !strings.Contains(errs[0].Error(), strconv.Itoa(statusCode)) {
+		t.Fatalf("Expected error to contain HTTP response code %d, got error: %s\n", statusCode, errs[0])
+	}
+}