diff --git a/prober/dns.go b/prober/dns.go index 3d301b9b..6d74c327 100644 --- a/prober/dns.go +++ b/prober/dns.go @@ -18,6 +18,7 @@ import ( "log/slog" "net" "regexp" + "strconv" "time" "github.com/miekg/dns" @@ -28,37 +29,36 @@ import ( ) // validRRs checks a slice of RRs received from the server against a DNSRRValidator. -func validRRs(rrs *[]dns.RR, v *config.DNSRRValidator, logger *slog.Logger) bool { +func validRRs(rrType string, rrs *[]dns.RR, v *config.DNSRRValidator, logger *slog.Logger) ProbeResult { + logger.Info("Validating " + rrType) + failure_reason := rrType + " validation Failed" var anyMatch = false var allMatch = true // Fail the probe if there are no RRs of a given type, but a regexp match is required // (i.e. FailIfNotMatchesRegexp or FailIfNoneMatchesRegexp is set). if len(*rrs) == 0 && len(v.FailIfNotMatchesRegexp) > 0 { - logger.Error("fail_if_not_matches_regexp specified but no RRs returned") - return false + return ProbeFailure(failure_reason, "problem", "fail_if_not_matches_regexp specified but no RRs returned") } if len(*rrs) == 0 && len(v.FailIfNoneMatchesRegexp) > 0 { - logger.Error("fail_if_none_matches_regexp specified but no RRs returned") - return false + return ProbeFailure(failure_reason, "problem", "fail_if_none_matches_regexp specified but no RRs returned") } for _, rr := range *rrs { logger.Info("Validating RR", "rr", rr) for _, re := range v.FailIfMatchesRegexp { match, err := regexp.MatchString(re, rr.String()) if err != nil { - logger.Error("Error matching regexp", "regexp", re, "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure(failure_reason, "problem", "Error matching regexp", "regexp", re) } if match { - logger.Error("At least one RR matched regexp", "regexp", re, "rr", rr) - return false + return ProbeFailure(failure_reason, "problem", "At least one RR matched regexp", "regexp", re) } } for _, re := range v.FailIfAllMatchRegexp { match, err := regexp.MatchString(re, rr.String()) if err != nil { - logger.Error("Error matching regexp", "regexp", re, "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure(failure_reason, "problem", "Error matching regexp", "regexp", re) } if !match { allMatch = false @@ -67,19 +67,18 @@ func validRRs(rrs *[]dns.RR, v *config.DNSRRValidator, logger *slog.Logger) bool for _, re := range v.FailIfNotMatchesRegexp { match, err := regexp.MatchString(re, rr.String()) if err != nil { - logger.Error("Error matching regexp", "regexp", re, "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure(failure_reason, "problem", "Error matching regexp", "regexp", re) } if !match { - logger.Error("At least one RR did not match regexp", "regexp", re, "rr", rr) - return false + return ProbeFailure(failure_reason, "problem", "At least one RR did not match regexp", "regexp", re) } } for _, re := range v.FailIfNoneMatchesRegexp { match, err := regexp.MatchString(re, rr.String()) if err != nil { - logger.Error("Error matching regexp", "regexp", re, "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure(failure_reason, "problem", "Error matching regexp", "regexp", re) } if match { anyMatch = true @@ -87,18 +86,16 @@ func validRRs(rrs *[]dns.RR, v *config.DNSRRValidator, logger *slog.Logger) bool } } if len(v.FailIfAllMatchRegexp) > 0 && !allMatch { - logger.Error("Not all RRs matched regexp") - return false + return ProbeFailure(failure_reason, "problem", "Not all RRs matched regexp") } if len(v.FailIfNoneMatchesRegexp) > 0 && !anyMatch { - logger.Error("None of the RRs did matched any regexp") - return false + return ProbeFailure(failure_reason, "problem", "None of the RRs matched any regexp") } - return true + return ProbeSuccess() } // validRcode checks rcode in the response against a list of valid rcodes. -func validRcode(rcode int, valid []string, logger *slog.Logger) bool { +func validRcode(rcode int, valid []string, logger *slog.Logger) ProbeResult { var validRcodes []int // If no list of valid rcodes is specified, only NOERROR is considered valid. if valid == nil { @@ -107,8 +104,8 @@ func validRcode(rcode int, valid []string, logger *slog.Logger) bool { for _, rcode := range valid { rc, ok := dns.StringToRcode[rcode] if !ok { - logger.Error("Invalid rcode", "rcode", rcode, "known_rcode", dns.RcodeToString) - return false + logger.Info("Known rcodes", "known_rcode", dns.RcodeToString) + return ProbeFailure("Invalid rcode", "rcode", rcode) } validRcodes = append(validRcodes, rc) } @@ -116,14 +113,14 @@ func validRcode(rcode int, valid []string, logger *slog.Logger) bool { for _, rc := range validRcodes { if rcode == rc { logger.Info("Rcode is valid", "rcode", rcode, "string_rcode", dns.RcodeToString[rcode]) - return true + return ProbeSuccess() } } - logger.Error("Rcode is not one of the valid rcodes", "rcode", rcode, "string_rcode", dns.RcodeToString[rcode], "valid_rcodes", validRcodes) - return false + logger.Info("Valid Rrcodes", "valid_rcodes", validRcodes) + return ProbeFailure("Rcode is not one of the valid rcodes", "rcode", strconv.Itoa(rcode), "string_rcode", dns.RcodeToString[rcode]) } -func ProbeDNS(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) bool { +func ProbeDNS(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) ProbeResult { var dialProtocol string probeDNSDurationGaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "probe_dns_duration_seconds", @@ -161,8 +158,8 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry var ok bool qc, ok = dns.StringToClass[module.DNS.QueryClass] if !ok { - logger.Error("Invalid query class", "Class seen", module.DNS.QueryClass, "Existing classes", dns.ClassToString) - return false + logger.Info("Existing query classes", "existing_classes", dns.ClassToString) + return ProbeFailure("Invalid query class", "Class seen", module.DNS.QueryClass) } } @@ -171,8 +168,8 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry var ok bool qt, ok = dns.StringToType[module.DNS.QueryType] if !ok { - logger.Error("Invalid query type", "Type seen", module.DNS.QueryType, "Existing types", dns.TypeToString) - return false + logger.Info("Existing query types", "existing_types", dns.TypeToString) + return ProbeFailure("Invalid query type", "Type seen", module.DNS.QueryType) } } var probeDNSSOAGauge prometheus.Gauge @@ -182,8 +179,7 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry module.DNS.TransportProtocol = "udp" } if module.DNS.TransportProtocol != "udp" && module.DNS.TransportProtocol != "tcp" { - logger.Error("Configuration error: Expected transport protocol udp or tcp", "protocol", module.DNS.TransportProtocol) - return false + return ProbeFailure("Configuration error: Expected transport protocol udp or tcp", "protocol", module.DNS.TransportProtocol) } targetAddr, port, err := net.SplitHostPort(target) @@ -196,10 +192,9 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry } targetAddr = target } - ip, lookupTime, err := chooseProtocol(ctx, module.DNS.IPProtocol, module.DNS.IPProtocolFallback, targetAddr, registry, logger) - if err != nil { - logger.Error("Error resolving address", "err", err) - return false + ip, lookupTime, resolveResult := chooseProtocol(ctx, module.DNS.IPProtocol, module.DNS.IPProtocolFallback, targetAddr, registry, logger) + if !resolveResult.success { + return resolveResult } probeDNSDurationGaugeVec.WithLabelValues("resolve").Add(lookupTime) targetIP := net.JoinHostPort(ip.String(), port) @@ -214,8 +209,7 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry if module.DNS.TransportProtocol == "tcp" { dialProtocol += "-tls" } else { - logger.Error("Configuration error: Expected transport protocol tcp for DoT", "protocol", module.DNS.TransportProtocol) - return false + return ProbeFailure("Configuration error: Expected transport protocol tcp for DoT", "protocol", module.DNS.TransportProtocol) } } @@ -225,8 +219,8 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry if module.DNS.DNSOverTLS { tlsConfig, err := pconfig.NewTLSConfig(&module.DNS.TLSConfig) if err != nil { - logger.Error("Failed to create TLS configuration", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Failed to create TLS configuration") } if tlsConfig.ServerName == "" { // Use target-hostname as default for TLS-servername. @@ -240,8 +234,7 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry if len(module.DNS.SourceIPAddress) > 0 { srcIP := net.ParseIP(module.DNS.SourceIPAddress) if srcIP == nil { - logger.Error("Error parsing source ip address", "srcIP", module.DNS.SourceIPAddress) - return false + return ProbeFailure("Error parsing source ip address", "srcIP", module.DNS.SourceIPAddress) } logger.Info("Using local address", "srcIP", srcIP) client.Dialer = &net.Dialer{} @@ -270,8 +263,8 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry probeDNSDurationGaugeVec.WithLabelValues("connect").Set((time.Since(requestStart) - rtt).Seconds()) probeDNSDurationGaugeVec.WithLabelValues("request").Set(rtt.Seconds()) if err != nil { - logger.Error("Error while sending a DNS query", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error while sending a DNS query") } logger.Info("Got response", "response", response) @@ -294,23 +287,15 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry } } - if !validRcode(response.Rcode, module.DNS.ValidRcodes, logger) { - return false - } - logger.Info("Validating Answer RRs") - if !validRRs(&response.Answer, &module.DNS.ValidateAnswer, logger) { - logger.Error("Answer RRs validation failed") - return false + result := validRcode(response.Rcode, module.DNS.ValidRcodes, logger) + if result.success { + result = validRRs("Answer RRs", &response.Answer, &module.DNS.ValidateAnswer, logger) } - logger.Info("Validating Authority RRs") - if !validRRs(&response.Ns, &module.DNS.ValidateAuthority, logger) { - logger.Error("Authority RRs validation failed") - return false + if result.success { + result = validRRs("Authority RRs", &response.Ns, &module.DNS.ValidateAuthority, logger) } - logger.Info("Validating Additional RRs") - if !validRRs(&response.Extra, &module.DNS.ValidateAdditional, logger) { - logger.Error("Additional RRs validation failed") - return false + if result.success { + result = validRRs("Additional RRs", &response.Extra, &module.DNS.ValidateAdditional, logger) } - return true + return result } diff --git a/prober/dns_test.go b/prober/dns_test.go index 4e4fdff2..7dd455d3 100644 --- a/prober/dns_test.go +++ b/prober/dns_test.go @@ -17,6 +17,7 @@ import ( "context" "net" "os" + "reflect" "runtime" "testing" "time" @@ -95,8 +96,8 @@ func TestRecursiveDNSResponse(t *testing.T) { } tests := []struct { - Probe config.DNSProbe - ShouldSucceed bool + Probe config.DNSProbe + expectedResult ProbeResult }{ { config.DNSProbe{ @@ -104,7 +105,7 @@ func TestRecursiveDNSResponse(t *testing.T) { IPProtocolFallback: true, QueryName: "example.com", Recursion: true, - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -113,7 +114,8 @@ func TestRecursiveDNSResponse(t *testing.T) { QueryName: "example.com", Recursion: true, ValidRcodes: []string{"SERVFAIL", "NXDOMAIN"}, - }, false, + }, ProbeFailure("Rcode is not one of the valid rcodes", "rcode", "0", "string_rcode", + "NOERROR"), }, { config.DNSProbe{ @@ -125,7 +127,7 @@ func TestRecursiveDNSResponse(t *testing.T) { FailIfMatchesRegexp: []string{".*7200.*"}, FailIfNotMatchesRegexp: []string{".*3600.*"}, }, - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -136,7 +138,7 @@ func TestRecursiveDNSResponse(t *testing.T) { ValidateAuthority: config.DNSRRValidator{ FailIfMatchesRegexp: []string{".*7200.*"}, }, - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -147,7 +149,7 @@ func TestRecursiveDNSResponse(t *testing.T) { ValidateAdditional: config.DNSRRValidator{ FailIfNotMatchesRegexp: []string{".*3600.*"}, }, - }, false, + }, ProbeFailure("Additional RRs validation Failed", "problem", "fail_if_not_matches_regexp specified but no RRs returned"), }, { config.DNSProbe{ @@ -155,7 +157,7 @@ func TestRecursiveDNSResponse(t *testing.T) { IPProtocolFallback: true, QueryName: "example.com", Recursion: false, - }, false, + }, ProbeFailure("Rcode is not one of the valid rcodes", "rcode", "5", "string_rcode", "REFUSED"), }, } @@ -171,8 +173,8 @@ func TestRecursiveDNSResponse(t *testing.T) { testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeDNS(testCTX, addr.String(), config.Module{Timeout: time.Second, DNS: test.Probe}, registry, promslog.NewNopLogger()) - if result != test.ShouldSucceed { - t.Fatalf("Test %d had unexpected result: %v", i, result) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Fatalf("Test %d had unexpected result: expected %s, got %s", i, test.expectedResult, result) } mfs, err := registry.Gather() if err != nil { @@ -252,15 +254,15 @@ func TestAuthoritativeDNSResponse(t *testing.T) { } tests := []struct { - Probe config.DNSProbe - ShouldSucceed bool + Probe config.DNSProbe + expectedResult ProbeResult }{ { config.DNSProbe{ IPProtocol: "ip4", IPProtocolFallback: true, QueryName: "example.com", - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -268,7 +270,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { IPProtocolFallback: true, QueryName: "example.com", QueryType: "SOA", - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -281,7 +283,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { FailIfMatchesRegexp: []string{".*IN.*"}, FailIfNotMatchesRegexp: []string{".*CH.*"}, }, - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -289,7 +291,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { IPProtocolFallback: true, QueryName: "example.com", ValidRcodes: []string{"SERVFAIL", "NXDOMAIN"}, - }, false, + }, ProbeFailure("Rcode is not one of the valid rcodes", "rcode", "0", "string_rcode", "NOERROR"), }, { config.DNSProbe{ @@ -300,7 +302,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { FailIfMatchesRegexp: []string{".*3600.*"}, FailIfNotMatchesRegexp: []string{".*3600.*"}, }, - }, false, + }, ProbeFailure("Answer RRs validation Failed", "problem", "At least one RR matched regexp", "regexp", ".*3600.*"), }, { config.DNSProbe{ @@ -311,7 +313,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { FailIfMatchesRegexp: []string{".*7200.*"}, FailIfNotMatchesRegexp: []string{".*7200.*"}, }, - }, false, + }, ProbeFailure("Answer RRs validation Failed", "problem", "At least one RR did not match regexp", "regexp", ".*7200.*"), }, { config.DNSProbe{ @@ -321,7 +323,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { ValidateAuthority: config.DNSRRValidator{ FailIfNotMatchesRegexp: []string{"ns.*.isp.net"}, }, - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -331,7 +333,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { ValidateAdditional: config.DNSRRValidator{ FailIfNotMatchesRegexp: []string{"^ns.*.isp"}, }, - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -341,7 +343,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { ValidateAdditional: config.DNSRRValidator{ FailIfMatchesRegexp: []string{"^ns.*.isp"}, }, - }, false, + }, ProbeFailure("Additional RRs validation Failed", "problem", "At least one RR matched regexp", "regexp", "^ns.*.isp"), }, { config.DNSProbe{ @@ -351,7 +353,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { ValidateAdditional: config.DNSRRValidator{ FailIfAllMatchRegexp: []string{".*127.0.0.*"}, }, - }, false, + }, ProbeFailure("Additional RRs validation Failed", "problem", "Not all RRs matched regexp"), }, { config.DNSProbe{ @@ -361,7 +363,7 @@ func TestAuthoritativeDNSResponse(t *testing.T) { ValidateAdditional: config.DNSRRValidator{ FailIfNoneMatchesRegexp: []string{".*127.0.0.3.*"}, }, - }, false, + }, ProbeFailure("Additional RRs validation Failed", "problem", "None of the RRs matched any regexp"), }, } @@ -375,8 +377,8 @@ func TestAuthoritativeDNSResponse(t *testing.T) { testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeDNS(testCTX, addr.String(), config.Module{Timeout: time.Second, DNS: test.Probe}, registry, promslog.NewNopLogger()) - if result != test.ShouldSucceed { - t.Fatalf("Test %d had unexpected result: %v", i, result) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Fatalf("Test %d had unexpected result: expected %s, got %s", i, test.expectedResult, result) } mfs, err := registry.Gather() if err != nil { @@ -403,15 +405,15 @@ func TestServfailDNSResponse(t *testing.T) { } tests := []struct { - Probe config.DNSProbe - ShouldSucceed bool + Probe config.DNSProbe + expectedResult ProbeResult }{ { config.DNSProbe{ IPProtocol: "ip4", IPProtocolFallback: true, QueryName: "example.com", - }, false, + }, ProbeFailure("Rcode is not one of the valid rcodes", "rcode", "2", "string_rcode", "SERVFAIL"), }, { config.DNSProbe{ @@ -419,7 +421,7 @@ func TestServfailDNSResponse(t *testing.T) { IPProtocolFallback: true, QueryName: "example.com", ValidRcodes: []string{"SERVFAIL", "NXDOMAIN"}, - }, true, + }, ProbeSuccess(), }, { config.DNSProbe{ @@ -427,7 +429,7 @@ func TestServfailDNSResponse(t *testing.T) { IPProtocolFallback: true, QueryName: "example.com", QueryType: "NOT_A_VALID_QUERY_TYPE", - }, false, + }, ProbeFailure("Invalid query type", "Type seen", "NOT_A_VALID_QUERY_TYPE"), }, { config.DNSProbe{ @@ -435,7 +437,7 @@ func TestServfailDNSResponse(t *testing.T) { IPProtocolFallback: true, QueryName: "example.com", ValidRcodes: []string{"NOT_A_VALID_RCODE"}, - }, false, + }, ProbeFailure("Invalid rcode", "rcode", "NOT_A_VALID_RCODE"), }, } @@ -450,8 +452,8 @@ func TestServfailDNSResponse(t *testing.T) { testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeDNS(testCTX, addr.String(), config.Module{Timeout: time.Second, DNS: test.Probe}, registry, promslog.NewNopLogger()) - if result != test.ShouldSucceed { - t.Fatalf("Test %d had unexpected result: %v", i, result) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Fatalf("Test %d had unexpected result: expected %s, got %s", i, test.expectedResult, result) } mfs, err := registry.Gather() if err != nil { @@ -510,8 +512,9 @@ func TestDNSProtocol(t *testing.T) { testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeDNS(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) - if !result { + if !result.success { t.Fatalf("DNS protocol: \"%v\", preferred \"ip6\" connection test failed, expected success.", protocol) + t.Fatalf("Failure reason: %s", result) } mfs, err := registry.Gather() if err != nil { @@ -536,8 +539,9 @@ func TestDNSProtocol(t *testing.T) { testCTX, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result = ProbeDNS(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) - if !result { + if !result.success { t.Fatalf("DNS protocol: \"%v\", preferred \"ip4\" connection test failed, expected success.", protocol) + t.Fatalf("Failure reason: %s", result) } mfs, err = registry.Gather() if err != nil { @@ -562,8 +566,9 @@ func TestDNSProtocol(t *testing.T) { testCTX, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result = ProbeDNS(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) - if !result { + if !result.success { t.Fatalf("DNS protocol: \"%v\" connection test failed, expected success.", protocol) + t.Fatalf("Failure reason: %s", result) } mfs, err = registry.Gather() if err != nil { @@ -588,13 +593,18 @@ func TestDNSProtocol(t *testing.T) { defer cancel() result = ProbeDNS(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) if protocol == "udp" { - if !result { + if !result.success { t.Fatalf("DNS test connection with protocol %s failed, expected success.", protocol) + t.Fatalf("Failure reason: %s", result) } } else { - if result { + if result.success { t.Fatalf("DNS test connection with protocol %s succeeded, expected failure.", protocol) } + expectedReason := ProbeFailure("Error while sending a DNS query") + if !reflect.DeepEqual(result, expectedReason) { + t.Fatalf("Test unexpected result: expected %s, got %s", expectedReason, result) + } } mfs, err = registry.Gather() if err != nil { @@ -630,8 +640,8 @@ func TestDNSMetrics(t *testing.T) { testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeDNS(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) - if !result { - t.Fatalf("DNS test connection failed, expected success.") + if !result.success { + t.Fatalf("DNS test connection failed, expected success, got %s.", result) } mfs, err := registry.Gather() if err != nil { diff --git a/prober/grpc.go b/prober/grpc.go index fe8d1eca..5a131d62 100644 --- a/prober/grpc.go +++ b/prober/grpc.go @@ -74,7 +74,7 @@ func (c *gRPCHealthCheckClient) Check(ctx context.Context, service string) (bool return false, returnStatus.Code(), nil, "", err } -func ProbeGRPC(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (success bool) { +func ProbeGRPC(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (result ProbeResult) { var ( durationGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ @@ -128,8 +128,8 @@ func ProbeGRPC(ctx context.Context, target string, module config.Module, registr targetURL, err := url.Parse(target) if err != nil { - logger.Error("Could not parse target URL", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Could not parse target URL") } targetHost, targetPort, err := net.SplitHostPort(targetURL.Host) @@ -140,14 +140,13 @@ func ProbeGRPC(ctx context.Context, target string, module config.Module, registr tlsConfig, err := pconfig.NewTLSConfig(&module.GRPC.TLSConfig) if err != nil { - logger.Error("Error creating TLS configuration", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error creating TLS configuration") } - ip, lookupTime, err := chooseProtocol(ctx, module.GRPC.PreferredIPProtocol, module.GRPC.IPProtocolFallback, targetHost, registry, logger) - if err != nil { - logger.Error("Error resolving address", "err", err) - return false + ip, lookupTime, resolveResult := chooseProtocol(ctx, module.GRPC.PreferredIPProtocol, module.GRPC.IPProtocolFallback, targetHost, registry, logger) + if !resolveResult.success { + return resolveResult } durationGaugeVec.WithLabelValues("resolve").Add(lookupTime) checkStart := time.Now() @@ -211,12 +210,14 @@ func ProbeGRPC(ctx context.Context, target string, module config.Module, registr } statusCodeGauge.Set(float64(statusCode)) - if !ok || err != nil { - logger.Error("can't connect grpc server:", "err", err) - success = false + if err != nil { + logger.Error(err.Error()) + return ProbeFailure("Can't connect to the grpc server") + } else if !ok { + return ProbeFailure("Can't connect to the grpc server") } else { logger.Debug("connect the grpc server successfully") - success = true + result = ProbeSuccess() } return diff --git a/prober/grpc_test.go b/prober/grpc_test.go index 2a2c4aeb..182d3285 100644 --- a/prober/grpc_test.go +++ b/prober/grpc_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "os" + "reflect" "testing" "time" @@ -72,8 +73,8 @@ func TestGRPCConnection(t *testing.T) { }, }, registry, promslog.NewNopLogger()) - if !result { - t.Fatalf("GRPC probe failed") + if !result.success { + t.Fatalf("GRPC probe failed, got %s", result) } mfs, err := registry.Gather() @@ -142,7 +143,7 @@ func TestMultipleGRPCservices(t *testing.T) { }, }, registryService1, promslog.NewNopLogger()) - if !resultService1 { + if !resultService1.success { t.Fatalf("GRPC probe failed for service1") } @@ -154,9 +155,13 @@ func TestMultipleGRPCservices(t *testing.T) { }, }, registryService2, promslog.NewNopLogger()) - if resultService2 { + if resultService2.success { t.Fatalf("GRPC probe succeed for service2") } + expectedReason2 := ProbeFailure("Can't connect to the grpc server") + if !reflect.DeepEqual(resultService2, expectedReason2) { + t.Fatalf("Test unexpected result: expected %s, got %s", expectedReason2, resultService2) + } registryService3 := prometheus.NewRegistry() resultService3 := ProbeGRPC(testCTX, "localhost:"+port, @@ -166,9 +171,13 @@ func TestMultipleGRPCservices(t *testing.T) { }, }, registryService3, promslog.NewNopLogger()) - if resultService3 { + if resultService3.success { t.Fatalf("GRPC probe succeed for service3") } + expectedReason3 := ProbeFailure("Can't connect to the grpc server") + if !reflect.DeepEqual(resultService3, expectedReason3) { + t.Fatalf("Test unexpected result: expected %s, got %s", expectedReason2, resultService2) + } } func TestGRPCTLSConnection(t *testing.T) { @@ -242,8 +251,8 @@ func TestGRPCTLSConnection(t *testing.T) { }, }, registry, promslog.NewNopLogger()) - if !result { - t.Fatalf("GRPC probe failed") + if !result.success { + t.Fatalf("GRPC probe failed, got %s", result) } mfs, err := registry.Gather() @@ -306,8 +315,9 @@ func TestNoTLSConnection(t *testing.T) { }, }, registry, promslog.NewNopLogger()) - if result { - t.Fatalf("GRPC probe succeed") + expectedResult := ProbeFailure("Can't connect to the grpc server") + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("Test unexpected result: expected %s, got %s", expectedResult, result) } mfs, err := registry.Gather() @@ -363,8 +373,9 @@ func TestGRPCServiceNotFound(t *testing.T) { }, }, registry, promslog.NewNopLogger()) - if result { - t.Fatalf("GRPC probe succeed") + expectedResult := ProbeFailure("Can't connect to the grpc server") + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("Test unexpected result: expected %s, got %s", expectedResult, result) } mfs, err := registry.Gather() @@ -416,8 +427,9 @@ func TestGRPCHealthCheckUnimplemented(t *testing.T) { }, }, registry, promslog.NewNopLogger()) - if result { - t.Fatalf("GRPC probe succeed") + expectedResult := ProbeFailure("Can't connect to the grpc server") + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("Test unexpected result: expected %s, got %s", expectedResult, result) } mfs, err := registry.Gather() @@ -434,6 +446,10 @@ func TestGRPCHealthCheckUnimplemented(t *testing.T) { } func TestGRPCAbsentFailedTLS(t *testing.T) { + if os.Getenv("CI") == "true" { + t.Skip("skipping; CI is failing on ipv6 dns requests") + } + testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() registry := prometheus.NewRegistry() @@ -446,8 +462,9 @@ func TestGRPCAbsentFailedTLS(t *testing.T) { }, }, registry, promslog.NewNopLogger()) - if result { - t.Fatalf("GRPC probe succeeded, should have failed") + expectedResult := ProbeFailure("Can't connect to the grpc server") + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("Test unexpected result: expected %s, got %s", expectedResult, result) } mfs, err := registry.Gather() diff --git a/prober/handler.go b/prober/handler.go index b8112eaf..1abfd584 100644 --- a/prober/handler.go +++ b/prober/handler.go @@ -125,18 +125,18 @@ func Handler(w http.ResponseWriter, r *http.Request, c *config.Config, logger *s registry := prometheus.NewRegistry() registry.MustRegister(probeSuccessGauge) registry.MustRegister(probeDurationGauge) - success := prober(ctx, target, module, registry, slLogger) + probeResult := prober(ctx, target, module, registry, slLogger) duration := time.Since(start).Seconds() probeDurationGauge.Set(duration) - if success { + probeResult.log(slLogger, duration) + if probeResult.success { probeSuccessGauge.Set(1) - slLogger.Info("Probe succeeded", "duration_seconds", duration) } else { - slLogger.Error("Probe failed", "duration_seconds", duration) + registry.MustRegister(probeResult.failureInfoGauge()) } debugOutput := DebugOutput(&module, &sl.buffer, registry) - rh.Add(moduleName, target, debugOutput, success) + rh.Add(moduleName, target, debugOutput, probeResult.success) if r.URL.Query().Get("debug") == "true" { w.Header().Set("Content-Type", "text/plain") diff --git a/prober/http.go b/prober/http.go index 28243116..4cdf6a3c 100644 --- a/prober/http.go +++ b/prober/http.go @@ -45,38 +45,36 @@ import ( "github.com/prometheus/blackbox_exporter/config" ) -func matchRegularExpressions(reader io.Reader, httpConfig config.HTTPProbe, logger *slog.Logger) bool { +func matchRegularExpressions(reader io.Reader, httpConfig config.HTTPProbe, logger *slog.Logger) ProbeResult { body, err := io.ReadAll(reader) if err != nil { - logger.Error("Error reading HTTP body", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error reading HTTP body") } for _, expression := range httpConfig.FailIfBodyMatchesRegexp { if expression.Match(body) { - logger.Error("Body matched regular expression", "regexp", expression) - return false + return ProbeFailure("Body matched regular expression", "regexp", expression.String()) } } for _, expression := range httpConfig.FailIfBodyNotMatchesRegexp { if !expression.Match(body) { - logger.Error("Body did not match regular expression", "regexp", expression) - return false + return ProbeFailure("Body did not match regular expression", "regexp", expression.String()) } } - return true + return ProbeSuccess() } -func matchCELExpressions(ctx context.Context, reader io.Reader, httpConfig config.HTTPProbe, logger *slog.Logger) bool { +func matchCELExpressions(ctx context.Context, reader io.Reader, httpConfig config.HTTPProbe, logger *slog.Logger) ProbeResult { body, err := io.ReadAll(reader) if err != nil { - logger.Error("Error reading HTTP body", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error reading HTTP body") } var bodyJSON any if err := json.Unmarshal(body, &bodyJSON); err != nil { - logger.Error("Error unmarshalling HTTP body to JSON", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error unmarshalling HTTP body to JSON") } evalPayload := map[string]interface{}{ @@ -86,45 +84,43 @@ func matchCELExpressions(ctx context.Context, reader io.Reader, httpConfig confi if httpConfig.FailIfBodyJsonMatchesCEL != nil { result, details, err := httpConfig.FailIfBodyJsonMatchesCEL.ContextEval(ctx, evalPayload) if err != nil { - logger.Error("Error evaluating CEL expression", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error evaluating CEL expression") } if result.Type() != cel.BoolType { - logger.Error("CEL evaluation result is not a boolean", "details", details) - return false + logger.Info("CEL evaluation details", "details", details) + return ProbeFailure("CEL evaluation result is not a boolean") } if result.Type() == cel.BoolType && result.Value().(bool) { - logger.Error("Body matched CEL expression", "expression", httpConfig.FailIfBodyJsonMatchesCEL.Expression) - return false + return ProbeFailure("Body matched CEL expression", "expression", httpConfig.FailIfBodyJsonMatchesCEL.Expression) } } if httpConfig.FailIfBodyJsonNotMatchesCEL != nil { result, details, err := httpConfig.FailIfBodyJsonNotMatchesCEL.ContextEval(ctx, evalPayload) if err != nil { - logger.Error("Error evaluating CEL expression", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error evaluating CEL expression") } if result.Type() != cel.BoolType { - logger.Error("CEL evaluation result is not a boolean", "details", details) - return false + logger.Info("CEL evaluation details", "details", details) + return ProbeFailure("CEL evaluation result is not a boolean") + } if result.Type() == cel.BoolType && !result.Value().(bool) { - logger.Error("Body did not match CEL expression", "expression", httpConfig.FailIfBodyJsonNotMatchesCEL.Expression) - return false + return ProbeFailure("Body did not match CEL expression", "expression", httpConfig.FailIfBodyJsonNotMatchesCEL.Expression) } } - return true + return ProbeSuccess() } -func matchRegularExpressionsOnHeaders(header http.Header, httpConfig config.HTTPProbe, logger *slog.Logger) bool { +func matchRegularExpressionsOnHeaders(header http.Header, httpConfig config.HTTPProbe, logger *slog.Logger) ProbeResult { for _, headerMatchSpec := range httpConfig.FailIfHeaderMatchesRegexp { values := header[textproto.CanonicalMIMEHeaderKey(headerMatchSpec.Header)] if len(values) == 0 { if !headerMatchSpec.AllowMissing { - logger.Error("Missing required header", "header", headerMatchSpec.Header) - return false + return ProbeFailure("Missing required header", "header", headerMatchSpec.Header) } else { continue // No need to match any regex on missing headers. } @@ -132,9 +128,8 @@ func matchRegularExpressionsOnHeaders(header http.Header, httpConfig config.HTTP for _, val := range values { if headerMatchSpec.Regexp.MatchString(val) { - logger.Error("Header matched regular expression", "header", headerMatchSpec.Header, - "regexp", headerMatchSpec.Regexp, "value_count", len(values)) - return false + return ProbeFailure("Header matched regular expression", "header", headerMatchSpec.Header, + "regexp", headerMatchSpec.Regexp.String(), "value_count", strconv.Itoa(len(values))) } } } @@ -142,8 +137,7 @@ func matchRegularExpressionsOnHeaders(header http.Header, httpConfig config.HTTP values := header[textproto.CanonicalMIMEHeaderKey(headerMatchSpec.Header)] if len(values) == 0 { if !headerMatchSpec.AllowMissing { - logger.Error("Missing required header", "header", headerMatchSpec.Header) - return false + return ProbeFailure("Missing required header", "header", headerMatchSpec.Header) } else { continue // No need to match any regex on missing headers. } @@ -159,13 +153,12 @@ func matchRegularExpressionsOnHeaders(header http.Header, httpConfig config.HTTP } if !anyHeaderValueMatched { - logger.Error("Header did not match regular expression", "header", headerMatchSpec.Header, - "regexp", headerMatchSpec.Regexp, "value_count", len(values)) - return false + return ProbeFailure("Header did not match regular expression", "header", headerMatchSpec.Header, + "regexp", headerMatchSpec.Regexp.String(), "value_count", strconv.Itoa(len(values))) } } - return true + return ProbeSuccess() } // roundTripTrace holds timings for a single HTTP roundtrip. @@ -288,7 +281,7 @@ func (bc *byteCounter) Read(p []byte) (int, error) { var userAgentDefaultHeader = fmt.Sprintf("Blackbox Exporter/%s", version.Version) -func ProbeHTTP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (success bool) { +func ProbeHTTP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (result ProbeResult) { var redirects int var ( durationGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ @@ -382,8 +375,8 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr targetURL, err := url.Parse(target) if err != nil { - logger.Error("Could not parse target URL", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Could not parse target URL") } targetHost := targetURL.Hostname() @@ -392,11 +385,11 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr var ip *net.IPAddr if !module.HTTP.SkipResolvePhaseWithProxy || module.HTTP.HTTPClientConfig.ProxyURL.URL == nil || module.HTTP.HTTPClientConfig.ProxyFromEnvironment { var lookupTime float64 - ip, lookupTime, err = chooseProtocol(ctx, module.HTTP.IPProtocol, module.HTTP.IPProtocolFallback, targetHost, registry, logger) + var resolveResult ProbeResult + ip, lookupTime, resolveResult = chooseProtocol(ctx, module.HTTP.IPProtocol, module.HTTP.IPProtocolFallback, targetHost, registry, logger) durationGaugeVec.WithLabelValues("resolve").Add(lookupTime) - if err != nil { - logger.Error("Error resolving address", "err", err) - return false + if !resolveResult.success { + return resolveResult } } @@ -417,21 +410,21 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr } client, err := pconfig.NewClientFromConfig(httpClientConfig, "http_probe", pconfig.WithKeepAlivesDisabled()) if err != nil { - logger.Error("Error generating HTTP client", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error generating HTTP client") } httpClientConfig.TLSConfig.ServerName = "" noServerName, err := pconfig.NewRoundTripperFromConfig(httpClientConfig, "http_probe", pconfig.WithKeepAlivesDisabled()) if err != nil { - logger.Error("Error generating HTTP client without ServerName", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error generating HTTP client without ServerName") } jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) if err != nil { - logger.Error("Error generating cookiejar", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error generating cookiejar") } client.Jar = jar @@ -534,7 +527,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr if resp == nil { resp = &http.Response{} if err != nil { - logger.Error("Error for HTTP request", "err", err) + logger.Error("Error for HTTP request", "err", err.Error()) + result = ProbeFailure("HTTP request failed") + // no return here, since there are cases where an error here + // might be acceptable after all. } } else { requestErrored := (err != nil) @@ -543,23 +539,23 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr if len(httpConfig.ValidStatusCodes) != 0 { for _, code := range httpConfig.ValidStatusCodes { if resp.StatusCode == code { - success = true + result = ProbeSuccess() break } } - if !success { - logger.Info("Invalid HTTP response status code", "status_code", resp.StatusCode, - "valid_status_codes", fmt.Sprintf("%v", httpConfig.ValidStatusCodes)) + if !result.success { + logger.Info("Valid status codes", "codes", httpConfig.ValidStatusCodes) + result = ProbeFailure("Invalid HTTP response status code", "status_code", strconv.Itoa(resp.StatusCode)) } } else if 200 <= resp.StatusCode && resp.StatusCode < 300 { - success = true + result = ProbeSuccess() } else { - logger.Info("Invalid HTTP response status code, wanted 2xx", "status_code", resp.StatusCode) + result = ProbeFailure("Invalid HTTP response status code, wanted 2xx", "status_code", strconv.Itoa(resp.StatusCode)) } - if success && (len(httpConfig.FailIfHeaderMatchesRegexp) > 0 || len(httpConfig.FailIfHeaderNotMatchesRegexp) > 0) { - success = matchRegularExpressionsOnHeaders(resp.Header, httpConfig, logger) - if success { + if result.success && (len(httpConfig.FailIfHeaderMatchesRegexp) > 0 || len(httpConfig.FailIfHeaderNotMatchesRegexp) > 0) { + result = matchRegularExpressionsOnHeaders(resp.Header, httpConfig, logger) + if result.success { probeFailedDueToRegex.Set(0) } else { probeFailedDueToRegex.Set(1) @@ -572,8 +568,8 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr if httpConfig.Compression != "" { dec, err := getDecompressionReader(httpConfig.Compression, resp.Body) if err != nil { - logger.Info("Failed to get decompressor for HTTP response body", "err", err) - success = false + logger.Error(err.Error()) + result = ProbeFailure("Failed to get decompressor for HTTP response body") } else if dec != nil { // Since we are replacing the original resp.Body with the decoder, we need to make sure // we close the original body. We cannot close it right away because the decompressor @@ -601,18 +597,18 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr byteCounter := &byteCounter{ReadCloser: resp.Body} - if success && (len(httpConfig.FailIfBodyMatchesRegexp) > 0 || len(httpConfig.FailIfBodyNotMatchesRegexp) > 0) { - success = matchRegularExpressions(byteCounter, httpConfig, logger) - if success { + if result.success && (len(httpConfig.FailIfBodyMatchesRegexp) > 0 || len(httpConfig.FailIfBodyNotMatchesRegexp) > 0) { + result = matchRegularExpressions(byteCounter, httpConfig, logger) + if result.success { probeFailedDueToRegex.Set(0) } else { probeFailedDueToRegex.Set(1) } } - if success && (httpConfig.FailIfBodyJsonMatchesCEL != nil || httpConfig.FailIfBodyJsonNotMatchesCEL != nil) { - success = matchCELExpressions(ctx, byteCounter, httpConfig, logger) - if success { + if result.success && (httpConfig.FailIfBodyJsonMatchesCEL != nil || httpConfig.FailIfBodyJsonNotMatchesCEL != nil) { + result = matchCELExpressions(ctx, byteCounter, httpConfig, logger) + if result.success { probeFailedDueToCEL.Set(0) } else { probeFailedDueToCEL.Set(1) @@ -622,8 +618,8 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr if !requestErrored { _, err = io.Copy(io.Discard, byteCounter) if err != nil { - logger.Info("Failed to read HTTP response body", "err", err) - success = false + logger.Error(err.Error()) + result = ProbeFailure("Failed to read HTTP response body") } respBodyBytes = byteCounter.n @@ -661,8 +657,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr } } if !found { - logger.Error("Invalid HTTP version number", "version", resp.Proto) - success = false + result = ProbeFailure("Invalid HTTP version number", "version", resp.Proto) } } } @@ -721,18 +716,17 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(resp.TLS).Unix())) probeSSLLastInformation.WithLabelValues(getFingerprint(resp.TLS), getSubject(resp.TLS), getIssuer(resp.TLS), getDNSNames(resp.TLS), getSerialNumber(resp.TLS)).Set(1) if httpConfig.FailIfSSL { - logger.Error("Final request was over SSL") - success = false + result = ProbeFailure("Final request was over SSL") } - } else if httpConfig.FailIfNotSSL && success { - logger.Error("Final request was not over SSL") - success = false + } else if httpConfig.FailIfNotSSL && result.success { + result = ProbeFailure("Final request was not over SSL") } statusCodeGauge.Set(float64(resp.StatusCode)) contentLengthGauge.Set(float64(resp.ContentLength)) bodyUncompressedLengthGauge.Set(float64(respBodyBytes)) redirectsGauge.Set(float64(redirects)) + return } diff --git a/prober/http_test.go b/prober/http_test.go index 6922418f..48ae0601 100644 --- a/prober/http_test.go +++ b/prober/http_test.go @@ -23,13 +23,13 @@ import ( "encoding/pem" "fmt" "io" - "log/slog" "net" "net/http" "net/http/httptest" "net/textproto" "net/url" "os" + "reflect" "strconv" "strings" "testing" @@ -47,18 +47,18 @@ func TestHTTPStatusCodes(t *testing.T) { tests := []struct { StatusCode int ValidStatusCodes []int - ShouldSucceed bool + expectedResult ProbeResult }{ - {200, []int{}, true}, - {201, []int{}, true}, - {299, []int{}, true}, - {300, []int{}, false}, - {404, []int{}, false}, - {404, []int{200, 404}, true}, - {200, []int{200, 404}, true}, - {201, []int{200, 404}, false}, - {404, []int{404}, true}, - {200, []int{404}, false}, + {200, []int{}, ProbeSuccess()}, + {201, []int{}, ProbeSuccess()}, + {299, []int{}, ProbeSuccess()}, + {300, []int{}, ProbeFailure("Invalid HTTP response status code, wanted 2xx", "status_code", "300")}, + {404, []int{}, ProbeFailure("Invalid HTTP response status code, wanted 2xx", "status_code", "404")}, + {404, []int{200, 404}, ProbeSuccess()}, + {200, []int{200, 404}, ProbeSuccess()}, + {201, []int{200, 404}, ProbeFailure("Invalid HTTP response status code", "status_code", "201")}, + {404, []int{404}, ProbeSuccess()}, + {200, []int{404}, ProbeFailure("Invalid HTTP response status code", "status_code", "200")}, } for i, test := range tests { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -71,22 +71,25 @@ func TestHTTPStatusCodes(t *testing.T) { defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, ValidStatusCodes: test.ValidStatusCodes}}, registry, promslog.NewNopLogger()) - body := recorder.Body.String() - if result != test.ShouldSucceed { - t.Fatalf("Test %d had unexpected result: %s", i, body) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Fatalf("Test %d had unexpected result: expected %s got %s", i, test.expectedResult, result) + body := recorder.Body.String() + t.Log(body) } + body := recorder.Body.String() + t.Log(body) } } func TestValidHTTPVersion(t *testing.T) { tests := []struct { ValidHTTPVersions []string - ShouldSucceed bool + expectedResult ProbeResult }{ - {[]string{}, true}, - {[]string{"HTTP/1.1"}, true}, - {[]string{"HTTP/1.1", "HTTP/2.0"}, true}, - {[]string{"HTTP/2.0"}, false}, + {[]string{}, ProbeSuccess()}, + {[]string{"HTTP/1.1"}, ProbeSuccess()}, + {[]string{"HTTP/1.1", "HTTP/2.0"}, ProbeSuccess()}, + {[]string{"HTTP/2.0"}, ProbeFailure("Invalid HTTP version number", "version", "HTTP/1.1")}, } for i, test := range tests { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -100,8 +103,8 @@ func TestValidHTTPVersion(t *testing.T) { ValidHTTPVersions: test.ValidHTTPVersions, }}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if result != test.ShouldSucceed { - t.Fatalf("Test %v had unexpected result: %s", i, body) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Fatalf("Test %d had unexpected result: expected %s, got %s. %s", i, test.expectedResult, result, body) } } } @@ -112,7 +115,7 @@ func TestContentLength(t *testing.T) { contentLength int uncompressedBodyLength int handler http.HandlerFunc - expectFailure bool + expectedResult ProbeResult } testmsg := []byte(strings.Repeat("hello world", 10)) @@ -121,6 +124,7 @@ func TestContentLength(t *testing.T) { testcases := map[string]testdata{ "identity": { + expectedResult: ProbeSuccess(), msg: testmsg, contentLength: len(testmsg), uncompressedBodyLength: len(testmsg), @@ -132,6 +136,7 @@ func TestContentLength(t *testing.T) { }, "no content-encoding": { + expectedResult: ProbeSuccess(), msg: testmsg, contentLength: len(testmsg), uncompressedBodyLength: len(testmsg), @@ -143,6 +148,7 @@ func TestContentLength(t *testing.T) { // Unknown Content-Encoding, we should let this pass thru. "unknown content-encoding": { + expectedResult: ProbeSuccess(), msg: testmsg, contentLength: len(testmsg), uncompressedBodyLength: len(testmsg), @@ -155,7 +161,7 @@ func TestContentLength(t *testing.T) { // 401 response, verify that the content-length is still computed correctly. "401": { - expectFailure: true, + expectedResult: ProbeFailure("Invalid HTTP response status code, wanted 2xx", "status_code", "404"), msg: notfoundMsg, contentLength: len(notfoundMsg), uncompressedBodyLength: len(notfoundMsg), @@ -174,6 +180,7 @@ func TestContentLength(t *testing.T) { fw.Write([]byte(msg)) fw.Close() return testdata{ + expectedResult: ProbeSuccess(), msg: msg, contentLength: len(buf.Bytes()), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(buf.Bytes()), // No decompression. @@ -194,6 +201,7 @@ func TestContentLength(t *testing.T) { fw.Write([]byte(msg)) fw.Close() return testdata{ + expectedResult: ProbeSuccess(), msg: msg, contentLength: len(buf.Bytes()), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(buf.Bytes()), // No decompression. @@ -213,6 +221,7 @@ func TestContentLength(t *testing.T) { gw.Write([]byte(msg)) gw.Close() return testdata{ + expectedResult: ProbeSuccess(), msg: msg, contentLength: len(buf.Bytes()), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(buf.Bytes()), // No decompression. @@ -243,10 +252,10 @@ func TestContentLength(t *testing.T) { }, registry, promslog.New(&promslog.Config{Writer: &logbuf})) - if !tc.expectFailure && !result { - t.Fatalf("probe failed unexpectedly: %s", logbuf.String()) - } else if tc.expectFailure && result { - t.Fatalf("probe succeeded unexpectedly: %s", logbuf.String()) + + if !reflect.DeepEqual(result, tc.expectedResult) { + t.Fatalf("Test had unexpected result: expected %v, got %v.", tc.expectedResult, result) + t.Log(logbuf.String()) } mfs, err := registry.Gather() @@ -272,7 +281,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { contentLength int uncompressedBodyLength int handler http.HandlerFunc - expectFailure bool + expectedResult ProbeResult httpConfig config.HTTPProbe } @@ -286,6 +295,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ + expectedResult: ProbeSuccess(), contentLength: buf.Len(), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(msg), handler: func(w http.ResponseWriter, r *http.Request) { @@ -307,6 +317,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ + expectedResult: ProbeSuccess(), contentLength: len(buf.Bytes()), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(msg), handler: func(w http.ResponseWriter, r *http.Request) { @@ -329,6 +340,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ + expectedResult: ProbeSuccess(), contentLength: len(buf.Bytes()), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(msg), handler: func(w http.ResponseWriter, r *http.Request) { @@ -344,6 +356,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { }(), "identity": { + expectedResult: ProbeSuccess(), contentLength: len(testmsg), uncompressedBodyLength: len(testmsg), handler: func(w http.ResponseWriter, r *http.Request) { @@ -367,7 +380,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ - expectFailure: true, + expectedResult: ProbeFailure("Failed to read HTTP response body"), contentLength: buf.Len(), // Content length is the length of the compressed buffer. uncompressedBodyLength: 0, handler: func(w http.ResponseWriter, r *http.Request) { @@ -389,7 +402,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ - expectFailure: false, + expectedResult: ProbeSuccess(), contentLength: buf.Len(), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(msg), handler: func(w http.ResponseWriter, r *http.Request) { @@ -414,7 +427,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ - expectFailure: false, + expectedResult: ProbeSuccess(), contentLength: buf.Len(), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(msg), handler: func(w http.ResponseWriter, r *http.Request) { @@ -439,7 +452,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ - expectFailure: false, + expectedResult: ProbeSuccess(), contentLength: buf.Len(), // Content length is the length of the compressed buffer. uncompressedBodyLength: len(msg), handler: func(w http.ResponseWriter, r *http.Request) { @@ -464,7 +477,7 @@ func TestHandlingOfCompressionSetting(t *testing.T) { enc.Write(msg) enc.Close() return testdata{ - expectFailure: false, + expectedResult: ProbeSuccess(), contentLength: buf.Len(), uncompressedBodyLength: buf.Len(), // content won't be uncompressed handler: func(w http.ResponseWriter, r *http.Request) { @@ -497,10 +510,8 @@ func TestHandlingOfCompressionSetting(t *testing.T) { }, registry, promslog.New(&promslog.Config{Writer: &logbuf})) - if !tc.expectFailure && !result { - t.Fatalf("probe failed unexpectedly: %s", logbuf.String()) - } else if tc.expectFailure && result { - t.Fatalf("probe succeeded unexpectedly: %s", logbuf.String()) + if !reflect.DeepEqual(result, tc.expectedResult) { + t.Fatalf("Test had unexpected result: expected %s, got %s", tc.expectedResult, result) } mfs, err := registry.Gather() @@ -534,34 +545,36 @@ func TestMaxResponseLength(t *testing.T) { target string compression string expectedMetrics map[string]float64 - expectFailure bool + expectedResult ProbeResult }{ "short": { - target: "/short", + expectedResult: ProbeSuccess(), + target: "/short", expectedMetrics: map[string]float64{ "probe_http_uncompressed_body_length": float64(max - 1), "probe_http_content_length": float64(max - 1), }, }, "long": { - target: "/long", - expectFailure: true, + expectedResult: ProbeFailure("Failed to read HTTP response body"), + target: "/long", expectedMetrics: map[string]float64{ "probe_http_content_length": float64(max + 1), }, }, "short compressed": { - target: "/short-compressed", - compression: "gzip", + expectedResult: ProbeSuccess(), + target: "/short-compressed", + compression: "gzip", expectedMetrics: map[string]float64{ "probe_http_content_length": float64(shortGzippedPayload.Len()), "probe_http_uncompressed_body_length": float64(max - 1), }, }, "long compressed": { - target: "/long-compressed", - compression: "gzip", - expectFailure: true, + expectedResult: ProbeFailure("Failed to read HTTP response body"), + target: "/long-compressed", + compression: "gzip", expectedMetrics: map[string]float64{ "probe_http_content_length": float64(longGzippedPayload.Len()), "probe_http_uncompressed_body_length": max, // it should stop decompressing at max bytes @@ -620,11 +633,8 @@ func TestMaxResponseLength(t *testing.T) { promslog.NewNopLogger(), ) - switch { - case tc.expectFailure && result: - t.Fatalf("test passed unexpectedly") - case !tc.expectFailure && !result: - t.Fatalf("test failed unexpectedly") + if !reflect.DeepEqual(result, tc.expectedResult) { + t.Fatalf("Test had unexpected result: expected %s, got %s", tc.expectedResult, result) } mfs, err := registry.Gather() @@ -652,8 +662,8 @@ func TestRedirectFollowed(t *testing.T) { defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, HTTPClientConfig: pconfig.DefaultHTTPClientConfig}}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("Redirect test failed unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Redirect test failed unexpectedly, got %s %s", result, body) } mfs, err := registry.Gather() @@ -680,8 +690,8 @@ func TestRedirectNotFollowed(t *testing.T) { result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, HTTPClientConfig: pconfig.HTTPClientConfig{FollowRedirects: false}, ValidStatusCodes: []int{302}}}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("Redirect test failed unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Redirect test failed unexpectedly, got %s %s", result, body) } } @@ -728,8 +738,8 @@ func TestRedirectionLimit(t *testing.T) { config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, HTTPClientConfig: pconfig.DefaultHTTPClientConfig}}, registry, promslog.NewNopLogger()) - if result { - t.Fatalf("Probe succeeded unexpectedly") + if result.success { + t.Fatalf("Probe succeeded unexpectedly, got %s", result) } if tooManyRedirects { @@ -763,8 +773,8 @@ func TestPost(t *testing.T) { result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, Method: "POST"}}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("Post test failed unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Post test failed unexpectedly, got %s %s", result, body) } } @@ -786,8 +796,8 @@ func TestBasicAuth(t *testing.T) { }, }}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("HTTP probe failed, got %s", body) + if !result.success { + t.Fatalf("HTTP probe failed, got %s %s", result, body) } } @@ -808,8 +818,8 @@ func TestBearerToken(t *testing.T) { }, }}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("HTTP probe failed, got %s", body) + if !result.success { + t.Fatalf("HTTP probe failed, got %s %s", result, body) } } @@ -825,8 +835,9 @@ func TestFailIfNotSSL(t *testing.T) { result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfNotSSL: true}}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if result { - t.Fatalf("Fail if not SSL test succeeded unexpectedly, got %s", body) + expectedResult := ProbeFailure("Final request was not over SSL") + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("Test had unexpected result: expected %s, got %s %s", expectedResult, result, body) } mfs, err := registry.Gather() if err != nil { @@ -838,37 +849,8 @@ func TestFailIfNotSSL(t *testing.T) { checkRegistryResults(expectedResults, mfs, t) } -type logRecorder struct { - msgs map[string]bool - next *slog.Logger -} - -func (lr *logRecorder) Enabled(ctx context.Context, level slog.Level) bool { - return lr.next.Enabled(ctx, level) -} - -func (lr *logRecorder) Handle(ctx context.Context, r slog.Record) error { - if lr.msgs == nil { - lr.msgs = make(map[string]bool) - } - - lr.msgs[r.Message] = true - return nil -} - -func (lr *logRecorder) WithAttrs(attrs []slog.Attr) slog.Handler { - lr.next = slog.New(lr.next.Handler().WithAttrs(attrs)) - return lr -} - -func (lr *logRecorder) WithGroup(name string) slog.Handler { - lr.next = slog.New(lr.next.Handler().WithGroup(name)) - return lr -} - func TestFailIfNotSSLLogMsg(t *testing.T) { const ( - Msg = "Final request was not over SSL" Timeout = time.Second * 10 ) @@ -895,42 +877,34 @@ func TestFailIfNotSSLLogMsg(t *testing.T) { badServerURL := fmt.Sprintf("http://%s/", listener.Addr().String()) for title, tc := range map[string]struct { - Config config.Module - URL string - Success bool - MessageExpected bool + Config config.Module + URL string + expectedResult ProbeResult }{ "SSL expected, message": { - Config: config.Module{HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfNotSSL: true}}, - URL: goodServer.URL, - Success: false, - MessageExpected: true, + Config: config.Module{HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfNotSSL: true}}, + URL: goodServer.URL, + expectedResult: ProbeFailure("Final request was not over SSL"), }, "No SSL expected, no message": { - Config: config.Module{HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfNotSSL: false}}, - URL: goodServer.URL, - Success: true, - MessageExpected: false, + Config: config.Module{HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfNotSSL: false}}, + URL: goodServer.URL, + expectedResult: ProbeSuccess(), }, "SSL expected, no message": { - Config: config.Module{HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfNotSSL: true}}, - URL: badServerURL, - Success: false, - MessageExpected: false, + Config: config.Module{HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfNotSSL: true}}, + URL: badServerURL, + expectedResult: ProbeFailure("HTTP request failed"), }, } { t.Run(title, func(t *testing.T) { - recorder := logRecorder{next: promslog.NewNopLogger()} registry := prometheus.NewRegistry() testCTX, cancel := context.WithTimeout(context.Background(), Timeout) defer cancel() - result := ProbeHTTP(testCTX, tc.URL, tc.Config, registry, slog.New(&recorder)) - if result != tc.Success { - t.Fatalf("Expected success=%v, got=%v", tc.Success, result) - } - if seen := recorder.msgs[Msg]; seen != tc.MessageExpected { - t.Fatalf("SSL message expected=%v, seen=%v", tc.MessageExpected, seen) + result := ProbeHTTP(testCTX, tc.URL, tc.Config, registry, promslog.NewNopLogger()) + if !reflect.DeepEqual(result, tc.expectedResult) { + t.Fatalf("Test had unexpected result: expected %s, got %s", tc.expectedResult, result) } }) } @@ -940,67 +914,67 @@ func TestFailIfBodyMatchesCEL(t *testing.T) { testcases := map[string]struct { respBody string celExpression string - expectedResult bool + expectedResult ProbeResult }{ "celExpression matches": { respBody: `{"foo": {"bar": "baz"}}`, celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Body matched CEL expression", "expression", "body.foo.bar == 'baz'"), }, "celExpression does not match": { respBody: `{"foo": {"bar": "baz"}}`, celExpression: "body.foo.bar == 'qux'", - expectedResult: true, + expectedResult: ProbeSuccess(), }, "celExpression does not match with empty body": { respBody: `{}`, celExpression: "body.foo.bar == 'qux'", - expectedResult: false, + expectedResult: ProbeFailure("Error evaluating CEL expression"), }, "celExpression result not boolean": { respBody: `{"foo": {"bar": "baz"}}`, celExpression: "body.foo.bar", - expectedResult: false, + expectedResult: ProbeFailure("CEL evaluation result is not a boolean"), }, "body is not json": { respBody: "hello world", celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Error unmarshalling HTTP body to JSON"), }, "body is empty json object": { respBody: "{}", celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Error evaluating CEL expression"), }, "body is json string": { respBody: `"foo"`, celExpression: "body == 'foo'", - expectedResult: false, + expectedResult: ProbeFailure("Body matched CEL expression", "expression", "body == 'foo'"), }, "body is json list": { respBody: `["foo","bar","baz"]`, celExpression: "body[2] == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Body matched CEL expression", "expression", "body[2] == 'baz'"), }, "body is json boolean": { respBody: `true`, celExpression: "body", - expectedResult: false, + expectedResult: ProbeFailure("Body matched CEL expression", "expression", "body"), }, "body is empty": { respBody: "", celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Error unmarshalling HTTP body to JSON"), }, "body returns emoji": { respBody: "🤠🤠🤠", celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Error unmarshalling HTTP body to JSON"), }, "body returns json with emojis": { respBody: `{"foo": {"bar": "🤠🤠🤠"}}`, celExpression: "body.foo.bar == '😿😿😿'", - expectedResult: true, + expectedResult: ProbeSuccess(), }, } @@ -1013,15 +987,12 @@ func TestFailIfBodyMatchesCEL(t *testing.T) { celProgram := config.MustNewCELProgram(testcase.celExpression) - recorder := httptest.NewRecorder() registry := prometheus.NewRegistry() testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyJsonMatchesCEL: &celProgram}}, registry, promslog.NewNopLogger()) - if testcase.expectedResult && !result { - t.Fatalf("CEL test failed unexpectedly, got %s", recorder.Body.String()) - } else if !testcase.expectedResult && result { - t.Fatalf("CEL test succeeded unexpectedly, got %s", recorder.Body.String()) + if !reflect.DeepEqual(result, testcase.expectedResult) { + t.Fatalf("CEL Test had unexpected result: expected %s, got %s", testcase.expectedResult, result) } mfs, err := registry.Gather() if err != nil { @@ -1034,7 +1005,7 @@ func TestFailIfBodyMatchesCEL(t *testing.T) { return 0 } expectedResults := map[string]float64{ - "probe_failed_due_to_cel": boolToFloat(!testcase.expectedResult), + "probe_failed_due_to_cel": boolToFloat(!testcase.expectedResult.success), "probe_http_content_length": float64(len(testcase.respBody)), // Issue #673: check that this is correctly populated when using regex validations. "probe_http_uncompressed_body_length": float64(len(testcase.respBody)), // Issue #673, see above. } @@ -1047,67 +1018,67 @@ func TestFailIfBodyNotMatchesCEL(t *testing.T) { testcases := map[string]struct { respBody string celExpression string - expectedResult bool + expectedResult ProbeResult }{ "cel matches": { respBody: `{"foo": {"bar": "baz"}}`, celExpression: "body.foo.bar == 'baz'", - expectedResult: true, + expectedResult: ProbeSuccess(), }, "cel does not match": { respBody: `{"foo": {"bar": "baz"}}`, celExpression: "body.foo.bar == 'qux'", - expectedResult: false, + expectedResult: ProbeFailure("Body did not match CEL expression", "expression", "body.foo.bar == 'qux'"), }, "cel does not match with empty body": { respBody: `{}`, celExpression: "body.foo.bar == 'qux'", - expectedResult: false, + expectedResult: ProbeFailure("Error evaluating CEL expression"), }, "cel result not boolean": { respBody: `{"foo": {"bar": "baz"}}`, celExpression: "body.foo.bar", - expectedResult: false, + expectedResult: ProbeFailure("CEL evaluation result is not a boolean"), }, "body is not json": { respBody: "hello world", celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Error unmarshalling HTTP body to JSON"), }, "body is empty json object": { respBody: "{}", celExpression: "!has(body.foo)", - expectedResult: true, + expectedResult: ProbeSuccess(), }, "body is json string": { respBody: `"foo"`, celExpression: "body == 'foo'", - expectedResult: true, + expectedResult: ProbeSuccess(), }, "body is json list": { respBody: `["foo","bar","baz"]`, celExpression: "body[2] == 'baz'", - expectedResult: true, + expectedResult: ProbeSuccess(), }, "body is json boolean": { respBody: `true`, celExpression: "body", - expectedResult: true, + expectedResult: ProbeSuccess(), }, "body is empty": { respBody: "", celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Error unmarshalling HTTP body to JSON"), }, "body returns emoji": { respBody: "🤠🤠🤠", celExpression: "body.foo.bar == 'baz'", - expectedResult: false, + expectedResult: ProbeFailure("Error unmarshalling HTTP body to JSON"), }, "body returns json with emojis": { respBody: `{"foo": {"bar": "🤠🤠🤠"}}`, celExpression: "body.foo.bar == '🤠🤠🤠'", - expectedResult: true, + expectedResult: ProbeSuccess(), }, } @@ -1120,15 +1091,12 @@ func TestFailIfBodyNotMatchesCEL(t *testing.T) { celProgram := config.MustNewCELProgram(testcase.celExpression) - recorder := httptest.NewRecorder() registry := prometheus.NewRegistry() testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyJsonNotMatchesCEL: &celProgram}}, registry, promslog.NewNopLogger()) - if testcase.expectedResult && !result { - t.Fatalf("CEL test failed unexpectedly, got %s", recorder.Body.String()) - } else if !testcase.expectedResult && result { - t.Fatalf("CEL test succeeded unexpectedly, got %s", recorder.Body.String()) + if !reflect.DeepEqual(result, testcase.expectedResult) { + t.Fatalf("CEL Test had unexpected result: expected %s, got %s", testcase.expectedResult, result) } mfs, err := registry.Gather() if err != nil { @@ -1141,7 +1109,7 @@ func TestFailIfBodyNotMatchesCEL(t *testing.T) { return 0 } expectedResults := map[string]float64{ - "probe_failed_due_to_cel": boolToFloat(!testcase.expectedResult), + "probe_failed_due_to_cel": boolToFloat(!testcase.expectedResult.success), } checkRegistryResults(expectedResults, mfs, t) }) @@ -1152,30 +1120,30 @@ func TestFailIfBodyMatchesRegexp(t *testing.T) { testcases := map[string]struct { respBody string regexps []config.Regexp - expectedResult bool + expectedResult ProbeResult }{ "one regex, match": { respBody: "Bad news: could not connect to database server", regexps: []config.Regexp{config.MustNewRegexp("could not connect to database")}, - expectedResult: false, + expectedResult: ProbeFailure("Body matched regular expression", "regexp", "could not connect to database"), }, "one regex, no match": { respBody: "Download the latest version here", regexps: []config.Regexp{config.MustNewRegexp("could not connect to database")}, - expectedResult: true, + expectedResult: ProbeSuccess(), }, "multiple regexes, match": { respBody: "internal error", regexps: []config.Regexp{config.MustNewRegexp("could not connect to database"), config.MustNewRegexp("internal error")}, - expectedResult: false, + expectedResult: ProbeFailure("Body matched regular expression", "regexp", "internal error"), }, "multiple regexes, no match": { respBody: "hello world", regexps: []config.Regexp{config.MustNewRegexp("could not connect to database"), config.MustNewRegexp("internal error")}, - expectedResult: true, + expectedResult: ProbeSuccess(), }, } @@ -1186,15 +1154,12 @@ func TestFailIfBodyMatchesRegexp(t *testing.T) { })) defer ts.Close() - recorder := httptest.NewRecorder() registry := prometheus.NewRegistry() testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyMatchesRegexp: testcase.regexps}}, registry, promslog.NewNopLogger()) - if testcase.expectedResult && !result { - t.Fatalf("Regexp test failed unexpectedly, got %s", recorder.Body.String()) - } else if !testcase.expectedResult && result { - t.Fatalf("Regexp test succeeded unexpectedly, got %s", recorder.Body.String()) + if !reflect.DeepEqual(result, testcase.expectedResult) { + t.Fatalf("Test had unexpected result: expected %s, got %s", testcase.expectedResult, result) } mfs, err := registry.Gather() if err != nil { @@ -1207,7 +1172,7 @@ func TestFailIfBodyMatchesRegexp(t *testing.T) { return 0 } expectedResults := map[string]float64{ - "probe_failed_due_to_regex": boolToFloat(!testcase.expectedResult), + "probe_failed_due_to_regex": boolToFloat(!testcase.expectedResult.success), "probe_http_content_length": float64(len(testcase.respBody)), // Issue #673: check that this is correctly populated when using regex validations. "probe_http_uncompressed_body_length": float64(len(testcase.respBody)), // Issue #673, see above. } @@ -1229,8 +1194,9 @@ func TestFailIfBodyNotMatchesRegexp(t *testing.T) { result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyNotMatchesRegexp: []config.Regexp{config.MustNewRegexp("Download the latest version here")}}}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if result { - t.Fatalf("Regexp test succeeded unexpectedly, got %s", body) + expectedResult := ProbeFailure("Body did not match regular expression", "regexp", "Download the latest version here") + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("Test had unexpected result: expected %s, got %s %s", expectedResult, result, body) } ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1243,8 +1209,8 @@ func TestFailIfBodyNotMatchesRegexp(t *testing.T) { result = ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyNotMatchesRegexp: []config.Regexp{config.MustNewRegexp("Download the latest version here")}}}, registry, promslog.NewNopLogger()) body = recorder.Body.String() - if !result { - t.Fatalf("Regexp test failed unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Regexp test failed unexpectedly, got %s %s", result, body) } // With multiple regexps configured, verify that any non-matching regexp @@ -1259,8 +1225,9 @@ func TestFailIfBodyNotMatchesRegexp(t *testing.T) { result = ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyNotMatchesRegexp: []config.Regexp{config.MustNewRegexp("Download the latest version here"), config.MustNewRegexp("Copyright 2015")}}}, registry, promslog.NewNopLogger()) body = recorder.Body.String() - if result { - t.Fatalf("Regexp test succeeded unexpectedly, got %s", body) + expectedResult = ProbeFailure("Body did not match regular expression", "regexp", "Copyright 2015") + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("Test had unexpected result: expected %s, got %s %s", expectedResult, result, body) } ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1273,26 +1240,81 @@ func TestFailIfBodyNotMatchesRegexp(t *testing.T) { result = ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyNotMatchesRegexp: []config.Regexp{config.MustNewRegexp("Download the latest version here"), config.MustNewRegexp("Copyright 2015")}}}, registry, promslog.NewNopLogger()) body = recorder.Body.String() - if !result { - t.Fatalf("Regexp test failed unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Regexp test failed unexpectedly, got %s %s", result, body) } } func TestFailIfHeaderMatchesRegexp(t *testing.T) { tests := []struct { - Rule config.HeaderMatch - Values []string - ShouldSucceed bool + Rule config.HeaderMatch + Values []string + expectedResult ProbeResult }{ - {config.HeaderMatch{"Content-Type", config.MustNewRegexp("text/javascript"), false}, []string{"text/javascript"}, false}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp("text/javascript"), false}, []string{"application/octet-stream"}, true}, - {config.HeaderMatch{"content-type", config.MustNewRegexp("text/javascript"), false}, []string{"application/octet-stream"}, true}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp(".*"), false}, []string{""}, false}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp(".*"), false}, []string{}, false}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp(".*"), true}, []string{""}, false}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp(".*"), true}, []string{}, true}, - {config.HeaderMatch{"Set-Cookie", config.MustNewRegexp(".*Domain=\\.example\\.com.*"), false}, []string{"gid=1; Expires=Tue, 19-Mar-2019 20:08:29 GMT; Domain=.example.com; Path=/"}, false}, - {config.HeaderMatch{"Set-Cookie", config.MustNewRegexp(".*Domain=\\.example\\.com.*"), false}, []string{"zz=4; expires=Mon, 01-Jan-1990 00:00:00 GMT; Domain=www.example.com; Path=/", "gid=1; Expires=Tue, 19-Mar-2019 20:08:29 GMT; Domain=.example.com; Path=/"}, false}, + { + config.HeaderMatch{Header: "Content-Type", + Regexp: config.MustNewRegexp("text/javascript"), + AllowMissing: false}, + []string{"text/javascript"}, + ProbeFailure("Header matched regular expression", "header", "Content-Type", "regexp", "text/javascript", "value_count", "1"), + }, + { + config.HeaderMatch{Header: "Content-Type", + Regexp: config.MustNewRegexp("text/javascript"), + AllowMissing: false}, + []string{"application/octet-stream"}, + ProbeSuccess(), + }, + { + config.HeaderMatch{Header: "content-type", + Regexp: config.MustNewRegexp("text/javascript"), + AllowMissing: false}, + []string{"application/octet-stream"}, + ProbeSuccess(), + }, + { + config.HeaderMatch{Header: "Content-Type", + Regexp: config.MustNewRegexp(".*"), + AllowMissing: false}, + []string{""}, + ProbeFailure("Header matched regular expression", "header", "Content-Type", "regexp", ".*", "value_count", "1"), + }, + { + config.HeaderMatch{Header: "Content-Type", + Regexp: config.MustNewRegexp(".*"), + AllowMissing: false}, + []string{}, + ProbeFailure("Missing required header", "header", "Content-Type"), + }, + { + config.HeaderMatch{Header: "Content-Type", + Regexp: config.MustNewRegexp(".*"), + AllowMissing: true}, + []string{""}, + ProbeFailure("Header matched regular expression", "header", "Content-Type", "regexp", ".*", "value_count", "1"), + }, + { + config.HeaderMatch{Header: "Content-Type", + Regexp: config.MustNewRegexp(".*"), + AllowMissing: true}, + []string{}, + ProbeSuccess(), + }, + { + config.HeaderMatch{Header: "Set-Cookie", + Regexp: config.MustNewRegexp(".*Domain=\\.example\\.com.*"), + AllowMissing: false}, + []string{"gid=1; Expires=Tue, 19-Mar-2019 20:08:29 GMT; Domain=.example.com; Path=/"}, + ProbeFailure("Header matched regular expression", "header", "Set-Cookie", "regexp", ".*Domain=\\.example\\.com.*", "value_count", "1"), + }, + { + config.HeaderMatch{Header: "Set-Cookie", + Regexp: config.MustNewRegexp(".*Domain=\\.example\\.com.*"), + AllowMissing: false}, + []string{"zz=4; expires=Mon, 01-Jan-1990 00:00:00 GMT; Domain=www.example.com; Path=/", + "gid=1; Expires=Tue, 19-Mar-2019 20:08:29 GMT; Domain=.example.com; Path=/"}, + ProbeFailure("Header matched regular expression", "header", "Set-Cookie", "regexp", ".*Domain=\\.example\\.com.*", "value_count", "2"), + }, } for i, test := range tests { @@ -1307,8 +1329,8 @@ func TestFailIfHeaderMatchesRegexp(t *testing.T) { defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfHeaderMatchesRegexp: []config.HeaderMatch{test.Rule}}}, registry, promslog.NewNopLogger()) - if result != test.ShouldSucceed { - t.Fatalf("Test %d had unexpected result: succeeded: %t, expected: %+v", i, result, test) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Fatalf("Test %d had unexpected result: expected %s, got %s", i, test.expectedResult, result) } mfs, err := registry.Gather() @@ -1319,7 +1341,7 @@ func TestFailIfHeaderMatchesRegexp(t *testing.T) { "probe_failed_due_to_regex": 1, } - if test.ShouldSucceed { + if test.expectedResult.success { expectedResults["probe_failed_due_to_regex"] = 0 } @@ -1329,18 +1351,77 @@ func TestFailIfHeaderMatchesRegexp(t *testing.T) { func TestFailIfHeaderNotMatchesRegexp(t *testing.T) { tests := []struct { - Rule config.HeaderMatch - Values []string - ShouldSucceed bool + Rule config.HeaderMatch + Values []string + expectedResult ProbeResult }{ - {config.HeaderMatch{"Content-Type", config.MustNewRegexp("text/javascript"), false}, []string{"text/javascript"}, true}, - {config.HeaderMatch{"content-type", config.MustNewRegexp("text/javascript"), false}, []string{"text/javascript"}, true}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp("text/javascript"), false}, []string{"application/octet-stream"}, false}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp(".*"), false}, []string{""}, true}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp(".*"), false}, []string{}, false}, - {config.HeaderMatch{"Content-Type", config.MustNewRegexp(".*"), true}, []string{}, true}, - {config.HeaderMatch{"Set-Cookie", config.MustNewRegexp(".*Domain=\\.example\\.com.*"), false}, []string{"zz=4; expires=Mon, 01-Jan-1990 00:00:00 GMT; Domain=www.example.com; Path=/"}, false}, - {config.HeaderMatch{"Set-Cookie", config.MustNewRegexp(".*Domain=\\.example\\.com.*"), false}, []string{"zz=4; expires=Mon, 01-Jan-1990 00:00:00 GMT; Domain=www.example.com; Path=/", "gid=1; Expires=Tue, 19-Mar-2019 20:08:29 GMT; Domain=.example.com; Path=/"}, true}, + { + config.HeaderMatch{ + Header: "Content-Type", + Regexp: config.MustNewRegexp("text/javascript"), + AllowMissing: false, + }, + + []string{"text/javascript"}, + ProbeSuccess(), + }, + { + config.HeaderMatch{ + Header: "content-type", + Regexp: config.MustNewRegexp("text/javascript"), + AllowMissing: false, + }, + []string{"text/javascript"}, + ProbeSuccess(), + }, + { + config.HeaderMatch{ + Header: "Content-Type", + Regexp: config.MustNewRegexp("text/javascript"), + AllowMissing: false, + }, + []string{"application/octet-stream"}, + ProbeFailure("Header did not match regular expression", "header", "Content-Type", "regexp", "text/javascript", "value_count", "1"), + }, + { + config.HeaderMatch{ + Header: "Content-Type", + Regexp: config.MustNewRegexp(".*"), + AllowMissing: false}, []string{""}, + ProbeSuccess(), + }, + { + config.HeaderMatch{ + Header: "Content-Type", + Regexp: config.MustNewRegexp(".*"), + AllowMissing: false}, + []string{}, + ProbeFailure("Missing required header", "header", "Content-Type"), + }, + { + config.HeaderMatch{ + Header: "Content-Type", + Regexp: config.MustNewRegexp(".*"), + AllowMissing: true}, + []string{}, + ProbeSuccess(), + }, + { + config.HeaderMatch{ + Header: "Set-Cookie", + Regexp: config.MustNewRegexp(".*Domain=\\.example\\.com.*"), + AllowMissing: false}, + []string{"zz=4; expires=Mon, 01-Jan-1990 00:00:00 GMT; Domain=www.example.com; Path=/"}, + ProbeFailure("Header did not match regular expression", "header", "Set-Cookie", "regexp", ".*Domain=\\.example\\.com.*", "value_count", "1"), + }, + { + config.HeaderMatch{ + Header: "Set-Cookie", + Regexp: config.MustNewRegexp(".*Domain=\\.example\\.com.*"), + AllowMissing: false}, + []string{"zz=4; expires=Mon, 01-Jan-1990 00:00:00 GMT; Domain=www.example.com; Path=/", "gid=1; Expires=Tue, 19-Mar-2019 20:08:29 GMT; Domain=.example.com; Path=/"}, + ProbeSuccess(), + }, } for i, test := range tests { @@ -1355,8 +1436,8 @@ func TestFailIfHeaderNotMatchesRegexp(t *testing.T) { defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfHeaderNotMatchesRegexp: []config.HeaderMatch{test.Rule}}}, registry, promslog.NewNopLogger()) - if result != test.ShouldSucceed { - t.Fatalf("Test %d had unexpected result: succeeded: %t, expected: %+v", i, result, test) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Fatalf("Test %d had unexpected result: expected %s, got %s", i, test.expectedResult, result) } mfs, err := registry.Gather() @@ -1367,7 +1448,7 @@ func TestFailIfHeaderNotMatchesRegexp(t *testing.T) { "probe_failed_due_to_regex": 1, } - if test.ShouldSucceed { + if test.expectedResult.success { expectedResults["probe_failed_due_to_regex"] = 0 } @@ -1403,8 +1484,8 @@ func TestHTTPHeaders(t *testing.T) { IPProtocolFallback: true, Headers: headers, }}, registry, promslog.NewNopLogger()) - if !result { - t.Fatalf("Probe failed unexpectedly.") + if !result.success { + t.Fatalf("Probe failed unexpectedly. %s", result) } } @@ -1425,8 +1506,8 @@ func TestFailIfSelfSignedCA(t *testing.T) { }, }}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if result { - t.Fatalf("Fail if selfsigned CA test succeeded unexpectedly, got %s", body) + if result.success { + t.Fatalf("Fail if selfsigned CA test succeeded unexpectedly, got %s %s", result, body) } mfs, err := registry.Gather() if err != nil { @@ -1455,8 +1536,8 @@ func TestSucceedIfSelfSignedCA(t *testing.T) { }, }}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("Fail if (not strict) selfsigned CA test fails unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Fail if (not strict) selfsigned CA test fails unexpectedly, got %s %s", result, body) } mfs, err := registry.Gather() if err != nil { @@ -1485,8 +1566,8 @@ func TestTLSConfigIsIgnoredForPlainHTTP(t *testing.T) { }, }}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("Fail if InsecureSkipVerify affects simple http fails unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Fail if InsecureSkipVerify affects simple http fails unexpectedly, got %s %s", result, body) } mfs, err := registry.Gather() if err != nil { @@ -1551,8 +1632,8 @@ func TestHTTPUsesTargetAsTLSServerName(t *testing.T) { url = strings.ReplaceAll(url, "[::1]", "localhost") result := ProbeHTTP(context.Background(), url, module, registry, promslog.NewNopLogger()) - if !result { - t.Fatalf("TLS probe failed unexpectedly") + if !result.success { + t.Fatalf("TLS probe failed unexpectedly, got %s", result) } } @@ -1571,8 +1652,8 @@ func TestRedirectToTLSHostWorks(t *testing.T) { defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, HTTPClientConfig: pconfig.DefaultHTTPClientConfig}}, registry, promslog.NewNopLogger()) - if !result { - t.Fatalf("Redirect test failed unexpectedly") + if !result.success { + t.Fatalf("Redirect test failed unexpectedly, got %s", result) } } @@ -1595,8 +1676,8 @@ func TestHTTPPhases(t *testing.T) { }, }}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("HTTP Phases test failed unexpectedly, got %s", body) + if !result.success { + t.Fatalf("HTTP Phases test failed unexpectedly, got %s %s", result, body) } mfs, err := registry.Gather() @@ -1645,8 +1726,8 @@ func TestCookieJar(t *testing.T) { defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, HTTPClientConfig: pconfig.DefaultHTTPClientConfig}}, registry, promslog.NewNopLogger()) body := recorder.Body.String() - if !result { - t.Fatalf("Redirect test failed unexpectedly, got %s", body) + if !result.success { + t.Fatalf("Redirect test failed unexpectedly, got %s %s", result, body) } } @@ -1665,8 +1746,8 @@ func TestSkipResolvePhase(t *testing.T) { defer cancel() result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, HTTPClientConfig: pconfig.DefaultHTTPClientConfig, SkipResolvePhaseWithProxy: true}}, registry, promslog.NewNopLogger()) - if !result { - t.Fatalf("Probe unsuccessful") + if !result.success { + t.Fatalf("Probe unsuccessful %s", result) } mfs, err := registry.Gather() if err != nil { @@ -1761,8 +1842,8 @@ func TestBody(t *testing.T) { registry, promslog.NewNopLogger(), ) - if !result { - t.Fatalf("Body test %d failed unexpectedly.", i) + if !result.success { + t.Fatalf("Body test %d failed unexpectedly, got %s.", i, result) } } } diff --git a/prober/icmp.go b/prober/icmp.go index 2f07e6b0..5d1dee7d 100644 --- a/prober/icmp.go +++ b/prober/icmp.go @@ -62,7 +62,7 @@ func getICMPSequence() uint16 { return icmpSequence } -func ProbeICMP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (success bool) { +func ProbeICMP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (result ProbeResult) { var ( requestType icmp.Type replyType icmp.Type @@ -87,19 +87,17 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr registry.MustRegister(durationGaugeVec) - dstIPAddr, lookupTime, err := chooseProtocol(ctx, module.ICMP.IPProtocol, module.ICMP.IPProtocolFallback, target, registry, logger) + dstIPAddr, lookupTime, resolveResult := chooseProtocol(ctx, module.ICMP.IPProtocol, module.ICMP.IPProtocolFallback, target, registry, logger) - if err != nil { - logger.Error("Error resolving address", "err", err) - return false + if !resolveResult.success { + return resolveResult } durationGaugeVec.WithLabelValues("resolve").Add(lookupTime) var srcIP net.IP if len(module.ICMP.SourceIPAddress) > 0 { if srcIP = net.ParseIP(module.ICMP.SourceIPAddress); srcIP == nil { - logger.Error("Error parsing source ip address", "srcIP", module.ICMP.SourceIPAddress) - return false + return ProbeFailure("Error parsing source ip address", "srcIP", module.ICMP.SourceIPAddress) } logger.Info("Using source address", "srcIP", srcIP) } @@ -111,6 +109,7 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr // Unprivileged sockets are supported on Darwin and Linux only. tryUnprivileged := runtime.GOOS == "darwin" || runtime.GOOS == "linux" + var err error if dstIPAddr.IP.To4() == nil { requestType = ipv6.ICMPTypeEchoRequest replyType = ipv6.ICMPTypeEchoReply @@ -369,7 +368,7 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr registry.MustRegister(hopLimitGauge) } logger.Info("Found matching reply packet") - return true + return ProbeSuccess() } } } diff --git a/prober/prober.go b/prober/prober.go index 5988c032..7eb80539 100644 --- a/prober/prober.go +++ b/prober/prober.go @@ -16,19 +16,118 @@ package prober import ( "context" "log/slog" + "strconv" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/blackbox_exporter/config" ) -type ProbeFn func(ctx context.Context, target string, config config.Module, registry *prometheus.Registry, logger *slog.Logger) bool +// Encodes whether a probe was successful. +// For failed probes additional Details about the failure Reasons are included +// and published through the `probe_failure_info` metric. +type ProbeResult struct { + success bool + failureReason string + failureDetails []string +} + +// Creates the ProbeResult for a failed probe. +// +// Expects an odd number of string arguments. +// +// Example: +// Calling ProbeFailure("problem", "label1", "value1", "label2", "value2") +// will result in the metric: +// +// `probe_failure_info{reason="problem", label1="value1", label2="value2"}` +// +// The corresponding gauge can be obtained with the ProbeResult.failureInfoGauge() +// method. +func ProbeFailure(reason string, details ...string) ProbeResult { + // golangci statically checks this at compile time. So if there is a call + // to this function that could lead to a panic at runtime, this would also trigger + // a linter error in the build process. + if len(details)%2 != 0 { + panic("Must be called with an odd number of string arguments.") + } + + return ProbeResult{success: false, failureReason: reason, failureDetails: details} +} + +func ProbeSuccess() ProbeResult { + return ProbeResult{success: true, failureReason: "", failureDetails: nil} +} + +func (r *ProbeResult) failureInfoGauge() *prometheus.GaugeVec { + if r.success { + panic("Must only be called on failed probes.") + } else if r.failureReason == "" { + // Should not happen, but there theoretically might be an + // inconsistent state of the struct. + r.failureReason = "Unknown" + } + + labels := []string{"reason"} + + for i := 0; i < len(r.failureDetails); i += 2 { + labels = append(labels, r.failureDetails[i]) + } + values := []string{r.failureReason} + + for j := 1; j < len(r.failureDetails); j += 2 { + values = append(values, r.failureDetails[j]) + } + failure_info_gauge := prometheus.NewGaugeVec(probeFailureInfo, labels) + failure_info_gauge.WithLabelValues(values...).Set(1) + + return failure_info_gauge + +} + +func (r *ProbeResult) log(logger *slog.Logger, duration float64) { + if r.success { + logger.Info("Probe succeeded", "duration_seconds", duration) + } else { + if r.failureReason == "" { + // Should not happen, but there theoretically might be an + // inconsistent state of the struct. + r.failureReason = "Probe failed for unknown reason" + } + // converting the []string slice to an []any slice is a bit finicky + logDetails := make([]any, 0, len(r.failureDetails)+4) + logDetails = append(logDetails, "reason") + logDetails = append(logDetails, r.failureReason) + for _, d := range r.failureDetails { + logDetails = append(logDetails, d) + } + logDetails = append(logDetails, "duration") + logDetails = append(logDetails, duration) + logger.Error("Probe failed", logDetails...) + } +} + +func (r ProbeResult) String() string { + if r.success { + return "Probe successful" + } else { + ret := "Probe failed," + ret += " reason: " + strconv.Quote(r.failureReason) + for i := 0; i < len(r.failureDetails); i += 2 { + ret += ", " + r.failureDetails[i] + ": " + strconv.Quote(r.failureDetails[i+1]) + } + return ret + } +} + +type ProbeFn func(ctx context.Context, target string, config config.Module, registry *prometheus.Registry, logger *slog.Logger) ProbeResult const ( helpSSLEarliestCertExpiry = "Returns last SSL chain expiry in unixtime" helpSSLChainExpiryInTimeStamp = "Returns last SSL chain expiry in timestamp" helpProbeTLSInfo = "Returns the TLS version used or NaN when unknown" helpProbeTLSCipher = "Returns the TLS cipher negotiated during handshake" + helpProbeFailureInfo = "Returns the reason the probe failed" ) var ( @@ -51,4 +150,9 @@ var ( Name: "probe_tls_cipher_info", Help: helpProbeTLSCipher, } + + probeFailureInfo = prometheus.GaugeOpts{ + Name: "probe_failure_info", + Help: helpProbeFailureInfo, + } ) diff --git a/prober/tcp.go b/prober/tcp.go index f2f1396d..612eaf9f 100644 --- a/prober/tcp.go +++ b/prober/tcp.go @@ -27,19 +27,18 @@ import ( "github.com/prometheus/blackbox_exporter/config" ) -func dialTCP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (net.Conn, error) { +func dialTCP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) (net.Conn, ProbeResult) { var dialProtocol, dialTarget string dialer := &net.Dialer{} targetAddress, port, err := net.SplitHostPort(target) if err != nil { - logger.Error("Error splitting target address and port", "err", err) - return nil, err + logger.Error(err.Error()) + return nil, ProbeFailure("Error splitting target address and port") } - ip, _, err := chooseProtocol(ctx, module.TCP.IPProtocol, module.TCP.IPProtocolFallback, targetAddress, registry, logger) - if err != nil { - logger.Error("Error resolving address", "err", err) - return nil, err + ip, _, resolveResult := chooseProtocol(ctx, module.TCP.IPProtocol, module.TCP.IPProtocolFallback, targetAddress, registry, logger) + if !resolveResult.success { + return nil, resolveResult } if ip.IP.To4() == nil { @@ -52,7 +51,7 @@ func dialTCP(ctx context.Context, target string, module config.Module, registry srcIP := net.ParseIP(module.TCP.SourceIPAddress) if srcIP == nil { logger.Error("Error parsing source ip address", "srcIP", module.TCP.SourceIPAddress) - return nil, fmt.Errorf("error parsing source ip address: %s", module.TCP.SourceIPAddress) + return nil, ProbeFailure("error parsing source ip address") } logger.Info("Using local address", "srcIP", srcIP) dialer.LocalAddr = &net.TCPAddr{IP: srcIP} @@ -62,12 +61,18 @@ func dialTCP(ctx context.Context, target string, module config.Module, registry if !module.TCP.TLS { logger.Info("Dialing TCP without TLS") - return dialer.DialContext(ctx, dialProtocol, dialTarget) + conn, err := dialer.DialContext(ctx, dialProtocol, dialTarget) + if err != nil { + logger.Error("Dialing TCP without TLS failed", "err", err) + return nil, ProbeFailure("Dialing TCP failed", "tls", "false") + } + + return conn, ProbeSuccess() } tlsConfig, err := pconfig.NewTLSConfig(&module.TCP.TLSConfig) if err != nil { logger.Error("Error creating TLS configuration", "err", err) - return nil, err + return nil, ProbeFailure("Error creating TLS configuration") } if len(tlsConfig.ServerName) == 0 { @@ -84,7 +89,13 @@ func dialTCP(ctx context.Context, target string, module config.Module, registry dialer.Deadline = timeoutDeadline logger.Info("Dialing TCP with TLS") - return tls.DialWithDialer(dialer, dialProtocol, dialTarget, tlsConfig) + conn, err := tls.DialWithDialer(dialer, dialProtocol, dialTarget, tlsConfig) + if err != nil { + logger.Error("Dialing TCP with TLS failed", "err", err) + return nil, ProbeFailure("Dialing TCP failed", "tls", "true") + } + + return conn, ProbeSuccess() } func probeExpectInfo(registry *prometheus.Registry, qr *config.QueryResponse, bytes []byte, match []int) { @@ -105,7 +116,7 @@ func probeExpectInfo(registry *prometheus.Registry, qr *config.QueryResponse, by metric.WithLabelValues(values...).Set(1) } -func ProbeTCP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) bool { +func ProbeTCP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger *slog.Logger) ProbeResult { probeSSLEarliestCertExpiry := prometheus.NewGauge(sslEarliestCertExpiryGaugeOpts) probeSSLLastChainExpiryTimestampSeconds := prometheus.NewGauge(sslChainExpiryInTimeStampGaugeOpts) probeSSLLastInformation := prometheus.NewGaugeVec( @@ -126,10 +137,9 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry registry.MustRegister(probeFailedDueToRegex) deadline, _ := ctx.Deadline() - conn, err := dialTCP(ctx, target, module, registry, logger) - if err != nil { - logger.Error("Error dialing TCP", "err", err) - return false + conn, dialResult := dialTCP(ctx, target, module, registry, logger) + if !dialResult.success { + return dialResult } defer conn.Close() logger.Info("Successfully dialed") @@ -138,8 +148,8 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry // If a deadline cannot be set, better fail the probe by returning an error // now rather than blocking forever. if err := conn.SetDeadline(deadline); err != nil { - logger.Error("Error setting deadline", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Error setting deadline") } if module.TCP.TLS { state := conn.(*tls.Conn).ConnectionState() @@ -165,13 +175,13 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry } } if scanner.Err() != nil { - logger.Error("Error reading from connection", "err", scanner.Err().Error()) - return false + logger.Error("Scanner Error", "error", scanner.Err()) + return ProbeFailure("Error reading from connection") } if match == nil { probeFailedDueToRegex.Set(1) - logger.Error("Regexp did not match", "regexp", qr.Expect.Regexp, "line", scanner.Text()) - return false + logger.Error("Regexp did not match", "regexp", qr.Expect.Regexp.String(), "line", scanner.Text()) + return ProbeFailure("Regexp did not match") } probeFailedDueToRegex.Set(0) send = string(qr.Expect.Expand(nil, []byte(send), scanner.Bytes(), match)) @@ -182,16 +192,16 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry if send != "" { logger.Debug("Sending line", "line", send) if _, err := fmt.Fprintf(conn, "%s\n", send); err != nil { - logger.Error("Failed to send", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Failed to send") } } if qr.StartTLS { // Upgrade TCP connection to TLS. tlsConfig, err := pconfig.NewTLSConfig(&module.TCP.TLSConfig) if err != nil { - logger.Error("Failed to create TLS configuration", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("Failed to create TLS configuration") } if tlsConfig.ServerName == "" { // Use target-hostname as default for TLS-servername. @@ -203,8 +213,8 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry // Initiate TLS handshake (required here to get TLS state). if err := tlsConn.Handshake(); err != nil { - logger.Error("TLS Handshake (client) failed", "err", err) - return false + logger.Error(err.Error()) + return ProbeFailure("TLS Handshake (client) failed") } logger.Info("TLS Handshake (client) succeeded.") conn = net.Conn(tlsConn) @@ -219,5 +229,5 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry probeSSLLastInformation.WithLabelValues(getFingerprint(&state), getSubject(&state), getIssuer(&state), getDNSNames(&state), getSerialNumber(&state)).Set(1) } } - return true + return ProbeSuccess() } diff --git a/prober/tcp_test.go b/prober/tcp_test.go index 52624296..a84795fd 100644 --- a/prober/tcp_test.go +++ b/prober/tcp_test.go @@ -54,7 +54,7 @@ func TestTCPConnection(t *testing.T) { testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() registry := prometheus.NewRegistry() - if !ProbeTCP(testCTX, ln.Addr().String(), config.Module{TCP: config.TCPProbe{IPProtocolFallback: true}}, registry, promslog.NewNopLogger()) { + if !ProbeTCP(testCTX, ln.Addr().String(), config.Module{TCP: config.TCPProbe{IPProtocolFallback: true}}, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -65,7 +65,7 @@ func TestTCPConnectionFails(t *testing.T) { registry := prometheus.NewRegistry() testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - if ProbeTCP(testCTX, ":0", config.Module{TCP: config.TCPProbe{}}, registry, promslog.NewNopLogger()) { + if ProbeTCP(testCTX, ":0", config.Module{TCP: config.TCPProbe{}}, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module succeeded, expected failure.") } } @@ -154,7 +154,7 @@ func TestTCPConnectionWithTLS(t *testing.T) { registry := prometheus.NewRegistry() go serverFunc() // Test name-verification failure (IP without IPs in cert's SAN). - if ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()) { + if ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module succeeded, expected failure.") } <-ch @@ -163,7 +163,7 @@ func TestTCPConnectionWithTLS(t *testing.T) { go serverFunc() // Test name-verification with name from target. target := net.JoinHostPort("localhost", listenPort) - if !ProbeTCP(testCTX, target, module, registry, promslog.NewNopLogger()) { + if !ProbeTCP(testCTX, target, module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -172,7 +172,7 @@ func TestTCPConnectionWithTLS(t *testing.T) { go serverFunc() // Test name-verification against name from tls_config. module.TCP.TLSConfig.ServerName = "localhost" - if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()) { + if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -303,7 +303,7 @@ func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) { go serverFunc() // Test name-verification with name from target. target := net.JoinHostPort("localhost", listenPort) - if !ProbeTCP(testCTX, target, module, registry, promslog.NewNopLogger()) { + if !ProbeTCP(testCTX, target, module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -424,7 +424,7 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) { // Do the client side of this test. registry := prometheus.NewRegistry() - if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()) { + if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -476,7 +476,7 @@ func TestTCPConnectionQueryResponseIRC(t *testing.T) { ch <- struct{}{} }() registry := prometheus.NewRegistry() - if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()) { + if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -495,7 +495,7 @@ func TestTCPConnectionQueryResponseIRC(t *testing.T) { ch <- struct{}{} }() registry = prometheus.NewRegistry() - if ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()) { + if ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module succeeded, expected failure.") } mfs, err := registry.Gather() @@ -555,7 +555,7 @@ func TestTCPConnectionQueryResponseMatching(t *testing.T) { ch <- version }() registry := prometheus.NewRegistry() - if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()) { + if !ProbeTCP(testCTX, ln.Addr().String(), module, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module failed, expected success.") } if got, want := <-ch, "OpenSSH_6.9p1"; got != want { @@ -616,7 +616,7 @@ func TestTCPConnectionProtocol(t *testing.T) { registry := prometheus.NewRegistry() result := ProbeTCP(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) - if !result { + if !result.success { t.Fatalf("TCP protocol: \"tcp\", prefer: \"ip4\" connection test failed, expected success.") } mfs, err := registry.Gather() @@ -637,7 +637,7 @@ func TestTCPConnectionProtocol(t *testing.T) { registry = prometheus.NewRegistry() result = ProbeTCP(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) - if !result { + if !result.success { t.Fatalf("TCP protocol: \"tcp\", prefer: \"ip6\" connection test failed, expected success.") } mfs, err = registry.Gather() @@ -656,7 +656,7 @@ func TestTCPConnectionProtocol(t *testing.T) { registry = prometheus.NewRegistry() result = ProbeTCP(testCTX, net.JoinHostPort("localhost", port), module, registry, promslog.NewNopLogger()) - if !result { + if !result.success { t.Fatalf("TCP protocol: \"tcp\" connection test failed, expected success.") } mfs, err = registry.Gather() @@ -695,7 +695,7 @@ func TestPrometheusTimeoutTCP(t *testing.T) { Expect: config.MustNewRegexp("SSH-2.0-(OpenSSH_6.9p1) Debian-2"), }, }, - }}, registry, promslog.NewNopLogger()) { + }}, registry, promslog.NewNopLogger()).success { t.Fatalf("TCP module succeeded, expected timeout failure.") } <-ch diff --git a/prober/utils.go b/prober/utils.go index 3dc4153c..859cec42 100644 --- a/prober/utils.go +++ b/prober/utils.go @@ -15,7 +15,6 @@ package prober import ( "context" - "fmt" "hash/fnv" "log/slog" "net" @@ -30,7 +29,7 @@ var protocolToGauge = map[string]float64{ } // Returns the IP for the IPProtocol and lookup time. -func chooseProtocol(ctx context.Context, IPProtocol string, fallbackIPProtocol bool, target string, registry *prometheus.Registry, logger *slog.Logger) (ip *net.IPAddr, lookupTime float64, err error) { +func chooseProtocol(ctx context.Context, IPProtocol string, fallbackIPProtocol bool, target string, registry *prometheus.Registry, logger *slog.Logger) (ip *net.IPAddr, lookupTime float64, success ProbeResult) { var fallbackProtocol string probeDNSLookupTimeSeconds := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "probe_dns_lookup_time_seconds", @@ -74,17 +73,17 @@ func chooseProtocol(ctx context.Context, IPProtocol string, fallbackIPProtocol b logger.Info("Resolved target address", "target", target, "ip", ip.String()) probeIPProtocolGauge.Set(protocolToGauge[IPProtocol]) probeIPAddrHash.Set(ipHash(ip)) - return &net.IPAddr{IP: ip}, lookupTime, nil + return &net.IPAddr{IP: ip}, lookupTime, ProbeSuccess() } } logger.Error("Resolution with IP protocol failed", "target", target, "ip_protocol", IPProtocol, "err", err) - return nil, 0.0, err + return nil, 0.0, ProbeFailure("DNS resolution failed", "target", target, "ip_protocol", IPProtocol) } ips, err := resolver.LookupIPAddr(ctx, target) if err != nil { logger.Error("Resolution with IP protocol failed", "target", target, "err", err) - return nil, 0.0, err + return nil, 0.0, ProbeFailure("DNS resolution failed", "target", target) } // Return the IP in the requested protocol. @@ -96,7 +95,7 @@ func chooseProtocol(ctx context.Context, IPProtocol string, fallbackIPProtocol b logger.Info("Resolved target address", "target", target, "ip", ip.String()) probeIPProtocolGauge.Set(4) probeIPAddrHash.Set(ipHash(ip.IP)) - return &ip, lookupTime, nil + return &ip, lookupTime, ProbeSuccess() } // ip4 as fallback @@ -107,7 +106,7 @@ func chooseProtocol(ctx context.Context, IPProtocol string, fallbackIPProtocol b logger.Info("Resolved target address", "target", target, "ip", ip.String()) probeIPProtocolGauge.Set(6) probeIPAddrHash.Set(ipHash(ip.IP)) - return &ip, lookupTime, nil + return &ip, lookupTime, ProbeSuccess() } // ip6 as fallback @@ -117,7 +116,7 @@ func chooseProtocol(ctx context.Context, IPProtocol string, fallbackIPProtocol b // Unable to find ip and no fallback set. if fallback == nil || !fallbackIPProtocol { - return nil, 0.0, fmt.Errorf("unable to find ip; no fallback") + return nil, 0.0, ProbeFailure("DNS resolution failed", "details", "unable to find ip; no fallback") } // Use fallback ip protocol. @@ -128,7 +127,7 @@ func chooseProtocol(ctx context.Context, IPProtocol string, fallbackIPProtocol b } probeIPAddrHash.Set(ipHash(fallback.IP)) logger.Info("Resolved target address", "target", target, "ip", fallback.String()) - return fallback, lookupTime, nil + return fallback, lookupTime, ProbeSuccess() } func ipHash(ip net.IP) float64 { diff --git a/prober/utils_test.go b/prober/utils_test.go index bc85d23d..dd22a40f 100644 --- a/prober/utils_test.go +++ b/prober/utils_test.go @@ -24,6 +24,7 @@ import ( "fmt" "math/big" "net" + "reflect" "slices" "testing" "time" @@ -151,9 +152,9 @@ func TestChooseProtocol(t *testing.T) { registry := prometheus.NewPedanticRegistry() logger := promslog.New(&promslog.Config{}) - ip, _, err := chooseProtocol(ctx, "ip4", true, "ipv6.google.com", registry, logger) - if err != nil { - t.Error(err) + ip, _, result := chooseProtocol(ctx, "ip4", true, "ipv6.google.com", registry, logger) + if !result.success { + t.Error(result) } if ip == nil || ip.IP.To4() != nil { t.Error("with fallback it should answer") @@ -161,11 +162,9 @@ func TestChooseProtocol(t *testing.T) { registry = prometheus.NewPedanticRegistry() - ip, _, err = chooseProtocol(ctx, "ip4", false, "ipv6.google.com", registry, logger) - if err != nil && !err.(*net.DNSError).IsNotFound { - t.Error(err) - } else if err == nil { - t.Error("should set error") + ip, _, result = chooseProtocol(ctx, "ip4", false, "ipv6.google.com", registry, logger) + if !reflect.DeepEqual(result, ProbeFailure("DNS resolution failed", "target", "ipv6.google.com", "ip_protocol", "ip4")) { + t.Error(result) } if ip != nil { t.Error("without fallback it should not answer")