Implement dynamic probes

README.md

@@ -143,6 +143,48 @@ scrape_configs:
         target_label: vhost  # and store it in 'vhost' label
 ```
 
+## Dynamic Probes
+
+In addition to static probes, configured via a [configuration file](CONFIGURATION.md), blackbox exporter supports the probe configuration via HTTP
+query parameter on the endpoint `/scrape/dynamic`.
+
+Most of the probe configuration can be configured via HTTP query parameter with the schema `<prober>.<setting>`, e.g. `http.method=POST`.
+HTTP query parameter supports only the configuration of top-level parameter. 
+
+If the value of the property is an array, the query parameter needs to be passed with an suffix `[]`, e.g.: `http.valid_http_versions[]`
+
+The configuration of sub level parameters needs to be passed as JSON document, e.g.
+`http.http_client_config={"tls_config":{"insecure_skip_verify":true}}` instead `http.http_client_config.tls_config.insecure_skip_verify=true`.
+
+### Examples
+
+- `http://localhost:9115/probe/dynamic?target=example.com&prober=http`
+- `http://localhost:9115/probe/dynamic?target=expired.badssl.com&prober=http&http.http_client_config={"tls_config":{"insecure_skip_verify":true}}`
+
+Example config:
+```yml
+scrape_configs:
+  - job_name: 'blackbox'
+    metrics_path: /probe/dynamic
+    params:
+      prober: [http]
+      http.method: ["OPTIONS"]
+      http.valid_http_versions[]: ["HTTP/2"]
+      http.valid_status_codes[]: ["204","301","405"]
+      http.http_client_config: ['{"tls_config":{"insecure_skip_verify":true}}']
+    static_configs:
+      - targets:
+          - https://expired.badssl.com
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115
+```
+
+
 ## Permissions
 
 The ICMP probe requires elevated privileges to function:

config/binding.go

@@ -0,0 +1,82 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"bytes"
+	"reflect"
+
+	"github.com/bytedance/go-tagexpr/v2/binding"
+	"github.com/prometheus/common/config"
+	"gopkg.in/yaml.v3"
+)
+
+func InitializeBinding() {
+	binding.MustRegTypeUnmarshal(reflect.TypeOf(Regexp{}), func(v string, emptyAsZero bool) (reflect.Value, error) {
+		if v == "" && emptyAsZero {
+			return reflect.ValueOf(Regexp{}), nil
+		}
+
+		t, err := NewRegexp(v)
+		if err != nil {
+			return reflect.ValueOf(Regexp{}), err
+		}
+
+		return reflect.ValueOf(t), nil
+	})
+
+	binding.MustRegTypeUnmarshal(reflect.TypeOf(HeaderMatch{}), func(v string, emptyAsZero bool) (reflect.Value, error) {
+		if v == "" && emptyAsZero {
+			return reflect.ValueOf(HeaderMatch{}), nil
+		}
+
+		var c = &HeaderMatch{}
+
+		decoder := yaml.NewDecoder(bytes.NewBufferString(v))
+		decoder.KnownFields(true)
+
+		if err := decoder.Decode(c); err != nil {
+			return reflect.ValueOf(HeaderMatch{}), err
+		}
+
+		if err := c.Validate(); err != nil {
+			return reflect.ValueOf(HeaderMatch{}), err
+		}
+
+		return reflect.ValueOf(*c), nil
+	})
+
+	binding.MustRegTypeUnmarshal(reflect.TypeOf(config.HTTPClientConfig{}), binderYamlDecoder(config.HTTPClientConfig{}))
+	binding.MustRegTypeUnmarshal(reflect.TypeOf(config.TLSConfig{}), binderYamlDecoder(config.TLSConfig{}))
+	binding.MustRegTypeUnmarshal(reflect.TypeOf(DNSRRValidator{}), binderYamlDecoder(DNSRRValidator{}))
+}
+
+func binderYamlDecoder[T any](s T) func(v string, emptyAsZero bool) (reflect.Value, error) {
+	return func(v string, emptyAsZero bool) (reflect.Value, error) {
+		if v == "" && emptyAsZero {
+			return reflect.ValueOf(s), nil
+		}
+
+		var c = &s
+
+		decoder := yaml.NewDecoder(bytes.NewBufferString(v))
+		decoder.KnownFields(true)
+
+		if err := decoder.Decode(c); err != nil {
+			return reflect.ValueOf(s), err
+		}
+
+		return reflect.ValueOf(*c), nil
+	}
+}

config/config.go

@@ -193,7 +193,7 @@ func MustNewRegexp(s string) Regexp {
 }
 
 type Module struct {
-	Prober  string        `yaml:"prober,omitempty"`
+	Prober  string        `yaml:"prober,omitempty" query:"prober"`
 	Timeout time.Duration `yaml:"timeout,omitempty"`
 	HTTP    HTTPProbe     `yaml:"http,omitempty"`
 	TCP     TCPProbe      `yaml:"tcp,omitempty"`
@@ -204,33 +204,33 @@ type Module struct {
 
 type HTTPProbe struct {
 	// Defaults to 2xx.
-	ValidStatusCodes             []int                   `yaml:"valid_status_codes,omitempty"`
-	ValidHTTPVersions            []string                `yaml:"valid_http_versions,omitempty"`
-	IPProtocol                   string                  `yaml:"preferred_ip_protocol,omitempty"`
-	IPProtocolFallback           bool                    `yaml:"ip_protocol_fallback,omitempty"`
-	SkipResolvePhaseWithProxy    bool                    `yaml:"skip_resolve_phase_with_proxy,omitempty"`
-	NoFollowRedirects            *bool                   `yaml:"no_follow_redirects,omitempty"`
-	FailIfSSL                    bool                    `yaml:"fail_if_ssl,omitempty"`
-	FailIfNotSSL                 bool                    `yaml:"fail_if_not_ssl,omitempty"`
-	Method                       string                  `yaml:"method,omitempty"`
-	Headers                      map[string]string       `yaml:"headers,omitempty"`
-	FailIfBodyMatchesRegexp      []Regexp                `yaml:"fail_if_body_matches_regexp,omitempty"`
-	FailIfBodyNotMatchesRegexp   []Regexp                `yaml:"fail_if_body_not_matches_regexp,omitempty"`
-	FailIfHeaderMatchesRegexp    []HeaderMatch           `yaml:"fail_if_header_matches,omitempty"`
-	FailIfHeaderNotMatchesRegexp []HeaderMatch           `yaml:"fail_if_header_not_matches,omitempty"`
-	Body                         string                  `yaml:"body,omitempty"`
+	ValidStatusCodes             []int                   `yaml:"valid_status_codes,omitempty" query:"http.valid_status_codes[]"`
+	ValidHTTPVersions            []string                `yaml:"valid_http_versions,omitempty" query:"http.valid_http_versions[]"`
+	IPProtocol                   string                  `yaml:"preferred_ip_protocol,omitempty" query:"http.preferred_ip_protocol"`
+	IPProtocolFallback           bool                    `yaml:"ip_protocol_fallback,omitempty" query:"http.ip_protocol_fallback"`
+	SkipResolvePhaseWithProxy    bool                    `yaml:"skip_resolve_phase_with_proxy,omitempty" query:"http.skip_resolve_phase_with_proxy"`
+	NoFollowRedirects            *bool                   `yaml:"no_follow_redirects,omitempty" query:"http.no_follow_redirects"`
+	FailIfSSL                    bool                    `yaml:"fail_if_ssl,omitempty" query:"http.fail_if_ssl"`
+	FailIfNotSSL                 bool                    `yaml:"fail_if_not_ssl,omitempty" query:"http.fail_if_not_ssl"`
+	Method                       string                  `yaml:"method,omitempty" query:"http.method"`
+	Headers                      map[string]string       `yaml:"headers,omitempty" query:"http.headers"`
+	FailIfBodyMatchesRegexp      []Regexp                `yaml:"fail_if_body_matches_regexp,omitempty" query:"http.fail_if_body_matches_regexp[]"`
+	FailIfBodyNotMatchesRegexp   []Regexp                `yaml:"fail_if_body_not_matches_regexp,omitempty" query:"http.fail_if_body_not_matches_regexp[]"`
+	FailIfHeaderMatchesRegexp    []HeaderMatch           `yaml:"fail_if_header_matches,omitempty" query:"http.fail_if_header_matches[]"`
+	FailIfHeaderNotMatchesRegexp []HeaderMatch           `yaml:"fail_if_header_not_matches,omitempty" query:"http.fail_if_header_not_matches[]"`
+	Body                         string                  `yaml:"body,omitempty" query:"http.body"`
 	BodyFile                     string                  `yaml:"body_file,omitempty"`
-	HTTPClientConfig             config.HTTPClientConfig `yaml:"http_client_config,inline"`
-	Compression                  string                  `yaml:"compression,omitempty"`
-	BodySizeLimit                units.Base2Bytes        `yaml:"body_size_limit,omitempty"`
+	HTTPClientConfig             config.HTTPClientConfig `yaml:"http_client_config,inline" query:"http.http_client_config"`
+	Compression                  string                  `yaml:"compression,omitempty" query:"http.compression"`
+	BodySizeLimit                units.Base2Bytes        `yaml:"body_size_limit,omitempty" query:"http.body_size_limit"`
 }
 
 type GRPCProbe struct {
-	Service             string           `yaml:"service,omitempty"`
-	TLS                 bool             `yaml:"tls,omitempty"`
-	TLSConfig           config.TLSConfig `yaml:"tls_config,omitempty"`
-	IPProtocolFallback  bool             `yaml:"ip_protocol_fallback,omitempty"`
-	PreferredIPProtocol string           `yaml:"preferred_ip_protocol,omitempty"`
+	Service             string           `yaml:"service,omitempty" query:"grpc.service"`
+	TLS                 bool             `yaml:"tls,omitempty" query:"grpc.tls"`
+	TLSConfig           config.TLSConfig `yaml:"tls_config,omitempty" query:"grpc.tls_config"`
+	IPProtocolFallback  bool             `yaml:"ip_protocol_fallback,omitempty" query:"grpc.ip_protocol_fallback"`
+	PreferredIPProtocol string           `yaml:"preferred_ip_protocol,omitempty" query:"grpc.preferred_ip_protocols"`
 }
 
 type HeaderMatch struct {
@@ -246,38 +246,38 @@ type QueryResponse struct {
 }
 
 type TCPProbe struct {
-	IPProtocol         string           `yaml:"preferred_ip_protocol,omitempty"`
-	IPProtocolFallback bool             `yaml:"ip_protocol_fallback,omitempty"`
-	SourceIPAddress    string           `yaml:"source_ip_address,omitempty"`
-	QueryResponse      []QueryResponse  `yaml:"query_response,omitempty"`
-	TLS                bool             `yaml:"tls,omitempty"`
-	TLSConfig          config.TLSConfig `yaml:"tls_config,omitempty"`
+	IPProtocol         string           `yaml:"preferred_ip_protocol,omitempty" query:"tcp.preferred_ip_protocol"`
+	IPProtocolFallback bool             `yaml:"ip_protocol_fallback,omitempty" query:"tcp.ip_protocol_fallback"`
+	SourceIPAddress    string           `yaml:"source_ip_address,omitempty" query:"tcp.source_ip_address"`
+	QueryResponse      []QueryResponse  `yaml:"query_response,omitempty" query:"tcp.query_response[]"`
+	TLS                bool             `yaml:"tls,omitempty" query:"tcp.tls"`
+	TLSConfig          config.TLSConfig `yaml:"tls_config,omitempty" query:"tcp.tls_config"`
 }
 
 type ICMPProbe struct {
-	IPProtocol         string `yaml:"preferred_ip_protocol,omitempty"` // Defaults to "ip6".
-	IPProtocolFallback bool   `yaml:"ip_protocol_fallback,omitempty"`
-	SourceIPAddress    string `yaml:"source_ip_address,omitempty"`
-	PayloadSize        int    `yaml:"payload_size,omitempty"`
-	DontFragment       bool   `yaml:"dont_fragment,omitempty"`
-	TTL                int    `yaml:"ttl,omitempty"`
+	IPProtocol         string `yaml:"preferred_ip_protocol,omitempty" query:"icmp.preferred_ip_protocol"` // Defaults to "ip6".
+	IPProtocolFallback bool   `yaml:"ip_protocol_fallback,omitempty" query:"icmp.ip_protocol_fallback"`
+	SourceIPAddress    string `yaml:"source_ip_address,omitempty" query:"icmp.source_ip_address"`
+	PayloadSize        int    `yaml:"payload_size,omitempty" query:"icmp.payload_size"`
+	DontFragment       bool   `yaml:"dont_fragment,omitempty" query:"icmp.dont_fragment"`
+	TTL                int    `yaml:"ttl,omitempty" query:"icmp.ttl"`
 }
 
 type DNSProbe struct {
-	IPProtocol         string           `yaml:"preferred_ip_protocol,omitempty"`
-	IPProtocolFallback bool             `yaml:"ip_protocol_fallback,omitempty"`
-	DNSOverTLS         bool             `yaml:"dns_over_tls,omitempty"`
-	TLSConfig          config.TLSConfig `yaml:"tls_config,omitempty"`
-	SourceIPAddress    string           `yaml:"source_ip_address,omitempty"`
-	TransportProtocol  string           `yaml:"transport_protocol,omitempty"`
-	QueryClass         string           `yaml:"query_class,omitempty"` // Defaults to IN.
-	QueryName          string           `yaml:"query_name,omitempty"`
-	QueryType          string           `yaml:"query_type,omitempty"`        // Defaults to ANY.
-	Recursion          bool             `yaml:"recursion_desired,omitempty"` // Defaults to true.
-	ValidRcodes        []string         `yaml:"valid_rcodes,omitempty"`      // Defaults to NOERROR.
-	ValidateAnswer     DNSRRValidator   `yaml:"validate_answer_rrs,omitempty"`
-	ValidateAuthority  DNSRRValidator   `yaml:"validate_authority_rrs,omitempty"`
-	ValidateAdditional DNSRRValidator   `yaml:"validate_additional_rrs,omitempty"`
+	IPProtocol         string           `yaml:"preferred_ip_protocol,omitempty" query:"dns.preferred_ip_protocol"`
+	IPProtocolFallback bool             `yaml:"ip_protocol_fallback,omitempty" query:"dns.ip_protocol_fallback"`
+	DNSOverTLS         bool             `yaml:"dns_over_tls,omitempty" query:"dns.dns_over_tls"`
+	TLSConfig          config.TLSConfig `yaml:"tls_config,omitempty" query:"dns.tls_config"`
+	SourceIPAddress    string           `yaml:"source_ip_address,omitempty" query:"dns.source_ip_address"`
+	TransportProtocol  string           `yaml:"transport_protocol,omitempty" query:"dns.transport_protocol"`
+	QueryClass         string           `yaml:"query_class,omitempty" query:"dns.query_class"` // Defaults to IN.
+	QueryName          string           `yaml:"query_name,omitempty" query:"dns.query_name"`
+	QueryType          string           `yaml:"query_type,omitempty" query:"dns.query_type"`               // Defaults to ANY.
+	Recursion          bool             `yaml:"recursion_desired,omitempty" query:"dns.recursion_desired"` // Defaults to true.
+	ValidRcodes        []string         `yaml:"valid_rcodes,omitempty" query:"dns.valid_rcodes[]"`         // Defaults to NOERROR.
+	ValidateAnswer     DNSRRValidator   `yaml:"validate_answer_rrs,omitempty" query:"dns.validate_answer_rrs"`
+	ValidateAuthority  DNSRRValidator   `yaml:"validate_authority_rrs,omitempty" query:"dns.validate_authority_rrs"`
+	ValidateAdditional DNSRRValidator   `yaml:"validate_additional_rrs,omitempty" query:"dns.validate_additional_rrs"`
 }
 
 type DNSRRValidator struct {
@@ -314,6 +314,14 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return err
 	}
 
+	if err := s.Validate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *HTTPProbe) Validate() error {
 	// BodySizeLimit == 0 means no limit. By leaving it at 0 we
 	// avoid setting up the limiter.
 	if s.BodySizeLimit < 0 || s.BodySizeLimit == math.MaxInt64 {
@@ -344,7 +352,6 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			}
 		}
 	}
-
 	return nil
 }
 
@@ -365,6 +372,15 @@ func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err := unmarshal((*plain)(s)); err != nil {
 		return err
 	}
+
+	if err := s.Validate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *DNSProbe) Validate() error {
 	if s.QueryName == "" {
 		return errors.New("query name must be set for DNS module")
 	}
@@ -378,7 +394,6 @@ func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			return fmt.Errorf("query type '%s' is not valid", s.QueryType)
 		}
 	}
-
 	return nil
 }
 
@@ -409,6 +424,13 @@ func (s *ICMPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return err
 	}
 
+	if err := s.Validate(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *ICMPProbe) Validate() error {
 	if runtime.GOOS == "windows" && s.DontFragment {
 		return errors.New("\"dont_fragment\" is not supported on windows platforms")
 	}
@@ -439,14 +461,21 @@ func (s *HeaderMatch) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return err
 	}
 
+	if err := s.Validate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *HeaderMatch) Validate() error {
 	if s.Header == "" {
 		return errors.New("header name must be set for HTTP header matchers")
 	}
 
 	if s.Regexp.Regexp == nil || s.Regexp.Regexp.String() == "" {
 		return errors.New("regexp must be set for HTTP header matchers")
 	}
-
 	return nil
 }
 

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/alecthomas/kingpin/v2 v2.3.2
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 	github.com/andybalholm/brotli v1.0.5
+	github.com/bytedance/go-tagexpr/v2 v2.9.8
 	github.com/go-kit/log v0.2.1
 	github.com/miekg/dns v1.1.57
 	github.com/prometheus/client_golang v1.17.0
@@ -19,6 +20,8 @@ require (
 )
 
 require (
+	github.com/andeya/ameda v1.5.3 // indirect
+	github.com/andeya/goutil v1.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
@@ -28,7 +31,10 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
+	github.com/nyaruka/phonenumbers v1.0.55 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect