Skip to content

Commit

Permalink
Add support for grpc health check
Browse files Browse the repository at this point in the history
Signed-off-by: Alessandro Verzicco <[email protected]>
  • Loading branch information
alessandro-verzicco committed Oct 18, 2021
1 parent 72910e5 commit caee2ee
Show file tree
Hide file tree
Showing 7 changed files with 409 additions and 0 deletions.
10 changes: 10 additions & 0 deletions blackbox.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,16 @@ modules:
tls: true
tls_config:
insecure_skip_verify: false
grpc:
prober: grpc
grpc:
tls: true
preferred_ip_protocol: "ip4"
grpc_plain:
prober: grpc
grpc:
tls: false
preferred_ip_protocol: "ip4"
ssh_banner:
prober: tcp
tcp:
Expand Down
23 changes: 23 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ var (
HTTPClientConfig: config.DefaultHTTPClientConfig,
}

// DefaultGRPCProbe set default value for HTTPProbe
DefaultGRPCProbe = GRPCProbe{
IPProtocolFallback: true,
}

// DefaultTCPProbe set default value for TCPProbe
DefaultTCPProbe = TCPProbe{
IPProtocolFallback: true,
Expand Down Expand Up @@ -187,6 +192,7 @@ type Module struct {
TCP TCPProbe `yaml:"tcp,omitempty"`
ICMP ICMPProbe `yaml:"icmp,omitempty"`
DNS DNSProbe `yaml:"dns,omitempty"`
GRPC GRPCProbe `yaml:"grpc,omitempty"`
}

type HTTPProbe struct {
Expand All @@ -209,6 +215,13 @@ type HTTPProbe struct {
Compression string `yaml:"compression,omitempty"`
}

type GRPCProbe struct {
TLS bool `yaml:"tls,omitempty"`
TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"`
IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"`
PreferredIPProtocol string `yaml:"preferred_ip_protocol,omitempty"`
}

type HeaderMatch struct {
Header string `yaml:"header,omitempty"`
Regexp Regexp `yaml:"regexp,omitempty"`
Expand Down Expand Up @@ -307,6 +320,16 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (s *GRPCProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
*s = DefaultGRPCProbe
type plain GRPCProbe
if err := unmarshal((*plain)(s)); err != nil {
return err
}
return nil
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
*s = DefaultDNSProbe
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/prometheus/common v0.23.0
github.com/prometheus/exporter-toolkit v0.5.1
golang.org/x/net v0.0.0-20210505214959-0714010a04ed
google.golang.org/grpc v1.26.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
Expand Down Expand Up @@ -400,6 +401,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
Expand All @@ -409,6 +411,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
Expand Down
1 change: 1 addition & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ var (
"tcp": prober.ProbeTCP,
"icmp": prober.ProbeICMP,
"dns": prober.ProbeDNS,
"grpc": prober.ProbeGRPC,
}

moduleUnknownCounter = prometheus.NewCounter(prometheus.CounterOpts{
Expand Down
201 changes: 201 additions & 0 deletions prober/grpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prober

import (
"context"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/prometheus/blackbox_exporter/config"
"github.com/prometheus/client_golang/prometheus"
pconfig "github.com/prometheus/common/config"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
"net"
"net/url"
"strings"
)

// Health defines a health-check connection.
type Health interface {
Check(c context.Context) (bool, codes.Code, *peer.Peer, error)
}

type healthClient struct {
client grpc_health_v1.HealthClient
conn *grpc.ClientConn
}

func NewGrpcHealthClient(conn *grpc.ClientConn) Health {
client := new(healthClient)
client.client = grpc_health_v1.NewHealthClient(conn)
client.conn = conn
return client
}

func (c *healthClient) Close() error {
return c.conn.Close()
}

func (c *healthClient) Check(ctx context.Context) (bool, codes.Code, *peer.Peer, error) {
var res *grpc_health_v1.HealthCheckResponse
var err error
req := new(grpc_health_v1.HealthCheckRequest)

serverPeer := new(peer.Peer)
res, err = c.client.Check(ctx, req, grpc.Peer(serverPeer))
if err == nil {
if res.GetStatus() == grpc_health_v1.HealthCheckResponse_SERVING {
return true, codes.OK, serverPeer, nil
}
return false, codes.OK, serverPeer, nil
}

returnStatus, _ := status.FromError(err)

return false, returnStatus.Code(), nil, err
}

func ProbeGRPC(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger log.Logger) (success bool) {

var (
durationGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "probe_grpc_duration_seconds",
Help: "Duration of gRPC request by phase",
}, []string{"phase"})

isSSLGauge = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "probe_grpc_ssl",
Help: "Indicates if SSL was used for the connection",
})

statusCodeGauge = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "probe_grpc_status_code",
Help: "Response gRPC status code",
})

probeSSLEarliestCertExpiryGauge = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "probe_ssl_earliest_cert_expiry",
Help: "Returns earliest SSL cert expiry in unixtime",
})

probeTLSVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "probe_tls_version_info",
Help: "Contains the TLS version used",
},
[]string{"version"},
)
)

for _, lv := range []string{"resolve"} {
durationGaugeVec.WithLabelValues(lv)
}

registry.MustRegister(durationGaugeVec)
registry.MustRegister(isSSLGauge)
registry.MustRegister(statusCodeGauge)
registry.MustRegister(probeSSLEarliestCertExpiryGauge)
registry.MustRegister(probeTLSVersion)

if !strings.HasPrefix(target, "http://") && !strings.HasPrefix(target, "https://") {
target = "http://" + target
}

targetURL, err := url.Parse(target)
if err != nil {
level.Error(logger).Log("msg", "Could not parse target URL", "err", err)
return false
}

targetHost, targetPort, err := net.SplitHostPort(targetURL.Host)
// If split fails, assuming it's a hostname without port part.
if err != nil {
targetHost = targetURL.Host
}

tlsConfig, err := pconfig.NewTLSConfig(&module.GRPC.TLSConfig)
if err != nil {
level.Error(logger).Log("msg", "Error creating TLS configuration", "err", err)
return false
}

ip, lookupTime, err := chooseProtocol(ctx, module.GRPC.PreferredIPProtocol, module.GRPC.IPProtocolFallback, targetHost, registry, logger)
if err != nil {
level.Error(logger).Log("msg", "Error resolving address", "err", err)
return false
}
durationGaugeVec.WithLabelValues("resolve").Add(lookupTime)

if len(tlsConfig.ServerName) == 0 {
// If there is no `server_name` in tls_config, use
// the hostname of the target.
tlsConfig.ServerName = targetHost
}

if targetPort == "" {
targetURL.Host = "[" + ip.String() + "]"
} else {
targetURL.Host = net.JoinHostPort(ip.String(), targetPort)
}

var opts []grpc.DialOption
target = targetHost + ":" + targetPort
if !module.GRPC.TLS {
level.Debug(logger).Log("msg", "Dialing GRPC without TLS")
opts = append(opts, grpc.WithInsecure())
if len(targetPort) == 0 {
target = targetHost + ":80"
}
} else {
creds := credentials.NewTLS(tlsConfig)
opts = append(opts, grpc.WithTransportCredentials(creds))
if len(targetPort) == 0 {
target = targetHost + ":443"
}
}

conn, err := grpc.Dial(target, opts...)

if err != nil {
level.Error(logger).Log("did not connect: %v", err)
}

client := NewGrpcHealthClient(conn)
defer conn.Close()
ok, statusCode, serverPeer, err := client.Check(context.Background())

tlsInfo, tlsOk := serverPeer.AuthInfo.(credentials.TLSInfo)
if tlsOk {
isSSLGauge.Set(float64(1))
probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(&tlsInfo.State).Unix()))
probeTLSVersion.WithLabelValues(getTLSVersion(&tlsInfo.State)).Set(1)
} else {
isSSLGauge.Set(float64(0))
}
statusCodeGauge.Set(float64(statusCode))

if !ok || err != nil {
level.Error(logger).Log("msg", "can't connect grpc server:", "err", err)
success = false
} else {
level.Debug(logger).Log("connect the grpc server successfully")
success = true
}

return
}
Loading

0 comments on commit caee2ee

Please sign in to comment.