From ee897829a74b56f2994614db987347e1b69aef7b Mon Sep 17 00:00:00 2001 From: Ryuma Yoshida Date: Fri, 27 Nov 2020 16:46:26 +0900 Subject: [PATCH 1/7] Implement the readiness endpoint for health checking Signed-off-by: Ryuma Yoshida --- conf/agent/agent_full.conf | 33 +++++++++-------- conf/server/server_full.conf | 3 ++ doc/spire_agent.md | 1 + doc/spire_server.md | 1 + pkg/agent/agent.go | 27 ++++++++++++-- pkg/common/health/config.go | 37 ++++++++++++++----- pkg/common/health/health.go | 13 +++++-- pkg/server/server.go | 24 ++++++++++-- .../k8s-reconcile/conf/agent/spire-agent.yaml | 29 ++++++++++----- .../conf/server/spire-server.yaml | 21 ++++++++--- .../suites/k8s/conf/agent/spire-agent.yaml | 29 ++++++++++----- .../suites/k8s/conf/server/spire-server.yaml | 27 +++++++++----- 12 files changed, 178 insertions(+), 67 deletions(-) diff --git a/conf/agent/agent_full.conf b/conf/agent/agent_full.conf index 65d6b2acb0..2fff120379 100644 --- a/conf/agent/agent_full.conf +++ b/conf/agent/agent_full.conf @@ -1,5 +1,5 @@ # This is the SPIRE Agent configuration file including all possible configuration -# options. +# options. # agent: Contains core configuration parameters. agent { @@ -24,16 +24,16 @@ agent { # server_address: DNS name or IP address of the SPIRE server. server_address = "127.0.0.1" - + # server_port: Port number of the SPIRE server. server_port = "8081" - + # socket_path: Location to bind the workload API socket. Default: /tmp/agent.sock. socket_path = "/tmp/agent.sock" - + # trust_bundle_path: Path to the SPIRE server CA bundle. trust_bundle_path = "./conf/agent/dummy_root_ca.crt" - + # trust_bundle_url: URL to download the initial SPIRE server trust bundle. # trust_bundle_url = "" @@ -56,9 +56,9 @@ agent { # Each nested object has the following format: # # PluginType "plugin_name" { -# +# # # plugin_cmd: Path to the plugin implementation binary (optional, not -# # needed for built-ins) +# # needed for built-ins) # plugin_cmd = # # # plugin_checksum: An optional sha256 of the plugin binary (optional, @@ -149,7 +149,7 @@ plugins { # cluster: Name of the cluster. It must correspond to a cluster # configured in the server plugin. # cluster = "" - + # token_path: Path to the service account token on disk. # Default: /run/secrets/kubernetes.io/serviceaccount/token. # token_path = "/run/secrets/kubernetes.io/serviceaccount/token" @@ -180,7 +180,7 @@ plugins { # certificate_path: The path to the certificate bundle on disk. The # file must contain one or more PEM blocks, starting with the identity # certificate followed by any intermediate certificates necessary for - # chain-of-trust validation. + # chain-of-trust validation. # certificate_path = "" # intermediates_path: Optional. The path to a chain of intermediate @@ -204,7 +204,7 @@ plugins { # docker_version = "" } } - + # WorkloadAttestor "k8s": A workload attestor which allows selectors based # on Kubernetes constructs such ns (namespace) and sa (service account). WorkloadAttestor "k8s" { @@ -244,7 +244,7 @@ plugins { # node_name_env = "MY_NODE_NAME" # node_name: The name of the node. Overrides the value obtained by - # the environment variable specified by node_name_env. + # the environment variable specified by node_name_env. # node_name = "" } } @@ -276,7 +276,7 @@ plugins { # port = 9988 # } -# DogStatsd = [ +# DogStatsd = [ # # List of DogStatsd addresses. # { address = "localhost:8125" }, # { address = "collector.example.org:1337" }, @@ -301,7 +301,7 @@ plugins { # } # health_checks: If health checking is desired use this section to configure -# and expose an additional server endpoint for such purpose. +# and expose an additional agent endpoint for such purpose. # health_checks { # # listener_enabled: Enables health checks endpoint. # listener_enabled = true @@ -312,9 +312,12 @@ plugins { # # bind_port: HTTP Port number of the health checks endpoint. Default: 80. # # bind_port = "80" -# # live_path: HTTP resource path for checking server liveness. Default: /live. +# # live_path: HTTP resource path for checking agent liveness. Default: /live. # # live_path = "/live" -# # ready_path: HTTP resource path for checking server readiness. Default: /ready. +# # ready_path: HTTP resource path for checking agent readiness. Default: /ready. # # ready_path = "/ready" + +# # checking_readiness_interval: Interval for checking agent readiness. Default: 1m. +# # checking_readiness_interval = "1m" # } diff --git a/conf/server/server_full.conf b/conf/server/server_full.conf index df331d5046..088d41edea 100644 --- a/conf/server/server_full.conf +++ b/conf/server/server_full.conf @@ -694,4 +694,7 @@ plugins { # # ready_path: HTTP resource path for checking server readiness. Default: /ready. # # ready_path = "/ready" + +# # checking_readiness_interval: Interval for checking server readiness. Default: 1m. +# # checking_readiness_interval = "1m" # } diff --git a/doc/spire_agent.md b/doc/spire_agent.md index 5864609d92..ed53b81893 100644 --- a/doc/spire_agent.md +++ b/doc/spire_agent.md @@ -112,6 +112,7 @@ health_checks { bind_port = "80" live_path = "/live" ready_path = "/ready" + checking_readiness_interval = "1m" } ``` diff --git a/doc/spire_server.md b/doc/spire_server.md index 20a1a70592..40c9c34032 100644 --- a/doc/spire_server.md +++ b/doc/spire_server.md @@ -188,6 +188,7 @@ health_checks { bind_port = "80" live_path = "/live" ready_path = "/ready" + checking_readiness_interval = "1m" } ``` diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 767c27cb10..620c2f4b63 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -2,6 +2,7 @@ package agent import ( "context" + "errors" "fmt" "net/http" _ "net/http/pprof" //nolint: gosec // import registers routes on DefaultServeMux @@ -9,9 +10,9 @@ import ( "path" "runtime" "sync" - "time" "github.com/spiffe/go-spiffe/v2/spiffeid" + api_workload "github.com/spiffe/spire/api/workload" admin_api "github.com/spiffe/spire/pkg/agent/api" node_attestor "github.com/spiffe/spire/pkg/agent/attestor/node" workload_attestor "github.com/spiffe/spire/pkg/agent/attestor/workload" @@ -30,6 +31,8 @@ import ( "github.com/spiffe/spire/proto/spire/api/server/bundle/v1" _ "golang.org/x/net/trace" // registers handlers on the DefaultServeMux "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type Agent struct { @@ -97,7 +100,7 @@ func (a *Agent) Run(ctx context.Context) error { endpoints := a.newEndpoints(cat, metrics, manager) - if err := healthChecks.AddCheck("agent", a, time.Minute); err != nil { + if err := healthChecks.AddCheck("agent", a); err != nil { return fmt.Errorf("failed adding healthcheck: %v", err) } @@ -261,5 +264,23 @@ func (a *Agent) agentSVIDPath() string { // Status is used as a top-level health check for the Agent. func (a *Agent) Status() (interface{}, error) { - return nil, nil + client := api_workload.NewX509Client(&api_workload.X509ClientConfig{ + Addr: a.c.BindAddress, + FailOnError: true, + }) + defer client.Stop() + + errCh := make(chan error, 1) + go func() { + errCh <- client.Start() + }() + + err := <-errCh + if status.Code(err) == codes.Unavailable { + return nil, errors.New("workload api is unavailable") //nolint: golint // error is (ab)used for CLI output + } + + return health.Details{ + Message: "successfully created a workload api client to fetch x509 svid", + }, nil } diff --git a/pkg/common/health/config.go b/pkg/common/health/config.go index 1c2ce2db5c..17ad985739 100644 --- a/pkg/common/health/config.go +++ b/pkg/common/health/config.go @@ -15,9 +15,27 @@ type Config struct { ReadyPath string `hcl:"ready_path"` LivePath string `hcl:"live_path"` + // Interval for checking readiness + CheckingReadinessInterval string `hcl:"checking_readiness_interval"` + UnusedKeys []string `hcl:",unusedKeys"` } +// getAddress returns an address suitable for use as http.Server.Addr. +func (c *Config) getAddress() string { + host := "localhost" + if c.BindAddress != "" { + host = c.BindAddress + } + + port := "80" + if c.BindPort != "" { + port = c.BindPort + } + + return fmt.Sprintf("%s:%s", host, port) +} + // getReadyPath returns the configured value or a default func (c *Config) getReadyPath() string { if c.ReadyPath == "" { @@ -36,17 +54,16 @@ func (c *Config) getLivePath() string { return c.LivePath } -// getAddress returns an address suitable for use as http.Server.Addr. -func (c *Config) getAddress() string { - host := "localhost" - if c.BindAddress != "" { - host = c.BindAddress +// getCheckingReadinessInterval returns the configured value or a default +func (c *Config) getCheckingReadinessInterval() string { + if c.CheckingReadinessInterval == "" { + return "1m" } - port := "80" - if c.BindPort != "" { - port = c.BindPort - } + return c.CheckingReadinessInterval +} - return fmt.Sprintf("%s:%s", host, port) +// Details are additional data to be used when the system is ready +type Details struct { + Message string `json:"message,omitempty"` } diff --git a/pkg/common/health/health.go b/pkg/common/health/health.go index 8b8ecd663c..a18e2d8897 100644 --- a/pkg/common/health/health.go +++ b/pkg/common/health/health.go @@ -2,6 +2,7 @@ package health import ( "context" + "fmt" "net/http" "sync" "time" @@ -46,16 +47,22 @@ func NewChecker(config Config, log logrus.FieldLogger) *Checker { } } - hc.StatusListener = &statusListener{} - hc.Logger = &logadapter{FieldLogger: log.WithField(telemetry.SubsystemName, "health")} + l := log.WithField(telemetry.SubsystemName, "health") + hc.StatusListener = &statusListener{log: l} + hc.Logger = &logadapter{FieldLogger: l} return &Checker{config: config, server: server, hc: hc, log: log} } -func (c *Checker) AddCheck(name string, checker health.ICheckable, interval time.Duration) error { +func (c *Checker) AddCheck(name string, checker health.ICheckable) error { c.mutex.Lock() defer c.mutex.Unlock() + interval, err := time.ParseDuration(c.config.getCheckingReadinessInterval()) + if err != nil { + return fmt.Errorf("could not parse interval for checking readiness %q: %v", c.config.getCheckingReadinessInterval(), err) + } + return c.hc.AddCheck(&health.Config{ Name: name, Checker: checker, diff --git a/pkg/server/server.go b/pkg/server/server.go index c1e159a485..ecbab1a42a 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -2,6 +2,7 @@ package server import ( "context" + "errors" "fmt" "net/http" _ "net/http/pprof" //nolint: gosec // import registers routes on DefaultServeMux @@ -9,10 +10,10 @@ import ( "os" "runtime" "sync" - "time" "github.com/andres-erbsen/clock" "github.com/spiffe/go-spiffe/v2/spiffeid" + server_util "github.com/spiffe/spire/cmd/spire-server/util" "github.com/spiffe/spire/pkg/common/health" "github.com/spiffe/spire/pkg/common/hostservices/metricsservice" common_services "github.com/spiffe/spire/pkg/common/plugin/hostservices" @@ -30,6 +31,7 @@ import ( "github.com/spiffe/spire/pkg/server/plugin/hostservices" "github.com/spiffe/spire/pkg/server/registration" "github.com/spiffe/spire/pkg/server/svid" + "github.com/spiffe/spire/proto/spire/api/server/bundle/v1" "google.golang.org/grpc" ) @@ -159,7 +161,7 @@ func (s *Server) run(ctx context.Context) (err error) { registrationManager := s.newRegistrationManager(cat, metrics) - if err := healthChecks.AddCheck("server", s, time.Minute); err != nil { + if err := healthChecks.AddCheck("server", s); err != nil { return fmt.Errorf("failed adding healthcheck: %v", err) } @@ -385,5 +387,21 @@ func (s *Server) validateTrustDomain(ctx context.Context, ds datastore.DataStore // Status is used as a top-level health check for the Server. func (s *Server) Status() (interface{}, error) { - return nil, nil + client, err := server_util.NewServerClient(s.config.BindUDSAddress.Name) + if err != nil { + return nil, errors.New("cannot create registration client") + } + bundleClient := client.NewBundleClient() + + // Currently using the ability to fetch a bundle as the health check. This + // **could** be problematic if the Upstream CA signing process is lengthy. + // As currently coded however, the registration API isn't served until after + // the server CA has been signed by upstream. + if _, err := bundleClient.GetBundle(context.Background(), &bundle.GetBundleRequest{}); err != nil { + return nil, errors.New("unable to fetch bundle") + } + + return health.Details{ + Message: "successfully fetched bundle", + }, nil } diff --git a/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml b/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml index eea78dcce1..b38ec8d7c9 100644 --- a/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml +++ b/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml @@ -78,6 +78,15 @@ data: } } + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "80" + live_path = "/live" + ready_path = "/ready" + checking_readiness_interval = "5s" + } + --- apiVersion: apps/v1 @@ -129,13 +138,15 @@ spec: - name: spire-token mountPath: /var/run/secrets/tokens livenessProbe: - exec: - command: ["/opt/spire/bin/spire-agent", "healthcheck", "-socketPath", "/run/spire/sockets/agent.sock"] + httpGet: + path: /live + port: 80 initialDelaySeconds: 10 periodSeconds: 10 readinessProbe: - exec: - command: ["/opt/spire/bin/spire-agent", "healthcheck", "-socketPath", "/run/spire/sockets/agent.sock", "--shallow"] + httpGet: + path: /ready + port: 80 initialDelaySeconds: 10 periodSeconds: 10 volumes: @@ -151,8 +162,8 @@ spec: type: DirectoryOrCreate - name: spire-token projected: - sources: - - serviceAccountToken: - path: spire-agent - expirationSeconds: 7200 - audience: spire-server + sources: + - serviceAccountToken: + path: spire-agent + expirationSeconds: 7200 + audience: spire-server diff --git a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml index 9db7228033..bf5a897e2e 100644 --- a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml +++ b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml @@ -157,6 +157,15 @@ data: } } + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "80" + live_path = "/live" + ready_path = "/ready" + checking_readiness_interval = "5s" + } + --- apiVersion: v1 @@ -212,13 +221,15 @@ spec: mountPath: /run/spire/sockets readOnly: false livenessProbe: - exec: - command: ["/opt/spire/bin/spire-server", "healthcheck", "-registrationUDSPath", "/run/spire/sockets/registration.sock"] + httpGet: + path: /live + port: 80 initialDelaySeconds: 5 periodSeconds: 5 readinessProbe: - exec: - command: ["/opt/spire/bin/spire-server", "healthcheck", "-registrationUDSPath", "/run/spire/sockets/registration.sock", "--shallow"] + httpGet: + path: /ready + port: 80 initialDelaySeconds: 5 periodSeconds: 5 - name: k8s-workload-registrar @@ -279,5 +290,3 @@ spec: ports: - port: 443 targetPort: registrar-port - - diff --git a/test/integration/suites/k8s/conf/agent/spire-agent.yaml b/test/integration/suites/k8s/conf/agent/spire-agent.yaml index eea78dcce1..b38ec8d7c9 100644 --- a/test/integration/suites/k8s/conf/agent/spire-agent.yaml +++ b/test/integration/suites/k8s/conf/agent/spire-agent.yaml @@ -78,6 +78,15 @@ data: } } + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "80" + live_path = "/live" + ready_path = "/ready" + checking_readiness_interval = "5s" + } + --- apiVersion: apps/v1 @@ -129,13 +138,15 @@ spec: - name: spire-token mountPath: /var/run/secrets/tokens livenessProbe: - exec: - command: ["/opt/spire/bin/spire-agent", "healthcheck", "-socketPath", "/run/spire/sockets/agent.sock"] + httpGet: + path: /live + port: 80 initialDelaySeconds: 10 periodSeconds: 10 readinessProbe: - exec: - command: ["/opt/spire/bin/spire-agent", "healthcheck", "-socketPath", "/run/spire/sockets/agent.sock", "--shallow"] + httpGet: + path: /ready + port: 80 initialDelaySeconds: 10 periodSeconds: 10 volumes: @@ -151,8 +162,8 @@ spec: type: DirectoryOrCreate - name: spire-token projected: - sources: - - serviceAccountToken: - path: spire-agent - expirationSeconds: 7200 - audience: spire-server + sources: + - serviceAccountToken: + path: spire-agent + expirationSeconds: 7200 + audience: spire-server diff --git a/test/integration/suites/k8s/conf/server/spire-server.yaml b/test/integration/suites/k8s/conf/server/spire-server.yaml index 7b8f74d62d..ebe3a4b63b 100644 --- a/test/integration/suites/k8s/conf/server/spire-server.yaml +++ b/test/integration/suites/k8s/conf/server/spire-server.yaml @@ -147,6 +147,15 @@ data: } } + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "80" + live_path = "/live" + ready_path = "/ready" + checking_readiness_interval = "5s" + } + --- apiVersion: v1 @@ -156,9 +165,9 @@ metadata: namespace: spire data: k8s-workload-registrar.conf: | - cert_path = "/run/spire/k8s-workload-registrar/certs/server-cert.pem" - key_path = "/run/spire/k8s-workload-registrar/secret/server-key.pem" - cacert_path = "/run/spire/k8s-workload-registrar/certs/cacert.pem" + cert_path = "/run/spire/k8s-workload-registrar/certs/server-cert.pem" + key_path = "/run/spire/k8s-workload-registrar/secret/server-key.pem" + cacert_path = "/run/spire/k8s-workload-registrar/certs/cacert.pem" trust_domain = "example.org" cluster = "example-cluster" server_socket_path = "/run/spire/sockets/registration.sock" @@ -242,13 +251,15 @@ spec: mountPath: /run/spire/sockets readOnly: false livenessProbe: - exec: - command: ["/opt/spire/bin/spire-server", "healthcheck", "-registrationUDSPath", "/run/spire/sockets/registration.sock"] + httpGet: + path: /live + port: 80 initialDelaySeconds: 5 periodSeconds: 5 readinessProbe: - exec: - command: ["/opt/spire/bin/spire-server", "healthcheck", "-registrationUDSPath", "/run/spire/sockets/registration.sock", "--shallow"] + httpGet: + path: /ready + port: 80 initialDelaySeconds: 5 periodSeconds: 5 - name: k8s-workload-registrar @@ -321,5 +332,3 @@ spec: ports: - port: 443 targetPort: registrar-port - - From fd5512c3cb3af72472ee82e779955ff194392542 Mon Sep 17 00:00:00 2001 From: Ryuma Yoshida Date: Tue, 5 Jan 2021 11:06:43 +0900 Subject: [PATCH 2/7] Add missing package Signed-off-by: Ryuma Yoshida --- pkg/server/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/server/server.go b/pkg/server/server.go index 830d28f0a5..79ff0a8c00 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -12,6 +12,7 @@ import ( "sync" "github.com/andres-erbsen/clock" + server_util "github.com/spiffe/spire/cmd/spire-server/util" "github.com/spiffe/spire/pkg/common/health" "github.com/spiffe/spire/pkg/common/hostservices/metricsservice" common_services "github.com/spiffe/spire/pkg/common/plugin/hostservices" From 703c6244e47925046a0f603ccd61584444af3d5b Mon Sep 17 00:00:00 2001 From: Ryuma Yoshida Date: Wed, 13 Jan 2021 14:29:46 +0900 Subject: [PATCH 3/7] Remove the checking_readiness_interval parameter Signed-off-by: Ryuma Yoshida --- conf/agent/agent_full.conf | 3 --- conf/server/server_full.conf | 3 --- doc/spire_agent.md | 1 - doc/spire_server.md | 1 - pkg/common/health/config.go | 12 ------------ pkg/common/health/health.go | 10 +++------- .../suites/k8s-reconcile/conf/agent/spire-agent.yaml | 1 - .../k8s-reconcile/conf/server/spire-server.yaml | 1 - .../suites/k8s/conf/agent/spire-agent.yaml | 1 - .../suites/k8s/conf/server/spire-server.yaml | 1 - 10 files changed, 3 insertions(+), 31 deletions(-) diff --git a/conf/agent/agent_full.conf b/conf/agent/agent_full.conf index 2fff120379..d776f7ac9d 100644 --- a/conf/agent/agent_full.conf +++ b/conf/agent/agent_full.conf @@ -317,7 +317,4 @@ plugins { # # ready_path: HTTP resource path for checking agent readiness. Default: /ready. # # ready_path = "/ready" - -# # checking_readiness_interval: Interval for checking agent readiness. Default: 1m. -# # checking_readiness_interval = "1m" # } diff --git a/conf/server/server_full.conf b/conf/server/server_full.conf index 1478cbd3a2..e608a0fa2c 100644 --- a/conf/server/server_full.conf +++ b/conf/server/server_full.conf @@ -698,7 +698,4 @@ plugins { # # ready_path: HTTP resource path for checking server readiness. Default: /ready. # # ready_path = "/ready" - -# # checking_readiness_interval: Interval for checking server readiness. Default: 1m. -# # checking_readiness_interval = "1m" # } diff --git a/doc/spire_agent.md b/doc/spire_agent.md index ed53b81893..5864609d92 100644 --- a/doc/spire_agent.md +++ b/doc/spire_agent.md @@ -112,7 +112,6 @@ health_checks { bind_port = "80" live_path = "/live" ready_path = "/ready" - checking_readiness_interval = "1m" } ``` diff --git a/doc/spire_server.md b/doc/spire_server.md index 4db49ee8d3..50a80f9f88 100644 --- a/doc/spire_server.md +++ b/doc/spire_server.md @@ -188,7 +188,6 @@ health_checks { bind_port = "80" live_path = "/live" ready_path = "/ready" - checking_readiness_interval = "1m" } ``` diff --git a/pkg/common/health/config.go b/pkg/common/health/config.go index 17ad985739..af3eea05b0 100644 --- a/pkg/common/health/config.go +++ b/pkg/common/health/config.go @@ -15,9 +15,6 @@ type Config struct { ReadyPath string `hcl:"ready_path"` LivePath string `hcl:"live_path"` - // Interval for checking readiness - CheckingReadinessInterval string `hcl:"checking_readiness_interval"` - UnusedKeys []string `hcl:",unusedKeys"` } @@ -54,15 +51,6 @@ func (c *Config) getLivePath() string { return c.LivePath } -// getCheckingReadinessInterval returns the configured value or a default -func (c *Config) getCheckingReadinessInterval() string { - if c.CheckingReadinessInterval == "" { - return "1m" - } - - return c.CheckingReadinessInterval -} - // Details are additional data to be used when the system is ready type Details struct { Message string `json:"message,omitempty"` diff --git a/pkg/common/health/health.go b/pkg/common/health/health.go index a18e2d8897..7201bb0c01 100644 --- a/pkg/common/health/health.go +++ b/pkg/common/health/health.go @@ -2,7 +2,6 @@ package health import ( "context" - "fmt" "net/http" "sync" "time" @@ -13,6 +12,8 @@ import ( "github.com/spiffe/spire/pkg/common/telemetry" ) +const readyCheckInterval = time.Minute + // health.Checker is responsible for running health checks and serving the healthcheck HTTP paths type Checker struct { config Config @@ -58,15 +59,10 @@ func (c *Checker) AddCheck(name string, checker health.ICheckable) error { c.mutex.Lock() defer c.mutex.Unlock() - interval, err := time.ParseDuration(c.config.getCheckingReadinessInterval()) - if err != nil { - return fmt.Errorf("could not parse interval for checking readiness %q: %v", c.config.getCheckingReadinessInterval(), err) - } - return c.hc.AddCheck(&health.Config{ Name: name, Checker: checker, - Interval: interval, + Interval: readyCheckInterval, Fatal: true, }) } diff --git a/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml b/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml index b38ec8d7c9..c1c64972c7 100644 --- a/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml +++ b/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml @@ -84,7 +84,6 @@ data: bind_port = "80" live_path = "/live" ready_path = "/ready" - checking_readiness_interval = "5s" } --- diff --git a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml index 084031278a..1e3a406a59 100644 --- a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml +++ b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml @@ -163,7 +163,6 @@ data: bind_port = "80" live_path = "/live" ready_path = "/ready" - checking_readiness_interval = "5s" } --- diff --git a/test/integration/suites/k8s/conf/agent/spire-agent.yaml b/test/integration/suites/k8s/conf/agent/spire-agent.yaml index b38ec8d7c9..c1c64972c7 100644 --- a/test/integration/suites/k8s/conf/agent/spire-agent.yaml +++ b/test/integration/suites/k8s/conf/agent/spire-agent.yaml @@ -84,7 +84,6 @@ data: bind_port = "80" live_path = "/live" ready_path = "/ready" - checking_readiness_interval = "5s" } --- diff --git a/test/integration/suites/k8s/conf/server/spire-server.yaml b/test/integration/suites/k8s/conf/server/spire-server.yaml index b50d3b8bf0..c001254e41 100644 --- a/test/integration/suites/k8s/conf/server/spire-server.yaml +++ b/test/integration/suites/k8s/conf/server/spire-server.yaml @@ -153,7 +153,6 @@ data: bind_port = "80" live_path = "/live" ready_path = "/ready" - checking_readiness_interval = "5s" } --- From 4a3c7451ee88e259f44a173626e69e6cb82dcb03 Mon Sep 17 00:00:00 2001 From: Ryuma Yoshida Date: Wed, 13 Jan 2021 14:45:37 +0900 Subject: [PATCH 4/7] Avoid the gRPC connection leaks in the health system Signed-off-by: Ryuma Yoshida --- cmd/spire-server/cli/healthcheck/healthcheck.go | 2 ++ pkg/server/server.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmd/spire-server/cli/healthcheck/healthcheck.go b/cmd/spire-server/cli/healthcheck/healthcheck.go index 44e3fd19a6..fec5d9a855 100644 --- a/cmd/spire-server/cli/healthcheck/healthcheck.go +++ b/cmd/spire-server/cli/healthcheck/healthcheck.go @@ -83,6 +83,8 @@ func (c *healthCheckCommand) run() error { } return errors.New("cannot create registration client") } + defer client.Release() + bundleClient := client.NewBundleClient() // Currently using the ability to fetch a bundle as the health check. This diff --git a/pkg/server/server.go b/pkg/server/server.go index 79ff0a8c00..6634c09a04 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -390,6 +390,8 @@ func (s *Server) Status() (interface{}, error) { if err != nil { return nil, errors.New("cannot create registration client") } + defer client.Release() + bundleClient := client.NewBundleClient() // Currently using the ability to fetch a bundle as the health check. This From 358825e1da313168c652427dc02f419b17f8466b Mon Sep 17 00:00:00 2001 From: Ryuma Yoshida Date: Wed, 13 Jan 2021 14:55:46 +0900 Subject: [PATCH 5/7] Update the bind_port in the health_checks block example Signed-off-by: Ryuma Yoshida --- conf/agent/agent_full.conf | 2 +- conf/server/server_full.conf | 2 +- doc/spire_agent.md | 2 +- doc/spire_server.md | 2 +- .../suites/k8s-reconcile/conf/agent/spire-agent.yaml | 6 +++--- .../suites/k8s-reconcile/conf/server/spire-server.yaml | 6 +++--- test/integration/suites/k8s/conf/agent/spire-agent.yaml | 6 +++--- test/integration/suites/k8s/conf/server/spire-server.yaml | 6 +++--- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/conf/agent/agent_full.conf b/conf/agent/agent_full.conf index d776f7ac9d..583ed87f1c 100644 --- a/conf/agent/agent_full.conf +++ b/conf/agent/agent_full.conf @@ -310,7 +310,7 @@ plugins { # # bind_address = "localhost" # # bind_port: HTTP Port number of the health checks endpoint. Default: 80. -# # bind_port = "80" +# # bind_port = "8080" # # live_path: HTTP resource path for checking agent liveness. Default: /live. # # live_path = "/live" diff --git a/conf/server/server_full.conf b/conf/server/server_full.conf index e608a0fa2c..2de0615598 100644 --- a/conf/server/server_full.conf +++ b/conf/server/server_full.conf @@ -691,7 +691,7 @@ plugins { # # bind_address = "localhost" # # bind_port: HTTP Port number of the health checks endpoint. Default: 80. -# # bind_port = "80" +# # bind_port = "8080" # # live_path: HTTP resource path for checking server liveness. Default: /live. # # live_path = "/live" diff --git a/doc/spire_agent.md b/doc/spire_agent.md index 5864609d92..04021160e1 100644 --- a/doc/spire_agent.md +++ b/doc/spire_agent.md @@ -109,7 +109,7 @@ The agent can expose additional endpoint that can be used for health checking. I health_checks { listener_enabled = true bind_address = "localhost" - bind_port = "80" + bind_port = "8080" live_path = "/live" ready_path = "/ready" } diff --git a/doc/spire_server.md b/doc/spire_server.md index 50a80f9f88..0f162b6cd5 100644 --- a/doc/spire_server.md +++ b/doc/spire_server.md @@ -185,7 +185,7 @@ The server can expose an additional endpoint that can be used for health checkin health_checks { listener_enabled = true bind_address = "localhost" - bind_port = "80" + bind_port = "8080" live_path = "/live" ready_path = "/ready" } diff --git a/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml b/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml index c1c64972c7..a0992f27b9 100644 --- a/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml +++ b/test/integration/suites/k8s-reconcile/conf/agent/spire-agent.yaml @@ -81,7 +81,7 @@ data: health_checks { listener_enabled = true bind_address = "0.0.0.0" - bind_port = "80" + bind_port = "8080" live_path = "/live" ready_path = "/ready" } @@ -139,13 +139,13 @@ spec: livenessProbe: httpGet: path: /live - port: 80 + port: 8080 initialDelaySeconds: 10 periodSeconds: 10 readinessProbe: httpGet: path: /ready - port: 80 + port: 8080 initialDelaySeconds: 10 periodSeconds: 10 volumes: diff --git a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml index 1e3a406a59..a91ae54e09 100644 --- a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml +++ b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml @@ -160,7 +160,7 @@ data: health_checks { listener_enabled = true bind_address = "0.0.0.0" - bind_port = "80" + bind_port = "8080" live_path = "/live" ready_path = "/ready" } @@ -222,13 +222,13 @@ spec: livenessProbe: httpGet: path: /live - port: 80 + port: 8080 initialDelaySeconds: 5 periodSeconds: 5 readinessProbe: httpGet: path: /ready - port: 80 + port: 8080 initialDelaySeconds: 5 periodSeconds: 5 - name: k8s-workload-registrar diff --git a/test/integration/suites/k8s/conf/agent/spire-agent.yaml b/test/integration/suites/k8s/conf/agent/spire-agent.yaml index c1c64972c7..a0992f27b9 100644 --- a/test/integration/suites/k8s/conf/agent/spire-agent.yaml +++ b/test/integration/suites/k8s/conf/agent/spire-agent.yaml @@ -81,7 +81,7 @@ data: health_checks { listener_enabled = true bind_address = "0.0.0.0" - bind_port = "80" + bind_port = "8080" live_path = "/live" ready_path = "/ready" } @@ -139,13 +139,13 @@ spec: livenessProbe: httpGet: path: /live - port: 80 + port: 8080 initialDelaySeconds: 10 periodSeconds: 10 readinessProbe: httpGet: path: /ready - port: 80 + port: 8080 initialDelaySeconds: 10 periodSeconds: 10 volumes: diff --git a/test/integration/suites/k8s/conf/server/spire-server.yaml b/test/integration/suites/k8s/conf/server/spire-server.yaml index c001254e41..cbc804cd66 100644 --- a/test/integration/suites/k8s/conf/server/spire-server.yaml +++ b/test/integration/suites/k8s/conf/server/spire-server.yaml @@ -150,7 +150,7 @@ data: health_checks { listener_enabled = true bind_address = "0.0.0.0" - bind_port = "80" + bind_port = "8080" live_path = "/live" ready_path = "/ready" } @@ -252,13 +252,13 @@ spec: livenessProbe: httpGet: path: /live - port: 80 + port: 8080 initialDelaySeconds: 5 periodSeconds: 5 readinessProbe: httpGet: path: /ready - port: 80 + port: 8080 initialDelaySeconds: 5 periodSeconds: 5 - name: k8s-workload-registrar From a79c88c85d288c271ebef0c529f8706f99916396 Mon Sep 17 00:00:00 2001 From: Ryuma Yoshida Date: Wed, 13 Jan 2021 15:44:40 +0900 Subject: [PATCH 6/7] Fix suites/k8s-reconcile Signed-off-by: Ryuma Yoshida --- .../suites/k8s-reconcile/conf/server/spire-server.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml index a91ae54e09..9adad1704d 100644 --- a/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml +++ b/test/integration/suites/k8s-reconcile/conf/server/spire-server.yaml @@ -179,6 +179,7 @@ data: cluster = "example-cluster" server_socket_path = "/run/spire/sockets/registration.sock" leader_election = true + metrics_addr = "0.0.0.0:18080" --- From 1fb4a0d24fc665119e7c03a7e45b420ce563914d Mon Sep 17 00:00:00 2001 From: Ryuma Yoshida Date: Sat, 16 Jan 2021 22:31:22 +0900 Subject: [PATCH 7/7] Address PR comments Signed-off-by: Ryuma Yoshida --- conf/agent/agent_full.conf | 2 +- conf/server/server_full.conf | 2 +- pkg/server/server.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/agent/agent_full.conf b/conf/agent/agent_full.conf index 583ed87f1c..d776f7ac9d 100644 --- a/conf/agent/agent_full.conf +++ b/conf/agent/agent_full.conf @@ -310,7 +310,7 @@ plugins { # # bind_address = "localhost" # # bind_port: HTTP Port number of the health checks endpoint. Default: 80. -# # bind_port = "8080" +# # bind_port = "80" # # live_path: HTTP resource path for checking agent liveness. Default: /live. # # live_path = "/live" diff --git a/conf/server/server_full.conf b/conf/server/server_full.conf index 2de0615598..e608a0fa2c 100644 --- a/conf/server/server_full.conf +++ b/conf/server/server_full.conf @@ -691,7 +691,7 @@ plugins { # # bind_address = "localhost" # # bind_port: HTTP Port number of the health checks endpoint. Default: 80. -# # bind_port = "8080" +# # bind_port = "80" # # live_path: HTTP resource path for checking server liveness. Default: /live. # # live_path = "/live" diff --git a/pkg/server/server.go b/pkg/server/server.go index 6634c09a04..ce54e5208d 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -396,7 +396,7 @@ func (s *Server) Status() (interface{}, error) { // Currently using the ability to fetch a bundle as the health check. This // **could** be problematic if the Upstream CA signing process is lengthy. - // As currently coded however, the registration API isn't served until after + // As currently coded however, the API isn't served until after // the server CA has been signed by upstream. if _, err := bundleClient.GetBundle(context.Background(), &bundle.GetBundleRequest{}); err != nil { return nil, errors.New("unable to fetch bundle")