diff --git a/doc/telemetry.md b/doc/telemetry.md index a7c2862567..afbafe5a0b 100644 --- a/doc/telemetry.md +++ b/doc/telemetry.md @@ -74,7 +74,8 @@ The following metrics are emitted: | Counter | `server_ca`, `sign`, `x509_ca_svid` | | The CA has successfully signed an X.509 CA SVID. | Counter | `server_ca`, `sign`, `x509_svid` | | The CA has successfully signed an X.509 SVID. | Call Counter | `svid`, `rotate` | | The Server's SVID is being rotated. -| Gauge | `started` | `version` | | The version of the Server. +| Gauge | `started` | `version` | The version of the Server. +| Gauge | `uptime_in_ms` | | The uptime of the Server in milliseconds. ## SPIRE Agent @@ -99,5 +100,6 @@ The following metrics are emitted: | Call Counter | `workload_api`, `workload_attestation` | | The Workload API is performing a workload attestation. | Call Counter | `workload_api`, `workload_attestor` | `attestor` | The Workload API is invoking a given attestor. | Gauge | `started` | `version` | The version of the Agent. +| Gauge | `uptime_in_ms` | | The uptime of the Agent in milliseconds. Note: These are the keys and labels that SPIRE emits, but the format of the metric once ingested could vary depending on the metric collector. E.g. once in StatsD, the metric emitted when rotating an Agent SVID (`agent_svid`, `rotate`) can be found as `spire_agent_agent_svid_rotate_internal_host-agent-0`, where `host-agent-0` is the hostname and `spire-agent` is the service name. diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index d1610a95b0..657db0999d 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -69,6 +69,7 @@ func (a *Agent) Run(ctx context.Context) error { }) telemetry.EmitVersion(metrics) + uptime.ReportMetrics(ctx, metrics) cat, err := catalog.Load(ctx, catalog.Config{ Log: a.c.Log.WithField(telemetry.SubsystemName, telemetry.Catalog), diff --git a/pkg/common/telemetry/uptime.go b/pkg/common/telemetry/uptime.go new file mode 100644 index 0000000000..72a0513e68 --- /dev/null +++ b/pkg/common/telemetry/uptime.go @@ -0,0 +1,5 @@ +package telemetry + +func EmitUptime(m Metrics, v float32) { + m.SetGauge([]string{"uptime_in_ms"}, v) +} diff --git a/pkg/common/uptime/uptime.go b/pkg/common/uptime/uptime.go index 41d19b875e..9c6231be58 100644 --- a/pkg/common/uptime/uptime.go +++ b/pkg/common/uptime/uptime.go @@ -1,9 +1,38 @@ package uptime -import "time" +import ( + "context" + "time" -var start = time.Now() + "github.com/andres-erbsen/clock" + "github.com/spiffe/spire/pkg/common/telemetry" +) + +// Report every 10 seconds. +const reportInterval = time.Second * 10 + +var ( + clk = clock.New() + start = clk.Now() +) func Uptime() time.Duration { - return time.Since(start) + return clk.Now().Sub(start) +} + +func reportMetrics(ctx context.Context, interval time.Duration, m telemetry.Metrics) { + t := clk.Ticker(interval) + defer t.Stop() + for { + telemetry.EmitUptime(m, float32(Uptime()/time.Millisecond)) + select { + case <-t.C: + case <-ctx.Done(): + return + } + } +} + +func ReportMetrics(ctx context.Context, metrics telemetry.Metrics) { + go reportMetrics(ctx, reportInterval, metrics) } diff --git a/pkg/common/uptime/uptime_test.go b/pkg/common/uptime/uptime_test.go new file mode 100644 index 0000000000..257d66ff87 --- /dev/null +++ b/pkg/common/uptime/uptime_test.go @@ -0,0 +1,44 @@ +package uptime + +import ( + "context" + "testing" + "time" + + "github.com/spiffe/spire/test/clock" + "github.com/spiffe/spire/test/fakes/fakemetrics" + "github.com/stretchr/testify/assert" +) + +func TestReportMetrics(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + for _, tt := range []struct { + name string + reportInterval time.Duration + testUpTime time.Duration + expectedMetrics []fakemetrics.MetricItem + }{ + { + name: "report uptime metrics with 10 milliseconds interval", + reportInterval: 10 * time.Millisecond, + testUpTime: 200 * time.Millisecond, + expectedMetrics: []fakemetrics.MetricItem{ + {Type: fakemetrics.SetGaugeType, Key: []string{"uptime_in_ms"}, Val: 200}, + }, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + metrics := fakemetrics.New() + // overwrite the variable to use mock clock. + clk = clock.NewMock(t) + start = clk.Now().Add(-tt.testUpTime) + reportMetrics(ctx, 0*time.Nanosecond, metrics) + assert.Equal(t, tt.expectedMetrics, metrics.AllMetrics()) + }) + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go index ce54e5208d..e832ab4b17 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -86,6 +86,7 @@ func (s *Server) run(ctx context.Context) (err error) { }) telemetry.EmitVersion(metrics) + uptime.ReportMetrics(ctx, metrics) // Create the identity provider host service. It will not be functional // until the call to SetDeps() below. There is some tricky initialization