Report uptime as metrics for spire server and agent with config

doc/telemetry.md

@@ -74,7 +74,8 @@ The following metrics are emitted:
 | Counter | `server_ca`, `sign`, `x509_ca_svid` | | The CA has successfully signed an X.509 CA SVID.
 | Counter | `server_ca`, `sign`, `x509_svid` | | The CA has successfully signed an X.509 SVID.
 | Call Counter | `svid`, `rotate` | | The Server's SVID is being rotated.
-| Gauge | `started` | `version` | | The version of the Server.
+| Gauge | `started` | `version` | The version of the Server.
+| Gauge | `uptime_in_ms` |  | The uptime of the Server in millisecond.
 
 ## SPIRE Agent
 
@@ -99,5 +100,6 @@ The following metrics are emitted:
 | Call Counter | `workload_api`, `workload_attestation` | | The Workload API is performing a workload attestation.
 | Call Counter | `workload_api`, `workload_attestor` | `attestor` | The Workload API is invoking a given attestor.
 | Gauge | `started` | `version` | The version of the Agent.
+| Gauge | `uptime_in_ms` |  | The uptime of the Agent in millisecond.
 
 Note: These are the keys and labels that SPIRE emits, but the format of the metric once ingested could vary depending on the metric collector. E.g. once in StatsD, the metric emitted when rotating an Agent SVID (`agent_svid`, `rotate`) can be found as `spire_agent_agent_svid_rotate_internal_host-agent-0`, where `host-agent-0` is the hostname and `spire-agent` is the service name.

pkg/agent/agent.go

@@ -66,6 +66,7 @@ func (a *Agent) Run(ctx context.Context) error {
 	})
 
 	telemetry.EmitVersion(metrics)
+	uptime.ReportMetrics(ctx, metrics)
 
 	cat, err := catalog.Load(ctx, catalog.Config{
 		Log: a.c.Log.WithField(telemetry.SubsystemName, telemetry.Catalog),

pkg/common/telemetry/uptime.go

@@ -0,0 +1,5 @@
+package telemetry
+
+func EmitUptime(m Metrics, v float32) {
+	m.SetGauge([]string{"uptime_in_ms"}, v)
+}

pkg/common/uptime/uptime.go

@@ -1,9 +1,38 @@
 package uptime
 
-import "time"
+import (
+	"context"
+	"time"
 
-var start = time.Now()
+	"github.com/andres-erbsen/clock"
+	"github.com/spiffe/spire/pkg/common/telemetry"
+)
+
+// by default, report every 10 seconds.
+const _defaultReportInterval = time.Second * 10
+
+var (
+	clk   = clock.New()
+	start = clk.Now()
+)
 
 func Uptime() time.Duration {
-	return time.Since(start)
+	return clk.Now().Sub(start)
+}
+
+func reportMetrics(ctx context.Context, interval time.Duration, m telemetry.Metrics) {
+	t := clk.Ticker(interval)
+	defer t.Stop()
+	for {
+		telemetry.EmitUptime(m, float32(Uptime()/time.Millisecond))
+		select {
+		case <-t.C:
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func ReportMetrics(ctx context.Context, metrics telemetry.Metrics) {
+	go reportMetrics(ctx, _defaultReportInterval, metrics)
 }

pkg/common/uptime/uptime_test.go

@@ -0,0 +1,44 @@
+package uptime
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/spiffe/spire/test/clock"
+	"github.com/spiffe/spire/test/fakes/fakemetrics"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReportMetrics(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+
+	for _, tt := range []struct {
+		name            string
+		reportInterval  time.Duration
+		testUpTime      time.Duration
+		expectedMetrics []fakemetrics.MetricItem
+	}{
+		{
+			name:           "report uptime metrics with 10 millisecond internal",
+			reportInterval: 10 * time.Millisecond,
+			testUpTime:     200 * time.Millisecond,
+			expectedMetrics: []fakemetrics.MetricItem{
+				{Type: fakemetrics.SetGaugeType, Key: []string{"uptime_in_ms"}, Val: 200},
+			},
+		},
+	} {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(ctx)
+			defer cancel()
+			metrics := fakemetrics.New()
+			// overwrite the variable to use mock clock.
+			clk = clock.NewMock(t)
+			start = clk.Now().Add(-tt.testUpTime)
+			reportMetrics(ctx, 0*time.Nanosecond, metrics)
+			assert.Equal(t, tt.expectedMetrics, metrics.AllMetrics())
+		})
+	}
+}

pkg/server/server.go

@@ -84,6 +84,7 @@ func (s *Server) run(ctx context.Context) (err error) {
 	})
 
 	telemetry.EmitVersion(metrics)
+	uptime.ReportMetrics(ctx, metrics)
 
 	// Create the identity provider host service. It will not be functional
 	// until the call to SetDeps() below. There is some tricky initialization