Include upgrader version in metrics and inventory

api/proto/teleport/legacy/client/proto/authservice.proto

@@ -2260,6 +2260,8 @@ message UpstreamInventoryHello {
   // ExternalUpgrader identifies the external upgrader that the instance is configured to
   // export schedules to (e.g. 'kube'). Empty if no upgrader is defined.
   string ExternalUpgrader = 5;
+  // ExternalUpgraderVersion identifies the external upgrader version. Empty if no upgrader is defined.
+  string ExternalUpgraderVersion = 6;
 }
 
 // UpstreamInventoryAgentMetadata is the message sent up the inventory control stream containing

api/proto/teleport/legacy/types/types.proto

@@ -676,6 +676,9 @@ message InstanceSpecV1 {
   // ExternalUpgrader identifies the external upgrader that the instance is configured to
   // export schedules to (e.g. 'kube'). Empty if no upgrader is defined.
   string ExternalUpgrader = 7 [(gogoproto.jsontag) = "ext_upgrader,omitempty"];
+
+  // ExternalUpgraderVersion identifies the external upgrader version. Empty if no upgrader is defined.
+  string ExternalUpgraderVersion = 8 [(gogoproto.jsontag) = "ext_upgrader_version,omitempty"];
 }
 
 // InstanceControlLogEntry represents an entry in a given instance's control log. The control log of

api/types/instance.go

@@ -157,6 +157,10 @@ type Instance interface {
 	// upgraders.
 	GetExternalUpgrader() string
 
+	// GetExternalUpgraderVersion gets the reported upgrader version. This value corresponds
+	// to the TELEPORT_EXT_UPGRADER_VERSION env var that is set when agents are configured.
+	GetExternalUpgraderVersion() string
+
 	// SyncLogAndResourceExpiry filters expired entries from the control log and updates
 	// the resource-level expiry. All calculations are performed relative to the value of
 	// the LastSeen field, and the supplied TTL is used only as a default. The actual TTL
@@ -274,6 +278,10 @@ func (i *InstanceV1) GetExternalUpgrader() string {
 	return i.Spec.ExternalUpgrader
 }
 
+func (i *InstanceV1) GetExternalUpgraderVersion() string {
+	return i.Spec.ExternalUpgraderVersion
+}
+
 func (i *InstanceV1) GetControlLog() []InstanceControlLogEntry {
 	return i.Spec.ControlLog
 }

docs/pages/management/operations/enroll-agent-into-automatic-updates.mdx

@@ -45,9 +45,9 @@ If you have a Teleport Enterprise Cloud account, you can find agents that need t
 
 ```code
 $ tctl inventory ls --upgrader=none
-Server ID                            Hostname          Services Version Upgrader
------------------------------------- ----------------- -------- ------- --------
-4fb2d97d-884a-4566-b477-c805d477df09 agent.example.com Node     v1.2.3  none
+Server ID                            Hostname          Services Agent Version Upgrader Upgrader Version
+------------------------------------ ----------------- -------- ------------- -------- ----------------
+4fb2d97d-884a-4566-b477-c805d477df09 agent.example.com Node     v1.2.3        none     none
 ...
 ```
 
@@ -56,9 +56,27 @@ your oldest agents, you can limit your search using the `--older-than` filter:
 
 ```code
 $ tctl inventory ls --upgrader=none --older-than=v1.2.3
-Server ID                            Hostname        Services Version Upgrader
------------------------------------- --------------- -------- ------- --------
-1e6578b6-9530-448e-8013-d32641324abb old.example.com Node     v1.1.1  none
+Server ID                            Hostname        Services Agent Version Upgrader Upgrader Version
+------------------------------------ --------------- -------- ------------- -------- ----------------
+1e6578b6-9530-448e-8013-d32641324abb old.example.com Node     v1.1.1        none     none
+...
+```
+
+The `tctl inventory ls` command will also display the installed version of the Teleport Upgrader, which is versioned independently from Teleport itself:
+```code
+$ tctl inventory ls
+Server ID                            Hostname          Services Agent Version Upgrader Upgrader Version
+------------------------------------ ----------------- -------- ------------- -------- ----------------
+4fb2d97d-884a-4566-b477-c805d477df09 agent.example.com Node     v1.2.3        unit     v14.2.0
+...
+```
+
+If the Upgrader Version displays `none`, that means the Upgrader version is too old to report the version:
+```code
+$ tctl inventory ls
+Server ID                            Hostname          Services Agent Version Upgrader Upgrader Version
+------------------------------------ ----------------- -------- ------------- -------- ----------------
+4fb2d97d-884a-4566-b477-c805d477df09 agent.example.com Node     v1.2.3        unit     none
 ...
 ```
 

examples/chart/teleport-kube-agent/templates/deployment.yaml

@@ -133,6 +133,8 @@ spec:
         {{- if .Values.updater.enabled }}
         - name: TELEPORT_EXT_UPGRADER
           value: kube
+        - name: TELEPORT_EXT_UPGRADER_VERSION
+          value: {{ include "teleport-kube-agent.version" . }}
         {{- end }}
         {{- if (gt (len .Values.extraEnv) 0) }}
           {{- toYaml .Values.extraEnv | nindent 8 }}

examples/chart/teleport-kube-agent/templates/statefulset.yaml

@@ -143,6 +143,8 @@ spec:
           {{- if .Values.updater.enabled }}
           - name: TELEPORT_EXT_UPGRADER
             value: kube
+          - name: TELEPORT_EXT_UPGRADER_VERSION
+            value: {{ include "teleport-kube-agent.version" . }}
           {{- end }}
           {{- if .Values.tls.existingCASecretName }}
           - name: SSL_CERT_FILE

examples/chart/teleport-kube-agent/tests/deployment_test.yaml

@@ -719,3 +719,24 @@ tests:
       - equal:
           path: spec.template.spec.containers[0].securityContext
           value: null
+
+  - it: should enable maintenance schedule export when updater is enabled if action is Upgrade
+    template: deployment.yaml
+    values:
+      - ../.lint/updater.yaml
+    set:
+      # unit test does not support lookup functions, so to test the behavior we use this undoc value
+      # https://github.com/helm/helm/issues/8137
+      unitTestUpgrade: true
+      teleportVersionOverride: 13.4.5
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: TELEPORT_EXT_UPGRADER
+            value: kube
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: TELEPORT_EXT_UPGRADER_VERSION
+            value: 13.4.5

examples/chart/teleport-kube-agent/tests/statefulset_test.yaml

@@ -713,14 +713,20 @@ tests:
   - it: should enable maintenance schedule export when updater is enabled
     template: statefulset.yaml
     values:
-      - ../.lint/existing-tls-secret-with-ca.yaml
       - ../.lint/updater.yaml
+    set:
+      teleportVersionOverride: 13.4.5
     asserts:
       - contains:
           path: spec.template.spec.containers[0].env
           content:
             name: TELEPORT_EXT_UPGRADER
             value: kube
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: TELEPORT_EXT_UPGRADER_VERSION
+            value: 13.4.5
 
   - it: should set the installation method environment variable
     template: statefulset.yaml

lib/auth/auth.go

@@ -638,7 +638,10 @@ var (
 			Name:      teleport.MetricUpgraderCounts,
 			Help:      "Tracks the number of instances advertising each upgrader",
 		},
-		[]string{teleport.TagUpgrader},
+		[]string{
+			teleport.TagUpgrader,
+			teleport.TagVersion,
+		},
 	)
 
 	accessRequestsCreatedMetric = prometheus.NewCounterVec(
@@ -1209,8 +1212,14 @@ func (a *Server) doInstancePeriodics(ctx context.Context) {
 	// set instance metric values
 	totalInstancesMetric.Set(float64(imp.TotalInstances()))
 	enrolledInUpgradesMetric.Set(float64(imp.TotalEnrolledInUpgrades()))
-	for _, upgrader := range []string{types.UpgraderKindKubeController, types.UpgraderKindSystemdUnit} {
-		upgraderCountsMetric.WithLabelValues(upgrader).Set(float64(imp.InstancesWithUpgrader(upgrader)))
+
+	for upgraderType, upgraderVersions := range imp.upgraderCounts {
+		for version, count := range upgraderVersions {
+			upgraderCountsMetric.With(prometheus.Labels{
+				teleport.TagUpgrader: upgraderType,
+				teleport.TagVersion:  version,
+			}).Set(float64(count))
+		}
 	}
 
 	// create/delete upgrade enroll prompt as appropriate

lib/auth/periodic.go

@@ -103,39 +103,39 @@ func inspectVersionCounts(counts map[string]int) (median string, total int, ok b
 
 // instanceMetricsPeriodic is an aggregator for general instance metrics.
 type instanceMetricsPeriodic struct {
-	upgraderCounts map[string]int
+	upgraderCounts map[string]map[string]int
 	totalInstances int
 }
 
 func newInstanceMetricsPeriodic() *instanceMetricsPeriodic {
 	return &instanceMetricsPeriodic{
-		upgraderCounts: make(map[string]int),
+		upgraderCounts: make(map[string]map[string]int),
 	}
 }
 
 // VisitInstance adds an instance to ongoing aggregations.
 func (i *instanceMetricsPeriodic) VisitInstance(instance types.Instance) {
 	i.totalInstances++
 	if upgrader := instance.GetExternalUpgrader(); upgrader != "" {
-		i.upgraderCounts[upgrader]++
+		if _, exists := i.upgraderCounts[upgrader]; !exists {
+			i.upgraderCounts[upgrader] = make(map[string]int)
+		}
+		i.upgraderCounts[upgrader][instance.GetExternalUpgraderVersion()]++
 	}
 }
 
 // TotalEnrolledInUpgrades gets the total number of instances that have some upgrader defined.
 func (i *instanceMetricsPeriodic) TotalEnrolledInUpgrades() int {
 	var total int
-	for _, count := range i.upgraderCounts {
-		total += count
+	for _, upgraderVersion := range i.upgraderCounts {
+		for _, count := range upgraderVersion {
+			total += count
+		}
 	}
 
 	return total
 }
 
-// InstancesWithUpgrader gets the number of instances that advertise the given upgrader.
-func (i *instanceMetricsPeriodic) InstancesWithUpgrader(upgrader string) int {
-	return i.upgraderCounts[upgrader]
-}
-
 // TotalInstances gets the total number of known instances.
 func (i *instanceMetricsPeriodic) TotalInstances() int {
 	return i.totalInstances

lib/auth/periodic_test.go

@@ -31,67 +31,115 @@ import (
 func TestInstanceMetricsPeriodic(t *testing.T) {
 	tts := []struct {
 		desc           string
+		specs          []types.InstanceSpecV1
+		expectedCounts map[string]map[string]int
 		upgraders      []string
-		expectCounts   map[string]int
 		expectEnrolled int
 	}{
 		{
 			desc: "mixed",
+			specs: []types.InstanceSpecV1{
+				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "13.0.0"},
+				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "14.0.0"},
+				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "13.0.0"},
+				{},
+				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "14.0.0"},
+				{},
+			},
 			upgraders: []string{
+				"kube",
 				"kube",
 				"unit",
 				"",
 				"unit",
 				"",
 			},
-			expectCounts: map[string]int{
-				"kube": 1,
-				"unit": 2,
+			expectedCounts: map[string]map[string]int{
+				"kube": {
+					"13.0.0": 1,
+					"14.0.0": 1,
+				},
+				"unit": {
+					"13.0.0": 1,
+					"14.0.0": 1,
+				},
 			},
-			expectEnrolled: 3,
+			expectEnrolled: 4,
 		},
 		{
 			desc: "all-unenrolled",
+			specs: []types.InstanceSpecV1{
+				{},
+				{},
+			},
 			upgraders: []string{
 				"",
 				"",
 			},
+			expectedCounts: map[string]map[string]int{},
 		},
 		{
 			desc: "all-enrolled",
+			specs: []types.InstanceSpecV1{
+				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "13.0.0"},
+				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "13.0.0"},
+				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "13.0.0"},
+				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "13.0.0"},
+			},
 			upgraders: []string{
 				"kube",
 				"kube",
 				"unit",
 				"unit",
 			},
-			expectCounts: map[string]int{
-				"kube": 2,
-				"unit": 2,
+			expectedCounts: map[string]map[string]int{
+				"kube": {
+					"13.0.0": 2,
+				},
+				"unit": {
+					"13.0.0": 2,
+				},
 			},
 			expectEnrolled: 4,
 		},
 		{
-			desc: "nothing",
+			desc: "nil version",
+			specs: []types.InstanceSpecV1{
+				{ExternalUpgrader: "kube"},
+				{ExternalUpgrader: "unit"},
+			},
+			upgraders: []string{
+				"kube",
+				"unit",
+			},
+			expectedCounts: map[string]map[string]int{
+				"kube": {
+					"": 1,
+				},
+				"unit": {
+					"": 1,
+				},
+			},
+			expectEnrolled: 2,
+		},
+		{
+			desc:           "nothing",
+			expectedCounts: map[string]map[string]int{},
 		},
 	}
 
 	for _, tt := range tts {
 		t.Run(tt.desc, func(t *testing.T) {
 			periodic := newInstanceMetricsPeriodic()
 
-			for _, upgrader := range tt.upgraders {
-				instance, err := types.NewInstance(uuid.New().String(), types.InstanceSpecV1{
-					ExternalUpgrader: upgrader,
-				})
+			for _, upgrader := range tt.specs {
+				instance, err := types.NewInstance(uuid.New().String(), upgrader)
 				require.NoError(t, err)
 
 				periodic.VisitInstance(instance)
 			}
 
-			for upgrader, count := range tt.expectCounts {
-				require.Equal(t, count, periodic.InstancesWithUpgrader(upgrader), "upgrader=%q, tt=%q", upgrader, tt.desc)
-			}
+			require.Equal(t, tt.expectedCounts, periodic.upgraderCounts, "tt=%q", tt.desc)
 
 			require.Equal(t, tt.expectEnrolled, periodic.TotalEnrolledInUpgrades(), "tt=%q", tt.desc)