Export auth server version for upgrader

lib/service/service.go

@@ -1001,6 +1001,7 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) {
 		exporter, err := uw.NewExporter(uw.ExporterConfig[inventory.DownstreamSender]{
 			Driver:                   driver,
 			ExportFunc:               process.exportUpgradeWindows,
+			ExportVersionFunc:        process.exportAuthVersion,
 			AuthConnectivitySentinel: process.inventoryHandle.Sender(),
 		})
 		if err != nil {
@@ -1359,6 +1360,26 @@ func (process *TeleportProcess) exportUpgradeWindows(ctx context.Context, req pr
 	return clt.ExportUpgradeWindows(ctx, req)
 }
 
+func (process *TeleportProcess) exportAuthVersion(ctx context.Context) (string, error) {
+	if auth := process.getLocalAuth(); auth != nil {
+		resp, err := auth.Ping(ctx)
+		if err != nil {
+			return "", trace.Wrap(err)
+		}
+		return resp.ServerVersion, nil
+	}
+
+	clt := process.getInstanceClient()
+	if clt == nil {
+		return "", trace.Errorf("instance client not yet initialized")
+	}
+	resp, err := clt.Ping(ctx)
+	if err != nil {
+		return "", trace.Wrap(err)
+	}
+	return resp.ServerVersion, nil
+}
+
 // adminCreds returns admin UID and GID settings based on the OS
 func adminCreds() (*int, *int, error) {
 	if runtime.GOOS != constants.LinuxOS {

lib/versioncontrol/upgradewindow/upgradewindow.go

@@ -40,16 +40,25 @@ const (
 	// kubeSchedKey is the key under which the kube controller schedule is exported
 	kubeSchedKey = "agent-maintenance-schedule"
 
+	// kubeVersionKey is the key under which the kube controller version is exported
+	kubeVersionKey = "agent-auth-version"
+
 	// unitScheduleFile is the name of the file to which the unit schedule is exported.
 	unitScheduleFile = "schedule"
 
+	// unitVersionFile is the name of the file to which the version is exported.
+	unitVersionFile = "auth-version"
+
 	// unitConfigDir is the configuration directory of the teleport-upgrade unit.
 	unitConfigDir = "/etc/teleport-upgrade.d"
 )
 
 // ExportFunc represents the ExportUpgradeWindows rpc exposed by auth servers.
 type ExportFunc func(ctx context.Context, req proto.ExportUpgradeWindowsRequest) (proto.ExportUpgradeWindowsResponse, error)
 
+// ExportVersionFunc exports the auth version.
+type ExportVersionFunc func(ctx context.Context) (string, error)
+
 // contextLike lets us abstract over the difference between basic contexts and context-like values such
 // as control stream senders or resource watchers. the exporter uses a contextLike value to decide wether
 // or not auth connectivity appears healthy. during normal runtime, we end up using the inventory control
@@ -61,15 +70,18 @@ type contextLike interface {
 type testEvent string
 
 const (
-	resetFromExport  testEvent = "reset-from-export"
-	resetFromRun     testEvent = "reset-from-run"
-	exportAttempt    testEvent = "export-attempt"
-	exportSuccess    testEvent = "export-success"
-	exportFailure    testEvent = "export-failure"
-	getExportErr     testEvent = "get-export-err"
-	syncExportErr    testEvent = "sync-export-err"
-	sentinelAcquired testEvent = "sentinel-acquired"
-	sentinelLost     testEvent = "sentinel-lost"
+	resetFromExport      testEvent = "reset-from-export"
+	resetFromRun         testEvent = "reset-from-run"
+	exportAttempt        testEvent = "export-attempt"
+	exportSuccess        testEvent = "export-success"
+	exportVersionSuccess testEvent = "export-version-success"
+	exportFailure        testEvent = "export-failure"
+	getExportErr         testEvent = "get-export-err"
+	getExportVersionErr  testEvent = "get-export-version-err"
+	syncExportErr        testEvent = "sync-export-err"
+	syncExportVersionErr testEvent = "sync-export-version-err"
+	sentinelAcquired     testEvent = "sentinel-acquired"
+	sentinelLost         testEvent = "sentinel-lost"
 )
 
 // ExporterConfig configures a maintenance window exporter.
@@ -80,6 +92,9 @@ type ExporterConfig[C contextLike] struct {
 	// ExportFunc gets the current maintenance window.
 	ExportFunc ExportFunc
 
+	// ExportVersionFunc gets the current auth server version.
+	ExportVersionFunc ExportVersionFunc
+
 	// AuthConnectivitySentinel is a channel that yields context-like values indicating the current health of
 	// auth connectivity. When connectivity to auth is established, a context-like value should be sent over
 	// the channel. If auth connectivity is subsequently lost, that context-like value must be canceled.
@@ -111,6 +126,10 @@ func (c *ExporterConfig[C]) CheckAndSetDefaults() error {
 		return trace.BadParameter("exporter config missing required parameter 'ExportFunc'")
 	}
 
+	if c.ExportVersionFunc == nil {
+		return trace.BadParameter("exporter config missing required parameter 'ExportVersionFunc'")
+	}
+
 	if c.AuthConnectivitySentinel == nil {
 		return trace.BadParameter("exporter config missing required parameter 'AuthConnectivitySentinel'")
 	}
@@ -295,6 +314,25 @@ func (e *Exporter[C]) exportWithRetry(ctx context.Context) {
 
 		log.Infof("Successfully synced %q upgrader maintenance window value.", e.cfg.Driver.Kind())
 		e.event(exportSuccess)
+
+		version, err := e.cfg.ExportVersionFunc(ctx)
+		if err != nil {
+			log.Warnf("Failed to import %q auth version: %v", e.cfg.Driver.Kind(), err)
+			e.retry.Inc()
+			e.event(getExportVersionErr)
+			continue
+		}
+
+		if err := e.cfg.Driver.SyncAuthVersion(ctx, version); err != nil {
+			log.Warnf("Failed to sync %q auth version: %v", e.cfg.Driver.Kind(), err)
+			e.retry.Inc()
+			e.event(syncExportVersionErr)
+			continue
+		}
+
+		log.Infof("Successfully synced %q auth version value.", e.cfg.Driver.Kind())
+		e.event(exportVersionSuccess)
+
 		return
 	}
 }
@@ -310,6 +348,9 @@ type Driver interface {
 	// info.
 	Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsResponse) error
 
+	// SyncAuthVersion exports the auth server version.
+	SyncAuthVersion(ctx context.Context, version string) error
+
 	// Reset forcibly clears any previously exported maintenance window values. This should be
 	// called if teleport experiences prolonged loss of auth connectivity, which may be an indicator
 	// that the control plane has been upgraded s.t. this agent is no longer compatible.
@@ -374,6 +415,15 @@ func (e *kubeDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsRes
 	return trace.Wrap(err)
 }
 
+func (e *kubeDriver) SyncAuthVersion(ctx context.Context, version string) error {
+	_, err := e.cfg.Backend.Put(ctx, backend.Item{
+		Key:   []byte(kubeVersionKey),
+		Value: []byte(version),
+	})
+
+	return trace.Wrap(err)
+}
+
 func (e *kubeDriver) Reset(ctx context.Context) error {
 	// kube backend doesn't support deletes right now, so just set
 	// the key to empty.
@@ -427,6 +477,18 @@ func (e *systemdDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindows
 	return nil
 }
 
+func (e *systemdDriver) SyncAuthVersion(ctx context.Context, version string) error {
+	if err := os.MkdirAll(e.cfg.ConfigDir, defaults.DirectoryPermissions); err != nil {
+		return trace.Wrap(err)
+	}
+
+	if err := os.WriteFile(e.versionFile(), []byte(version), defaults.FilePermissions); err != nil {
+		return trace.Errorf("failed to write version file: %v", err)
+	}
+
+	return nil
+}
+
 func (e *systemdDriver) Reset(_ context.Context) error {
 	if _, err := os.Stat(e.scheduleFile()); os.IsNotExist(err) {
 		return nil
@@ -445,3 +507,7 @@ func (e *systemdDriver) Reset(_ context.Context) error {
 func (e *systemdDriver) scheduleFile() string {
 	return filepath.Join(e.cfg.ConfigDir, unitScheduleFile)
 }
+
+func (e *systemdDriver) versionFile() string {
+	return filepath.Join(e.cfg.ConfigDir, unitVersionFile)
+}

lib/versioncontrol/upgradewindow/upgradewindow_test.go

@@ -104,6 +104,18 @@ func TestKubeControllerDriver(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, "", bk.data[key])
+
+	// verify basic version creation
+	err = driver.SyncAuthVersion(ctx, "14.1.5")
+	require.NoError(t, err)
+
+	keyVersion := "agent-auth-version"
+	require.Equal(t, "14.1.5", bk.data[keyVersion])
+
+	// verify overwrite of existing version
+	err = driver.SyncAuthVersion(ctx, "14.2.0")
+	require.NoError(t, err)
+	require.Equal(t, "14.2.0", bk.data[keyVersion])
 }
 
 // TestSystemdUnitDriver verifies the basic behavior of the systemd unit export driver.
@@ -178,14 +190,34 @@ func TestSystemdUnitDriver(t *testing.T) {
 	sb, err = os.ReadFile(schedPath)
 	require.NoError(t, err)
 	require.Equal(t, "", string(sb))
+
+	// verify basic version creation
+	err = driver.SyncAuthVersion(ctx, "14.1.5")
+	require.NoError(t, err)
+
+	versionPath := filepath.Join(dir, "version")
+
+	vb, err := os.ReadFile(versionPath)
+	require.NoError(t, err)
+
+	require.Equal(t, "14.1.5", string(vb))
+
+	// verify overwrite of existing version
+	err = driver.SyncAuthVersion(ctx, "14.2.0")
+	require.NoError(t, err)
+
+	vb, err = os.ReadFile(versionPath)
+	require.NoError(t, err)
+	require.Equal(t, "14.2.0", string(vb))
 }
 
 // fakeDriver is used to inject custom behavior into a dummy Driver instance.
 type fakeDriver struct {
-	mu    sync.Mutex
-	kind  string
-	sync  func(context.Context, proto.ExportUpgradeWindowsResponse) error
-	reset func(context.Context) error
+	mu          sync.Mutex
+	kind        string
+	sync        func(context.Context, proto.ExportUpgradeWindowsResponse) error
+	syncVersion func(context.Context, string) error
+	reset       func(context.Context) error
 }
 
 func (d *fakeDriver) Kind() string {
@@ -207,6 +239,16 @@ func (d *fakeDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsRes
 	return nil
 }
 
+func (d *fakeDriver) SyncAuthVersion(ctx context.Context, version string) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.syncVersion != nil {
+		return d.syncVersion(ctx, version)
+	}
+
+	return nil
+}
+
 func (d *fakeDriver) Reset(ctx context.Context) error {
 	d.mu.Lock()
 	defer d.mu.Unlock()
@@ -255,6 +297,11 @@ func TestExporterBasics(t *testing.T) {
 		return
 	}
 
+	exportVersion := func(ctx context.Context) (version string, err error) {
+		version = "fake-version"
+		return
+	}
+
 	driver := new(fakeDriver)
 
 	driver.withLock(func() {
@@ -269,6 +316,7 @@ func TestExporterBasics(t *testing.T) {
 	exporter, err := NewExporter(ExporterConfig[context.Context]{
 		Driver:                   driver,
 		ExportFunc:               export,
+		ExportVersionFunc:        exportVersion,
 		AuthConnectivitySentinel: sc,
 		UnhealthyThreshold:       time.Millisecond * 200,
 		ExportInterval:           time.Millisecond * 300,