diff --git a/e b/e index 6914f4af55c97..750daf11dde70 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit 6914f4af55c97bfdbb5ed3f89ff6ddeaaf65994a +Subproject commit 750daf11dde7076a93b89a88c1658fe1b532a833 diff --git a/lib/service/service.go b/lib/service/service.go index 34525fa860c15..514e40212f3ab 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -1001,6 +1001,7 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { exporter, err := uw.NewExporter(uw.ExporterConfig[inventory.DownstreamSender]{ Driver: driver, ExportFunc: process.exportUpgradeWindows, + ExportVersionFunc: process.exportAuthVersion, AuthConnectivitySentinel: process.inventoryHandle.Sender(), }) if err != nil { @@ -1359,6 +1360,26 @@ func (process *TeleportProcess) exportUpgradeWindows(ctx context.Context, req pr return clt.ExportUpgradeWindows(ctx, req) } +func (process *TeleportProcess) exportAuthVersion(ctx context.Context) (string, error) { + if auth := process.getLocalAuth(); auth != nil { + resp, err := auth.Ping(ctx) + if err != nil { + return "", trace.Wrap(err) + } + return resp.ServerVersion, nil + } + + clt := process.getInstanceClient() + if clt == nil { + return "", trace.Errorf("instance client not yet initialized") + } + resp, err := clt.Ping(ctx) + if err != nil { + return "", trace.Wrap(err) + } + return resp.ServerVersion, nil +} + // adminCreds returns admin UID and GID settings based on the OS func adminCreds() (*int, *int, error) { if runtime.GOOS != constants.LinuxOS { diff --git a/lib/versioncontrol/upgradewindow/upgradewindow.go b/lib/versioncontrol/upgradewindow/upgradewindow.go index 51a05a7cd5b93..36bf9a0f2079a 100644 --- a/lib/versioncontrol/upgradewindow/upgradewindow.go +++ b/lib/versioncontrol/upgradewindow/upgradewindow.go @@ -40,9 +40,15 @@ const ( // kubeSchedKey is the key under which the kube controller schedule is exported kubeSchedKey = "agent-maintenance-schedule" + // kubeVersionKey is the key under which the kube controller version is exported + kubeVersionKey = "agent-auth-version" + // unitScheduleFile is the name of the file to which the unit schedule is exported. unitScheduleFile = "schedule" + // unitVersionFile is the name of the file to which the version is exported. + unitVersionFile = "auth-version" + // unitConfigDir is the configuration directory of the teleport-upgrade unit. unitConfigDir = "/etc/teleport-upgrade.d" ) @@ -50,6 +56,9 @@ const ( // ExportFunc represents the ExportUpgradeWindows rpc exposed by auth servers. type ExportFunc func(ctx context.Context, req proto.ExportUpgradeWindowsRequest) (proto.ExportUpgradeWindowsResponse, error) +// ExportVersionFunc exports the auth version. +type ExportVersionFunc func(ctx context.Context) (string, error) + // contextLike lets us abstract over the difference between basic contexts and context-like values such // as control stream senders or resource watchers. the exporter uses a contextLike value to decide wether // or not auth connectivity appears healthy. during normal runtime, we end up using the inventory control @@ -61,15 +70,18 @@ type contextLike interface { type testEvent string const ( - resetFromExport testEvent = "reset-from-export" - resetFromRun testEvent = "reset-from-run" - exportAttempt testEvent = "export-attempt" - exportSuccess testEvent = "export-success" - exportFailure testEvent = "export-failure" - getExportErr testEvent = "get-export-err" - syncExportErr testEvent = "sync-export-err" - sentinelAcquired testEvent = "sentinel-acquired" - sentinelLost testEvent = "sentinel-lost" + resetFromExport testEvent = "reset-from-export" + resetFromRun testEvent = "reset-from-run" + exportAttempt testEvent = "export-attempt" + exportSuccess testEvent = "export-success" + exportVersionSuccess testEvent = "export-version-success" + exportFailure testEvent = "export-failure" + getExportErr testEvent = "get-export-err" + getExportVersionErr testEvent = "get-export-version-err" + syncExportErr testEvent = "sync-export-err" + syncExportVersionErr testEvent = "sync-export-version-err" + sentinelAcquired testEvent = "sentinel-acquired" + sentinelLost testEvent = "sentinel-lost" ) // ExporterConfig configures a maintenance window exporter. @@ -80,6 +92,9 @@ type ExporterConfig[C contextLike] struct { // ExportFunc gets the current maintenance window. ExportFunc ExportFunc + // ExportVersionFunc gets the current auth server version. + ExportVersionFunc ExportVersionFunc + // AuthConnectivitySentinel is a channel that yields context-like values indicating the current health of // auth connectivity. When connectivity to auth is established, a context-like value should be sent over // the channel. If auth connectivity is subsequently lost, that context-like value must be canceled. @@ -111,6 +126,10 @@ func (c *ExporterConfig[C]) CheckAndSetDefaults() error { return trace.BadParameter("exporter config missing required parameter 'ExportFunc'") } + if c.ExportVersionFunc == nil { + return trace.BadParameter("exporter config missing required parameter 'ExportVersionFunc'") + } + if c.AuthConnectivitySentinel == nil { return trace.BadParameter("exporter config missing required parameter 'AuthConnectivitySentinel'") } @@ -295,6 +314,25 @@ func (e *Exporter[C]) exportWithRetry(ctx context.Context) { log.Infof("Successfully synced %q upgrader maintenance window value.", e.cfg.Driver.Kind()) e.event(exportSuccess) + + version, err := e.cfg.ExportVersionFunc(ctx) + if err != nil { + log.Warnf("Failed to import %q auth version: %v", e.cfg.Driver.Kind(), err) + e.retry.Inc() + e.event(getExportVersionErr) + continue + } + + if err := e.cfg.Driver.SyncAuthVersion(ctx, version); err != nil { + log.Warnf("Failed to sync %q auth version: %v", e.cfg.Driver.Kind(), err) + e.retry.Inc() + e.event(syncExportVersionErr) + continue + } + + log.Infof("Successfully synced %q auth version value.", e.cfg.Driver.Kind()) + e.event(exportVersionSuccess) + return } } @@ -310,6 +348,9 @@ type Driver interface { // info. Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsResponse) error + // SyncAuthVersion exports the auth server version. + SyncAuthVersion(ctx context.Context, version string) error + // Reset forcibly clears any previously exported maintenance window values. This should be // called if teleport experiences prolonged loss of auth connectivity, which may be an indicator // that the control plane has been upgraded s.t. this agent is no longer compatible. @@ -374,6 +415,15 @@ func (e *kubeDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsRes return trace.Wrap(err) } +func (e *kubeDriver) SyncAuthVersion(ctx context.Context, version string) error { + _, err := e.cfg.Backend.Put(ctx, backend.Item{ + Key: []byte(kubeVersionKey), + Value: []byte(version), + }) + + return trace.Wrap(err) +} + func (e *kubeDriver) Reset(ctx context.Context) error { // kube backend doesn't support deletes right now, so just set // the key to empty. @@ -427,6 +477,18 @@ func (e *systemdDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindows return nil } +func (e *systemdDriver) SyncAuthVersion(ctx context.Context, version string) error { + if err := os.MkdirAll(e.cfg.ConfigDir, defaults.DirectoryPermissions); err != nil { + return trace.Wrap(err) + } + + if err := os.WriteFile(e.versionFile(), []byte(version), defaults.FilePermissions); err != nil { + return trace.Errorf("failed to write version file: %v", err) + } + + return nil +} + func (e *systemdDriver) Reset(_ context.Context) error { if _, err := os.Stat(e.scheduleFile()); os.IsNotExist(err) { return nil @@ -445,3 +507,7 @@ func (e *systemdDriver) Reset(_ context.Context) error { func (e *systemdDriver) scheduleFile() string { return filepath.Join(e.cfg.ConfigDir, unitScheduleFile) } + +func (e *systemdDriver) versionFile() string { + return filepath.Join(e.cfg.ConfigDir, unitVersionFile) +} diff --git a/lib/versioncontrol/upgradewindow/upgradewindow_test.go b/lib/versioncontrol/upgradewindow/upgradewindow_test.go index 9de86beb666f7..284592836ec32 100644 --- a/lib/versioncontrol/upgradewindow/upgradewindow_test.go +++ b/lib/versioncontrol/upgradewindow/upgradewindow_test.go @@ -104,6 +104,18 @@ func TestKubeControllerDriver(t *testing.T) { require.NoError(t, err) require.Equal(t, "", bk.data[key]) + + // verify basic version creation + err = driver.SyncAuthVersion(ctx, "14.1.5") + require.NoError(t, err) + + keyVersion := "agent-auth-version" + require.Equal(t, "14.1.5", bk.data[keyVersion]) + + // verify overwrite of existing version + err = driver.SyncAuthVersion(ctx, "14.2.0") + require.NoError(t, err) + require.Equal(t, "14.2.0", bk.data[keyVersion]) } // TestSystemdUnitDriver verifies the basic behavior of the systemd unit export driver. @@ -178,14 +190,34 @@ func TestSystemdUnitDriver(t *testing.T) { sb, err = os.ReadFile(schedPath) require.NoError(t, err) require.Equal(t, "", string(sb)) + + // verify basic version creation + err = driver.SyncAuthVersion(ctx, "14.1.5") + require.NoError(t, err) + + versionPath := filepath.Join(dir, "version") + + vb, err := os.ReadFile(versionPath) + require.NoError(t, err) + + require.Equal(t, "14.1.5", string(vb)) + + // verify overwrite of existing version + err = driver.SyncAuthVersion(ctx, "14.2.0") + require.NoError(t, err) + + vb, err = os.ReadFile(versionPath) + require.NoError(t, err) + require.Equal(t, "14.2.0", string(vb)) } // fakeDriver is used to inject custom behavior into a dummy Driver instance. type fakeDriver struct { - mu sync.Mutex - kind string - sync func(context.Context, proto.ExportUpgradeWindowsResponse) error - reset func(context.Context) error + mu sync.Mutex + kind string + sync func(context.Context, proto.ExportUpgradeWindowsResponse) error + syncVersion func(context.Context, string) error + reset func(context.Context) error } func (d *fakeDriver) Kind() string { @@ -207,6 +239,16 @@ func (d *fakeDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsRes return nil } +func (d *fakeDriver) SyncAuthVersion(ctx context.Context, version string) error { + d.mu.Lock() + defer d.mu.Unlock() + if d.syncVersion != nil { + return d.syncVersion(ctx, version) + } + + return nil +} + func (d *fakeDriver) Reset(ctx context.Context) error { d.mu.Lock() defer d.mu.Unlock() @@ -255,6 +297,11 @@ func TestExporterBasics(t *testing.T) { return } + exportVersion := func(ctx context.Context) (version string, err error) { + version = "fake-version" + return + } + driver := new(fakeDriver) driver.withLock(func() { @@ -269,6 +316,7 @@ func TestExporterBasics(t *testing.T) { exporter, err := NewExporter(ExporterConfig[context.Context]{ Driver: driver, ExportFunc: export, + ExportVersionFunc: exportVersion, AuthConnectivitySentinel: sc, UnhealthyThreshold: time.Millisecond * 200, ExportInterval: time.Millisecond * 300,