From b9b2e3ed10ea4840529528ac0e4e6546eadfd09d Mon Sep 17 00:00:00 2001 From: Bernard Kim Date: Wed, 22 Nov 2023 16:27:02 -0800 Subject: [PATCH 1/5] Export auth server version --- lib/service/service.go | 21 +++++ .../upgradewindow/upgradewindow.go | 84 +++++++++++++++++-- .../upgradewindow/upgradewindow_test.go | 56 ++++++++++++- 3 files changed, 148 insertions(+), 13 deletions(-) diff --git a/lib/service/service.go b/lib/service/service.go index 2279a1af15cd8..30641927ac7e7 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -1001,6 +1001,7 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { exporter, err := uw.NewExporter(uw.ExporterConfig[inventory.DownstreamSender]{ Driver: driver, ExportFunc: process.exportUpgradeWindows, + ExportVersionFunc: process.exportAuthVersion, AuthConnectivitySentinel: process.inventoryHandle.Sender(), }) if err != nil { @@ -1359,6 +1360,26 @@ func (process *TeleportProcess) exportUpgradeWindows(ctx context.Context, req pr return clt.ExportUpgradeWindows(ctx, req) } +func (process *TeleportProcess) exportAuthVersion(ctx context.Context) (string, error) { + if auth := process.getLocalAuth(); auth != nil { + resp, err := auth.Ping(ctx) + if err != nil { + return "", trace.Wrap(err) + } + return resp.ServerVersion, nil + } + + clt := process.getInstanceClient() + if clt == nil { + return "", trace.Errorf("instance client not yet initialized") + } + resp, err := clt.Ping(ctx) + if err != nil { + return "", trace.Wrap(err) + } + return resp.ServerVersion, nil +} + // adminCreds returns admin UID and GID settings based on the OS func adminCreds() (*int, *int, error) { if runtime.GOOS != constants.LinuxOS { diff --git a/lib/versioncontrol/upgradewindow/upgradewindow.go b/lib/versioncontrol/upgradewindow/upgradewindow.go index 51a05a7cd5b93..065b8415f071e 100644 --- a/lib/versioncontrol/upgradewindow/upgradewindow.go +++ b/lib/versioncontrol/upgradewindow/upgradewindow.go @@ -40,9 +40,15 @@ const ( // kubeSchedKey is the key under which the kube controller schedule is exported kubeSchedKey = "agent-maintenance-schedule" + // kubeVersionKey is the key under which the kube controller version is exported + kubeVersionKey = "agent-auth-version" + // unitScheduleFile is the name of the file to which the unit schedule is exported. unitScheduleFile = "schedule" + // unitVersionFile is the name of the file to which the version is exported. + unitVersionFile = "version" + // unitConfigDir is the configuration directory of the teleport-upgrade unit. unitConfigDir = "/etc/teleport-upgrade.d" ) @@ -50,6 +56,9 @@ const ( // ExportFunc represents the ExportUpgradeWindows rpc exposed by auth servers. type ExportFunc func(ctx context.Context, req proto.ExportUpgradeWindowsRequest) (proto.ExportUpgradeWindowsResponse, error) +// ExportVersionFunc exports the auth version. +type ExportVersionFunc func(ctx context.Context) (string, error) + // contextLike lets us abstract over the difference between basic contexts and context-like values such // as control stream senders or resource watchers. the exporter uses a contextLike value to decide wether // or not auth connectivity appears healthy. during normal runtime, we end up using the inventory control @@ -61,15 +70,18 @@ type contextLike interface { type testEvent string const ( - resetFromExport testEvent = "reset-from-export" - resetFromRun testEvent = "reset-from-run" - exportAttempt testEvent = "export-attempt" - exportSuccess testEvent = "export-success" - exportFailure testEvent = "export-failure" - getExportErr testEvent = "get-export-err" - syncExportErr testEvent = "sync-export-err" - sentinelAcquired testEvent = "sentinel-acquired" - sentinelLost testEvent = "sentinel-lost" + resetFromExport testEvent = "reset-from-export" + resetFromRun testEvent = "reset-from-run" + exportAttempt testEvent = "export-attempt" + exportSuccess testEvent = "export-success" + exportVersionSuccess testEvent = "export-version-success" + exportFailure testEvent = "export-failure" + getExportErr testEvent = "get-export-err" + getExportVersionErr testEvent = "get-export-version-err" + syncExportErr testEvent = "sync-export-err" + syncExportVersionErr testEvent = "sync-export-version-err" + sentinelAcquired testEvent = "sentinel-acquired" + sentinelLost testEvent = "sentinel-lost" ) // ExporterConfig configures a maintenance window exporter. @@ -80,6 +92,9 @@ type ExporterConfig[C contextLike] struct { // ExportFunc gets the current maintenance window. ExportFunc ExportFunc + // ExportVersionFunc gets the current auth server version. + ExportVersionFunc ExportVersionFunc + // AuthConnectivitySentinel is a channel that yields context-like values indicating the current health of // auth connectivity. When connectivity to auth is established, a context-like value should be sent over // the channel. If auth connectivity is subsequently lost, that context-like value must be canceled. @@ -111,6 +126,10 @@ func (c *ExporterConfig[C]) CheckAndSetDefaults() error { return trace.BadParameter("exporter config missing required parameter 'ExportFunc'") } + if c.ExportVersionFunc == nil { + return trace.BadParameter("exporter config missing required parameter 'ExportVersionFunc'") + } + if c.AuthConnectivitySentinel == nil { return trace.BadParameter("exporter config missing required parameter 'AuthConnectivitySentinel'") } @@ -295,6 +314,25 @@ func (e *Exporter[C]) exportWithRetry(ctx context.Context) { log.Infof("Successfully synced %q upgrader maintenance window value.", e.cfg.Driver.Kind()) e.event(exportSuccess) + + version, err := e.cfg.ExportVersionFunc(ctx) + if err != nil { + log.Warnf("Failed to import %q auth version: %v", e.cfg.Driver.Kind(), err) + e.retry.Inc() + e.event(getExportVersionErr) + continue + } + + if err := e.cfg.Driver.SyncAuthVersion(ctx, version); err != nil { + log.Warnf("Failed to sync %q auth version: %v", e.cfg.Driver.Kind(), err) + e.retry.Inc() + e.event(syncExportVersionErr) + continue + } + + log.Infof("Successfully synced %q auth version value.", e.cfg.Driver.Kind()) + e.event(exportVersionSuccess) + return } } @@ -310,6 +348,9 @@ type Driver interface { // info. Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsResponse) error + // SyncAuthVersion exports the auth server version. + SyncAuthVersion(ctx context.Context, version string) error + // Reset forcibly clears any previously exported maintenance window values. This should be // called if teleport experiences prolonged loss of auth connectivity, which may be an indicator // that the control plane has been upgraded s.t. this agent is no longer compatible. @@ -374,6 +415,15 @@ func (e *kubeDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsRes return trace.Wrap(err) } +func (e *kubeDriver) SyncAuthVersion(ctx context.Context, version string) error { + _, err := e.cfg.Backend.Put(ctx, backend.Item{ + Key: []byte(kubeVersionKey), + Value: []byte(version), + }) + + return trace.Wrap(err) +} + func (e *kubeDriver) Reset(ctx context.Context) error { // kube backend doesn't support deletes right now, so just set // the key to empty. @@ -427,6 +477,18 @@ func (e *systemdDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindows return nil } +func (e *systemdDriver) SyncAuthVersion(ctx context.Context, version string) error { + if err := os.MkdirAll(e.cfg.ConfigDir, defaults.DirectoryPermissions); err != nil { + return trace.Wrap(err) + } + + if err := os.WriteFile(e.versionFile(), []byte(version), defaults.FilePermissions); err != nil { + return trace.Errorf("failed to write version file: %v", err) + } + + return nil +} + func (e *systemdDriver) Reset(_ context.Context) error { if _, err := os.Stat(e.scheduleFile()); os.IsNotExist(err) { return nil @@ -445,3 +507,7 @@ func (e *systemdDriver) Reset(_ context.Context) error { func (e *systemdDriver) scheduleFile() string { return filepath.Join(e.cfg.ConfigDir, unitScheduleFile) } + +func (e *systemdDriver) versionFile() string { + return filepath.Join(e.cfg.ConfigDir, unitVersionFile) +} diff --git a/lib/versioncontrol/upgradewindow/upgradewindow_test.go b/lib/versioncontrol/upgradewindow/upgradewindow_test.go index 9de86beb666f7..284592836ec32 100644 --- a/lib/versioncontrol/upgradewindow/upgradewindow_test.go +++ b/lib/versioncontrol/upgradewindow/upgradewindow_test.go @@ -104,6 +104,18 @@ func TestKubeControllerDriver(t *testing.T) { require.NoError(t, err) require.Equal(t, "", bk.data[key]) + + // verify basic version creation + err = driver.SyncAuthVersion(ctx, "14.1.5") + require.NoError(t, err) + + keyVersion := "agent-auth-version" + require.Equal(t, "14.1.5", bk.data[keyVersion]) + + // verify overwrite of existing version + err = driver.SyncAuthVersion(ctx, "14.2.0") + require.NoError(t, err) + require.Equal(t, "14.2.0", bk.data[keyVersion]) } // TestSystemdUnitDriver verifies the basic behavior of the systemd unit export driver. @@ -178,14 +190,34 @@ func TestSystemdUnitDriver(t *testing.T) { sb, err = os.ReadFile(schedPath) require.NoError(t, err) require.Equal(t, "", string(sb)) + + // verify basic version creation + err = driver.SyncAuthVersion(ctx, "14.1.5") + require.NoError(t, err) + + versionPath := filepath.Join(dir, "version") + + vb, err := os.ReadFile(versionPath) + require.NoError(t, err) + + require.Equal(t, "14.1.5", string(vb)) + + // verify overwrite of existing version + err = driver.SyncAuthVersion(ctx, "14.2.0") + require.NoError(t, err) + + vb, err = os.ReadFile(versionPath) + require.NoError(t, err) + require.Equal(t, "14.2.0", string(vb)) } // fakeDriver is used to inject custom behavior into a dummy Driver instance. type fakeDriver struct { - mu sync.Mutex - kind string - sync func(context.Context, proto.ExportUpgradeWindowsResponse) error - reset func(context.Context) error + mu sync.Mutex + kind string + sync func(context.Context, proto.ExportUpgradeWindowsResponse) error + syncVersion func(context.Context, string) error + reset func(context.Context) error } func (d *fakeDriver) Kind() string { @@ -207,6 +239,16 @@ func (d *fakeDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsRes return nil } +func (d *fakeDriver) SyncAuthVersion(ctx context.Context, version string) error { + d.mu.Lock() + defer d.mu.Unlock() + if d.syncVersion != nil { + return d.syncVersion(ctx, version) + } + + return nil +} + func (d *fakeDriver) Reset(ctx context.Context) error { d.mu.Lock() defer d.mu.Unlock() @@ -255,6 +297,11 @@ func TestExporterBasics(t *testing.T) { return } + exportVersion := func(ctx context.Context) (version string, err error) { + version = "fake-version" + return + } + driver := new(fakeDriver) driver.withLock(func() { @@ -269,6 +316,7 @@ func TestExporterBasics(t *testing.T) { exporter, err := NewExporter(ExporterConfig[context.Context]{ Driver: driver, ExportFunc: export, + ExportVersionFunc: exportVersion, AuthConnectivitySentinel: sc, UnhealthyThreshold: time.Millisecond * 200, ExportInterval: time.Millisecond * 300, From 040bf7563cf841250878e0f3543b35945c4b7ac3 Mon Sep 17 00:00:00 2001 From: Bernard Kim Date: Wed, 22 Nov 2023 16:27:19 -0800 Subject: [PATCH 2/5] Update e --- e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e b/e index 59dff8ba447e1..03d039d9c7adc 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit 59dff8ba447e1db904cd6f2fe28c211d3425fc23 +Subproject commit 03d039d9c7adc8ceec800cfe6659e828c6f91857 From 4aa395b24cc7668a6ae8c861544c5639dca70f06 Mon Sep 17 00:00:00 2001 From: Bernard Kim Date: Wed, 22 Nov 2023 17:20:15 -0800 Subject: [PATCH 3/5] Bump e --- e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e b/e index 03d039d9c7adc..04732b6593936 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit 03d039d9c7adc8ceec800cfe6659e828c6f91857 +Subproject commit 04732b6593936f1346d50e5227fe728bf2c5c3a6 From 3137cd44c49ae3bc0cac1fe189d756f45ec95afb Mon Sep 17 00:00:00 2001 From: Bernard Kim Date: Mon, 27 Nov 2023 09:59:11 -0800 Subject: [PATCH 4/5] Change key version -> auth-version --- lib/versioncontrol/upgradewindow/upgradewindow.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/versioncontrol/upgradewindow/upgradewindow.go b/lib/versioncontrol/upgradewindow/upgradewindow.go index 065b8415f071e..36bf9a0f2079a 100644 --- a/lib/versioncontrol/upgradewindow/upgradewindow.go +++ b/lib/versioncontrol/upgradewindow/upgradewindow.go @@ -47,7 +47,7 @@ const ( unitScheduleFile = "schedule" // unitVersionFile is the name of the file to which the version is exported. - unitVersionFile = "version" + unitVersionFile = "auth-version" // unitConfigDir is the configuration directory of the teleport-upgrade unit. unitConfigDir = "/etc/teleport-upgrade.d" From 573be84ee00b186824f466e9f444b7a6b002bdb7 Mon Sep 17 00:00:00 2001 From: Bernard Kim Date: Mon, 27 Nov 2023 09:59:39 -0800 Subject: [PATCH 5/5] Bump e --- e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e b/e index 04732b6593936..750daf11dde70 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit 04732b6593936f1346d50e5227fe728bf2c5c3a6 +Subproject commit 750daf11dde7076a93b89a88c1658fe1b532a833