feat: HSM/KMS CA rotation reminders via cluster alerts

integration/hsm/helpers.go

@@ -166,11 +166,11 @@ func (t *teleportService) waitForLocalAdditionalKeys(ctx context.Context) error
 		if err != nil {
 			return trace.Wrap(err)
 		}
-		hasUsableKeys, err := t.process.GetAuthServer().GetKeyStore().HasUsableAdditionalKeys(ctx, ca)
+		usableKeysResult, err := t.process.GetAuthServer().GetKeyStore().HasUsableAdditionalKeys(ctx, ca)
 		if err != nil {
 			return trace.Wrap(err)
 		}
-		if hasUsableKeys {
+		if usableKeysResult.CAHasUsableKeys {
 			break
 		}
 	}

integration/hsm/hsm_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/gravitational/trace"
+	"github.com/jonboulle/clockwork"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -40,6 +41,7 @@ import (
 	"github.com/gravitational/teleport/lib/backend"
 	"github.com/gravitational/teleport/lib/backend/etcdbk"
 	"github.com/gravitational/teleport/lib/backend/lite"
+	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/modules"
 	"github.com/gravitational/teleport/lib/service/servicecfg"
 	"github.com/gravitational/teleport/lib/utils"
@@ -523,6 +525,16 @@ func TestHSMMigrate(t *testing.T) {
 
 	testClient(t)
 
+	// Make sure a cluster alert is created.
+	alerts, err := auth1.process.GetAuthServer().GetClusterAlerts(ctx, types.GetClusterAlertsRequest{})
+	require.NoError(t, err)
+	require.Len(t, alerts, 1)
+	alert := alerts[0]
+	assert.Equal(t, types.AlertSeverity_MEDIUM, alert.Spec.Severity)
+	assert.Contains(t, alert.Spec.Message, "configured to use PKCS#11 HSM keys")
+	assert.Contains(t, alert.Spec.Message, "the following CAs do not contain any keys of that type:")
+	assert.Contains(t, alert.Spec.Message, "host")
+
 	authServices := teleportServices{auth1, auth2}
 	allServices := teleportServices{auth1, auth2, proxy}
 
@@ -572,6 +584,13 @@ func TestHSMMigrate(t *testing.T) {
 		stage.verify(t)
 	}
 
+	// Make sure the cluster alert no longer mentions the host CA.
+	alerts, err = auth1.process.GetAuthServer().GetClusterAlerts(ctx, types.GetClusterAlertsRequest{})
+	require.NoError(t, err)
+	require.Len(t, alerts, 1)
+	alert = alerts[0]
+	assert.NotContains(t, alert.Spec.Message, "host")
+
 	// Phase 2: migrate auth2 to HSM
 	auth2.process.Close()
 	require.NoError(t, auth2.waitForShutdown(ctx))
@@ -584,6 +603,11 @@ func TestHSMMigrate(t *testing.T) {
 
 	testClient(t)
 
+	// There should now be 2 cluster alerts (one for each auth using HSM).
+	alerts, err = auth1.process.GetAuthServer().GetClusterAlerts(ctx, types.GetClusterAlertsRequest{})
+	require.NoError(t, err)
+	assert.Len(t, alerts, 2)
+
 	// Do another full rotation to get HSM keys for auth2 into the CA.
 	for _, stage := range stages {
 		log.Debugf("TestHSMMigrate: Sending rotate request %s", stage.targetPhase)
@@ -597,3 +621,99 @@ func TestHSMMigrate(t *testing.T) {
 
 	testClient(t)
 }
+
+// TestHSMRevert tests a single-auth server migration from HSM keys back to
+// software keys.
+func TestHSMRevert(t *testing.T) {
+	requireHSMAvailable(t)
+
+	clock := clockwork.NewFakeClock()
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	log := utils.NewLoggerForTests()
+
+	log.Debug("TestHSMRevert: starting auth server")
+	auth1Config := newHSMAuthConfig(t, liteBackendConfig(t), log)
+	auth1Config.Clock = clock
+	auth1 := newTeleportService(t, auth1Config, "auth1")
+
+	log.Debug("TestHSMRevert: waiting for auth server to start")
+	err := auth1.start(ctx)
+	require.NoError(t, err, trace.DebugReport(err))
+	t.Cleanup(func() {
+		require.NoError(t, auth1.process.GetAuthServer().GetKeyStore().DeleteUnusedKeys(ctx, nil))
+	})
+
+	// Switch config back to default (software) and restart.
+	auth1.process.Close()
+	require.NoError(t, auth1.waitForShutdown(ctx))
+	auth1Config.Auth.KeyStore = keystore.Config{}
+	auth1 = newTeleportService(t, auth1Config, "auth1")
+	require.NoError(t, auth1.start(ctx))
+
+	// Make sure a cluster alert is created.
+	alerts, err := auth1.process.GetAuthServer().GetClusterAlerts(ctx, types.GetClusterAlertsRequest{})
+	require.NoError(t, err)
+	require.Len(t, alerts, 1)
+	alert := alerts[0]
+	assert.Equal(t, types.AlertSeverity_HIGH, alert.Spec.Severity)
+	assert.Contains(t, alert.Spec.Message, "configured to use raw software keys")
+	assert.Contains(t, alert.Spec.Message, "the following CAs do not contain any keys of that type:")
+	assert.Contains(t, alert.Spec.Message, "The Auth Service is currently unable to sign certificates")
+
+	for _, caType := range types.CertAuthTypes {
+		log.Debugf("TestHSMRevert: sending rotation request init for CA %s", caType)
+		err = auth1.process.GetAuthServer().RotateCertAuthority(ctx, types.RotateRequest{
+			Type:        caType,
+			TargetPhase: types.RotationPhaseInit,
+			Mode:        types.RotationModeManual,
+		})
+		require.NoError(t, err)
+		if caType == types.HostCA {
+			require.NoError(t, auth1.waitForPhaseChange(ctx))
+		}
+
+		log.Debugf("TestHSMRevert: sending rotation request update_clients for CA %s", caType)
+		err = auth1.process.GetAuthServer().RotateCertAuthority(ctx, types.RotateRequest{
+			Type:        caType,
+			TargetPhase: types.RotationPhaseUpdateClients,
+			Mode:        types.RotationModeManual,
+		})
+		require.NoError(t, err)
+		if caType == types.HostCA {
+			require.NoError(t, auth1.waitForRestart(ctx))
+		}
+
+		log.Debugf("TestHSMRevert: sending rotation request update_servers for CA %s", caType)
+		err = auth1.process.GetAuthServer().RotateCertAuthority(ctx, types.RotateRequest{
+			Type:        caType,
+			TargetPhase: types.RotationPhaseUpdateServers,
+			Mode:        types.RotationModeManual,
+		})
+		require.NoError(t, err)
+		if caType == types.HostCA {
+			require.NoError(t, auth1.waitForRestart(ctx))
+		}
+
+		log.Debugf("TestHSMRevert: sending rotation request standby for CA %s", caType)
+		err = auth1.process.GetAuthServer().RotateCertAuthority(ctx, types.RotateRequest{
+			Type:        caType,
+			TargetPhase: types.RotationPhaseStandby,
+			Mode:        types.RotationModeManual,
+		})
+		require.NoError(t, err)
+		if caType == types.HostCA {
+			require.NoError(t, auth1.waitForRestart(ctx))
+		}
+	}
+
+	// Make sure the cluster alert gets cleared.
+	// Advance far enough for auth.runPeriodicOperations to call
+	// auth.autoRotateCertAuthorities which reconciles the alert state.
+	clock.Advance(2 * defaults.HighResPollingPeriod)
+	assert.EventuallyWithT(t, func(t *assert.CollectT) {
+		alerts, err = auth1.process.GetAuthServer().GetClusterAlerts(ctx, types.GetClusterAlertsRequest{})
+		require.NoError(t, err)
+		assert.Empty(t, alerts)
+	}, 5*time.Second, 100*time.Millisecond)
+}

lib/auth/auth.go

@@ -6071,12 +6071,12 @@ func newKeySet(ctx context.Context, keyStore *keystore.Manager, caID types.CertA
 // ensureLocalAdditionalKeys adds additional trusted keys to the CA if they are not
 // already present.
 func (a *Server) ensureLocalAdditionalKeys(ctx context.Context, ca types.CertAuthority) error {
-	hasUsableKeys, err := a.keyStore.HasUsableAdditionalKeys(ctx, ca)
+	usableKeysResult, err := a.keyStore.HasUsableAdditionalKeys(ctx, ca)
 	if err != nil {
 		return trace.Wrap(err)
 	}
-	if hasUsableKeys {
-		// nothing to do
+	if usableKeysResult.CAHasPreferredKeyType {
+		// Nothing to do.
 		return nil
 	}
 
@@ -6085,11 +6085,11 @@ func (a *Server) ensureLocalAdditionalKeys(ctx context.Context, ca types.CertAut
 		return trace.Wrap(err)
 	}
 
-	// The CA still needs an update while the keystore does not have any usable
-	// keys in the CA.
+	// The CA still needs an update while the CA does not contain any keys of
+	// the preferred type.
 	needsUpdate := func(ca types.CertAuthority) (bool, error) {
-		hasUsableKeys, err := a.keyStore.HasUsableAdditionalKeys(ctx, ca)
-		return !hasUsableKeys, trace.Wrap(err)
+		usableKeysResult, err := a.keyStore.HasUsableAdditionalKeys(ctx, ca)
+		return !usableKeysResult.CAHasPreferredKeyType, trace.Wrap(err)
 	}
 	err = a.addAdditionalTrustedKeysAtomic(ctx, ca, newKeySet, needsUpdate)
 	if err != nil {