[v18] disable mysql health check

docs/pages/enroll-resources/database-access/guides/health-checks.mdx

@@ -198,6 +198,25 @@ Refer to the [Database Service troubleshooting guide](../troubleshooting.mdx) fo
 
 ## FAQ
 
+### Why are MySQL health checks disabled?
+
+MySQL tracks connection errors for each remote host that tries to connect to the database.
+Each TCP health check is counted as a connection error and eventually MySQL will
+block a host when `sum_connect_errors >= max_connect_errors`.
+As a result, TCP health checks eventually cause MySQL databases to block Teleport.
+
+To re-enable TCP health checks:
+- set `max_connect_errors` to its maximum value on the database to effectively disable the automatic host blocking.
+- set the environment variable `TELEPORT_ENABLE_MYSQL_DB_HEALTH_CHECKS=1` on your Teleport Database Service instance(s).
+
+<Admonition type="warning">
+[MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/host-cache.html#:~:text=The%20value%20of%20the%20max_connect_errors,host%20from%20further%20connection%20requests) notes:
+> The value of the max_connect_errors system variable determines how many successive interrupted connection requests the server permits before blocking a host. After max_connect_errors failed requests without a successful connection, the server assumes that something is wrong (for example, that someone is trying to break in), and blocks the host from further connection requests.
+
+Setting `max_connect_errors` to its maximum value will effectively disable MySQL's host blocking feature.
+See https://dev.mysql.com/doc/refman/8.4/en/server-system-variables.html#sysvar_max_connect_errors
+</Admonition>
+
 ### How to disable database health checks?
 
 You can disable health checks for databases by updating your cluster's `health_check_config` resources to only match specific databases.

lib/srv/db/server.go

@@ -24,7 +24,9 @@ import (
 	"errors"
 	"log/slog"
 	"net"
+	"os"
 	"runtime/debug"
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -96,7 +98,9 @@ func init() {
 	objects.RegisterObjectFetcher(postgres.NewObjectFetcher, defaults.ProtocolPostgres)
 
 	endpoints.RegisterResolver(postgres.NewEndpointsResolver, defaults.ProtocolPostgres, defaults.ProtocolCockroachDB)
-	endpoints.RegisterResolver(mysql.NewEndpointsResolver, defaults.ProtocolMySQL)
+	if isMySQLHealthCheckEnabled() {
+		endpoints.RegisterResolver(mysql.NewEndpointsResolver, defaults.ProtocolMySQL)
+	}
 	endpoints.RegisterResolver(mongodb.NewEndpointsResolver, defaults.ProtocolMongoDB)
 
 	endpoints.RegisterResolver(cassandra.NewEndpointsResolver, defaults.ProtocolCassandra)
@@ -111,6 +115,14 @@ func init() {
 	endpoints.RegisterResolver(sqlserver.NewEndpointsResolver, defaults.ProtocolSQLServer)
 }
 
+// isMySQLHealthCheckEnabled returns true if MySQL health checks are enabled.
+// The default MySQL settings will automatically block a host after it dials
+// MySQL without handshaking 100 times, so we make MySQL health checks opt-in.
+func isMySQLHealthCheckEnabled() bool {
+	enabled, err := strconv.ParseBool(os.Getenv("TELEPORT_ENABLE_MYSQL_DB_HEALTH_CHECKS"))
+	return err == nil && enabled
+}
+
 // Config is the configuration for a database proxy server.
 type Config struct {
 	// Clock used to control time.
@@ -520,7 +532,7 @@ func New(ctx context.Context, config Config) (*Server, error) {
 // startDatabase performs initialization actions for the provided database
 // such as starting dynamic labels and initializing CA certificate.
 func (s *Server) startDatabase(ctx context.Context, database types.Database) error {
-	if err := s.startHealthCheck(ctx, database); err != nil {
+	if err := s.startHealthCheck(ctx, database); err != nil && endpoints.IsRegistered(database) {
 		s.log.DebugContext(ctx, "Failed to start database health checker",
 			"db", database.GetName(),
 			"error", err,
@@ -1487,7 +1499,11 @@ func (s *Server) stopHealthCheck(db types.Database) error {
 // getTargetHealth returns the target health for the database.
 func (s *Server) getTargetHealth(ctx context.Context, db types.Database) types.TargetHealth {
 	if err := checkSupportsHealthChecks(db); err != nil {
-		return types.TargetHealth{}
+		return types.TargetHealth{
+			Status:           string(types.TargetHealthStatusUnknown),
+			TransitionReason: string(types.TargetHealthTransitionReasonDisabled),
+			Message:          err.Error(),
+		}
 	}
 
 	health, err := s.cfg.healthCheckManager.GetTargetHealth(db)

lib/srv/db/server_test.go

@@ -714,6 +714,10 @@ func TestHealthCheck(t *testing.T) {
 		withSnowflake("snowflake")(t, ctx, testCtx),
 	}
 	for _, db := range databases {
+		if db.GetProtocol() == defaults.ProtocolMySQL {
+			require.False(t, endpoints.IsRegistered(db), "health checks for MySQL protocol should be disabled")
+			continue
+		}
 		require.True(t, endpoints.IsRegistered(db), "database %v does not have a registered endpoint resolver", db.GetName())
 	}
 	dynamoListenAddr := net.JoinHostPort("localhost", testCtx.dynamodb["dynamodb"].db.Port())
@@ -756,6 +760,12 @@ func TestHealthCheck(t *testing.T) {
 			t.Parallel()
 			dbServer, err := testCtx.server.getServerInfo(ctx, db)
 			require.NoError(t, err)
+			if db.GetProtocol() == defaults.ProtocolMySQL {
+				require.Equal(t, "unknown", dbServer.GetTargetHealth().Status)
+				require.Equal(t, "disabled", dbServer.GetTargetHealth().TransitionReason)
+				require.Equal(t, `endpoint health checks for database protocol "mysql" are not supported`, dbServer.GetTargetHealth().Message)
+				return
+			}
 			require.EventuallyWithT(t, func(t *assert.CollectT) {
 				assert.Equal(t, types.TargetHealthStatusHealthy, dbServer.GetTargetHealthStatus())
 			}, 30*time.Second, time.Millisecond*250, "waiting for database %s to become healthy", db.GetName())