diff --git a/src/SqlBindingUtilities.cs b/src/SqlBindingUtilities.cs index 013191459..d13a89d24 100644 --- a/src/SqlBindingUtilities.cs +++ b/src/SqlBindingUtilities.cs @@ -237,7 +237,7 @@ internal static async Task OpenAsyncWithSqlErrorHandling(this SqlConnection conn } /// - /// Checks whether an exception is a fatal SqlException. It is deteremined to be fatal + /// Checks whether an exception is a fatal SqlException. It is determined to be fatal /// if the Class value of the Exception is 20 or higher, see /// https://learn.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlexception#remarks /// for details @@ -250,6 +250,16 @@ internal static bool IsFatalSqlException(this Exception e) return (e as SqlException)?.Class >= 20 || (e.InnerException as SqlException)?.Class >= 20; } + /// + /// Checks whether the connection state is currently Broken or Closed + /// + /// The connection to check + /// True if the connection is broken or closed, false otherwise + internal static bool IsBrokenOrClosed(this SqlConnection conn) + { + return conn.State == ConnectionState.Broken || conn.State == ConnectionState.Closed; + } + /// /// Attempts to ensure that this connection is open, if it currently is in a broken state /// then it will close the connection and re-open it. @@ -266,7 +276,7 @@ internal static async Task TryEnsureConnected(this SqlConnection conn, string connectionName, CancellationToken token) { - if (forceReconnect || conn.State.HasFlag(ConnectionState.Broken | ConnectionState.Closed)) + if (forceReconnect || conn.IsBrokenOrClosed()) { logger.LogWarning($"{connectionName} is broken, attempting to reconnect..."); logger.LogDebugWithThreadId($"BEGIN RetryOpen{connectionName}"); diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index c93fe42d4..4e4cc3148 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -304,8 +304,10 @@ private async Task RunChangeConsumptionLoopAsync() await this.ReleaseLeasesAsync(connection, token); } } - catch (Exception e) when (e.IsFatalSqlException()) + catch (Exception e) when (e.IsFatalSqlException() || connection.IsBrokenOrClosed()) { + // Retry connection if there was a fatal SQL exception or something else caused the connection to be closed + // since that indicates some other issue occurred (such as dropped network) and may be able to be recovered this._logger.LogError($"Fatal SQL Client exception processing changes. Will attempt to reestablish connection in {this._pollingIntervalInMs}ms. Exception = {e.Message}"); forceReconnect = true; } @@ -435,9 +437,9 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo this._rowsToProcess = new List>(); this._logger.LogError($"Failed to check for changes in table '{this._userTable.FullName}' due to exception: {e.GetType()}. Exception message: {e.Message}"); TelemetryInstance.TrackException(TelemetryErrorName.GetChanges, e, this._telemetryProps); - if (e.IsFatalSqlException()) + if (e.IsFatalSqlException() || connection.IsBrokenOrClosed()) { - // If we get a fatal SQL Client exception here let it bubble up so we can try to re-establish the connection + // If we get a fatal SQL Client exception or the connection is broken let it bubble up so we can try to re-establish the connection throw; } } @@ -546,8 +548,10 @@ private async void RunLeaseRenewalLoopAsync() { await this.RenewLeasesAsync(connection, token); } - catch (Exception e) when (e.IsFatalSqlException()) + catch (Exception e) when (e.IsFatalSqlException() || connection.IsBrokenOrClosed()) { + // Retry connection if there was a fatal SQL exception or something else caused the connection to be closed + // since that indicates some other issue occurred (such as dropped network) and may be able to be recovered forceReconnect = true; }