Skip to content

fix(admin): retry on network errors by resetting controller connection #3406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
DCjanus wants to merge 2 commits into from
Closed

fix(admin): retry on network errors by resetting controller connection #3406

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8c61b9b
fix(admin): retry on network errors by resetting controller connection
DCjanus Dec 15, 2025
21f2702
chore: fix spelling in admin network retry comment
DCjanus Dec 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions admin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import (
"io"
"maps"
"math/rand"
"net"
"strconv"
"sync"
"syscall"
"time"
)

Expand Down Expand Up @@ -211,16 +213,23 @@ func (ca *clusterAdmin) refreshController() (*Broker, error) {
}

// isRetriableControllerError returns `true` if the given error type unwraps to
// an `ErrNotController` or `EOF` response from Kafka
// an `ErrNotController`, `EOF` response from Kafka, or a network-level error.
func isRetriableControllerError(err error) bool {
return errors.Is(err, ErrNotController) || errors.Is(err, io.EOF)
return errors.Is(err, ErrNotController) || errors.Is(err, io.EOF) || isNetworkError(err)
}

// isRetriableGroupCoordinatorError returns `true` if the given error type
// unwraps to an `ErrNotCoordinatorForConsumer`,
// `ErrConsumerCoordinatorNotAvailable` or `EOF` response from Kafka
// `ErrConsumerCoordinatorNotAvailable`, `EOF` response from Kafka, or a network-level error
func isRetriableGroupCoordinatorError(err error) bool {
return errors.Is(err, ErrNotCoordinatorForConsumer) || errors.Is(err, ErrConsumerCoordinatorNotAvailable) || errors.Is(err, io.EOF)
return errors.Is(err, ErrNotCoordinatorForConsumer) || errors.Is(err, ErrConsumerCoordinatorNotAvailable) || errors.Is(err, io.EOF) || isNetworkError(err)
}

// isNetworkError categorizes transient network failures that should trigger
// a reconnect/retry path (e.g. broken pipe, connection reset/timeout).
func isNetworkError(err error) bool {
var ne net.Error
return errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.EPIPE) || errors.As(err, &ne)
}

// retryOnError will repeatedly call the given (error-returning) func in the
Expand All @@ -234,6 +243,13 @@ func (ca *clusterAdmin) retryOnError(retryable func(error) bool, fn func() error
if err == nil || attemptsRemaining <= 0 || !retryable(err) {
return err
}
if isNetworkError(err) {
// Close the cached controller connection so the next attempt will reopen it.
// Ref: https://github.com/IBM/sarama/issues/1162 — admin calls can get stuck on a broken controller connection.
if ctrl, e := ca.client.Controller(); e == nil {
_ = ctrl.Close()
}
}
Logger.Printf(
"admin/request retrying after %dms... (%d attempts remaining)\n",
ca.conf.Admin.Retry.Backoff/time.Millisecond, attemptsRemaining)
Expand Down