Skip to content

Add typed ClientChannelCloseException for client-rooted PUT termination #3280

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
beijxu wants to merge 10 commits into
base: master
Choose a base branch
from
Open

Add typed ClientChannelCloseException for client-rooted PUT termination #3280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4a48efd
Add typed ClientChannelCloseException for client-rooted PUT termination
beijxu Aug 9, 2026
9e596b5
Fix dead javadoc @link references in NettyMessageProcessorTest
beijxu Aug 9, 2026
8730e80
Tighten idle-timeout comment: document structural false-positive and …
beijxu Aug 9, 2026
c1d3aee
Add three-tier client-termination classification (sure/possible/other)
beijxu Aug 9, 2026
81e0af6
Document why markPossibleClientTermination's check-then-act is race-safe
beijxu Aug 9, 2026
8123512
Propagate typed client-termination exceptions into response path (Pat…
beijxu Aug 9, 2026
1438b79
Fix vacuous Path B assertion in channelInactive test
beijxu Aug 9, 2026
d48baf9
Remove unused NettyRequest.closeDueToPossibleClientTermination()
beijxu Aug 9, 2026
4d97535
Harden client-close classification per reviewer feedback (PR #3280)
beijxu Aug 11, 2026
8165718
C3: fail safe to possible tier when RestServerState is null
beijxu Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ambry-rest/src/main/java/com/github/ambry/rest/FrontendNettyChannelInitializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,6 @@ protected void initChannel(SocketChannel ch) throws Exception {
}
// custom processing class that interfaces with a RestRequestService.
pipeline.addLast("processor",
new NettyMessageProcessor(nettyMetrics, nettyConfig, performanceConfig, requestHandler));
new NettyMessageProcessor(nettyMetrics, nettyConfig, performanceConfig, requestHandler, restServerState));
}
}
80 changes: 74 additions & 6 deletions ambry-rest/src/main/java/com/github/ambry/rest/NettyMessageProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@

import com.github.ambry.config.NettyConfig;
import com.github.ambry.config.PerformanceConfig;
import com.github.ambry.utils.Utils;
import com.github.ambry.utils.ClientChannelCloseException;
import com.github.ambry.utils.PossibleClientChannelCloseException;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.http.HttpContent;
Expand All @@ -28,7 +29,6 @@
import io.netty.handler.timeout.IdleState;
import io.netty.handler.timeout.IdleStateEvent;
import java.io.IOException;
import java.nio.channels.ClosedChannelException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -72,6 +72,7 @@ public class NettyMessageProcessor extends SimpleChannelInboundHandler<HttpObjec
private final NettyConfig nettyConfig;
private final PerformanceConfig performanceConfig;
private final RestRequestHandler requestHandler;
private final RestServerState restServerState;
private static final Logger logger = LoggerFactory.getLogger(NettyMessageProcessor.class);

// variables that will live through the life of the channel.
Expand All @@ -95,13 +96,18 @@ public class NettyMessageProcessor extends SimpleChannelInboundHandler<HttpObjec
* @param nettyConfig the configuration object to use.
* @param performanceConfig the configuration object to use for SLO evaluation.
* @param requestHandler the {@link RestRequestHandler} that can be used to submit requests that need to be handled.
* @param restServerState the {@link RestServerState} used to distinguish a client-initiated channel close from a
* server-initiated one (e.g. during {@link RestServer#shutdown()}). Used to gate the
* high-confidence {@link ClientChannelCloseException} classification in
* {@link #channelInactive(ChannelHandlerContext)}.
*/
public NettyMessageProcessor(NettyMetrics nettyMetrics, NettyConfig nettyConfig, PerformanceConfig performanceConfig,
RestRequestHandler requestHandler) {
RestRequestHandler requestHandler, RestServerState restServerState) {
this.nettyMetrics = nettyMetrics;
this.nettyConfig = nettyConfig;
this.performanceConfig = performanceConfig;
this.requestHandler = requestHandler;
this.restServerState = restServerState;
logger.trace("Instantiated NettyMessageProcessor");
}

Expand Down Expand Up @@ -143,12 +149,37 @@ public void channelInactive(ChannelHandlerContext ctx) {
// and can short-circuit best-effort work (e.g. named-blob metadata commit in AmbryIdConverterFactory).
// NettyRequest.close() is idempotent via channelOpen.compareAndSet(true,false), so double-close on the
// normal-completion path is a no-op.
//
// Classify the pending readInto callback. Reaching this branch means the request was never closed by the
// normal completion path (which always closes the request - see NettyResponseChannel#completeRequest - before
// scheduling the network channel close), so the channel went inactive out-of-band. However, channelInactive
// also fires for SERVER-initiated closes: RestServer#shutdown() marks the service down and then shuts down the
// Netty server, whose worker event-loop shutdownGracefully() closes in-flight connections. To avoid labeling a
// server-shutdown abort as a high-confidence client termination, only tag the "sure" tier
// (ClientChannelCloseException) while the service is up; otherwise fall back to the "possible" tier.
// Even when up, "sure" means the close originated from our TCP peer (which may be a load balancer/proxy rather
// than the end client) - but for router health accounting and response classification that is the meaningful
// distinction from a locally-initiated shutdown close.
// If restServerState is unavailable (null), we cannot establish service liveness, so fail safe to the
// "possible" tier rather than over-claiming a sure client termination.
boolean serviceUp = restServerState != null && restServerState.isServiceUp();
try {
request.close();
if (serviceUp) {
request.closeDueToClientTermination();
} else {
request.markPossibleClientTermination();
request.close();
}
} catch (Exception e) {
logger.warn("Exception while closing request {} on channelInactive", request.getUri(), e);
}
onRequestAborted(Utils.convertToClientTerminationException(new ClosedChannelException()));
// Use the same typed exception delivered to readInto() here too, so a consumer of the response-completion path
// (onRequestAborted -> RestResponseChannel#close/onResponseComplete) can also detect this tier via instanceof,
// not just via the message-based Utils#isPossibleClientTermination check. This is behavior-neutral: both typed
// exceptions are recognized by isPossibleClientTermination() just like the previous
// Utils.convertToClientTerminationException(...) wrap was, so the response status code and
// client-early-termination metrics emitted downstream (see NettyResponseChannel#getErrorResponse) are unchanged.
onRequestAborted(serviceUp ? new ClientChannelCloseException() : new PossibleClientChannelCloseException());
} else {
close();
}
Expand All @@ -170,6 +201,21 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E
try {
if (request != null && request.isOpen() && cause instanceof Exception) {
nettyMetrics.processorExceptionCaughtCount.inc();
if (cause instanceof IOException) {
// NOTE: an IOException reaching this handler for an in-flight request is likely client-rooted (Netty's
// own handling of this client-facing channel, e.g. "connection reset"/"broken pipe" while reading further
// request content) rather than a business/destination write failure (those are reported directly to the
// RestResponseChannel by FrontendRestRequestService/AsyncRequestResponseHandler and never reach this
// pipeline handler). However, exclusivity isn't proven the way it is for channelInactive() (below) - a
// destination-write failure could in principle propagate here via Netty's implicit exception routing - so
// this is tagged as "possible" (PossibleClientChannelCloseException), not "sure"
// (ClientChannelCloseException).
try {
request.markPossibleClientTermination();

@nicolaslopezbravo nicolaslopezbravo Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor, but this drops the useful part of the diagnosis. cause here carries the actual socket message ("Connection reset by peer", "Broken pipe"), and the readInto callback ends up with a cause-less, message-less singleton instead.

It's also asymmetric: onRequestAborted((Exception) cause) a few lines below still forwards the real exception to the response path, so whether the underlying reason is visible depends on which side you're reading. During an incident that's exactly the detail you want.

Suggest an overload that preserves it — ClosedChannelException has no cause constructor, so it needs initCause, which also means not using the shared singleton on this path:

void markPossibleClientTermination(Throwable cause) {
  if (channelException == CLOSED_CHANNEL_EXCEPTION) {
    PossibleClientChannelCloseException e = new PossibleClientChannelCloseException();
    e.initCause(cause);
    channelException = e;
  }
}

The no-arg version can stay as-is for the idle-timeout call site, which genuinely has no cause to attach.

@beijxu beijxu Aug 12, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right that the asymmetry is real — the readInto callback gets the message-less singleton while onRequestAborted((Exception) cause) forwards the real socket exception — and your point that initCause needs a fresh instance (the singleton is shared/class-loaded) is correct.

That said, for the classification goal this is observability-only, so I'm inclined to leave it:

  • Nothing on the callback side consumes the message/cause. RouterUtils.isSystemHealthError() and Utils.isPossibleClientTermination() both classify purely by instanceof, so the tier is already correct (possible) with or without the cause attached.
  • The real cause isn't lost during an incident — it's still logged on the response path via onResponseComplete's log(exception), which receives the actual cause. Only the router-side log line is less specific.

So the enrichment is a genuine nicety but doesn't change how any close is tiered. If diagnosing these on the router side turns out to be painful in practice I'm happy to add the markPossibleClientTermination(Throwable cause) overload (fresh instance + initCause, no-arg kept for the idle path) as a follow-up — but I'd keep it out of this PR since it's not part of the sure/possible/server separation.

} catch (Exception e) {
logger.warn("Exception while marking request {} as possibly client-terminated", request.getUri(), e);
}
}
onRequestAborted((Exception) cause);
} else if (isOpen()) {
if (cause instanceof RestServiceException) {
Expand Down Expand Up @@ -221,7 +267,29 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object event) {
nettyConfig.nettyServerIdleTimeSeconds);
nettyMetrics.idleConnectionCloseCount.inc();
if (request != null && request.isOpen()) {
onRequestAborted(Utils.convertToClientTerminationException(new ClosedChannelException()));
// NOTE: idle-timeout is tagged as "possible" (PossibleClientChannelCloseException), not "sure"
// (ClientChannelCloseException). NettyRequest suspends reads (autoRead=false) while the amount of data
// buffered for a slow/backpressured downstream consumer exceeds nettyServerRequestBufferWatermark (see
// NettyRequest#continueReadIfPossible); while reads are suspended, no channelRead events can occur no
// matter how active the client is, so IdleStateHandler's ALL_IDLE can fire purely because OUR OWN
// downstream write is stalled - not because the client is idle or has failed. Worse, this isn't just a
// narrow race: NettyRequest#writeContent unconditionally re-enables autoRead the instant the last client
// chunk arrives, before the corresponding destination write is even issued - so a slow destination write
// on that final chunk leaves the channel silent in both directions with autoRead==true for the entire
// idle window, a deterministic (not merely racy) false-positive shape. Tagging this as a "sure" client
// termination would risk hiding a real server/destination-side slowness problem, violating the "bias
// toward NOT-client on ambiguity" invariant; only channelInactive() has been proven exclusively
// client-rooted. A correct "sure" follow-up would need to gate on "no destination write currently in
// flight" rather than on autoRead state - out of scope here.
try {
request.markPossibleClientTermination();
} catch (Exception e) {
logger.warn("Exception while marking request {} as possibly client-terminated", request.getUri(), e);
}
// See the comment on the equivalent onRequestAborted(...) call in channelInactive() above: using the typed
// "possible" exception here is behavior-neutral for the same reason (isPossibleClientTermination()
// recognizes it unconditionally, exactly as it did the previous message-based wrap).
onRequestAborted(new PossibleClientChannelCloseException());
} else {
close();
}
Expand Down
56 changes: 55 additions & 1 deletion ambry-rest/src/main/java/com/github/ambry/rest/NettyRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import com.github.ambry.commons.Callback;
import com.github.ambry.router.AsyncWritableChannel;
import com.github.ambry.router.FutureResult;
import com.github.ambry.utils.ClientChannelCloseException;
import com.github.ambry.utils.PossibleClientChannelCloseException;
import io.netty.buffer.ByteBuf;
import io.netty.channel.Channel;
import io.netty.channel.DefaultMaxBytesRecvByteBufAllocator;
Expand Down Expand Up @@ -65,6 +67,9 @@ public class NettyRequest implements RestRequest {
// is <=0, it is assumed that there is no limit on the size of unacknowledged data.
static int bufferWatermark = -1;
private static final ClosedChannelException CLOSED_CHANNEL_EXCEPTION = new ClosedChannelException();
private static final ClientChannelCloseException CLIENT_CHANNEL_CLOSE_EXCEPTION = new ClientChannelCloseException();
private static final PossibleClientChannelCloseException POSSIBLE_CLIENT_CHANNEL_CLOSE_EXCEPTION =
new PossibleClientChannelCloseException();

protected final HttpRequest request;
protected final Channel channel;
Expand Down Expand Up @@ -301,6 +306,48 @@ public void close() {
}
}

/**
* Marks this request's pending read (if any) as terminated because of a high-confidence, client-rooted event
* (e.g. the client disconnected or reset the connection). Must be called, if at all, before {@link #close()} so
* that {@link ClientChannelCloseException} - rather than the default {@link ClosedChannelException} - is delivered
* to the pending {@link #readInto} callback. Idempotent and safe to call even if there is no pending read.
*/
void markClientTerminated() {
channelException = CLIENT_CHANNEL_CLOSE_EXCEPTION;
}
Comment on lines +315 to +317

@nicolaslopezbravo nicolaslopezbravo Aug 10, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

channelException isn't a dedicated field for this classification — validateState() also writes a real RestServiceException into it (lines 608/612) when the content length doesn't match the header. This assignment is unconditional, so a genuine BadRequest can be silently replaced by the client-close tag.

Realistic trigger: an undersized body is only detected on LastHttpContent, so a truncated PUT followed by a client reset hits validateState first and then channelInactive. The 400 is lost and the request lands in the client-early-termination bucket instead.

Suggested change
void markClientTerminated() {
channelException = CLIENT_CHANNEL_CLOSE_EXCEPTION;
}
void markClientTerminated() {
// Only overwrite the default sentinel. A real error already recorded here (e.g. the RestServiceException from
// validateState) is the more specific cause and must win.
if (channelException == CLOSED_CHANNEL_EXCEPTION) {
channelException = CLIENT_CHANNEL_CLOSE_EXCEPTION;
}
}

Checking for the untouched sentinel rather than excluding one specific value keeps any real error that was already recorded, and stays correct if another writer to this field is added later.

@beijxu beijxu Aug 12, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks — I traced this end to end and I'd like to push back on the impact before changing the guard, because in the context of the goal here (separating sure-client vs possible-client vs server/unclassified connection closes) this guard is actually doing the right thing.

Two findings:

  1. The 400 is not lost. When validateState() throws the BadRequest, it propagates channelRead0 → exceptionCaught, where cause is a RestServiceException (not IOException), so the markPossibleClientTermination branch is skipped and onRequestAborted((Exception) cause) runs. The client-facing error response is built from that passed exception via getErrorResponse(exception) — it never reads channelException. So the client still gets its 400 regardless of what later overwrites the field. The only thing channelException feeds is the readInto callback tier.

  2. On the readInto tier, keeping ClientChannelCloseException is the correct classification, not a regression. In the scenario you describe the connection closed because the client reset (channelInactive with serviceUp==true). That is a genuine sure-client close, so tagging it sure is right. Applying the suggested == CLOSED_CHANNEL_EXCEPTION guard would instead preserve the RestServiceException, which RouterUtils.isSystemHealthError() treats as true (a server health error) — i.e. it would reclassify a client-caused close as a server problem, the opposite of this PR's goal. There's also no durability impact: an aborted PUT commits nothing.

Given that, I'd prefer to leave markClientTerminated() as-is. If we ever do want to protect a business error from being overwritten, the right predicate is !(channelException instanceof RestServiceException) (not identity against the sentinel), so a real client reset can still upgrade a prior "possible" tag to "sure" — but since the client response is unaffected and the sure tag is the correct tier here, I don't think it's warranted. Happy to reconsider if you see a consumer of the readInto-side exception that I'm missing.


/**
* Convenience method that marks this request as client-terminated (see {@link #markClientTerminated()}) and then
* closes it, in one call. Use this at call sites that close the request directly (as opposed to sites where the
* close happens later via a different code path, e.g. through {@link NettyResponseChannel}), so the "mark before
* close" ordering requirement can never be broken by a future edit that reorders or drops one of the two calls.
*/
void closeDueToClientTermination() {
markClientTerminated();
close();
}

/**
* Marks this request's pending read (if any) as terminated because of an event that is plausibly, but not
* confirmably, client-rooted (e.g. an idle timeout, which can equally be caused by a slow destination write; or an
* {@link java.io.IOException} reaching Netty's {@code exceptionCaught}, which is usually but not provably
* client-facing). Must be called, if at all, before {@link #close()} so that
* {@link PossibleClientChannelCloseException} - rather than the default {@link ClosedChannelException} - is
* delivered to the pending {@link #readInto} callback. Does not overwrite an already-set
* {@link #markClientTerminated() high-confidence} tag, so that a "sure" classification is never downgraded to
* "possible" if both were somehow triggered for the same request. Idempotent and safe to call even if there is no
* pending read.
*/
void markPossibleClientTermination() {
// Check-then-act, not atomic/CAS - safe because channelInactive/exceptionCaught/userEventTriggered all fire on
// the same Netty channel's single-threaded event loop for a given request, so these calls are never actually
// concurrent with each other; the volatile field just ensures the eventual invokeCallback() on another thread
// sees the final write.
if (channelException != CLIENT_CHANNEL_CLOSE_EXCEPTION) {

@nicolaslopezbravo nicolaslopezbravo Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same issue as markClientTerminated above: this guard only prevents downgrading the "sure" tag, but it still overwrites a RestServiceException written by validateState().

Suggested change
if (channelException != CLIENT_CHANNEL_CLOSE_EXCEPTION) {
if (channelException == CLOSED_CHANNEL_EXCEPTION) {

Checking against the default sentinel ("has anything meaningful been written here yet?") rather than excluding one specific value covers both cases, and stays correct if another writer to this field is added later.

Worth a test where channelException already holds a RestServiceException when a mark fires — the current suite doesn't cover that ordering.

@beijxu beijxu Aug 12, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed the guard is imprecise, but I checked reachability and I don't think a RestServiceException can realistically still be in channelException when this runs, so I'd rather not add code for a path that can't execute.

For validateState's RestServiceException to survive to a later markPossibleClientTermination() on the same open request:

  • The exceptionCaught IOException path can't get there — after validateState throws, cause is a RestServiceException, so the instanceof IOException gate is false and markPossibleClientTermination() is never called on that path.
  • That leaves only the idle-timeout path (userEventTriggered ALL_IDLE) firing inside the millisecond-scale error-response write window. Idle timeout is nettyServerIdleTimeSeconds of total silence in both directions, so landing it inside that window isn't realistically reachable.

Functionally it also has no effect on the tiering: "possible" is already the lowest live tier, and the != SURE guard already prevents the one downgrade that would matter (sure → possible). So the suggested == CLOSED_CHANNEL_EXCEPTION is strictly-correct hardening but with de-minimis reachability and no classification change. I'll leave it as-is unless you feel strongly; if we do change it I'd apply the same !(channelException instanceof RestServiceException) predicate as the sibling method for consistency rather than identity-against-sentinel.

channelException = POSSIBLE_CLIENT_CHANNEL_CLOSE_EXCEPTION;
}
}

@Override
public RestRequestMetricsTracker getMetricsTracker() {
return restRequestMetricsTracker;
Expand Down Expand Up @@ -336,7 +383,14 @@ public Future<Long> readInto(AsyncWritableChannel asyncWritableChannel, Callback
try {
if (!isOpen()) {
nettyMetrics.requestAlreadyClosedError.inc();
tempWrapper.invokeCallback(new ClosedChannelException());
// Deliver the stored channelException (not a fresh ClosedChannelException) so a client-termination
// classification recorded via markClientTerminated()/markPossibleClientTermination() before close() is
// preserved even when the read is registered AFTER the request already closed. This is the realistic
// AsyncRequestResponseHandler queue-then-read ordering: a client can disconnect (channelInactive ->
// closeDueToClientTermination) before the router calls readInto(), at which point close() had no
// callbackWrapper to deliver to. channelException defaults to a plain ClosedChannelException when the
// request was never tagged, so the untagged path is unchanged.
tempWrapper.invokeCallback(channelException);
} else if (writeChannel != null) {
throw new IllegalStateException("ReadableStreamChannel cannot be read more than once");
}
Expand Down
31 changes: 22 additions & 9 deletions ambry-rest/src/main/java/com/github/ambry/rest/NettyResponseChannel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -805,17 +805,30 @@ private HttpResponseStatus getHttpResponseStatus(ResponseStatus responseStatus)
* @param shouldCloseRequest used to determine if we want to close the request or not.
*/
private void completeRequest(boolean closeNetworkChannel, boolean shouldDelay, boolean shouldCloseRequest) {
if ((closeNetworkChannel || forceClose) && ctx.channel().isOpen()) {
if (shouldDelay && (request != null && request.getRestMethod().equals(RestMethod.POST))
&& this.nettyConfig.nettyServerCloseDelayTimeoutMs > 0) {
nettyMetrics.delayedCloseScheduledCount.inc();
writeFuture.addListener(DELAYED_CLOSE);
} else {
writeFuture.addListener(ChannelFutureListener.CLOSE);
// Close the request (and flip its isOpen() state to false) before scheduling the network channel close below.
// If writeFuture is already complete, addListener(...) fires its listener synchronously, which can close the
// network channel and trigger channelInactive() re-entrantly on this same call stack. NettyMessageProcessor's
// channelInactive() uses request.isOpen() to decide whether channel inactivity is client-rooted (as opposed to
// this server-initiated completion) - closeRequest() must therefore run first so that check is never fooled by
// a server-initiated close still in progress. The network channel close below must always be scheduled even if
// closeRequest() throws (that was always attempted first previously), so it runs in a finally block; the original
// exception then propagates to the caller's existing failure handling (e.g. onResponseComplete's catch, which
// increments responseCompleteTasksError and fails the writeFuture) rather than being downgraded to a log-only
// event.
try {
closeRequest(shouldCloseRequest);
} finally {
if ((closeNetworkChannel || forceClose) && ctx.channel().isOpen()) {
if (shouldDelay && (request != null && request.getRestMethod().equals(RestMethod.POST))
&& this.nettyConfig.nettyServerCloseDelayTimeoutMs > 0) {
nettyMetrics.delayedCloseScheduledCount.inc();
writeFuture.addListener(DELAYED_CLOSE);
} else {
writeFuture.addListener(ChannelFutureListener.CLOSE);
}
logger.trace("Requested closing of channel {}", ctx.channel());
}
logger.trace("Requested closing of channel {}", ctx.channel());
}
closeRequest(shouldCloseRequest);
}

/**
Expand Down
Loading
Loading