From b51829a9745fcccf9286d931c3eed413e9bd8438 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Tue, 6 Aug 2024 10:44:36 +0200 Subject: [PATCH 01/19] add SerializedInvoker assertion Signed-off-by: Ludovic Orban --- .../jetty/client/transport/HttpReceiver.java | 2 + .../jetty/util/thread/SerializedInvoker.java | 12 ++++++ .../util/thread/SerializedInvokerTest.java | 42 ++++++++++--------- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 174643b7189b..e8e63a03f04a 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -320,6 +320,8 @@ protected void responseHeaders(HttpExchange exchange) */ protected void responseContentAvailable() { + if (!invoker.isInvoking()) + throw new IllegalStateException(); if (LOG.isDebugEnabled()) LOG.debug("Response content available on {}", this); contentSource.onDataAvailable(); diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index 72bcd693a732..f440ca669e9c 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -35,6 +35,15 @@ public class SerializedInvoker private static final Logger LOG = LoggerFactory.getLogger(SerializedInvoker.class); private final AtomicReference _tail = new AtomicReference<>(); + private int recursionCounter; + + /** + * @return whether this invoker is currently executing a task + */ + public boolean isInvoking() + { + return recursionCounter > 0; + } /** * Arrange for a task to be invoked, mutually excluded from other tasks. @@ -188,10 +197,13 @@ public void run() LOG.debug("Running link {} of {}", link, SerializedInvoker.this); try { + recursionCounter++; link._task.run(); + recursionCounter--; } catch (Throwable t) { + recursionCounter--; if (LOG.isDebugEnabled()) LOG.debug("Failed while running link {} of {}", link, SerializedInvoker.this, t); onError(link._task, t); diff --git a/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java b/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java index 2fc9011d7794..2242683dd174 100644 --- a/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java +++ b/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java @@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,17 +24,12 @@ public class SerializedInvokerTest { - SerializedInvoker _serialedInvoker; + private SerializedInvoker _serializedInvoker; @BeforeEach public void beforeEach() { - _serialedInvoker = new SerializedInvoker(); - } - - @AfterEach - public void afterEach() - { + _serializedInvoker = new SerializedInvoker(); } @Test @@ -45,24 +39,27 @@ public void testSimple() throws Exception Task task2 = new Task(); Task task3 = new Task(); - Runnable todo = _serialedInvoker.offer(task1); - assertNull(_serialedInvoker.offer(task2)); - assertNull(_serialedInvoker.offer(task3)); + Runnable todo = _serializedInvoker.offer(task1); + assertNull(_serializedInvoker.offer(task2)); + assertNull(_serializedInvoker.offer(task3)); assertFalse(task1.hasRun()); assertFalse(task2.hasRun()); assertFalse(task3.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); todo.run(); assertTrue(task1.hasRun()); assertTrue(task2.hasRun()); assertTrue(task3.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); Task task4 = new Task(); - todo = _serialedInvoker.offer(task4); + todo = _serializedInvoker.offer(task4); todo.run(); assertTrue(task4.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); } @Test @@ -72,22 +69,25 @@ public void testMulti() Task task2 = new Task(); Task task3 = new Task(); - Runnable todo = _serialedInvoker.offer(null, task1, null, task2, null, task3, null); + Runnable todo = _serializedInvoker.offer(null, task1, null, task2, null, task3, null); assertFalse(task1.hasRun()); assertFalse(task2.hasRun()); assertFalse(task3.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); todo.run(); assertTrue(task1.hasRun()); assertTrue(task2.hasRun()); assertTrue(task3.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); Task task4 = new Task(); - todo = _serialedInvoker.offer(task4); + todo = _serializedInvoker.offer(task4); todo.run(); assertTrue(task4.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); } @Test @@ -99,7 +99,7 @@ public void testRecursive() @Override public void run() { - assertNull(_serialedInvoker.offer(task3)); + assertNull(_serializedInvoker.offer(task3)); super.run(); } }; @@ -108,30 +108,33 @@ public void run() @Override public void run() { - assertNull(_serialedInvoker.offer(task2)); + assertNull(_serializedInvoker.offer(task2)); super.run(); } }; - Runnable todo = _serialedInvoker.offer(task1); + Runnable todo = _serializedInvoker.offer(task1); assertFalse(task1.hasRun()); assertFalse(task2.hasRun()); assertFalse(task3.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); todo.run(); assertTrue(task1.hasRun()); assertTrue(task2.hasRun()); assertTrue(task3.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); Task task4 = new Task(); - todo = _serialedInvoker.offer(task4); + todo = _serializedInvoker.offer(task4); todo.run(); assertTrue(task4.hasRun()); + assertFalse(_serializedInvoker.isInvoking()); } - public static class Task implements Runnable + public class Task implements Runnable { CountDownLatch _run = new CountDownLatch(1); @@ -143,6 +146,7 @@ boolean hasRun() @Override public void run() { + assertTrue(_serializedInvoker.isInvoking()); _run.countDown(); } } From 9f3549bf22bafaf2f26ac59f6c795486a6a9bfe2 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Tue, 6 Aug 2024 11:15:54 +0200 Subject: [PATCH 02/19] add SerializedInvoker assertion Signed-off-by: Ludovic Orban --- .../org/eclipse/jetty/client/transport/HttpReceiver.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index e8e63a03f04a..4abeb296b5ca 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -738,6 +738,9 @@ private void processDemand() if (LOG.isDebugEnabled()) LOG.debug("Processing demand on {}", this); + if (!invoker.isInvoking()) + throw new IllegalStateException(); + Content.Chunk current; try (AutoLock ignored = lock.lock()) { @@ -777,9 +780,15 @@ private void invokeDemandCallback(boolean invoke) try { if (invoke) + { invoker.run(demandCallback); + } else + { + if (!invoker.isInvoking()) + throw new IllegalStateException(); demandCallback.run(); + } } catch (Throwable x) { From dede1225cddc544268da40cb50fbd1faffe57586 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Wed, 7 Aug 2024 10:31:57 +0200 Subject: [PATCH 03/19] make invoker assertion per-thread + name invokers to help debugging Signed-off-by: Ludovic Orban --- .../jetty/client/transport/HttpReceiver.java | 8 ++-- .../org/eclipse/jetty/http/MultiPart.java | 2 +- .../jetty/io/content/AsyncContent.java | 2 +- .../io/content/ByteBufferContentSource.java | 2 +- .../jetty/io/content/ChunksContentSource.java | 2 +- .../io/content/ContentSourceTransformer.java | 2 +- .../io/content/InputStreamContentSource.java | 2 +- .../jetty/io/content/PathContentSource.java | 2 +- .../io/internal/ByteChannelContentSource.java | 2 +- .../server/internal/HttpChannelState.java | 9 +++- .../jetty/util/thread/SerializedExecutor.java | 2 +- .../jetty/util/thread/SerializedInvoker.java | 32 +++++++++---- .../util/thread/SerializedInvokerTest.java | 46 +++++++++++++------ 13 files changed, 77 insertions(+), 36 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 4abeb296b5ca..a0edfdc78865 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -67,7 +67,7 @@ public abstract class HttpReceiver { private static final Logger LOG = LoggerFactory.getLogger(HttpReceiver.class); - private final SerializedInvoker invoker = new SerializedInvoker(); + private final SerializedInvoker invoker = new SerializedInvoker(HttpReceiver.class); private final HttpChannel channel; private ResponseState responseState = ResponseState.IDLE; private NotifiableContentSource contentSource; @@ -320,7 +320,7 @@ protected void responseHeaders(HttpExchange exchange) */ protected void responseContentAvailable() { - if (!invoker.isInvoking()) + if (!invoker.isCurrentThreadInvoking()) throw new IllegalStateException(); if (LOG.isDebugEnabled()) LOG.debug("Response content available on {}", this); @@ -738,7 +738,7 @@ private void processDemand() if (LOG.isDebugEnabled()) LOG.debug("Processing demand on {}", this); - if (!invoker.isInvoking()) + if (!invoker.isCurrentThreadInvoking()) throw new IllegalStateException(); Content.Chunk current; @@ -785,7 +785,7 @@ private void invokeDemandCallback(boolean invoke) } else { - if (!invoker.isInvoking()) + if (!invoker.isCurrentThreadInvoking()) throw new IllegalStateException(); demandCallback.run(); } diff --git a/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPart.java b/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPart.java index 452cd394bbd5..fd9d89673f34 100644 --- a/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPart.java +++ b/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPart.java @@ -564,7 +564,7 @@ public String toString() public abstract static class AbstractContentSource implements Content.Source, Closeable { private final AutoLock lock = new AutoLock(); - private final SerializedInvoker invoker = new SerializedInvoker(); + private final SerializedInvoker invoker = new SerializedInvoker(AbstractContentSource.class); private final Queue parts = new ArrayDeque<>(); private final String boundary; private final ByteBuffer firstBoundary; diff --git a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/AsyncContent.java b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/AsyncContent.java index 6bd5eeebc32a..02bc1d0cf6b2 100644 --- a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/AsyncContent.java +++ b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/AsyncContent.java @@ -50,7 +50,7 @@ public String toString() }; private final AutoLock.WithCondition lock = new AutoLock.WithCondition(); - private final SerializedInvoker invoker = new SerializedInvoker(); + private final SerializedInvoker invoker = new SerializedInvoker(AsyncContent.class); private final Queue chunks = new ArrayDeque<>(); private Content.Chunk persistentFailure; private boolean readClosed; diff --git a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ByteBufferContentSource.java b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ByteBufferContentSource.java index 23a4e6fb97d3..6273cab9e76a 100644 --- a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ByteBufferContentSource.java +++ b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ByteBufferContentSource.java @@ -31,7 +31,7 @@ public class ByteBufferContentSource implements Content.Source { private final AutoLock lock = new AutoLock(); - private final SerializedInvoker invoker = new SerializedInvoker(); + private final SerializedInvoker invoker = new SerializedInvoker(ByteBufferContentSource.class); private final long length; private final Collection byteBuffers; private Iterator iterator; diff --git a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ChunksContentSource.java b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ChunksContentSource.java index a00680251cf4..b065efeb618b 100644 --- a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ChunksContentSource.java +++ b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ChunksContentSource.java @@ -33,7 +33,7 @@ public class ChunksContentSource implements Content.Source { private final AutoLock lock = new AutoLock(); - private final SerializedInvoker invoker = new SerializedInvoker(); + private final SerializedInvoker invoker = new SerializedInvoker(ChunksContentSource.class); private final long length; private final Collection chunks; private Iterator iterator; diff --git a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ContentSourceTransformer.java b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ContentSourceTransformer.java index 760e4f1f42d6..c4a6a9fe9c54 100644 --- a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ContentSourceTransformer.java +++ b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/ContentSourceTransformer.java @@ -39,7 +39,7 @@ public abstract class ContentSourceTransformer implements Content.Source protected ContentSourceTransformer(Content.Source rawSource) { - this(rawSource, new SerializedInvoker()); + this(rawSource, new SerializedInvoker(ContentSourceTransformer.class)); } protected ContentSourceTransformer(Content.Source rawSource, SerializedInvoker invoker) diff --git a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/InputStreamContentSource.java b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/InputStreamContentSource.java index bc14ba9d9e95..dcce4d5c7ce2 100644 --- a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/InputStreamContentSource.java +++ b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/InputStreamContentSource.java @@ -38,7 +38,7 @@ public class InputStreamContentSource implements Content.Source { private final AutoLock lock = new AutoLock(); - private final SerializedInvoker invoker = new SerializedInvoker(); + private final SerializedInvoker invoker = new SerializedInvoker(InputStreamContentSource.class); private final InputStream inputStream; private ByteBufferPool.Sized bufferPool; private Runnable demandCallback; diff --git a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/PathContentSource.java b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/PathContentSource.java index f731280f737e..0481761f5b98 100644 --- a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/PathContentSource.java +++ b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/content/PathContentSource.java @@ -41,7 +41,7 @@ public class PathContentSource implements Content.Source // TODO in 12.1.x reimplement this class based on ByteChannelContentSource private final AutoLock lock = new AutoLock(); - private final SerializedInvoker invoker = new SerializedInvoker(); + private final SerializedInvoker invoker = new SerializedInvoker(PathContentSource.class); private final Path path; private final long length; private final ByteBufferPool byteBufferPool; diff --git a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/internal/ByteChannelContentSource.java b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/internal/ByteChannelContentSource.java index 65336ac6520d..c8713b2b39bf 100644 --- a/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/internal/ByteChannelContentSource.java +++ b/jetty-core/jetty-io/src/main/java/org/eclipse/jetty/io/internal/ByteChannelContentSource.java @@ -39,7 +39,7 @@ public class ByteChannelContentSource implements Content.Source { private final AutoLock lock = new AutoLock(); - private final SerializedInvoker _invoker = new SerializedInvoker(); + private final SerializedInvoker _invoker = new SerializedInvoker(ByteChannelContentSource.class); private final ByteBufferPool.Sized _byteBufferPool; private ByteChannel _byteChannel; private final long _offset; diff --git a/jetty-core/jetty-server/src/main/java/org/eclipse/jetty/server/internal/HttpChannelState.java b/jetty-core/jetty-server/src/main/java/org/eclipse/jetty/server/internal/HttpChannelState.java index e97331b573ae..f41e61c729b1 100644 --- a/jetty-core/jetty-server/src/main/java/org/eclipse/jetty/server/internal/HttpChannelState.java +++ b/jetty-core/jetty-server/src/main/java/org/eclipse/jetty/server/internal/HttpChannelState.java @@ -128,8 +128,8 @@ public HttpChannelState(ConnectionMetaData connectionMetaData) { _connectionMetaData = connectionMetaData; // The SerializedInvoker is used to prevent infinite recursion of callbacks calling methods calling callbacks etc. - _readInvoker = new HttpChannelSerializedInvoker(); - _writeInvoker = new HttpChannelSerializedInvoker(); + _readInvoker = new HttpChannelSerializedInvoker(HttpChannelState.class.getSimpleName() + "#readInvoker"); + _writeInvoker = new HttpChannelSerializedInvoker(HttpChannelState.class.getSimpleName() + "#writeInvoker"); } @Override @@ -1813,6 +1813,11 @@ private void completing() private class HttpChannelSerializedInvoker extends SerializedInvoker { + public HttpChannelSerializedInvoker(String name) + { + super(name); + } + @Override protected void onError(Runnable task, Throwable failure) { diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedExecutor.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedExecutor.java index 5c09087c5226..9377d4cad70b 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedExecutor.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedExecutor.java @@ -27,7 +27,7 @@ */ public class SerializedExecutor implements Executor { - private final SerializedInvoker _invoker = new SerializedInvoker() + private final SerializedInvoker _invoker = new SerializedInvoker(SerializedExecutor.class) { @Override protected void onError(Runnable task, Throwable t) diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index f440ca669e9c..3be05a7a475c 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -35,14 +35,30 @@ public class SerializedInvoker private static final Logger LOG = LoggerFactory.getLogger(SerializedInvoker.class); private final AtomicReference _tail = new AtomicReference<>(); - private int recursionCounter; + private final String _name; + private volatile Thread _invokerThread; + + public SerializedInvoker() + { + this("anonymous"); + } + + public SerializedInvoker(Class clazz) + { + this(clazz.getSimpleName()); + } + + public SerializedInvoker(String name) + { + _name = name; + } /** - * @return whether this invoker is currently executing a task + * @return whether the current thread is currently executing a task using this invoker */ - public boolean isInvoking() + public boolean isCurrentThreadInvoking() { - return recursionCounter > 0; + return _invokerThread == Thread.currentThread(); } /** @@ -147,7 +163,7 @@ public void run(Runnable... tasks) @Override public String toString() { - return String.format("%s@%x{tail=%s}", getClass().getSimpleName(), hashCode(), _tail); + return String.format("%s@%x{name=%s,tail=%s,invoker=%s}", getClass().getSimpleName(), hashCode(), _name, _tail, _invokerThread); } protected void onError(Runnable task, Throwable t) @@ -197,13 +213,13 @@ public void run() LOG.debug("Running link {} of {}", link, SerializedInvoker.this); try { - recursionCounter++; + _invokerThread = Thread.currentThread(); link._task.run(); - recursionCounter--; + _invokerThread = null; } catch (Throwable t) { - recursionCounter--; + _invokerThread = null; if (LOG.isDebugEnabled()) LOG.debug("Failed while running link {} of {}", link, SerializedInvoker.this, t); onError(link._task, t); diff --git a/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java b/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java index 2242683dd174..16a5dff80d23 100644 --- a/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java +++ b/jetty-core/jetty-util/src/test/java/org/eclipse/jetty/util/thread/SerializedInvokerTest.java @@ -14,7 +14,10 @@ package org.eclipse.jetty.util.thread; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,11 +28,19 @@ public class SerializedInvokerTest { private SerializedInvoker _serializedInvoker; + private ExecutorService _executor; @BeforeEach public void beforeEach() { - _serializedInvoker = new SerializedInvoker(); + _serializedInvoker = new SerializedInvoker(SerializedInvokerTest.class); + _executor = Executors.newSingleThreadExecutor(); + } + + @AfterEach + public void afterEach() + { + _executor.shutdownNow(); } @Test @@ -46,20 +57,20 @@ public void testSimple() throws Exception assertFalse(task1.hasRun()); assertFalse(task2.hasRun()); assertFalse(task3.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); todo.run(); assertTrue(task1.hasRun()); assertTrue(task2.hasRun()); assertTrue(task3.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); Task task4 = new Task(); todo = _serializedInvoker.offer(task4); todo.run(); assertTrue(task4.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); } @Test @@ -74,20 +85,20 @@ public void testMulti() assertFalse(task1.hasRun()); assertFalse(task2.hasRun()); assertFalse(task3.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); todo.run(); assertTrue(task1.hasRun()); assertTrue(task2.hasRun()); assertTrue(task3.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); Task task4 = new Task(); todo = _serializedInvoker.offer(task4); todo.run(); assertTrue(task4.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); } @Test @@ -118,25 +129,25 @@ public void run() assertFalse(task1.hasRun()); assertFalse(task2.hasRun()); assertFalse(task3.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); todo.run(); assertTrue(task1.hasRun()); assertTrue(task2.hasRun()); assertTrue(task3.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); Task task4 = new Task(); todo = _serializedInvoker.offer(task4); todo.run(); assertTrue(task4.hasRun()); - assertFalse(_serializedInvoker.isInvoking()); + assertFalse(_serializedInvoker.isCurrentThreadInvoking()); } public class Task implements Runnable { - CountDownLatch _run = new CountDownLatch(1); + final CountDownLatch _run = new CountDownLatch(1); boolean hasRun() { @@ -146,8 +157,17 @@ boolean hasRun() @Override public void run() { - assertTrue(_serializedInvoker.isInvoking()); - _run.countDown(); + try + { + assertTrue(_serializedInvoker.isCurrentThreadInvoking()); + assertFalse(_executor.submit(() -> _serializedInvoker.isCurrentThreadInvoking()).get()); + + _run.countDown(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } } } } From e7e6a4b2d4e7b8d3e26f38f60a098c921a575d56 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Thu, 8 Aug 2024 14:21:29 +0200 Subject: [PATCH 04/19] fix merge Signed-off-by: Ludovic Orban --- .../java/org/eclipse/jetty/client/transport/HttpReceiver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 99f3fe0a5bac..ba3295ab2ada 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -321,8 +321,6 @@ protected void responseHeaders(HttpExchange exchange) */ protected void responseContentAvailable(HttpExchange exchange) { - if (!invoker.isCurrentThreadInvoking()) - throw new IllegalStateException(); if (LOG.isDebugEnabled()) LOG.debug("Invoking responseContentAvailable on {}", this); @@ -346,6 +344,8 @@ protected void responseContentAvailable(HttpExchange exchange) */ protected void responseContentAvailable() { + if (!invoker.isCurrentThreadInvoking()) + throw new IllegalStateException(); contentSource.onDataAvailable(); } From 5f4ca3ec3e56b5185286aa0517bd175a7405953e Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Thu, 8 Aug 2024 17:30:19 +0200 Subject: [PATCH 05/19] fix test Signed-off-by: Ludovic Orban --- .../jetty/client/transport/HttpReceiver.java | 23 +++++------ .../internal/HttpReceiverOverHTTP.java | 2 +- .../internal/HttpReceiverOverFCGI.java | 2 +- .../jetty/util/thread/SerializedInvoker.java | 39 +++++++++++-------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index ba3295ab2ada..b9fb48aeab8c 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -324,6 +324,14 @@ protected void responseContentAvailable(HttpExchange exchange) if (LOG.isDebugEnabled()) LOG.debug("Invoking responseContentAvailable on {}", this); + // No need to invoke onDataAvailable() if we currently are invoking from a demand callback. + if (invoker.isCurrentThreadInvoking()) + { + if (LOG.isDebugEnabled()) + LOG.debug("Skipping invocation of onDataAvailable on {}", this); + return; + } + invoker.run(() -> { if (LOG.isDebugEnabled()) @@ -332,23 +340,10 @@ protected void responseContentAvailable(HttpExchange exchange) if (exchange.isResponseCompleteOrTerminated()) return; - responseContentAvailable(); + contentSource.onDataAvailable(); }); } - /** - * Method to be invoked when response content is available to be read. - *

- * This method directly invokes the demand callback, assuming the caller - * is already serialized with other events. - */ - protected void responseContentAvailable() - { - if (!invoker.isCurrentThreadInvoking()) - throw new IllegalStateException(); - contentSource.onDataAvailable(); - } - /** * Method to be invoked when the response is successful. *

diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java index de6ff4973a3d..3d3c66ce6321 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java @@ -458,7 +458,7 @@ public boolean content(ByteBuffer buffer) if (LOG.isDebugEnabled()) LOG.debug("Setting action to responseContentAvailable on {}", this); - if (getAndSetAction(this::responseContentAvailable) != null) + if (getAndSetAction(() -> responseContentAvailable(exchange)) != null) throw new IllegalStateException(); if (getHttpConnection().isFillInterested()) throw new IllegalStateException(); diff --git a/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java b/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java index 8c810e215acd..9b3ee61277d9 100644 --- a/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java +++ b/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java @@ -117,7 +117,7 @@ void content(Content.Chunk chunk) // Retain the chunk because it is stored for later reads. chunk.retain(); this.chunk = chunk; - responseContentAvailable(); + responseContentAvailable(exchange); } void end(HttpExchange exchange) diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index 3be05a7a475c..e82572dcb567 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -206,27 +206,32 @@ Link next() @Override public void run() { - Link link = this; - while (link != null) + _invokerThread = Thread.currentThread(); + try { - if (LOG.isDebugEnabled()) - LOG.debug("Running link {} of {}", link, SerializedInvoker.this); - try + Link link = this; + while (link != null) { - _invokerThread = Thread.currentThread(); - link._task.run(); - _invokerThread = null; - } - catch (Throwable t) - { - _invokerThread = null; if (LOG.isDebugEnabled()) - LOG.debug("Failed while running link {} of {}", link, SerializedInvoker.this, t); - onError(link._task, t); + LOG.debug("Running link {} of {}", link, SerializedInvoker.this); + try + { + link._task.run(); + } + catch (Throwable t) + { + if (LOG.isDebugEnabled()) + LOG.debug("Failed while running link {} of {}", link, SerializedInvoker.this, t); + onError(link._task, t); + } + link = link.next(); + if (link == null && LOG.isDebugEnabled()) + LOG.debug("Next link is null, execution is over in {}", SerializedInvoker.this); } - link = link.next(); - if (link == null && LOG.isDebugEnabled()) - LOG.debug("Next link is null, execution is over in {}", SerializedInvoker.this); + } + finally + { + _invokerThread = null; } } From 93f58ccd8f9e70c7d0b4027d8d86074a1f610fd3 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Mon, 12 Aug 2024 14:10:20 +0200 Subject: [PATCH 06/19] responseContentAvailable can be called for a pending demand while invoking Signed-off-by: Ludovic Orban --- .../jetty/client/transport/HttpReceiver.java | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index b9fb48aeab8c..83e45e785034 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -321,18 +321,7 @@ protected void responseHeaders(HttpExchange exchange) */ protected void responseContentAvailable(HttpExchange exchange) { - if (LOG.isDebugEnabled()) - LOG.debug("Invoking responseContentAvailable on {}", this); - - // No need to invoke onDataAvailable() if we currently are invoking from a demand callback. - if (invoker.isCurrentThreadInvoking()) - { - if (LOG.isDebugEnabled()) - LOG.debug("Skipping invocation of onDataAvailable on {}", this); - return; - } - - invoker.run(() -> + Runnable runnable = () -> { if (LOG.isDebugEnabled()) LOG.debug("Executing responseContentAvailable on {}", this); @@ -341,7 +330,15 @@ protected void responseContentAvailable(HttpExchange exchange) return; contentSource.onDataAvailable(); - }); + }; + + if (LOG.isDebugEnabled()) + LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Running" : "Invoking" , this); + + if (invoker.isCurrentThreadInvoking()) + runnable.run(); + else + invoker.run(runnable); } /** @@ -697,7 +694,6 @@ private class ContentSource implements NotifiableContentSource private final AtomicReference demandCallbackRef = new AtomicReference<>(); private final AutoLock lock = new AutoLock(); - private final Runnable processDemand = this::processDemand; private Content.Chunk currentChunk; @Override @@ -752,7 +748,7 @@ public void demand(Runnable demandCallback) throw new IllegalStateException(); // The processDemand method may call HttpReceiver.read(boolean) // so it must be called by the invoker. - invoker.run(processDemand); + invoker.run(this::processDemand); } private void processDemand() From 7766942acf737911c52ec78c867e51a15790ce17 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Mon, 12 Aug 2024 14:46:15 +0200 Subject: [PATCH 07/19] fix checkstyle Signed-off-by: Ludovic Orban --- .../java/org/eclipse/jetty/client/transport/HttpReceiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 83e45e785034..6dda6581f831 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -333,7 +333,7 @@ protected void responseContentAvailable(HttpExchange exchange) }; if (LOG.isDebugEnabled()) - LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Running" : "Invoking" , this); + LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Running" : "Invoking", this); if (invoker.isCurrentThreadInvoking()) runnable.run(); From 3302b39ee541a87c9eeed8e34f77da3b07da8037 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Tue, 13 Aug 2024 18:04:46 +0200 Subject: [PATCH 08/19] always invoke Signed-off-by: Ludovic Orban --- .../jetty/client/transport/HttpReceiver.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 6dda6581f831..fd0cbc11d9ee 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -321,7 +321,10 @@ protected void responseHeaders(HttpExchange exchange) */ protected void responseContentAvailable(HttpExchange exchange) { - Runnable runnable = () -> + if (LOG.isDebugEnabled()) + LOG.debug("Invoking responseContentAvailable on {}", this); + + invoker.run(() -> { if (LOG.isDebugEnabled()) LOG.debug("Executing responseContentAvailable on {}", this); @@ -330,15 +333,7 @@ protected void responseContentAvailable(HttpExchange exchange) return; contentSource.onDataAvailable(); - }; - - if (LOG.isDebugEnabled()) - LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Running" : "Invoking", this); - - if (invoker.isCurrentThreadInvoking()) - runnable.run(); - else - invoker.run(runnable); + }); } /** From 6c470d3863d3ea8058fb49f23bb7a13e3b8c7e61 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Wed, 14 Aug 2024 10:24:05 +0200 Subject: [PATCH 09/19] fix race condition Signed-off-by: Ludovic Orban --- .../jetty/util/thread/SerializedInvoker.java | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index e82572dcb567..07588f63df40 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -206,32 +206,31 @@ Link next() @Override public void run() { - _invokerThread = Thread.currentThread(); - try + Link link = this; + while (link != null) { - Link link = this; - while (link != null) + if (LOG.isDebugEnabled()) + LOG.debug("Running link {} of {}", link, SerializedInvoker.this); + _invokerThread = Thread.currentThread(); + try + { + link._task.run(); + } + catch (Throwable t) { if (LOG.isDebugEnabled()) - LOG.debug("Running link {} of {}", link, SerializedInvoker.this); - try - { - link._task.run(); - } - catch (Throwable t) - { - if (LOG.isDebugEnabled()) - LOG.debug("Failed while running link {} of {}", link, SerializedInvoker.this, t); - onError(link._task, t); - } - link = link.next(); - if (link == null && LOG.isDebugEnabled()) - LOG.debug("Next link is null, execution is over in {}", SerializedInvoker.this); + LOG.debug("Failed while running link {} of {}", link, SerializedInvoker.this, t); + onError(link._task, t); } - } - finally - { - _invokerThread = null; + finally + { + // _invokerThread must be nulled before calling link.next() as + // once the latter has executed, another thread can enter Link.run(). + _invokerThread = null; + } + link = link.next(); + if (link == null && LOG.isDebugEnabled()) + LOG.debug("Next link is null, execution is over in {}", SerializedInvoker.this); } } From 48c9d14bfc2cab494610ac3c10dd7c676ae2fd41 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Wed, 14 Aug 2024 14:34:06 +0200 Subject: [PATCH 10/19] improve serviceability by recording stack traces and adding the ability to dump them Signed-off-by: Ludovic Orban --- .../jetty/util/thread/SerializedInvoker.java | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index 07588f63df40..ec35514d770c 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -13,8 +13,12 @@ package org.eclipse.jetty.util.thread; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jetty.util.component.Dumpable; import org.eclipse.jetty.util.thread.Invocable.InvocationType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,7 +88,7 @@ public Runnable offer(Runnable task) { // Wrap the given task with another one that's going to delegate run() to the wrapped task while the // wrapper's toString() returns a description of the place in code where SerializedInvoker.run() was called. - task = new NamedRunnable(task, deriveTaskName(task)); + task = new NamedRunnable(task); } } Link link = new Link(task); @@ -97,17 +101,6 @@ public Runnable offer(Runnable task) return null; } - protected String deriveTaskName(Runnable task) - { - StackTraceElement[] stackTrace = new Exception().getStackTrace(); - for (StackTraceElement stackTraceElement : stackTrace) - { - String className = stackTraceElement.getClassName(); - if (!className.equals(SerializedInvoker.class.getName()) && !className.equals(getClass().getName())) - return "Queued at " + stackTraceElement; - } - return task.toString(); - } /** * Arrange for tasks to be invoked, mutually excluded from other tasks. @@ -171,7 +164,7 @@ protected void onError(Runnable task, Throwable t) LOG.warn("Serialized invocation error", t); } - private class Link implements Runnable, Invocable + private class Link implements Runnable, Invocable, Dumpable { private final Runnable _task; private final AtomicReference _next = new AtomicReference<>(); @@ -181,6 +174,24 @@ public Link(Runnable task) _task = task; } + @Override + public void dump(Appendable out, String indent) throws IOException + { + if (_task instanceof NamedRunnable nr) + { + StringWriter sw = new StringWriter(); + nr.stack.printStackTrace(new PrintWriter(sw)); + Dumpable.dumpObjects(out, indent, nr.toString(), sw.toString()); + } + else + { + Dumpable.dumpObjects(out, indent, _task); + } + Link link = _next.get(); + if (link != null) + link.dump(out, indent); + } + @Override public InvocationType getInvocationType() { @@ -241,10 +252,36 @@ public String toString() } } - private record NamedRunnable(Runnable delegate, String name) implements Runnable + private class NamedRunnable implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(NamedRunnable.class); + private final Runnable delegate; + private final String name; + private final Throwable stack; + + public NamedRunnable(Runnable delegate) + { + this.delegate = delegate; + this.stack = new Throwable(); + this.name = deriveTaskName(delegate, stack); + } + + protected String deriveTaskName(Runnable task, Throwable stack) + { + StackTraceElement[] stackTrace = stack.getStackTrace(); + for (StackTraceElement stackTraceElement : stackTrace) + { + String className = stackTraceElement.getClassName(); + if (!className.equals(SerializedInvoker.class.getName()) && + !className.equals(SerializedInvoker.this.getClass().getName()) && + !className.equals(getClass().getName())) + return "Queued by " + Thread.currentThread().getName() + " at " + stackTraceElement; + } + return task.toString(); + } + + @Override public void run() { From 31b2b9be6986d8d5436175f6bfdc254f1acb1752 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Wed, 14 Aug 2024 14:35:42 +0200 Subject: [PATCH 11/19] explicitly tell the receiver when to use the invoker Signed-off-by: Ludovic Orban --- .../jetty/client/transport/HttpReceiver.java | 29 ++++++++++++++----- .../internal/HttpReceiverOverHTTP.java | 4 +-- .../internal/HttpReceiverOverFCGI.java | 4 +-- .../internal/HttpReceiverOverHTTP2.java | 2 +- .../internal/HttpReceiverOverHTTP3.java | 2 +- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index fd0cbc11d9ee..8753fe6dc75f 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -317,14 +317,12 @@ protected void responseHeaders(HttpExchange exchange) * This method takes care of ensuring the {@link Content.Source} passed to * {@link Response.ContentSourceListener#onContentSource(Response, Content.Source)} * calls the demand callback. - * The call to the demand callback is serialized with other events. + * @param exchange the HTTP exchange + * @param invoke when true, the call to the demand callback is serialized with other events. */ - protected void responseContentAvailable(HttpExchange exchange) + protected void responseContentAvailable(HttpExchange exchange, boolean invoke) { - if (LOG.isDebugEnabled()) - LOG.debug("Invoking responseContentAvailable on {}", this); - - invoker.run(() -> + Runnable runnable = () -> { if (LOG.isDebugEnabled()) LOG.debug("Executing responseContentAvailable on {}", this); @@ -333,7 +331,22 @@ protected void responseContentAvailable(HttpExchange exchange) return; contentSource.onDataAvailable(); - }); + }; + + if (invoke) + { + if (LOG.isDebugEnabled()) + LOG.debug("Invoking responseContentAvailable on {}", this); + invoker.run(runnable); + } + else + { + if (!invoker.isCurrentThreadInvoking()) + throw new IllegalStateException(); + if (LOG.isDebugEnabled()) + LOG.debug("Calling responseContentAvailable on {}", this); + runnable.run(); + } } /** @@ -727,6 +740,8 @@ public void onDataAvailable() { if (LOG.isDebugEnabled()) LOG.debug("onDataAvailable on {}", this); + if (!invoker.isCurrentThreadInvoking()) + throw new IllegalStateException(); // The onDataAvailable() method is only ever called // by the invoker so avoid using the invoker again. invokeDemandCallback(false); diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java index 3d3c66ce6321..8b9a52a7bb18 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java @@ -81,7 +81,7 @@ void receive() { HttpExchange exchange = getHttpExchange(); if (exchange != null) - responseContentAvailable(exchange); + responseContentAvailable(exchange, true); } } @@ -458,7 +458,7 @@ public boolean content(ByteBuffer buffer) if (LOG.isDebugEnabled()) LOG.debug("Setting action to responseContentAvailable on {}", this); - if (getAndSetAction(() -> responseContentAvailable(exchange)) != null) + if (getAndSetAction(() -> responseContentAvailable(exchange, false)) != null) throw new IllegalStateException(); if (getHttpConnection().isFillInterested()) throw new IllegalStateException(); diff --git a/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java b/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java index 9b3ee61277d9..af41b59e2296 100644 --- a/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java +++ b/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java @@ -42,7 +42,7 @@ void receive() { HttpExchange exchange = getHttpExchange(); if (exchange != null) - responseContentAvailable(exchange); + responseContentAvailable(exchange, true); } } @@ -117,7 +117,7 @@ void content(Content.Chunk chunk) // Retain the chunk because it is stored for later reads. chunk.retain(); this.chunk = chunk; - responseContentAvailable(exchange); + responseContentAvailable(exchange, false); } void end(HttpExchange exchange) diff --git a/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java b/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java index bddb5d7396f0..16ba2bc8b2ca 100644 --- a/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java +++ b/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java @@ -211,7 +211,7 @@ public Runnable onDataAvailable() HttpExchange exchange = getHttpExchange(); if (exchange == null) return null; - return new Invocable.ReadyTask(Invocable.InvocationType.NON_BLOCKING, () -> responseContentAvailable(exchange)); + return new Invocable.ReadyTask(Invocable.InvocationType.NON_BLOCKING, () -> responseContentAvailable(exchange, true)); } @Override diff --git a/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java b/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java index 660fad9b9bee..8efb9f7d0606 100644 --- a/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java +++ b/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java @@ -127,7 +127,7 @@ public void onDataAvailable(Stream.Client stream) if (exchange == null) return; - responseContentAvailable(exchange); + responseContentAvailable(exchange, true); } @Override From 27704f24e3239502ccff1f3ccb13516a7b926297 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Wed, 14 Aug 2024 14:53:29 +0200 Subject: [PATCH 12/19] fix checkstyle Signed-off-by: Ludovic Orban --- .../java/org/eclipse/jetty/util/thread/SerializedInvoker.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index ec35514d770c..46c56e11b99e 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -101,7 +101,6 @@ public Runnable offer(Runnable task) return null; } - /** * Arrange for tasks to be invoked, mutually excluded from other tasks. * @param tasks The tasks to invoke @@ -281,7 +280,6 @@ protected String deriveTaskName(Runnable task, Throwable stack) return task.toString(); } - @Override public void run() { From 3a880683921cc6653bf4602512e813aa8c7ae5c7 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Wed, 14 Aug 2024 14:34:06 +0200 Subject: [PATCH 13/19] improve serviceability by recording stack traces and adding the ability to dump them Signed-off-by: Ludovic Orban --- .../jetty/util/thread/SerializedInvoker.java | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index 07588f63df40..46c56e11b99e 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -13,8 +13,12 @@ package org.eclipse.jetty.util.thread; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jetty.util.component.Dumpable; import org.eclipse.jetty.util.thread.Invocable.InvocationType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,7 +88,7 @@ public Runnable offer(Runnable task) { // Wrap the given task with another one that's going to delegate run() to the wrapped task while the // wrapper's toString() returns a description of the place in code where SerializedInvoker.run() was called. - task = new NamedRunnable(task, deriveTaskName(task)); + task = new NamedRunnable(task); } } Link link = new Link(task); @@ -97,18 +101,6 @@ public Runnable offer(Runnable task) return null; } - protected String deriveTaskName(Runnable task) - { - StackTraceElement[] stackTrace = new Exception().getStackTrace(); - for (StackTraceElement stackTraceElement : stackTrace) - { - String className = stackTraceElement.getClassName(); - if (!className.equals(SerializedInvoker.class.getName()) && !className.equals(getClass().getName())) - return "Queued at " + stackTraceElement; - } - return task.toString(); - } - /** * Arrange for tasks to be invoked, mutually excluded from other tasks. * @param tasks The tasks to invoke @@ -171,7 +163,7 @@ protected void onError(Runnable task, Throwable t) LOG.warn("Serialized invocation error", t); } - private class Link implements Runnable, Invocable + private class Link implements Runnable, Invocable, Dumpable { private final Runnable _task; private final AtomicReference _next = new AtomicReference<>(); @@ -181,6 +173,24 @@ public Link(Runnable task) _task = task; } + @Override + public void dump(Appendable out, String indent) throws IOException + { + if (_task instanceof NamedRunnable nr) + { + StringWriter sw = new StringWriter(); + nr.stack.printStackTrace(new PrintWriter(sw)); + Dumpable.dumpObjects(out, indent, nr.toString(), sw.toString()); + } + else + { + Dumpable.dumpObjects(out, indent, _task); + } + Link link = _next.get(); + if (link != null) + link.dump(out, indent); + } + @Override public InvocationType getInvocationType() { @@ -241,10 +251,35 @@ public String toString() } } - private record NamedRunnable(Runnable delegate, String name) implements Runnable + private class NamedRunnable implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(NamedRunnable.class); + private final Runnable delegate; + private final String name; + private final Throwable stack; + + public NamedRunnable(Runnable delegate) + { + this.delegate = delegate; + this.stack = new Throwable(); + this.name = deriveTaskName(delegate, stack); + } + + protected String deriveTaskName(Runnable task, Throwable stack) + { + StackTraceElement[] stackTrace = stack.getStackTrace(); + for (StackTraceElement stackTraceElement : stackTrace) + { + String className = stackTraceElement.getClassName(); + if (!className.equals(SerializedInvoker.class.getName()) && + !className.equals(SerializedInvoker.this.getClass().getName()) && + !className.equals(getClass().getName())) + return "Queued by " + Thread.currentThread().getName() + " at " + stackTraceElement; + } + return task.toString(); + } + @Override public void run() { From e3b4581562567f488e2688409345602068f203c0 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Thu, 15 Aug 2024 13:26:06 +0200 Subject: [PATCH 14/19] document why we need to either invoke or call Signed-off-by: Ludovic Orban --- .../jetty/client/transport/HttpReceiver.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index fd0cbc11d9ee..342c5efb8c10 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -321,10 +321,7 @@ protected void responseHeaders(HttpExchange exchange) */ protected void responseContentAvailable(HttpExchange exchange) { - if (LOG.isDebugEnabled()) - LOG.debug("Invoking responseContentAvailable on {}", this); - - invoker.run(() -> + Runnable runnable = () -> { if (LOG.isDebugEnabled()) LOG.debug("Executing responseContentAvailable on {}", this); @@ -333,7 +330,29 @@ protected void responseContentAvailable(HttpExchange exchange) return; contentSource.onDataAvailable(); - }); + }; + + // Given: + // - This method must always call onDataAvailable() as the ContentSource is the only source of truth knowing if there is a + // pending demand or not, and that must always be done by the invoker. + // - This method is called sometimes from the context of the invoker (e.g.: read() from within a demand callback) and + // sometimes not (e.g.: read() from a naked thread). + // Then: + // If the consuming loop is a read-demand one (i.e.: demand is used to loop) then read() enqueues a onDataAvailable() invocation + // then demand() enqueues a processDemand() invocation before control is returned to the invoker. + // onDataAvailable() runs the demand callback that enqueues another onDataAvailable() invocation because of read() then another + // processDemand() invocation because of demand(). + // There are then two processDemand() invocations enqueued, and since they call HttpReceiver.read(true) when no content is readily available, + // they can both try to register fill interest and eventually throw a ReadPendingException. + // So onDataAvailable() must always be executed from the invoker but also immediately. + + if (LOG.isDebugEnabled()) + LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Invoking" : "Calling", this); + + if (invoker.isCurrentThreadInvoking()) + runnable.run(); + else + invoker.run(runnable); } /** From 9e055df8f046094c16214b35d781ec0a16c837a6 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Thu, 15 Aug 2024 13:35:47 +0200 Subject: [PATCH 15/19] fix bad merge Signed-off-by: Ludovic Orban --- .../jetty/client/transport/internal/HttpReceiverOverHTTP.java | 4 ++-- .../fcgi/client/transport/internal/HttpReceiverOverFCGI.java | 4 ++-- .../client/transport/internal/HttpReceiverOverHTTP2.java | 2 +- .../client/transport/internal/HttpReceiverOverHTTP3.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java index 8b9a52a7bb18..3d3c66ce6321 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpReceiverOverHTTP.java @@ -81,7 +81,7 @@ void receive() { HttpExchange exchange = getHttpExchange(); if (exchange != null) - responseContentAvailable(exchange, true); + responseContentAvailable(exchange); } } @@ -458,7 +458,7 @@ public boolean content(ByteBuffer buffer) if (LOG.isDebugEnabled()) LOG.debug("Setting action to responseContentAvailable on {}", this); - if (getAndSetAction(() -> responseContentAvailable(exchange, false)) != null) + if (getAndSetAction(() -> responseContentAvailable(exchange)) != null) throw new IllegalStateException(); if (getHttpConnection().isFillInterested()) throw new IllegalStateException(); diff --git a/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java b/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java index af41b59e2296..9b3ee61277d9 100644 --- a/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java +++ b/jetty-core/jetty-fcgi/jetty-fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/transport/internal/HttpReceiverOverFCGI.java @@ -42,7 +42,7 @@ void receive() { HttpExchange exchange = getHttpExchange(); if (exchange != null) - responseContentAvailable(exchange, true); + responseContentAvailable(exchange); } } @@ -117,7 +117,7 @@ void content(Content.Chunk chunk) // Retain the chunk because it is stored for later reads. chunk.retain(); this.chunk = chunk; - responseContentAvailable(exchange, false); + responseContentAvailable(exchange); } void end(HttpExchange exchange) diff --git a/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java b/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java index 16ba2bc8b2ca..bddb5d7396f0 100644 --- a/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java +++ b/jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/internal/HttpReceiverOverHTTP2.java @@ -211,7 +211,7 @@ public Runnable onDataAvailable() HttpExchange exchange = getHttpExchange(); if (exchange == null) return null; - return new Invocable.ReadyTask(Invocable.InvocationType.NON_BLOCKING, () -> responseContentAvailable(exchange, true)); + return new Invocable.ReadyTask(Invocable.InvocationType.NON_BLOCKING, () -> responseContentAvailable(exchange)); } @Override diff --git a/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java b/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java index 8efb9f7d0606..660fad9b9bee 100644 --- a/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java +++ b/jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/internal/HttpReceiverOverHTTP3.java @@ -127,7 +127,7 @@ public void onDataAvailable(Stream.Client stream) if (exchange == null) return; - responseContentAvailable(exchange, true); + responseContentAvailable(exchange); } @Override From 63d31266f603afc3fe65128e2bb4f6d3394118b8 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Fri, 16 Aug 2024 12:19:55 +0200 Subject: [PATCH 16/19] see what tests fail when not running Signed-off-by: Ludovic Orban --- .../java/org/eclipse/jetty/client/transport/HttpReceiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 577b21ee7f59..0142f1d636dd 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -350,7 +350,7 @@ protected void responseContentAvailable(HttpExchange exchange) LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Invoking" : "Calling", this); if (invoker.isCurrentThreadInvoking()) - runnable.run(); + return; else invoker.run(runnable); } From 7807703128c348024d68eba99ef09fb2ec4591c2 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Fri, 16 Aug 2024 12:20:09 +0200 Subject: [PATCH 17/19] revert change Signed-off-by: Ludovic Orban --- .../java/org/eclipse/jetty/client/transport/HttpReceiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 0142f1d636dd..577b21ee7f59 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -350,7 +350,7 @@ protected void responseContentAvailable(HttpExchange exchange) LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Invoking" : "Calling", this); if (invoker.isCurrentThreadInvoking()) - return; + runnable.run(); else invoker.run(runnable); } From 6e9d18402cd1a285282a2473f0f026870d74e183 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Fri, 16 Aug 2024 12:56:18 +0200 Subject: [PATCH 18/19] add comments Signed-off-by: Ludovic Orban --- .../org/eclipse/jetty/client/transport/HttpReceiver.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java index 577b21ee7f59..2eb2d28c6731 100644 --- a/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java +++ b/jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpReceiver.java @@ -334,6 +334,11 @@ protected void responseContentAvailable(HttpExchange exchange) // There are then two processDemand() invocations enqueued, and since they call HttpReceiver.read(true) when no content is readily available, // they can both try to register fill interest and eventually throw a ReadPendingException. // So onDataAvailable() must always be executed from the invoker but also immediately. + // The root cause is that for H1/FCGI the parser calls this method not knowing if it is reacting to a read() or a demand(). + // Make sure these 3 tests pass when modifying this method: + // - ForwardProxyWithDynamicTransportTest.testProxyConcurrentLoad() + // - ConnectionPoolTest.testConnectionPoolFactory() + // - HttpClientTest.testContentSourceListenerDemandInSpawnedThread() Runnable runnable = () -> { @@ -350,7 +355,7 @@ protected void responseContentAvailable(HttpExchange exchange) LOG.debug("{} responseContentAvailable on {}", invoker.isCurrentThreadInvoking() ? "Invoking" : "Calling", this); if (invoker.isCurrentThreadInvoking()) - runnable.run(); + runnable.run(); // This is needed by H2, but it could be just return for h1/fcgi. ForwardProxyWithDynamicTransportTest.testProxyConcurrentLoad fails when we do not run the runnable here. else invoker.run(runnable); } From 32000ac590d37b72780a10789ce131e58e791db3 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Wed, 28 Aug 2024 10:41:27 +0200 Subject: [PATCH 19/19] add javadoc + rename ctor variable Signed-off-by: Ludovic Orban --- .../jetty/util/thread/SerializedInvoker.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java index 46c56e11b99e..a429a201c849 100644 --- a/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java +++ b/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/SerializedInvoker.java @@ -42,16 +42,27 @@ public class SerializedInvoker private final String _name; private volatile Thread _invokerThread; + /** + * Create a new instance whose name is {@code anonymous}. + */ public SerializedInvoker() { this("anonymous"); } - public SerializedInvoker(Class clazz) + /** + * Create a new instance whose name is derived from the given class. + * @param nameFrom the class to use as a name. + */ + public SerializedInvoker(Class nameFrom) { - this(clazz.getSimpleName()); + this(nameFrom.getSimpleName()); } + /** + * Create a new instance with the given name. + * @param name the name. + */ public SerializedInvoker(String name) { _name = name;