Immediate remote cancels

examples/debugging/multi_subactor_root_errors.py

@@ -1,3 +1,8 @@
+'''
+Test that a nested nursery will avoid clobbering
+the debugger latched by a broken child.
+
+'''
 import trio
 import tractor
 
@@ -35,6 +40,7 @@ async def main():
     """
     async with tractor.open_nursery(
         debug_mode=True,
+        # loglevel='cancel',
     ) as n:
 
         # spawn both actors

newsfragments/245.feature.rst

@@ -0,0 +1,13 @@
+Change the core message loop to handle task and actor-runtime cancel
+requests immediately instead of scheduling them as is done for rpc-task
+requests.
+
+In order to obtain more reliable teardown mechanics for (complex) actor
+trees it's important that we specially treat cancel requests as having
+higher priority. Previously, it was possible that task cancel requests
+could actually also themselves be cancelled if a "actor-runtime" cancel
+request was received (can happen during messy multi actor crashes that
+propagate). Instead cancels now block the msg loop until serviced and
+a response is relayed back to the requester. This also allows for
+improved debugger support since we have determinism guarantees about
+which processes must wait before hard killing their children.

newsfragments/HOWTO.rst

@@ -4,5 +4,5 @@ now and use the default `fragment set`_.
 
 
 .. _towncrier docs: https://github.com/twisted/towncrier#quick-start
-.. _pluggy release readme: https://github.com/twisted/towncrier#quick-start
+.. _pluggy release readme: https://github.com/pytest-dev/pluggy/blob/main/changelog/README.rst
 .. _fragment set: https://github.com/twisted/towncrier#news-fragments

tests/test_cancellation.py

@@ -1,5 +1,6 @@
 """
 Cancellation and error propagation
+
 """
 import os
 import signal
@@ -365,14 +366,26 @@ async def test_nested_multierrors(loglevel, start_method):
                     # to happen before an actor is spawned
                     if isinstance(subexc, trio.Cancelled):
                         continue
-                    else:
+
+                    elif isinstance(subexc, tractor.RemoteActorError):
                         # on windows it seems we can't exactly be sure wtf
                         # will happen..
                         assert subexc.type in (
                             tractor.RemoteActorError,
                             trio.Cancelled,
                             trio.MultiError
                         )
+
+                    elif isinstance(subexc, trio.MultiError):
+                        for subsub in subexc.exceptions:
+
+                            if subsub in (tractor.RemoteActorError,):
+                                subsub = subsub.type
+
+                            assert type(subsub) in (
+                                trio.Cancelled,
+                                trio.MultiError,
+                            )
                 else:
                     assert isinstance(subexc, tractor.RemoteActorError)
 
@@ -381,13 +394,14 @@ async def test_nested_multierrors(loglevel, start_method):
                     # on windows sometimes spawning is just too slow and
                     # we get back the (sent) cancel signal instead
                     if platform.system() == 'Windows':
-                        assert (subexc.type is trio.MultiError) or (
-                            subexc.type is tractor.RemoteActorError)
+                        if isinstance(subexc, tractor.RemoteActorError):
+                            assert subexc.type in (trio.MultiError, tractor.RemoteActorError)
+                        else:
+                            assert isinstance(subexc, trio.MultiError)
                     else:
                         assert subexc.type is trio.MultiError
                 else:
-                    assert (subexc.type is tractor.RemoteActorError) or (
-                        subexc.type is trio.Cancelled)
+                    assert subexc.type in (tractor.RemoteActorError, trio.Cancelled)
 
 
 @no_windows
@@ -448,6 +462,7 @@ async def main():
     with pytest.raises(KeyboardInterrupt):
         trio.run(main)
 
+
 async def spin_for(period=3):
     "Sync sleep."
     time.sleep(period)

tests/test_debugger.py

@@ -236,7 +236,8 @@ def test_subactor_breakpoint(spawn):
 
 
 def test_multi_subactors(spawn):
-    """Multiple subactors, both erroring and breakpointing as well as
+    """
+    Multiple subactors, both erroring and breakpointing as well as
     a nested subactor erroring.
     """
     child = spawn(r'multi_subactors')
@@ -259,6 +260,7 @@ def test_multi_subactors(spawn):
     # first name_error failure
     child.expect(r"\(Pdb\+\+\)")
     before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('name_error'" in before
     assert "NameError" in before
 
     # continue again
@@ -267,6 +269,7 @@ def test_multi_subactors(spawn):
     # 2nd name_error failure
     child.expect(r"\(Pdb\+\+\)")
     before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('name_error_1'" in before
     assert "NameError" in before
 
     # breakpoint loop should re-engage
@@ -275,6 +278,19 @@ def test_multi_subactors(spawn):
     before = str(child.before.decode())
     assert "Attaching pdb to actor: ('breakpoint_forever'" in before
 
+    # wait for spawn error to show up
+    while 'breakpoint_forever' in before:
+        child.sendline('c')
+        child.expect(r"\(Pdb\+\+\)")
+        before = str(child.before.decode())
+
+    # 2nd depth nursery should trigger
+    # child.sendline('c')
+    # child.expect(r"\(Pdb\+\+\)")
+    # before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('spawn_error'" in before
+    assert "RemoteActorError: ('name_error_1'" in before
+
     # now run some "continues" to show re-entries
     for _ in range(5):
         child.sendline('c')
@@ -284,16 +300,24 @@ def test_multi_subactors(spawn):
     child.sendline('q')
     child.expect(r"\(Pdb\+\+\)")
     before = str(child.before.decode())
+    # debugger attaches to root
     assert "Attaching to pdb in crashed actor: ('root'" in before
+    # expect a multierror with exceptions for each sub-actor
     assert "RemoteActorError: ('breakpoint_forever'" in before
+    assert "RemoteActorError: ('name_error'" in before
+    assert "RemoteActorError: ('spawn_error'" in before
+    assert "RemoteActorError: ('name_error_1'" in before
     assert 'bdb.BdbQuit' in before
 
     # process should exit
     child.sendline('c')
     child.expect(pexpect.EOF)
-
+    # repeat of previous multierror for final output
     before = str(child.before.decode())
     assert "RemoteActorError: ('breakpoint_forever'" in before
+    assert "RemoteActorError: ('name_error'" in before
+    assert "RemoteActorError: ('spawn_error'" in before
+    assert "RemoteActorError: ('name_error_1'" in before
     assert 'bdb.BdbQuit' in before
 
 
@@ -387,16 +411,29 @@ def test_multi_subactors_root_errors(spawn):
     before = str(child.before.decode())
     assert "NameError: name 'doggypants' is not defined" in before
 
-    # continue again
+    # continue again to catch 2nd name error from
+    # actor 'name_error_1' (which is 2nd depth).
     child.sendline('c')
     child.expect(r"\(Pdb\+\+\)")
+    before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('name_error_1'" in before
+    assert "NameError" in before
 
-    # should now get attached in root with assert error
+    child.sendline('c')
+    child.expect(r"\(Pdb\+\+\)")
     before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('spawn_error'" in before
+    # boxed error from previous step
+    assert "RemoteActorError: ('name_error_1'" in before
+    assert "NameError" in before
 
-    # should have come just after priot prompt
+    child.sendline('c')
+    child.expect(r"\(Pdb\+\+\)")
+    before = str(child.before.decode())
     assert "Attaching to pdb in crashed actor: ('root'" in before
-    assert "AssertionError" in before
+    # boxed error from first level failure
+    assert "RemoteActorError: ('name_error'" in before
+    assert "NameError" in before
 
     # warnings assert we probably don't need
     # assert "Cancelling nursery in ('spawn_error'," in before
@@ -406,6 +443,7 @@ def test_multi_subactors_root_errors(spawn):
     child.expect(pexpect.EOF)
 
     before = str(child.before.decode())
+    # error from root actor and root task that created top level nursery
     assert "AssertionError" in before
 
 

tests/test_pubsub.py

@@ -180,6 +180,7 @@ async def main():
                     'streamer',
                     enable_modules=[__name__],
                 )
+                name = 'streamer'
 
             even_portal = await n.run_in_actor(
                 subs,