From d7b50a97344fbb2afbafda76e2624074e56e5548 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 12:05:59 -0600 Subject: [PATCH 01/36] Handle warnings in tests --- jupyter_client/client.py | 9 +++++++ jupyter_client/kernelspec.py | 2 ++ jupyter_client/manager.py | 11 +++++--- .../provisioning/local_provisioner.py | 5 ++++ jupyter_client/tests/conftest.py | 25 ++++++++++++++++--- jupyter_client/tests/test_client.py | 8 +++--- jupyter_client/tests/test_kernelmanager.py | 4 +++ jupyter_client/threaded.py | 2 +- jupyter_client/utils.py | 1 + pyproject.toml | 12 +++++++++ 10 files changed, 68 insertions(+), 11 deletions(-) diff --git a/jupyter_client/client.py b/jupyter_client/client.py index d71eac5ab..8727f406f 100644 --- a/jupyter_client/client.py +++ b/jupyter_client/client.py @@ -11,6 +11,7 @@ import zmq.asyncio from traitlets import Any # type: ignore +from traitlets import Bool from traitlets import Instance from traitlets import Type @@ -92,7 +93,10 @@ class KernelClient(ConnectionFileMixin): # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.asyncio.Context) + _created_context: Bool = Bool(False) + def _context_default(self) -> zmq.asyncio.Context: + self._created_context = True return zmq.asyncio.Context() # The classes to use for the various channels @@ -282,6 +286,9 @@ def start_channels( :meth:`start_kernel`. If the channels have been stopped and you call this, :class:`RuntimeError` will be raised. """ + # Re-create the context if needed. + if self._created_context: + self.context = self._context_default() if iopub: self.iopub_channel.start() if shell: @@ -311,6 +318,8 @@ def stop_channels(self) -> None: self.hb_channel.stop() if self.control_channel.is_alive(): self.control_channel.stop() + if self._created_context: + self.context.destroy() @property def channels_running(self) -> bool: diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 9252c8243..8d83842ea 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -31,6 +31,8 @@ class KernelSpec(HasTraits): argv = List() + name = Unicode() + mimetype = Unicode() display_name = Unicode() language = Unicode() env = Dict() diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index bc3190f25..39d27454d 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -387,8 +387,9 @@ async def _async_request_shutdown(self, restart: bool = False) -> None: content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) # ensure control socket is connected - self._connect_control_socket() - self.session.send(self._control_socket, msg) + if self._control_socket and not self._control_socket.closed: + self._connect_control_socket() + self.session.send(self._control_socket, msg) assert self.provisioner is not None await self.provisioner.shutdown_requested(restart=restart) self._shutdown_status = _ShutdownStatus.ShutdownRequest @@ -453,7 +454,6 @@ async def _async_cleanup_resources(self, restart: bool = False) -> None: cleanup_resources = run_sync(_async_cleanup_resources) - @in_pending_state async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False): """Attempts to stop the kernel process cleanly. @@ -476,7 +476,10 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) # Stop monitoring for restarting while we shutdown. self.stop_restarter() - await ensure_async(self.interrupt_kernel()) + # If the kernel has already started, interrupt it to give it a + # chance to clean up. + if self.has_kernel: + await ensure_async(self.interrupt_kernel()) if now: await ensure_async(self._kill_kernel()) diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 043068680..08cb9aa14 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -60,6 +60,11 @@ async def wait(self) -> Optional[int]: # Process is no longer alive, wait and clear ret = self.process.wait() + # Make sure all the fds get closed. + for attr in ['stdout', 'stderr', 'stdin']: + fid = getattr(self.process, attr) + if fid: + fid.close() self.process = None # allow has_process to now return False return ret diff --git a/jupyter_client/tests/conftest.py b/jupyter_client/tests/conftest.py index 8f9ad7378..69376d415 100644 --- a/jupyter_client/tests/conftest.py +++ b/jupyter_client/tests/conftest.py @@ -7,14 +7,32 @@ from .utils import test_env +try: + import resource +except ImportError: + # Windows + resource = None + pjoin = os.path.join -if os.name == "nt" and sys.version_info >= (3, 7): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +# Handle resource limit +if resource is not None: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + + DEFAULT_SOFT = 4096 + if hard >= DEFAULT_SOFT: + soft = DEFAULT_SOFT + old_soft, old_hard = resource.getrlimit(resource.RLIMIT_NOFILE) + hard = old_hard + if old_soft < soft: + if hard < soft: + hard = soft + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) -@pytest.fixture + +@pytest.fixture(autouse=True) def event_loop(): # Make sure we test against a selector event loop # since pyzmq doesn't like the proactor loop. @@ -22,6 +40,7 @@ def event_loop(): if os.name == "nt" and sys.version_info >= (3, 7): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) loop = asyncio.SelectorEventLoop() + asyncio.set_event_loop(loop) try: yield loop finally: diff --git a/jupyter_client/tests/test_client.py b/jupyter_client/tests/test_client.py index a422462b3..75134cf57 100644 --- a/jupyter_client/tests/test_client.py +++ b/jupyter_client/tests/test_client.py @@ -22,14 +22,16 @@ class TestKernelClient(TestCase): def setUp(self): self.env_patch = test_env() self.env_patch.start() - self.addCleanup(self.env_patch.stop) try: KernelSpecManager().get_kernel_spec(NATIVE_KERNEL_NAME) except NoSuchKernel: pytest.skip() self.km, self.kc = start_new_kernel(kernel_name=NATIVE_KERNEL_NAME) - self.addCleanup(self.kc.stop_channels) - self.addCleanup(self.km.shutdown_kernel) + + def tearDown(self): + self.env_patch.stop() + self.km.shutdown_kernel() + self.kc.stop_channels() def test_execute_interactive(self): kc = self.kc diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index 983ca8095..f53a3ae27 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -441,6 +441,8 @@ def execute(cmd): km.shutdown_kernel() assert km.context.closed + kc.stop_channels() + @pytest.mark.asyncio class TestAsyncKernelManager: @@ -525,6 +527,8 @@ async def execute(cmd): # verify that subprocesses were interrupted assert reply["user_expressions"]["poll"] == [-signal.SIGINT] * N + await km.shutdown_kernel() + @pytest.mark.timeout(10) async def test_start_new_async_kernel(self, install_kernel, start_async_kernel): km, kc = start_async_kernel diff --git a/jupyter_client/threaded.py b/jupyter_client/threaded.py index 54e44a3d7..9a4bf4c58 100644 --- a/jupyter_client/threaded.py +++ b/jupyter_client/threaded.py @@ -47,7 +47,7 @@ def __init__( self, socket: Optional[zmq.Socket], session: Optional[Session], - loop: Optional[zmq.eventloop.ioloop.ZMQIOLoop], + loop: Optional[Any], ) -> None: """Create a channel. diff --git a/jupyter_client/utils.py b/jupyter_client/utils.py index f2f3c4dc4..c2d6d5070 100644 --- a/jupyter_client/utils.py +++ b/jupyter_client/utils.py @@ -18,6 +18,7 @@ def wrapped(*args, **kwargs): import nest_asyncio # type: ignore nest_asyncio.apply(loop) + future = asyncio.ensure_future(coro(*args, **kwargs)) try: return loop.run_until_complete(future) diff --git a/pyproject.toml b/pyproject.toml index 2336b4ddd..f576bfad1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,3 +27,15 @@ tag_template = "v{new_version}" [[tool.tbump.file]] src = "jupyter_client/_version.py" + +[tool.pytest.ini_options] +norecursedirs = "dist build" +addopts= "-r sxX" +filterwarnings= [ + # Fail on warnings + "error", + + # Workarounds for https://github.com/pytest-dev/pytest-asyncio/issues/77 + "ignore:unclosed Date: Sun, 6 Mar 2022 18:37:58 -0600 Subject: [PATCH 02/36] set strict asyncio mode --- jupyter_client/tests/conftest.py | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyter_client/tests/conftest.py b/jupyter_client/tests/conftest.py index 69376d415..3d0e33b46 100644 --- a/jupyter_client/tests/conftest.py +++ b/jupyter_client/tests/conftest.py @@ -32,7 +32,7 @@ resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) -@pytest.fixture(autouse=True) +@pytest.fixture def event_loop(): # Make sure we test against a selector event loop # since pyzmq doesn't like the proactor loop. @@ -40,7 +40,7 @@ def event_loop(): if os.name == "nt" and sys.version_info >= (3, 7): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) loop = asyncio.SelectorEventLoop() - asyncio.set_event_loop(loop) + try: yield loop finally: diff --git a/pyproject.toml b/pyproject.toml index f576bfad1..72b9a04e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ src = "jupyter_client/_version.py" [tool.pytest.ini_options] norecursedirs = "dist build" addopts= "-r sxX" +asyncio_mode = "strict" filterwarnings= [ # Fail on warnings "error", From eaf21953f9318b50bb68a44b7233de4e35f30f59 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 18:43:28 -0600 Subject: [PATCH 03/36] switch to auto mode --- pyproject.toml | 2 +- requirements-test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 72b9a04e7..59671336a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ src = "jupyter_client/_version.py" [tool.pytest.ini_options] norecursedirs = "dist build" addopts= "-r sxX" -asyncio_mode = "strict" +asyncio_mode = "auto" filterwarnings= [ # Fail on warnings "error", diff --git a/requirements-test.txt b/requirements-test.txt index f3d81fe98..5a8fcb052 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -7,6 +7,6 @@ mock mypy pre-commit pytest -pytest-asyncio +pytest-asyncio>=0.17 pytest-cov pytest-timeout From c068a82eaebbd5927d0efba3993e8957d077b8c7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 19:09:25 -0600 Subject: [PATCH 04/36] fix multikernelmanager tests --- jupyter_client/manager.py | 55 +++++++++++-------- .../tests/test_multikernelmanager.py | 28 ++++++---- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 39d27454d..d12bf810a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -52,33 +52,37 @@ class _ShutdownStatus(Enum): SigkillRequest = "SigkillRequest" -def in_pending_state(method): +def in_pending_state(attr='_ready'): """Sets the kernel to a pending state by creating a fresh Future for the KernelManager's `ready` attribute. Once the method is finished, set the Future's results. """ - @functools.wraps(method) - async def wrapper(self, *args, **kwargs): - # Create a future for the decorated method - try: - self._ready = Future() - except RuntimeError: - # No event loop running, use concurrent future - self._ready = CFuture() - try: - # call wrapped method, await, and set the result or exception. - out = await method(self, *args, **kwargs) - # Add a small sleep to ensure tests can capture the state before done - await asyncio.sleep(0.01) - self._ready.set_result(None) - return out - except Exception as e: - self._ready.set_exception(e) - self.log.exception(self._ready.exception()) - raise e + def inner(method): + @functools.wraps(method) + async def wrapper(self, *args, **kwargs): + # Create a future for the decorated method + try: + fut = Future() + except RuntimeError: + # No event loop running, use concurrent future + fut = CFuture() + setattr(self, attr, fut) + try: + # call wrapped method, await, and set the result or exception. + out = await method(self, *args, **kwargs) + # Add a small sleep to ensure tests can capture the state before done + await asyncio.sleep(0.01) + fut.set_result(None) + return out + except Exception as e: + fut.set_exception(e) + self.log.exception(fut.exception()) + raise e + + return wrapper - return wrapper + return inner class KernelManager(ConnectionFileMixin): @@ -91,6 +95,7 @@ def __init__(self, *args, **kwargs): super().__init__(**kwargs) self._shutdown_status = _ShutdownStatus.Unset # Create a place holder future. + self._shutdown_ready = None try: self._ready = Future() except RuntimeError: @@ -182,6 +187,11 @@ def ready(self) -> Future: """A future that resolves when the kernel process has started for the first time""" return self._ready + @property + def shutdown_ready(self) -> Future: + """A future that resolves when a shutdown has completed""" + return self._shutdown_ready + @property def ipykernel(self) -> bool: return self.kernel_name in {"python", "python2", "python3"} @@ -360,7 +370,7 @@ async def _async_post_start_kernel(self, **kw) -> None: post_start_kernel = run_sync(_async_post_start_kernel) - @in_pending_state + @in_pending_state() async def _async_start_kernel(self, **kw): """Starts a kernel on this host in a separate process. @@ -454,6 +464,7 @@ async def _async_cleanup_resources(self, restart: bool = False) -> None: cleanup_resources = run_sync(_async_cleanup_resources) + @in_pending_state('_shutdown_ready') async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False): """Attempts to stop the kernel process cleanly. diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index 8cd953d60..9a9fd9197 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -44,6 +44,10 @@ def setUp(self): self.env_patch.start() super().setUp() + def tearDown(self) -> None: + self.env_patch.stop() + return super().tearDown() + # static so picklable for multiprocessing on Windows @staticmethod def _get_tcp_km(): @@ -243,6 +247,10 @@ def setUp(self): self.env_patch.start() super().setUp() + def tearDown(self): + self.env_patch.stop() + super().setUp() + # static so picklable for multiprocessing on Windows @staticmethod def _get_tcp_km(): @@ -382,9 +390,8 @@ async def test_use_pending_kernels(self): k = km.get_kernel(kid) assert isinstance(k, AsyncKernelManager) await ensure_future(km.shutdown_kernel(kid, now=True)) - # Wait for the kernel to shutdown - await kernel.ready - assert kid not in km, f"{kid} not in {km}" + await kernel.shutdown_ready + assert kid not in km, f"{kid} aws in {km}" @gen_test async def test_use_pending_kernels_early_restart(self): @@ -396,9 +403,8 @@ async def test_use_pending_kernels_early_restart(self): await km.restart_kernel(kid, now=True) await kernel.ready await ensure_future(km.shutdown_kernel(kid, now=True)) - # Wait for the kernel to shutdown - await kernel.ready - assert kid not in km, f"{kid} not in {km}" + await kernel.shutdown_ready + assert kid not in km, f"{kid} was in {km}" @gen_test async def test_use_pending_kernels_early_shutdown(self): @@ -412,9 +418,8 @@ async def test_use_pending_kernels_early_shutdown(self): await kernel.ready # Shutdown once the kernel is ready await ensure_future(km.shutdown_kernel(kid, now=True)) - # Wait for the kernel to shutdown - await kernel.ready - assert kid not in km, f"{kid} not in {km}" + await kernel.shutdown_ready + assert kid not in km, f"{kid} was in {km}" @gen_test async def test_use_pending_kernels_early_interrupt(self): @@ -427,9 +432,8 @@ async def test_use_pending_kernels_early_interrupt(self): # Now wait for the kernel to be ready. await kernel.ready await ensure_future(km.shutdown_kernel(kid, now=True)) - # Wait for the kernel to shutdown - await kernel.ready - assert kid not in km, f"{kid} not in {km}" + await kernel.shutdown_ready + assert kid not in km, f"{kid} was in {km}" @gen_test async def test_tcp_cinfo(self): From 7ccc0cff547c67461a7427e4dae06fd38100c2d2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 19:11:29 -0600 Subject: [PATCH 05/36] fix provisioning --- jupyter_client/tests/test_provisioning.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index f8063f272..de15ea011 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -66,6 +66,11 @@ async def wait(self) -> Optional[int]: # Process is no longer alive, wait and clear ret = self.process.wait() + # Make sure all the fds get closed. + for attr in ['stdout', 'stderr', 'stdin']: + fid = getattr(self.process, attr) + if fid: + fid.close() self.process = None return ret From 4db6e4eda0b8d04a4abdc0ddbdbcf2da74c3ef7a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 19:27:23 -0600 Subject: [PATCH 06/36] cleanup --- jupyter_client/tests/conftest.py | 4 ++++ jupyter_client/tests/test_session.py | 4 +++- jupyter_client/threaded.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jupyter_client/tests/conftest.py b/jupyter_client/tests/conftest.py index 3d0e33b46..c1cf231a5 100644 --- a/jupyter_client/tests/conftest.py +++ b/jupyter_client/tests/conftest.py @@ -32,6 +32,10 @@ resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) +if os.name == "nt" and sys.version_info >= (3, 7): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + @pytest.fixture def event_loop(): # Make sure we test against a selector event loop diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index edf7a2bcd..19ff48124 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -9,6 +9,7 @@ import pytest import zmq +from tornado import ioloop from zmq.eventloop.zmqstream import ZMQStream from zmq.tests import BaseZMQTestCase @@ -169,7 +170,8 @@ def test_tracking(self): a, b = self.create_bound_pair(zmq.PAIR, zmq.PAIR) s = self.session s.copy_threshold = 1 - ZMQStream(a) + loop = ioloop.IOLoop(make_current=False) + ZMQStream(a, io_loop=loop) msg = s.send(a, "hello", track=False) self.assertTrue(msg["tracker"] is ss.DONE) msg = s.send(a, "hello", track=True) diff --git a/jupyter_client/threaded.py b/jupyter_client/threaded.py index 9a4bf4c58..54e44a3d7 100644 --- a/jupyter_client/threaded.py +++ b/jupyter_client/threaded.py @@ -47,7 +47,7 @@ def __init__( self, socket: Optional[zmq.Socket], session: Optional[Session], - loop: Optional[Any], + loop: Optional[zmq.eventloop.ioloop.ZMQIOLoop], ) -> None: """Create a channel. From 8de3e86cfda892d463154c29b8feed7db1e783d2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 19:30:10 -0600 Subject: [PATCH 07/36] fix for min version --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 59671336a..97c612d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,4 +39,7 @@ filterwarnings= [ # Workarounds for https://github.com/pytest-dev/pytest-asyncio/issues/77 "ignore:unclosed Date: Sun, 6 Mar 2022 19:30:59 -0600 Subject: [PATCH 08/36] fix toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97c612d0c..bc2b9757f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,8 +38,8 @@ filterwarnings= [ # Workarounds for https://github.com/pytest-dev/pytest-asyncio/issues/77 "ignore:unclosed Date: Sun, 6 Mar 2022 19:35:47 -0600 Subject: [PATCH 09/36] try pyzmq 18 --- pyproject.toml | 3 --- requirements.txt | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bc2b9757f..54db66d98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,4 @@ filterwarnings= [ # Workarounds for https://github.com/pytest-dev/pytest-asyncio/issues/77 "ignore:unclosed =4.9.2 nest-asyncio>=1.5.1 python-dateutil>=2.1 -pyzmq>=17 +pyzmq>=18 tornado>=5.0 traitlets From 40d38906409ca0d7d021c23ce8b428ad5586342d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 19:36:19 -0600 Subject: [PATCH 10/36] use bugfix version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ae2a7681b..eb91878a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ entrypoints jupyter_core>=4.9.2 nest-asyncio>=1.5.1 python-dateutil>=2.1 -pyzmq>=18 +pyzmq>=18.0.2 tornado>=5.0 traitlets From afdacfbdb4f135df9492dd4fe9e11a695bbeeda1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 19:39:54 -0600 Subject: [PATCH 11/36] do not fail on warnings for min deps --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b6371119a..abe1503f5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,7 +96,7 @@ jobs: - name: Install miniumum versions uses: jupyterlab/maintainer-tools/.github/actions/install-minimums@v1 - name: Run the unit tests - run: pytest -vv jupyter_client || pytest -vv jupyter_client --lf + run: pytest -vv jupyter_client -W default || pytest -vv jupyter_client --lf -W default test_prereleases: name: Test Prereleases From 87832a26d69b546f7577259548e84cfd8251a1f3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 6 Mar 2022 19:46:48 -0600 Subject: [PATCH 12/36] fix event loop warning --- jupyter_client/tests/test_multikernelmanager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index 9a9fd9197..e0fa7740a 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -245,6 +245,13 @@ class TestAsyncKernelManager(AsyncTestCase): def setUp(self): self.env_patch = test_env() self.env_patch.start() + # Work around deprecation warning in Python 3.10+ + # when there is not running event loop and + # get_event_loop() is called by tornado. + try: + asyncio.get_running_loop() + except RuntimeError: + asyncio.set_event_loop(asyncio.new_event_loop()) super().setUp() def tearDown(self): From c3b432a4475c1278efc2e948c7700668bbae929d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 05:31:22 -0600 Subject: [PATCH 13/36] py10 fixups --- jupyter_client/tests/test_multikernelmanager.py | 7 ------- jupyter_client/utils.py | 4 ++-- pyproject.toml | 7 +++++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index e0fa7740a..9a9fd9197 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -245,13 +245,6 @@ class TestAsyncKernelManager(AsyncTestCase): def setUp(self): self.env_patch = test_env() self.env_patch.start() - # Work around deprecation warning in Python 3.10+ - # when there is not running event loop and - # get_event_loop() is called by tornado. - try: - asyncio.get_running_loop() - except RuntimeError: - asyncio.set_event_loop(asyncio.new_event_loop()) super().setUp() def tearDown(self): diff --git a/jupyter_client/utils.py b/jupyter_client/utils.py index c2d6d5070..1063ce4d8 100644 --- a/jupyter_client/utils.py +++ b/jupyter_client/utils.py @@ -11,7 +11,7 @@ def run_sync(coro): def wrapped(*args, **kwargs): try: - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -19,7 +19,7 @@ def wrapped(*args, **kwargs): nest_asyncio.apply(loop) - future = asyncio.ensure_future(coro(*args, **kwargs)) + future = asyncio.ensure_future(coro(*args, **kwargs), loop=loop) try: return loop.run_until_complete(future) except BaseException as e: diff --git a/pyproject.toml b/pyproject.toml index 54db66d98..d2d01e134 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,4 +39,11 @@ filterwarnings= [ # Workarounds for https://github.com/pytest-dev/pytest-asyncio/issues/77 "ignore:unclosed Date: Mon, 7 Mar 2022 05:51:13 -0600 Subject: [PATCH 14/36] remove problematic test --- jupyter_client/tests/test_utils.py | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 jupyter_client/tests/test_utils.py diff --git a/jupyter_client/tests/test_utils.py b/jupyter_client/tests/test_utils.py deleted file mode 100644 index dd6849192..000000000 --- a/jupyter_client/tests/test_utils.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -from unittest import mock - -import pytest - -from jupyter_client.utils import run_sync - - -@pytest.fixture -def loop(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return loop - - -def test_run_sync_clean_up_task(loop): - async def coro_never_called(): - pytest.fail("The call to this coroutine is not expected") - - # Ensure that run_sync cancels the pending task - with mock.patch.object(loop, "run_until_complete") as patched_loop: - patched_loop.side_effect = KeyboardInterrupt - with mock.patch("asyncio.ensure_future") as patched_ensure_future: - mock_future = mock.Mock() - patched_ensure_future.return_value = mock_future - with pytest.raises(KeyboardInterrupt): - run_sync(coro_never_called)() - mock_future.cancel.assert_called_once() - # Suppress 'coroutine ... was never awaited' warning - patched_ensure_future.call_args[0][0].close() From 602f361dbc86cd2544586fc63738e63e6befdb4c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 05:58:46 -0600 Subject: [PATCH 15/36] handle loop warning --- jupyter_client/manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index d12bf810a..e4e2a4497 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -9,6 +9,7 @@ import sys import typing as t import uuid +import warnings from asyncio.futures import Future from concurrent.futures import Future as CFuture from contextlib import contextmanager @@ -63,7 +64,8 @@ def inner(method): async def wrapper(self, *args, **kwargs): # Create a future for the decorated method try: - fut = Future() + with warnings.catch_warnings(): + fut = Future() except RuntimeError: # No event loop running, use concurrent future fut = CFuture() @@ -97,7 +99,8 @@ def __init__(self, *args, **kwargs): # Create a place holder future. self._shutdown_ready = None try: - self._ready = Future() + with warnings.catch_warnings(): + self._ready = Future() except RuntimeError: # No event loop running, use concurrent future self._ready = CFuture() From 8945cc25c1dad1b677f000397c29213dee6db666 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 06:10:51 -0600 Subject: [PATCH 16/36] fix windows tempdir handling --- jupyter_client/tests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jupyter_client/tests/utils.py b/jupyter_client/tests/utils.py index 5ca313469..8d027e4a2 100644 --- a/jupyter_client/tests/utils.py +++ b/jupyter_client/tests/utils.py @@ -62,7 +62,11 @@ def start(self): def stop(self): self.env_patch.stop() - self.test_dir.cleanup() + try: + self.test_dir.cleanup() + except PermissionError: + if os.name != 'nt': + raise def __enter__(self): self.start() From 1edb75fafa521da8d6f05487c07e1762f7c9e3ee Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 06:25:17 -0600 Subject: [PATCH 17/36] more cleanup --- jupyter_client/manager.py | 9 ++++----- jupyter_client/tests/test_multikernelmanager.py | 5 +++-- jupyter_client/tests/utils.py | 2 +- pyproject.toml | 4 ++++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index e4e2a4497..3abdc9f94 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -9,7 +9,6 @@ import sys import typing as t import uuid -import warnings from asyncio.futures import Future from concurrent.futures import Future as CFuture from contextlib import contextmanager @@ -64,8 +63,8 @@ def inner(method): async def wrapper(self, *args, **kwargs): # Create a future for the decorated method try: - with warnings.catch_warnings(): - fut = Future() + asyncio.get_running_loop() + fut = Future() except RuntimeError: # No event loop running, use concurrent future fut = CFuture() @@ -99,8 +98,8 @@ def __init__(self, *args, **kwargs): # Create a place holder future. self._shutdown_ready = None try: - with warnings.catch_warnings(): - self._ready = Future() + asyncio.get_running_loop() + self._ready = Future() except RuntimeError: # No event loop running, use concurrent future self._ready = CFuture() diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index 9a9fd9197..64516092c 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -469,8 +469,9 @@ async def test_start_sequence_ipc_kernels(self): def tcp_lifecycle_with_loop(self): # Ensure each thread has an event loop - asyncio.set_event_loop(asyncio.new_event_loop()) - asyncio.get_event_loop().run_until_complete(self.raw_tcp_lifecycle()) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self.raw_tcp_lifecycle()) # static so picklable for multiprocessing on Windows @classmethod diff --git a/jupyter_client/tests/utils.py b/jupyter_client/tests/utils.py index 8d027e4a2..ffaba2e9a 100644 --- a/jupyter_client/tests/utils.py +++ b/jupyter_client/tests/utils.py @@ -64,7 +64,7 @@ def stop(self): self.env_patch.stop() try: self.test_dir.cleanup() - except PermissionError: + except (PermissionError, NotADirectoryError): if os.name != 'nt': raise diff --git a/pyproject.toml b/pyproject.toml index d2d01e134..6bb8ba1f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,4 +46,8 @@ filterwarnings= [ # Workaround for distutils.Version used in ipykernel "ignore:The distutils package is deprecated and slated for removal:DeprecationWarning:ipykernel", + + # ZMQ uses Future internally, which raises a DeprecationWarning + # When there is no loop running. + "ignore:There is no current event loop:DeprecationWarning:zmq" ] From 2db6359bac92b010a3ec019f02446cac37e0b374 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 06:37:28 -0600 Subject: [PATCH 18/36] more cleanup --- jupyter_client/tests/test_multikernelmanager.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index 64516092c..c972ad63f 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -484,11 +484,8 @@ async def raw_tcp_lifecycle(cls, test_kid=None): # static so picklable for multiprocessing on Windows @classmethod def raw_tcp_lifecycle_sync(cls, test_kid=None): - loop = asyncio.get_event_loop() - if loop.is_running(): - # Forked MP, make new loop - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) loop.run_until_complete(cls.raw_tcp_lifecycle(test_kid=test_kid)) @gen_test From 77f5e4bc66aa0e0716979f69480e84deabf258a6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 06:53:38 -0600 Subject: [PATCH 19/36] use latest nest-asyncio --- jupyter_client/manager.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 3abdc9f94..d2ec3a71a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -95,8 +95,8 @@ class KernelManager(ConnectionFileMixin): def __init__(self, *args, **kwargs): super().__init__(**kwargs) self._shutdown_status = _ShutdownStatus.Unset - # Create a place holder future. self._shutdown_ready = None + # Create a place holder future. try: asyncio.get_running_loop() self._ready = Future() diff --git a/requirements.txt b/requirements.txt index eb91878a4..fa0a4ec85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ entrypoints jupyter_core>=4.9.2 -nest-asyncio>=1.5.1 +nest-asyncio>=1.5.4 python-dateutil>=2.1 pyzmq>=18.0.2 tornado>=5.0 From d217f5973abbd992779af5ada1688431974bb48a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 08:03:40 -0600 Subject: [PATCH 20/36] try updated tornado --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fa0a4ec85..8a1c41d13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ jupyter_core>=4.9.2 nest-asyncio>=1.5.4 python-dateutil>=2.1 pyzmq>=18.0.2 -tornado>=5.0 +tornado>=6.1 traitlets From 2c506ecb6b1b61a793d8a848828631f9768b50f9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 08:31:46 -0600 Subject: [PATCH 21/36] try newer asyncio-test --- requirements-test.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 5a8fcb052..83a9562cc 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -7,6 +7,6 @@ mock mypy pre-commit pytest -pytest-asyncio>=0.17 +pytest-asyncio>=0.18 pytest-cov pytest-timeout diff --git a/requirements.txt b/requirements.txt index 8a1c41d13..fa0a4ec85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ jupyter_core>=4.9.2 nest-asyncio>=1.5.4 python-dateutil>=2.1 pyzmq>=18.0.2 -tornado>=6.1 +tornado>=5.0 traitlets From 7a2f6d17fe4bc65cd93e52c111829bcc0362e10d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 08:46:56 -0600 Subject: [PATCH 22/36] try newer pyzmq --- .github/workflows/main.yml | 2 +- jupyter_client/ssh/tunnel.py | 4 +--- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index abe1503f5..646c007fc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,7 +73,7 @@ jobs: run: mypy jupyter_client --exclude '\/tests|kernelspecapp|ioloop|runapp' --install-types --non-interactive - name: Run the tests with coverage - run: pytest --cov jupyter_client -v jupyter_client + run: pytest --cov jupyter_client -vv jupyter_client || || pytest --cov jupyter_client -vv jupyter_client --lf - name: Build the docs run: | diff --git a/jupyter_client/ssh/tunnel.py b/jupyter_client/ssh/tunnel.py index 88e10323f..80f7ae67b 100644 --- a/jupyter_client/ssh/tunnel.py +++ b/jupyter_client/ssh/tunnel.py @@ -36,8 +36,6 @@ class SSHException(Exception): # type: ignore except ImportError: pexpect = None -from zmq.utils.strtypes import b - def select_random_ports(n): """Select and return n random ports that are available.""" @@ -56,7 +54,7 @@ def select_random_ports(n): # ----------------------------------------------------------------------------- # Check for passwordless login # ----------------------------------------------------------------------------- -_password_pat = re.compile(b(r"pass(word|phrase):"), re.IGNORECASE) +_password_pat = re.compile(r"pass(word|phrase):".encode("utf8"), re.IGNORECASE) def try_passwordless_ssh(server, keyfile, paramiko=None): diff --git a/requirements.txt b/requirements.txt index fa0a4ec85..973680cc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ entrypoints jupyter_core>=4.9.2 nest-asyncio>=1.5.4 python-dateutil>=2.1 -pyzmq>=18.0.2 +pyzmq>=22 tornado>=5.0 traitlets From 9868fa844b068e5a6808471c48a2442af3c3878e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 10:02:45 -0600 Subject: [PATCH 23/36] try newer ipykernel --- .github/workflows/main.yml | 2 +- requirements-test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 646c007fc..c7c704495 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,7 +73,7 @@ jobs: run: mypy jupyter_client --exclude '\/tests|kernelspecapp|ioloop|runapp' --install-types --non-interactive - name: Run the tests with coverage - run: pytest --cov jupyter_client -vv jupyter_client || || pytest --cov jupyter_client -vv jupyter_client --lf + run: pytest --cov jupyter_client -vv jupyter_client || pytest --cov jupyter_client -vv jupyter_client --lf - name: Build the docs run: | diff --git a/requirements-test.txt b/requirements-test.txt index 83a9562cc..3dcd4b159 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ codecov coverage -ipykernel>=5.5.6 +ipykernel>=6.9 ipython jedi<0.18; python_version<="3.6" mock From 9251cc1b46565ce41742dd5bead1e811d4ea3035 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 10:11:38 -0600 Subject: [PATCH 24/36] try removing zmq warning guard --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bb8ba1f0..d2d01e134 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,4 @@ filterwarnings= [ # Workaround for distutils.Version used in ipykernel "ignore:The distutils package is deprecated and slated for removal:DeprecationWarning:ipykernel", - - # ZMQ uses Future internally, which raises a DeprecationWarning - # When there is no loop running. - "ignore:There is no current event loop:DeprecationWarning:zmq" ] From 457e34466957604983e2e51c256ca5f51ec2990a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 10:21:25 -0600 Subject: [PATCH 25/36] restore future warning --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d2d01e134..e50e8d2f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,4 +46,10 @@ filterwarnings= [ # Workaround for distutils.Version used in ipykernel "ignore:The distutils package is deprecated and slated for removal:DeprecationWarning:ipykernel", + + # ZMQ uses Future internally, which raises a DeprecationWarning + # When there is no loop running. + # We could eventually find a way to make sure these are only created + # when there is a running event loop. + "ignore:There is no current event loop:DeprecationWarning:zmq" ] From 0a754622bdc218d17fc23137d9c1a899001c3760 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 10:29:42 -0600 Subject: [PATCH 26/36] address review --- jupyter_client/client.py | 5 +++-- jupyter_client/tests/conftest.py | 11 +++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/jupyter_client/client.py b/jupyter_client/client.py index 8727f406f..4e5e7e0d4 100644 --- a/jupyter_client/client.py +++ b/jupyter_client/client.py @@ -286,8 +286,8 @@ def start_channels( :meth:`start_kernel`. If the channels have been stopped and you call this, :class:`RuntimeError` will be raised. """ - # Re-create the context if needed. - if self._created_context: + # Create the context if needed. + if not self._created_context: self.context = self._context_default() if iopub: self.iopub_channel.start() @@ -319,6 +319,7 @@ def stop_channels(self) -> None: if self.control_channel.is_alive(): self.control_channel.stop() if self._created_context: + self._created_context = False self.context.destroy() @property diff --git a/jupyter_client/tests/conftest.py b/jupyter_client/tests/conftest.py index c1cf231a5..2ab60d5b8 100644 --- a/jupyter_client/tests/conftest.py +++ b/jupyter_client/tests/conftest.py @@ -17,6 +17,7 @@ # Handle resource limit +# Ensure a minimal soft limit of DEFAULT_SOFT if the current hard limit is at least that much. if resource is not None: soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) @@ -24,12 +25,10 @@ if hard >= DEFAULT_SOFT: soft = DEFAULT_SOFT - old_soft, old_hard = resource.getrlimit(resource.RLIMIT_NOFILE) - hard = old_hard - if old_soft < soft: - if hard < soft: - hard = soft - resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + if hard < soft: + hard = soft + + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) if os.name == "nt" and sys.version_info >= (3, 7): From b0a321d16ce6b0f4fc2eb57d23a9d71942350ebb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 13:38:21 -0600 Subject: [PATCH 27/36] bump tornado --- requirements-test.txt | 1 - requirements.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 3dcd4b159..a50d1f7ac 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,7 +2,6 @@ codecov coverage ipykernel>=6.9 ipython -jedi<0.18; python_version<="3.6" mock mypy pre-commit diff --git a/requirements.txt b/requirements.txt index 973680cc1..d812010de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ jupyter_core>=4.9.2 nest-asyncio>=1.5.4 python-dateutil>=2.1 pyzmq>=22 -tornado>=5.0 +tornado>=6.1 traitlets From ff8e482f9424ff5b2b1625de42b63132f76323da Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 13:56:16 -0600 Subject: [PATCH 28/36] bump zmq version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d812010de..2561ca798 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ entrypoints jupyter_core>=4.9.2 nest-asyncio>=1.5.4 python-dateutil>=2.1 -pyzmq>=22 +pyzmq>=23 tornado>=6.1 traitlets From d670d8c089df6090ac7c65221861b3fd1fa0d584 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 14:00:18 -0600 Subject: [PATCH 29/36] fix version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2561ca798..63ff741b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ entrypoints jupyter_core>=4.9.2 nest-asyncio>=1.5.4 python-dateutil>=2.1 -pyzmq>=23 +pyzmq>=22.3 tornado>=6.1 traitlets From 9ec08a8c4415c5820afe729ecf796bb0998359c7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 14:07:25 -0600 Subject: [PATCH 30/36] relax a couple versions --- requirements-test.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index a50d1f7ac..0631d5b6e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ codecov coverage -ipykernel>=6.9 +ipykernel>=6.2 ipython mock mypy diff --git a/requirements.txt b/requirements.txt index 63ff741b4..a6ca1eb48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ jupyter_core>=4.9.2 nest-asyncio>=1.5.4 python-dateutil>=2.1 pyzmq>=22.3 -tornado>=6.1 +tornado>=6.0 traitlets From 968b9c75997018cb9dd875e938d8849ad1ea575b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 14:09:44 -0600 Subject: [PATCH 31/36] clean up ci --- .github/workflows/downstream.yml | 2 ++ .github/workflows/main.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 97957f15a..90ef7c442 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -12,9 +12,11 @@ concurrency: jobs: tests: runs-on: ubuntu-latest + timeout-minutes: 20 strategy: matrix: python-version: ["3.9"] + fail-fast: false steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c7c704495..c85ca9e49 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,6 +46,8 @@ jobs: build-n-test-n-coverage: name: Build, test and code coverage + timeout-minutes: 15 + runs-on: ${{ matrix.os }} strategy: @@ -87,6 +89,7 @@ jobs: test_miniumum_verisons: name: Test Minimum Versions runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v2 - name: Base Setup @@ -100,6 +103,7 @@ jobs: test_prereleases: name: Test Prereleases + timeout-minutes: 10 runs-on: ubuntu-latest steps: - name: Checkout From 35645f805aba84d809e2efc84ca19091e526a268 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 14:29:22 -0600 Subject: [PATCH 32/36] try a newer ipykernel --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 0631d5b6e..80f245dcd 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ codecov coverage -ipykernel>=6.2 +ipykernel>=6.5 ipython mock mypy From 8c4c47257f1b4a47fbdc3edc93fcc772e3c3335c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Mar 2022 18:04:16 -0600 Subject: [PATCH 33/36] fix shutdown all behavior --- jupyter_client/multikernelmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 79a589ef9..a5a3600b8 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -324,7 +324,7 @@ async def _async_shutdown_all(self, now: bool = False) -> None: # should be awaited before exiting this method. if self._using_pending_kernels(): for km in self._kernels.values(): - await km.ready + await km.shutdown_ready shutdown_all = run_sync(_async_shutdown_all) From d108c0512ffd2ee3f3510ddacdc33db4eb72727e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Mar 2022 06:04:45 -0600 Subject: [PATCH 34/36] wip refactor --- jupyter_client/manager.py | 77 ++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index d2ec3a71a..7e602fd67 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -9,8 +9,7 @@ import sys import typing as t import uuid -from asyncio.futures import Future -from concurrent.futures import Future as CFuture +from concurrent.futures import Future from contextlib import contextmanager from enum import Enum @@ -52,23 +51,23 @@ class _ShutdownStatus(Enum): SigkillRequest = "SigkillRequest" -def in_pending_state(attr='_ready'): - """Sets the kernel to a pending state by - creating a fresh Future for the KernelManager's `ready` - attribute. Once the method is finished, set the Future's results. - """ +def in_pending_state(future_name='_start_future', flag='starting'): + """Manages the future for a pending operation""" def inner(method): @functools.wraps(method) async def wrapper(self, *args, **kwargs): - # Create a future for the decorated method - try: - asyncio.get_running_loop() - fut = Future() - except RuntimeError: - # No event loop running, use concurrent future - fut = CFuture() - setattr(self, attr, fut) + # If we are aleady starting, error out. + if flag == 'starting': + if self.starting: + raise RuntimeError('Pending Start') + # If we are already shutting down, bail. + elif flag == 'shutting_down': + if self.shutting_down: + return + fut = Future() + setattr(self, future_name, fut) + setattr(self, flag, True) try: # call wrapped method, await, and set the result or exception. out = await method(self, *args, **kwargs) @@ -80,6 +79,9 @@ async def wrapper(self, *args, **kwargs): fut.set_exception(e) self.log.exception(fut.exception()) raise e + finally: + setattr(self, flag, False) + setattr(self, future_name, None) return wrapper @@ -96,13 +98,9 @@ def __init__(self, *args, **kwargs): super().__init__(**kwargs) self._shutdown_status = _ShutdownStatus.Unset self._shutdown_ready = None - # Create a place holder future. - try: - asyncio.get_running_loop() - self._ready = Future() - except RuntimeError: - # No event loop running, use concurrent future - self._ready = CFuture() + # Add placeholders for start and shutdown futures + self._start_future = None + self._shutdown_future = None _created_context: Bool = Bool(False) @@ -184,16 +182,6 @@ def kernel_spec(self) -> t.Optional[kernelspec.KernelSpec]: def _default_cache_ports(self) -> bool: return self.transport == "tcp" - @property - def ready(self) -> Future: - """A future that resolves when the kernel process has started for the first time""" - return self._ready - - @property - def shutdown_ready(self) -> Future: - """A future that resolves when a shutdown has completed""" - return self._shutdown_ready - @property def ipykernel(self) -> bool: return self.kernel_name in {"python", "python2", "python3"} @@ -208,6 +196,7 @@ def ipykernel(self) -> bool: True, config=True, help="""Should we autorestart the kernel if it dies.""" ) + starting: bool = False shutting_down: bool = False def __del__(self) -> None: @@ -259,6 +248,26 @@ def client(self, **kwargs) -> KernelClient: # Kernel management # -------------------------------------------------------------------------- + async def wait_for_start_ready(self): + """Wait for a pending start to be ready.""" + if not self._start_future: + raise RuntimeError('Not starting') + future = self._start_future + await asyncio.ensure_future(future) + # Recurse if needed. + if self._start_future and self._start_future != future: + await self.wait_for_start_ready() + + async def wait_for_shutdown_ready(self): + """Wait for a pending shutdown to be ready.""" + if not self._shutdown_future: + raise RuntimeError('Not shutting down') + future = self._shutdown_future + await asyncio.ensure_future(future) + # Recurse if needed. + if self._shutdown_future and self._shutdown_future != future: + await self.wait_for_shutdown_ready() + def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] @@ -341,7 +350,6 @@ async def _async_pre_start_kernel(self, **kw) -> t.Tuple[t.List[str], t.Dict[str keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - self.shutting_down = False self.kernel_id = self.kernel_id or kw.pop('kernel_id', str(uuid.uuid4())) # save kwargs for use in restart self._launch_args = kw.copy() @@ -466,7 +474,7 @@ async def _async_cleanup_resources(self, restart: bool = False) -> None: cleanup_resources = run_sync(_async_cleanup_resources) - @in_pending_state('_shutdown_ready') + @in_pending_state('_shutdown_future', 'shutting_down') async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False): """Attempts to stop the kernel process cleanly. @@ -504,6 +512,7 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) await ensure_async(self.finish_shutdown(restart=restart)) await ensure_async(self.cleanup_resources(restart=restart)) + self.shutting_down = False shutdown_kernel = run_sync(_async_shutdown_kernel) From 66c6dab2c4e85eb8988ef2f5f3cc5568b7a1e075 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Mar 2022 14:56:51 -0600 Subject: [PATCH 35/36] more cleanup --- jupyter_client/manager.py | 29 +++++---- jupyter_client/multikernelmanager.py | 64 ++++--------------- jupyter_client/tests/test_kernelmanager.py | 6 +- .../tests/test_multikernelmanager.py | 44 ++++++------- jupyter_client/tests/utils.py | 4 +- 5 files changed, 52 insertions(+), 95 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 7e602fd67..7c117360c 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -65,7 +65,12 @@ async def wrapper(self, *args, **kwargs): elif flag == 'shutting_down': if self.shutting_down: return - fut = Future() + fut = getattr(self, future_name) + + # Create a new future as needed. + if fut.done: + fut = Future() + setattr(self, future_name, fut) setattr(self, flag, True) try: @@ -81,7 +86,6 @@ async def wrapper(self, *args, **kwargs): raise e finally: setattr(self, flag, False) - setattr(self, future_name, None) return wrapper @@ -98,9 +102,9 @@ def __init__(self, *args, **kwargs): super().__init__(**kwargs) self._shutdown_status = _ShutdownStatus.Unset self._shutdown_ready = None - # Add placeholders for start and shutdown futures - self._start_future = None - self._shutdown_future = None + # Add initial start and shutdown futures. + self._start_future = Future() + self._shutdown_future = Future() _created_context: Bool = Bool(False) @@ -250,22 +254,18 @@ def client(self, **kwargs) -> KernelClient: async def wait_for_start_ready(self): """Wait for a pending start to be ready.""" - if not self._start_future: - raise RuntimeError('Not starting') future = self._start_future - await asyncio.ensure_future(future) + await asyncio.ensure_future(asyncio.wrap_future(future)) # Recurse if needed. - if self._start_future and self._start_future != future: + if self._start_future != future: await self.wait_for_start_ready() async def wait_for_shutdown_ready(self): """Wait for a pending shutdown to be ready.""" - if not self._shutdown_future: - raise RuntimeError('Not shutting down') future = self._shutdown_future - await asyncio.ensure_future(future) + await asyncio.ensure_future(asyncio.wrap_future(future)) # Recurse if needed. - if self._shutdown_future and self._shutdown_future != future: + if self._shutdown_future != future: await self.wait_for_shutdown_ready() def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: @@ -493,6 +493,9 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ + if self.starting: + await self.wait_for_start_ready() + self.shutting_down = True # Used by restarter to prevent race condition # Stop monitoring for restarting while we shutdown. self.stop_restarter() diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index a5a3600b8..d6f891196 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -214,35 +214,17 @@ async def _async_start_kernel(self, kernel_name: t.Optional[str] = None, **kwarg self._kernels[kernel_id] = km else: await fut - # raise an exception if one occurred during kernel startup. - if km.ready.exception(): - raise km.ready.exception() # type: ignore return kernel_id start_kernel = run_sync(_async_start_kernel) - async def _shutdown_kernel_when_ready( - self, - kernel_id: str, - now: t.Optional[bool] = False, - restart: t.Optional[bool] = False, - ) -> None: - """Wait for a pending kernel to be ready - before shutting the kernel down. - """ - # Only do this if using pending kernels - if self._using_pending_kernels(): - kernel = self._kernels[kernel_id] - await kernel.ready - # Once out of a pending state, we can call shutdown. - await ensure_async(self.shutdown_kernel(kernel_id, now=now, restart=restart)) - async def _async_shutdown_kernel( self, kernel_id: str, now: t.Optional[bool] = False, restart: t.Optional[bool] = False, + force_wait: t.Optional[bool] = False, ) -> None: """Shutdown a kernel by its kernel uuid. @@ -254,35 +236,20 @@ async def _async_shutdown_kernel( Should the kernel be shutdown forcibly using a signal. restart : bool Will the kernel be restarted? + force_wait: bool + Whether to wait for the shutdown to complete, even when using + pending kernels """ self.log.info("Kernel shutdown: %s" % kernel_id) - # If we're using pending kernels, block shutdown when a kernel is pending. - if self._using_pending_kernels() and kernel_id in self._pending_kernels: - raise RuntimeError("Kernel is in a pending state. Cannot shutdown.") - # If the kernel is still starting, wait for it to be ready. - elif kernel_id in self._pending_kernels: - kernel = self._pending_kernels[kernel_id] - try: - await kernel - km = self.get_kernel(kernel_id) - await km.ready - except Exception: - self.remove_kernel(kernel_id) - return + if kernel_id in self._pending_kernels: + await self._pending_kernels[kernel_id] km = self.get_kernel(kernel_id) - # If a pending kernel raised an exception, remove it. - if km.ready.exception(): - self.remove_kernel(kernel_id) - return stopper = ensure_async(km.shutdown_kernel(now, restart)) fut = asyncio.ensure_future(self._remove_kernel_when_ready(kernel_id, stopper)) self._pending_kernels[kernel_id] = fut # Await the kernel if not using pending kernels. - if not self._using_pending_kernels(): + if not self._using_pending_kernels() or force_wait: await fut - # raise an exception if one occurred during kernel shutdown. - if km.ready.exception(): - raise km.ready.exception() # type: ignore shutdown_kernel = run_sync(_async_shutdown_kernel) @@ -318,13 +285,10 @@ async def _async_shutdown_all(self, now: bool = False) -> None: """Shutdown all kernels.""" kids = self.list_kernel_ids() kids += list(self._pending_kernels) - futs = [ensure_async(self._shutdown_kernel_when_ready(kid, now=now)) for kid in set(kids)] + futs = [ + ensure_async(self.shutdown_kernel(kid, now=now, force_wait=True)) for kid in set(kids) + ] await asyncio.gather(*futs) - # When using "shutdown all", all pending kernels - # should be awaited before exiting this method. - if self._using_pending_kernels(): - for km in self._kernels.values(): - await km.shutdown_ready shutdown_all = run_sync(_async_shutdown_all) @@ -337,8 +301,6 @@ def interrupt_kernel(self, kernel_id: str) -> None: The id of the kernel to interrupt. """ kernel = self.get_kernel(kernel_id) - if not kernel.ready.done(): - raise RuntimeError("Kernel is in a pending state. Cannot interrupt.") out = kernel.interrupt_kernel() self.log.info("Kernel interrupted: %s" % kernel_id) return out @@ -375,9 +337,6 @@ async def _async_restart_kernel(self, kernel_id: str, now: bool = False) -> None it is given a chance to perform a clean shutdown or not. """ kernel = self.get_kernel(kernel_id) - if self._using_pending_kernels(): - if not kernel.ready.done(): - raise RuntimeError("Kernel is in a pending state. Cannot restart.") out = await ensure_async(kernel.restart_kernel(now=now)) self.log.info("Kernel restarted: %s" % kernel_id) return out @@ -545,8 +504,7 @@ class AsyncMultiKernelManager(MultiKernelManager): use_pending_kernels = Bool( False, - help="""Whether to make kernels available before the process has started. The - kernel has a `.ready` future which can be awaited before connecting""", + help="""Whether to make kernels available before the process has started.""", ).tag(config=True) start_kernel = MultiKernelManager._async_start_kernel diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index f53a3ae27..368138eb5 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -195,8 +195,7 @@ class TestKernelManager: def test_lifecycle(self, km): km.start_kernel(stdout=PIPE, stderr=PIPE) assert km.is_alive() - is_done = km.ready.done() - assert is_done + assert km.has_kernel km.restart_kernel(now=True) assert km.is_alive() km.interrupt_kernel() @@ -450,8 +449,7 @@ async def test_lifecycle(self, async_km): await async_km.start_kernel(stdout=PIPE, stderr=PIPE) is_alive = await async_km.is_alive() assert is_alive - is_ready = async_km.ready.done() - assert is_ready + assert async_km.has_kernel await async_km.restart_kernel(now=True) is_alive = await async_km.is_alive() assert is_alive diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index c972ad63f..0fc87bac7 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -79,7 +79,7 @@ def _run_lifecycle(km, test_kid=None): else: kid = km.start_kernel(stdout=PIPE, stderr=PIPE) assert km.is_alive(kid) - assert km.get_kernel(kid).ready.done() + assert km.get_kernel(kid).has_kernel assert kid in km assert kid in km.list_kernel_ids() assert len(km) == 1, f"{len(km)} != {1}" @@ -376,12 +376,12 @@ async def test_use_pending_kernels(self): km = self._get_pending_kernels_km() kid = await ensure_future(km.start_kernel(stdout=PIPE, stderr=PIPE)) kernel = km.get_kernel(kid) - assert not kernel.ready.done() + assert kernel.starting assert kid in km assert kid in km.list_kernel_ids() assert len(km) == 1, f"{len(km)} != {1}" # Wait for the kernel to start. - await kernel.ready + await kernel.wait_for_start_ready() await km.restart_kernel(kid, now=True) out = await km.is_alive(kid) assert out @@ -390,7 +390,7 @@ async def test_use_pending_kernels(self): k = km.get_kernel(kid) assert isinstance(k, AsyncKernelManager) await ensure_future(km.shutdown_kernel(kid, now=True)) - await kernel.shutdown_ready + await kernel.wait_for_shutdown_ready() assert kid not in km, f"{kid} aws in {km}" @gen_test @@ -398,27 +398,21 @@ async def test_use_pending_kernels_early_restart(self): km = self._get_pending_kernels_km() kid = await ensure_future(km.start_kernel(stdout=PIPE, stderr=PIPE)) kernel = km.get_kernel(kid) - assert not kernel.ready.done() - with pytest.raises(RuntimeError): - await km.restart_kernel(kid, now=True) - await kernel.ready - await ensure_future(km.shutdown_kernel(kid, now=True)) - await kernel.shutdown_ready - assert kid not in km, f"{kid} was in {km}" + assert kernel.starting + await km.restart_kernel(kid, now=True) + await kernel.wait_for_shutdown_ready() + await kernel.wait_for_start_ready() + assert kid in km @gen_test async def test_use_pending_kernels_early_shutdown(self): km = self._get_pending_kernels_km() kid = await ensure_future(km.start_kernel(stdout=PIPE, stderr=PIPE)) kernel = km.get_kernel(kid) - assert not kernel.ready.done() + assert kernel.starting # Try shutting down while the kernel is pending - with pytest.raises(RuntimeError): - await ensure_future(km.shutdown_kernel(kid, now=True)) - await kernel.ready - # Shutdown once the kernel is ready await ensure_future(km.shutdown_kernel(kid, now=True)) - await kernel.shutdown_ready + await kernel.wait_for_shutdown_ready() assert kid not in km, f"{kid} was in {km}" @gen_test @@ -426,13 +420,16 @@ async def test_use_pending_kernels_early_interrupt(self): km = self._get_pending_kernels_km() kid = await ensure_future(km.start_kernel(stdout=PIPE, stderr=PIPE)) kernel = km.get_kernel(kid) - assert not kernel.ready.done() - with pytest.raises(RuntimeError): - await km.interrupt_kernel(kid) + assert kernel.starting + if kernel.has_kernel: + await km.interrupt_kernel() + else: + with pytest.raises(RuntimeError): + await km.interrupt_kernel(kid) # Now wait for the kernel to be ready. - await kernel.ready + await kernel.wait_for_start_ready() await ensure_future(km.shutdown_kernel(kid, now=True)) - await kernel.shutdown_ready + await kernel.wait_for_shutdown_ready() assert kid not in km, f"{kid} was in {km}" @gen_test @@ -589,7 +586,8 @@ async def test_bad_kernelspec_pending(self): km.start_kernel(kernel_name="bad", stdout=PIPE, stderr=PIPE) ) with pytest.raises(FileNotFoundError): - await km.get_kernel(kernel_id).ready + await km.get_kernel(kernel_id).wait_for_start_ready() assert kernel_id in km.list_kernel_ids() await ensure_future(km.shutdown_kernel(kernel_id)) + await km.get_kernel(kernel_id).wait_for_shutdown_ready() assert kernel_id not in km.list_kernel_ids() diff --git a/jupyter_client/tests/utils.py b/jupyter_client/tests/utils.py index ffaba2e9a..a15db384c 100644 --- a/jupyter_client/tests/utils.py +++ b/jupyter_client/tests/utils.py @@ -139,7 +139,7 @@ def start_kernel(self, **kw): """Record call and defer to superclass""" @subclass_recorder - def shutdown_kernel(self, now=False, restart=False): + def shutdown_kernel(self, now=False, restart=False, force_wait=False): """Record call and defer to superclass""" @subclass_recorder @@ -212,7 +212,7 @@ def start_kernel(self, kernel_name=None, **kwargs): """Record call and defer to superclass""" @subclass_recorder - def shutdown_kernel(self, kernel_id, now=False, restart=False): + def shutdown_kernel(self, kernel_id, now=False, restart=False, force_wait=False): """Record call and defer to superclass""" @subclass_recorder From 40bacf097ce6d29e2dc8ee0f61ca3de84e5904dd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Mar 2022 15:19:24 -0600 Subject: [PATCH 36/36] fix test --- jupyter_client/tests/test_multikernelmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index 0fc87bac7..1207eb59e 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -422,7 +422,7 @@ async def test_use_pending_kernels_early_interrupt(self): kernel = km.get_kernel(kid) assert kernel.starting if kernel.has_kernel: - await km.interrupt_kernel() + await km.interrupt_kernel(kid) else: with pytest.raises(RuntimeError): await km.interrupt_kernel(kid)