diff --git a/jupyter_client/client.py b/jupyter_client/client.py
index a56a9ec30..5ffa069d9 100644
--- a/jupyter_client/client.py
+++ b/jupyter_client/client.py
@@ -117,10 +117,15 @@ def _context_default(self) -> zmq.asyncio.Context:
     allow_stdin: bool = True
 
     def __del__(self):
+        """Handle garbage collection.  Destroy context if applicable."""
         if self._created_context and self.context and not self.context.closed:
-            if self.log:
-                self.log.debug("Destroying zmq context for %s", self)
-            self.context.destroy()
+            if self.channels_running:
+                if self.log:
+                    self.log.warning("Could not destroy zmq context for %s", self)
+            else:
+                if self.log:
+                    self.log.debug("Destroying zmq context for %s", self)
+                self.context.destroy()
         try:
             super_del = super().__del__
         except AttributeError:
diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py
index daa5a6170..aec04d546 100644
--- a/jupyter_client/multikernelmanager.py
+++ b/jupyter_client/multikernelmanager.py
@@ -116,6 +116,7 @@ def _context_default(self) -> zmq.Context:
     _kernels = Dict()
 
     def __del__(self):
+        """Handle garbage collection.  Destroy context if applicable."""
         if self._created_context and self.context and not self.context.closed:
             if self.log:
                 self.log.debug("Destroying zmq context for %s", self)