diff --git a/notebook/services/kernels/kernelmanager.py b/notebook/services/kernels/kernelmanager.py index b5c4c9f5d2..292cb62393 100644 --- a/notebook/services/kernels/kernelmanager.py +++ b/notebook/services/kernels/kernelmanager.py @@ -11,15 +11,17 @@ from tornado import gen, web from tornado.concurrent import Future -from tornado.ioloop import IOLoop +from tornado.ioloop import IOLoop, PeriodicCallback from jupyter_client.multikernelmanager import MultiKernelManager -from traitlets import Dict, List, Unicode, TraitError, default, validate +from traitlets import Dict, List, Unicode, TraitError, Integer, default, validate from notebook.utils import to_os_path from notebook._tz import utcnow, isoformat from ipython_genutils.py3compat import getcwd +from datetime import datetime, timedelta + class MappingKernelManager(MultiKernelManager): """A KernelManager that handles notebook mapping and HTTP error handling""" @@ -34,6 +36,10 @@ def _default_kernel_manager_class(self): _kernel_connections = Dict() + _culler_callback = None + + _initialized_culler = False + @default('root_dir') def _default_root_dir(self): try: @@ -52,6 +58,18 @@ def _update_root_dir(self, proposal): raise TraitError("kernel root dir %r is not a directory" % value) return value + cull_idle_timeout_minimum = 300 # 5 minutes + cull_idle_timeout = Integer(0, config=True, + help="""Timeout (in seconds) after which a kernel is considered idle and ready to be culled. Values of 0 or + lower disable culling. The minimum timeout is 300 seconds (5 minutes). Positive values less than the minimum value + will be set to the minimum.""" + ) + + cull_interval_default = 300 # 5 minutes + cull_interval = Integer(cull_interval_default, config=True, + help="""The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value.""" + ) + #------------------------------------------------------------------------- # Methods for managing kernels and sessions #------------------------------------------------------------------------- @@ -105,6 +123,11 @@ def start_kernel(self, kernel_id=None, path=None, **kwargs): else: self._check_kernel_id(kernel_id) self.log.info("Using existing kernel: %s" % kernel_id) + + # Initialize culling if not already + if not self._initialized_culler: + self.initialize_culler() + # py2-compat raise gen.Return(kernel_id) @@ -225,3 +248,49 @@ def record_activity(msg_list): kernel._activity_stream.on_recv(record_activity) + def initialize_culler(self): + """Start idle culler if 'cull_idle_timeout' is greater than zero. + + Regardless of that value, set flag that we've been here. + """ + if not self._initialized_culler and self.cull_idle_timeout > 0: + if self._culler_callback is None: + if self.cull_idle_timeout < self.cull_idle_timeout_minimum: + self.log.warning("'cull_idle_timeout' (%s) is less than the minimum value (%s) and has been set to the minimum.", + self.cull_idle_timeout, self.cull_idle_timeout_minimum) + self.cull_idle_timeout = self.cull_idle_timeout_minimum + loop = IOLoop.current() + if self.cull_interval <= 0: #handle case where user set invalid value + self.log.warning("Invalid value for 'cull_interval' detected (%s) - using default value (%s).", + self.cull_interval, self.cull_interval_default) + self.cull_interval = self.cull_interval_default + self._culler_callback = PeriodicCallback( + self.cull_kernels, 1000*self.cull_interval, loop) + self.log.info("Culling kernels with idle durations > %s seconds at %s second intervals ...", + self.cull_idle_timeout, self.cull_interval) + self._culler_callback.start() + + self._initialized_culler = True + + def cull_kernels(self): + self.log.debug("Polling every %s seconds for kernels idle > %s seconds...", + self.cull_interval, self.cull_idle_timeout) + """Create a separate list of kernels to avoid conflicting updates while iterating""" + for kernel_id in list(self._kernels): + try: + self.cull_kernel_if_idle(kernel_id) + except Exception as e: + self.log.exception("The following exception was encountered while checking the idle duration of kernel %s: %s", + kernel_id, e) + + def cull_kernel_if_idle(self, kernel_id): + kernel = self._kernels[kernel_id] + self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s", kernel_id, kernel.kernel_name, kernel.last_activity) + if kernel.last_activity is not None: + dt_now = utcnow() + dt_idle = dt_now - kernel.last_activity + if dt_idle > timedelta(seconds=self.cull_idle_timeout): # exceeds timeout, can be culled + idle_duration = int(dt_idle.total_seconds()) + self.log.warning("Culling kernel '%s' (%s) due to %s seconds of inactivity.", kernel.kernel_name, kernel_id, idle_duration) + self.shutdown_kernel(kernel_id) +