"""A MultiTerminalManager for use in the notebook webserver - raises HTTPErrors - creates REST API models """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from datetime import timedelta import terminado from tornado import web from tornado.ioloop import IOLoop from tornado.ioloop import PeriodicCallback from traitlets import Integer from traitlets.config import LoggingConfigurable from ..prometheus.metrics import TERMINAL_CURRENTLY_RUNNING_TOTAL from jupyter_server._tz import isoformat from jupyter_server._tz import utcnow class TerminalManager(LoggingConfigurable, terminado.NamedTermManager): """ """ _culler_callback = None _initialized_culler = False cull_inactive_timeout = Integer( 0, config=True, help="""Timeout (in seconds) in which a terminal has been inactive and ready to be culled. Values of 0 or lower disable culling.""", ) cull_interval_default = 300 # 5 minutes cull_interval = Integer( cull_interval_default, config=True, help="""The interval (in seconds) on which to check for terminals exceeding the inactive timeout value.""", ) # ------------------------------------------------------------------------- # Methods for managing terminals # ------------------------------------------------------------------------- def __init__(self, *args, **kwargs): super(TerminalManager, self).__init__(*args, **kwargs) def create(self, **kwargs): """Create a new terminal.""" name, term = self.new_named_terminal(**kwargs) # Monkey-patch last-activity, similar to kernels. Should we need # more functionality per terminal, we can look into possible sub- # classing or containment then. term.last_activity = utcnow() model = self.get_terminal_model(name) # Increase the metric by one because a new terminal was created TERMINAL_CURRENTLY_RUNNING_TOTAL.inc() # Ensure culler is initialized self._initialize_culler() return model def get(self, name): """Get terminal 'name'.""" model = self.get_terminal_model(name) return model def list(self): """Get a list of all running terminals.""" models = [self.get_terminal_model(name) for name in self.terminals] # Update the metric below to the length of the list 'terms' TERMINAL_CURRENTLY_RUNNING_TOTAL.set(len(models)) return models async def terminate(self, name, force=False): """Terminate terminal 'name'.""" self._check_terminal(name) await super(TerminalManager, self).terminate(name, force=force) # Decrease the metric below by one # because a terminal has been shutdown TERMINAL_CURRENTLY_RUNNING_TOTAL.dec() async def terminate_all(self): """Terminate all terminals.""" terms = [name for name in self.terminals] for term in terms: await self.terminate(term, force=True) def get_terminal_model(self, name): """Return a JSON-safe dict representing a terminal. For use in representing terminals in the JSON APIs. """ self._check_terminal(name) term = self.terminals[name] model = { "name": name, "last_activity": isoformat(term.last_activity), } return model def _check_terminal(self, name): """Check a that terminal 'name' exists and raise 404 if not.""" if name not in self.terminals: raise web.HTTPError(404, u"Terminal not found: %s" % name) def _initialize_culler(self): """Start culler if 'cull_inactive_timeout' is greater than zero. Regardless of that value, set flag that we've been here. """ if not self._initialized_culler and self.cull_inactive_timeout > 0: if self._culler_callback is None: loop = IOLoop.current() if self.cull_interval <= 0: # handle case where user set invalid value self.log.warning( "Invalid value for 'cull_interval' detected (%s) - using default value (%s).", self.cull_interval, self.cull_interval_default, ) self.cull_interval = self.cull_interval_default self._culler_callback = PeriodicCallback( self._cull_terminals, 1000 * self.cull_interval ) self.log.info( "Culling terminals with inactivity > %s seconds at %s second intervals ...", self.cull_inactive_timeout, self.cull_interval, ) self._culler_callback.start() self._initialized_culler = True async def _cull_terminals(self): self.log.debug( "Polling every %s seconds for terminals inactive for > %s seconds...", self.cull_interval, self.cull_inactive_timeout, ) # Create a separate list of terminals to avoid conflicting updates while iterating for name in list(self.terminals): try: await self._cull_inactive_terminal(name) except Exception as e: self.log.exception( "The following exception was encountered while checking the " "activity of terminal {}: {}".format(name, e) ) async def _cull_inactive_terminal(self, name): try: term = self.terminals[name] except KeyError: return # KeyErrors are somewhat expected since the terminal can be terminated as the culling check is made. self.log.debug("name=%s, last_activity=%s", name, term.last_activity) if hasattr(term, "last_activity"): dt_now = utcnow() dt_inactive = dt_now - term.last_activity # Compute idle properties is_time = dt_inactive > timedelta(seconds=self.cull_inactive_timeout) # Cull the kernel if all three criteria are met if is_time: inactivity = int(dt_inactive.total_seconds()) self.log.warning( "Culling terminal '%s' due to %s seconds of inactivity.", name, inactivity ) await self.terminate(name, force=True)