Use jupyter_server instead of notebook (#927)

Author: kevin-bates

.github/workflows/build.yml

@@ -37,7 +37,7 @@ jobs:
         curl -L -o $HOME/.sbt/launchers/1.3.12/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.3.12/sbt-launch.jar
     - name: Install Python dependencies
       run: |
-        python -m pip install --upgrade setuptools pip websocket-client flake8 pytest coverage codecov
+        python -m pip install --upgrade setuptools pip websocket-client flake8 pytest pytest-tornasync ipykernel coverage codecov
     - name: Build Jupyter Enterprise Gateway conda env
       run: |
         SA="source $CONDA_HOME/bin/activate" make env
@@ -61,7 +61,7 @@ jobs:
         timeout_minutes: 3
         max_attempts: 1
         command: |
-          pytest enterprise_gateway/tests
+          pytest -v -s enterprise_gateway/tests
     - name: Run integration tests
       run: |
         SA="source $CONDA_HOME/bin/activate" make itest-yarn

Makefile

@@ -111,10 +111,10 @@ test-debug:
 test: TEST?=
 test: ## Run unit tests
 ifeq ($(TEST),)
-	$(SA) $(ENV) && pytest -v $(TEST_DEBUG_OPTS) enterprise_gateway/tests
+	$(SA) $(ENV) && pytest -v -s $(TEST_DEBUG_OPTS) enterprise_gateway/tests
 else
 # e.g., make test TEST="test_gatewayapp.TestGatewayAppConfig"
-	$(SA) $(ENV) && pytest -v $(TEST_DEBUG_OPTS) enterprise_gateway/tests/$(TEST)
+	$(SA) $(ENV) && pytest -v -s $(TEST_DEBUG_OPTS) enterprise_gateway/tests/$(TEST)
 endif
 
 release: POST_SDIST=upload
@@ -173,21 +173,21 @@ ITEST_OPTIONS?=
 
 ITEST_YARN_PORT?=8888
 ITEST_YARN_HOST?=localhost:$(ITEST_YARN_PORT)
-ITEST_YARN_TESTS?=enterprise_gateway.itests
+ITEST_YARN_TESTS?=enterprise_gateway/itests
 
 ITEST_KERNEL_LAUNCH_TIMEOUT=120
 
 LOG_LEVEL=INFO
 
 itest-yarn-debug: ## Run integration tests (optionally) against docker demo (YARN) container with print statements
-	make LOG_LEVEL=DEBUG TEST_DEBUG_OPTS="--nocapture --nologcapture --logging-level=10" itest-yarn
+	make LOG_LEVEL=DEBUG TEST_DEBUG_OPTS="--log-level=10" itest-yarn
 
 PREP_ITEST_YARN?=1
 itest-yarn: ## Run integration tests (optionally) against docker demo (YARN) container
 ifeq (1, $(PREP_ITEST_YARN))
 	make itest-yarn-prep
 endif
-	($(SA) $(ENV) && GATEWAY_HOST=$(ITEST_YARN_HOST) LOG_LEVEL=$(LOG_LEVEL) KERNEL_USERNAME=$(ITEST_USER) KERNEL_LAUNCH_TIMEOUT=$(ITEST_KERNEL_LAUNCH_TIMEOUT) SPARK_VERSION=$(SPARK_VERSION) ITEST_HOSTNAME_PREFIX=$(ITEST_HOSTNAME_PREFIX) nosetests -v $(TEST_DEBUG_OPTS) $(ITEST_YARN_TESTS))
+	($(SA) $(ENV) && GATEWAY_HOST=$(ITEST_YARN_HOST) LOG_LEVEL=$(LOG_LEVEL) KERNEL_USERNAME=$(ITEST_USER) KERNEL_LAUNCH_TIMEOUT=$(ITEST_KERNEL_LAUNCH_TIMEOUT) SPARK_VERSION=$(SPARK_VERSION) ITEST_HOSTNAME_PREFIX=$(ITEST_HOSTNAME_PREFIX) pytest -v -s $(TEST_DEBUG_OPTS) $(ITEST_YARN_TESTS))
 	@echo "Run \`docker logs itest-yarn\` to see enterprise-gateway log."
 
 PREP_TIMEOUT?=60
@@ -201,7 +201,7 @@ itest-yarn-prep:
 # This should get cleaned up once docker support is more mature
 ITEST_DOCKER_PORT?=8889
 ITEST_DOCKER_HOST?=localhost:$(ITEST_DOCKER_PORT)
-ITEST_DOCKER_TESTS?=enterprise_gateway.itests.test_r_kernel.TestRKernelLocal enterprise_gateway.itests.test_python_kernel.TestPythonKernelLocal enterprise_gateway.itests.test_scala_kernel.TestScalaKernelLocal
+ITEST_DOCKER_TESTS?=enterprise_gateway/itests/test_r_kernel.py::TestRKernelLocal enterprise_gateway/itests/test_python_kernel.py::TestPythonKernelLocal enterprise_gateway/itests/test_scala_kernel.py::TestScalaKernelLocal
 ITEST_DOCKER_KERNELS=PYTHON_KERNEL_LOCAL_NAME=python_docker SCALA_KERNEL_LOCAL_NAME=scala_docker R_KERNEL_LOCAL_NAME=R_docker
 
 itest-docker-debug: ## Run integration tests (optionally) against docker container with print statements
@@ -212,7 +212,7 @@ itest-docker: ## Run integration tests (optionally) against docker swarm
 ifeq (1, $(PREP_ITEST_DOCKER))
 	make itest-docker-prep
 endif
-	($(SA) $(ENV) && GATEWAY_HOST=$(ITEST_DOCKER_HOST) LOG_LEVEL=$(LOG_LEVEL) KERNEL_USERNAME=$(ITEST_USER) KERNEL_LAUNCH_TIMEOUT=$(ITEST_KERNEL_LAUNCH_TIMEOUT) $(ITEST_DOCKER_KERNELS) nosetests -v $(TEST_DEBUG_OPTS) $(ITEST_DOCKER_TESTS))
+	($(SA) $(ENV) && GATEWAY_HOST=$(ITEST_DOCKER_HOST) LOG_LEVEL=$(LOG_LEVEL) KERNEL_USERNAME=$(ITEST_USER) KERNEL_LAUNCH_TIMEOUT=$(ITEST_KERNEL_LAUNCH_TIMEOUT) $(ITEST_DOCKER_KERNELS) pytest -v -s $(TEST_DEBUG_OPTS) $(ITEST_DOCKER_TESTS))
 	@echo "Run \`docker service logs itest-docker\` to see enterprise-gateway log."
 
 PREP_TIMEOUT?=60

enterprise_gateway/base/handlers.py

@@ -3,8 +3,8 @@
 """Tornado handlers for the base of the API."""
 
 import json
-import notebook._version
-from notebook.base.handlers import APIHandler
+import jupyter_server._version
+from jupyter_server.base.handlers import APIHandler
 from tornado import web
 from ..mixins import TokenAuthorizationMixin, CORSMixin, JSONErrorsMixin
 from .._version import __version__
@@ -15,14 +15,14 @@ class APIVersionHandler(TokenAuthorizationMixin,
                         JSONErrorsMixin,
                         APIHandler):
     """"
-    Extends the notebook server base API handler with token auth, CORS, and
-    JSON errors to produce version information for notebook and gateway.
+    Extends the jupyter_server base API handler with token auth, CORS, and
+    JSON errors to produce version information for jupyter_server and gateway.
     """
     def get(self):
         # not authenticated, so give as few info as possible
-        # to be backwards compatibile, use only 'version' for the notebook version
+        # to be backwards compatibile, use only 'version' for the jupyter_server version
         # and be more specific for gateway_version
-        self.finish(json.dumps({"version": notebook.__version__, "gateway_version": __version__}))
+        self.finish(json.dumps({"version": jupyter_server.__version__, "gateway_version": __version__}))
 
 
 class NotFoundHandler(JSONErrorsMixin, web.RequestHandler):

enterprise_gateway/enterprisegatewayapp.py

@@ -14,20 +14,16 @@
 import time
 import weakref
 
-# Install the pyzmq ioloop. This has to be done before anything else from
-# tornado is imported.
 from zmq.eventloop import ioloop
-ioloop.install()
-
 from tornado import httpserver
 from tornado import web
 from tornado.log import enable_pretty_logging
 
 from traitlets.config import Configurable
 from jupyter_core.application import JupyterApp, base_aliases
 from jupyter_client.kernelspec import KernelSpecManager
-from notebook.notebookapp import random_ports
-from notebook.utils import url_path_join
+from jupyter_server.serverapp import random_ports
+from jupyter_server.utils import url_path_join
 
 from ._version import __version__
 
@@ -175,7 +171,7 @@ def init_webapp(self):
         Adds the various managers and web-front configuration values to the
         Tornado settings for reference by the handlers.
         """
-        # Enable the same pretty logging the notebook uses
+        # Enable the same pretty logging the server uses
         enable_pretty_logging()
 
         # Configure the tornado logging level too
@@ -204,7 +200,7 @@ def init_webapp(self):
             eg_list_kernels=self.list_kernels,
             eg_authorized_users=self.authorized_users,
             eg_unauthorized_users=self.unauthorized_users,
-            # Also set the allow_origin setting used by notebook so that the
+            # Also set the allow_origin setting used by jupyter_server so that the
             # check_origin method used everywhere respects the value
             allow_origin=self.allow_origin,
             # Always allow remote access (has been limited to localhost >= notebook 5.6)

enterprise_gateway/services/api/handlers.py

@@ -3,8 +3,8 @@
 """Tornado handlers for kernel specs."""
 import os
 
-from notebook.utils import maybe_future
-from tornado import gen, web
+from jupyter_server.utils import ensure_async
+from tornado import web
 from ...mixins import CORSMixin
 
 
@@ -25,14 +25,13 @@ def initialize(self):
         """
         web.StaticFileHandler.initialize(self, path=os.path.dirname(__file__))
 
-    @gen.coroutine
-    def get(self):
+    async def get(self):
         """Handler for a get on a specific handler
         """
         resource_name, content_type = self.get_resource_metadata()
         self.set_header('Content-Type', content_type)
         res = web.StaticFileHandler.get(self, resource_name)
-        yield maybe_future(res)
+        await ensure_async(res)
 
     def options(self, **kwargs):
         """Method for properly handling CORS pre-flight"""

enterprise_gateway/services/kernels/handlers.py

@@ -5,18 +5,18 @@
 import os
 
 import tornado
-import notebook.services.kernels.handlers as notebook_handlers
+import jupyter_server.services.kernels.handlers as jupyter_server_handlers
 from jupyter_client.jsonutil import date_default
-from tornado import gen, web
+from tornado import web
 from functools import partial
 from ...mixins import TokenAuthorizationMixin, CORSMixin, JSONErrorsMixin
 
 
 class MainKernelHandler(TokenAuthorizationMixin,
                         CORSMixin,
                         JSONErrorsMixin,
-                        notebook_handlers.MainKernelHandler):
-    """Extends the notebook main kernel handler with token auth, CORS, and
+                        jupyter_server_handlers.MainKernelHandler):
+    """Extends the jupyter_server main kernel handler with token auth, CORS, and
     JSON errors.
     """
 
@@ -28,8 +28,7 @@ def env_whitelist(self):
     def env_process_whitelist(self):
         return self.settings['eg_env_process_whitelist']
 
-    @gen.coroutine
-    def post(self):
+    async def post(self):
         """Overrides the super class method to manage env in the request body.
 
         Max kernel limits are now enforced in RemoteMappingKernelManager.start_kernel().
@@ -88,13 +87,13 @@ def post(self):
                                                        env=env,
                                                        kernel_headers=kernel_headers)
             try:
-                yield super(MainKernelHandler, self).post()
+                await super(MainKernelHandler, self).post()
             finally:
                 self.kernel_manager.start_kernel = orig_start
         else:
-            yield super(MainKernelHandler, self).post()
+            await super(MainKernelHandler, self).post()
 
-    def get(self):
+    async def get(self):
         """Overrides the super class method to honor the kernel listing
         configuration setting.
 
@@ -108,7 +107,7 @@ def get(self):
         if not self.settings.get('eg_list_kernels'):
             raise tornado.web.HTTPError(403, 'Forbidden')
         else:
-            super(MainKernelHandler, self).get()
+            await super(MainKernelHandler, self).get()
 
     def options(self, **kwargs):
         """Method for properly handling CORS pre-flight"""
@@ -118,8 +117,8 @@ def options(self, **kwargs):
 class KernelHandler(TokenAuthorizationMixin,
                     CORSMixin,
                     JSONErrorsMixin,
-                    notebook_handlers.KernelHandler):
-    """Extends the notebook kernel handler with token auth, CORS, and
+                    jupyter_server_handlers.KernelHandler):
+    """Extends the jupyter_server kernel handler with token auth, CORS, and
     JSON errors.
     """
 
@@ -136,7 +135,7 @@ def get(self, kernel_id):
 
 
 default_handlers = []
-for path, cls in notebook_handlers.default_handlers:
+for path, cls in jupyter_server_handlers.default_handlers:
     if cls.__name__ in globals():
         # Use the same named class from here if it exists
         default_handlers.append((path, globals()[cls.__name__]))

enterprise_gateway/services/kernels/remotemanager.py

@@ -10,7 +10,7 @@
 from tornado import web
 from ipython_genutils.py3compat import unicode_type
 from ipython_genutils.importstring import import_item
-from notebook.services.kernels.kernelmanager import AsyncMappingKernelManager
+from jupyter_server.services.kernels.kernelmanager import AsyncMappingKernelManager
 from jupyter_client.ioloop.manager import AsyncIOLoopKernelManager
 from traitlets import directional_link, log as traitlets_log
 

enterprise_gateway/services/kernelspecs/handlers.py

@@ -3,9 +3,9 @@
 """Tornado handlers for kernel specs."""
 import json
 
-from notebook.base.handlers import IPythonHandler
-from notebook.services.kernelspecs.handlers import is_kernelspec_model, kernelspec_model
-from notebook.utils import maybe_future, url_unescape
+from jupyter_server.base.handlers import JupyterHandler
+from jupyter_server.services.kernelspecs.handlers import is_kernelspec_model, kernelspec_model
+from jupyter_server.utils import ensure_async, url_unescape
 from tornado import web
 from ...base.handlers import APIHandler
 from ...mixins import TokenAuthorizationMixin, CORSMixin, JSONErrorsMixin
@@ -75,7 +75,7 @@ async def get(self):
             if kernel_user:
                 self.log.debug("Searching kernels for user '%s' " % kernel_user)
 
-        kspecs = await maybe_future(ksm.get_all_specs())
+        kspecs = await ensure_async(ksm.get_all_specs())
 
         list_kernels_found = []
         for kernel_name, kernel_info in kspecs.items():
@@ -117,7 +117,7 @@ async def get(self, kernel_name):
         if kernel_user_filter:
             kernel_user = kernel_user_filter[0].decode("utf-8")
         try:
-            spec = await maybe_future(ksm.get_kernel_spec(kernel_name))
+            spec = await ensure_async(ksm.get_kernel_spec(kernel_name))
         except KeyError:
             raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name)
         if is_kernelspec_model(spec):
@@ -139,7 +139,7 @@ class KernelSpecResourceHandler(TokenAuthorizationMixin,
                                 CORSMixin,
                                 JSONErrorsMixin,
                                 web.StaticFileHandler,
-                                IPythonHandler):
+                                JupyterHandler):
     SUPPORTED_METHODS = ('GET', 'HEAD')
 
     @property
@@ -153,7 +153,7 @@ def initialize(self):
     async def get(self, kernel_name, path, include_body=True):
         ksm = self.kernel_spec_cache
         try:
-            kernelspec = await maybe_future(ksm.get_kernel_spec(kernel_name))
+            kernelspec = await ensure_async(ksm.get_kernel_spec(kernel_name))
             self.root = kernelspec.resource_dir
         except KeyError as e:
             raise web.HTTPError(404,
@@ -168,7 +168,7 @@ def head(self, kernel_name, path):
 
 kernel_name_regex = r"(?P<kernel_name>[\w\.\-%]+)"
 
-# Extends the default handlers from the notebook package with token auth, CORS
+# Extends the default handlers from the jupyter_server package with token auth, CORS
 # and JSON errors.
 default_handlers = [
     (r"/api/kernelspecs", MainKernelSpecHandler),

enterprise_gateway/services/kernelspecs/kernelspec_cache.py

@@ -7,7 +7,7 @@
 from watchdog.events import FileSystemEventHandler, FileMovedEvent
 
 from jupyter_client.kernelspec import KernelSpec
-from notebook.utils import maybe_future
+from jupyter_server.utils import ensure_async
 from traitlets.config import SingletonConfigurable
 from traitlets.traitlets import CBool, default
 from typing import Dict, Optional, Union
@@ -56,7 +56,7 @@ async def get_kernel_spec(self, kernel_name: str) -> KernelSpec:
         """
         kernelspec = self.get_item(kernel_name)
         if not kernelspec:
-            kernelspec = await maybe_future(self.kernel_spec_manager.get_kernel_spec(kernel_name))
+            kernelspec = await ensure_async(self.kernel_spec_manager.get_kernel_spec(kernel_name))
             if kernelspec:
                 self.put_item(kernel_name, kernelspec)
         return kernelspec
@@ -77,7 +77,7 @@ async def get_all_specs(self) -> Dict[str, CacheItemType]:
         """
         kernelspecs = self.get_all_items()
         if not kernelspecs:
-            kernelspecs = await maybe_future(self.kernel_spec_manager.get_all_specs())
+            kernelspecs = await ensure_async(self.kernel_spec_manager.get_all_specs())
             if kernelspecs:
                 self.put_all_items(kernelspecs)
         return kernelspecs
@@ -182,8 +182,8 @@ def _initialize(self):
                         self.observed_dirs.add(kernel_dir)
                         self.observer.schedule(KernelSpecChangeHandler(self), kernel_dir, recursive=True)
                     else:
-                        self.log.warn("KernelSpecCache: kernel_dir '{kernel_dir}' does not exist"
-                                      " and will not be observed.".format(kernel_dir=kernel_dir))
+                        self.log.warning("KernelSpecCache: kernel_dir '{kernel_dir}' does not exist"
+                                         " and will not be observed.".format(kernel_dir=kernel_dir))
             self.observer.start()
 
     @staticmethod

enterprise_gateway/services/processproxies/conductor.py

@@ -15,7 +15,7 @@
 
 from .processproxy import RemoteProcessProxy
 
-from notebook.utils import url_unescape
+from jupyter_server.utils import url_unescape
 
 from random import randint
 

enterprise_gateway/services/processproxies/processproxy.py

@@ -25,7 +25,7 @@
 from calendar import timegm
 from ipython_genutils.py3compat import with_metaclass
 from jupyter_client import launch_kernel, localinterfaces
-from notebook import _tz
+from jupyter_server import _tz
 from zmq.ssh import tunnel
 
 try: