diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d5fe74ebf..6193fc09e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,12 +6,15 @@ on: branches: '*' jobs: build: - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [ '3.6', '3.7', '3.8' ] + os: [ubuntu, macos, windows] + python-version: [ '3.6', '3.7', '3.8', '3.9', 'pypy3' ] + exclude: + - os: windows + python-version: pypy3 steps: - name: Checkout uses: actions/checkout@v1 @@ -20,15 +23,37 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: 'x64' + - name: Upgrade packaging dependencies + run: | + pip install --upgrade pip setuptools wheel + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Cache pip + uses: actions/cache@v1 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('setup.py') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + ${{ runner.os }}-pip- - name: Install the Python dependencies run: | - pip install -e .[test] + pip install -e .[test] codecov + - name: List installed packages + run: | + pip freeze + pip check - name: Run the tests run: | - pytest + pytest -vv --cov jupyter_server --cov-branch --cov-report term-missing:skip-covered - name: Install the Python dependencies for the examples run: | cd examples/simple && pip install -e . - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py + - name: Coverage + run: | + codecov diff --git a/MANIFEST.in b/MANIFEST.in index 3d1d1d6cca..e8b3aad87e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,7 +6,7 @@ include CHANGELOG.md include setupbase.py # include everything in package_data -include jupyter_server/**/* +recursive-include jupyter_server * # Documentation graft docs @@ -26,3 +26,5 @@ global-exclude *.pyc global-exclude *.pyo global-exclude .git global-exclude .ipynb_checkpoints +global-exclude .pytest_cache +global-exclude .coverage diff --git a/examples/simple/README.md b/examples/simple/README.md index 3acfc4f7d0..f41eb5bb29 100644 --- a/examples/simple/README.md +++ b/examples/simple/README.md @@ -12,7 +12,7 @@ git clone https://github.com/jupyter/jupyter_server && \ cd examples/simple && \ conda create -y -n jupyter-server-example python=3.7 && \ conda activate jupyter-server-example && \ - pip install -e . + pip install -e .[test] ``` **OPTIONAL** If you want to build the Typescript code, you need [npm](https://www.npmjs.com) on your local environement. Compiled javascript is provided as artifact in this repository, so this Typescript build step is optional. The Typescript source and configuration have been taken from https://github.com/markellekelly/jupyter-server-example. diff --git a/examples/simple/setup.py b/examples/simple/setup.py index 97b04bd2fa..9040c55e86 100755 --- a/examples/simple/setup.py +++ b/examples/simple/setup.py @@ -34,11 +34,14 @@ def add_data_files(path): version = VERSION, description = 'Jupyter Server Example', long_description = open('README.md').read(), - python_requires = '>=3.5', + python_requires = '>=3.6', install_requires = [ 'jupyter_server', 'jinja2', ], + extras_require = { + 'test': ['pytest-jupyter'], + }, include_package_data=True, cmdclass = cmdclass, entry_points = { @@ -52,4 +55,4 @@ def add_data_files(path): if __name__ == '__main__': - setup(**setup_args) \ No newline at end of file + setup(**setup_args) diff --git a/examples/simple/tests/test_handlers.py b/examples/simple/tests/test_handlers.py index 90ce37d672..7d231666bd 100644 --- a/examples/simple/tests/test_handlers.py +++ b/examples/simple/tests/test_handlers.py @@ -2,7 +2,7 @@ @pytest.fixture -def server_config(template_dir): +def jp_server_config(jp_template_dir): return { "ServerApp": { "jpserver_extensions": { @@ -12,8 +12,8 @@ def server_config(template_dir): } -async def test_handler_default(fetch): - r = await fetch( +async def test_handler_default(jp_fetch): + r = await jp_fetch( 'simple_ext1/default', method='GET' ) @@ -22,8 +22,8 @@ async def test_handler_default(fetch): assert r.body.decode().index('Hello Simple 1 - I am the default...') > -1 -async def test_handler_template(fetch): - r = await fetch( +async def test_handler_template(jp_fetch): + r = await jp_fetch( 'simple_ext1/template1/test', method='GET' ) diff --git a/jupyter_server/pytest_plugin.py b/jupyter_server/pytest_plugin.py deleted file mode 100644 index 00aea6bd71..0000000000 --- a/jupyter_server/pytest_plugin.py +++ /dev/null @@ -1,285 +0,0 @@ -import os -import sys -import json -import shutil -import pytest -import asyncio -from binascii import hexlify - -import urllib.parse -import tornado -from tornado.escape import url_escape - -from traitlets.config import Config - -import jupyter_core.paths -from jupyter_server.extension import serverextension -from jupyter_server.serverapp import ServerApp -from jupyter_server.utils import url_path_join -from jupyter_server.services.contents.filemanager import FileContentsManager - -import nbformat - -# This shouldn't be needed anymore, since pytest_tornasync is found in entrypoints -pytest_plugins = "pytest_tornasync" - -# NOTE: This is a temporary fix for Windows 3.8 -# We have to override the io_loop fixture with an -# asyncio patch. This will probably be removed in -# the future. - -@pytest.fixture -def asyncio_patch(): - ServerApp()._init_asyncio_patch() - -@pytest.fixture -def io_loop(asyncio_patch): - loop = tornado.ioloop.IOLoop() - loop.make_current() - yield loop - loop.clear_current() - loop.close(all_fds=True) - - -def mkdir(tmp_path, *parts): - path = tmp_path.joinpath(*parts) - if not path.exists(): - path.mkdir(parents=True) - return path - - -server_config = pytest.fixture(lambda: {}) -home_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "home")) -data_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "data")) -config_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "config")) -runtime_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "runtime")) -root_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "root_dir")) -template_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "templates")) -system_jupyter_path = pytest.fixture( - lambda tmp_path: mkdir(tmp_path, "share", "jupyter") -) -env_jupyter_path = pytest.fixture( - lambda tmp_path: mkdir(tmp_path, "env", "share", "jupyter") -) -system_config_path = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "etc", "jupyter")) -env_config_path = pytest.fixture( - lambda tmp_path: mkdir(tmp_path, "env", "etc", "jupyter") -) -some_resource = u"The very model of a modern major general" -sample_kernel_json = { - 'argv':['cat', '{connection_file}'], - 'display_name': 'Test kernel', -} -argv = pytest.fixture(lambda: []) - -@pytest.fixture -def environ( - monkeypatch, - tmp_path, - home_dir, - data_dir, - config_dir, - runtime_dir, - root_dir, - system_jupyter_path, - system_config_path, - env_jupyter_path, - env_config_path, -): - monkeypatch.setenv("HOME", str(home_dir)) - monkeypatch.setenv("PYTHONPATH", os.pathsep.join(sys.path)) - - # Get path to nbconvert template directory *before* - # monkeypatching the paths env variable. - possible_paths = jupyter_core.paths.jupyter_path('nbconvert', 'templates') - nbconvert_path = None - for path in possible_paths: - if os.path.exists(path): - nbconvert_path = path - break - - nbconvert_target = data_dir / 'nbconvert' / 'templates' - - # monkeypatch.setenv("JUPYTER_NO_CONFIG", "1") - monkeypatch.setenv("JUPYTER_CONFIG_DIR", str(config_dir)) - monkeypatch.setenv("JUPYTER_DATA_DIR", str(data_dir)) - monkeypatch.setenv("JUPYTER_RUNTIME_DIR", str(runtime_dir)) - monkeypatch.setattr( - jupyter_core.paths, "SYSTEM_JUPYTER_PATH", [str(system_jupyter_path)] - ) - monkeypatch.setattr(jupyter_core.paths, "ENV_JUPYTER_PATH", [str(env_jupyter_path)]) - monkeypatch.setattr( - jupyter_core.paths, "SYSTEM_CONFIG_PATH", [str(system_config_path)] - ) - monkeypatch.setattr(jupyter_core.paths, "ENV_CONFIG_PATH", [str(env_config_path)]) - - # copy nbconvert templates to new tmp data_dir. - if nbconvert_path: - shutil.copytree(nbconvert_path, str(nbconvert_target)) - - -@pytest.fixture -def extension_environ(env_config_path, monkeypatch): - """Monkeypatch a Jupyter Extension's config path into each test's environment variable""" - monkeypatch.setattr(serverextension, "ENV_CONFIG_PATH", [str(env_config_path)]) - - -@pytest.fixture(scope='function') -def configurable_serverapp( - environ, - server_config, - argv, - http_port, - tmp_path, - root_dir, - io_loop, -): - ServerApp.clear_instance() - - def _configurable_serverapp( - config=server_config, - argv=argv, - environ=environ, - http_port=http_port, - tmp_path=tmp_path, - root_dir=root_dir, - **kwargs - ): - c = Config(config) - c.NotebookNotary.db_file = ":memory:" - token = hexlify(os.urandom(4)).decode("ascii") - url_prefix = "/" - app = ServerApp.instance( - # Set the log level to debug for testing purposes - log_level='DEBUG', - port=http_port, - port_retries=0, - open_browser=False, - root_dir=str(root_dir), - base_url=url_prefix, - config=c, - allow_root=True, - token=token, - **kwargs - ) - - app.init_signal = lambda: None - app.log.propagate = True - app.log.handlers = [] - # Initialize app without httpserver - app.initialize(argv=argv, new_httpserver=False) - app.log.propagate = True - app.log.handlers = [] - # Start app without ioloop - app.start_app() - return app - - return _configurable_serverapp - - -@pytest.fixture(scope="function") -def serverapp(server_config, argv, configurable_serverapp): - app = configurable_serverapp(config=server_config, argv=argv) - yield app - app.remove_server_info_file() - app.remove_browser_open_file() - app.cleanup_kernels() - - -@pytest.fixture -def app(serverapp): - """app fixture is needed by pytest_tornasync plugin""" - return serverapp.web_app - - -@pytest.fixture -def auth_header(serverapp): - return {"Authorization": "token {token}".format(token=serverapp.token)} - - -@pytest.fixture -def http_port(http_server_port): - return http_server_port[-1] - - -@pytest.fixture -def base_url(): - return "/" - - -@pytest.fixture -def fetch(http_server_client, auth_header, base_url): - """fetch fixture that handles auth, base_url, and path""" - def client_fetch(*parts, headers={}, params={}, **kwargs): - # Handle URL strings - path_url = url_escape(url_path_join(base_url, *parts), plus=False) - params_url = urllib.parse.urlencode(params) - url = path_url + "?" + params_url - # Add auth keys to header - headers.update(auth_header) - # Make request. - return http_server_client.fetch( - url, headers=headers, request_timeout=20, **kwargs - ) - return client_fetch - - -@pytest.fixture -def ws_fetch(auth_header, http_port): - """websocket fetch fixture that handles auth, base_url, and path""" - def client_fetch(*parts, headers={}, params={}, **kwargs): - # Handle URL strings - path = url_escape(url_path_join(*parts), plus=False) - urlparts = urllib.parse.urlparse('ws://localhost:{}'.format(http_port)) - urlparts = urlparts._replace( - path=path, - query=urllib.parse.urlencode(params) - ) - url = urlparts.geturl() - # Add auth keys to header - headers.update(auth_header) - # Make request. - req = tornado.httpclient.HTTPRequest( - url, - headers=auth_header, - connect_timeout=120 - ) - return tornado.websocket.websocket_connect(req) - return client_fetch - - -@pytest.fixture -def kernelspecs(data_dir): - spec_names = ['sample', 'sample 2'] - for name in spec_names: - sample_kernel_dir = data_dir.joinpath('kernels', name) - sample_kernel_dir.mkdir(parents=True) - # Create kernel json file - sample_kernel_file = sample_kernel_dir.joinpath('kernel.json') - sample_kernel_file.write_text(json.dumps(sample_kernel_json)) - # Create resources text - sample_kernel_resources = sample_kernel_dir.joinpath('resource.txt') - sample_kernel_resources.write_text(some_resource) - - -@pytest.fixture(params=[True, False]) -def contents_manager(request, tmp_path): - return FileContentsManager(root_dir=str(tmp_path), use_atomic_writing=request.param) - - -@pytest.fixture -def create_notebook(root_dir): - """Create a notebook in the test's home directory.""" - def inner(nbpath): - nbpath = root_dir.joinpath(nbpath) - # Check that the notebook has the correct file extension. - if nbpath.suffix != '.ipynb': - raise Exception("File extension for notebook must be .ipynb") - # If the notebook path has a parent directory, make sure it's created. - parent = nbpath.parent - parent.mkdir(parents=True, exist_ok=True) - # Create a notebook string and write to file. - nb = nbformat.v4.new_notebook() - nbtext = nbformat.writes(nb, version=4) - nbpath.write_text(nbtext) - return inner diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 0386e1bcc3..13bc0940be 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -41,17 +41,16 @@ from jupyter_server.transutils import trans, _ from jupyter_server.utils import secure_write, run_sync -# check for tornado 3.1.0 +# the minimum viable tornado version: needs to be kept in sync with setup.py +MIN_TORNADO = (6, 1, 0) + try: import tornado -except ImportError as e: - raise ImportError(_("The Jupyter Server requires tornado >= 4.0")) from e -try: - version_info = tornado.version_info -except AttributeError as e: - raise ImportError(_("The Jupyter Server requires tornado >= 4.0, but you have < 1.1.0")) from e -if version_info < (4,0): - raise ImportError(_("The Jupyter Server requires tornado >= 4.0, but you have %s") % tornado.version) + assert tornado.version_info >= MIN_TORNADO +except (ImportError, AttributeError, AssertionError) as e: # pragma: no cover + raise ImportError( + _("The Jupyter Server requires tornado >=%s.%s.%s") % MIN_TORNADO + ) from e from tornado import httpserver from tornado import ioloop @@ -1604,31 +1603,12 @@ def init_httpserver(self): @staticmethod def _init_asyncio_patch(): - """set default asyncio policy to be compatible with tornado - Tornado 6 (at least) is not compatible with the default - asyncio implementation on Windows - Pick the older SelectorEventLoopPolicy on Windows - if the known-incompatible default policy is in use. - do this as early as possible to make it a low priority and overrideable - ref: https://github.com/tornadoweb/tornado/issues/2608 - FIXME: if/when tornado supports the defaults in asyncio, - remove and bump tornado requirement for py38 - """ - if sys.platform.startswith("win") and sys.version_info >= (3, 8): - import asyncio - try: - from asyncio import ( - WindowsProactorEventLoopPolicy, - WindowsSelectorEventLoopPolicy, - ) - except ImportError: - pass - # not affected - else: - if type(asyncio.get_event_loop_policy()) is WindowsProactorEventLoopPolicy: - # WindowsProactorEventLoopPolicy is not compatible with tornado 6 - # fallback to the pre-3.8 default of Selector - asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) + """no longer needed with tornado 6.1""" + warnings.warn( + """ServerApp._init_asyncio_patch called, and is longer needed for """ + """tornado 6.1+, and will be removed in a future release.""", + DeprecationWarning + ) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True): @@ -1648,7 +1628,6 @@ def initialize(self, argv=None, find_extensions=True, new_httpserver=True): If True, a tornado HTTPServer instance will be created and configured for the Server Web Application. This will set the http_server attribute of this class. """ - self._init_asyncio_patch() # Parse command line, load ServerApp config files, # and update ServerApp config. super(ServerApp, self).initialize(argv) diff --git a/setup.py b/setup.py index f52748983c..c796720558 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ ], install_requires = [ 'jinja2', - 'tornado>=5.0', + 'tornado>=6.1.0', 'pyzmq>=17', 'ipython_genutils', 'traitlets>=4.2.1', @@ -48,7 +48,7 @@ 'Send2Trash', 'terminado>=0.8.3', 'prometheus_client', - "pywin32>=1.0 ; sys_platform == 'win32'" + "pywin32>=1.0 ; sys_platform == 'win32'", ], extras_require = { 'test': ['coverage', 'requests', @@ -59,9 +59,6 @@ entry_points = { 'console_scripts': [ 'jupyter-server = jupyter_server.serverapp:main', - ], - 'pytest11': [ - 'pytest_jupyter_server = jupyter_server.pytest_plugin' ] }, ) diff --git a/tests/test_files.py b/tests/test_files.py index 7391275357..a1c3872354 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -1,5 +1,6 @@ import os import pytest +from pathlib import Path import tornado from .utils import expected_http_error @@ -10,68 +11,47 @@ new_output) -async def test_hidden_files(jp_fetch, jp_serverapp, jp_root_dir): - not_hidden = [ - u'å b', - u'å b/ç. d', - ] - hidden = [ - u'.å b', - u'å b/.ç d', - ] - dirs = not_hidden + hidden - - for d in dirs: - path = jp_root_dir / d.replace('/', os.sep) - path.mkdir(parents=True, exist_ok=True) - path.joinpath('foo').write_text('foo') - path.joinpath('.foo').write_text('.foo') - - for d in not_hidden: - r = await jp_fetch( - 'files', d, 'foo', - method='GET' - ) - assert r.body.decode() == 'foo' - - with pytest.raises(tornado.httpclient.HTTPClientError) as e: - await jp_fetch( - 'files', d, '.foo', - method='GET' - ) - assert expected_http_error(e, 404) - - for d in hidden: - for foo in ('foo', '.foo'): - with pytest.raises(tornado.httpclient.HTTPClientError) as e: - await jp_fetch( - 'files', d, foo, - method='GET' - ) - assert expected_http_error(e, 404) +@pytest.fixture(params=[ + [False, ['å b']], + [False, ['å b', 'ç. d']], + [True, ['.å b']], + [True, ['å b', '.ç d']] +]) +def maybe_hidden(request): + return request.param + + +async def fetch_expect_200(jp_fetch, *path_parts): + r = await jp_fetch('files', *path_parts, method='GET') + assert (r.body.decode() == path_parts[-1]), (path_parts, r.body) + + +async def fetch_expect_404(jp_fetch, *path_parts): + with pytest.raises(tornado.httpclient.HTTPClientError) as e: + await jp_fetch('files', *path_parts, method='GET') + assert expected_http_error(e, 404), [path_parts, e] + + +async def test_hidden_files(jp_fetch, jp_serverapp, jp_root_dir, maybe_hidden): + is_hidden, path_parts = maybe_hidden + path = Path(jp_root_dir, *path_parts) + path.mkdir(parents=True, exist_ok=True) + + foos = ['foo', '.foo'] + for foo in foos: + (path / foo).write_text(foo) + + if is_hidden: + for foo in foos: + await fetch_expect_404(jp_fetch, *path_parts, foo) + else: + await fetch_expect_404(jp_fetch, *path_parts, '.foo') + await fetch_expect_200(jp_fetch, *path_parts, 'foo') jp_serverapp.contents_manager.allow_hidden = True - for d in not_hidden: - r = await jp_fetch( - 'files', d, 'foo', - method='GET' - ) - assert r.body.decode() == 'foo' - - r = await jp_fetch( - 'files', d, '.foo', - method='GET' - ) - assert r.body.decode() == '.foo' - - for d in hidden: - for foo in ('foo', '.foo'): - r = await jp_fetch( - 'files', d, foo, - method='GET' - ) - assert r.body.decode() == foo + for foo in foos: + await fetch_expect_200(jp_fetch, *path_parts, foo) async def test_contents_manager(jp_fetch, jp_serverapp, jp_root_dir):