From 36173446ae50d8926eabe9e1abc991d1545da336 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 00:22:56 -0500 Subject: [PATCH 01/26] remote pytest11 entrypoint --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 49598d707a..502ded3afd 100644 --- a/setup.py +++ b/setup.py @@ -60,9 +60,6 @@ entry_points = { 'console_scripts': [ 'jupyter-server = jupyter_server.serverapp:main', - ], - 'pytest11': [ - 'pytest_jupyter_server = jupyter_server.pytest_plugin' ] }, ) From ec6a9c666bc393a62d49713113dd5a4dd373a65b Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 00:29:25 -0500 Subject: [PATCH 02/26] try dotted import --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9d694425b2..81b9dbfe6e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1 @@ -pytest_plugins = ['pytest_jupyter_server'] \ No newline at end of file +pytest_plugins = ['jupyter_server.pytest_plugin'] From f35144d9359f59d2f3ae814d4cc4ede81a37a567 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 01:03:45 -0500 Subject: [PATCH 03/26] add conftest to example --- examples/simple/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 examples/simple/tests/conftest.py diff --git a/examples/simple/tests/conftest.py b/examples/simple/tests/conftest.py new file mode 100644 index 0000000000..81b9dbfe6e --- /dev/null +++ b/examples/simple/tests/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ['jupyter_server.pytest_plugin'] From d7709fa25f6c5b38e6e94a96f7ecc1b84da519e0 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 17:58:21 -0500 Subject: [PATCH 04/26] remove un-needed conftest.py --- examples/simple/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 examples/simple/tests/conftest.py diff --git a/examples/simple/tests/conftest.py b/examples/simple/tests/conftest.py deleted file mode 100644 index 81b9dbfe6e..0000000000 --- a/examples/simple/tests/conftest.py +++ /dev/null @@ -1 +0,0 @@ -pytest_plugins = ['jupyter_server.pytest_plugin'] From 2a3a2a0a469dffe44df7278354eaf2e11f488d85 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 18:00:41 -0500 Subject: [PATCH 05/26] tweak some metadata --- examples/simple/setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/simple/setup.py b/examples/simple/setup.py index 97b04bd2fa..9040c55e86 100755 --- a/examples/simple/setup.py +++ b/examples/simple/setup.py @@ -34,11 +34,14 @@ def add_data_files(path): version = VERSION, description = 'Jupyter Server Example', long_description = open('README.md').read(), - python_requires = '>=3.5', + python_requires = '>=3.6', install_requires = [ 'jupyter_server', 'jinja2', ], + extras_require = { + 'test': ['pytest-jupyter'], + }, include_package_data=True, cmdclass = cmdclass, entry_points = { @@ -52,4 +55,4 @@ def add_data_files(path): if __name__ == '__main__': - setup(**setup_args) \ No newline at end of file + setup(**setup_args) From b96c939d97ac721dd191c573ede5dee3d6d71dbb Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 18:04:41 -0500 Subject: [PATCH 06/26] update fixtures in example --- examples/simple/tests/test_handlers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/simple/tests/test_handlers.py b/examples/simple/tests/test_handlers.py index 90ce37d672..47e42c7a54 100644 --- a/examples/simple/tests/test_handlers.py +++ b/examples/simple/tests/test_handlers.py @@ -2,7 +2,7 @@ @pytest.fixture -def server_config(template_dir): +def server_config(jp_template_dir): return { "ServerApp": { "jpserver_extensions": { @@ -12,7 +12,7 @@ def server_config(template_dir): } -async def test_handler_default(fetch): +async def test_handler_default(jp_fetch): r = await fetch( 'simple_ext1/default', method='GET' @@ -22,7 +22,7 @@ async def test_handler_default(fetch): assert r.body.decode().index('Hello Simple 1 - I am the default...') > -1 -async def test_handler_template(fetch): +async def test_handler_template(jp_fetch): r = await fetch( 'simple_ext1/template1/test', method='GET' From 933b4f55bd93a5b7018484878e7ed224b577d899 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 19:59:30 -0500 Subject: [PATCH 07/26] actually use renamed fixtures --- examples/simple/tests/test_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple/tests/test_handlers.py b/examples/simple/tests/test_handlers.py index 47e42c7a54..76fbc1a07d 100644 --- a/examples/simple/tests/test_handlers.py +++ b/examples/simple/tests/test_handlers.py @@ -13,7 +13,7 @@ def server_config(jp_template_dir): async def test_handler_default(jp_fetch): - r = await fetch( + r = await jp_fetch( 'simple_ext1/default', method='GET' ) @@ -23,7 +23,7 @@ async def test_handler_default(jp_fetch): async def test_handler_template(jp_fetch): - r = await fetch( + r = await jp_fetch( 'simple_ext1/template1/test', method='GET' ) From 20ec9ecfffd62ce8708f60dadc5833dbd3549b35 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 20:23:28 -0500 Subject: [PATCH 08/26] add some more ci trappings --- .github/workflows/main.yml | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d5fe74ebf..a287ede5ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,12 +6,12 @@ on: branches: '*' jobs: build: - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [ '3.6', '3.7', '3.8' ] + os: [ubuntu, macos, windows] + python-version: [ '3.6', '3.7', '3.8', '3.9', 'pypy3' ] steps: - name: Checkout uses: actions/checkout@v1 @@ -20,9 +20,28 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: 'x64' + - name: Upgrade packaging dependencies + run: | + pip install --upgrade pip setuptools wheel + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Cache pip + uses: actions/cache@v1 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('setup.py') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + ${{ runner.os }}-pip- - name: Install the Python dependencies run: | - pip install -e .[test] + pip install -e .[test] codecov + - name: List installed packages + run: | + pip freeze + pip check - name: Run the tests run: | pytest @@ -32,3 +51,6 @@ jobs: - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py + - name: Coverage + run: | + codecov From d925e670e0a904a8b75d2412056b951e924bc2c7 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 20:30:41 -0500 Subject: [PATCH 09/26] recursive-include to grab spec.yaml --- MANIFEST.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 3d1d1d6cca..e8b3aad87e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,7 +6,7 @@ include CHANGELOG.md include setupbase.py # include everything in package_data -include jupyter_server/**/* +recursive-include jupyter_server * # Documentation graft docs @@ -26,3 +26,5 @@ global-exclude *.pyc global-exclude *.pyo global-exclude .git global-exclude .ipynb_checkpoints +global-exclude .pytest_cache +global-exclude .coverage From fd94f12cff0990b3ae3196907dab95eeb63422ef Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 20:59:41 -0500 Subject: [PATCH 10/26] inject server config properly --- .github/workflows/main.yml | 3 +++ examples/simple/README.md | 2 +- examples/simple/tests/test_handlers.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a287ede5ed..46d23804e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,9 @@ jobs: matrix: os: [ubuntu, macos, windows] python-version: [ '3.6', '3.7', '3.8', '3.9', 'pypy3' ] + exclude: + - os: windows + PYTHON_VERSION: pypy3 steps: - name: Checkout uses: actions/checkout@v1 diff --git a/examples/simple/README.md b/examples/simple/README.md index 3acfc4f7d0..f41eb5bb29 100644 --- a/examples/simple/README.md +++ b/examples/simple/README.md @@ -12,7 +12,7 @@ git clone https://github.com/jupyter/jupyter_server && \ cd examples/simple && \ conda create -y -n jupyter-server-example python=3.7 && \ conda activate jupyter-server-example && \ - pip install -e . + pip install -e .[test] ``` **OPTIONAL** If you want to build the Typescript code, you need [npm](https://www.npmjs.com) on your local environement. Compiled javascript is provided as artifact in this repository, so this Typescript build step is optional. The Typescript source and configuration have been taken from https://github.com/markellekelly/jupyter-server-example. diff --git a/examples/simple/tests/test_handlers.py b/examples/simple/tests/test_handlers.py index 76fbc1a07d..7d231666bd 100644 --- a/examples/simple/tests/test_handlers.py +++ b/examples/simple/tests/test_handlers.py @@ -2,7 +2,7 @@ @pytest.fixture -def server_config(jp_template_dir): +def jp_server_config(jp_template_dir): return { "ServerApp": { "jpserver_extensions": { From 3c99dff0e79f4cb0a7d42e4fc861d7e05adce088 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 21:00:42 -0500 Subject: [PATCH 11/26] fix workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 46d23804e5..b43089a30f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: python-version: [ '3.6', '3.7', '3.8', '3.9', 'pypy3' ] exclude: - os: windows - PYTHON_VERSION: pypy3 + python-version: pypy3 steps: - name: Checkout uses: actions/checkout@v1 From 02579a7304ea6c74d7402e692cb28c8d428b29bc Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 13 Nov 2020 21:06:16 -0500 Subject: [PATCH 12/26] collect coverage --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b43089a30f..2e1ba728e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,7 @@ jobs: pip check - name: Run the tests run: | - pytest + pytest -vv --cov=nbformat - name: Install the Python dependencies for the examples run: | cd examples/simple && pip install -e . From 47f07a495b8d0c940f249ee70b0d767fd72894c7 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 10:24:03 -0500 Subject: [PATCH 13/26] paramaterize (un)hidden file tests --- .github/workflows/main.yml | 2 +- tests/test_files.py | 98 +++++++++++++++----------------------- 2 files changed, 40 insertions(+), 60 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e1ba728e2..6193fc09e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,7 @@ jobs: pip check - name: Run the tests run: | - pytest -vv --cov=nbformat + pytest -vv --cov jupyter_server --cov-branch --cov-report term-missing:skip-covered - name: Install the Python dependencies for the examples run: | cd examples/simple && pip install -e . diff --git a/tests/test_files.py b/tests/test_files.py index 7391275357..a3037b3d9b 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -1,5 +1,6 @@ import os import pytest +from pathlib import Path import tornado from .utils import expected_http_error @@ -10,68 +11,47 @@ new_output) -async def test_hidden_files(jp_fetch, jp_serverapp, jp_root_dir): - not_hidden = [ - u'å b', - u'å b/ç. d', - ] - hidden = [ - u'.å b', - u'å b/.ç d', - ] - dirs = not_hidden + hidden - - for d in dirs: - path = jp_root_dir / d.replace('/', os.sep) - path.mkdir(parents=True, exist_ok=True) - path.joinpath('foo').write_text('foo') - path.joinpath('.foo').write_text('.foo') - - for d in not_hidden: - r = await jp_fetch( - 'files', d, 'foo', - method='GET' - ) - assert r.body.decode() == 'foo' - - with pytest.raises(tornado.httpclient.HTTPClientError) as e: - await jp_fetch( - 'files', d, '.foo', - method='GET' - ) - assert expected_http_error(e, 404) - - for d in hidden: - for foo in ('foo', '.foo'): - with pytest.raises(tornado.httpclient.HTTPClientError) as e: - await jp_fetch( - 'files', d, foo, - method='GET' - ) - assert expected_http_error(e, 404) +@pytest.fixture(params=[ + [False, ['å b']], + [False, ['å b', 'ç. d']], + [True, ['.å b']], + [True, ['å b', '.ç d']] +]) +def maybe_hidden(request): + return request.param + + +async def fetch_expect_200(jp_fetch, *path_parts): + r = await jp_fetch('files', *path_parts, method='GET') + assert (r.body.decode() == path_parts[-1]), path_parts + + +async def fetch_expect_404(jp_fetch, *path_parts): + with pytest.raises(tornado.httpclient.HTTPClientError) as e: + await jp_fetch('files', *path_parts, method='GET') + assert expected_http_error(e, 404), path_parts + + +async def test_hidden_files(jp_fetch, jp_serverapp, jp_root_dir, maybe_hidden): + is_hidden, path_parts = maybe_hidden + path = Path(jp_root_dir, *path_parts) + path.mkdir(parents=True, exist_ok=True) + + foos = ['foo', '.foo'] + for foo in foos: + (path / foo).write_text(foo) + + if is_hidden: + for foo in foos: + await fetch_expect_404(jp_fetch, *path_parts, foo) + else: + await fetch_expect_404(jp_fetch, *path_parts, '.foo') + await fetch_expect_200(jp_fetch, *path_parts, 'foo') jp_serverapp.contents_manager.allow_hidden = True - for d in not_hidden: - r = await jp_fetch( - 'files', d, 'foo', - method='GET' - ) - assert r.body.decode() == 'foo' - - r = await jp_fetch( - 'files', d, '.foo', - method='GET' - ) - assert r.body.decode() == '.foo' - - for d in hidden: - for foo in ('foo', '.foo'): - r = await jp_fetch( - 'files', d, foo, - method='GET' - ) - assert r.body.decode() == foo + for foo in foos: + await fetch_expect_200(jp_fetch, *path_parts, foo) async def test_contents_manager(jp_fetch, jp_serverapp, jp_root_dir): From ba40f81fb45da336f9ef5b3413203e555722e927 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 10:33:34 -0500 Subject: [PATCH 14/26] log some more --- tests/test_files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_files.py b/tests/test_files.py index a3037b3d9b..35da314e58 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -23,13 +23,14 @@ def maybe_hidden(request): async def fetch_expect_200(jp_fetch, *path_parts): r = await jp_fetch('files', *path_parts, method='GET') - assert (r.body.decode() == path_parts[-1]), path_parts + assert (r.body.decode() == path_parts[-1]), (path_parts, r.body) async def fetch_expect_404(jp_fetch, *path_parts): with pytest.raises(tornado.httpclient.HTTPClientError) as e: - await jp_fetch('files', *path_parts, method='GET') - assert expected_http_error(e, 404), path_parts + r = await jp_fetch('files', *path_parts, method='GET') + print(r.body) + assert expected_http_error(e, 404), [path_parts, e] async def test_hidden_files(jp_fetch, jp_serverapp, jp_root_dir, maybe_hidden): From d026e8357af6a668dc9ebcbe8d11b6ece97eafaa Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 11:25:57 -0500 Subject: [PATCH 15/26] try adding winpty --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8f2e4d2b5..ab1eea4af2 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,9 @@ 'Send2Trash', 'terminado>=0.8.3', 'prometheus_client', - "pywin32>=1.0 ; sys_platform == 'win32'" + "pywin32>=1.0 ; sys_platform == 'win32'", + # TODO: terminado needs to add this? + "pywinpt>=0.5 ; sys_platform == 'win32'" ], extras_require = { 'test': ['coverage', 'requests', From b86e72d1828a6282eb313473db1d8373d201f77b Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 11:35:40 -0500 Subject: [PATCH 16/26] fix pywinpty name --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab1eea4af2..3688dcd14b 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", # TODO: terminado needs to add this? - "pywinpt>=0.5 ; sys_platform == 'win32'" + "pywinpty>=0.5 ; sys_platform == 'win32'" ], extras_require = { 'test': ['coverage', 'requests', From 631e07553a692a128dc4808800a39bf52dfc4ac1 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 12:10:59 -0500 Subject: [PATCH 17/26] rever winpty, try longer request timeouts --- setup.py | 2 -- tests/test_files.py | 11 +++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 3688dcd14b..0fb3764929 100644 --- a/setup.py +++ b/setup.py @@ -49,8 +49,6 @@ 'terminado>=0.8.3', 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", - # TODO: terminado needs to add this? - "pywinpty>=0.5 ; sys_platform == 'win32'" ], extras_require = { 'test': ['coverage', 'requests', diff --git a/tests/test_files.py b/tests/test_files.py index 35da314e58..b53976eed6 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -21,14 +21,21 @@ def maybe_hidden(request): return request.param + +# double the defaults +TIMEOUTS = dict( + connect_timeout=40.0, + request_timeout=40.0 +) + async def fetch_expect_200(jp_fetch, *path_parts): - r = await jp_fetch('files', *path_parts, method='GET') + r = await jp_fetch('files', *path_parts, method='GET', **TIMEOUTS) assert (r.body.decode() == path_parts[-1]), (path_parts, r.body) async def fetch_expect_404(jp_fetch, *path_parts): with pytest.raises(tornado.httpclient.HTTPClientError) as e: - r = await jp_fetch('files', *path_parts, method='GET') + r = await jp_fetch('files', *path_parts, method='GET', **TIMEOUTS) print(r.body) assert expected_http_error(e, 404), [path_parts, e] From 0d0ac3c5badd2371ed06b55405b829668d50c607 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 12:13:53 -0500 Subject: [PATCH 18/26] just connect_timeout --- tests/test_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_files.py b/tests/test_files.py index b53976eed6..fd9f3c5097 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -25,7 +25,8 @@ def maybe_hidden(request): # double the defaults TIMEOUTS = dict( connect_timeout=40.0, - request_timeout=40.0 + # already set upstream? + # request_timeout=40.0 ) async def fetch_expect_200(jp_fetch, *path_parts): From 8694e53c58c32a787482750a6ff159b2dc57282f Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 12:19:25 -0500 Subject: [PATCH 19/26] try with unlimited connect_timeout --- tests/test_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_files.py b/tests/test_files.py index fd9f3c5097..281db480b3 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -22,9 +22,9 @@ def maybe_hidden(request): -# double the defaults TIMEOUTS = dict( - connect_timeout=40.0, + # default is 20.0 + connect_timeout=0.0, # already set upstream? # request_timeout=40.0 ) From 20816c428bbeff9626e6e13737c5062c5ce35eef Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 12:31:42 -0500 Subject: [PATCH 20/26] overload jp_fetch --- tests/test_files.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_files.py b/tests/test_files.py index 281db480b3..e9b4c5104a 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -25,10 +25,28 @@ def maybe_hidden(request): TIMEOUTS = dict( # default is 20.0 connect_timeout=0.0, - # already set upstream? - # request_timeout=40.0 + # needs patch below + request_timeout=0.0 ) +# shouldn't be overloading this +@pytest.fixture +def jp_fetch(jp_serverapp, http_server_client, jp_auth_header, jp_base_url): + def client_fetch(*parts, headers={}, params={}, **kwargs): + # Handle URL strings + path_url = url_escape(url_path_join(jp_base_url, *parts), plus=False) + params_url = urllib.parse.urlencode(params) + url = path_url + "?" + params_url + # Add auth keys to header + headers.update(jp_auth_header) + # Make request. + kwargs.setdefault("request_timeout", 20.0) + return http_server_client.fetch( + url, headers=headers, **kwargs + ) + return client_fetch + + async def fetch_expect_200(jp_fetch, *path_parts): r = await jp_fetch('files', *path_parts, method='GET', **TIMEOUTS) assert (r.body.decode() == path_parts[-1]), (path_parts, r.body) From 0e8a00fed71650169107ad884d3500502b0b48c0 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 13:16:56 -0500 Subject: [PATCH 21/26] fix imports --- tests/test_files.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_files.py b/tests/test_files.py index e9b4c5104a..268537d085 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -29,7 +29,12 @@ def maybe_hidden(request): request_timeout=0.0 ) -# shouldn't be overloading this +# HACK: shouldn't be overloading this + +from jupyter_server.utils import url_path_join +import urllib.parse +from tornado.escape import url_escape + @pytest.fixture def jp_fetch(jp_serverapp, http_server_client, jp_auth_header, jp_base_url): def client_fetch(*parts, headers={}, params={}, **kwargs): @@ -46,6 +51,7 @@ def client_fetch(*parts, headers={}, params={}, **kwargs): ) return client_fetch +# /HACK async def fetch_expect_200(jp_fetch, *path_parts): r = await jp_fetch('files', *path_parts, method='GET', **TIMEOUTS) From 480d6d693ddd5a4cc42543a7caf6ad74b7958bc9 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 13:32:30 -0500 Subject: [PATCH 22/26] try new tornado hotness --- jupyter_server/pytest_plugin.py | 285 -------------------------------- jupyter_server/serverapp.py | 51 +++--- setup.py | 2 +- 3 files changed, 27 insertions(+), 311 deletions(-) delete mode 100644 jupyter_server/pytest_plugin.py diff --git a/jupyter_server/pytest_plugin.py b/jupyter_server/pytest_plugin.py deleted file mode 100644 index 00aea6bd71..0000000000 --- a/jupyter_server/pytest_plugin.py +++ /dev/null @@ -1,285 +0,0 @@ -import os -import sys -import json -import shutil -import pytest -import asyncio -from binascii import hexlify - -import urllib.parse -import tornado -from tornado.escape import url_escape - -from traitlets.config import Config - -import jupyter_core.paths -from jupyter_server.extension import serverextension -from jupyter_server.serverapp import ServerApp -from jupyter_server.utils import url_path_join -from jupyter_server.services.contents.filemanager import FileContentsManager - -import nbformat - -# This shouldn't be needed anymore, since pytest_tornasync is found in entrypoints -pytest_plugins = "pytest_tornasync" - -# NOTE: This is a temporary fix for Windows 3.8 -# We have to override the io_loop fixture with an -# asyncio patch. This will probably be removed in -# the future. - -@pytest.fixture -def asyncio_patch(): - ServerApp()._init_asyncio_patch() - -@pytest.fixture -def io_loop(asyncio_patch): - loop = tornado.ioloop.IOLoop() - loop.make_current() - yield loop - loop.clear_current() - loop.close(all_fds=True) - - -def mkdir(tmp_path, *parts): - path = tmp_path.joinpath(*parts) - if not path.exists(): - path.mkdir(parents=True) - return path - - -server_config = pytest.fixture(lambda: {}) -home_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "home")) -data_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "data")) -config_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "config")) -runtime_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "runtime")) -root_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "root_dir")) -template_dir = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "templates")) -system_jupyter_path = pytest.fixture( - lambda tmp_path: mkdir(tmp_path, "share", "jupyter") -) -env_jupyter_path = pytest.fixture( - lambda tmp_path: mkdir(tmp_path, "env", "share", "jupyter") -) -system_config_path = pytest.fixture(lambda tmp_path: mkdir(tmp_path, "etc", "jupyter")) -env_config_path = pytest.fixture( - lambda tmp_path: mkdir(tmp_path, "env", "etc", "jupyter") -) -some_resource = u"The very model of a modern major general" -sample_kernel_json = { - 'argv':['cat', '{connection_file}'], - 'display_name': 'Test kernel', -} -argv = pytest.fixture(lambda: []) - -@pytest.fixture -def environ( - monkeypatch, - tmp_path, - home_dir, - data_dir, - config_dir, - runtime_dir, - root_dir, - system_jupyter_path, - system_config_path, - env_jupyter_path, - env_config_path, -): - monkeypatch.setenv("HOME", str(home_dir)) - monkeypatch.setenv("PYTHONPATH", os.pathsep.join(sys.path)) - - # Get path to nbconvert template directory *before* - # monkeypatching the paths env variable. - possible_paths = jupyter_core.paths.jupyter_path('nbconvert', 'templates') - nbconvert_path = None - for path in possible_paths: - if os.path.exists(path): - nbconvert_path = path - break - - nbconvert_target = data_dir / 'nbconvert' / 'templates' - - # monkeypatch.setenv("JUPYTER_NO_CONFIG", "1") - monkeypatch.setenv("JUPYTER_CONFIG_DIR", str(config_dir)) - monkeypatch.setenv("JUPYTER_DATA_DIR", str(data_dir)) - monkeypatch.setenv("JUPYTER_RUNTIME_DIR", str(runtime_dir)) - monkeypatch.setattr( - jupyter_core.paths, "SYSTEM_JUPYTER_PATH", [str(system_jupyter_path)] - ) - monkeypatch.setattr(jupyter_core.paths, "ENV_JUPYTER_PATH", [str(env_jupyter_path)]) - monkeypatch.setattr( - jupyter_core.paths, "SYSTEM_CONFIG_PATH", [str(system_config_path)] - ) - monkeypatch.setattr(jupyter_core.paths, "ENV_CONFIG_PATH", [str(env_config_path)]) - - # copy nbconvert templates to new tmp data_dir. - if nbconvert_path: - shutil.copytree(nbconvert_path, str(nbconvert_target)) - - -@pytest.fixture -def extension_environ(env_config_path, monkeypatch): - """Monkeypatch a Jupyter Extension's config path into each test's environment variable""" - monkeypatch.setattr(serverextension, "ENV_CONFIG_PATH", [str(env_config_path)]) - - -@pytest.fixture(scope='function') -def configurable_serverapp( - environ, - server_config, - argv, - http_port, - tmp_path, - root_dir, - io_loop, -): - ServerApp.clear_instance() - - def _configurable_serverapp( - config=server_config, - argv=argv, - environ=environ, - http_port=http_port, - tmp_path=tmp_path, - root_dir=root_dir, - **kwargs - ): - c = Config(config) - c.NotebookNotary.db_file = ":memory:" - token = hexlify(os.urandom(4)).decode("ascii") - url_prefix = "/" - app = ServerApp.instance( - # Set the log level to debug for testing purposes - log_level='DEBUG', - port=http_port, - port_retries=0, - open_browser=False, - root_dir=str(root_dir), - base_url=url_prefix, - config=c, - allow_root=True, - token=token, - **kwargs - ) - - app.init_signal = lambda: None - app.log.propagate = True - app.log.handlers = [] - # Initialize app without httpserver - app.initialize(argv=argv, new_httpserver=False) - app.log.propagate = True - app.log.handlers = [] - # Start app without ioloop - app.start_app() - return app - - return _configurable_serverapp - - -@pytest.fixture(scope="function") -def serverapp(server_config, argv, configurable_serverapp): - app = configurable_serverapp(config=server_config, argv=argv) - yield app - app.remove_server_info_file() - app.remove_browser_open_file() - app.cleanup_kernels() - - -@pytest.fixture -def app(serverapp): - """app fixture is needed by pytest_tornasync plugin""" - return serverapp.web_app - - -@pytest.fixture -def auth_header(serverapp): - return {"Authorization": "token {token}".format(token=serverapp.token)} - - -@pytest.fixture -def http_port(http_server_port): - return http_server_port[-1] - - -@pytest.fixture -def base_url(): - return "/" - - -@pytest.fixture -def fetch(http_server_client, auth_header, base_url): - """fetch fixture that handles auth, base_url, and path""" - def client_fetch(*parts, headers={}, params={}, **kwargs): - # Handle URL strings - path_url = url_escape(url_path_join(base_url, *parts), plus=False) - params_url = urllib.parse.urlencode(params) - url = path_url + "?" + params_url - # Add auth keys to header - headers.update(auth_header) - # Make request. - return http_server_client.fetch( - url, headers=headers, request_timeout=20, **kwargs - ) - return client_fetch - - -@pytest.fixture -def ws_fetch(auth_header, http_port): - """websocket fetch fixture that handles auth, base_url, and path""" - def client_fetch(*parts, headers={}, params={}, **kwargs): - # Handle URL strings - path = url_escape(url_path_join(*parts), plus=False) - urlparts = urllib.parse.urlparse('ws://localhost:{}'.format(http_port)) - urlparts = urlparts._replace( - path=path, - query=urllib.parse.urlencode(params) - ) - url = urlparts.geturl() - # Add auth keys to header - headers.update(auth_header) - # Make request. - req = tornado.httpclient.HTTPRequest( - url, - headers=auth_header, - connect_timeout=120 - ) - return tornado.websocket.websocket_connect(req) - return client_fetch - - -@pytest.fixture -def kernelspecs(data_dir): - spec_names = ['sample', 'sample 2'] - for name in spec_names: - sample_kernel_dir = data_dir.joinpath('kernels', name) - sample_kernel_dir.mkdir(parents=True) - # Create kernel json file - sample_kernel_file = sample_kernel_dir.joinpath('kernel.json') - sample_kernel_file.write_text(json.dumps(sample_kernel_json)) - # Create resources text - sample_kernel_resources = sample_kernel_dir.joinpath('resource.txt') - sample_kernel_resources.write_text(some_resource) - - -@pytest.fixture(params=[True, False]) -def contents_manager(request, tmp_path): - return FileContentsManager(root_dir=str(tmp_path), use_atomic_writing=request.param) - - -@pytest.fixture -def create_notebook(root_dir): - """Create a notebook in the test's home directory.""" - def inner(nbpath): - nbpath = root_dir.joinpath(nbpath) - # Check that the notebook has the correct file extension. - if nbpath.suffix != '.ipynb': - raise Exception("File extension for notebook must be .ipynb") - # If the notebook path has a parent directory, make sure it's created. - parent = nbpath.parent - parent.mkdir(parents=True, exist_ok=True) - # Create a notebook string and write to file. - nb = nbformat.v4.new_notebook() - nbtext = nbformat.writes(nb, version=4) - nbpath.write_text(nbtext) - return inner diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 0386e1bcc3..12f6134f36 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1604,31 +1604,32 @@ def init_httpserver(self): @staticmethod def _init_asyncio_patch(): - """set default asyncio policy to be compatible with tornado - Tornado 6 (at least) is not compatible with the default - asyncio implementation on Windows - Pick the older SelectorEventLoopPolicy on Windows - if the known-incompatible default policy is in use. - do this as early as possible to make it a low priority and overrideable - ref: https://github.com/tornadoweb/tornado/issues/2608 - FIXME: if/when tornado supports the defaults in asyncio, - remove and bump tornado requirement for py38 - """ - if sys.platform.startswith("win") and sys.version_info >= (3, 8): - import asyncio - try: - from asyncio import ( - WindowsProactorEventLoopPolicy, - WindowsSelectorEventLoopPolicy, - ) - except ImportError: - pass - # not affected - else: - if type(asyncio.get_event_loop_policy()) is WindowsProactorEventLoopPolicy: - # WindowsProactorEventLoopPolicy is not compatible with tornado 6 - # fallback to the pre-3.8 default of Selector - asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) + return + # """set default asyncio policy to be compatible with tornado + # Tornado 6 (at least) is not compatible with the default + # asyncio implementation on Windows + # Pick the older SelectorEventLoopPolicy on Windows + # if the known-incompatible default policy is in use. + # do this as early as possible to make it a low priority and overrideable + # ref: https://github.com/tornadoweb/tornado/issues/2608 + # FIXME: if/when tornado supports the defaults in asyncio, + # remove and bump tornado requirement for py38 + # """ + # if sys.platform.startswith("win") and sys.version_info >= (3, 8): + # import asyncio + # try: + # from asyncio import ( + # WindowsProactorEventLoopPolicy, + # WindowsSelectorEventLoopPolicy, + # ) + # except ImportError: + # pass + # # not affected + # else: + # if type(asyncio.get_event_loop_policy()) is WindowsProactorEventLoopPolicy: + # # WindowsProactorEventLoopPolicy is not compatible with tornado 6 + # # fallback to the pre-3.8 default of Selector + # asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True): diff --git a/setup.py b/setup.py index 0fb3764929..c796720558 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ ], install_requires = [ 'jinja2', - 'tornado>=5.0', + 'tornado>=6.1.0', 'pyzmq>=17', 'ipython_genutils', 'traitlets>=4.2.1', From a0640b3d6a30f815079e2233afcdb4d74719d11c Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 14 Nov 2020 13:47:56 -0500 Subject: [PATCH 23/26] more work on removing asyncio patch --- jupyter_server/serverapp.py | 33 ++++++--------------------------- tests/test_files.py | 37 ++----------------------------------- 2 files changed, 8 insertions(+), 62 deletions(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 12f6134f36..1a32841cf4 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1604,32 +1604,12 @@ def init_httpserver(self): @staticmethod def _init_asyncio_patch(): - return - # """set default asyncio policy to be compatible with tornado - # Tornado 6 (at least) is not compatible with the default - # asyncio implementation on Windows - # Pick the older SelectorEventLoopPolicy on Windows - # if the known-incompatible default policy is in use. - # do this as early as possible to make it a low priority and overrideable - # ref: https://github.com/tornadoweb/tornado/issues/2608 - # FIXME: if/when tornado supports the defaults in asyncio, - # remove and bump tornado requirement for py38 - # """ - # if sys.platform.startswith("win") and sys.version_info >= (3, 8): - # import asyncio - # try: - # from asyncio import ( - # WindowsProactorEventLoopPolicy, - # WindowsSelectorEventLoopPolicy, - # ) - # except ImportError: - # pass - # # not affected - # else: - # if type(asyncio.get_event_loop_policy()) is WindowsProactorEventLoopPolicy: - # # WindowsProactorEventLoopPolicy is not compatible with tornado 6 - # # fallback to the pre-3.8 default of Selector - # asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) + """no longer needed with tornado 6.1""" + warnings.warn( + """ServerApp._init_asyncio_patch called, and is longer needed for """ + """tornado 6.1+, and will be removed in a future release.""", + DeprecationWarning + ) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True): @@ -1649,7 +1629,6 @@ def initialize(self, argv=None, find_extensions=True, new_httpserver=True): If True, a tornado HTTPServer instance will be created and configured for the Server Web Application. This will set the http_server attribute of this class. """ - self._init_asyncio_patch() # Parse command line, load ServerApp config files, # and update ServerApp config. super(ServerApp, self).initialize(argv) diff --git a/tests/test_files.py b/tests/test_files.py index 268537d085..738b5879df 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -21,47 +21,14 @@ def maybe_hidden(request): return request.param - -TIMEOUTS = dict( - # default is 20.0 - connect_timeout=0.0, - # needs patch below - request_timeout=0.0 -) - -# HACK: shouldn't be overloading this - -from jupyter_server.utils import url_path_join -import urllib.parse -from tornado.escape import url_escape - -@pytest.fixture -def jp_fetch(jp_serverapp, http_server_client, jp_auth_header, jp_base_url): - def client_fetch(*parts, headers={}, params={}, **kwargs): - # Handle URL strings - path_url = url_escape(url_path_join(jp_base_url, *parts), plus=False) - params_url = urllib.parse.urlencode(params) - url = path_url + "?" + params_url - # Add auth keys to header - headers.update(jp_auth_header) - # Make request. - kwargs.setdefault("request_timeout", 20.0) - return http_server_client.fetch( - url, headers=headers, **kwargs - ) - return client_fetch - -# /HACK - async def fetch_expect_200(jp_fetch, *path_parts): - r = await jp_fetch('files', *path_parts, method='GET', **TIMEOUTS) + r = await jp_fetch('files', *path_parts, method='GET') assert (r.body.decode() == path_parts[-1]), (path_parts, r.body) async def fetch_expect_404(jp_fetch, *path_parts): with pytest.raises(tornado.httpclient.HTTPClientError) as e: - r = await jp_fetch('files', *path_parts, method='GET', **TIMEOUTS) - print(r.body) + r = await jp_fetch('files', *path_parts, method='GET') assert expected_http_error(e, 404), [path_parts, e] From da0fef56eb4f765f0746de2a44a0c4807895921d Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 15 Nov 2020 13:23:11 -0500 Subject: [PATCH 24/26] updates from review --- jupyter_server/serverapp.py | 17 ++++++++--------- tests/conftest.py | 0 tests/test_files.py | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 tests/conftest.py diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 1a32841cf4..091ef4b7a8 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -41,17 +41,16 @@ from jupyter_server.transutils import trans, _ from jupyter_server.utils import secure_write, run_sync -# check for tornado 3.1.0 +# the minimum viable tornado version: needs to be kept in sync with setup.py +MIN_TORNADO = (6, 1) + try: import tornado -except ImportError as e: - raise ImportError(_("The Jupyter Server requires tornado >= 4.0")) from e -try: - version_info = tornado.version_info -except AttributeError as e: - raise ImportError(_("The Jupyter Server requires tornado >= 4.0, but you have < 1.1.0")) from e -if version_info < (4,0): - raise ImportError(_("The Jupyter Server requires tornado >= 4.0, but you have %s") % tornado.version) + assert tornado.version_info >= MIN_TORNADO +except (ImportError, AttributeError, AssertionError) as e: # pragma: no cover + raise ImportError( + _("The Jupyter Server requires tornado >=%s", ".".join(MIN_TORNADO)) + ) from e from tornado import httpserver from tornado import ioloop diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/test_files.py b/tests/test_files.py index 738b5879df..a1c3872354 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -28,7 +28,7 @@ async def fetch_expect_200(jp_fetch, *path_parts): async def fetch_expect_404(jp_fetch, *path_parts): with pytest.raises(tornado.httpclient.HTTPClientError) as e: - r = await jp_fetch('files', *path_parts, method='GET') + await jp_fetch('files', *path_parts, method='GET') assert expected_http_error(e, 404), [path_parts, e] From e09ba054f002bf62c48435472d15619e2e3fcf8c Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 15 Nov 2020 19:54:21 -0500 Subject: [PATCH 25/26] fix minimum tornado error message --- jupyter_server/serverapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 091ef4b7a8..e3d152dcd7 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -42,14 +42,14 @@ from jupyter_server.utils import secure_write, run_sync # the minimum viable tornado version: needs to be kept in sync with setup.py -MIN_TORNADO = (6, 1) +MIN_TORNADO = (6, 1, 0) try: import tornado assert tornado.version_info >= MIN_TORNADO except (ImportError, AttributeError, AssertionError) as e: # pragma: no cover raise ImportError( - _("The Jupyter Server requires tornado >=%s", ".".join(MIN_TORNADO)) + _("The Jupyter Server requires tornado >=%s.%s.%s", *MIN_TORNADO) ) from e from tornado import httpserver From 2ce063a3758bbf67702bbfcd929ece7abcb2d41c Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 16 Nov 2020 17:00:26 -0500 Subject: [PATCH 26/26] move min tornado outside translation block --- jupyter_server/serverapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index e3d152dcd7..13bc0940be 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -49,7 +49,7 @@ assert tornado.version_info >= MIN_TORNADO except (ImportError, AttributeError, AssertionError) as e: # pragma: no cover raise ImportError( - _("The Jupyter Server requires tornado >=%s.%s.%s", *MIN_TORNADO) + _("The Jupyter Server requires tornado >=%s.%s.%s") % MIN_TORNADO ) from e from tornado import httpserver