From 62cfaa27da939fe99bb9ecf3d4b1a19d6e280a0c Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 13:47:04 -0500 Subject: [PATCH 01/18] bump minimum python to 3.6, add 3.9, pypy --- .github/workflows/tests.yml | 40 +++++++++++++++++++++++++------------ setup.py | 6 ++++-- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0105c34f..bd29cdcd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,31 +9,45 @@ jobs: name: ${{ matrix.OS }} - Py${{ matrix.PYTHON_VERSION }} runs-on: ${{ matrix.OS }} strategy: - fail-fast: false + fail-fast: false matrix: - OS: ['ubuntu-latest', 'windows-latest'] - PYTHON_VERSION: ['3.5', '3.6', '3.7','3.8'] + OS: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + PYTHON_VERSION: ['3.6', '3.9', 'pypy3'] + exclude: + exclude: + - os: windows-latest + python-version: pypy3 steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 with: python-version: ${{ matrix.PYTHON_VERSION }} - - name: Install test dependencies + - name: Install python dependencies run: | - pip install --upgrade pip setuptools - pip install .[test] - pip install codecov - - name: Install nbformat + pip install --upgrade pip setuptools wheel + - name: Get pip cache dir + id: pip-cache run: | - pip install . - pip freeze - - name: List dependencies + echo "::set-output name=dir::$(pip cache dir)" + - name: Cache pip + uses: actions/cache@v1 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ matrix.PYTHON_VERSION }}-${{ hashFiles('setup.py') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.PYTHON_VERSION }}- + ${{ runner.os }}-pip- + - name: Install nbformat and test dependencies run: | - pip list + pip install --upgrade .[test] codecov + - name: List installed packages + run: | + pip freeze + pip check - name: Run tests run: | - py.test nbformat/tests -v --cov=nbformat + pytest nbformat/tests -v --cov=nbformat - name: Coverage run: | codecov diff --git a/setup.py b/setup.py index 5fb46c8d..de741e16 100644 --- a/setup.py +++ b/setup.py @@ -61,20 +61,22 @@ author_email = 'jupyter@googlegroups.com', url = 'http://jupyter.org', license = 'BSD', - python_requires = '>=3.5', + python_requires = '>=3.6', platforms = "Linux, Mac OS X, Windows", keywords = ['Interactive', 'Interpreter', 'Shell', 'Web'], classifiers = [ + 'Framework :: Jupyter', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], ) From db16b8131ceec76422f41db98328dd0dcdf9ae26 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 13:48:40 -0500 Subject: [PATCH 02/18] add aiofiles --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index de741e16..9724ff22 100644 --- a/setup.py +++ b/setup.py @@ -85,10 +85,11 @@ setuptools_args = {} install_requires = setuptools_args['install_requires'] = [ + 'aiofiles', 'ipython_genutils', - 'traitlets>=4.1', 'jsonschema>=2.4,!=2.5.0', 'jupyter_core', + 'traitlets>=4.1', ] extras_require = setuptools_args['extras_require'] = { From e07e8728fcd735656b91dac1feb7b6521e9057e3 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 13:50:19 -0500 Subject: [PATCH 03/18] make aiofiles extra --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9724ff22..e1f465dc 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,6 @@ setuptools_args = {} install_requires = setuptools_args['install_requires'] = [ - 'aiofiles', 'ipython_genutils', 'jsonschema>=2.4,!=2.5.0', 'jupyter_core', @@ -93,8 +92,9 @@ ] extras_require = setuptools_args['extras_require'] = { + 'async': ['aiofiles'], 'fast': ['fastjsonschema'], - 'test': ['fastjsonschema', 'testpath', 'pytest', 'pytest-cov'], + 'test': ['aiofiles', 'fastjsonschema', 'testpath', 'pytest', 'pytest-cov'], } if 'setuptools' in sys.modules: From 270c25286cf3266f237e186c709fcc4996fccc82 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 13:52:28 -0500 Subject: [PATCH 04/18] tweak workflow --- .github/workflows/tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bd29cdcd..3000ad8f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,9 +14,8 @@ jobs: OS: ['ubuntu-latest', 'macos-latest', 'windows-latest'] PYTHON_VERSION: ['3.6', '3.9', 'pypy3'] exclude: - exclude: - - os: windows-latest - python-version: pypy3 + - os: windows-latest + python-version: pypy3 steps: - uses: actions/checkout@v2 - name: Set up Python From 683a6839fc238a2daddfec83d6f05ad0ddc9e359 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 13:54:26 -0500 Subject: [PATCH 05/18] tweak workflow --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3000ad8f..e7d040df 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: PYTHON_VERSION: ['3.6', '3.9', 'pypy3'] exclude: - os: windows-latest - python-version: pypy3 + PYTHON_VERSION: pypy3 steps: - uses: actions/checkout@v2 - name: Set up Python From 18458d167a97a9562cdd1ad3777451fab6c3229c Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 13:55:46 -0500 Subject: [PATCH 06/18] avoid duplicate runs --- .github/workflows/tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e7d040df..1fe83316 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,12 @@ name: Run tests on: push: + branches: + - master + pull_request: + branches: + - "*" jobs: tests: From 4b02a5854515ecbb967383a8191b72c85ccf0a1b Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 14:08:59 -0500 Subject: [PATCH 07/18] add hypothesis and pytest-asyncio --- .gitignore | 1 + setup.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 82ec99b6..07dea63d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ __pycache__ .#* .coverage .cache +.hypothesis diff --git a/setup.py b/setup.py index e1f465dc..d82a0a6f 100644 --- a/setup.py +++ b/setup.py @@ -94,9 +94,11 @@ extras_require = setuptools_args['extras_require'] = { 'async': ['aiofiles'], 'fast': ['fastjsonschema'], - 'test': ['aiofiles', 'fastjsonschema', 'testpath', 'pytest', 'pytest-cov'], + 'test': ['hypothesis', 'pytest', 'pytest-asyncio', 'pytest-cov', 'testpath'], } +extras_require['test'] = sorted(set(sum(extras_require.values(), []))) + if 'setuptools' in sys.modules: setup_args.update(setuptools_args) setup_args['entry_points'] = { From 7462cbb62d97a453a0fea756a782bc7b47b0940d Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 14:43:18 -0500 Subject: [PATCH 08/18] add aiofiles pin --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d82a0a6f..3041b117 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ ] extras_require = setuptools_args['extras_require'] = { - 'async': ['aiofiles'], + 'async': ['aiofiles>=0.6.0'], 'fast': ['fastjsonschema'], 'test': ['hypothesis', 'pytest', 'pytest-asyncio', 'pytest-cov', 'testpath'], } From 220e7112dad55f7e67cd8883c2057444d5a2649d Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 15:17:24 -0500 Subject: [PATCH 09/18] start asynchronous work in anger --- .github/workflows/tests.yml | 8 ++++ nbformat/asynchronous.py | 80 ++++++++++++++++++++++++++++++++++++ nbformat/tests/strategies.py | 78 +++++++++++++++++++++++++++++++++++ nbformat/tests/test_async.py | 25 +++++++++++ 4 files changed, 191 insertions(+) create mode 100644 nbformat/asynchronous.py create mode 100644 nbformat/tests/strategies.py create mode 100644 nbformat/tests/test_async.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fe83316..91e777ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,6 +42,14 @@ jobs: restore-keys: | ${{ runner.os }}-pip-${{ matrix.PYTHON_VERSION }}- ${{ runner.os }}-pip- + - name: Cache hypotheses + uses: actions/cache@v1 + with: + path: .hypothesis + key: ${{ runner.os }}-hypothesis-${{ matrix.PYTHON_VERSION }}-${{ hashFiles('setup.py') }} + restore-keys: | + ${{ runner.os }}-hypothesis-${{ matrix.PYTHON_VERSION }}- + ${{ runner.os }}-hypothesis- - name: Install nbformat and test dependencies run: | pip install --upgrade .[test] codecov diff --git a/nbformat/asynchronous.py b/nbformat/asynchronous.py new file mode 100644 index 00000000..ce6c4870 --- /dev/null +++ b/nbformat/asynchronous.py @@ -0,0 +1,80 @@ +""" experimental asynchronous API for nbformat +""" +import asyncio +import os +from pathlib import Path +import aiofiles + +from . import reads, writes, read, write, validate, NO_CONVERT, ValidationError + +def _loop(): + return asyncio.get_running_loop() + + +# shim calls for tracing, etc. +def _reads(s, as_version, kwargs_): + return reads(s, as_version, **kwargs_) + + +def _read(fp, as_version, kwargs_): + return read(fp, as_version, **kwargs_) + + +def _writes(nb, version, kwargs_): + return writes(nb, version, **kwargs_) + + +def _write(nb, fp, version, kwargs_): + return writes(nb, fp, version, **kwargs_) + +def _validate(nbdict, ref, version, version_minor, relax_add_props, nbjson): + return validate(nbdict, ref, version, version_minor, relax_add_props, nbjson) + + +__all__ = [ + "NO_CONVERT", + # synchronous API + "reads", + "read", + "writes", + "write", + "validate", + # asynchronous API + "areads", + "aread", + "awrites", + "awrite", + "avalidate" +] + + +async def areads(s, as_version, **kwargs): + return await _loop().run_in_executor(None, _reads, s, as_version, kwargs) + + +async def aread(fp, as_version, **kwargs): + with aiofiles.open(fp) as afp: + return await areads(await afp.read(), as_version, **kwargs) + + return await _loop().run_in_executor(None, _read, fp, as_version, kwargs) + + +async def awrites(nb, version=NO_CONVERT, **kwargs): + return await _loop().run_in_executor(None, _writes, nb, version, kwargs) + + +async def awrite(nb, fp, version=NO_CONVERT, **kwargs): + if isinstance(fp, str): + with aiofiles.open(fp, "w+") as afp: + return await awrites(nb, await afp.read(), version, **kwargs) + elif isinstance(fp, Path): + with aiofiles.open(str(fp), "w+") as afp: + return await awrites(nb, await afp.read(), version, **kwargs) + + return await _loop().run_in_executor(None, _write, nb, fp, version, kwargs) + + +async def avalidate(nbdict=None, ref=None, version=None, version_minor=None, + relax_add_props=False, nbjson=None): + return await _loop().run_in_executor(None, _validate, nbdict, ref, version, + version_minor, relax_add_props, nbjson) diff --git a/nbformat/tests/strategies.py b/nbformat/tests/strategies.py new file mode 100644 index 00000000..75866583 --- /dev/null +++ b/nbformat/tests/strategies.py @@ -0,0 +1,78 @@ +"""nbformat strategies for hypothesis""" +import pytest +import re +from pathlib import Path +from hypothesis import given, strategies as st, assume, settings, HealthCheck + +from nbformat import validate, reads, writes +from nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook + +HERE = Path(__file__).parent +ALL_NOTEBOOKS = [ + reads( + p.read_text(), + int(re.findall(r"""nbformat":\s+(\d+)""", p.read_text())[0]) + ) + for p in HERE.glob("*.ipynb") +] + +def _is_valid(nb): + try: + validate(nb) + return True + except: + return False + +VALID_NOTEBOOKS = [nb for nb in ALL_NOTEBOOKS if _is_valid(nb)] +INVALID_NOTEBOOKS = [nb for nb in ALL_NOTEBOOKS if nb not in VALID_NOTEBOOKS] + +CELL_TYPES = [new_code_cell, new_markdown_cell] +# , nbformat.new_text_cell, nbformat.new_notebook_cell] + +a_cell_generator = st.sampled_from(CELL_TYPES) +a_test_notebook = st.sampled_from(ALL_NOTEBOOKS) +a_valid_test_notebook = st.sampled_from(VALID_NOTEBOOKS) +an_invalid_test_notebook = st.sampled_from(INVALID_NOTEBOOKS) + + +@st.composite +def a_cell(draw): + Cell = draw(a_cell_generator) + cell = Cell() + cell.source = draw(st.text()) + return cell + + +@st.composite +def a_new_notebook(draw): + notebook = new_notebook() + cell_count = draw(st.integers(min_value=1, max_value=100)) + notebook.cells = [draw(a_cell()) for i in range(cell_count)] + + return notebook + + +@st.composite +def a_valid_notebook(draw): + if draw(st.booleans()): + return draw(a_valid_test_notebook) + + return draw(a_new_notebook()) + + +@st.composite +def an_invalid_notebook(draw): + # TODO: some mutations to make a valid notebook invalid + return draw(an_invalid_test_notebook) + + +@st.composite +def a_valid_notebook_with_string(draw): + notebook = draw(a_valid_notebook()) + return notebook, writes(notebook) + + +@st.composite +def an_invalid_notebook_with_string(draw): + notebook = draw(an_invalid_notebook()) + return notebook, writes(notebook) diff --git a/nbformat/tests/test_async.py b/nbformat/tests/test_async.py new file mode 100644 index 00000000..a08b9c56 --- /dev/null +++ b/nbformat/tests/test_async.py @@ -0,0 +1,25 @@ +import pytest +from hypothesis import given, settings, HealthCheck + +from nbformat import reads +from nbformat.asynchronous import areads, avalidate, ValidationError + +from . import strategies as nbs + + +@pytest.mark.asyncio +@given(nb_txt=nbs.a_valid_notebook_with_string()) +@settings(suppress_health_check=[HealthCheck.too_slow]) +async def test_areads_valid(nb_txt, caplog): + nb, txt = nb_txt + await areads(txt, nb.nbformat) + assert "Notebook JSON" not in caplog.text + + +@pytest.mark.asyncio +@given(nb_txt=nbs.an_invalid_notebook_with_string()) +@settings(suppress_health_check=[HealthCheck.too_slow]) +async def test_areads_invalid(nb_txt, caplog): + nb, txt = nb_txt + await areads(txt, nb.nbformat) + assert "Notebook JSON" in caplog.text From 0018293483f282369c93412a6b5de9cc516e400a Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 15:22:31 -0500 Subject: [PATCH 10/18] try get_event_loop --- nbformat/asynchronous.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/nbformat/asynchronous.py b/nbformat/asynchronous.py index ce6c4870..e4efb51d 100644 --- a/nbformat/asynchronous.py +++ b/nbformat/asynchronous.py @@ -8,7 +8,7 @@ from . import reads, writes, read, write, validate, NO_CONVERT, ValidationError def _loop(): - return asyncio.get_running_loop() + return asyncio.get_event_loop() # shim calls for tracing, etc. @@ -33,12 +33,7 @@ def _validate(nbdict, ref, version, version_minor, relax_add_props, nbjson): __all__ = [ "NO_CONVERT", - # synchronous API - "reads", - "read", - "writes", - "write", - "validate", + "ValidationError", # asynchronous API "areads", "aread", From b90d9784fe045c4ebadaa82cd2388ca99302a6e9 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 15:23:26 -0500 Subject: [PATCH 11/18] encoding for windows --- nbformat/tests/strategies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbformat/tests/strategies.py b/nbformat/tests/strategies.py index 75866583..29c20b61 100644 --- a/nbformat/tests/strategies.py +++ b/nbformat/tests/strategies.py @@ -10,7 +10,7 @@ HERE = Path(__file__).parent ALL_NOTEBOOKS = [ reads( - p.read_text(), + p.read_text(encoding="utf-8"), int(re.findall(r"""nbformat":\s+(\d+)""", p.read_text())[0]) ) for p in HERE.glob("*.ipynb") From d0f51c74424374f0d4fb8c58926f07233461dd30 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 15:35:28 -0500 Subject: [PATCH 12/18] add decorator for settings --- nbformat/tests/strategies.py | 3 +++ nbformat/tests/test_async.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/nbformat/tests/strategies.py b/nbformat/tests/strategies.py index 29c20b61..16e44a0e 100644 --- a/nbformat/tests/strategies.py +++ b/nbformat/tests/strategies.py @@ -29,6 +29,9 @@ def _is_valid(nb): CELL_TYPES = [new_code_cell, new_markdown_cell] # , nbformat.new_text_cell, nbformat.new_notebook_cell] +# Most tests will need this decorator, because fileio and threads are slow +base_settings = settings(suppress_health_check=[HealthCheck.too_slow], deadline=None) + a_cell_generator = st.sampled_from(CELL_TYPES) a_test_notebook = st.sampled_from(ALL_NOTEBOOKS) a_valid_test_notebook = st.sampled_from(VALID_NOTEBOOKS) diff --git a/nbformat/tests/test_async.py b/nbformat/tests/test_async.py index a08b9c56..236bf6be 100644 --- a/nbformat/tests/test_async.py +++ b/nbformat/tests/test_async.py @@ -1,5 +1,5 @@ import pytest -from hypothesis import given, settings, HealthCheck +from hypothesis import given from nbformat import reads from nbformat.asynchronous import areads, avalidate, ValidationError @@ -9,7 +9,7 @@ @pytest.mark.asyncio @given(nb_txt=nbs.a_valid_notebook_with_string()) -@settings(suppress_health_check=[HealthCheck.too_slow]) +@nbs.base_settings async def test_areads_valid(nb_txt, caplog): nb, txt = nb_txt await areads(txt, nb.nbformat) @@ -18,7 +18,7 @@ async def test_areads_valid(nb_txt, caplog): @pytest.mark.asyncio @given(nb_txt=nbs.an_invalid_notebook_with_string()) -@settings(suppress_health_check=[HealthCheck.too_slow]) +@nbs.base_settings async def test_areads_invalid(nb_txt, caplog): nb, txt = nb_txt await areads(txt, nb.nbformat) From 5d50cd4473a986581a6a2dc597ea341d5ef473d3 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 16:58:10 -0500 Subject: [PATCH 13/18] complete rest of basic tests --- nbformat/asynchronous.py | 22 +++--------- nbformat/tests/strategies.py | 8 ++--- nbformat/tests/test_async.py | 67 +++++++++++++++++++++++++++++++----- 3 files changed, 65 insertions(+), 32 deletions(-) diff --git a/nbformat/asynchronous.py b/nbformat/asynchronous.py index e4efb51d..8aa84f00 100644 --- a/nbformat/asynchronous.py +++ b/nbformat/asynchronous.py @@ -16,17 +16,10 @@ def _reads(s, as_version, kwargs_): return reads(s, as_version, **kwargs_) -def _read(fp, as_version, kwargs_): - return read(fp, as_version, **kwargs_) - - def _writes(nb, version, kwargs_): return writes(nb, version, **kwargs_) -def _write(nb, fp, version, kwargs_): - return writes(nb, fp, version, **kwargs_) - def _validate(nbdict, ref, version, version_minor, relax_add_props, nbjson): return validate(nbdict, ref, version, version_minor, relax_add_props, nbjson) @@ -48,25 +41,18 @@ async def areads(s, as_version, **kwargs): async def aread(fp, as_version, **kwargs): - with aiofiles.open(fp) as afp: + async with aiofiles.open(fp) as afp: return await areads(await afp.read(), as_version, **kwargs) - return await _loop().run_in_executor(None, _read, fp, as_version, kwargs) - async def awrites(nb, version=NO_CONVERT, **kwargs): return await _loop().run_in_executor(None, _writes, nb, version, kwargs) async def awrite(nb, fp, version=NO_CONVERT, **kwargs): - if isinstance(fp, str): - with aiofiles.open(fp, "w+") as afp: - return await awrites(nb, await afp.read(), version, **kwargs) - elif isinstance(fp, Path): - with aiofiles.open(str(fp), "w+") as afp: - return await awrites(nb, await afp.read(), version, **kwargs) - - return await _loop().run_in_executor(None, _write, nb, fp, version, kwargs) + nb_str = await awrites(nb, version, **kwargs) + async with aiofiles.open(fp, "w+", encoding="utf-8") as afp: + await afp.write(nb_str) async def avalidate(nbdict=None, ref=None, version=None, version_minor=None, diff --git a/nbformat/tests/strategies.py b/nbformat/tests/strategies.py index 16e44a0e..f0e3b259 100644 --- a/nbformat/tests/strategies.py +++ b/nbformat/tests/strategies.py @@ -8,12 +8,10 @@ from nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook HERE = Path(__file__).parent +ALL_NOTEBOOK_TEXT = [p.read_text(encoding="utf-8") for p in HERE.glob("*.ipynb")] ALL_NOTEBOOKS = [ - reads( - p.read_text(encoding="utf-8"), - int(re.findall(r"""nbformat":\s+(\d+)""", p.read_text())[0]) - ) - for p in HERE.glob("*.ipynb") + reads(nb_text, int(re.findall(r"""nbformat":\s+(\d+)""", nb_text)[0])) + for nb_text in ALL_NOTEBOOK_TEXT ] def _is_valid(nb): diff --git a/nbformat/tests/test_async.py b/nbformat/tests/test_async.py index 236bf6be..c52b2b82 100644 --- a/nbformat/tests/test_async.py +++ b/nbformat/tests/test_async.py @@ -1,25 +1,74 @@ +import os import pytest +import contextlib from hypothesis import given -from nbformat import reads -from nbformat.asynchronous import areads, avalidate, ValidationError +from nbformat.asynchronous import aread, areads, awrite, awrites, avalidate, ValidationError +from nbformat.json_compat import VALIDATORS +import tempfile from . import strategies as nbs +@contextlib.contextmanager +def json_validator(validator_name): + os.environ["NBFORMAT_VALIDATOR"] = validator_name + try: + yield + finally: + os.environ.pop("NBFORMAT_VALIDATOR") + + + +# some issues with setting the environment mean they can just be parametrized +async def _valid(nb_txt, caplog): + nb, txt = nb_txt + read_nb = await areads(txt, nb.nbformat) + assert "Notebook JSON" not in caplog.text + + await avalidate(read_nb) + + with tempfile.TemporaryDirectory() as td: + nb_path = os.path.join(td, "notebook.ipynb") + await awrite(read_nb, nb_path) + await aread(nb_path, nb["nbformat"]) + +@given(nb_txt=nbs.a_valid_notebook_with_string()) +@nbs.base_settings @pytest.mark.asyncio +async def test_async_valid_default(nb_txt, caplog): + with json_validator("jsonschema"): + await _valid(nb_txt, caplog) + + @given(nb_txt=nbs.a_valid_notebook_with_string()) @nbs.base_settings -async def test_areads_valid(nb_txt, caplog): +@pytest.mark.asyncio +async def test_async_valid_fast(nb_txt, caplog): + with json_validator("fastjsonschema"): + await _valid(nb_txt, caplog) + + +async def _invalid(nb_txt, caplog): nb, txt = nb_txt - await areads(txt, nb.nbformat) - assert "Notebook JSON" not in caplog.text + read_nb = await areads(txt, nb.nbformat) + assert "Notebook JSON" in caplog.text + + with pytest.raises(ValidationError): + await avalidate(read_nb) +@given(nb_txt=nbs.an_invalid_notebook_with_string()) +@nbs.base_settings @pytest.mark.asyncio +async def test_async_invalid_default(nb_txt, caplog): + with json_validator("jsonschema"): + await _invalid(nb_txt, caplog) + + @given(nb_txt=nbs.an_invalid_notebook_with_string()) @nbs.base_settings -async def test_areads_invalid(nb_txt, caplog): - nb, txt = nb_txt - await areads(txt, nb.nbformat) - assert "Notebook JSON" in caplog.text +@pytest.mark.asyncio +async def test_async_invalid_false(nb_txt, caplog): + with json_validator("fastjsonschema"): + await _invalid(nb_txt, caplog) From 3516b36ab631821ac43fe04971ecf347aaed0148 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 17:18:06 -0500 Subject: [PATCH 14/18] hoist env var and encoding to constants --- docs/conf.py | 4 ---- nbformat/__init__.py | 8 +++++--- nbformat/asynchronous.py | 27 +++++++++++++++----------- nbformat/constants.py | 9 +++++++++ nbformat/json_compat.py | 4 +++- nbformat/sign.py | 3 ++- nbformat/tests/base.py | 4 +++- nbformat/tests/strategies.py | 11 ++++++++--- nbformat/tests/test_async.py | 31 +++++++++++++++++------------- nbformat/tests/test_validator.py | 7 ++++--- nbformat/v3/tests/formattest.py | 18 ++++++++--------- nbformat/v3/tests/nbexamples.py | 2 -- nbformat/v4/tests/formattest.py | 3 ++- nbformat/v4/tests/nbexamples.py | 4 ---- nbformat/v4/tests/test_convert.py | 1 - nbformat/v4/tests/test_json.py | 1 + nbformat/v4/tests/test_validate.py | 3 ++- setup.py | 1 - 18 files changed, 81 insertions(+), 60 deletions(-) create mode 100644 nbformat/constants.py diff --git a/docs/conf.py b/docs/conf.py index b4d73a6a..16e21129 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # nbformat documentation build configuration file, created by # sphinx-quickstart on Thu May 14 17:26:52 2015. @@ -44,9 +43,6 @@ # source_suffix = ['.rst', '.md'] source_suffix = '.rst' -# The encoding of source files. -#source_encoding = 'utf-8-sig' - # The master toctree document. master_doc = 'index' diff --git a/nbformat/__init__.py b/nbformat/__init__.py index fa05fc27..100e8188 100644 --- a/nbformat/__init__.py +++ b/nbformat/__init__.py @@ -3,7 +3,7 @@ Use this module to read or write notebook files as particular nbformat versions. """ -# Copyright (c) IPython Development Team. +# Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import io @@ -15,6 +15,7 @@ from . import v3 from . import v4 from .sentinel import Sentinel +from .constants import DEFAULT_ENCODING, ENV_VAR_VALIDATOR __all__ = ['versions', 'validate', 'ValidationError', 'convert', 'from_dict', 'NotebookNode', 'current_nbformat', 'current_nbformat_minor', @@ -29,6 +30,7 @@ 4: v4, } + from .validator import validate, ValidationError from .converter import convert from . import reader @@ -137,7 +139,7 @@ def read(fp, as_version, **kwargs): try: buf = fp.read() except AttributeError: - with io.open(fp, encoding='utf-8') as f: + with io.open(fp, encoding=DEFAULT_ENCODING) as f: return reads(f.read(), as_version, **kwargs) return reads(buf, as_version, **kwargs) @@ -170,7 +172,7 @@ def write(nb, fp, version=NO_CONVERT, **kwargs): if not s.endswith(u'\n'): fp.write(u'\n') except AttributeError: - with io.open(fp, 'w', encoding='utf-8') as f: + with io.open(fp, 'w', encoding=DEFAULT_ENCODING) as f: f.write(s) if not s.endswith(u'\n'): f.write(u'\n') diff --git a/nbformat/asynchronous.py b/nbformat/asynchronous.py index 8aa84f00..bdadd017 100644 --- a/nbformat/asynchronous.py +++ b/nbformat/asynchronous.py @@ -1,11 +1,15 @@ -""" experimental asynchronous API for nbformat -""" +"""asynchronous API for nbformat""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + import asyncio import os from pathlib import Path import aiofiles from . import reads, writes, read, write, validate, NO_CONVERT, ValidationError +from .constants import DEFAULT_ENCODING def _loop(): return asyncio.get_event_loop() @@ -25,14 +29,15 @@ def _validate(nbdict, ref, version, version_minor, relax_add_props, nbjson): __all__ = [ - "NO_CONVERT", - "ValidationError", + 'NO_CONVERT', + 'DEFAULT_ENCODING', + 'ValidationError', # asynchronous API - "areads", - "aread", - "awrites", - "awrite", - "avalidate" + 'areads', + 'aread', + 'awrites', + 'awrite', + 'avalidate' ] @@ -41,7 +46,7 @@ async def areads(s, as_version, **kwargs): async def aread(fp, as_version, **kwargs): - async with aiofiles.open(fp) as afp: + async with aiofiles.open(fp, encoding=DEFAULT_ENCODING) as afp: return await areads(await afp.read(), as_version, **kwargs) @@ -51,7 +56,7 @@ async def awrites(nb, version=NO_CONVERT, **kwargs): async def awrite(nb, fp, version=NO_CONVERT, **kwargs): nb_str = await awrites(nb, version, **kwargs) - async with aiofiles.open(fp, "w+", encoding="utf-8") as afp: + async with aiofiles.open(fp, 'w+', encoding=DEFAULT_ENCODING) as afp: await afp.write(nb_str) diff --git a/nbformat/constants.py b/nbformat/constants.py new file mode 100644 index 00000000..4591df42 --- /dev/null +++ b/nbformat/constants.py @@ -0,0 +1,9 @@ +"""constants used throughout nbformat""" +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# while JSON allows for other encodings, utf-8 is most widely supported +DEFAULT_ENCODING = 'utf-8' + +# environment variable +ENV_VAR_VALIDATOR = 'NBFORMAT_VALIDATOR' diff --git a/nbformat/json_compat.py b/nbformat/json_compat.py index 936d3c87..f2f1aec2 100644 --- a/nbformat/json_compat.py +++ b/nbformat/json_compat.py @@ -18,6 +18,8 @@ fastjsonschema = None _JsonSchemaException = ValidationError +from .constants import ENV_VAR_VALIDATOR + class JsonSchemaValidator: name = "jsonschema" @@ -78,5 +80,5 @@ def get_current_validator(): """ Return the default validator based on the value of an environment variable. """ - validator_name = os.environ.get("NBFORMAT_VALIDATOR", "jsonschema") + validator_name = os.environ.get(ENV_VAR_VALIDATOR, "jsonschema") return _validator_for_name(validator_name) diff --git a/nbformat/sign.py b/nbformat/sign.py index f7fe36c9..0087abf3 100644 --- a/nbformat/sign.py +++ b/nbformat/sign.py @@ -29,6 +29,7 @@ from jupyter_core.application import JupyterApp, base_flags from . import read, reads, NO_CONVERT, __version__ +from .constants import DEFAULT_ENCODING from ._compat import encodebytes try: @@ -569,7 +570,7 @@ def sign_notebook_file(self, notebook_path): if not os.path.exists(notebook_path): self.log.error("Notebook missing: %s" % notebook_path) self.exit(1) - with io.open(notebook_path, encoding='utf8') as f: + with io.open(notebook_path, encoding=DEFAULT_ENCODING) as f: nb = read(f, NO_CONVERT) self.sign_notebook(nb, notebook_path) diff --git a/nbformat/tests/base.py b/nbformat/tests/base.py index 312c22bb..bd80cda6 100644 --- a/nbformat/tests/base.py +++ b/nbformat/tests/base.py @@ -9,11 +9,13 @@ import unittest import io +from ..constants import DEFAULT_ENCODING + class TestsBase(unittest.TestCase): """Base tests class.""" @classmethod - def fopen(cls, f, mode=u'r',encoding='utf-8'): + def fopen(cls, f, mode=u'r',encoding=DEFAULT_ENCODING): return io.open(os.path.join(cls._get_files_path(), f), mode, encoding=encoding) @classmethod diff --git a/nbformat/tests/strategies.py b/nbformat/tests/strategies.py index f0e3b259..99cbaddd 100644 --- a/nbformat/tests/strategies.py +++ b/nbformat/tests/strategies.py @@ -1,16 +1,21 @@ """nbformat strategies for hypothesis""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + import pytest import re from pathlib import Path from hypothesis import given, strategies as st, assume, settings, HealthCheck -from nbformat import validate, reads, writes +from nbformat import validate, reads, writes, DEFAULT_ENCODING from nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook + HERE = Path(__file__).parent -ALL_NOTEBOOK_TEXT = [p.read_text(encoding="utf-8") for p in HERE.glob("*.ipynb")] +ALL_NOTEBOOK_TEXT = [p.read_text(encoding=DEFAULT_ENCODING) for p in HERE.glob('*.ipynb')] ALL_NOTEBOOKS = [ - reads(nb_text, int(re.findall(r"""nbformat":\s+(\d+)""", nb_text)[0])) + reads(nb_text, int(re.findall(r'''nbformat":\s+(\d+)''', nb_text)[0])) for nb_text in ALL_NOTEBOOK_TEXT ] diff --git a/nbformat/tests/test_async.py b/nbformat/tests/test_async.py index c52b2b82..6b011b5d 100644 --- a/nbformat/tests/test_async.py +++ b/nbformat/tests/test_async.py @@ -1,22 +1,27 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + import os import pytest import contextlib +import tempfile + from hypothesis import given -from nbformat.asynchronous import aread, areads, awrite, awrites, avalidate, ValidationError -from nbformat.json_compat import VALIDATORS -import tempfile +from ..constants import ENV_VAR_VALIDATOR +from ..asynchronous import aread, areads, awrite, awrites, avalidate, ValidationError +from ..json_compat import VALIDATORS from . import strategies as nbs @contextlib.contextmanager def json_validator(validator_name): - os.environ["NBFORMAT_VALIDATOR"] = validator_name + os.environ[ENV_VAR_VALIDATOR] = validator_name try: yield finally: - os.environ.pop("NBFORMAT_VALIDATOR") + os.environ.pop(ENV_VAR_VALIDATOR) @@ -24,20 +29,20 @@ def json_validator(validator_name): async def _valid(nb_txt, caplog): nb, txt = nb_txt read_nb = await areads(txt, nb.nbformat) - assert "Notebook JSON" not in caplog.text + assert 'Notebook JSON' not in caplog.text await avalidate(read_nb) with tempfile.TemporaryDirectory() as td: - nb_path = os.path.join(td, "notebook.ipynb") + nb_path = os.path.join(td, 'notebook.ipynb') await awrite(read_nb, nb_path) - await aread(nb_path, nb["nbformat"]) + await aread(nb_path, nb['nbformat']) @given(nb_txt=nbs.a_valid_notebook_with_string()) @nbs.base_settings @pytest.mark.asyncio async def test_async_valid_default(nb_txt, caplog): - with json_validator("jsonschema"): + with json_validator('jsonschema'): await _valid(nb_txt, caplog) @@ -45,14 +50,14 @@ async def test_async_valid_default(nb_txt, caplog): @nbs.base_settings @pytest.mark.asyncio async def test_async_valid_fast(nb_txt, caplog): - with json_validator("fastjsonschema"): + with json_validator('fastjsonschema'): await _valid(nb_txt, caplog) async def _invalid(nb_txt, caplog): nb, txt = nb_txt read_nb = await areads(txt, nb.nbformat) - assert "Notebook JSON" in caplog.text + assert 'Notebook JSON' in caplog.text with pytest.raises(ValidationError): await avalidate(read_nb) @@ -62,7 +67,7 @@ async def _invalid(nb_txt, caplog): @nbs.base_settings @pytest.mark.asyncio async def test_async_invalid_default(nb_txt, caplog): - with json_validator("jsonschema"): + with json_validator('jsonschema'): await _invalid(nb_txt, caplog) @@ -70,5 +75,5 @@ async def test_async_invalid_default(nb_txt, caplog): @nbs.base_settings @pytest.mark.asyncio async def test_async_invalid_false(nb_txt, caplog): - with json_validator("fastjsonschema"): + with json_validator('fastjsonschema'): await _invalid(nb_txt, caplog) diff --git a/nbformat/tests/test_validator.py b/nbformat/tests/test_validator.py index 7723dc91..aace13cb 100644 --- a/nbformat/tests/test_validator.py +++ b/nbformat/tests/test_validator.py @@ -9,6 +9,7 @@ from .base import TestsBase from jsonschema import ValidationError from nbformat import read +from ..constants import ENV_VAR_VALIDATOR from ..validator import isvalid, validate, iter_validate from ..json_compat import VALIDATORS @@ -19,14 +20,14 @@ @pytest.fixture(autouse=True) def clean_env_before_and_after_tests(): """Fixture to clean up env variables before and after tests.""" - os.environ.pop("NBFORMAT_VALIDATOR", None) + os.environ.pop(ENV_VAR_VALIDATOR, None) yield - os.environ.pop("NBFORMAT_VALIDATOR", None) + os.environ.pop(ENV_VAR_VALIDATOR, None) # Helpers def set_validator(validator_name): - os.environ["NBFORMAT_VALIDATOR"] = validator_name + os.environ[ENV_VAR_VALIDATOR] = validator_name @pytest.mark.parametrize("validator_name", VALIDATORS) diff --git a/nbformat/v3/tests/formattest.py b/nbformat/v3/tests/formattest.py index e8667586..ded63a80 100644 --- a/nbformat/v3/tests/formattest.py +++ b/nbformat/v3/tests/formattest.py @@ -6,6 +6,7 @@ pjoin = os.path.join +from ...constants import DEFAULT_ENCODING from ..nbbase import ( NotebookNode, new_code_cell, new_text_cell, new_worksheet, new_notebook @@ -16,7 +17,7 @@ def open_utf8(fname, mode): - return io.open(fname, mode=mode, encoding='utf-8') + return io.open(fname, mode=mode, encoding=DEFAULT_ENCODING) class NBFormatTest: """Mixin for writing notebook format tests""" @@ -25,16 +26,16 @@ class NBFormatTest: nb0_ref = None ext = None mod = None - + def setUp(self): self.wd = tempfile.mkdtemp() - + def tearDown(self): shutil.rmtree(self.wd) - + def assertNBEquals(self, nba, nbb): self.assertEqual(nba, nbb) - + def test_writes(self): s = self.mod.writes(nb0) if self.nb0_ref: @@ -51,13 +52,10 @@ def test_roundtrip(self): def test_write_file(self): with open_utf8(pjoin(self.wd, "nb0.%s" % self.ext), 'w') as f: self.mod.write(nb0, f) - + def test_read_file(self): with open_utf8(pjoin(self.wd, "nb0.%s" % self.ext), 'w') as f: self.mod.write(nb0, f) - + with open_utf8(pjoin(self.wd, "nb0.%s" % self.ext), 'r') as f: nb = self.mod.read(f) - - - diff --git a/nbformat/v3/tests/nbexamples.py b/nbformat/v3/tests/nbexamples.py index bdafc75c..0e0a5475 100644 --- a/nbformat/v3/tests/nbexamples.py +++ b/nbformat/v3/tests/nbexamples.py @@ -148,5 +148,3 @@ print "ünîcødé" """ % (nbformat, nbformat_minor) - - diff --git a/nbformat/v4/tests/formattest.py b/nbformat/v4/tests/formattest.py index 853083ec..6c67bb10 100644 --- a/nbformat/v4/tests/formattest.py +++ b/nbformat/v4/tests/formattest.py @@ -6,11 +6,12 @@ pjoin = os.path.join +from ...constants import DEFAULT_ENCODING from .nbexamples import nb0 def open_utf8(fname, mode): - return io.open(fname, mode=mode, encoding='utf-8') + return io.open(fname, mode=mode, encoding=DEFAULT_ENCODING) class NBFormatTest: """Mixin for writing notebook format tests""" diff --git a/nbformat/v4/tests/nbexamples.py b/nbformat/v4/tests/nbexamples.py index 9602740b..2133a3f9 100644 --- a/nbformat/v4/tests/nbexamples.py +++ b/nbformat/v4/tests/nbexamples.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import os from ..._compat import encodebytes @@ -131,5 +129,3 @@ 'language': 'python', } ) - - diff --git a/nbformat/v4/tests/test_convert.py b/nbformat/v4/tests/test_convert.py index e4492d30..12f0113f 100644 --- a/nbformat/v4/tests/test_convert.py +++ b/nbformat/v4/tests/test_convert.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import copy from nbformat import validate diff --git a/nbformat/v4/tests/test_json.py b/nbformat/v4/tests/test_json.py index ea157e54..84d52e2b 100644 --- a/nbformat/v4/tests/test_json.py +++ b/nbformat/v4/tests/test_json.py @@ -3,6 +3,7 @@ from unittest import TestCase from ..._compat import decodebytes +from ...constants import DEFAULT_ENCODING from ..nbjson import reads, writes from .. import nbjson, nbformat, nbformat_minor from .nbexamples import nb0 diff --git a/nbformat/v4/tests/test_validate.py b/nbformat/v4/tests/test_validate.py index a3f98371..93881f7e 100644 --- a/nbformat/v4/tests/test_validate.py +++ b/nbformat/v4/tests/test_validate.py @@ -9,6 +9,7 @@ import pytest from nbformat.validator import validate, ValidationError +from ...constants import DEFAULT_ENCODING from ..nbjson import reads from ..nbbase import ( nbformat, @@ -100,6 +101,6 @@ def test_invalid_raw_cell(): def test_sample_notebook(): here = os.path.dirname(__file__) - with io.open(os.path.join(here, os.pardir, os.pardir, 'tests', "test4.ipynb"), encoding='utf-8') as f: + with io.open(os.path.join(here, os.pardir, os.pardir, 'tests', "test4.ipynb"), encoding=DEFAULT_ENCODING) as f: nb = reads(f.read()) validate4(nb) diff --git a/setup.py b/setup.py index 3041b117..f673b48a 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. From 0a58dc756f603a9c8273f8e5cb3b63977e1f17c0 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 21:10:21 -0500 Subject: [PATCH 15/18] rename methods --- nbformat/asynchronous.py | 35 ++++++++++---------- nbformat/tests/test_async.py | 62 ++++++++++++++++++++---------------- setup.py | 2 +- 3 files changed, 54 insertions(+), 45 deletions(-) diff --git a/nbformat/asynchronous.py b/nbformat/asynchronous.py index bdadd017..1f65e673 100644 --- a/nbformat/asynchronous.py +++ b/nbformat/asynchronous.py @@ -8,7 +8,10 @@ from pathlib import Path import aiofiles -from . import reads, writes, read, write, validate, NO_CONVERT, ValidationError +from . import ( + NO_CONVERT, ValidationError, + reads as reads_sync, writes as writes_sync, validate as validate_sync +) from .constants import DEFAULT_ENCODING def _loop(): @@ -17,15 +20,15 @@ def _loop(): # shim calls for tracing, etc. def _reads(s, as_version, kwargs_): - return reads(s, as_version, **kwargs_) + return reads_sync(s, as_version, **kwargs_) def _writes(nb, version, kwargs_): - return writes(nb, version, **kwargs_) + return writes_sync(nb, version, **kwargs_) def _validate(nbdict, ref, version, version_minor, relax_add_props, nbjson): - return validate(nbdict, ref, version, version_minor, relax_add_props, nbjson) + return validate_sync(nbdict, ref, version, version_minor, relax_add_props, nbjson) __all__ = [ @@ -33,34 +36,34 @@ def _validate(nbdict, ref, version, version_minor, relax_add_props, nbjson): 'DEFAULT_ENCODING', 'ValidationError', # asynchronous API - 'areads', - 'aread', - 'awrites', - 'awrite', - 'avalidate' + 'reads', + 'read', + 'writes', + 'write', + 'validate' ] -async def areads(s, as_version, **kwargs): +async def reads(s, as_version, **kwargs): return await _loop().run_in_executor(None, _reads, s, as_version, kwargs) -async def aread(fp, as_version, **kwargs): +async def read(fp, as_version, **kwargs): async with aiofiles.open(fp, encoding=DEFAULT_ENCODING) as afp: - return await areads(await afp.read(), as_version, **kwargs) + return await reads(await afp.read(), as_version, **kwargs) -async def awrites(nb, version=NO_CONVERT, **kwargs): +async def writes(nb, version=NO_CONVERT, **kwargs): return await _loop().run_in_executor(None, _writes, nb, version, kwargs) -async def awrite(nb, fp, version=NO_CONVERT, **kwargs): - nb_str = await awrites(nb, version, **kwargs) +async def write(nb, fp, version=NO_CONVERT, **kwargs): + nb_str = await writes(nb, version, **kwargs) async with aiofiles.open(fp, 'w+', encoding=DEFAULT_ENCODING) as afp: await afp.write(nb_str) -async def avalidate(nbdict=None, ref=None, version=None, version_minor=None, +async def validate(nbdict=None, ref=None, version=None, version_minor=None, relax_add_props=False, nbjson=None): return await _loop().run_in_executor(None, _validate, nbdict, ref, version, version_minor, relax_add_props, nbjson) diff --git a/nbformat/tests/test_async.py b/nbformat/tests/test_async.py index 6b011b5d..2954ed63 100644 --- a/nbformat/tests/test_async.py +++ b/nbformat/tests/test_async.py @@ -9,10 +9,11 @@ from hypothesis import given from ..constants import ENV_VAR_VALIDATOR -from ..asynchronous import aread, areads, awrite, awrites, avalidate, ValidationError +from ..asynchronous import read, reads, write, writes, validate, ValidationError from ..json_compat import VALIDATORS from . import strategies as nbs +from testfixtures import LogCapture @contextlib.contextmanager @@ -25,55 +26,60 @@ def json_validator(validator_name): -# some issues with setting the environment mean they can just be parametrized -async def _valid(nb_txt, caplog): - nb, txt = nb_txt - read_nb = await areads(txt, nb.nbformat) - assert 'Notebook JSON' not in caplog.text +# some issues with setting environment variables mean they can just be parametrized +# pytest's caplog conflicts with hypothesis +async def _valid(nb, txt): + """use the asynchronous API with a valid notebook, round-tripping to disk""" + with LogCapture() as caplog: + read_nb = await reads(txt, nb.nbformat) + assert 'Notebook JSON' not in str(caplog) - await avalidate(read_nb) + await validate(read_nb) + + with tempfile.TemporaryDirectory() as td: + nb_path = os.path.join(td, 'notebook.ipynb') + await write(read_nb, nb_path) + await read(nb_path, nb['nbformat']) + + +async def _invalid(nb, txt): + """use the asynchronous API with an invalid notebook, round-tripping to disk""" + + with LogCapture() as caplog: + read_nb = await reads(txt, nb.nbformat) + assert 'Notebook JSON' in str(caplog) + + with pytest.raises(ValidationError): + await validate(read_nb) - with tempfile.TemporaryDirectory() as td: - nb_path = os.path.join(td, 'notebook.ipynb') - await awrite(read_nb, nb_path) - await aread(nb_path, nb['nbformat']) @given(nb_txt=nbs.a_valid_notebook_with_string()) @nbs.base_settings @pytest.mark.asyncio -async def test_async_valid_default(nb_txt, caplog): +async def test_async_valid_default(nb_txt): with json_validator('jsonschema'): - await _valid(nb_txt, caplog) + await _valid(*nb_txt) @given(nb_txt=nbs.a_valid_notebook_with_string()) @nbs.base_settings @pytest.mark.asyncio -async def test_async_valid_fast(nb_txt, caplog): +async def test_async_valid_fast(nb_txt): with json_validator('fastjsonschema'): - await _valid(nb_txt, caplog) - - -async def _invalid(nb_txt, caplog): - nb, txt = nb_txt - read_nb = await areads(txt, nb.nbformat) - assert 'Notebook JSON' in caplog.text - - with pytest.raises(ValidationError): - await avalidate(read_nb) + await _valid(*nb_txt) @given(nb_txt=nbs.an_invalid_notebook_with_string()) @nbs.base_settings @pytest.mark.asyncio -async def test_async_invalid_default(nb_txt, caplog): +async def test_async_invalid_default(nb_txt): with json_validator('jsonschema'): - await _invalid(nb_txt, caplog) + await _invalid(*nb_txt) @given(nb_txt=nbs.an_invalid_notebook_with_string()) @nbs.base_settings @pytest.mark.asyncio -async def test_async_invalid_false(nb_txt, caplog): +async def test_async_invalid_fast(nb_txt): with json_validator('fastjsonschema'): - await _invalid(nb_txt, caplog) + await _invalid(*nb_txt) diff --git a/setup.py b/setup.py index f673b48a..e48c692b 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,7 @@ extras_require = setuptools_args['extras_require'] = { 'async': ['aiofiles>=0.6.0'], 'fast': ['fastjsonschema'], - 'test': ['hypothesis', 'pytest', 'pytest-asyncio', 'pytest-cov', 'testpath'], + 'test': ['hypothesis', 'pytest', 'pytest-asyncio', 'pytest-cov', 'testfixtures', 'testpath'], } extras_require['test'] = sorted(set(sum(extras_require.values(), []))) From 3a47282982bc7153cd1ddd13f09fda8316df22f8 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 22:04:36 -0500 Subject: [PATCH 16/18] handle some use cases like jupyter_server --- nbformat/asynchronous.py | 35 ++++++++++++++++++++++++++++------- nbformat/tests/test_async.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/nbformat/asynchronous.py b/nbformat/asynchronous.py index 1f65e673..429100da 100644 --- a/nbformat/asynchronous.py +++ b/nbformat/asynchronous.py @@ -5,8 +5,10 @@ import asyncio import os +import io from pathlib import Path import aiofiles +from aiofiles.threadpool.text import AsyncTextIOWrapper from . import ( NO_CONVERT, ValidationError, @@ -14,7 +16,13 @@ ) from .constants import DEFAULT_ENCODING +AIOFILES_OPENABLE = (str, bytes, os.PathLike) + + def _loop(): + """get the current event loop + this may need some more work later + """ return asyncio.get_event_loop() @@ -48,19 +56,32 @@ async def reads(s, as_version, **kwargs): return await _loop().run_in_executor(None, _reads, s, as_version, kwargs) -async def read(fp, as_version, **kwargs): - async with aiofiles.open(fp, encoding=DEFAULT_ENCODING) as afp: - return await reads(await afp.read(), as_version, **kwargs) - - async def writes(nb, version=NO_CONVERT, **kwargs): return await _loop().run_in_executor(None, _writes, nb, version, kwargs) +async def read(fp, as_version, **kwargs): + if isinstance(fp, AIOFILES_OPENABLE): + async with aiofiles.open(fp, encoding=DEFAULT_ENCODING) as afp: + nb_str = afp.read() + elif isinstance(fp, io.TextIOWrapper): + nb_str = await AsyncTextIOWrapper(fp, loop=_loop(), executor=None).read() + else: + raise NotImplementedError(f"Don't know how to read {type(fp)}") + + return await reads(nb_str, as_version, **kwargs) + + async def write(nb, fp, version=NO_CONVERT, **kwargs): nb_str = await writes(nb, version, **kwargs) - async with aiofiles.open(fp, 'w+', encoding=DEFAULT_ENCODING) as afp: - await afp.write(nb_str) + + if isinstance(fp, AIOFILES_OPENABLE): + async with aiofiles.open(fp, 'w+', encoding=DEFAULT_ENCODING) as afp: + return await afp.write(nb_str) + elif isinstance(fp, io.TextIOWrapper): + return await AsyncTextIOWrapper(fp, loop=_loop(), executor=None).write(nb_str) + else: + raise NotImplementedError(f"Don't know how to write {type(fp)}") async def validate(nbdict=None, ref=None, version=None, version_minor=None, diff --git a/nbformat/tests/test_async.py b/nbformat/tests/test_async.py index 2954ed63..b493aa8e 100644 --- a/nbformat/tests/test_async.py +++ b/nbformat/tests/test_async.py @@ -2,18 +2,21 @@ # Distributed under the terms of the Modified BSD License. import os +import io import pytest import contextlib import tempfile +from pathlib import Path +import aiofiles from hypothesis import given +from testfixtures import LogCapture -from ..constants import ENV_VAR_VALIDATOR -from ..asynchronous import read, reads, write, writes, validate, ValidationError +from ..constants import ENV_VAR_VALIDATOR, DEFAULT_ENCODING +from ..asynchronous import read, reads, write, writes, validate, ValidationError, NO_CONVERT from ..json_compat import VALIDATORS from . import strategies as nbs -from testfixtures import LogCapture @contextlib.contextmanager @@ -83,3 +86,27 @@ async def test_async_invalid_default(nb_txt): async def test_async_invalid_fast(nb_txt): with json_validator('fastjsonschema'): await _invalid(*nb_txt) + + +@given(nb_txt=nbs.a_valid_notebook_with_string()) +@nbs.base_settings +@pytest.mark.asyncio +async def test_async_like_jupyter_server(nb_txt): + """ the atomic write stuff is rather complex, but it's basically `io.open` + """ + nb, txt = nb_txt + with tempfile.TemporaryDirectory() as td: + nb_path = Path(td) / 'notebook.ipynb' + + # like _save_notebook[1] + with io.open(nb_path, 'w+', encoding=DEFAULT_ENCODING) as fp: + await write(nb, fp) + + # like _read_notebook[2] + with io.open(nb_path, 'r', encoding=DEFAULT_ENCODING) as fp: + await read(fp, as_version=nb["nbformat"]) + +""" +[1]: https://github.com/jupyter/jupyter_server/blob/1.0.5/jupyter_server/services/contents/fileio.py#L279-L282 +[2]: https://github.com/jupyter/jupyter_server/blob/1.0.5/jupyter_server/services/contents/fileio.py#L254-L258 +""" From aa29688c978348423fc78c9cbab8078774c66bb1 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 22:10:24 -0500 Subject: [PATCH 17/18] don't forget await --- nbformat/asynchronous.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbformat/asynchronous.py b/nbformat/asynchronous.py index 429100da..a65de46b 100644 --- a/nbformat/asynchronous.py +++ b/nbformat/asynchronous.py @@ -63,7 +63,7 @@ async def writes(nb, version=NO_CONVERT, **kwargs): async def read(fp, as_version, **kwargs): if isinstance(fp, AIOFILES_OPENABLE): async with aiofiles.open(fp, encoding=DEFAULT_ENCODING) as afp: - nb_str = afp.read() + nb_str = await afp.read() elif isinstance(fp, io.TextIOWrapper): nb_str = await AsyncTextIOWrapper(fp, loop=_loop(), executor=None).read() else: From 6157f831654bebe225007b76504d0cb6306a890d Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 7 Nov 2020 22:36:09 -0500 Subject: [PATCH 18/18] test plain old open --- nbformat/tests/test_async.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nbformat/tests/test_async.py b/nbformat/tests/test_async.py index b493aa8e..23ccc0a3 100644 --- a/nbformat/tests/test_async.py +++ b/nbformat/tests/test_async.py @@ -88,6 +88,22 @@ async def test_async_invalid_fast(nb_txt): await _invalid(*nb_txt) +@given(nb_txt=nbs.a_valid_notebook_with_string()) +@nbs.base_settings +@pytest.mark.asyncio +async def test_async_builtin_open(nb_txt): + """someone's probably using `open`""" + nb, txt = nb_txt + with tempfile.TemporaryDirectory() as td: + nb_path = Path(td) / 'notebook.ipynb' + + with open(nb_path, 'w+', encoding=DEFAULT_ENCODING) as fp: + await write(nb, fp) + + with open(nb_path, 'r', encoding=DEFAULT_ENCODING) as fp: + await read(fp, as_version=nb["nbformat"]) + + @given(nb_txt=nbs.a_valid_notebook_with_string()) @nbs.base_settings @pytest.mark.asyncio @@ -106,6 +122,7 @@ async def test_async_like_jupyter_server(nb_txt): with io.open(nb_path, 'r', encoding=DEFAULT_ENCODING) as fp: await read(fp, as_version=nb["nbformat"]) + """ [1]: https://github.com/jupyter/jupyter_server/blob/1.0.5/jupyter_server/services/contents/fileio.py#L279-L282 [2]: https://github.com/jupyter/jupyter_server/blob/1.0.5/jupyter_server/services/contents/fileio.py#L254-L258