[wip] add async

.github/workflows/tests.yml

@@ -2,38 +2,64 @@ name: Run tests
 
 on:
   push:
+    branches:
+      - master
+
   pull_request:
+    branches:
+      - "*"
 
 jobs:
   tests:
     name: ${{ matrix.OS }} - Py${{ matrix.PYTHON_VERSION }}
     runs-on: ${{ matrix.OS }}
     strategy:
-      fail-fast: false 
+      fail-fast: false
       matrix:
-        OS: ['ubuntu-latest', 'windows-latest']
-        PYTHON_VERSION: ['3.5', '3.6', '3.7','3.8']
+        OS: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        PYTHON_VERSION: ['3.6', '3.9', 'pypy3']
+        exclude:
+        - os: windows-latest
+          PYTHON_VERSION: pypy3
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.PYTHON_VERSION }}
-    - name: Install test dependencies
+    - name: Install python dependencies
       run: |
-        pip install --upgrade pip setuptools
-        pip install .[test]
-        pip install codecov
-    - name: Install nbformat
+        pip install --upgrade pip setuptools wheel
+    - name: Get pip cache dir
+      id: pip-cache
       run: |
-        pip install .
-        pip freeze
-    - name: List dependencies
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: Cache pip
+      uses: actions/cache@v1
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ matrix.PYTHON_VERSION }}-${{ hashFiles('setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.PYTHON_VERSION }}-
+          ${{ runner.os }}-pip-
+    - name: Cache hypotheses
+      uses: actions/cache@v1
+      with:
+        path: .hypothesis
+        key: ${{ runner.os }}-hypothesis-${{ matrix.PYTHON_VERSION }}-${{ hashFiles('setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-hypothesis-${{ matrix.PYTHON_VERSION }}-
+          ${{ runner.os }}-hypothesis-
+    - name: Install nbformat and test dependencies
+      run: |
+        pip install --upgrade .[test] codecov
+    - name: List installed packages
       run: |
-        pip list
+        pip freeze
+        pip check
     - name: Run tests
       run: |
-        py.test nbformat/tests -v --cov=nbformat
+        pytest nbformat/tests -v --cov=nbformat
     - name: Coverage
       run: |
         codecov

.gitignore

@@ -16,3 +16,4 @@ __pycache__
 .#*
 .coverage
 .cache
+.hypothesis

docs/conf.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # nbformat documentation build configuration file, created by
 # sphinx-quickstart on Thu May 14 17:26:52 2015.
@@ -44,9 +43,6 @@
 # source_suffix = ['.rst', '.md']
 source_suffix = '.rst'
 
-# The encoding of source files.
-#source_encoding = 'utf-8-sig'
-
 # The master toctree document.
 master_doc = 'index'
 

nbformat/__init__.py

@@ -3,7 +3,7 @@
 Use this module to read or write notebook files as particular nbformat versions.
 """
 
-# Copyright (c) IPython Development Team.
+# Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import io
 
@@ -15,6 +15,7 @@
 from . import v3
 from . import v4
 from .sentinel import Sentinel
+from .constants import DEFAULT_ENCODING, ENV_VAR_VALIDATOR
 
 __all__ = ['versions', 'validate', 'ValidationError', 'convert', 'from_dict',
            'NotebookNode', 'current_nbformat', 'current_nbformat_minor',
@@ -29,6 +30,7 @@
     4: v4,
 }
 
+
 from .validator import validate, ValidationError
 from .converter import convert
 from . import reader
@@ -137,7 +139,7 @@ def read(fp, as_version, **kwargs):
     try:
         buf = fp.read()
     except AttributeError:
-        with io.open(fp, encoding='utf-8') as f:
+        with io.open(fp, encoding=DEFAULT_ENCODING) as f:
             return reads(f.read(), as_version, **kwargs)
 
     return reads(buf, as_version, **kwargs)
@@ -170,7 +172,7 @@ def write(nb, fp, version=NO_CONVERT, **kwargs):
         if not s.endswith(u'\n'):
             fp.write(u'\n')
     except AttributeError:
-        with io.open(fp, 'w', encoding='utf-8') as f:
+        with io.open(fp, 'w', encoding=DEFAULT_ENCODING) as f:
             f.write(s)
             if not s.endswith(u'\n'):
                 f.write(u'\n')

nbformat/asynchronous.py

@@ -0,0 +1,90 @@
+"""asynchronous API for nbformat"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import asyncio
+import os
+import io
+from pathlib import Path
+import aiofiles
+from aiofiles.threadpool.text import AsyncTextIOWrapper
+
+from . import (
+    NO_CONVERT, ValidationError,
+    reads as reads_sync, writes as writes_sync, validate as validate_sync
+)
+from .constants import DEFAULT_ENCODING
+
+AIOFILES_OPENABLE = (str, bytes, os.PathLike)
+
+
+def _loop():
+    """get the current event loop
+       this may need some more work later
+    """
+    return asyncio.get_event_loop()
+
+
+# shim calls for tracing, etc.
+def _reads(s, as_version, kwargs_):
+    return reads_sync(s, as_version, **kwargs_)
+
+
+def _writes(nb, version, kwargs_):
+    return writes_sync(nb, version, **kwargs_)
+
+
+def _validate(nbdict, ref, version, version_minor, relax_add_props, nbjson):
+    return validate_sync(nbdict, ref, version, version_minor, relax_add_props, nbjson)
+
+
+__all__ = [
+    'NO_CONVERT',
+    'DEFAULT_ENCODING',
+    'ValidationError',
+    # asynchronous API
+    'reads',
+    'read',
+    'writes',
+    'write',
+    'validate'
+]
+
+
+async def reads(s, as_version, **kwargs):
+    return await _loop().run_in_executor(None, _reads, s, as_version, kwargs)
+
+
+async def writes(nb, version=NO_CONVERT, **kwargs):
+    return await _loop().run_in_executor(None, _writes, nb, version, kwargs)
+
+
+async def read(fp, as_version, **kwargs):
+    if isinstance(fp, AIOFILES_OPENABLE):
+        async with aiofiles.open(fp, encoding=DEFAULT_ENCODING) as afp:
+            nb_str = await afp.read()
+    elif isinstance(fp, io.TextIOWrapper):
+        nb_str = await AsyncTextIOWrapper(fp, loop=_loop(), executor=None).read()
+    else:
+        raise NotImplementedError(f"Don't know how to read {type(fp)}")
+
+    return await reads(nb_str, as_version, **kwargs)
+
+
+async def write(nb, fp, version=NO_CONVERT, **kwargs):
+    nb_str = await writes(nb, version, **kwargs)
+
+    if isinstance(fp, AIOFILES_OPENABLE):
+        async with aiofiles.open(fp, 'w+', encoding=DEFAULT_ENCODING) as afp:
+            return await afp.write(nb_str)
+    elif isinstance(fp, io.TextIOWrapper):
+        return await AsyncTextIOWrapper(fp, loop=_loop(), executor=None).write(nb_str)
+    else:
+        raise NotImplementedError(f"Don't know how to write {type(fp)}")
+
+
+async def validate(nbdict=None, ref=None, version=None, version_minor=None,
+                    relax_add_props=False, nbjson=None):
+    return await _loop().run_in_executor(None, _validate, nbdict, ref, version,
+                                         version_minor, relax_add_props, nbjson)

nbformat/constants.py

@@ -0,0 +1,9 @@
+"""constants used throughout nbformat"""
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+# while JSON allows for other encodings, utf-8 is most widely supported
+DEFAULT_ENCODING = 'utf-8'
+
+# environment variable
+ENV_VAR_VALIDATOR = 'NBFORMAT_VALIDATOR'

nbformat/json_compat.py

@@ -18,6 +18,8 @@
     fastjsonschema = None
     _JsonSchemaException = ValidationError
 
+from .constants import ENV_VAR_VALIDATOR
+
 
 class JsonSchemaValidator:
     name = "jsonschema"
@@ -78,5 +80,5 @@ def get_current_validator():
     """
     Return the default validator based on the value of an environment variable.
     """
-    validator_name = os.environ.get("NBFORMAT_VALIDATOR", "jsonschema")
+    validator_name = os.environ.get(ENV_VAR_VALIDATOR, "jsonschema")
     return _validator_for_name(validator_name)

nbformat/sign.py

@@ -29,6 +29,7 @@
 from jupyter_core.application import JupyterApp, base_flags
 
 from . import read, reads, NO_CONVERT, __version__
+from .constants import DEFAULT_ENCODING
 from ._compat import encodebytes
 
 try:
@@ -569,7 +570,7 @@ def sign_notebook_file(self, notebook_path):
         if not os.path.exists(notebook_path):
             self.log.error("Notebook missing: %s" % notebook_path)
             self.exit(1)
-        with io.open(notebook_path, encoding='utf8') as f:
+        with io.open(notebook_path, encoding=DEFAULT_ENCODING) as f:
             nb = read(f, NO_CONVERT)
         self.sign_notebook(nb, notebook_path)
 

nbformat/tests/base.py

@@ -9,11 +9,13 @@
 import unittest
 import io
 
+from ..constants import DEFAULT_ENCODING
+
 class TestsBase(unittest.TestCase):
     """Base tests class."""
 
     @classmethod
-    def fopen(cls, f, mode=u'r',encoding='utf-8'):
+    def fopen(cls, f, mode=u'r',encoding=DEFAULT_ENCODING):
         return io.open(os.path.join(cls._get_files_path(), f), mode, encoding=encoding)
 
     @classmethod

nbformat/tests/strategies.py

@@ -0,0 +1,84 @@
+"""nbformat strategies for hypothesis"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import pytest
+import re
+from pathlib import Path
+from hypothesis import given, strategies as st, assume, settings, HealthCheck
+
+from nbformat import validate, reads, writes, DEFAULT_ENCODING
+from nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook
+
+
+HERE = Path(__file__).parent
+ALL_NOTEBOOK_TEXT = [p.read_text(encoding=DEFAULT_ENCODING) for p in HERE.glob('*.ipynb')]
+ALL_NOTEBOOKS = [
+    reads(nb_text, int(re.findall(r'''nbformat":\s+(\d+)''', nb_text)[0]))
+    for nb_text in ALL_NOTEBOOK_TEXT
+]
+
+def _is_valid(nb):
+    try:
+        validate(nb)
+        return True
+    except:
+        return False
+
+VALID_NOTEBOOKS = [nb for nb in ALL_NOTEBOOKS if _is_valid(nb)]
+INVALID_NOTEBOOKS = [nb for nb in ALL_NOTEBOOKS if nb not in VALID_NOTEBOOKS]
+
+CELL_TYPES = [new_code_cell, new_markdown_cell]
+# , nbformat.new_text_cell, nbformat.new_notebook_cell]
+
+# Most tests will need this decorator, because fileio and threads are slow
+base_settings = settings(suppress_health_check=[HealthCheck.too_slow], deadline=None)
+
+a_cell_generator = st.sampled_from(CELL_TYPES)
+a_test_notebook = st.sampled_from(ALL_NOTEBOOKS)
+a_valid_test_notebook = st.sampled_from(VALID_NOTEBOOKS)
+an_invalid_test_notebook = st.sampled_from(INVALID_NOTEBOOKS)
+
+
+@st.composite
+def a_cell(draw):
+    Cell = draw(a_cell_generator)
+    cell = Cell()
+    cell.source = draw(st.text())
+    return cell
+
+
+@st.composite
+def a_new_notebook(draw):
+    notebook = new_notebook()
+    cell_count = draw(st.integers(min_value=1, max_value=100))
+    notebook.cells = [draw(a_cell()) for i in range(cell_count)]
+
+    return notebook
+
+
+@st.composite
+def a_valid_notebook(draw):
+    if draw(st.booleans()):
+        return draw(a_valid_test_notebook)
+
+    return draw(a_new_notebook())
+
+
+@st.composite
+def an_invalid_notebook(draw):
+    # TODO: some mutations to make a valid notebook invalid
+    return draw(an_invalid_test_notebook)
+
+
+@st.composite
+def a_valid_notebook_with_string(draw):
+    notebook = draw(a_valid_notebook())
+    return notebook, writes(notebook)
+
+
+@st.composite
+def an_invalid_notebook_with_string(draw):
+    notebook = draw(an_invalid_notebook())
+    return notebook, writes(notebook)