From 08b79aa92456e1d3caffb163762faa69a88ebaa4 Mon Sep 17 00:00:00 2001 From: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> Date: Wed, 9 Aug 2023 10:26:23 -0700 Subject: [PATCH] Set up infrastructure for qiskit-tutorials migration (#10443) * Add infrastructure for building tutorials This first commit is a rebase of Eric's initial PR as of db1ce6254 onto `main`, fixing up some changes caused by the CI infrastructure changing a bit since the PR was first opened. Co-authored-by: Jake Lishman * Harden tutorials Azure job This moves much of the fetch- and process-related code into separate scripts that assert far more about the directory structure, and fail if they do not match the assumptions. We don't want to accidentally get out-of-sync while we're changing things and end up with a tutorials job that isn't really doing its job without us noticing. The tutorials-fetching script can now also be re-used in a separate GitHub Actions workflow that will handle the full tutorials-included documentation build and deployment in the future. The notebook-convertion step is moved into Python space, using `nbconvert` as a library in order to parallelise the build process for the tutorials, and to allow CI and developers calling `tox` directly to specify the output directories for the built tutorials. * Retarget tutorial-conversion utility as executor This reorganises the tutorial "conversion" utility to make it clearer that what it's actually doing is just executing the tutorials. The script itself is changed to default to editing the files inplace, while the `tox` job is updated to write the files into a special directory, making it easier to clean up a dirty build directory and making it so subsequent local executions will not pick up the converted files. * Allow configuration of tutorials execution There was a worry that not being able to configure these would make it more unpleasant to use `tox` for the jobs locally. --------- Co-authored-by: Jake Lishman (cherry picked from commit df2ddcadf3c6d59b8bab8b0b1ceff14234fa82b5) --- .azure/docs-linux.yml | 2 +- .azure/tutorials-linux.yml | 35 +++--- .gitignore | 1 + docs/conf.py | 44 ++++++-- docs/index.rst | 1 + docs/tutorials.rst | 37 +++++++ docs/tutorials/algorithms/placeholder.ipynb | 34 ++++++ docs/tutorials/circuits/placeholder.ipynb | 34 ++++++ .../circuits_advanced/placeholder.ipynb | 34 ++++++ docs/tutorials/operators/placeholder.ipynb | 34 ++++++ requirements-dev.txt | 9 +- requirements-tutorials.txt | 2 +- tools/execute_tutorials.py | 100 ++++++++++++++++++ tools/prepare_tutorials.bash | 61 +++++++++++ tox.ini | 10 ++ 15 files changed, 403 insertions(+), 35 deletions(-) create mode 100644 docs/tutorials.rst create mode 100644 docs/tutorials/algorithms/placeholder.ipynb create mode 100644 docs/tutorials/circuits/placeholder.ipynb create mode 100644 docs/tutorials/circuits_advanced/placeholder.ipynb create mode 100644 docs/tutorials/operators/placeholder.ipynb create mode 100644 tools/execute_tutorials.py create mode 100755 tools/prepare_tutorials.bash diff --git a/.azure/docs-linux.yml b/.azure/docs-linux.yml index 24c539b5e769..6e4e30491652 100644 --- a/.azure/docs-linux.yml +++ b/.azure/docs-linux.yml @@ -24,7 +24,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install -U "tox<4.4.0" sudo apt-get update - sudo apt-get install -y graphviz + sudo apt-get install -y graphviz pandoc displayName: 'Install dependencies' - bash: | diff --git a/.azure/tutorials-linux.yml b/.azure/tutorials-linux.yml index 3e8b3678cabd..197ca186615a 100644 --- a/.azure/tutorials-linux.yml +++ b/.azure/tutorials-linux.yml @@ -8,9 +8,7 @@ jobs: pool: {vmImage: 'ubuntu-latest'} variables: - QISKIT_SUPPRESS_PACKAGING_WARNINGS: Y PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip - QISKIT_CELL_TIMEOUT: 300 steps: - task: UsePythonVersion@0 @@ -20,40 +18,33 @@ jobs: - bash: | set -e - git clone https://github.com/Qiskit/qiskit-tutorials --depth=1 python -m pip install --upgrade pip setuptools wheel - python -m pip install -U \ - -c constraints.txt \ - -r requirements.txt \ - -r requirements-dev.txt \ - -r requirements-optional.txt \ - -r requirements-tutorials.txt \ - -e . + python -m pip install -U "tox<4.4.0" sudo apt-get update sudo apt-get install -y graphviz pandoc - pip check displayName: 'Install dependencies' - env: - SETUPTOOLS_ENABLE_FEATURES: "legacy-editable" - - bash: | - set -e - cd qiskit-tutorials - sphinx-build -b html . _build/html + - bash: tools/prepare_tutorials.bash algorithms circuits circuits_advanced operators + displayName: 'Download current tutorials' + + - bash: tox -e tutorials + displayName: "Execute tutorials" env: - QISKIT_PARALLEL: False + QISKIT_CELL_TIMEOUT: 300 - task: ArchiveFiles@2 inputs: - rootFolderOrFile: 'qiskit-tutorials/_build/html' + rootFolderOrFile: 'executed_tutorials' archiveType: tar - archiveFile: '$(Build.ArtifactStagingDirectory)/html_tutorials.tar.gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/executed_tutorials.tar.gz' verbose: true + condition: succeededOrFailed() - task: PublishBuildArtifacts@1 - displayName: 'Publish docs' + displayName: 'Publish updated tutorials' inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'html_tutorials' + artifactName: 'executed_tutorials' Parallel: true ParallelCount: 8 + condition: succeededOrFailed() diff --git a/.gitignore b/.gitignore index d131395fe509..4a7757bb3d0b 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ qiskit/transpiler/passes/**/cython/**/*.cpp qiskit/quantum_info/states/cython/*.cpp docs/stubs/* +executed_tutorials/ # Notebook testing images test/visual/mpl/circuit/circuit_results/*.png diff --git a/docs/conf.py b/docs/conf.py index d607712f4e54..42ab3bed8247 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,12 +14,12 @@ """Sphinx documentation builder.""" -# -- General configuration --------------------------------------------------- import datetime import doctest +import os project = "Qiskit" -copyright = f"2017-{datetime.date.today().year}, Qiskit Development Team" # pylint: disable=redefined-builtin +project_copyright = f"2017-{datetime.date.today().year}, Qiskit Development Team" author = "Qiskit Development Team" # The short X.Y version @@ -39,7 +39,8 @@ "reno.sphinxext", "sphinx_design", "matplotlib.sphinxext.plot_directive", - "sphinx.ext.doctest", + "qiskit_sphinx_theme", + "nbsphinx", ] templates_path = ["_templates"] @@ -77,7 +78,9 @@ "matplotlib": ("https://matplotlib.org/stable/", None), } -# -- Options for HTML output ------------------------------------------------- +# ---------------------------------------------------------------------------------- +# HTML theme +# ---------------------------------------------------------------------------------- html_theme = "qiskit_sphinx_theme" html_last_updated_fmt = "%Y/%m/%d" @@ -88,8 +91,9 @@ "style_external_links": True, } - -# -- Options for Autosummary and Autodoc ------------------------------------- +# ---------------------------------------------------------------------------------- +# Autodoc +# ---------------------------------------------------------------------------------- # Note that setting autodoc defaults here may not have as much of an effect as you may expect; any # documentation created by autosummary uses a template file (in autosummary in the templates path), @@ -131,7 +135,9 @@ napoleon_numpy_docstring = False -# -- Options for Doctest -------------------------------------------------------- +# ---------------------------------------------------------------------------------- +# Doctest +# ---------------------------------------------------------------------------------- doctest_default_flags = ( doctest.ELLIPSIS @@ -145,3 +151,27 @@ # >> code # output doctest_test_doctest_blocks = "" + +# ---------------------------------------------------------------------------------- +# Nbsphinx +# ---------------------------------------------------------------------------------- + +nbsphinx_timeout = int(os.getenv("QISKIT_CELL_TIMEOUT", "300")) +nbsphinx_execute = os.getenv("QISKIT_DOCS_BUILD_TUTORIALS", "never") +nbsphinx_widgets_path = "" +nbsphinx_thumbnails = {"**": "_static/images/logo.png"} + +nbsphinx_prolog = """ +{% set docname = env.doc2path(env.docname, base=None) %} + +.. only:: html + + .. role:: raw-html(raw) + :format: html + + .. note:: + This page was generated from `{{ docname }}`__. + + __ https://github.com/Qiskit/qiskit-terra/blob/main/{{ docname }} + +""" diff --git a/docs/index.rst b/docs/index.rst index 903fc64030c0..51de241b2351 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,6 +7,7 @@ Qiskit Terra documentation :hidden: How-to Guides + tutorials API References Explanation Migration Guides diff --git a/docs/tutorials.rst b/docs/tutorials.rst new file mode 100644 index 000000000000..5b230dd0e3a3 --- /dev/null +++ b/docs/tutorials.rst @@ -0,0 +1,37 @@ +.. _tutorials: + +========= +Tutorials +========= + +Quantum circuits +================ + +.. nbgallery:: + :glob: + + tutorials/circuits/* + +Advanced circuits +================= + +.. nbgallery:: + :glob: + + tutorials/circuits_advanced/* + +Algorithms +========== + +.. nbgallery:: + :glob: + + tutorials/algorithms/* + +Operators +========= + +.. nbgallery:: + :glob: + + tutorials/operators/* diff --git a/docs/tutorials/algorithms/placeholder.ipynb b/docs/tutorials/algorithms/placeholder.ipynb new file mode 100644 index 000000000000..6f1c1caae820 --- /dev/null +++ b/docs/tutorials/algorithms/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/tutorials/circuits/placeholder.ipynb b/docs/tutorials/circuits/placeholder.ipynb new file mode 100644 index 000000000000..6f1c1caae820 --- /dev/null +++ b/docs/tutorials/circuits/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/tutorials/circuits_advanced/placeholder.ipynb b/docs/tutorials/circuits_advanced/placeholder.ipynb new file mode 100644 index 000000000000..6f1c1caae820 --- /dev/null +++ b/docs/tutorials/circuits_advanced/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/tutorials/operators/placeholder.ipynb b/docs/tutorials/operators/placeholder.ipynb new file mode 100644 index 000000000000..6f1c1caae820 --- /dev/null +++ b/docs/tutorials/operators/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/requirements-dev.txt b/requirements-dev.txt index b373c7b08c3b..177180769d9a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -31,9 +31,10 @@ ddt>=1.2.0,!=1.4.0,!=1.4.3 # components of Terra use some of its optional dependencies in order to document # themselves. These are the requirements that are _only_ required for the docs # build, and are not used by Terra itself. - +Sphinx>=6.0 +qiskit-sphinx-theme~=1.13.0 +sphinx-design>=0.2.0 +nbsphinx~=0.9.2 +nbconvert~=7.7.1 # TODO: switch to stable release when 4.1 is released reno @ git+https://github.com/openstack/reno.git@81587f616f17904336cdc431e25c42b46cd75b8f -Sphinx>=5.0 -qiskit-sphinx-theme~=1.11.0 -sphinx-design>=0.2.0 diff --git a/requirements-tutorials.txt b/requirements-tutorials.txt index 5aa9d0c412c0..c87701dc97ad 100644 --- a/requirements-tutorials.txt +++ b/requirements-tutorials.txt @@ -2,7 +2,7 @@ # This may also include some requirements that are only in `requirements-dev.txt`, since those # aren't runtime dependencies or optionals of Terra. -networkx>=2.2 +networkx>=2.3 jupyter Sphinx nbsphinx diff --git a/tools/execute_tutorials.py b/tools/execute_tutorials.py new file mode 100644 index 000000000000..2ce0c0c07fe7 --- /dev/null +++ b/tools/execute_tutorials.py @@ -0,0 +1,100 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2023 +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +# pylint: disable=missing-function-docstring,broad-exception-caught + +""" +Utility script to parallelise the conversion of several Jupyter notebooks. + +If nbconvert starts offering built-in parallelisation this script can likely be dropped. +""" + +import argparse +import functools +import multiprocessing +import os +import pathlib +import sys +import typing + +import nbformat +from nbconvert.preprocessors import ExecutePreprocessor + + +def worker( + notebook_path: pathlib.Path, + in_root: pathlib.Path, + out_root: typing.Optional[pathlib.Path], + timeout: int = -1, +) -> typing.Optional[Exception]: + """Single parallel worker that spawns a Jupyter executor node, executes the given notebook + within it, and writes out the output.""" + try: + print(f"({os.getpid()}) Processing '{str(notebook_path)}'", flush=True) + processor = ExecutePreprocessor(timeout=timeout, kernel_name="python3") + with open(notebook_path, "r") as fptr: + notebook = nbformat.read(fptr, as_version=4) + # Run the notebook with the working directory set to the folder it resides in. + processor.preprocess(notebook, {"metadata": {"path": f"{notebook_path.parent}/"}}) + + # Ensure the output directory exists, and write to it. This overwrites the notebook with + # its executed form unless the '--out' flag was set. + out_root = in_root if out_root is None else out_root + out_path = out_root / notebook_path.relative_to(in_root) + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as fptr: + nbformat.write(notebook, fptr) + except Exception as exc: + return exc + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description="Execute tutorial Jupyter notebooks.") + parser.add_argument( + "notebook_dirs", type=pathlib.Path, nargs="*", help="Folders containing Jupyter notebooks." + ) + parser.add_argument( + "-o", + "--out", + type=pathlib.Path, + help="Output directory for files. Defaults to same location as input file, overwriting it.", + ) + parser.add_argument( + "-j", + "--num-processes", + type=int, + default=os.cpu_count(), + help="Number of processes to use.", + ) + args = parser.parse_args() + notebooks = sorted( + { + (notebook_path, in_root, args.out) + for in_root in args.notebook_dirs + for notebook_path in in_root.glob("**/*.ipynb") + } + ) + timeout = int(os.getenv("QISKIT_CELL_TIMEOUT", "300")) + print(f"Using {args.num_processes} process{'' if args.num_processes == 1 else 'es'}.") + with multiprocessing.Pool(args.num_processes) as pool: + failures = pool.starmap(functools.partial(worker, timeout=timeout), notebooks) + num_failures = 0 + for path, failure in zip(notebooks, failures): + if failure is not None: + print(f"'{path}' failed: {failure}", file=sys.stderr) + num_failures += 1 + return num_failures + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/prepare_tutorials.bash b/tools/prepare_tutorials.bash new file mode 100755 index 000000000000..bfa929423921 --- /dev/null +++ b/tools/prepare_tutorials.bash @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Clone the tutorials in from Qiskit/qiskit-tutorials, and put them in the right +# place in the documentation structure ready for a complete documentation build. +# In the initial transition from the metapackage structure, we are leaving +# placeholder files in the documentation directories so everything will build +# happily without the full structure having been cloned, but this may change in +# the future. +# +# If the placeholder files are removed in the future, this script will likely +# have become obsolete and the CI pipelines (or this script) should be updated +# to reflect that. +# +# Usage: +# prepare_tutorials.sh components ... +# +# components +# Subdirectories of `tutorials` in the tutorials repository that should be +# moved into the correct locations in the source tree. + +set -e + +if [[ $# -eq 0 ]]; then + echo "Usage: prepare_tutorials.sh components ..." >&2 + exit 1 +fi + +# Pull in the tutorials repository. +tmpdir="$(mktemp -d)" +git clone --depth=1 https://github.com/Qiskit/qiskit-tutorials "$tmpdir" +indir="${tmpdir}/tutorials" + +outdir="$(dirname "$(dirname "${BASH_SOURCE[0]}")")/docs/tutorials" +if [[ ! -d "$outdir" ]]; then + echo "Tutorials documentation directory '${outdir}' does not exist." >&2 + exit 2 +fi + +for component in "$@"; do + echo "Getting tutorials from '${component}'" + + if [[ ! -d "${indir}/${component}" ]]; then + echo "Component '${component}' not in tutorials repository." >&2 + exit 3 + fi + if [[ -d "${outdir}/${component}" && -f "${outdir}/${component}/placeholder.ipynb" ]]; then + rm "${outdir}/${component}/placeholder.ipynb" + if [[ -z "$(ls -A "${outdir}/${component}")" ]]; then + rm -r "${outdir}/${component}" + else + echo "Directory '${outdir}/${component}' contains files other than the placeholder. This script needs updating." >&2 + exit 4 + fi + else + echo "Directory '${outdir}/${component}' does not exist, or has no placeholder. This script needs updating." >&2 + exit 5 + fi + mv "${indir}/${component}" "${outdir}/${component}" +done + +rm -rf "${tmpdir}" diff --git a/tox.ini b/tox.ini index 36fb269f6133..d62dcc9fa6cc 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,7 @@ setenv = {[testenv]setenv} QISKIT_SUPPRESS_PACKAGING_WARNINGS=Y RUST_DEBUG=1 # Faster to compile. +passenv = {[testenv]passenv}, QISKIT_DOCS_BUILD_TUTORIALS deps = setuptools_rust # This is work around for the bug of tox 3 (see #8606 for more details.) -r{toxinidir}/requirements-dev.txt @@ -92,3 +93,12 @@ allowlist_externals = rm commands = rm -rf {toxinidir}/docs/stubs/ {toxinidir}/docs/_build + +[testenv:tutorials] +basepython = python3 +deps = + {[testenv:docs]deps} + -r{toxinidir}/requirements-tutorials.txt +passenv = {[testenv]passenv}, QISKIT_CELL_TIMEOUT +commands = + python tools/execute_tutorials.py {toxinidir}/docs/tutorials --out={toxinidir}/executed_tutorials {posargs}