diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index fe01b92036377..29f98ed36e2ec 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -200,7 +200,9 @@ jobs: architecture: x64 - name: Install Python linter dependencies run: | - pip3 install flake8 sphinx numpy + # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. + # See also https://github.com/sphinx-doc/sphinx/issues/7551. + pip3 install flake8 'sphinx<3.1.0' numpy pydata_sphinx_theme - name: Install R 3.6 uses: r-lib/actions/setup-r@v1 with: @@ -218,7 +220,9 @@ jobs: - name: Install dependencies for documentation generation run: | sudo apt-get install -y libcurl4-openssl-dev pandoc - pip install sphinx mkdocs numpy + # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. + # See also https://github.com/sphinx-doc/sphinx/issues/7551. + pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme gem install jekyll jekyll-redirect-from rouge sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')" - name: Scala linter diff --git a/.gitignore b/.gitignore index 198fdee39be95..0d8addeb10e21 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ python/lib/pyspark.zip python/.eggs/ python/deps python/docs/_site/ +python/docs/source/reference/api/ python/test_coverage/coverage_data python/test_coverage/htmlcov python/pyspark/python diff --git a/LICENSE b/LICENSE index af2cdd275d28d..8cec4f5ea5379 100644 --- a/LICENSE +++ b/LICENSE @@ -223,7 +223,7 @@ Python Software Foundation License ---------------------------------- pyspark/heapq3.py -python/docs/_static/copybutton.js +python/docs/source/_static/copybutton.js BSD 3-Clause ------------ diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile index 44d602415b262..a02a6b7bccf27 100644 --- a/dev/create-release/spark-rm/Dockerfile +++ b/dev/create-release/spark-rm/Dockerfile @@ -33,7 +33,10 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true # These arguments are just for reuse and not really meant to be customized. ARG APT_INSTALL="apt-get install --no-install-recommends -y" -ARG PIP_PKGS="sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1" +# TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. +# See also https://github.com/sphinx-doc/sphinx/issues/7551. +# We should use the latest Sphinx version once this is fixed. +ARG PIP_PKGS="sphinx==3.0.4 mkdocs==1.0.4 numpy==1.18.1 pydata_sphinx_theme==0.3.1" ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0" # Install extra needed repos and refresh. diff --git a/dev/lint-python b/dev/lint-python index 1fddbfa64b32c..48dd94e36fae8 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -173,7 +173,6 @@ function sphinx_test { return fi - # TODO(SPARK-32279): Install Sphinx in Python 3 of Jenkins machines PYTHON_HAS_SPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("sphinx") is not None)') if [[ "$PYTHON_HAS_SPHINX" == "False" ]]; then echo "$PYTHON_EXECUTABLE does not have Sphinx installed. Skipping Sphinx build for now." @@ -181,6 +180,23 @@ function sphinx_test { return fi + # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. + # See also https://github.com/sphinx-doc/sphinx/issues/7551. + PYTHON_HAS_SPHINX_3_0=$("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion; import sphinx; print(LooseVersion(sphinx.__version__) < LooseVersion("3.1.0"))') + if [[ "$PYTHON_HAS_SPHINX_3_0" == "False" ]]; then + echo "$PYTHON_EXECUTABLE has Sphinx 3.1+ installed but it requires lower then 3.1. Skipping Sphinx build for now." + echo + return + fi + + # TODO(SPARK-32391): Install pydata_sphinx_theme in Jenkins machines + PYTHON_HAS_THEME=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("pydata_sphinx_theme") is not None)') + if [[ "$PYTHON_HAS_THEME" == "False" ]]; then + echo "$PYTHON_EXECUTABLE does not have pydata_sphinx_theme installed. Skipping Sphinx build for now." + echo + return + fi + echo "starting $SPHINX_BUILD tests..." pushd python/docs &> /dev/null make clean &> /dev/null diff --git a/dev/requirements.txt b/dev/requirements.txt index baea9213dbc97..a862a6e986791 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -3,3 +3,4 @@ jira==1.0.3 PyGithub==1.26.0 Unidecode==0.04.19 sphinx +pydata_sphinx_theme diff --git a/dev/tox.ini b/dev/tox.ini index ba5df084daad7..e25595aa6c9a6 100644 --- a/dev/tox.ini +++ b/dev/tox.ini @@ -16,4 +16,4 @@ [pycodestyle] ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504 max-line-length=100 -exclude=python/pyspark/cloudpickle/*.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/* +exclude=python/pyspark/cloudpickle/*.py,heapq3.py,shared.py,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/* diff --git a/docs/README.md b/docs/README.md index 22039871cf63d..e2002a66b0433 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,8 +57,13 @@ Note: Other versions of roxygen2 might work in SparkR documentation generation b To generate API docs for any language, you'll need to install these libraries: + + ```sh -$ sudo pip install sphinx mkdocs numpy +$ sudo pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ``` ## Generating the Documentation HTML diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 8e2a06e4bc9a8..17da22bf8a433 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -126,8 +126,8 @@ puts "Making directory api/python" mkdir_p "api/python" - puts "cp -r ../python/docs/_build/html/. api/python" - cp_r("../python/docs/_build/html/.", "api/python") + puts "cp -r ../python/docs/build/html/. api/python" + cp_r("../python/docs/build/html/.", "api/python") end if not (ENV['SKIP_RDOC'] == '1') diff --git a/docs/img/spark-logo-reverse.png b/docs/img/spark-logo-reverse.png new file mode 100644 index 0000000000000..a3e4ed4bb3d08 Binary files /dev/null and b/docs/img/spark-logo-reverse.png differ diff --git a/python/docs/Makefile b/python/docs/Makefile index 4272b7488d9a0..763f493a0eb58 100644 --- a/python/docs/Makefile +++ b/python/docs/Makefile @@ -3,8 +3,8 @@ # You can set these variables from the command line. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build -SOURCEDIR ?= . -BUILDDIR ?= _build +SOURCEDIR ?= source +BUILDDIR ?= build export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.9-src.zip) diff --git a/python/docs/_static/pyspark.css b/python/docs/_static/pyspark.css deleted file mode 100644 index 41106f2f6e26d..0000000000000 --- a/python/docs/_static/pyspark.css +++ /dev/null @@ -1,90 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -body { - background-color: #ffffff; -} - -div.sphinxsidebar { - width: 274px; -} - -div.bodywrapper { - margin: 0 0 0 274px; -} - -div.sphinxsidebar ul { - margin-right: 10px; -} - -div.sphinxsidebar li a { - word-break: break-all; -} - -span.pys-tag { - font-size: 11px; - font-weight: bold; - margin: 0 0 0 2px; - padding: 1px 3px 1px 3px; - -moz-border-radius: 3px; - -webkit-border-radius: 3px; - border-radius: 3px; - text-align: center; - text-decoration: none; -} - -span.pys-tag-experimental { - background-color: rgb(37, 112, 128); - color: rgb(255, 255, 255); -} - -span.pys-tag-deprecated { - background-color: rgb(238, 238, 238); - color: rgb(62, 67, 73); -} - -div.pys-note-experimental { - background-color: rgb(88, 151, 165); - border-color: rgb(59, 115, 127); - color: rgb(255, 255, 255); -} - -div.pys-note-deprecated { -} - -.hasTooltip { - position:relative; -} -.hasTooltip span { - display:none; -} - -.hasTooltip:hover span.tooltip { - display: inline-block; - -moz-border-radius: 2px; - -webkit-border-radius: 2px; - border-radius: 2px; - background-color: rgb(250, 250, 250); - color: rgb(68, 68, 68); - font-weight: normal; - box-shadow: 1px 1px 3px rgb(127, 127, 127); - position: absolute; - padding: 0 3px 0 3px; - top: 1.3em; - left: 14px; - z-index: 9999 -} diff --git a/python/docs/_static/pyspark.js b/python/docs/_static/pyspark.js deleted file mode 100644 index 75e4c42492a48..0000000000000 --- a/python/docs/_static/pyspark.js +++ /dev/null @@ -1,99 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -$(function (){ - - function startsWith(s, prefix) { - return s && s.indexOf(prefix) === 0; - } - - function buildSidebarLinkMap() { - var linkMap = {}; - $('div.sphinxsidebar a.reference.internal').each(function (i,a) { - var href = $(a).attr('href'); - if (startsWith(href, '#module-')) { - var id = href.substr(8); - linkMap[id] = [$(a), null]; - } - }) - return linkMap; - }; - - function getAdNoteDivs(dd) { - var noteDivs = {}; - dd.find('> div.admonition.note > p.last').each(function (i, p) { - var text = $(p).text(); - if (!noteDivs.experimental && startsWith(text, 'Experimental')) { - noteDivs.experimental = $(p).parent(); - } - if (!noteDivs.deprecated && startsWith(text, 'Deprecated')) { - noteDivs.deprecated = $(p).parent(); - } - }); - return noteDivs; - } - - function getParentId(name) { - var last_idx = name.lastIndexOf('.'); - return last_idx == -1? '': name.substr(0, last_idx); - } - - function buildTag(text, cls, tooltip) { - return '' + text + '' - + tooltip + '' - } - - - var sidebarLinkMap = buildSidebarLinkMap(); - - $('dl.class, dl.function').each(function (i,dl) { - - dl = $(dl); - dt = dl.children('dt').eq(0); - dd = dl.children('dd').eq(0); - var id = dt.attr('id'); - var desc = dt.find('> .descname').text(); - var adNoteDivs = getAdNoteDivs(dd); - - if (id) { - var parent_id = getParentId(id); - - var r = sidebarLinkMap[parent_id]; - if (r) { - if (r[1] === null) { - r[1] = $('