apache · HyukjinKwon · May 29, 2020 · Jul 23, 2020 · Jul 23, 2020 · Jul 23, 2020
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -200,7 +200,9 @@ jobs:
         architecture: x64
     - name: Install Python linter dependencies
       run: |
-        pip3 install flake8 sphinx numpy
+        # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+        #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+        pip3 install flake8 'sphinx<3.1.0' numpy pydata_sphinx_theme
     - name: Install R 3.6
       uses: r-lib/actions/setup-r@v1
       with:
@@ -218,7 +220,9 @@ jobs:
     - name: Install dependencies for documentation generation
       run: |
         sudo apt-get install -y libcurl4-openssl-dev pandoc
-        pip install sphinx mkdocs numpy
+        # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+        #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+        pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme
         gem install jekyll jekyll-redirect-from rouge
         sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
     - name: Scala linter

diff --git a/.gitignore b/.gitignore
@@ -64,6 +64,7 @@ python/lib/pyspark.zip
 python/.eggs/
 python/deps
 python/docs/_site/
+python/docs/source/reference/api/
 python/test_coverage/coverage_data
 python/test_coverage/htmlcov
 python/pyspark/python

diff --git a/LICENSE b/LICENSE
@@ -223,7 +223,7 @@ Python Software Foundation License
 ----------------------------------
 
 pyspark/heapq3.py
-python/docs/_static/copybutton.js
+python/docs/source/_static/copybutton.js
 
 BSD 3-Clause
 ------------

diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
@@ -33,7 +33,10 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
-ARG PIP_PKGS="sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1"
+# TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+#   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+#   We should use the latest Sphinx version once this is fixed.
+ARG PIP_PKGS="sphinx==3.0.4 mkdocs==1.0.4 numpy==1.18.1 pydata_sphinx_theme==0.3.1"
 ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 
 # Install extra needed repos and refresh.

diff --git a/dev/lint-python b/dev/lint-python
@@ -173,14 +173,30 @@ function sphinx_test {
         return
     fi
 
-    # TODO(SPARK-32279): Install Sphinx in Python 3 of Jenkins machines
     PYTHON_HAS_SPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("sphinx") is not None)')
     if [[ "$PYTHON_HAS_SPHINX" == "False" ]]; then
         echo "$PYTHON_EXECUTABLE does not have Sphinx installed. Skipping Sphinx build for now."
         echo
         return
     fi
 
+    # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+    #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+    PYTHON_HAS_SPHINX_3_0=$("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion; import sphinx; print(LooseVersion(sphinx.__version__) < LooseVersion("3.1.0"))')
+    if [[ "$PYTHON_HAS_SPHINX_3_0" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE has Sphinx 3.1+ installed but it requires lower then 3.1. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
+    # TODO(SPARK-32391): Install pydata_sphinx_theme in Jenkins machines
+    PYTHON_HAS_THEME=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("pydata_sphinx_theme") is not None)')
+    if [[ "$PYTHON_HAS_THEME" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE does not have pydata_sphinx_theme installed. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
     echo "starting $SPHINX_BUILD tests..."
     pushd python/docs &> /dev/null
     make clean &> /dev/null

diff --git a/dev/requirements.txt b/dev/requirements.txt
@@ -3,3 +3,4 @@ jira==1.0.3
 PyGithub==1.26.0
 Unidecode==0.04.19
 sphinx
+pydata_sphinx_theme
diff --git a/dev/tox.ini b/dev/tox.ini
@@ -16,4 +16,4 @@
 [pycodestyle]
 ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504
 max-line-length=100
-exclude=python/pyspark/cloudpickle/*.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
+exclude=python/pyspark/cloudpickle/*.py,heapq3.py,shared.py,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
diff --git a/docs/README.md b/docs/README.md
@@ -57,8 +57,13 @@ Note: Other versions of roxygen2 might work in SparkR documentation generation b
 
 To generate API docs for any language, you'll need to install these libraries:
 
+<!--
+TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+See also https://github.com/sphinx-doc/sphinx/issues/7551.
+-->
+
 ```sh
-$ sudo pip install sphinx mkdocs numpy
+$ sudo pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme
 ```
 
 ## Generating the Documentation HTML

diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
@@ -126,8 +126,8 @@
     puts "Making directory api/python"
     mkdir_p "api/python"
 
-    puts "cp -r ../python/docs/_build/html/. api/python"
-    cp_r("../python/docs/_build/html/.", "api/python")
+    puts "cp -r ../python/docs/build/html/. api/python"
+    cp_r("../python/docs/build/html/.", "api/python")
   end
 
   if not (ENV['SKIP_RDOC'] == '1')

diff --git a/docs/img/spark-logo-reverse.png b/docs/img/spark-logo-reverse.png
diff --git a/python/docs/Makefile b/python/docs/Makefile
@@ -3,8 +3,8 @@
 # You can set these variables from the command line.
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     ?= .
-BUILDDIR      ?= _build
+SOURCEDIR     ?= source
+BUILDDIR      ?= build
 
 export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.9-src.zip)
 

diff --git a/python/docs/_static/pyspark.css b/python/docs/_static/pyspark.css
diff --git a/python/docs/_static/pyspark.js b/python/docs/_static/pyspark.js
diff --git a/python/docs/_templates/layout.html b/python/docs/_templates/layout.html
diff --git a/python/docs/index.rst b/python/docs/index.rst
diff --git a/python/docs/make2.bat b/python/docs/make2.bat
@@ -5,8 +5,8 @@ REM Command file for Sphinx documentation
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
 )
-set SOURCEDIR=.
-set BUILDDIR=_build
+set SOURCEDIR=source
+set BUILDDIR=build
 
 set PYTHONPATH=..;..\lib\py4j-0.10.9-src.zip