diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 5033ab00601a..a4c9ec304258 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -751,7 +751,7 @@ jobs: # See also https://issues.apache.org/jira/browse/SPARK-35375. # Pin the MarkupSafe to 2.0.1 to resolve the CI error. # See also https://issues.apache.org/jira/browse/SPARK-38279. - python3.9 -m pip install 'sphinx<3.1.0' mkdocs pydata_sphinx_theme sphinx-copybutton nbsphinx numpydoc 'jinja2<3.0.0' 'markupsafe==2.0.1' 'pyzmq<24.0.0' + python3.9 -m pip install 'sphinx==4.2.0' mkdocs 'pydata_sphinx_theme==0.13' sphinx-copybutton nbsphinx numpydoc jinja2 'markupsafe==2.0.1' 'pyzmq<24.0.0' python3.9 -m pip install ipython_genutils # See SPARK-38517 python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' python3.9 -m pip install 'docutils<0.18.0' # See SPARK-39421 diff --git a/dev/requirements.txt b/dev/requirements.txt index 7de55ec24968..a7af0907c726 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -31,12 +31,12 @@ pandas-stubs<1.2.0.54 mkdocs # Documentation (Python) -pydata_sphinx_theme +pydata_sphinx_theme==0.13 ipython nbsphinx numpydoc -jinja2<3.0.0 -sphinx<3.1.0 +jinja2 +sphinx==4.2.0 sphinx-plotly-directive sphinx-copybutton docutils<0.18.0 diff --git a/python/docs/source/_static/spark-logo-dark.png b/python/docs/source/_static/spark-logo-dark.png new file mode 100644 index 000000000000..7460faec37fc Binary files /dev/null and b/python/docs/source/_static/spark-logo-dark.png differ diff --git a/python/docs/source/_static/spark-logo-light.png b/python/docs/source/_static/spark-logo-light.png new file mode 100644 index 000000000000..41938560822c Binary files /dev/null and b/python/docs/source/_static/spark-logo-light.png differ diff --git a/python/docs/source/_templates/autosummary/accessor_attribute.rst b/python/docs/source/_templates/autosummary/accessor_attribute.rst new file mode 100644 index 000000000000..28a94614b98f --- /dev/null +++ b/python/docs/source/_templates/autosummary/accessor_attribute.rst @@ -0,0 +1,6 @@ +{{ fullname }} +{{ underline }} + +.. currentmodule:: {{ module + "." + objname.split(".")[0] }} + +.. autoattribute:: {{ ".".join(objname.split(".")[1:]) }} diff --git a/python/docs/source/_templates/autosummary/accessor_method.rst b/python/docs/source/_templates/autosummary/accessor_method.rst new file mode 100644 index 000000000000..dce014d7b5da --- /dev/null +++ b/python/docs/source/_templates/autosummary/accessor_method.rst @@ -0,0 +1,6 @@ +{{ fullname }} +{{ underline }} + +.. currentmodule:: {{ module + "." + objname.split(".")[0] }} + +.. automethod:: {{ ".".join(objname.split(".")[1:]) }} diff --git a/python/docs/source/_templates/autosummary/class_with_docs.rst b/python/docs/source/_templates/autosummary/class_with_docs.rst index 7c37b83c0e90..1141fa68a256 100644 --- a/python/docs/source/_templates/autosummary/class_with_docs.rst +++ b/python/docs/source/_templates/autosummary/class_with_docs.rst @@ -47,7 +47,9 @@ .. autosummary:: {% for item in attributes %} - ~{{ name }}.{{ item }} + {% if not (item == 'uid') %} + ~{{ name }}.{{ item }} + {% endif %} {%- endfor %} {% endif %} diff --git a/python/docs/source/_templates/autosummary/plot_class.rst b/python/docs/source/_templates/autosummary/plot_class.rst new file mode 100644 index 000000000000..5e6a73bd0ecc --- /dev/null +++ b/python/docs/source/_templates/autosummary/plot_class.rst @@ -0,0 +1,53 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +{{ fullname }} +{{ underline }} + +.. currentmodule:: {{ module + "." + objname.split(".")[0] }} + +.. automethod:: {{ ".".join(objname.split(".")[1:]) }} + +{% if '__init__' in methods %} + {% set caught_result = methods.remove('__init__') %} +{% endif %} + +{% block methods %} +{% if methods %} + + .. rubric:: Methods + + .. autosummary:: + {% for item in methods %} + ~{{ name.split(".")[1] }}.{{ item }} + {%- endfor %} + +{% endif %} +{% endblock %} + +{% block attributes_summary %} +{% if attributes %} + + .. rubric:: Attributes + + .. autosummary:: + {% for item in attributes %} + ~{{ name.split(".")[1] }}.{{ item }} + {%- endfor %} + +{% endif %} +{% endblock %} diff --git a/python/docs/source/conf.py b/python/docs/source/conf.py index b9884d55b3a1..81083c007b34 100644 --- a/python/docs/source/conf.py +++ b/python/docs/source/conf.py @@ -194,7 +194,11 @@ # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "navbar_end": ["version-switcher"] + "navbar_end": ["version-switcher", "theme-switcher"], + "logo": { + "image_light": "_static/spark-logo-light.png", + "image_dark": "_static/spark-logo-dark.png", + } } # Add any paths that contain custom themes here, relative to this directory. diff --git a/python/docs/source/reference/pyspark.pandas/frame.rst b/python/docs/source/reference/pyspark.pandas/frame.rst index 911999b56be5..12cf6e7db12f 100644 --- a/python/docs/source/reference/pyspark.pandas/frame.rst +++ b/python/docs/source/reference/pyspark.pandas/frame.rst @@ -299,6 +299,7 @@ in Spark. These can be accessed by ``DataFrame.spark.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst DataFrame.spark.frame DataFrame.spark.cache @@ -319,8 +320,8 @@ specific plotting methods of the form ``DataFrame.plot.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst - DataFrame.plot DataFrame.plot.area DataFrame.plot.barh DataFrame.plot.bar @@ -330,6 +331,10 @@ specific plotting methods of the form ``DataFrame.plot.``. DataFrame.plot.pie DataFrame.plot.scatter DataFrame.plot.density + +.. autosummary:: + :toctree: api/ + DataFrame.hist DataFrame.boxplot DataFrame.kde @@ -341,6 +346,7 @@ These can be accessed by ``DataFrame.pandas_on_spark.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst DataFrame.pandas_on_spark.apply_batch DataFrame.pandas_on_spark.transform_batch diff --git a/python/docs/source/reference/pyspark.pandas/indexing.rst b/python/docs/source/reference/pyspark.pandas/indexing.rst index 7ec4387bb679..301e849ffe28 100644 --- a/python/docs/source/reference/pyspark.pandas/indexing.rst +++ b/python/docs/source/reference/pyspark.pandas/indexing.rst @@ -129,8 +129,14 @@ in Spark. These can be accessed by ``Index.spark.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_attribute.rst Index.spark.column + +.. autosummary:: + :toctree: api/ + :template: autosummary/accessor_method.rst + Index.spark.transform Sorting @@ -308,9 +314,15 @@ in Spark. These can be accessed by ``MultiIndex.spark.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_attribute.rst MultiIndex.spark.data_type MultiIndex.spark.column + +.. autosummary:: + :toctree: api/ + :template: autosummary/accessor_method.rst + MultiIndex.spark.transform MultiIndex Sorting diff --git a/python/docs/source/reference/pyspark.pandas/io.rst b/python/docs/source/reference/pyspark.pandas/io.rst index 118dd49a4ada..fd41a03699ca 100644 --- a/python/docs/source/reference/pyspark.pandas/io.rst +++ b/python/docs/source/reference/pyspark.pandas/io.rst @@ -69,6 +69,11 @@ Generic Spark I/O :toctree: api/ read_spark_io + +.. autosummary:: + :toctree: api/ + :template: autosummary/accessor_method.rst + DataFrame.spark.to_spark_io Flat File / CSV diff --git a/python/docs/source/reference/pyspark.pandas/series.rst b/python/docs/source/reference/pyspark.pandas/series.rst index 01fb5aa87fb1..88d1861c6ccf 100644 --- a/python/docs/source/reference/pyspark.pandas/series.rst +++ b/python/docs/source/reference/pyspark.pandas/series.rst @@ -270,8 +270,14 @@ in Spark. These can be accessed by ``Series.spark.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_attribute.rst Series.spark.column + +.. autosummary:: + :toctree: api/ + :template: autosummary/accessor_method.rst + Series.spark.transform Series.spark.apply @@ -304,6 +310,7 @@ Datetime Properties .. autosummary:: :toctree: api/ + :template: autosummary/accessor_attribute.rst Series.dt.date Series.dt.year @@ -333,6 +340,7 @@ Datetime Methods .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst Series.dt.normalize Series.dt.strftime @@ -353,6 +361,7 @@ like ``Series.str.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst Series.str.capitalize Series.str.cat @@ -416,10 +425,16 @@ the ``Series.cat`` accessor. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_attribute.rst Series.cat.categories Series.cat.ordered Series.cat.codes + +.. autosummary:: + :toctree: api/ + :template: autosummary/accessor_method.rst + Series.cat.rename_categories Series.cat.reorder_categories Series.cat.add_categories @@ -438,8 +453,8 @@ specific plotting methods of the form ``Series.plot.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst - Series.plot Series.plot.area Series.plot.bar Series.plot.barh @@ -449,6 +464,10 @@ specific plotting methods of the form ``Series.plot.``. Series.plot.line Series.plot.pie Series.plot.kde + +.. autosummary:: + :toctree: api/ + Series.hist Serialization / IO / Conversion @@ -476,6 +495,7 @@ These can be accessed by ``Series.pandas_on_spark.``. .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst Series.pandas_on_spark.transform_batch diff --git a/python/docs/source/reference/pyspark.sql/spark_session.rst b/python/docs/source/reference/pyspark.sql/spark_session.rst index f25dbab5f6b9..f242e4439cf4 100644 --- a/python/docs/source/reference/pyspark.sql/spark_session.rst +++ b/python/docs/source/reference/pyspark.sql/spark_session.rst @@ -29,12 +29,21 @@ See also :class:`SparkSession`. :toctree: api/ SparkSession.active + +.. autosummary:: + :toctree: api/ + :template: autosummary/accessor_method.rst + SparkSession.builder.appName SparkSession.builder.config SparkSession.builder.enableHiveSupport SparkSession.builder.getOrCreate SparkSession.builder.master SparkSession.builder.remote + +.. autosummary:: + :toctree: api/ + SparkSession.catalog SparkSession.conf SparkSession.createDataFrame @@ -58,8 +67,13 @@ Spark Connect Only .. autosummary:: :toctree: api/ + :template: autosummary/accessor_method.rst SparkSession.builder.create + +.. autosummary:: + :toctree: api/ + SparkSession.addArtifact SparkSession.addArtifacts SparkSession.copyFromLocalToFs