diff --git a/dev/lint-python b/dev/lint-python index df8df037a5f6..f7c65bf4f6ea 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -22,6 +22,7 @@ SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")" # Exclude auto-generated configuration file. PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" )" PEP8_REPORT_PATH="$SPARK_ROOT_DIR/dev/pep8-report.txt" +PYDOCSTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pydocstyle-report.txt" PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt" PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt" SPHINXBUILD=${SPHINXBUILD:=sphinx-build} @@ -42,7 +43,7 @@ PEP8_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pep8-$PEP8_VERSION.py" PEP8_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/jcrocholl/pep8/$PEP8_VERSION/pep8.py" if [ ! -e "$PEP8_SCRIPT_PATH" ]; then - curl --silent -o "$PEP8_SCRIPT_PATH" "$PEP8_SCRIPT_REMOTE_PATH" + curl --silent -o "$PEP8_SCRIPT_PATH" "$PEP8_SCRIPT_REMOTE_PATH" curl_status="$?" if [ "$curl_status" -ne 0 ]; then @@ -83,6 +84,53 @@ else rm "$PEP8_REPORT_PATH" fi +#### Python Document Style Checks #### + +# Get PYDOCSTYLE at runtime so that we don't rely on it being installed on the build server. +# Using pep257.py which is the single file version of pydocstyle. +PYDOCSTYLE_VERSION="0.2.1" +PYDOCSTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pydocstyle-$PYDOCSTYLE_VERSION.py" +PYDOCSTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pydocstyle/$PYDOCSTYLE_VERSION/pep257.py" + +if [ ! -e "$PYDOCSTYLE_SCRIPT_PATH" ]; then + curl --silent -o "$PYDOCSTYLE_SCRIPT_PATH" "$PYDOCSTYLE_SCRIPT_REMOTE_PATH" + curl_status="$?" + + if [ "$curl_status" -ne 0 ]; then + echo "Failed to download pep257.py from \"$PYDOCSTYLE_SCRIPT_REMOTE_PATH\"." + exit "$curl_status" + fi +fi + + +# There is no need to write this output to a file +#+ first, but we do so so that the check status can +#+ be output before the report, like with the +#+ scalastyle and RAT checks. +python "$PYDOCSTYLE_SCRIPT_PATH" $PATHS_TO_CHECK >> "$PYDOCSTYLE_REPORT_PATH" +pydocstyle_status=echo "${PIPESTATUS[@]}" | grep -re 1 + +if [ "$compile_status" -eq 0 -a "$pydocstyle_status" -eq 0 ]; then + lint_status=0 +else + lint_status=1 +fi + +if [ "$lint_status" -ne 0 ]; then + echo "pydocstyle checks failed." + cat "$PYDOCSTYLE_REPORT_PATH" + rm "$PYDOCSTYLE_REPORT_PATH" + # As there are doc failures currently, we want to verify but do not want to fail python lint check based on it now. + #exit "$lint_status" +else + echo "pydocstyle checks passed." + rm "$PYDOCSTYLE_REPORT_PATH" +fi + +rm "$PYDOCSTYLE_SCRIPT_PATH" + +#### Python Sphinx Document Style Checks #### + # Check that the documentation builds acceptably, skip check if sphinx is not installed. if hash "$SPHINXBUILD" 2> /dev/null; then cd python/docs diff --git a/dev/run-tests.py b/dev/run-tests.py index 7e6f7ff06035..6fac324972ee 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -576,7 +576,10 @@ def main(): for f in changed_files): # run_java_style_checks() pass - if not changed_files or any(f.endswith(".py") for f in changed_files): + if not changed_files or any(f.endswith("lint-python") + or f.endswith("tox.ini") + or f.endswith(".py") + for f in changed_files): run_python_style_checks() if not changed_files or any(f.endswith(".R") for f in changed_files): run_sparkr_style_checks()