Merge branch 'master' into sort_values

Author: ivanovmg

.github/actions/build_pandas/action.yml

@@ -0,0 +1,17 @@
+name: Build pandas
+description: Rebuilds the C extensions and installs pandas
+runs:
+  using: composite
+  steps:
+
+    - name: Environment Detail
+      run: |
+        conda info
+        conda list
+      shell: bash -l {0}
+
+    - name: Build Pandas
+      run: |
+        python setup.py build_ext -j 2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
+      shell: bash -l {0}

.github/actions/setup/action.yml

@@ -0,0 +1,12 @@
+name: Set up pandas
+description: Runs all the setup steps required to have a built pandas ready to use
+runs:
+  using: composite
+  steps:
+    - name: Setting conda path
+      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
+      shell: bash -l {0}
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+      shell: bash -l {0}

.github/workflows/ci.yml

@@ -41,15 +41,8 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Environment Detail
-      run: |
-        conda info
-        conda list
-
     - name: Build Pandas
-      run: |
-        python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+      uses: ./.github/actions/build_pandas
 
     - name: Linting
       run: ci/code_checks.sh lint
@@ -100,14 +93,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-
     - name: Checkout
       uses: actions/checkout@v1
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
+    - name: Set up pandas
+      uses: ./.github/actions/setup
 
     - name: Build website
       run: |
@@ -139,21 +129,41 @@ jobs:
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
       if: github.event_name == 'push'
 
+    - name: Move docs into site directory
+      run: mv doc/build/html web/build/docs
+    - name: Save website as an artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: website
+        path: web/build
+        retention-days: 14
+
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
     steps:
 
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-
     - name: Checkout
       uses: actions/checkout@v1
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
+    - name: Set up pandas
+      uses: ./.github/actions/setup
 
     - name: Run tests
       run: |
         source activate pandas-dev
         pytest pandas/tests/frame/methods --array-manager
+        pytest pandas/tests/frame/test_reductions.py --array-manager
+        pytest pandas/tests/reductions/ --array-manager
+        pytest pandas/tests/generic/test_generic.py --array-manager
+        pytest pandas/tests/arithmetic/ --array-manager
+        pytest pandas/tests/groupby/aggregate/ --array-manager
+        pytest pandas/tests/reshape/merge --array-manager
+
+        # indexing subset (temporary since other tests don't pass yet)
+        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean --array-manager
+        pytest pandas/tests/frame/indexing/test_where.py --array-manager
+        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_multi_index --array-manager
+        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns --array-manager
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups --array-manager
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column --array-manager

.github/workflows/comment_bot.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Install-pre-commit
         run: python -m pip install --upgrade pre-commit
       - name: Run pre-commit
-        run: pre-commit run --all-files || (exit 0)
+        run: pre-commit run --from-ref=origin/master --to-ref=HEAD --all-files || (exit 0)
       - name: Commit results
         run: |
           git config user.name "$(git log -1 --pretty=format:%an)"

.github/workflows/database.yml

@@ -72,15 +72,8 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Environment Detail
-      run: |
-        conda info
-        conda list
-
     - name: Build Pandas
-      run: |
-        python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+      uses: ./.github/actions/build_pandas
 
     - name: Test
       run: ci/run_tests.sh
@@ -158,15 +151,8 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Environment Detail
-      run: |
-        conda info
-        conda list
-
     - name: Build Pandas
-      run: |
-        python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+      uses: ./.github/actions/build_pandas
 
     - name: Test
       run: ci/run_tests.sh
@@ -184,3 +170,11 @@ jobs:
 
     - name: Print skipped tests
       run: python ci/print_skipped.py
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        files: /tmp/test_coverage.xml
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: true

.pre-commit-config.yaml

@@ -24,12 +24,12 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.4
+    rev: v2.10.0
     hooks:
     -   id: pyupgrade
-        args: [--py37-plus]
+        args: [--py37-plus, --keep-runtime-typing]
 -   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.7.0
+    rev: v1.7.1
     hooks:
       - id: rst-backticks
       - id: rst-directive-colons
@@ -127,12 +127,18 @@ repos:
         types: [python]
         files: ^pandas/tests/
         exclude: ^pandas/tests/extension/
+    -   id: unwanted-patters-pytest-xfail
+        name: Check for use of pytest.xfail
+        entry: pytest\.xfail
+        language: pygrep
+        types: [python]
+        files: ^pandas/tests/
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace in tests'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py
         language: python
         types: [python]
-        files: ^pandas/tests/
+        files: ^pandas/tests/frame/
     -   id: FrameOrSeriesUnion
         name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
         entry: Union\[.*(Series,.*DataFrame|DataFrame,.*Series).*\]
@@ -157,6 +163,11 @@ repos:
         entry: np\.bool[^_8]
         language: pygrep
         types_or: [python, cython, rst]
+    -   id: np-object
+        name: Check for use of np.object instead of np.object_
+        entry: np\.object[^_8]
+        language: pygrep
+        types_or: [python, cython, rst]
     -   id: no-os-remove
         name: Check code for instances of os.remove
         entry: os\.remove
@@ -174,6 +185,12 @@ repos:
         language: pygrep
         types: [python]
         files: ^pandas/tests/
+    -   id: title-capitalization
+        name: Validate correct capitalization among titles in documentation
+        entry: python scripts/validate_rst_title_capitalization.py
+        language: python
+        types: [rst]
+        files: ^doc/source/(development|reference)/
 -   repo: https://github.com/asottile/yesqa
     rev: v1.2.2
     hooks:
@@ -192,6 +209,11 @@ repos:
         files: ^pandas/
         exclude: ^pandas/tests/
 -   repo: https://github.com/MarcoGorelli/no-string-hints
-    rev: v0.1.6
+    rev: v0.1.7
     hooks:
     -   id: no-string-hints
+-   repo: https://github.com/MarcoGorelli/abs-imports
+    rev: v0.1.2
+    hooks:
+    -   id: abs-imports
+        files: ^pandas/

MANIFEST.in

@@ -1,9 +1,4 @@
-include MANIFEST.in
-include LICENSE
 include RELEASE.md
-include README.md
-include setup.py
-include pyproject.toml
 
 graft doc
 prune doc/build
@@ -16,10 +11,12 @@ global-exclude *.bz2
 global-exclude *.csv
 global-exclude *.dta
 global-exclude *.feather
+global-exclude *.tar
 global-exclude *.gz
 global-exclude *.h5
 global-exclude *.html
 global-exclude *.json
+global-exclude *.jsonl
 global-exclude *.pickle
 global-exclude *.png
 global-exclude *.pyc
@@ -40,6 +37,11 @@ global-exclude .DS_Store
 global-exclude .git*
 global-exclude \#*
 
+# GH 39321
+# csv_dir_path fixture checks the existence of the directory
+# exclude the whole directory to avoid running related tests in sdist
+prune pandas/tests/io/parser/data
+
 include versioneer.py
 include pandas/_version.py
 include pandas/io/formats/templates/*.tpl

Makefile

@@ -25,16 +25,3 @@ doc:
 	cd doc; \
 	python make.py clean; \
 	python make.py html
-
-check:
-	python3 scripts/validate_unwanted_patterns.py \
-		--validation-type="private_function_across_module" \
-		--included-file-extensions="py" \
-		--excluded-file-paths=pandas/tests,asv_bench/ \
-		pandas/
-
-	python3 scripts/validate_unwanted_patterns.py \
-		--validation-type="private_import_across_module" \
-		--included-file-extensions="py" \
-		--excluded-file-paths=pandas/tests,asv_bench/,doc/
-		pandas/

asv_bench/benchmarks/algos/__init__.py

@@ -0,0 +1,12 @@
+"""
+algos/ directory is intended for individual functions from core.algorithms
+
+In many cases these algorithms are reachable in multiple ways:
+   algos.foo(x, y)
+   Series(x).foo(y)
+   Index(x).foo(y)
+   pd.array(x).foo(y)
+
+In most cases we profile the Series variant directly, trusting the performance
+of the others to be highly correlated.
+"""