Merge remote-tracking branch 'upstream/master' into SPARK-32940

Author: tanelk

.github/workflows/build_and_test.yml

@@ -25,6 +25,8 @@ on:
     - '**'
     - '!branch-*.*'
   schedule:
+    # master, Hadoop 2
+    - cron: '0 1 * * *'
     # master
     - cron: '0 4 * * *'
     # branch-3.2
@@ -89,7 +91,7 @@ jobs:
         java:
           - 8
         hadoop:
-          - hadoop3.2
+          - ${{ needs.configure-jobs.outputs.hadoop }}
         hive:
           - hive2.3
         # TODO(SPARK-32246): We don't test 'streaming-kinesis-asl' for now.

.github/workflows/cancel_duplicate_workflow_runs.yml

@@ -29,7 +29,7 @@ jobs:
     name: "Cancel duplicate workflow runs"
     runs-on: ubuntu-latest
     steps:
-      - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # @master
+      - uses: potiuk/cancel-workflow-runs@4723494a065d162f8e9efd071b98e0126e00f866 # @master
         name: "Cancel duplicate workflow runs"
         with:
           cancelMode: allDuplicates

.github/workflows/labeler.yml

@@ -44,7 +44,7 @@ jobs:
     #
     # However, these are not in a published release and the current `main` branch
     # has some issues upon testing.
-    - uses: actions/[email protected]
+    - uses: actions/labeler@5f867a63be70efff62b767459b009290364495eb # pin@2.2.0
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"
         sync-labels: true

.github/workflows/notify_test_workflow.yml

@@ -33,7 +33,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: "Notify test workflow"
-        uses: actions/github-script@v3
+        uses: actions/github-script@f05a81df23035049204b043b50c3322045ce7eb3 # pin@v3
         if: ${{ github.base_ref == 'master' }}
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish_snapshot.yml

@@ -36,18 +36,18 @@ jobs:
           - branch-3.1
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@master
+      uses: actions/checkout@61b9e3751b92087fd0b06925ba6dd6314e06f089 # pin@master
       with:
         ref: ${{ matrix.branch }}
     - name: Cache Maven local repository
-      uses: actions/cache@v2
+      uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353 # pin@v2
       with:
         path: ~/.m2/repository
         key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
           snapshot-maven-
     - name: Install Java 8
-      uses: actions/setup-java@v1
+      uses: actions/setup-java@d202f5dbf7256730fb690ec59f6381650114feb2 # pin@v1
       with:
         java-version: 8
     - name: Publish snapshot

.github/workflows/stale.yml

@@ -27,7 +27,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected]
+    - uses: actions/stale@c201d45ef4b0ccbd3bb0616f93bae13e73d0a080 # pin@v1.1.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: >

.github/workflows/test_report.yml

@@ -29,14 +29,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Download test results to report
-      uses: dawidd6/action-download-artifact@v2
+      uses: dawidd6/action-download-artifact@6f8f427fb41886a66b82ea11a5a15d1454c79415 # pin@v2
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         workflow: ${{ github.event.workflow_run.workflow_id }}
         commit: ${{ github.event.workflow_run.head_commit.id }}
         workflow_conclusion: completed
     - name: Publish test report
-      uses: scacap/action-surefire-report@v1
+      uses: scacap/action-surefire-report@482f012643ed0560e23ef605a79e8e87ca081648 # pin@v1
       with:
         check_name: Report test results
         github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/update_build_status.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: "Update build status"
-        uses: actions/github-script@v3
+        uses: actions/github-script@f05a81df23035049204b043b50c3322045ce7eb3 # pin@v3
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

LICENSE-binary

@@ -478,7 +478,7 @@ org.typelevel:spire-util_2.12
 org.typelevel:algebra_2.12:jar
 org.typelevel:cats-kernel_2.12
 org.typelevel:machinist_2.12
-net.razorvine:pyrolite
+net.razorvine:pickle
 org.slf4j:jcl-over-slf4j
 org.slf4j:jul-to-slf4j
 org.slf4j:slf4j-api

R/pkg/NAMESPACE

@@ -352,9 +352,11 @@ exportMethods("%<=>%",
               "map_values",
               "map_zip_with",
               "max",
+              "max_by",
               "md5",
               "mean",
               "min",
+              "min_by",
               "minute",
               "monotonically_increasing_id",
               "month",