Serialize dynamic filters on execution plan nodes (HashJoin, Aggregate, Sort)

.asf.yaml

@@ -51,12 +51,20 @@ github:
     main:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_status_checks:
+        contexts:
+          - "Check License Header"
+          - "Use prettier to check formatting of documents"
+          - "Check Markdown Links"
+          - "Validate required_status_checks in .asf.yaml"
+          - "Spell Check with Typos"
+          - "Circular Dependency Check"
+          - "Detect Unused Dependencies"
     # needs to be updated as part of the release process
     # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names
     # https://github.com/apache/infrastructure-asfyaml?tab=readme-ov-file#branch-protection
-    # Keeping set of protected branches for future releases
-    # Meanwhile creating a prerelease script that will update the branch protection names
-    # automatically. Keep track on it https://github.com/apache/datafusion/issues/17134
+    # these branches protection blocks autogenerated during release process which is described in
+    # https://github.com/apache/datafusion/tree/main/dev/release#2-add-a-protection-to-release-candidate-branch
     branch-50:
       required_pull_request_reviews:
         required_approving_review_count: 1
@@ -66,66 +74,15 @@ github:
     branch-52:
       required_pull_request_reviews:
         required_approving_review_count: 1
-    branch-53:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-54:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-55:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-56:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-57:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-58:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-59:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-60:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-61:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-62:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-63:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-64:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-65:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-66:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-67:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-68:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-69:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
-    branch-70:
-      required_pull_request_reviews:
-        required_approving_review_count: 1
   pull_requests:
     # enable updating head branches of pull requests
     allow_update_branch: true
     allow_auto_merge: true
+    # auto-delete head branches after being merged
+    del_branch_on_merge: true
 
 # publishes the content of the `asf-site` branch to
 # https://datafusion.apache.org/
 publish:
   whoami: asf-site
+

.github/actions/setup-builder/action.yaml

@@ -46,3 +46,17 @@ runs:
       # https://github.com/actions/checkout/issues/766
       shell: bash
       run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+    - name: Remove unnecessary preinstalled software
+      shell: bash
+      run: |
+        echo "Disk space before cleanup:"
+        df -h 
+        apt-get clean
+        # remove tool cache: about 8.5GB (github has host /opt/hostedtoolcache mounted as /__t)
+        rm -rf /__t/* || true
+        # remove Haskell runtime: about 6.3GB (host /usr/local/.ghcup)
+        rm -rf /host/usr/local/.ghcup || true
+        # remove Android library: about 7.8GB (host /usr/local/lib/android)
+        rm -rf /host/usr/local/lib/android || true
+        echo "Disk space after cleanup:"
+        df -h

.github/dependabot.yml

@@ -23,6 +23,7 @@ updates:
       interval: weekly
     target-branch: main
     labels: [auto-dependencies]
+    open-pull-requests-limit: 15
     ignore:
       # major version bumps of arrow* and parquet are handled manually
       - dependency-name: "arrow*"
@@ -44,10 +45,27 @@ updates:
         patterns:
           - "prost*"
           - "pbjson*"
+
+        # Catch-all: group only minor/patch into a single PR,
+        # excluding deps we want always separate (and excluding arrow/parquet which have their own group)
+      all-other-cargo-deps:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        exclude-patterns:
+          - "arrow*"
+          - "parquet"
+          - "object_store"
+          - "sqlparser"
+          - "prost*"
+          - "pbjson*"
+        update-types:
+          - "minor"
+          - "patch"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     open-pull-requests-limit: 10
     labels: [auto-dependencies]
   - package-ecosystem: "pip"

.github/workflows/audit.yml

@@ -33,17 +33,22 @@ on:
     paths:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
-  
+
   merge_group:
 
+permissions:
+  contents: read
+
 jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Install cargo-audit
-        uses: taiki-e/install-action@f535147c22906d77695e11cb199e764aa610a4fc  # v2.62.46
+        uses: taiki-e/install-action@481c34c1cf3a84c68b5e46f4eccfc82af798415a  # v2.75.23
         with:
           tool: cargo-audit
       - name: Run audit check
+        # Note: you can ignore specific RUSTSEC issues using the `--ignore` flag ,for example:
+        # run: cargo audit --ignore RUSTSEC-2026-0001
         run: cargo audit

.github/workflows/breaking_changes_detector.yml

@@ -0,0 +1,142 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Detect semver-incompatible (breaking) API changes in crates modified by a PR.
+#
+# Only public workspace crates that have file changes are checked.
+# Internal crates (benchmarks, test-utils, sqllogictest, doc) are excluded.
+#
+# This workflow only runs cargo-semver-checks and uploads the result as an
+# artifact. The actual PR comment is posted by a companion workflow
+# (`breaking_changes_detector_comment.yml`) that picks up the artifact via
+# `workflow_run`.
+#
+# Why split it?
+#   "The GITHUB_TOKEN has read-only permissions in pull requests from forked
+#    repositories."
+#     https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+# A read-only token cannot post comments, so on fork PRs the previous
+# single-workflow design failed with HTTP 403. We can't simply broaden the
+# trigger here either: cargo-semver-checks compiles PR code (build.rs, proc
+# macros), so granting this job a write token would expose it to any code
+# in the PR. And ASF infra policy independently forbids `pull_request_target`
+# for any workflow that exposes GITHUB_TOKEN
+# (https://infra.apache.org/github-actions-policy.html). The companion
+# `workflow_run` workflow runs in the base-repo context with write access
+# and never executes PR code.
+
+name: "Detect breaking changes"
+
+on:
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  check-semver:
+    name: Check semver
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+
+      # For fork PRs, `origin` points to the fork, not the upstream repo.
+      # Explicitly fetch the base branch from the upstream repo so we have
+      # a valid baseline ref for both diff and semver-checks.
+      - name: Fetch base branch
+        env:
+          BASE_REF: ${{ github.base_ref }}
+          REPO: ${{ github.repository }}
+        run: git fetch "https://github.com/${REPO}.git" "${BASE_REF}:refs/remotes/origin/${BASE_REF}"
+
+      - name: Determine changed crates
+        id: changed_crates
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          PACKAGES=$(ci/scripts/changed_crates.sh changed-crates "origin/${BASE_REF}")
+          echo "packages=$PACKAGES" >> "$GITHUB_OUTPUT"
+          echo "Changed crates: $PACKAGES"
+
+      # `datafusion-substrait` (and crates that depend on it via sqllogictest)
+      # have a build script that calls protoc, which is not preinstalled on
+      # ubuntu-latest runners.
+      - name: Install Protobuf Compiler
+        if: steps.changed_crates.outputs.packages != ''
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler
+
+      - name: Install cargo-semver-checks
+        if: steps.changed_crates.outputs.packages != ''
+        uses: taiki-e/install-action@94cb46f8d6e437890146ffbd78a778b78e623fb2  # v2.74.0
+        with:
+          tool: cargo-semver-checks
+
+      - name: Run cargo-semver-checks
+        id: check_semver
+        if: steps.changed_crates.outputs.packages != ''
+        env:
+          BASE_REF: ${{ github.base_ref }}
+          PACKAGES: ${{ steps.changed_crates.outputs.packages }}
+        run: |
+          set +e
+          # `tee` lets cargo's output stream live into the Actions log
+          # while we also keep a copy for the PR comment.
+          ci/scripts/changed_crates.sh semver-check "origin/${BASE_REF}" $PACKAGES \
+            2>&1 | tee /tmp/semver-output.txt
+          EXIT_CODE=${PIPESTATUS[0]}
+          # Pass the result through an output instead of failing the job:
+          # a detected breaking change should surface as a PR comment, not a
+          # red check, so PR authors aren't confused by an intentional break.
+          if [ "$EXIT_CODE" -eq 0 ]; then
+            echo "result=success" >> "$GITHUB_OUTPUT"
+          else
+            echo "result=failure" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Stage the data the companion comment workflow needs into a single
+      # directory. We default the result to "success" so the comment
+      # workflow clears any stale comment when the check step is skipped
+      # (e.g. no published crates changed).
+      - name: Stage artifact for comment workflow
+        if: always()
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          CHECK_RESULT: ${{ steps.check_semver.outputs.result || 'success' }}
+        run: |
+          mkdir -p semver-artifact
+          echo "$PR_NUMBER" > semver-artifact/pr_number
+          echo "$CHECK_RESULT" > semver-artifact/result
+          if [ -f /tmp/semver-output.txt ]; then
+            sed 's/\x1b\[[0-9;]*m//g' /tmp/semver-output.txt > semver-artifact/logs
+          else
+            : > semver-artifact/logs
+          fi
+
+      - name: Upload artifact
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: semver-check-result
+          path: semver-artifact/
+          retention-days: 1