feat: add a broadcast channel for job state event notifications

.github/workflows/build.yml

@@ -61,7 +61,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           enable-cache: true
 
@@ -80,7 +80,7 @@ jobs:
         with:
           python-version: "3.12"
 
-      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           enable-cache: true
 
@@ -103,7 +103,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7.0.1
-      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           enable-cache: true
       - name: Generate license file
@@ -154,7 +154,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           enable-cache: true
 
@@ -203,7 +203,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           enable-cache: true
 
@@ -248,7 +248,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           enable-cache: true
 
@@ -293,7 +293,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           enable-cache: true
 

.github/workflows/codeql.yml

@@ -48,11 +48,11 @@ jobs:
           persist-credentials: false
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@7188fc363630916deb702c7fdcf4e481b751f97a # v4
+        uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4
         with:
           languages: actions
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4
+        uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4
         with:
           category: "/language:actions"

.github/workflows/rust.yml

@@ -238,22 +238,16 @@ jobs:
           rustup toolchain install stable
           rustup default stable
       - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
-      - name: Install cargo-tomlfmt
-        uses: taiki-e/install-action@7572810d7dd469b651bb7793945692cf78da5dd7 # v2.85.0
+      - name: Install taplo
+        uses: taiki-e/install-action@67729d5c413db75907f0ad1e39bb04b9c868ff60 # v2.85.7
         with:
-          tool: cargo-tomlfmt@0.2.1
+          tool: taplo-cli@0.10.0
 
       - name: Check Cargo.toml formatting
         run: |
-          # if you encounter error, try rerun the command below, finally run 'git diff' to
-          # check which Cargo.toml introduces formatting violation
-          #
-          # ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read
+          # if you encounter an error, run 'ci/scripts/rust_toml_fmt.sh --write'
+          # (or 'taplo format') locally to fix the formatting, then commit the result.
           ci/scripts/rust_toml_fmt.sh
-          if test -f "./Cargo.toml.bak"; then
-              echo "cargo tomlfmt found format violations"
-              exit 1
-          fi
 
   datafusion-proto-sync-check:
     name: Check vendored DataFusion proto is in sync

.github/workflows/tpcds.yml

@@ -52,14 +52,30 @@ on:
 
 jobs:
   tpcds-sf1:
-    name: TPC-DS SF1 (all queries, static planner)
+    name: TPC-DS SF1 (static planner, ${{ matrix.label }})
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-    # A passing run takes ~7 minutes. Cap the job well under the 6-hour
+    # A passing leg takes ~7 minutes. Cap the job well under the 6-hour
     # default so a hung query fails fast and frees the runner rather than
     # sitting for hours with no useful signal.
     timeout-minutes: 45
+    strategy:
+      # One red leg must not cancel the other; the two task-packing modes are
+      # independent signals.
+      fail-fast: false
+      matrix:
+        include:
+          - label: "1 partition per task"
+            task_args: ""
+            slug: "mpt1"
+          # 4 rather than a larger cap because the scheduler clamps the slice to
+          # the executor's free vcores (`budget.vcores.min(cap)`), and the
+          # executor below runs `--concurrent-tasks 4`. A higher cap would be
+          # unreachable and the label would overstate what is covered.
+          - label: "4 partitions per task"
+            task_args: "-c ballista.scheduler.max_partitions_per_task=4"
+            slug: "mpt4"
     steps:
       - name: Install dependencies
         run: |
@@ -76,6 +92,10 @@ jobs:
           rust-version: stable
 
       - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+        with:
+          # Share the build cache across all matrix legs — the compiled
+          # binaries are identical; only the per-suite CLI args differ.
+          shared-key: tpcds-sf1
 
       - name: Build Ballista binaries
         run: |
@@ -157,27 +177,22 @@ jobs:
           done
           nc -z 127.0.0.1 50051 || { echo "executor did not start"; exit 1; }
 
-          # Run the suite under the default (static) planner. The tpcds binary
-          # internally loops all non-skipped queries and exits non-zero on any
-          # failure, so a single invocation covers the whole suite.
+          # This matrix leg runs the suite under the default (static) planner at
+          # one task-packing setting. The tpcds binary internally loops all
+          # non-skipped queries and exits non-zero on any failure, so a single
+          # invocation covers the whole suite. The other leg runs in parallel.
           #
-          # Coverage for the adaptive planner (AQE on) and for single-partition
-          # execution is being added separately; both currently fail on
-          # pre-existing bugs.
-          run_suite() {
-            local label="$1"; shift
-            echo "::group::[$label] TPC-DS suite"
-            ./target/tpch-ci/tpcds \
-              --host 127.0.0.1 --port 50050 \
-              --path "$DATA_DIR" \
-              --partitions 16 \
-              --verify \
-              "$@"
-            echo "::endgroup::"
-          }
-
-          run_suite "static planner" \
-            -c datafusion.optimizer.prefer_hash_join=false
+          # Coverage for the adaptive planner (AQE on) is being added
+          # separately; it currently fails on pre-existing bugs.
+          echo "::group::[static planner, ${{ matrix.label }}] TPC-DS suite"
+          ./target/tpch-ci/tpcds \
+            --host 127.0.0.1 --port 50050 \
+            --path "$DATA_DIR" \
+            --partitions 16 \
+            --verify \
+            -c datafusion.optimizer.prefer_hash_join=false \
+            ${{ matrix.task_args }}
+          echo "::endgroup::"
 
       - name: Upload cluster logs on failure
         # `!success()` rather than `failure()` so a timed-out or cancelled
@@ -186,7 +201,7 @@ jobs:
         if: ${{ !success() }}
         uses: actions/upload-artifact@v7
         with:
-          name: tpcds-sf1-cluster-logs
+          name: tpcds-sf1-cluster-logs-${{ matrix.slug }}
           retention-days: 14
           path: |
             ${{ runner.temp }}/scheduler.log

.github/workflows/tpch.yml

@@ -98,7 +98,7 @@ jobs:
             -p ballista-benchmarks
 
       - name: Install tpchgen-cli
-        uses: taiki-e/install-action@7572810d7dd469b651bb7793945692cf78da5dd7 # v2.85.0
+        uses: taiki-e/install-action@67729d5c413db75907f0ad1e39bb04b9c868ff60 # v2.85.7
         with:
           tool: tpchgen-cli@2.0.2
 

.github/workflows/web-tui.yml

@@ -68,12 +68,12 @@ jobs:
       - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
 
       - name: Install Trunk
-        uses: taiki-e/install-action@7572810d7dd469b651bb7793945692cf78da5dd7 # v2.85.0
+        uses: taiki-e/install-action@67729d5c413db75907f0ad1e39bb04b9c868ff60 # v2.85.7
         with:
           tool: trunk@0.21.14
 
       - name: Install cargo-get
-        uses: taiki-e/install-action@7572810d7dd469b651bb7793945692cf78da5dd7 # v2.85.0
+        uses: taiki-e/install-action@67729d5c413db75907f0ad1e39bb04b9c868ff60 # v2.85.7
         with:
           tool: cargo-get@1.4.0