ci(bazel): source the bazel-ci container from a repo variable and run byoo first

.github/workflows/bazel.yml

@@ -52,13 +52,19 @@ concurrency:
   group: bazel-${{ github.ref }}
   cancel-in-progress: true
 
-# NOTE: keep the bazel-ci image tag as a string literal in
-# container.image and any tag-bump bookkeeping. GitHub Actions
-# evaluates job-level `container.image` before the workflow-level
-# `env:` context is reliably available, so referencing
-# `${{ env.BAZEL_CI_IMAGE }}` here causes the matrix to expand to
-# zero jobs on push events (validated empirically; PR-event runs
-# happen to work). The duplication is intentional.
+# The bazel-ci job container comes from the repository variable
+# BAZEL_CI_IMAGE, so the image is bumped in one place (repo settings) instead
+# of being edited in every workflow that runs in it.
+#
+# It must be `vars`, NOT `env`. GitHub Actions evaluates job-level
+# `container.image` before the workflow-level `env:` context is reliably
+# available, so `${{ env.BAZEL_CI_IMAGE }}` made the matrix expand to zero jobs
+# on push events (validated empirically; PR-event runs happened to work). The
+# `vars` context does not have that ordering problem.
+#
+# The `||` fallback keeps CI working if the variable is unset or unavailable
+# (for example on a fork). Keep the fallback in step with the variable; it is a
+# safety net, not the source of truth.
 
 jobs:
   detect:
@@ -108,7 +114,13 @@ jobs:
           # Row fields:
           #   id|path|tests_skip|component_kind|ci_lane
           # Static non-Java rows use only the first three fields.
-          ROWS='root|.|false
+          # Order matters: rows are dispatched top-down and max-parallel caps
+          # how many run at once, so a long row placed late waits for a slot
+          # before it even starts. byoo-otel-collector is the longest row (its
+          # collector genrule dominates the build), so it leads. Keep the
+          # longest-first ordering when adding rows.
+          ROWS='byoo-otel-collector|src/compute-plane-services/byoo-otel-collector|false
+          root|.|false
           grpc-proxy|src/invocation-plane-services/grpc-proxy|false|service|docker-host
           nats-auth-callout|src/control-plane-services/nats-auth-callout|false
           ratelimiter|src/invocation-plane-services/ratelimiter|false
@@ -118,7 +130,6 @@ jobs:
           nvca|src/compute-plane-services/nvca|false
           ess-agent|src/compute-plane-services/ess-agent|false
           image-credential-helper|src/compute-plane-services/image-credential-helper|false
-          byoo-otel-collector|src/compute-plane-services/byoo-otel-collector|false
           nvcf-unbound|src/compute-plane-services/nvcf-unbound|false
           worker-init|src/compute-plane-services/worker-init|false
           worker-llm-credentials|src/compute-plane-services/worker-llm-credentials|false
@@ -327,7 +338,7 @@ jobs:
       # Bazelisk then selects the root .bazelversion release. Update this tag
       # only after the corresponding internal image has been published and
       # mirrored.
-      image: ghcr.io/nvidia/nvcf/bazel-ci:0.13.0
+      image: ${{ vars.BAZEL_CI_IMAGE || 'ghcr.io/nvidia/nvcf/bazel-ci:0.13.0' }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/chart-push-manual.yml

@@ -34,7 +34,7 @@ jobs:
     name: package + push chart to ncp-dev
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/nvidia/nvcf/bazel-ci:0.8.0
+      image: ${{ vars.BAZEL_CI_IMAGE || 'ghcr.io/nvidia/nvcf/bazel-ci:0.13.0' }}
     defaults:
       run:
         # In container jobs Actions falls back to plain `sh` (dash), which

.github/workflows/image-push-manual.yml

@@ -18,21 +18,12 @@ on:
   workflow_dispatch:
     inputs:
       service_path:
-        description: Service subtree to build and push
+        description: >-
+          Service subtree to build and push, e.g.
+          src/invocation-plane-services/grpc-proxy. Any subtree with an
+          oci_image_index target works; the list is not maintained here.
         required: true
-        type: choice
-        options:
-          - src/invocation-plane-services/grpc-proxy
-          - src/invocation-plane-services/http-invocation
-          - src/invocation-plane-services/llm-api-gateway
-          - src/invocation-plane-services/ratelimiter
-          - src/invocation-plane-services/vanity-gateway
-          - src/control-plane-services/nats-auth-callout
-          - src/control-plane-services/function-autoscaler
-          - src/control-plane-services/helm-reval
-          - src/compute-plane-services/nvca
-          - src/compute-plane-services/ess-agent
-          - src/compute-plane-services/image-credential-helper
+        type: string
 
 permissions:
   contents: read
@@ -48,7 +39,7 @@ jobs:
     name: push to ncp-dev
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/nvidia/nvcf/bazel-ci:0.8.0
+      image: ${{ vars.BAZEL_CI_IMAGE || 'ghcr.io/nvidia/nvcf/bazel-ci:0.13.0' }}
     defaults:
       run:
         # In container jobs Actions falls back to plain `sh` (dash), which
@@ -116,28 +107,83 @@ jobs:
           printf '{"auths":{"%s":{"auth":"%s"}}}\n' "${REGISTRY}" "${auth}" > "$HOME/.docker/config.json"
           chmod 600 "$HOME/.docker/config.json"
 
+      # Resolve where Bazel must run and how to scope the query. A subtree with
+      # its own MODULE.bazel is queried from inside itself; a service that
+      # lives in the repo-root module (the Java services) is queried from the
+      # root with a path-scoped pattern. Deriving this rather than hardcoding
+      # it means a new service needs no change to this workflow.
+      - name: Resolve Bazel module layout
+        id: layout
+        env:
+          SVC_PATH: ${{ github.event.inputs.service_path }}
+        run: |
+          set -euo pipefail
+          if [ ! -d "$SVC_PATH" ]; then
+            echo "ERROR: no such subtree: $SVC_PATH" >&2
+            echo "Subtrees owning a Bazel module:" >&2
+            find src -maxdepth 4 -name MODULE.bazel -printf '  %h\n' 2>/dev/null | sort >&2
+            exit 1
+          fi
+          if [ -f "$SVC_PATH/MODULE.bazel" ]; then
+            echo "workdir=$SVC_PATH" >> "$GITHUB_OUTPUT"
+            echo "scope=//..." >> "$GITHUB_OUTPUT"
+            echo "layout: standalone module rooted at $SVC_PATH"
+          else
+            echo "workdir=." >> "$GITHUB_OUTPUT"
+            echo "scope=//$SVC_PATH/..." >> "$GITHUB_OUTPUT"
+            echo "layout: root module, scoped to //$SVC_PATH/..."
+          fi
+
       - name: Build and push multi-arch image(s)
-        working-directory: ${{ github.event.inputs.service_path }}
+        working-directory: ${{ steps.layout.outputs.workdir }}
         env:
           SVC_PATH: ${{ github.event.inputs.service_path }}
+          SCOPE: ${{ steps.layout.outputs.scope }}
           TAG: ${{ steps.meta.outputs.tag }}
           REGISTRY: ${{ secrets.NCP_DEV_REGISTRY }}
         run: |
           set -euo pipefail
           REGISTRY="${REGISTRY%/}"
           svc="$(basename "$SVC_PATH")"
           export BAZEL_DISK_CACHE="${HOME}/.bazel-disk-cache"
-          mapfile -t indexes < <(bazel query --remote_cache= 'kind("oci_image_index", //...)')
+          # Run the query directly rather than inside process substitution.
+          # With `mapfile < <(bazel query ...)` only stdout reaches mapfile, so a
+          # failing query (BUILD error, unloadable package) yields an empty array
+          # and would be misreported below as "no image targets" instead of
+          # surfacing the real error.
+          if ! query_out="$(bazel query --remote_cache= "kind(\"oci_image_index\", ${SCOPE})")"; then
+            echo "ERROR: bazel query failed for scope ${SCOPE}" >&2
+            exit 1
+          fi
+          # Build the array by hand: a here-string of empty output would produce
+          # a single empty element rather than an empty array.
+          indexes=()
+          while IFS= read -r line; do
+            [ -n "$line" ] && indexes+=("$line")
+          done <<< "$query_out"
           if [ "${#indexes[@]}" -eq 0 ]; then
             echo "ERROR: no oci_image_index targets under ${SVC_PATH}" >&2
+            echo "The subtree must declare an image target (go_oci_image, java_oci_image, ...)." >&2
             exit 1
           fi
           echo "discovered: ${indexes[*]}"
           for tgt in "${indexes[@]}"; do
             name="${tgt##*:}"; name="${name%_index}"
+            # Two naming conventions exist in the tree and they mean different
+            # things, distinguished by the separator:
+            #   image                 -> the service's sole image; repo is the service
+            #   <component>_image     -> a sub-component; repo is <service>-<component>
+            #                            (nvcf-unbound webhook, llm-api-gateway
+            #                            rate_limit_sync_worker, nvsnap agent/server)
+            #   <image-name>-image    -> the target already carries the full image
+            #                            name; use it as-is, do NOT prefix the
+            #                            service (byoo-otel-collector, cloud-tasks)
+            # Previously the hyphenated form fell through to the default and
+            # produced names like byoo-otel-collector-byoo-otel-collector-image.
             case "$name" in
               image)   repo="${svc}" ;;
               *_image) sub="$(printf '%s' "${name%_image}" | tr '_' '-')"; repo="${svc}-${sub}" ;;
+              *-image) repo="${name%-image}" ;;
               *)       sub="$(printf '%s' "$name" | tr '_' '-')"; repo="${svc}-${sub}" ;;
             esac
             dest="${REGISTRY}/${repo}"

BAZEL.md

@@ -433,13 +433,20 @@ build).
 
 ## CI
 
-The public GitHub Bazel matrix in `.github/workflows/bazel.yml` consumes
-`ghcr.io/nvidia/nvcf/bazel-ci:0.12.0`. That image is built in the internal
+Every workflow that needs the Bazel toolchain runs in the `bazel-ci` job
+container, sourced from the repository variable `BAZEL_CI_IMAGE` (currently
+`ghcr.io/nvidia/nvcf/bazel-ci`). Each workflow carries the current pin as a
+`||` fallback so CI still runs when the variable is unavailable, for example on
+a fork.
+
+That image is built in the internal
 [`nvcf/bazel-ci-templates`](https://gitlab-master.nvidia.com/nvcf/bazel-ci-templates)
 project, stamped with a version, and mirrored to GHCR. The mirror is currently
 manual; automation is planned. To change the image's Bazel, Java, or operating
 system tooling, update the internal template first, publish and mirror a new
-tag, and only then update the pinned `container.image` in `.github/workflows/bazel.yml`.
+tag, then update the `BAZEL_CI_IMAGE` repository variable. Do not hardcode the
+tag per workflow: it drifted to three different versions across workflows and
+this document before the variable was introduced.
 
 The root `ci/Dockerfile.bazel` and `.github/workflows/bazel-ci-image.yml` were a
 stale, divergent copy (no Java, older Bazel) and have been removed. The image is