Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions .github/workflows/bazel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,13 @@ jobs:
- name: Check Java import boundaries
run: bash tools/ci/check-java-import-boundaries

# Behavioral tests for the CI shell this workflow depends on. actionlint
# covers syntax; these cover the decisions.
- name: Test CI shell helpers
run: |
bash tools/ci/test-bazel-cache-upload-mode
bash tools/ci/test-bazel-remote-probe

- name: Compute changed subtrees
id: detect
env:
Expand Down Expand Up @@ -430,14 +437,14 @@ jobs:
if [ -n "$CACHE_TOKEN" ] && [ -n "$CACHE_ENDPOINT" ]; then
printf '%s\n' "${{ vars.BAZEL_REMOTE_CACHE_CA }}" > "$RUNNER_TEMP/cache-ca.pem"
echo "CACHE_READY=1" >> "$GITHUB_ENV"
# Upload results only from main pushes; every other trigger
# (PRs) is read-only so it can never poison the cache.
if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then
upload=true
fi
# Who may write to the shared cache lives in a script so it is
# testable and stated once; see tools/ci/bazel-cache-upload-mode
# and its test for the rationale and the full event matrix.
upload="$(bash "$GITHUB_WORKSPACE/tools/ci/bazel-cache-upload-mode")"
echo "CACHE_UPLOAD=$upload" >> "$GITHUB_ENV"
# Make the mode visible in the log so a missing warm-write is never
# a silent guess: read-only PRs show upload=false, main pushes true.
# a silent guess: read-only PRs show upload=false, merge-queue runs
# and main pushes true.
if [ "$upload" = "true" ]; then
echo "remote cache ready: read-write (warming; upload=true)"
else
Expand Down Expand Up @@ -808,7 +815,7 @@ jobs:
printf '%s\n' "$CACHE_CA" > "$RUNNER_TEMP/cache-ca.pem"
echo "CACHE_READY=1" >> "$GITHUB_ENV"
upload=false
if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then upload=true; fi
upload="$(bash "$GITHUB_WORKSPACE/tools/ci/bazel-cache-upload-mode")"
echo "CACHE_UPLOAD=$upload" >> "$GITHUB_ENV"
echo "remote cache ready (upload=$upload)"
else
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,40 @@
# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
Comment thread
balajinvda marked this conversation as resolved.
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
# Env var is set but the file is missing or unreadable. Treat it
# the same as unset and degrade to local-only rather than failing
Expand Down
21 changes: 12 additions & 9 deletions src/compute-plane-services/nvca/scripts/.bazel-remote-probe
Original file line number Diff line number Diff line change
@@ -1,36 +1,39 @@
#!/usr/bin/env bash
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
else
probe_args=(-d '{"instance_name":""}')
cache_url="grpc://${host}:${port}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,40 @@
# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
# Env var is set but the file is missing or unreadable. Treat it
# the same as unset and degrade to local-only rather than failing
Expand Down
Original file line number Diff line number Diff line change
@@ -1,36 +1,39 @@
#!/usr/bin/env bash
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
else
probe_args=(-d '{"instance_name":""}')
cache_url="grpc://${host}:${port}"
Expand Down
10 changes: 6 additions & 4 deletions src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,12 @@ go_test(
local = True,
# external: testcontainers spins up the localstack Docker container,
# so the test's effective inputs (Docker daemon, localstack image
# tag) live outside Bazel's hermetic action hash. `local = True`
# forces local-only execution but does NOT bypass result caching;
# the `external` tag is what stops nvcfbarn from pinning a green
# forever. requires-docker is kept so callers can filter out
# tag) live outside Bazel's hermetic action hash. `local = True` forces
# local, unsandboxed execution and, measured on Bazel 9.1.1, also stops
# the result being reused from the disk/remote cache. It does not disable
# every cache: the `external` tag is what prevents Bazel reusing a
# previously cached test result. requires-docker is kept so callers
# can filter out
# docker-needing tests with --test_tag_filters=-requires-docker.
tags = [
"external",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,40 @@
# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
# Env var is set but the file is missing or unreadable. Treat it
# the same as unset and degrade to local-only rather than failing
Expand Down
Loading
Loading