diff --git a/Makefile b/Makefile index 56d5b14494..892f7a6165 100644 --- a/Makefile +++ b/Makefile @@ -194,6 +194,7 @@ script-test: $(call run-timed,bash internal/scaffold/fullsend-repo/scripts/pre-fetch-prior-review-test.sh) $(call run-timed,bash internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env-test.sh) $(call run-timed,bash hack/gitlab-runner-vm/executor/prepare_validation_test.sh) + $(call run-timed,bash hack/gitlab-runner-vm/executor/gateway_test.sh) $(call run-timed,bash hack/gitlab-runner-vm/lib_test.sh) $(call run-timed,bash internal/runtime/kill_stray_processes_test.sh) $(call run-timed,python3 skills/topissues/scripts/topissues_test.py) diff --git a/docs/guides/dev/cli-internals.md b/docs/guides/dev/cli-internals.md index fa416a333e..49a295aa4b 100644 --- a/docs/guides/dev/cli-internals.md +++ b/docs/guides/dev/cli-internals.md @@ -412,11 +412,14 @@ Vendoring commit messages use title + body (upload and stale delete). `github st │ └──────┬───────────┘ │ │ ▼ │ │ ┌──────────────────┐ │ -│ │ ImportProfile() │ Import openshell provider profiles │ +│ │ ImportProfileVerified() │ Import openshell provider profiles │ │ │ │ (from resolved openshell.profiles; │ -│ │ │ on GitLab, a fullsend-gitlab-forge │ -│ │ │ profile is auto-generated from the │ -│ │ │ forge host URL — see #6615) │ +│ │ │ drops the os.TempDir() content cache │ +│ │ │ and confirms the gateway lists each │ +│ │ │ profile — see #7218. On GitLab, a │ +│ │ │ fullsend-gitlab-forge profile is │ +│ │ │ auto-generated from the forge host URL │ +│ │ │ — see #6615) │ │ └──────┬───────────┘ │ │ ▼ │ │ ┌──────────────────┐ │ @@ -581,7 +584,8 @@ details, see [Agent runtimes](../../runtimes.md). |-----------|------------|---------| | `EnsureAvailable()` | Check `openshell` binary | Verify runtime available | | `CheckGateway()` | `openshell gateway ...` | Start inference gateway | -| `ImportProfile()` | `openshell provider profile import ...` | Import openshell provider profile | +| `ImportProfile()` | `openshell provider profile import ...` | Import openshell provider profile (hash-cached) | +| `ImportProfileVerified()` | Forget cache → import → `list-profiles` | Same, then confirm the gateway lists it (#7218) | | `EnsureProvider()` | `openshell provider ...` | Register model provider (bare-key form) | | `Create()` | `openshell sandbox create --image ...` | Spin up container | | `Exec()` | `openshell sandbox exec ...` | Run command in sandbox | diff --git a/hack/gitlab-runner-vm/README.md b/hack/gitlab-runner-vm/README.md index 6d89eb71d2..3a7a416d47 100644 --- a/hack/gitlab-runner-vm/README.md +++ b/hack/gitlab-runner-vm/README.md @@ -9,13 +9,21 @@ fullsend agent jobs. Each runner VM runs: - **gitlab-runner** (custom executor) — receives CI jobs from GitLab - **Podman** (rootless) — creates per-job containers -- **OpenShell gateway** — provides sandbox compute for fullsend agents +- **OpenShell gateway** — started per job in `prepare.sh`, torn down in + `cleanup.sh`. The VM does **not** keep a long-lived `systemd --user` + gateway: that accumulated a stale profile registry, a baked-in OpenShell + version, and leaked sandboxes (#7218). Each job recreates the gateway + with an empty registry, matching GitHub Actions (fresh, version-matched + install, throw the runner away). The ~6 GB image cache stays warm; only + the CLI (~39 MB) and a version-skewed supervisor (~30 MB) are fetched + when the job's pin differs from the host. Job containers use `--network=host` to reach the gateway. An OCI `createRuntime` hook injects the host CA trust bundle into every container (Debian and RHEL-family layouts) so jobs can verify internal TLS endpoints. Gateway mTLS credentials are mounted read-only from the runner user's -OpenShell config. +OpenShell config. `prepare.sh` reaps leftover `openshell-*` / `openshell.managed` +containers from an abruptly-killed prior job before starting the new gateway. This is a deployment variant of the container isolation model described in ADR-0036. It uses a Podman custom executor instead of Docker/Kubernetes @@ -117,7 +125,7 @@ GCP_PROJECT=my-gcp-project ./delete-gcp-vm.sh --list | `RUNNER_IMAGE` | yes (create) | — | Image pre-pulled as a warm cache; jobs must still set `image:` in `.gitlab-ci.yml` | | `RUNNER_TAG` | no | `fullsend-gitlab-runner` | Runner tag for job matching | | `RUNNER_ACCESS_LEVEL` | no | `not_protected` | `ref_protected` restricts the runner to protected branches and tags, so merge-request pipelines on unprotected source refs never match and sit `pending`. In project mode, `not_protected` means any job on any branch of the project can run; in group mode, any tag-matched job on any branch of any project invited into the group tree runs on this VM (see Security below) | -| `OPENSHELL_VERSION` | no | from `.github/scripts/openshell-version.sh` | OpenShell version (Renovate-tracked) | +| `OPENSHELL_VERSION` | no | from `.github/scripts/openshell-version.sh` | OpenShell version used at **VM provision** (Renovate-tracked). Per-job `prepare.sh` re-reads the job image's `openshell --version` and upgrades the host CLI + supervisor when they differ, so a stale VM pin cannot stick. | | `GITLAB_RUNNER_VERSION` | no | `19.2.1` | gitlab-runner version | | `REGISTRATION_TOKEN` | setup only | — | GitLab runner registration token | @@ -154,9 +162,10 @@ GCP_PROJECT=my-gcp-project ./delete-gcp-vm.sh --list - `setup.sh` — standalone VM configuration (called by create-openshift-vm.sh / create-gcp-vm.sh) - `gitlab-runner-version.sh` — central pin for the gitlab-runner version - `vm.yaml` — KubeVirt VirtualMachine template (OpenShift only) -- `executor/prepare.sh` — custom executor prepare stage +- `executor/prepare.sh` — custom executor prepare stage (reaps leftover OpenShell containers, starts a per-job gateway matched to the job image's OpenShell version) - `executor/run.sh` — custom executor run stage -- `executor/cleanup.sh` — custom executor cleanup stage +- `executor/cleanup.sh` — custom executor cleanup stage (stops the gateway, wipes `~/.local/state/openshell/{gateway,tls}`, reaps sandboxes) +- `executor/gateway.sh` — shared per-job gateway helpers sourced by prepare/cleanup ## Security notes @@ -176,7 +185,9 @@ GCP_PROJECT=my-gcp-project ./delete-gcp-vm.sh --list - Job containers share the host network namespace (`--network=host`) to reach the OpenShell gateway. The gateway binds to `0.0.0.0` (required for the Podman compute driver — sandbox containers register via - `host.containers.internal`). mTLS protects the endpoint. + `host.containers.internal`). mTLS protects the endpoint. The gateway process + itself is per-job: `prepare.sh` starts it against a wiped store, `cleanup.sh` + stops it. A leftover from a killed job is reaped at the next prepare. - Job containers receive read-only access to the runner's gateway mTLS credentials (`~/.config/openshell`). This is required for the fullsend agent inside job containers to authenticate to the gateway. Credential diff --git a/hack/gitlab-runner-vm/create-gcp-vm.sh b/hack/gitlab-runner-vm/create-gcp-vm.sh index be5557e863..4001568cd1 100755 --- a/hack/gitlab-runner-vm/create-gcp-vm.sh +++ b/hack/gitlab-runner-vm/create-gcp-vm.sh @@ -284,7 +284,7 @@ for tool in gcloud python3 curl timeout sha256sum; do fi done for _f in setup.sh create-gcp-vm.sh gitlab-runner-version.sh \ - executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh; do + executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh executor/gateway.sh; do if [ ! -f "${SCRIPT_DIR}/${_f}" ]; then echo "ERROR: required file not found: ${SCRIPT_DIR}/${_f}" >&2 _missing=1 @@ -503,7 +503,7 @@ cp "${SCRIPT_DIR}/setup.sh" "${_stage_dir}/" cp "${SCRIPT_DIR}/create-gcp-vm.sh" "${_stage_dir}/" cp "${SCRIPT_DIR}/gitlab-runner-version.sh" "${_stage_dir}/" mkdir -p "${_stage_dir}/executor" -for file in job_id.sh prepare.sh run.sh cleanup.sh; do +for file in job_id.sh prepare.sh run.sh cleanup.sh gateway.sh; do cp "${SCRIPT_DIR}/executor/${file}" "${_stage_dir}/executor/" done mkdir -p "${_stage_dir}/.github/scripts" @@ -538,7 +538,7 @@ echo "==> Verifying copied files" verify_copied_files() { { (cd "${SCRIPT_DIR}" && sha256sum setup.sh create-gcp-vm.sh gitlab-runner-version.sh \ - executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh) + executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh executor/gateway.sh) (cd "${REPO_ROOT}/.github/scripts" \ && sha256sum install-openshell.sh openshell-version.sh \ | sed 's| | .github/scripts/|') diff --git a/hack/gitlab-runner-vm/create-openshift-vm.sh b/hack/gitlab-runner-vm/create-openshift-vm.sh index 09df20cda1..5f60f63d07 100755 --- a/hack/gitlab-runner-vm/create-openshift-vm.sh +++ b/hack/gitlab-runner-vm/create-openshift-vm.sh @@ -190,7 +190,7 @@ for tool in oc virtctl python3 curl timeout sha256sum; do fi done for _f in setup.sh create-openshift-vm.sh vm.yaml gitlab-runner-version.sh \ - executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh; do + executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh executor/gateway.sh; do if [ ! -f "${SCRIPT_DIR}/${_f}" ]; then echo "ERROR: required file not found: ${SCRIPT_DIR}/${_f}" >&2 _missing=1 @@ -402,7 +402,7 @@ for file in setup.sh create-openshift-vm.sh vm.yaml gitlab-runner-version.sh; do -c "cat > ~/gitlab-runner-vm/${file}" < "${SCRIPT_DIR}/${file}" done -for file in job_id.sh prepare.sh run.sh cleanup.sh; do +for file in job_id.sh prepare.sh run.sh cleanup.sh gateway.sh; do virtctl -n "${NAMESPACE}" ssh "${VM_USER}"@vm/"${vm_name}" \ -t "-o StrictHostKeyChecking=no" -t "-o UserKnownHostsFile=/dev/null" \ -c "cat > ~/gitlab-runner-vm/executor/${file}" < "${SCRIPT_DIR}/executor/${file}" @@ -425,7 +425,7 @@ virtctl -n "${NAMESPACE}" ssh "${VM_USER}"@vm/"${vm_name}" \ echo "==> Verifying copied files" { (cd "${SCRIPT_DIR}" && sha256sum setup.sh create-openshift-vm.sh vm.yaml gitlab-runner-version.sh \ - executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh) + executor/job_id.sh executor/prepare.sh executor/run.sh executor/cleanup.sh executor/gateway.sh) (cd "${REPO_ROOT}/.github/scripts" \ && sha256sum install-openshell.sh openshell-version.sh \ | sed 's| | .github/scripts/|') diff --git a/hack/gitlab-runner-vm/executor/cleanup.sh b/hack/gitlab-runner-vm/executor/cleanup.sh index d9adde03bb..3c93597032 100755 --- a/hack/gitlab-runner-vm/executor/cleanup.sh +++ b/hack/gitlab-runner-vm/executor/cleanup.sh @@ -1,32 +1,42 @@ #!/usr/bin/env bash # GitLab Runner custom executor — cleanup stage. -# Stops and removes the job container. Always succeeds. +# Stops and removes the job container and the per-job OpenShell gateway. +# Always succeeds. # -e intentionally omitted — cleanup must not abort on individual failures. set -uo pipefail # shellcheck source=job_id.sh source "$(dirname "${BASH_SOURCE[0]}")/job_id.sh" -JOB_ID=$(resolve_job_id) || { - echo "WARN: could not read job id from JOB_RESPONSE_FILE — nothing to clean up" - exit 0 -} +# shellcheck source=gateway.sh +source "$(dirname "${BASH_SOURCE[0]}")/gateway.sh" +JOB_ID=$(resolve_job_id) || JOB_ID="" STATE_DIR="${HOME}/.local/state/gitlab-runner" -STATE_FILE="${STATE_DIR}/container-${JOB_ID}" -if [ -f "${STATE_FILE}" ]; then - CONTAINER_NAME=$(cat "${STATE_FILE}") - if [[ "${CONTAINER_NAME}" =~ ^runner-[0-9]+$ ]]; then - echo "Cleaning up container: ${CONTAINER_NAME}" - podman stop --time 10 "${CONTAINER_NAME}" 2>/dev/null || true - podman rm -f "${CONTAINER_NAME}" 2>/dev/null || true - else - # Skip only the podman calls — the staging copy of the gateway mTLS - # material below must still be removed. - echo "WARN: state file holds an unexpected container name (${CONTAINER_NAME}) — not touching podman" +if [ -n "${JOB_ID}" ]; then + STATE_FILE="${STATE_DIR}/container-${JOB_ID}" + + if [ -f "${STATE_FILE}" ]; then + CONTAINER_NAME=$(cat "${STATE_FILE}") + if [[ "${CONTAINER_NAME}" =~ ^runner-[0-9]+$ ]]; then + echo "Cleaning up container: ${CONTAINER_NAME}" + podman stop --time 10 "${CONTAINER_NAME}" 2>/dev/null || true + podman rm -f "${CONTAINER_NAME}" 2>/dev/null || true + else + # Skip only the podman calls — the staging copy of the gateway mTLS + # material below must still be removed. + echo "WARN: state file holds an unexpected container name (${CONTAINER_NAME}) — not touching podman" + fi + rm -f "${STATE_FILE}" fi - rm -f "${STATE_FILE}" + + OPENSHELL_STAGING="${STATE_DIR}/openshell-${JOB_ID}" + rm -rf "${OPENSHELL_STAGING}" 2>/dev/null || true +else + echo "WARN: could not read job id from JOB_RESPONSE_FILE — skipping job container cleanup" fi -OPENSHELL_STAGING="${STATE_DIR}/openshell-${JOB_ID}" -rm -rf "${OPENSHELL_STAGING}" 2>/dev/null || true +# Always tear the per-job gateway down (killing any sandboxes it still holds), +# even when the job id is missing. A leftover is also reaped by the next +# job's prepare.sh. +teardown_openshell_gateway || true diff --git a/hack/gitlab-runner-vm/executor/gateway.sh b/hack/gitlab-runner-vm/executor/gateway.sh new file mode 100755 index 0000000000..555ba96367 --- /dev/null +++ b/hack/gitlab-runner-vm/executor/gateway.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +# Per-job OpenShell gateway lifecycle for the GitLab custom executor. +# +# Sourced by prepare.sh and cleanup.sh. Not executed directly. +# +# GitLab runner VMs are long-lived. A systemd --user gateway that stays up +# across jobs accumulates a stale profile registry (`openshell provider +# profile import` is a no-op on "already exists") and a baked-in OpenShell +# version. GitHub Actions does not have this: each job installs a fresh, +# version-matched gateway with an empty registry, then throws the runner +# away. These helpers give that parity: create in prepare, tear down in +# cleanup, reap leftovers from an abruptly-killed prior job. +# +# The expensive image layers (runner image, supervisor) stay in the VM's +# Podman cache. Only the CLI (~39 MB) and a version-skewed supervisor +# (~30 MB) are fetched when the job's pin differs from the host. + +# Renovate-tracked pin for the OpenShell version/commit this host trusts, +# read from .github/scripts/openshell-version.sh (the same file +# install-openshell.sh and setup.sh source). Provides OPENSHELL_VERSION and +# OPENSHELL_SHA. Try the flattened EXECUTOR_DIR layout first (.github/scripts/ +# as a child of this script — install_executor in setup.sh ships it there +# since that's where prepare.sh/cleanup.sh actually source this file from at +# per-job runtime), then the VM source-tree layout (.github/scripts/ as a +# sibling of executor/), then the repo checkout layout. +_gateway_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_openshell_version_sh="${_gateway_dir}/.github/scripts/openshell-version.sh" +if [ ! -f "${_openshell_version_sh}" ]; then + _openshell_version_sh="${_gateway_dir}/../.github/scripts/openshell-version.sh" +fi +if [ ! -f "${_openshell_version_sh}" ]; then + _openshell_version_sh="${_gateway_dir}/../../../.github/scripts/openshell-version.sh" +fi +if [ -f "${_openshell_version_sh}" ]; then + # shellcheck source=../../../.github/scripts/openshell-version.sh + source "${_openshell_version_sh}" +fi + +# OpenShell's systemd user unit sets StateDirectory=openshell/gateway, which +# for a user service is ~/.local/state/openshell/gateway (SQLite). TLS +# material lives in ~/.local/state/openshell/tls. Wiping both and restarting +# yields an empty profile registry and new mTLS certs — a process restart +# alone does not. +openshell_state_root() { + printf '%s' "${HOME}/.local/state/openshell" +} + +# True when the job image's OpenShell version is known and differs from the +# host binary. An empty job version means we could not read one (stubbed +# runtime, image without a CLI) and should keep the host install. +openshell_versions_differ() { + local job_ver="$1" host_ver="$2" + [ -n "${job_ver}" ] && [ "${job_ver}" != "${host_ver}" ] +} + +# Print the first x.y.z in $1, or empty. Never fails (prepare.sh uses -e). +parse_openshell_version() { + local ver + ver=$(printf '%s\n' "$1" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) || true + printf '%s' "${ver}" +} + +job_image_openshell_version() { + local image="$1" out status=0 + # Hardened the same as the real job container (prepare.sh): a crafted + # `openshell` entrypoint should not get a weaker-isolation execution just + # to have its self-reported version read. + out=$(podman run --rm --cap-drop=ALL --security-opt=no-new-privileges \ + --pids-limit 4096 --entrypoint openshell -- "${image}" --version 2>&1) || status=$? + if [ "${status}" -ne 0 ]; then + echo "ERROR: could not probe OpenShell version from job image ${image} (podman exit ${status}): ${out}" >&2 + return 1 + fi + parse_openshell_version "${out}" +} + +host_openshell_version() { + local out + out=$(openshell --version 2>/dev/null || true) + parse_openshell_version "${out}" +} + +# $1 is a commit SHA (never a mutable release tag — see +# install_openshell_at_version for why). +openshell_install_script_url() { + printf 'https://raw.githubusercontent.com/NVIDIA/OpenShell/%s/install.sh' "$1" +} + +openshell_supervisor_image() { + printf 'ghcr.io/nvidia/openshell/supervisor:%s' "$1" +} + +stop_openshell_gateway() { + systemctl --user stop openshell-gateway.service 2>/dev/null || true + # Never leave the unit enabled: a reboot or lingering user session + # would resurrect the long-lived daemon this per-job model replaces. + systemctl --user disable openshell-gateway.service 2>/dev/null || true +} + +wipe_openshell_gateway_store() { + local root + root="$(openshell_state_root)" + rm -rf "${root}/gateway" "${root}/tls" +} + +# Sandboxes are Podman containers labeled openshell.managed=true (OpenShell +# Podman driver) and named openshell-*. A killed job leaves them behind; +# the leaked sandbox is what pinned the stale profile on runner-01. +reap_openshell_sandboxes() { + command -v podman >/dev/null 2>&1 || return 0 + local id name + while IFS= read -r id; do + [ -n "${id}" ] || continue + echo "Removing leftover OpenShell sandbox: ${id}" + podman rm -f -- "${id}" 2>/dev/null || true + done < <(podman ps -aq --filter label=openshell.managed=true 2>/dev/null || true) + + while IFS= read -r name; do + [ -n "${name}" ] || continue + case "${name}" in + openshell-*) + echo "Removing leftover OpenShell container: ${name}" + podman rm -f -- "${name}" 2>/dev/null || true + ;; + esac + done < <(podman ps -a --format '{{.Names}}' 2>/dev/null || true) +} + +teardown_openshell_gateway() { + reap_openshell_sandboxes + stop_openshell_gateway + wipe_openshell_gateway_store + openshell gateway remove openshell >/dev/null 2>&1 || true +} + +# prepare.sh calls this first so a prior job that skipped cleanup (runner +# crash, timeout kill) cannot leak a gateway or sandbox into this job. +reap_orphaned_openshell() { + echo "Reaping leftover OpenShell gateway/sandboxes from a previous job" + teardown_openshell_gateway +} + +wait_for_openshell_gateway() { + local i=1 + while [ "${i}" -le 10 ]; do + if systemctl --user is-active --quiet openshell-gateway.service; then + return 0 + fi + i=$((i + 1)) + sleep 1 + done + echo "ERROR: gateway did not start after 10s — check: journalctl --user -u openshell-gateway" >&2 + return 1 +} + +register_openshell_gateway() { + # ExecStartPre regenerates TLS certs after a store wipe. Drop the stale + # CLI registration (old certs) and re-add so mTLS succeeds. + openshell gateway remove openshell >/dev/null 2>&1 || true + + local add_err + if ! add_err=$(openshell gateway add --local https://127.0.0.1:17670 2>&1) \ + && ! openshell gateway select openshell >/dev/null 2>&1; then + echo "ERROR: could not register or select the OpenShell gateway: ${add_err}" >&2 + return 1 + fi + if ! openshell gateway list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | grep -Eq '^[[:space:]]*\*'; then + echo "ERROR: no active OpenShell gateway after add/select" >&2 + return 1 + fi + return 0 +} + +# Recreate the gateway process AND its store. A restart without the wipe +# keeps the stale profile registry — the original bug. +start_fresh_openshell_gateway() { + stop_openshell_gateway + reap_openshell_sandboxes + wipe_openshell_gateway_store + + systemctl --user daemon-reload + # start, not enable: the unit must not come back on reboot/linger. + if ! systemctl --user start openshell-gateway.service; then + echo "ERROR: systemctl failed to start openshell-gateway.service" >&2 + return 1 + fi + wait_for_openshell_gateway || return 1 + register_openshell_gateway || return 1 + echo "OpenShell gateway is running with an empty profile registry" +} + +pin_supervisor_image() { + local ver="$1" + local gateway_toml="${HOME}/.config/openshell/gateway.toml" + local supervisor_image + supervisor_image="$(openshell_supervisor_image "${ver}")" + mkdir -p "${HOME}/.config/openshell" + if [ -f "${gateway_toml}" ] && grep -q "supervisor_image" "${gateway_toml}"; then + sed -i "s|supervisor_image = .*|supervisor_image = \"${supervisor_image}\"|" "${gateway_toml}" + elif [ -f "${gateway_toml}" ] && grep -q '^\[openshell\.gateway\]' "${gateway_toml}"; then + sed -i "/^\[openshell\.gateway\]/a supervisor_image = \"${supervisor_image}\"" "${gateway_toml}" + elif [ -f "${gateway_toml}" ]; then + printf '\n[openshell.gateway]\nsupervisor_image = "%s"\n' "${supervisor_image}" >> "${gateway_toml}" + fi +} + +# Install the OpenShell version the job pins, but only when that version is +# the Renovate-tracked pin (OPENSHELL_VERSION/OPENSHELL_SHA, sourced above +# from .github/scripts/openshell-version.sh) — callers must verify that match +# before calling this (see ensure_job_openshell_gateway). install.sh is +# fetched from that commit SHA, never from a job-supplied release tag: the +# job image's self-reported `openshell --version` is job-controlled input, +# and a mutable vX.Y.Z tag could be retagged or rolled back, bypassing the +# commit-SHA allowlist .github/scripts/install-openshell.sh deliberately +# enforces for VM provisioning. +install_openshell_at_version() { + local ver="${1:-}" sha="${2:-}" + if ! printf '%s' "${ver}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "ERROR: invalid OpenShell version: ${ver}" >&2 + return 1 + fi + if ! printf '%s' "${sha}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "ERROR: invalid OpenShell commit SHA: ${sha}" >&2 + return 1 + fi + echo "Installing OpenShell ${ver} (${sha}, job-matched)" + local url + url="$(openshell_install_script_url "${sha}")" + local max_attempts=3 attempt=1 delay=5 + while true; do + if curl -LsSf --retry 3 --retry-delay 5 "${url}" \ + | OPENSHELL_VERSION="v${ver}" sh; then + break + fi + if [ "${attempt}" -ge "${max_attempts}" ]; then + echo "ERROR: OpenShell ${ver} install failed after ${max_attempts} attempts" >&2 + return 1 + fi + echo "WARN: OpenShell install attempt ${attempt}/${max_attempts} failed, retrying in ${delay}s..." >&2 + sleep "${delay}" + attempt=$((attempt + 1)) + delay=$((delay * 3)) + done + # The RPM %post may enable the user unit; pin it back to per-job. Stop + # first so a running unit doesn't block disable, and fail if disable does + # not succeed — otherwise a reboot before the next prepare/cleanup could + # resurrect the long-lived daemon this per-job model replaces. + systemctl --user stop openshell-gateway.service 2>/dev/null || true + if ! systemctl --user disable openshell-gateway.service 2>/dev/null; then + echo "ERROR: could not disable openshell-gateway.service after installing OpenShell ${ver}" >&2 + return 1 + fi +} + +# Pull the job image's OpenShell version onto the host if needed, then start +# a gateway with an empty registry. Called from prepare.sh after the image +# pull so `podman run --entrypoint openshell` can read the pin. +ensure_job_openshell_gateway() { + local image="$1" + local job_ver host_ver + job_ver="$(job_image_openshell_version "${image}")" || return 1 + host_ver="$(host_openshell_version)" + + if openshell_versions_differ "${job_ver}" "${host_ver}"; then + # The job image's self-reported version is job-controlled (any job that + # can set `image:` on this runner). Only ever install the host's + # Renovate-tracked pin — never a version the job merely claims to want — + # so a job cannot direct the host to fetch and run an + # arbitrary/retagged/older NVIDIA/OpenShell installer. + if [ "${job_ver}" != "${OPENSHELL_VERSION:-}" ]; then + echo "ERROR: job image reports OpenShell ${job_ver}, but this host only trusts the Renovate-pinned ${OPENSHELL_VERSION:-} (commit ${OPENSHELL_SHA:-})" >&2 + echo "ERROR: bump .github/scripts/openshell-version.sh (operator-approved) before running jobs that need a different OpenShell version" >&2 + return 1 + fi + echo "OpenShell host ${host_ver:-none} != job image ${job_ver}; installing the pinned version" + install_openshell_at_version "${job_ver}" "${OPENSHELL_SHA:-}" || return 1 + pin_supervisor_image "${job_ver}" + podman pull -- "$(openshell_supervisor_image "${job_ver}")" || return 1 + elif [ -n "${job_ver}" ]; then + echo "OpenShell ${job_ver} already matches the job image" + pin_supervisor_image "${job_ver}" + else + echo "Could not read OpenShell version from job image; using host ${host_ver:-unknown}" + fi + + start_fresh_openshell_gateway +} diff --git a/hack/gitlab-runner-vm/executor/gateway_test.sh b/hack/gitlab-runner-vm/executor/gateway_test.sh new file mode 100755 index 0000000000..cc5ef1a6ec --- /dev/null +++ b/hack/gitlab-runner-vm/executor/gateway_test.sh @@ -0,0 +1,369 @@ +#!/usr/bin/env bash +# Unit tests for gateway.sh helpers. Stubs podman/systemctl/openshell/curl +# so the tests run without a real OpenShell install. +# +# Usage: hack/gitlab-runner-vm/executor/gateway_test.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gateway.sh +source "${SCRIPT_DIR}/gateway.sh" + +FAILURES=0 +pass() { echo "ok $*"; } +fail() { + echo "FAIL $*" >&2 + FAILURES=$((FAILURES + 1)) +} + +FAKE_HOME=$(mktemp -d) +SHIM_DIR=$(mktemp -d) +trap 'rm -rf "${FAKE_HOME}" "${SHIM_DIR}"' EXIT +export HOME="${FAKE_HOME}" +export PATH="${SHIM_DIR}:${PATH}" + +PODMAN_LOG="${SHIM_DIR}/podman.log" +SYSTEMCTL_LOG="${SHIM_DIR}/systemctl.log" +OPENSHELL_LOG="${SHIM_DIR}/openshell.log" +CURL_LOG="${SHIM_DIR}/curl.log" + +# Default stubs. Individual tests rewrite them as needed. +printf '#!/bin/sh\necho "$@" >> "%s"\nexit 0\n' "${PODMAN_LOG}" > "${SHIM_DIR}/podman" +printf '#!/bin/sh\necho "$@" >> "%s"\nexit 0\n' "${SYSTEMCTL_LOG}" > "${SHIM_DIR}/systemctl" +printf '#!/bin/sh\necho "$@" >> "%s"\ncase "$1" in --version) echo "openshell 0.0.83";; gateway) echo " * openshell";; esac\nexit 0\n' "${OPENSHELL_LOG}" > "${SHIM_DIR}/openshell" +printf '#!/bin/sh\necho "$@" >> "%s"\nexit 0\n' "${CURL_LOG}" > "${SHIM_DIR}/curl" +chmod +x "${SHIM_DIR}/podman" "${SHIM_DIR}/systemctl" "${SHIM_DIR}/openshell" "${SHIM_DIR}/curl" + +reset_logs() { + : > "${PODMAN_LOG}" + : > "${SYSTEMCTL_LOG}" + : > "${OPENSHELL_LOG}" + : > "${CURL_LOG}" +} + +echo "== wiring (static) ==" +PREPARE="${SCRIPT_DIR}/prepare.sh" +CLEANUP="${SCRIPT_DIR}/cleanup.sh" +SETUP="${SCRIPT_DIR}/../setup.sh" +if grep -Fq 'source "$(dirname "${BASH_SOURCE[0]}")/gateway.sh"' "${PREPARE}" \ + && grep -Fq 'ensure_job_openshell_gateway' "${PREPARE}"; then + pass "prepare.sh sources gateway.sh and starts a per-job gateway" +else + fail "prepare.sh does not start a per-job gateway" +fi +if grep -Fq 'source "$(dirname "${BASH_SOURCE[0]}")/gateway.sh"' "${CLEANUP}" \ + && grep -Fq 'teardown_openshell_gateway' "${CLEANUP}"; then + pass "cleanup.sh sources gateway.sh and tears the gateway down" +else + fail "cleanup.sh does not tear the gateway down" +fi +if grep -Fq 'systemctl --user enable openshell-gateway.service' "${SETUP}"; then + fail "setup.sh still enables the long-lived openshell-gateway unit" +else + pass "setup.sh no longer enables a long-lived gateway" +fi +if grep -Fq 'configure_per_job_gateway' "${SETUP}"; then + pass "setup.sh configures the per-job gateway (disable after seed start)" +else + fail "setup.sh missing configure_per_job_gateway" +fi +if grep -Fq 'EXECUTOR_DIR}/.github/scripts' "${SETUP}"; then + pass "install_executor ships the version pin alongside the flattened executor scripts" +else + fail "install_executor does not copy openshell-version.sh into EXECUTOR_DIR" +fi + +echo "== flattened EXECUTOR_DIR layout resolves the version pin (regression: #7244) ==" +# install_executor copies job_id.sh/prepare.sh/run.sh/cleanup.sh/gateway.sh into +# a flat EXECUTOR_DIR with no .github/scripts sibling — that's what prepare.sh/ +# cleanup.sh actually source at per-job runtime. Reproduce that layout in an +# isolated temp dir (not under this repo checkout, so neither the VM-layout +# nor the repo-checkout relative guess can accidentally resolve) and confirm +# gateway.sh's flattened-layout guess picks up the pin. +FLAT_DIR=$(mktemp -d) +mkdir -p "${FLAT_DIR}/.github/scripts" +cp "${SCRIPT_DIR}/gateway.sh" "${FLAT_DIR}/gateway.sh" +# Use sentinel values distinct from the real pin, so a pass can only mean +# "read from this flattened file" — not a false pass from OPENSHELL_VERSION/ +# OPENSHELL_SHA leaking in via the environment (gateway.sh's real pin file +# exports both, and this test's own parent process already sourced gateway.sh +# once at the top of this file). +printf 'OPENSHELL_VERSION=9.9.9\nOPENSHELL_SHA=%s\nexport OPENSHELL_VERSION OPENSHELL_SHA\n' \ + "$(printf 'f%.0s' $(seq 1 40))" > "${FLAT_DIR}/.github/scripts/openshell-version.sh" +if [ -e "${FLAT_DIR}/../.github/scripts" ] || [ -e "${FLAT_DIR}/../../../.github/scripts" ]; then + fail "flattened-layout test fixture is not isolated: a relative guess other than the flattened one would resolve" +else + FLAT_OUT=$(env -u OPENSHELL_VERSION -u OPENSHELL_SHA bash -c \ + 'source "$1/gateway.sh"; printf "%s %s" "${OPENSHELL_VERSION:-}" "${OPENSHELL_SHA:-}"' _ "${FLAT_DIR}") + FLAT_VER="${FLAT_OUT%% *}" + FLAT_SHA="${FLAT_OUT##* }" + if [ "${FLAT_VER}" = "9.9.9" ] && [ "${FLAT_SHA}" = "$(printf 'f%.0s' $(seq 1 40))" ]; then + pass "gateway.sh resolves OPENSHELL_VERSION/OPENSHELL_SHA from a flattened EXECUTOR_DIR" + else + fail "gateway.sh did not resolve the version pin from a flattened EXECUTOR_DIR layout (version='${FLAT_VER}' sha='${FLAT_SHA}')" + fi +fi +rm -rf "${FLAT_DIR}" + +echo "== parse / version compare ==" +if [ "$(parse_openshell_version 'openshell 0.0.116 (commit abc)')" = "0.0.116" ]; then + pass "parse_openshell_version extracts semver" +else + fail "parse_openshell_version did not extract 0.0.116" +fi +if [ -z "$(parse_openshell_version 'not a version')" ]; then + pass "parse_openshell_version empty on garbage" +else + fail "parse_openshell_version should be empty on garbage" +fi +if openshell_versions_differ "0.0.116" "0.0.83"; then + pass "versions_differ true on mismatch" +else + fail "versions_differ should be true for 0.0.116 vs 0.0.83" +fi +if openshell_versions_differ "0.0.116" "0.0.116"; then + fail "versions_differ should be false when equal" +else + pass "versions_differ false on match" +fi +if openshell_versions_differ "" "0.0.83"; then + fail "versions_differ should be false when job version is unknown" +else + pass "versions_differ false when job version is empty" +fi +if [ "$(openshell_install_script_url abc123def)" = "https://raw.githubusercontent.com/NVIDIA/OpenShell/abc123def/install.sh" ]; then + pass "install URL is built from a commit SHA, not a job-chosen release tag" +else + fail "install URL was $(openshell_install_script_url abc123def)" +fi + +echo "== wipe store ==" +mkdir -p "${HOME}/.local/state/openshell/gateway" "${HOME}/.local/state/openshell/tls" +echo db > "${HOME}/.local/state/openshell/gateway/state.db" +echo cert > "${HOME}/.local/state/openshell/tls/ca.crt" +wipe_openshell_gateway_store +if [ ! -e "${HOME}/.local/state/openshell/gateway" ] \ + && [ ! -e "${HOME}/.local/state/openshell/tls" ]; then + pass "wipe_openshell_gateway_store removes gateway db and tls" +else + fail "wipe_openshell_gateway_store left state behind" +fi + +echo "== reap sandboxes ==" +reset_logs +# podman ps -aq --filter label=... prints an id; podman ps -a --format prints names. +cat > "${SHIM_DIR}/podman" <<'PODMAN' +#!/bin/sh +echo "$@" >> "${PODMAN_LOG}" +for a in "$@"; do + case "$a" in + label=openshell.managed=true) echo "ctr-managed"; exit 0 ;; + '{{.Names}}') echo "openshell-ws---abc"; echo "runner-1"; exit 0 ;; + esac +done +exit 0 +PODMAN +# The heredoc cannot expand PODMAN_LOG (quoted), so rewrite the log path. +sed -i "s|\${PODMAN_LOG}|${PODMAN_LOG}|" "${SHIM_DIR}/podman" +chmod +x "${SHIM_DIR}/podman" +reap_openshell_sandboxes +if grep -q 'rm -f -- ctr-managed' "${PODMAN_LOG}" \ + && grep -q 'rm -f -- openshell-ws---abc' "${PODMAN_LOG}" \ + && ! grep -q 'rm -f -- runner-1' "${PODMAN_LOG}"; then + pass "reap_openshell_sandboxes removes labeled and openshell-* containers only" +else + fail "reap_openshell_sandboxes podman log: $(tr '\n' '|' < "${PODMAN_LOG}")" +fi + +echo "== start_fresh wipes then starts (does not enable) ==" +reset_logs +mkdir -p "${HOME}/.local/state/openshell/gateway" +echo stale > "${HOME}/.local/state/openshell/gateway/state.db" +printf '#!/bin/sh\necho "$@" >> "%s"\nexit 0\n' "${PODMAN_LOG}" > "${SHIM_DIR}/podman" +chmod +x "${SHIM_DIR}/podman" +if start_fresh_openshell_gateway; then + if [ ! -e "${HOME}/.local/state/openshell/gateway/state.db" ]; then + pass "start_fresh_openshell_gateway wipes the stale store" + else + fail "start_fresh_openshell_gateway left the stale store" + fi + if grep -q 'start openshell-gateway.service' "${SYSTEMCTL_LOG}" \ + && ! grep -q 'enable openshell-gateway.service' "${SYSTEMCTL_LOG}"; then + pass "start_fresh starts the unit and never enables it" + else + fail "systemctl log: $(tr '\n' '|' < "${SYSTEMCTL_LOG}")" + fi + if grep -q 'gateway add --local' "${OPENSHELL_LOG}"; then + pass "start_fresh re-registers the CLI after cert regen" + else + fail "CLI was not re-registered: $(tr '\n' '|' < "${OPENSHELL_LOG}")" + fi +else + fail "start_fresh_openshell_gateway returned non-zero" +fi + +echo "== teardown stops gateway and wipes ==" +reset_logs +mkdir -p "${HOME}/.local/state/openshell/gateway" +echo db > "${HOME}/.local/state/openshell/gateway/state.db" +teardown_openshell_gateway +if [ ! -e "${HOME}/.local/state/openshell/gateway" ] \ + && grep -q 'stop openshell-gateway.service' "${SYSTEMCTL_LOG}"; then + pass "teardown_openshell_gateway stops the unit and wipes the store" +else + fail "teardown did not stop+wipe" +fi + +echo "== install_openshell_at_version rejects junk ==" +if install_openshell_at_version "../evil" 2>/dev/null; then + fail "install_openshell_at_version accepted a non-semver version" +else + pass "install_openshell_at_version rejects non-semver" +fi + +echo "== ensure_job_openshell_gateway installs on mismatch (only the Renovate pin) ==" +reset_logs +if [ -z "${OPENSHELL_VERSION:-}" ] || [ -z "${OPENSHELL_SHA:-}" ]; then + fail "OPENSHELL_VERSION/OPENSHELL_SHA were not sourced from .github/scripts/openshell-version.sh" +else + pass "gateway.sh sourced the Renovate-tracked OpenShell pin (${OPENSHELL_VERSION})" +fi +# Job image reports the Renovate-pinned version; host stub reports 0.0.1 (stale). +cat > "${SHIM_DIR}/podman" <> "${PODMAN_LOG}" +for a in "\$@"; do + case "\$a" in + --version) echo "openshell ${OPENSHELL_VERSION}"; exit 0 ;; + esac +done +# podman run --entrypoint openshell -- IMAGE --version +if [ "\$1" = "run" ]; then + echo "openshell ${OPENSHELL_VERSION}" + exit 0 +fi +exit 0 +PODMAN +chmod +x "${SHIM_DIR}/podman" +cat > "${SHIM_DIR}/openshell" <<'OS' +#!/bin/sh +case "$1" in + --version) echo "openshell 0.0.1";; + gateway) echo " * openshell";; +esac +exit 0 +OS +chmod +x "${SHIM_DIR}/openshell" + +# curl | sh: emit a no-op script so the pipe succeeds without a real install. +cat > "${SHIM_DIR}/curl" <> "${CURL_LOG}" +printf '#!/bin/sh\\necho stub-install\\n' +exit 0 +CURL +chmod +x "${SHIM_DIR}/curl" + +mkdir -p "${HOME}/.config/openshell" +printf '[openshell.gateway]\nsupervisor_image = "ghcr.io/nvidia/openshell/supervisor:0.0.1"\n' \ + > "${HOME}/.config/openshell/gateway.toml" + +if ensure_job_openshell_gateway "registry.example.com/runner:dev"; then + if grep -q "NVIDIA/OpenShell/${OPENSHELL_SHA}/install.sh" "${CURL_LOG}"; then + pass "mismatch installs from the Renovate-pinned commit SHA, not a job-chosen tag" + else + fail "curl log missing SHA-pinned URL: $(tr '\n' '|' < "${CURL_LOG}")" + fi + if grep -q -- '--cap-drop=ALL' "${PODMAN_LOG}" && grep -q -- '--security-opt=no-new-privileges' "${PODMAN_LOG}"; then + pass "job image version probe is hardened like the real job container" + else + fail "podman log missing hardening flags: $(tr '\n' '|' < "${PODMAN_LOG}")" + fi + if grep -q "supervisor:${OPENSHELL_VERSION}" "${HOME}/.config/openshell/gateway.toml"; then + pass "supervisor_image pinned to the matched version" + else + fail "supervisor_image not updated: $(cat "${HOME}/.config/openshell/gateway.toml")" + fi +else + fail "ensure_job_openshell_gateway returned non-zero on mismatch" +fi + +echo "== ensure_job_openshell_gateway refuses a job version outside the Renovate pin ==" +reset_logs +cat > "${SHIM_DIR}/podman" <<'PODMAN' +#!/bin/sh +echo "$@" >> "${PODMAN_LOG}" +if [ "$1" = "run" ]; then + echo "openshell 9.9.9" + exit 0 +fi +exit 0 +PODMAN +sed -i "s|\${PODMAN_LOG}|${PODMAN_LOG}|" "${SHIM_DIR}/podman" +chmod +x "${SHIM_DIR}/podman" +: > "${CURL_LOG}" +if ensure_job_openshell_gateway "registry.example.com/runner:dev" 2>/dev/null; then + fail "ensure_job_openshell_gateway must refuse a job-chosen version outside the Renovate pin" +elif [ -s "${CURL_LOG}" ]; then + fail "must not curl-install an unpinned version: $(tr '\n' '|' < "${CURL_LOG}")" +else + pass "job image requesting a non-pinned OpenShell version is refused without installing" +fi + +echo "== job_image_openshell_version fails hard on a podman error instead of silently keeping the host version ==" +reset_logs +cat > "${SHIM_DIR}/podman" <<'PODMAN' +#!/bin/sh +echo "$@" >> "${PODMAN_LOG}" +if [ "$1" = "run" ]; then + echo "OCI runtime exec failed: exec: openshell: executable file not found" >&2 + exit 126 +fi +exit 0 +PODMAN +sed -i "s|\${PODMAN_LOG}|${PODMAN_LOG}|" "${SHIM_DIR}/podman" +chmod +x "${SHIM_DIR}/podman" +if job_image_openshell_version "registry.example.com/runner:dev" >/dev/null 2>/dev/null; then + fail "job_image_openshell_version should fail on a podman error, not report a version" +else + pass "job_image_openshell_version propagates a hard podman error instead of masking it as 'no CLI'" +fi + +echo "== ensure_job_openshell_gateway skips install on match ==" +reset_logs +cat > "${SHIM_DIR}/openshell" <> "${OPENSHELL_LOG}" +case "\$1" in + --version) echo "openshell 0.0.116";; + gateway) echo " * openshell";; +esac +exit 0 +OS +chmod +x "${SHIM_DIR}/openshell" +cat > "${SHIM_DIR}/podman" <> "${PODMAN_LOG}" +if [ "\$1" = "run" ]; then + echo "openshell 0.0.116" + exit 0 +fi +exit 0 +PODMAN +chmod +x "${SHIM_DIR}/podman" +: > "${CURL_LOG}" +if ensure_job_openshell_gateway "registry.example.com/runner:dev"; then + if [ -s "${CURL_LOG}" ]; then + fail "matched versions should not curl-install: $(tr '\n' '|' < "${CURL_LOG}")" + else + pass "matched versions skip the install" + fi +else + fail "ensure_job_openshell_gateway returned non-zero on match" +fi + +if [ "${FAILURES}" -ne 0 ]; then + echo "${FAILURES} case(s) failed" >&2 + exit 1 +fi +echo "all cases passed" diff --git a/hack/gitlab-runner-vm/executor/prepare.sh b/hack/gitlab-runner-vm/executor/prepare.sh index 7ae595654e..ea61594211 100755 --- a/hack/gitlab-runner-vm/executor/prepare.sh +++ b/hack/gitlab-runner-vm/executor/prepare.sh @@ -67,6 +67,17 @@ STATE_FILE="${STATE_DIR}/container-${JOB_ID}" echo "Pulling image: ${IMAGE}" podman pull -- "${IMAGE}" +# Per-job OpenShell gateway: reap leftovers from a killed prior job, then +# start a version-matched gateway with an empty profile registry. The image +# pull above is required so we can read the job's OpenShell pin from it. +# shellcheck source=gateway.sh +source "$(dirname "${BASH_SOURCE[0]}")/gateway.sh" +reap_orphaned_openshell +ensure_job_openshell_gateway "${IMAGE}" || { + echo "ERROR: failed to start a per-job OpenShell gateway" >&2 + exit 1 +} + mkdir -p "${BUILDS_DIR}" "${CACHE_DIR}" # --network=host is required so the container can reach the OpenShell gateway. diff --git a/hack/gitlab-runner-vm/executor/prepare_validation_test.sh b/hack/gitlab-runner-vm/executor/prepare_validation_test.sh index 2c59cee637..4496b80609 100755 --- a/hack/gitlab-runner-vm/executor/prepare_validation_test.sh +++ b/hack/gitlab-runner-vm/executor/prepare_validation_test.sh @@ -26,9 +26,13 @@ if ! realpath -m / >/dev/null 2>&1; then fi fi -# Stub podman so the test exercises validation only — no container runtime. +# Stub podman / systemctl / openshell so the test exercises path validation +# only. prepare.sh now starts a per-job OpenShell gateway after the pull; +# those helpers must succeed against these no-op binaries. printf '#!/bin/sh\nexit 0\n' > "${SHIM_DIR}/podman" -chmod +x "${SHIM_DIR}/podman" +printf '#!/bin/sh\nexit 0\n' > "${SHIM_DIR}/systemctl" +printf '#!/bin/sh\ncase "$1" in --version) echo "openshell 0.0.116";; gateway) echo " * openshell";; esac\nexit 0\n' > "${SHIM_DIR}/openshell" +chmod +x "${SHIM_DIR}/podman" "${SHIM_DIR}/systemctl" "${SHIM_DIR}/openshell" FAKE_HOME=$(mktemp -d) trap 'rm -rf "${SHIM_DIR}" "${FAKE_HOME}"' EXIT diff --git a/hack/gitlab-runner-vm/lib_test.sh b/hack/gitlab-runner-vm/lib_test.sh index 888c7cc875..8914f879d2 100755 --- a/hack/gitlab-runner-vm/lib_test.sh +++ b/hack/gitlab-runner-vm/lib_test.sh @@ -214,6 +214,13 @@ else fail "create scripts missing 'cleanup_runner() { cleanup_vm; }' alias in RUNNER_TOKEN mode" fi +if grep -Fq 'executor/gateway.sh' "${CREATE_GCP}" \ + && grep -Fq 'executor/gateway.sh' "${CREATE_OCP}"; then + pass "both create scripts copy and checksum executor/gateway.sh" +else + fail "create scripts missing executor/gateway.sh in copy/checksum lists" +fi + if [ "${FAILURES}" -ne 0 ]; then echo "${FAILURES} case(s) failed" >&2 exit 1 diff --git a/hack/gitlab-runner-vm/setup.sh b/hack/gitlab-runner-vm/setup.sh index 138dc8210d..456dee6d0f 100755 --- a/hack/gitlab-runner-vm/setup.sh +++ b/hack/gitlab-runner-vm/setup.sh @@ -21,8 +21,10 @@ # gitlab-runner register. Use the API/UI to set tags. # GITLAB_RUNNER_VERSION — gitlab-runner version to install (default: 19.2.1) # -# Note: The OpenShell version is pinned in .github/scripts/openshell-version.sh -# (Renovate-tracked). The SHA-pinned installer installs the repo-pinned version. +# Note: The OpenShell version sourced here is the *provisioning* pin +# (.github/scripts/openshell-version.sh, Renovate-tracked). Per-job prepare.sh +# re-reads the job image's openshell --version and upgrades the host when they +# differ, so a VM that was created on an older pin still matches the job. set -euo pipefail GITLAB_URL="${GITLAB_URL:-}" @@ -42,6 +44,15 @@ if [ -f "${_openshell_version_sh}" ]; then fi OPENSHELL_VERSION="${OPENSHELL_VERSION:-0.0.116}" +# Source the executor's gateway helpers (wait_for_openshell_gateway) so +# configure_per_job_gateway can wait for the seed start instead of assuming +# it worked. +_gateway_sh="${SCRIPT_DIR}/executor/gateway.sh" +if [ -f "${_gateway_sh}" ]; then + # shellcheck source=executor/gateway.sh + source "${_gateway_sh}" +fi + EXECUTOR_DIR="${HOME}/gitlab-runner-executor" BUILDS_DIR="${HOME}/builds" CACHE_DIR="${HOME}/cache" @@ -617,46 +628,30 @@ EOF } # -------------------------------------------------------------------------- -# 5. Start gateway via RPM-provided systemd service +# 5. Per-job OpenShell gateway (no long-lived daemon) # -------------------------------------------------------------------------- -start_gateway() { - info "Starting OpenShell gateway" +# The RPM's user unit would otherwise stay up across every job and accumulate +# a stale profile registry plus a baked-in OpenShell version (#7218). prepare.sh +# starts a fresh, job-version-matched gateway; cleanup.sh tears it down. Setup +# only seeds config, PKI defaults, and image cache — then disables the unit so +# a reboot or lingering user session cannot resurrect it. +configure_per_job_gateway() { + info "Configuring per-job OpenShell gateway (no long-lived daemon)" - # Use the RPM-provided service which handles config seeding, PKI - # generation, and environment loading automatically. systemctl --user daemon-reload - systemctl --user enable openshell-gateway.service - systemctl --user restart openshell-gateway.service - - local i - for i in $(seq 1 10); do - if systemctl --user is-active --quiet openshell-gateway.service; then - break - fi - if [ "${i}" -eq 10 ]; then - fail "gateway did not start after 10s — check: journalctl --user -u openshell-gateway" - fi - sleep 1 - done + # Seed PKI + gateway.toml.default via a one-shot start, then stop and + # disable. The next job's prepare.sh starts it for real with a wiped store. + systemctl --user start openshell-gateway.service || true + if ! wait_for_openshell_gateway; then + fail "openshell-gateway.service did not become active during the seed start — check: journalctl --user -u openshell-gateway" + fi + systemctl --user stop openshell-gateway.service 2>/dev/null || true + systemctl --user disable openshell-gateway.service 2>/dev/null || true - # Register the gateway with the CLI so openshell commands can find it. - # The restart above triggers ExecStartPre which regenerates TLS - # certificates. Any existing CLI registration still references the old - # certs, so mTLS checks would fail. Remove the stale registration first, - # then re-add so the CLI picks up the new certificates. + # Drop any CLI registration from the seed start; prepare.sh re-adds it. openshell gateway remove openshell >/dev/null 2>&1 || true - local add_err - if ! add_err=$(openshell gateway add --local https://127.0.0.1:17670 2>&1) \ - && ! openshell gateway select openshell >/dev/null 2>&1; then - fail "could not register or select the OpenShell gateway: ${add_err}" - fi - if ! openshell gateway list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | grep -Eq '^[[:space:]]*\*'; then - fail "no active OpenShell gateway after add/select" - fi - ok "gateway registered and selected" - - ok "gateway is running" + ok "openshell-gateway.service disabled; jobs start it in prepare.sh" } # -------------------------------------------------------------------------- @@ -667,7 +662,7 @@ install_executor() { mkdir -p "${EXECUTOR_DIR}" - for script in job_id.sh prepare.sh run.sh cleanup.sh; do + for script in job_id.sh prepare.sh run.sh cleanup.sh gateway.sh; do local src="${SCRIPT_DIR}/executor/${script}" if [ ! -f "${src}" ]; then fail "executor script not found: ${src}" @@ -676,6 +671,19 @@ install_executor() { chmod +x "${EXECUTOR_DIR}/${script}" done + # install_executor flattens the executor scripts into EXECUTOR_DIR with no + # .github/scripts sibling, so gateway.sh's VM-layout and repo-checkout + # relative guesses for openshell-version.sh both miss once prepare.sh/ + # cleanup.sh source the flattened copy. Ship the same pin file alongside it + # so gateway.sh's flattened-layout guess (.github/scripts as a child of the + # gateway.sh dir) resolves. + if [ -f "${_openshell_version_sh}" ]; then + mkdir -p "${EXECUTOR_DIR}/.github/scripts" + cp "${_openshell_version_sh}" "${EXECUTOR_DIR}/.github/scripts/openshell-version.sh" + else + fail "openshell-version.sh not found (looked at ${_openshell_version_sh}); cannot provision the per-job gateway's version pin" + fi + ok "executor scripts installed" } @@ -775,18 +783,15 @@ verify() { echo " WARN: openshell version mismatch"; errors=$((errors + 1)) fi - if systemctl --user is-active --quiet openshell-gateway.service; then - ok "gateway running" + if systemctl --user is-enabled --quiet openshell-gateway.service 2>/dev/null; then + echo " WARN: openshell-gateway.service is enabled — jobs expect a per-job gateway"; errors=$((errors + 1)) else - echo " WARN: gateway not running"; errors=$((errors + 1)) + ok "openshell-gateway.service not enabled (per-job)" fi - - # The unit being active says nothing about CLI registration, which is what - # the agent inside job containers actually resolves the gateway through. - if openshell gateway list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | grep -Eq '^[[:space:]]*\*'; then - ok "gateway registered with the CLI" + if systemctl --user is-active --quiet openshell-gateway.service; then + echo " WARN: gateway is running at setup end — prepare.sh should start it per job"; errors=$((errors + 1)) else - echo " WARN: no active gateway in 'openshell gateway list'"; errors=$((errors + 1)) + ok "gateway not running (started per job in prepare.sh)" fi if systemctl --user is-active --quiet podman.socket; then @@ -821,7 +826,7 @@ verify() { echo " INFO: container CA trust smoke test skipped (image lacks curl)" fi - for script in job_id.sh prepare.sh run.sh cleanup.sh; do + for script in job_id.sh prepare.sh run.sh cleanup.sh gateway.sh; do if test -x "${EXECUTOR_DIR}/${script}"; then ok "${script} executable" else @@ -868,7 +873,7 @@ setup_podman install_openshell configure_gateway install_ca_hook -start_gateway +configure_per_job_gateway install_executor patch_config prepull_images diff --git a/internal/cli/run.go b/internal/cli/run.go index 0036a8dee0..8c109b01e3 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -1374,11 +1374,14 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep } printer.StepDone(fmt.Sprintf("Providers v2 enabled (%.1fs)", time.Since(provV2Start).Seconds())) - // Import URL-resolved profiles to the gateway. + // Import URL-resolved profiles to the gateway. ImportProfileVerified + // drops the os.TempDir() content cache and confirms the gateway lists + // the profile: a hash match against a freshly-recreated (empty) + // gateway would otherwise skip the send (#7218). for _, rp := range result.Profiles { profileStart := time.Now() printer.StepStart("Importing profile: " + rp.ID) - if err := sandbox.ImportProfile(ctx, rp.ID, rp.LocalPath); err != nil { + if err := sandbox.ImportProfileVerified(ctx, rp.ID, rp.LocalPath); err != nil { printer.StepFail("Failed to import profile " + rp.ID) return fmt.Errorf("importing profile %q: %w", rp.ID, err) } diff --git a/internal/cli/run_openai.go b/internal/cli/run_openai.go index 80cccaedae..02cfe54b43 100644 --- a/internal/cli/run_openai.go +++ b/internal/cli/run_openai.go @@ -479,10 +479,9 @@ func dropSkippedProviders(names []string, skipped map[string]struct{}) []string // layered into .fullsend/profiles at run time — importing that directory // wholesale would replace the canonical profiles the fleet resolves from // fullsend-ai/agents — and a repository install ships only a .gitkeep, so -// the runner brings its own, version-matched copy. ImportProfile is a -// delete-and-reimport with a per-id lock and content cache, so a profile a -// running sandbox still references stays in place and an unchanged one is -// not re-sent. +// the runner brings its own, version-matched copy. ImportProfileVerified +// drops the content cache, re-sends, and confirms the gateway lists it, so +// a hash match against a recreated (empty) gateway cannot skip the import. func ensureOpenAIProfile(ctx context.Context, profileID string, printer *ui.Printer) error { data, err := scaffold.FullsendRepoFile("profiles/" + profileID + ".yaml") if err != nil { @@ -502,35 +501,15 @@ func ensureOpenAIProfile(ctx context.Context, profileID string, printer *ui.Prin } start := time.Now() printer.StepStart("Importing provider profile: " + profileID) - // Never trust ImportProfile's content cache for this profile: the cache - // says whether *these bytes* were sent once, not what the gateway holds - // now (a same-id profile imported earlier in this run, or another - // gateway entirely). Re-send every run; it is one small import. - sandbox.ForgetProfileCache(profileID) - if err := sandbox.ImportProfile(ctx, profileID, tmp.Name()); err != nil { + // ImportProfileVerified drops the content cache, imports, and confirms + // the gateway lists the profile (retrying once). The cache says whether + // these bytes were sent once, not what the gateway holds now; skipping + // the send would leave the provider create below failing with + // "unsupported provider type or profile". + if err := sandbox.ImportProfileVerified(ctx, profileID, tmp.Name()); err != nil { printer.StepFail("Failed to import provider profile " + profileID) return fmt.Errorf("importing provider profile %q: %w", profileID, err) } - // ImportProfile's content cache can outlive the gateway it was written - // against (a recreated gateway, a cache from another machine's run); - // the provider create below would then fail with "unsupported provider - // type or profile". Confirm the gateway lists it and re-send otherwise. - present, err := sandbox.ProfileExists(ctx, profileID) - if err != nil { - printer.StepFail("Failed to list provider profiles") - return fmt.Errorf("checking provider profile %q: %w", profileID, err) - } - if !present { - sandbox.ForgetProfileCache(profileID) - if err := sandbox.ImportProfile(ctx, profileID, tmp.Name()); err != nil { - printer.StepFail("Failed to import provider profile " + profileID) - return fmt.Errorf("importing provider profile %q: %w", profileID, err) - } - if present, err = sandbox.ProfileExists(ctx, profileID); err != nil || !present { - printer.StepFail("Provider profile missing after import: " + profileID) - return fmt.Errorf("provider profile %q is not on the gateway after import (err=%v)", profileID, err) - } - } printer.StepDone(fmt.Sprintf("Provider profile ready: %s (%.1fs)", profileID, time.Since(start).Seconds())) return nil } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 2db8d9be7b..2da0dd56e3 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -195,6 +195,9 @@ func useFakeOpenshell(t *testing.T) { func useFakeOpenshellProviders(t *testing.T) { t.Helper() neutralizeAgentsRepoFallback(t) + // ImportProfileVerified keeps a per-id content cache under os.TempDir() + // and the providers-stub records imported ids there; isolate both. + t.Setenv("TMPDIR", t.TempDir()) stubDir, err := filepath.Abs(filepath.Join("testdata", "providers-stub")) require.NoError(t, err) origPath := os.Getenv("PATH") @@ -697,7 +700,7 @@ func TestRunAgent_WithURLBase(t *testing.T) { func TestRunAgent_ProviderProfileOrchestration(t *testing.T) { // Exercises the provider/profile orchestration block in runAgent // (steps 2a-2c): CheckGateway, checkProviderProfileIntegrity, - // EnableProvidersV2, ImportProfile, EnsureProvider, CreateWithRetry. + // EnableProvidersV2, ImportProfileVerified, EnsureProvider, CreateWithRetry. // Uses the providers-stub that passes all openshell commands. useFakeOpenshellProviders(t) diff --git a/internal/cli/testdata/providers-stub/openshell b/internal/cli/testdata/providers-stub/openshell index 46e5bf5718..6c829678d9 100755 --- a/internal/cli/testdata/providers-stub/openshell +++ b/internal/cli/testdata/providers-stub/openshell @@ -13,8 +13,34 @@ case "$1" in exit 0 ;; provider) + # Track imported profile ids so list-profiles can satisfy + # ImportProfileVerified (Forget → import → ProfileExists). + # TMPDIR is per-test when callers isolate it. + STATE="${TMPDIR:-/tmp}/fullsend-providers-stub.profiles" case "$2" in + list-profiles) + printf '[' + first=1 + if [ -f "$STATE" ]; then + while IFS= read -r id; do + [ -n "$id" ] || continue + if [ "$first" -eq 1 ]; then first=0; else printf ','; fi + printf '{"id":"%s"}' "$id" + done < "$STATE" + fi + printf ']\n' + exit 0 + ;; profile) + case "$3" in + delete) + if [ -n "$4" ] && [ -f "$STATE" ]; then + grep -v "^$4$" "$STATE" > "$STATE.tmp" || true + mv "$STATE.tmp" "$STATE" + fi + exit 0 + ;; + esac # Validate --file argument has .yaml/.yml extension when present. prev="" for arg in "$@"; do @@ -24,6 +50,12 @@ case "$1" in *.yaml|*.yml) ;; *) echo "openshell stub: --file must end in .yaml or .yml, got: $arg" >&2; exit 1 ;; esac + if [ -f "$arg" ]; then + id=$(awk '/^id:/{print $2; exit}' "$arg") + if [ -n "$id" ]; then + printf '%s\n' "$id" >> "$STATE" + fi + fi ;; esac prev="$arg" diff --git a/internal/sandbox/import_verified_test.go b/internal/sandbox/import_verified_test.go new file mode 100644 index 0000000000..68a9b36516 --- /dev/null +++ b/internal/sandbox/import_verified_test.go @@ -0,0 +1,195 @@ +package sandbox + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeVerifiedOpenshellStub installs an openshell binary that logs every +// invocation and answers `provider list-profiles` from listJSON. listJSON +// may be a path prefixed with "@" to read the listing from that file on +// each call (so a test can change the gateway's answer between imports). +func writeVerifiedOpenshellStub(t *testing.T, argsLog, listJSON string) { + t.Helper() + dir := filepath.Dir(argsLog) + script := "#!/bin/sh\n" + + "printf '%s\\n' \"$*\" >> " + shellQuote(argsLog) + "\n" + + "case \"$1 $2\" in\n" + + " 'provider list-profiles')\n" + if len(listJSON) > 0 && listJSON[0] == '@' { + script += " cat " + shellQuote(listJSON[1:]) + "\n" + } else { + script += " printf '%s\\n' " + shellQuote(listJSON) + "\n" + } + script += " exit 0 ;;\n" + + "esac\n" + + "exit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestImportProfileVerified_PresentAfterFirstImport(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + dir := t.TempDir() + profilePath := filepath.Join(dir, "vertex.yaml") + require.NoError(t, os.WriteFile(profilePath, []byte("id: fullsend-vertex-ai\n"), 0o644)) + argsLog := filepath.Join(dir, "args.log") + writeVerifiedOpenshellStub(t, argsLog, `[{"id":"fullsend-vertex-ai"}]`) + + require.NoError(t, ImportProfileVerified(context.Background(), "fullsend-vertex-ai", profilePath)) + + data, err := os.ReadFile(argsLog) + require.NoError(t, err) + lines := splitNonEmpty(string(data)) + require.Len(t, lines, 3, "delete, import, list: %q", lines) + assert.Equal(t, "provider profile delete fullsend-vertex-ai", lines[0]) + assert.Equal(t, "provider profile import --file "+profilePath, lines[1]) + assert.Equal(t, "provider list-profiles -o json", lines[2]) +} + +func TestImportProfileVerified_StaleCacheAgainstEmptyGateway(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + dir := t.TempDir() + profilePath := filepath.Join(dir, "vertex.yaml") + require.NoError(t, os.WriteFile(profilePath, []byte("id: fullsend-vertex-ai\n"), 0o644)) + + // The regression #7218 calls out: ImportProfile trusts os.TempDir() and + // skips the send on a hash match. Against a freshly-recreated (empty) + // gateway that skip leaves the profile unregistered. + hash, err := hashProfileFile(profilePath) + require.NoError(t, err) + require.NoError(t, os.WriteFile(profileFileCachePath("fullsend-vertex-ai"), []byte(hash), 0o600)) + + argsLog := filepath.Join(dir, "args.log") + listing := filepath.Join(dir, "listing.json") + require.NoError(t, os.WriteFile(listing, []byte("[]\n"), 0o644)) + writeVerifiedOpenshellStub(t, argsLog, "@"+listing) + + err = ImportProfileVerified(context.Background(), "fullsend-vertex-ai", profilePath) + require.Error(t, err, "empty listing after both attempts must fail") + assert.Contains(t, err.Error(), "not on the gateway after import") + + data, readErr := os.ReadFile(argsLog) + require.NoError(t, readErr) + lines := splitNonEmpty(string(data)) + // forget cache + delete + import + list (missing) + forget + delete + import + list. + require.Len(t, lines, 6, "%q", lines) + assert.Equal(t, "provider profile delete fullsend-vertex-ai", lines[0], "stale cache must not skip the first send") + assert.Equal(t, "provider profile import --file "+profilePath, lines[1]) + assert.Equal(t, "provider list-profiles -o json", lines[2]) + assert.Equal(t, "provider profile delete fullsend-vertex-ai", lines[3], "retry after empty listing") + assert.Equal(t, "provider profile import --file "+profilePath, lines[4]) + assert.Equal(t, "provider list-profiles -o json", lines[5]) +} + +func TestImportProfileVerified_RetrySucceedsWhenGatewayCatchesUp(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + dir := t.TempDir() + profilePath := filepath.Join(dir, "vertex.yaml") + require.NoError(t, os.WriteFile(profilePath, []byte("id: fullsend-vertex-ai\n"), 0o644)) + + argsLog := filepath.Join(dir, "args.log") + countFile := filepath.Join(dir, "list.count") + script := "#!/bin/sh\n" + + "printf '%s\\n' \"$*\" >> " + shellQuote(argsLog) + "\n" + + "case \"$1 $2\" in\n" + + " 'provider list-profiles')\n" + + " n=0; [ -f " + shellQuote(countFile) + " ] && n=$(cat " + shellQuote(countFile) + ")\n" + + " n=$((n + 1)); echo \"$n\" > " + shellQuote(countFile) + "\n" + + " if [ \"$n\" -eq 1 ]; then echo '[]'; else echo '[{\"id\":\"fullsend-vertex-ai\"}]'; fi\n" + + " exit 0 ;;\n" + + "esac\n" + + "exit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + + require.NoError(t, ImportProfileVerified(context.Background(), "fullsend-vertex-ai", profilePath)) + + data, err := os.ReadFile(argsLog) + require.NoError(t, err) + lines := splitNonEmpty(string(data)) + require.Len(t, lines, 6, "import, miss, retry, hit: %q", lines) + assert.Equal(t, "provider list-profiles -o json", lines[2]) + assert.Equal(t, "provider profile delete fullsend-vertex-ai", lines[3]) + assert.Equal(t, "provider list-profiles -o json", lines[5]) +} + +func TestImportProfileVerified_ImportError(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + dir := t.TempDir() + profilePath := filepath.Join(dir, "vertex.yaml") + require.NoError(t, os.WriteFile(profilePath, []byte("id: fullsend-vertex-ai\n"), 0o644)) + script := "#!/bin/sh\necho boom >&2; exit 1\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + + err := ImportProfileVerified(context.Background(), "fullsend-vertex-ai", profilePath) + require.Error(t, err) + assert.Contains(t, err.Error(), "profile import") +} + +func TestImportProfileVerified_ListError(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + dir := t.TempDir() + profilePath := filepath.Join(dir, "vertex.yaml") + require.NoError(t, os.WriteFile(profilePath, []byte("id: fullsend-vertex-ai\n"), 0o644)) + script := "#!/bin/sh\n" + + "case \"$1 $2\" in\n" + + " 'provider list-profiles') echo 'gateway unreachable' >&2; exit 1 ;;\n" + + "esac\n" + + "exit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + + err := ImportProfileVerified(context.Background(), "fullsend-vertex-ai", profilePath) + require.Error(t, err) + assert.Contains(t, err.Error(), "checking provider profile") +} + +// TestImportProfileVerified_RetryListError covers the retry path's +// ProfileExists call specifically: the first list must succeed (empty, so a +// retry is attempted) and the second list must fail, so a genuine listing +// error on the retry is reported as such (wrapped, via "checking provider +// profile") instead of being mislabeled as "not on the gateway after import". +func TestImportProfileVerified_RetryListError(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + dir := t.TempDir() + profilePath := filepath.Join(dir, "vertex.yaml") + require.NoError(t, os.WriteFile(profilePath, []byte("id: fullsend-vertex-ai\n"), 0o644)) + + argsLog := filepath.Join(dir, "args.log") + countFile := filepath.Join(dir, "list.count") + script := "#!/bin/sh\n" + + "printf '%s\\n' \"$*\" >> " + shellQuote(argsLog) + "\n" + + "case \"$1 $2\" in\n" + + " 'provider list-profiles')\n" + + " n=0; [ -f " + shellQuote(countFile) + " ] && n=$(cat " + shellQuote(countFile) + ")\n" + + " n=$((n + 1)); echo \"$n\" > " + shellQuote(countFile) + "\n" + + " if [ \"$n\" -eq 1 ]; then echo '[]'; else echo 'gateway unreachable' >&2; exit 1; fi\n" + + " exit 0 ;;\n" + + "esac\n" + + "exit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + + err := ImportProfileVerified(context.Background(), "fullsend-vertex-ai", profilePath) + require.Error(t, err) + assert.Contains(t, err.Error(), "checking provider profile") + assert.NotContains(t, err.Error(), "not on the gateway after import") +} + +func splitNonEmpty(s string) []string { + var out []string + for _, line := range strings.Split(strings.TrimSpace(s), "\n") { + if line != "" { + out = append(out, line) + } + } + return out +} diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 906deb45fb..b87691a5d6 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -354,6 +354,38 @@ func ForgetProfileCache(id string) { os.Remove(profileFileCachePath(id)) //nolint:errcheck } +// ImportProfileVerified imports a provider profile after dropping any local +// content cache, then confirms the gateway lists it. ImportProfile trusts a +// hash in os.TempDir() and skips the send on a match; that cache can outlive +// the gateway it was written against (a per-job GitLab gateway, a restarted +// local daemon). Callers that must have the profile present — the generic +// URL-resolved import path and the OpenAI scaffold import — use this instead +// of ImportProfile alone. See #7218. +func ImportProfileVerified(ctx context.Context, id, profilePath string) error { + ForgetProfileCache(id) + if err := ImportProfile(ctx, id, profilePath); err != nil { + return err + } + present, err := ProfileExists(ctx, id) + if err != nil { + return fmt.Errorf("checking provider profile %q: %w", id, err) + } + if present { + return nil + } + ForgetProfileCache(id) + if err := ImportProfile(ctx, id, profilePath); err != nil { + return err + } + if present, err = ProfileExists(ctx, id); err != nil { + return fmt.Errorf("checking provider profile %q: %w", id, err) + } + if !present { + return fmt.Errorf("provider profile %q is not on the gateway after import", id) + } + return nil +} + // reservedCredentialKeys are env var names that must not be used as provider // credential keys. Credential keys become env vars in the openshell child // process; allowing security-sensitive names would let a URL-fetched provider