diff --git a/.github/workflows/pr-self-hosted.yaml b/.github/workflows/pr-self-hosted.yaml index 1d4d89609cb..8f10b31e0ce 100644 --- a/.github/workflows/pr-self-hosted.yaml +++ b/.github/workflows/pr-self-hosted.yaml @@ -38,7 +38,7 @@ jobs: build-sandbox-images: runs-on: linux-amd64-cpu4 - timeout-minutes: 15 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -79,7 +79,7 @@ jobs: build-sandbox-images-arm64: runs-on: linux-arm64-cpu4 - timeout-minutes: 15 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index f9801ba7a18..76636265f33 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -27,7 +27,7 @@ permissions: jobs: build-sandbox-images: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 45 steps: - &checkout name: Checkout @@ -428,7 +428,7 @@ jobs: build-sandbox-images-arm64: if: inputs.run_arm64 runs-on: ubuntu-24.04-arm - timeout-minutes: 15 + timeout-minutes: 45 steps: - *checkout diff --git a/Dockerfile.base b/Dockerfile.base index 0141342b08b..161d09921de 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -52,6 +52,86 @@ # by OpenClaw CLI version bumps or the weekly docker-pin-check. # ──────────────────────────────────────────────────────────────────────── +ARG PERL_VERSION=5.44.0 +ARG PERL_SHA256=505cf43912e9480495c344c70260452e32aa2a73c546a026b3f100053b23ce91 +ARG PERL_PACKAGE_REVISION=1nemoclaw1 + +# Debian trixie has not published a Perl package containing the upstream +# fixes for CVE-2026-12087, CVE-2026-13221, and CVE-2026-57433. Build the +# fixed upstream release as native Debian packages so dpkg dependencies and +# vulnerability inventory both describe the runtime that is actually used. +FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba AS perl-builder + +ARG PERL_VERSION +ARG PERL_SHA256 +ARG PERL_PACKAGE_REVISION + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.12 \ + ca-certificates=20250419 \ + curl=8.14.1-2+deb13u4 \ + netbase=6.5 \ + xz-utils=5.8.1-1+deb13u1 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /tmp/perl-source + +# Perl Configure's d_syscallproto probe compiles and runs a target test through +# Buildx/QEMU. The QEMU-backed build returns a false negative for trixie's libc +# declaration, so pin the known target result instead of synthesizing a +# conflicting prototype. Remove this override when the QEMU-backed multi-arch +# build in .github/workflows/base-image.yaml passes without it. +RUN curl --proto '=https' --tlsv1.2 -fsSL \ + --retry 5 --retry-all-errors --retry-delay 2 --connect-timeout 15 --max-time 120 \ + -o /tmp/perl.tar.xz "https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.xz" \ + && printf '%s %s\n' "${PERL_SHA256}" /tmp/perl.tar.xz > /tmp/perl.sha256 \ + && sha256sum -c /tmp/perl.sha256 \ + && tar -xJf /tmp/perl.tar.xz -C /tmp/perl-source --strip-components=1 \ + && ./Configure -des \ + -Dprefix=/usr \ + -Dvendorprefix=/usr \ + -Dsiteprefix=/usr/local \ + -Dusethreads \ + -Duse64bitall \ + -Dd_syscallproto=define \ + -Dman1dir=none \ + -Dman3dir=none \ + && make -j"$(nproc)" \ + && make test \ + && make install DESTDIR=/tmp/perl-root + +RUN package_version="${PERL_VERSION}-${PERL_PACKAGE_REVISION}" \ + && architecture="$(dpkg --print-architecture)" \ + && mkdir -p /tmp/perl-root/DEBIAN /tmp/perl-meta/DEBIAN \ + && printf '%s\n' \ + 'Package: perl-base' \ + "Version: ${package_version}" \ + "Architecture: ${architecture}" \ + 'Essential: yes' \ + 'Priority: required' \ + 'Section: perl' \ + 'Multi-Arch: allowed' \ + 'Maintainer: NVIDIA NemoClaw Maintainers' \ + "Provides: libperl5.40 (= ${package_version}), perl-modules-5.40 (= ${package_version})" \ + 'Conflicts: libperl5.40, perl-modules-5.40' \ + "Breaks: perl (<< ${package_version})" \ + "Replaces: libperl5.40, perl-modules-5.40, perl (<< ${package_version})" \ + 'Description: Perl 5 language interpreter built for the NemoClaw sandbox' \ + > /tmp/perl-root/DEBIAN/control \ + && printf '%s\n' \ + 'Package: perl' \ + "Version: ${package_version}" \ + "Architecture: ${architecture}" \ + 'Priority: standard' \ + 'Section: perl' \ + 'Multi-Arch: allowed' \ + "Depends: perl-base (= ${package_version})" \ + 'Maintainer: NVIDIA NemoClaw Maintainers' \ + 'Description: Perl 5 language interpreter metapackage for the NemoClaw sandbox' \ + > /tmp/perl-meta/DEBIAN/control \ + && dpkg-deb --build --root-owner-group /tmp/perl-root /tmp/perl-base.deb \ + && dpkg-deb --build --root-owner-group /tmp/perl-meta /tmp/perl.deb + FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba # OpenShell blocks the link-local EC2 Instance Metadata Service. Keep AWS SDK @@ -134,6 +214,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && ln -s /usr/bin/python3 /usr/local/bin/python +COPY --from=perl-builder /tmp/perl-base.deb /tmp/perl.deb /tmp/ + +RUN apt-get update \ + && apt-get install -y --no-install-recommends /tmp/perl-base.deb /tmp/perl.deb \ + && rm -f /tmp/perl-base.deb /tmp/perl.deb \ + && rm -rf /var/lib/apt/lists/* \ + && test "$(perl -e 'print $^V')" = "v5.44.0" \ + && test "$(perl -MSocket -e 'print Socket->VERSION')" = "2.041" \ + && test "$(perl -MStorable -e 'print Storable->VERSION')" = "3.41" \ + && perl -MSocket=pack_ip_mreq_source -e \ + 'eval { pack_ip_mreq_source("\0" x 4, "\0" x 3) }; die "short source accepted" unless $@ =~ /Bad arg length/' \ + && perl -e \ + 'my $x = join "|", "aaa".."mzz"; my $y = join "|", "naa".."zzz"; use re "Debug"; "fnord" =~ m/(?:$x)|(?:$y)/' \ + && git --version \ + && test -z "$(dpkg --audit)" + # gosu for privilege separation (gateway vs sandbox user). # Install from GitHub release with checksum verification instead of # Debian's packaged gosu can lag upstream. Pinned to 1.19 (2025-09). diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 436145f68f1..fe88cdbc922 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -391,6 +391,11 @@ "test": "cross-checks the allowlist against every production archive install boundary", "category": "security" }, + { + "file": "test/perl-critical-cve-remediation.test.ts", + "test": "builds the fixed upstream release from a checksum-pinned source archive (#7338)", + "category": "security" + }, { "file": "test/platform-vitest-main-workflow.test.ts", "test": "keeps the WSL suite unprivileged with explicit root-only contracts", diff --git a/test/e2e/support/sandbox-images-workflow-boundary.test.ts b/test/e2e/support/sandbox-images-workflow-boundary.test.ts index 92b79cb85c2..5ad606eb77f 100644 --- a/test/e2e/support/sandbox-images-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-images-workflow-boundary.test.ts @@ -256,7 +256,7 @@ describe("sandbox image workflow boundary", () => { expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( expect.arrayContaining([ - "build-sandbox-images must retain its 15-minute producer budget", + "build-sandbox-images must retain its 45-minute producer budget", "runtime-overrides timeout must cover its 45-minute probe budget", "runtime-overrides must remain an independent consumer of build-sandbox-images", "OpenClaw producer must not run the failure-isolated runtime probe", diff --git a/test/helpers/base-apt-security-functions.ts b/test/helpers/base-apt-security-functions.ts index d19d875c5cf..d57928ac0e5 100644 --- a/test/helpers/base-apt-security-functions.ts +++ b/test/helpers/base-apt-security-functions.ts @@ -1,6 +1,59 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +export function dockerRunCommandBetween( + dockerfile: string, + startMarker: string, + endMarker: string, +): string { + const start = dockerfile.indexOf(startMarker); + const end = dockerfile.indexOf(endMarker, start); + if (start === -1 || end === -1 || end <= start) { + throw new Error(`Expected Dockerfile block between ${startMarker} and ${endMarker}`); + } + const runIndex = dockerfile.indexOf("RUN ", start); + if (runIndex === -1 || runIndex > end) { + throw new Error(`Expected RUN instruction after ${startMarker}`); + } + const runLines: string[] = []; + for (const line of dockerfile.slice(runIndex, end).split("\n")) { + runLines.push(line); + if (!line.trimEnd().endsWith("\\")) { + break; + } + } + const lastLine = runLines[runLines.length - 1]?.trimEnd() ?? ""; + if (lastLine.endsWith("\\")) { + throw new Error(`Expected complete RUN instruction before ${endMarker}`); + } + return runLines + .join("\n") + .trim() + .replace(/^RUN\s+/, "") + .replace(/\\\n/g, " "); +} + +export function runLoggedDockerShell(command: string, tmp: string, functionDefs: string[]) { + const logPath = path.join(tmp, "calls.log"); + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `call_log=${JSON.stringify(logPath)}`, + ...functionDefs, + command, + ].join("\n"); + const scriptPath = path.join(tmp, "run-docker-block.sh"); + fs.writeFileSync(scriptPath, script, { mode: 0o700 }); + return spawnSync("bash", [scriptPath], { + encoding: "utf-8", + timeout: 5000, + }); +} + export const BASE_APT_SECURITY_FUNCTIONS = [ [ "dpkg() {", diff --git a/test/perl-critical-cve-remediation.test.ts b/test/perl-critical-cve-remediation.test.ts new file mode 100644 index 00000000000..a10103012f5 --- /dev/null +++ b/test/perl-critical-cve-remediation.test.ts @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const dockerfile = fs.readFileSync(path.join(import.meta.dirname, "..", "Dockerfile.base"), "utf8"); +const baseImageWorkflow = fs.readFileSync( + path.join(import.meta.dirname, "..", ".github", "workflows", "base-image.yaml"), + "utf8", +); +const fixedPerlVersion = "5.44.0"; +const fixedPerlSha256 = "505cf43912e9480495c344c70260452e32aa2a73c546a026b3f100053b23ce91"; + +function stageNamed(name: string): string { + const start = dockerfile.indexOf(` AS ${name}`); + expect(start, `missing ${name} stage`).toBeGreaterThanOrEqual(0); + const next = dockerfile.indexOf("\nFROM ", start); + return dockerfile.slice(start, next >= 0 ? next : undefined); +} + +function completedStage(): string { + const start = dockerfile.lastIndexOf("\nFROM "); + expect(start, "missing completed image stage").toBeGreaterThanOrEqual(0); + return dockerfile.slice(start); +} + +function argumentDefault(name: string): string | undefined { + return dockerfile.match(new RegExp(`^ARG ${name}=([^\\s]+)$`, "mu"))?.[1]; +} + +describe("sandbox base critical Perl CVE remediation", () => { + // source-shape-contract: security -- Exact upstream version and checksum bind the replacement runtime to the reviewed CVE fixes + it("builds the fixed upstream release from a checksum-pinned source archive (#7338)", () => { + const builder = stageNamed("perl-builder"); + const download = builder.indexOf( + '-o /tmp/perl.tar.xz "https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.xz"', + ); + const checksum = builder.indexOf("sha256sum -c /tmp/perl.sha256"); + const extract = builder.indexOf("tar -xJf /tmp/perl.tar.xz"); + + expect(argumentDefault("PERL_VERSION")).toBe(fixedPerlVersion); + expect(argumentDefault("PERL_SHA256")).toBe(fixedPerlSha256); + expect(download).toBeGreaterThanOrEqual(0); + expect(checksum).toBeGreaterThan(download); + expect(extract).toBeGreaterThan(checksum); + expect(builder).toContain("-Dd_syscallproto=define"); + expect(builder).toContain("Perl Configure's d_syscallproto probe"); + expect(builder).toContain("Remove this override when the QEMU-backed multi-arch"); + expect(builder).toContain(".github/workflows/base-image.yaml"); + expect(baseImageWorkflow).toContain("docker/setup-qemu-action"); + expect(baseImageWorkflow).toContain("platforms: linux/amd64,linux/arm64"); + expect(baseImageWorkflow).toContain("dockerfile: Dockerfile.base"); + }); + + it("runs the upstream test suite before packaging the replacement runtime (#7338)", () => { + const builder = stageNamed("perl-builder"); + const compile = builder.indexOf('make -j"$(nproc)"'); + const test = builder.indexOf("make test"); + const install = builder.indexOf("make install DESTDIR=/tmp/perl-root"); + const packageBuild = builder.indexOf("dpkg-deb --build --root-owner-group"); + + expect(compile).toBeGreaterThanOrEqual(0); + expect(test).toBeGreaterThan(compile); + expect(install).toBeGreaterThan(test); + expect(packageBuild).toBeGreaterThan(install); + }); + + it("replaces the vulnerable distro packages without breaking dpkg ownership (#7338)", () => { + const builder = stageNamed("perl-builder"); + const runtime = completedStage(); + + expect(builder).toContain( + '"Provides: libperl5.40 (= ${package_version}), perl-modules-5.40 (= ${package_version})"', + ); + expect(builder).toContain("'Conflicts: libperl5.40, perl-modules-5.40'"); + expect(builder).toContain( + '"Replaces: libperl5.40, perl-modules-5.40, perl (<< ${package_version})"', + ); + expect(runtime).toContain( + "apt-get install -y --no-install-recommends /tmp/perl-base.deb /tmp/perl.deb", + ); + expect(runtime).toContain('test -z "$(dpkg --audit)"'); + }); + + it("fails the image build unless all three critical fixes are active (#7338)", () => { + const runtime = completedStage(); + + expect(runtime).toContain(`test "$(perl -e 'print $^V')" = "v${fixedPerlVersion}"`); + expect(runtime).toContain(`test "$(perl -MSocket -e 'print Socket->VERSION')" = "2.041"`); + expect(runtime).toContain(`test "$(perl -MStorable -e 'print Storable->VERSION')" = "3.41"`); + expect(runtime).toContain("pack_ip_mreq_source"); + expect(runtime).toContain('die "short source accepted"'); + expect(runtime).toContain('use re "Debug"'); + expect(runtime).toContain('"fnord" =~ m/(?:$x)|(?:$y)/'); + }); +}); diff --git a/test/runner.test.ts b/test/runner.test.ts index 56e17c1143f..67d61a93d89 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -1028,10 +1028,12 @@ describe("regression guards", () => { expect(baseSrc).toContain("ENV AWS_EC2_METADATA_DISABLED=true"); expect(runtimeSrc).toContain("ENV AWS_EC2_METADATA_DISABLED=true"); + const baseRuntimeStageStart = baseSrc.lastIndexOf("\nFROM "); + expect(baseRuntimeStageStart).toBeGreaterThan(-1); const runtimeStageStart = runtimeSrc.indexOf("# Stage 3: Runtime image"); expect(runtimeStageStart).toBeGreaterThan(-1); for (const [source, stageStart] of [ - [baseSrc, 0], + [baseSrc, baseRuntimeStageStart], [runtimeSrc, runtimeStageStart], ] as const) { const fromIndex = source.indexOf("\nFROM ", stageStart); diff --git a/test/sandbox-base-security-packages.test.ts b/test/sandbox-base-security-packages.test.ts new file mode 100644 index 00000000000..d479b64147c --- /dev/null +++ b/test/sandbox-base-security-packages.test.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + BASE_APT_SECURITY_FUNCTIONS, + dockerRunCommandBetween, + runLoggedDockerShell, +} from "./helpers/base-apt-security-functions"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const DOCKERFILE_BASE = path.join(ROOT, "Dockerfile.base"); + +describe("sandbox base security packages", () => { + it("rejects a sandbox security package when its expected checksum changes", () => { + const dockerfile = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-checksum-")); + const untouchedTail = path.join(tmp, "untouched-python-link"); + const command = dockerRunCommandBetween( + dockerfile, + "ENV DEBIAN_FRONTEND=noninteractive", + "# gosu for privilege separation", + ) + .replace("df928e3a8e4da79408d4b18e8cd80a03dffa90130d0698e50041aab5e14f9397", "0".repeat(64)) + .replaceAll("/var/lib/apt/lists", tmp) + .replaceAll("/tmp/nemoclaw-debian-security", path.join(tmp, "security-debs")) + .replaceAll("/usr/local/bin/python", untouchedTail) + .replaceAll("/usr/bin/python3", path.join(tmp, "python3")); + + try { + const result = runLoggedDockerShell(command, tmp, [ + 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', + ...BASE_APT_SECURITY_FUNCTIONS, + ]); + expect(result.status).not.toBe(0); + expect(fs.existsSync(untouchedTail)).toBe(false); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index 817a65852e7..9d8d36994a9 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -29,6 +29,11 @@ const DEEPAGENTS_DOCKERFILE_BASE = path.join( "Dockerfile.base", ); +function completedDockerStage(dockerfile: string): string { + const start = dockerfile.lastIndexOf("\nFROM "); + return start >= 0 ? dockerfile.slice(start) : dockerfile; +} + function dockerRunCommandBetween( dockerfile: string, startMarker: string, @@ -1039,40 +1044,15 @@ describe("sandbox provisioning: base runtime tools", () => { ["Hermes", HERMES_DOCKERFILE_BASE], ["Deep Agents Code", DEEPAGENTS_DOCKERFILE_BASE], ])("installs pinned nftables for OpenShell bypass enforcement in %s", (_agent, file) => { - const dockerfile = fs.readFileSync(file, "utf-8"); + const dockerfile = completedDockerStage(fs.readFileSync(file, "utf-8")); const aptInstall = dockerfile.match( /^RUN apt-get update && apt-get install -y --no-install-recommends \\\n(?:.*\\\n)*.*$/m, )?.[0]; expect(aptInstall).toBeDefined(); expect(aptInstall).toContain("nftables=1.1.3-1"); }); - it("rejects a sandbox security package when its expected checksum changes", () => { - const dockerfile = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-checksum-")); - const untouchedTail = path.join(tmp, "untouched-python-link"); - const command = dockerRunCommandBetween( - dockerfile, - "RUN apt-get update", - "# gosu for privilege separation", - ) - .replace("df928e3a8e4da79408d4b18e8cd80a03dffa90130d0698e50041aab5e14f9397", "0".repeat(64)) - .replaceAll("/var/lib/apt/lists", tmp) - .replaceAll("/tmp/nemoclaw-debian-security", path.join(tmp, "security-debs")) - .replaceAll("/usr/local/bin/python", untouchedTail) - .replaceAll("/usr/bin/python3", path.join(tmp, "python3")); - try { - const { result } = runLoggedDockerShell(command, tmp, [ - 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', - ...BASE_APT_SECURITY_FUNCTIONS, - ]); - expect(result.status).not.toBe(0); - expect(fs.existsSync(untouchedTail)).toBe(false); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); it("base apt layer requests procps, e2fsprogs, and the SFTP server", () => { - const dockerfile = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); + const dockerfile = completedDockerStage(fs.readFileSync(DOCKERFILE_BASE, "utf-8")); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-apt-")); const lists = path.join(tmp, "apt-lists"); const securityDebs = path.join(tmp, "security-debs"); @@ -1107,7 +1087,7 @@ describe("sandbox provisioning: base runtime tools", () => { }); it("symlinks bare `python` to python3 so agent tool calls don't fail with command-not-found (#1452)", () => { - const dockerfile = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); + const dockerfile = completedDockerStage(fs.readFileSync(DOCKERFILE_BASE, "utf-8")); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-pysymlink-")); const lists = path.join(tmp, "apt-lists"); const securityDebs = path.join(tmp, "security-debs"); diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index 7711cad733d..f2482c82cd6 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -649,8 +649,8 @@ function validateRuntimeImageReuse(errors: string[], workflow: SandboxImagesWork const producer = workflow.jobs[producerName] ?? {}; const runtimeName = "runtime-overrides"; const runtimeJob = workflow.jobs[runtimeName] ?? {}; - if (producer["timeout-minutes"] !== 15) { - errors.push("build-sandbox-images must retain its 15-minute producer budget"); + if (producer["timeout-minutes"] !== 45) { + errors.push("build-sandbox-images must retain its 45-minute producer budget"); } if (runtimeJob["timeout-minutes"] !== 60) { errors.push("runtime-overrides timeout must cover its 45-minute probe budget");