diff --git a/.github/scripts/docker-auth-setup.sh b/.github/scripts/docker-auth-setup.sh index c0235a22532..81fb84d12de 100755 --- a/.github/scripts/docker-auth-setup.sh +++ b/.github/scripts/docker-auth-setup.sh @@ -26,18 +26,20 @@ fi auth_marker="${DOCKER_CONFIG}/.nemoclaw-docker-login-attempted" : >"${auth_marker}" chmod 600 "${auth_marker}" +login_attempts=5 +retry_seconds=5 login_succeeded=0 -for attempt in 1 2 3; do +for ((attempt = 1; attempt <= login_attempts; attempt += 1)); do if printf '%s' "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then login_succeeded=1 break fi - if [[ "${attempt}" -lt 3 ]]; then - echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." - sleep 5 + if ((attempt < login_attempts)); then + echo "::warning::Docker Hub login attempt ${attempt}/${login_attempts} failed; retrying in ${retry_seconds}s." + sleep "${retry_seconds}" fi done if [[ "${login_succeeded}" -ne 1 ]]; then - echo "::error::Docker Hub login failed after 3 attempts." + echo "::error::Docker Hub login failed after ${login_attempts} attempts." exit 1 fi diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 64b9cf05d45..e8cf8babe4c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -793,7 +793,7 @@ jobs: # explicit because strict YAML decoders reject 100 or more aliases here. - &dockerhub-auth name: Authenticate to Docker Hub - uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928 + uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f with: auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }} username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }} diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 8df0d7d3648..f8bedaef612 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -66,19 +66,21 @@ jobs: auth_marker="${DOCKER_CONFIG}/.nemoclaw-docker-login-attempted" : > "${auth_marker}" chmod 600 "${auth_marker}" + login_attempts=5 + retry_seconds=5 login_succeeded=0 - for attempt in 1 2 3; do + for ((attempt = 1; attempt <= login_attempts; attempt += 1)); do if printf '%s' "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then login_succeeded=1 break fi - if [[ "${attempt}" -lt 3 ]]; then - echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." - sleep 5 + if ((attempt < login_attempts)); then + echo "::warning::Docker Hub login attempt ${attempt}/${login_attempts} failed; retrying in ${retry_seconds}s." + sleep "${retry_seconds}" fi done if [[ "${login_succeeded}" -ne 1 ]]; then - echo "::error::Docker Hub login failed after 3 attempts." + echo "::error::Docker Hub login failed after ${login_attempts} attempts." exit 1 fi diff --git a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts index 3982f088e65..85559abd2a6 100644 --- a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts +++ b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts @@ -23,7 +23,7 @@ const AUTH_STEP_NAME = "Authenticate to Docker Hub"; const CLEANUP_STEP_NAME = "Clean up Docker auth"; const CLEANUP_HELPER_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; const AUTH_HELPER_USES = - "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928"; + "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f"; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); const CLEANUP_HELPER_PATH = path.join(REPO_ROOT, ".github", "scripts", "docker-auth-cleanup.sh"); const AUTH_HELPER_PATH = path.join(REPO_ROOT, ".github", "scripts", "docker-auth-setup.sh"); @@ -444,6 +444,7 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => { const fakeBin = path.join(directory, "bin"); const runnerTemp = path.join(directory, "runner-temp"); const callsPath = path.join(directory, "docker-calls"); + const sleepsPath = path.join(directory, "sleep-calls"); const tokensPath = path.join(directory, "docker-tokens"); const githubEnv = path.join(directory, "github-env"); fs.mkdirSync(fakeBin); @@ -452,7 +453,10 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => { path.join(fakeBin, "timeout"), '#!/usr/bin/env bash\nset -euo pipefail\n[[ "$1" == "30s" ]]\nshift\nexec "$@"\n', ); - writeExecutable(path.join(fakeBin, "sleep"), "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable( + path.join(fakeBin, "sleep"), + '#!/usr/bin/env bash\nset -euo pipefail\n[[ "$#" -eq 1 && "$1" == "5" ]]\nprintf \'%s\\n\' "$1" >> "${SLEEP_CALLS}"\n', + ); writeExecutable( path.join(fakeBin, "docker"), `#!/usr/bin/env bash @@ -474,6 +478,7 @@ fi username?: string; }) => { fs.rmSync(callsPath, { force: true }); + fs.rmSync(sleepsPath, { force: true }); fs.rmSync(tokensPath, { force: true }); fs.rmSync(githubEnv, { force: true }); return spawnSync(AUTH_HELPER_PATH, [], { @@ -490,6 +495,7 @@ fi GITHUB_JOB: "live", PATH: `${fakeBin}:${process.env.PATH}`, RUNNER_TEMP: runnerTemp, + SLEEP_CALLS: sleepsPath, }, }); }; @@ -505,36 +511,56 @@ fi false, ); - const retried = runAuth({ + const recovered = runAuth({ authRequired: "1", - successAttempt: 3, + successAttempt: 4, token: "test-docker-token", username: "test-user", }); - expect(retried.status, retried.stderr).toBe(0); + expect(recovered.status, recovered.stderr).toBe(0); const authenticatedConfig = fs.readFileSync(githubEnv, "utf8").trim().split("=")[1]; const authMarker = path.join(authenticatedConfig, ".nemoclaw-docker-login-attempted"); expect(fs.existsSync(authMarker)).toBe(true); expect(fs.statSync(authMarker).mode & 0o777).toBe(0o600); - expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(3); + expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(4); expect(fs.readFileSync(callsPath, "utf8")).toContain("--password-stdin"); expect(fs.readFileSync(callsPath, "utf8")).not.toContain("test-docker-token"); expect(fs.readFileSync(tokensPath, "utf8").trim().split("\n")).toEqual([ "test-docker-token", "test-docker-token", "test-docker-token", + "test-docker-token", ]); + expect(fs.readFileSync(sleepsPath, "utf8").trim().split("\n")).toEqual(["5", "5", "5"]); + + const recoveredOnFinalAttempt = runAuth({ + authRequired: "1", + successAttempt: 5, + token: "test-docker-token", + username: "test-user", + }); + expect(recoveredOnFinalAttempt.status, recoveredOnFinalAttempt.stderr).toBe(0); + expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(5); + expect(fs.readFileSync(tokensPath, "utf8").trim().split("\n")).toEqual([ + "test-docker-token", + "test-docker-token", + "test-docker-token", + "test-docker-token", + "test-docker-token", + ]); + expect(fs.readFileSync(sleepsPath, "utf8").trim().split("\n")).toEqual(["5", "5", "5", "5"]); const exhausted = runAuth({ authRequired: "1", - successAttempt: 4, + successAttempt: 6, token: "test-docker-token", username: "test-user", }); expect(exhausted.status).toBe(1); - expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(3); + expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(5); + expect(fs.readFileSync(sleepsPath, "utf8").trim().split("\n")).toEqual(["5", "5", "5", "5"]); expect(`${exhausted.stdout}${exhausted.stderr}`).toContain( - "Docker Hub login failed after 3 attempts", + "Docker Hub login failed after 5 attempts", ); const missing = runAuth({ authRequired: "1", successAttempt: 1 }); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index c79f04330da..757ba125ccc 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -133,6 +133,15 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)\.github\/workflows\/e2e\.yaml$/, testsToRun: runTests(...E2E_WORKFLOW_CONTRACTS), }, + { + pattern: + /(?:^|\/)\.github\/(?:actions\/docker-auth-(?:cleanup|setup)\/action\.yaml|scripts\/docker-auth-(?:cleanup|setup)\.sh)$/, + testsToRun: runTests("test/e2e/support/dockerhub-auth-workflow-boundary.test.ts"), + }, + { + pattern: /(?:^|\/)\.github\/workflows\/sandbox-images-and-e2e\.yaml$/, + testsToRun: runTests("test/e2e/support/sandbox-images-workflow-boundary.test.ts"), + }, { pattern: /(?:^|\/)\.github\/workflows\/code-scanning\.yaml$/, testsToRun: runTests("test/code-scanning-workflow.test.ts"), diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index f451c048c42..e5d6494e0b2 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -68,6 +68,11 @@ const OPAQUE_INPUTS = [ "test/e2e/manifests/openclaw-nvidia.yaml", "test/e2e/docs/parity-inventory.generated.json", ".github/workflows/e2e.yaml", + ".github/actions/docker-auth-setup/action.yaml", + ".github/actions/docker-auth-cleanup/action.yaml", + ".github/scripts/docker-auth-setup.sh", + ".github/scripts/docker-auth-cleanup.sh", + ".github/workflows/sandbox-images-and-e2e.yaml", ".github/workflows/code-scanning.yaml", ".github/workflows/pr-review-advisor.yaml", "tools/pr-review-advisor/openshell-policy.yaml", @@ -156,6 +161,19 @@ describe("Vitest opaque-input watch triggers", () => { "test/e2e/support/e2e-migration-policy.test.ts", ]); expect(triggeredBy(".github/workflows/e2e.yaml")).toEqual(E2E_WORKFLOW_CONTRACTS); + for (const authPath of [ + ".github/actions/docker-auth-setup/action.yaml", + ".github/actions/docker-auth-cleanup/action.yaml", + ".github/scripts/docker-auth-setup.sh", + ".github/scripts/docker-auth-cleanup.sh", + ]) { + expect(triggeredBy(authPath)).toEqual([ + "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts", + ]); + } + expect(triggeredBy(".github/workflows/sandbox-images-and-e2e.yaml")).toEqual([ + "test/e2e/support/sandbox-images-workflow-boundary.test.ts", + ]); expect(triggeredBy(".github/workflows/code-scanning.yaml")).toEqual([ "test/code-scanning-workflow.test.ts", ]); diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index e964cdb9bdf..f25830657bd 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -39,7 +39,7 @@ const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance"; const CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 = "3578a053cede863f7aa4814d8399b4ca21ea0b77cee712e6d549c684818f11dd"; const CLI_ARTIFACT_WORKFLOW_CONTRACT_SHA256 = - "cf436f4885b22d05444f78362b9acb0dddd7027bb369ff47bf15c6a0ec66cf55"; + "c4e838dfa11056b3b47c9769b9a8b16fc422f8ceee518600d703e4a47a717826"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index e7aca8fd486..c0cf2526a8d 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -253,9 +253,13 @@ function validateCanonicalAuth(errors: string[], auth: SandboxImagesWorkflowStep 'auth_marker="${DOCKER_CONFIG}/.nemoclaw-docker-login-attempted"', ': > "${auth_marker}"', 'chmod 600 "${auth_marker}"', - "for attempt in 1 2 3; do", + "login_attempts=5", + "retry_seconds=5", + "for ((attempt = 1; attempt <= login_attempts; attempt += 1)); do", `if printf '%s' "\${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "\${DOCKERHUB_USERNAME}" --password-stdin; then`, - "Docker Hub login failed after 3 attempts", + "if ((attempt < login_attempts)); then", + 'sleep "${retry_seconds}"', + 'Docker Hub login failed after ${login_attempts} attempts', ]; for (const fragment of requiredFragments) { if (!run.includes(fragment)) { diff --git a/tools/e2e/workflow-boundary-policy.mts b/tools/e2e/workflow-boundary-policy.mts index 5b11e6fe8bd..ffde585d974 100644 --- a/tools/e2e/workflow-boundary-policy.mts +++ b/tools/e2e/workflow-boundary-policy.mts @@ -19,9 +19,9 @@ export const E2E_ACTION_PROVENANCE = { }, dockerAuth: { reference: - "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928", + "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f", actionSha256: "cf93dcbd19589a56d1d58225fd6b3f8ad2180705662ff79a3407f340b5dba4c0", - scriptSha256: "853a3f742f057c29ed465b63bed1ec8d8f306a1c046877a8556cadf290ef0cb6", + scriptSha256: "f4c7ba1d7c3dc5e82bacfdb85c94ed0838251dfaa88a081b4f64fba4f744b6dc", }, dockerCleanup: { reference: