fix(e2e): extend Docker Hub login retries

.github/scripts/docker-auth-setup.sh

@@ -26,18 +26,20 @@ fi
 auth_marker="${DOCKER_CONFIG}/.nemoclaw-docker-login-attempted"
 : >"${auth_marker}"
 chmod 600 "${auth_marker}"
+login_attempts=5
+retry_seconds=5
 login_succeeded=0
-for attempt in 1 2 3; do
+for ((attempt = 1; attempt <= login_attempts; attempt += 1)); do
   if printf '%s' "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then
     login_succeeded=1
     break
   fi
-  if [[ "${attempt}" -lt 3 ]]; then
-    echo "::warning::Docker Hub login attempt ${attempt} failed; retrying."
-    sleep 5
+  if ((attempt < login_attempts)); then
+    echo "::warning::Docker Hub login attempt ${attempt}/${login_attempts} failed; retrying in ${retry_seconds}s."
+    sleep "${retry_seconds}"
   fi
 done
 if [[ "${login_succeeded}" -ne 1 ]]; then
-  echo "::error::Docker Hub login failed after 3 attempts."
+  echo "::error::Docker Hub login failed after ${login_attempts} attempts."
   exit 1
 fi

.github/workflows/e2e.yaml

@@ -793,7 +793,7 @@ jobs:
       # explicit because strict YAML decoders reject 100 or more aliases here.
       - &dockerhub-auth
         name: Authenticate to Docker Hub
-        uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928
+        uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f
         with:
           auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }}
           username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }}

.github/workflows/sandbox-images-and-e2e.yaml

@@ -66,19 +66,21 @@ jobs:
           auth_marker="${DOCKER_CONFIG}/.nemoclaw-docker-login-attempted"
           : > "${auth_marker}"
           chmod 600 "${auth_marker}"
+          login_attempts=5
+          retry_seconds=5
           login_succeeded=0
-          for attempt in 1 2 3; do
+          for ((attempt = 1; attempt <= login_attempts; attempt += 1)); do
             if printf '%s' "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then
               login_succeeded=1
               break
             fi
-            if [[ "${attempt}" -lt 3 ]]; then
-              echo "::warning::Docker Hub login attempt ${attempt} failed; retrying."
-              sleep 5
+            if ((attempt < login_attempts)); then
+              echo "::warning::Docker Hub login attempt ${attempt}/${login_attempts} failed; retrying in ${retry_seconds}s."
+              sleep "${retry_seconds}"
             fi
           done
           if [[ "${login_succeeded}" -ne 1 ]]; then
-            echo "::error::Docker Hub login failed after 3 attempts."
+            echo "::error::Docker Hub login failed after ${login_attempts} attempts."
             exit 1
           fi
 

test/e2e/support/dockerhub-auth-workflow-boundary.test.ts

@@ -23,7 +23,7 @@ const AUTH_STEP_NAME = "Authenticate to Docker Hub";
 const CLEANUP_STEP_NAME = "Clean up Docker auth";
 const CLEANUP_HELPER_RUN = "bash .github/scripts/docker-auth-cleanup.sh";
 const AUTH_HELPER_USES =
-  "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928";
+  "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f";
 const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
 const CLEANUP_HELPER_PATH = path.join(REPO_ROOT, ".github", "scripts", "docker-auth-cleanup.sh");
 const AUTH_HELPER_PATH = path.join(REPO_ROOT, ".github", "scripts", "docker-auth-setup.sh");
@@ -444,6 +444,7 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => {
     const fakeBin = path.join(directory, "bin");
     const runnerTemp = path.join(directory, "runner-temp");
     const callsPath = path.join(directory, "docker-calls");
+    const sleepsPath = path.join(directory, "sleep-calls");
     const tokensPath = path.join(directory, "docker-tokens");
     const githubEnv = path.join(directory, "github-env");
     fs.mkdirSync(fakeBin);
@@ -452,7 +453,10 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => {
       path.join(fakeBin, "timeout"),
       '#!/usr/bin/env bash\nset -euo pipefail\n[[ "$1" == "30s" ]]\nshift\nexec "$@"\n',
     );
-    writeExecutable(path.join(fakeBin, "sleep"), "#!/usr/bin/env bash\nexit 0\n");
+    writeExecutable(
+      path.join(fakeBin, "sleep"),
+      '#!/usr/bin/env bash\nset -euo pipefail\n[[ "$#" -eq 1 && "$1" == "5" ]]\nprintf \'%s\\n\' "$1" >> "${SLEEP_CALLS}"\n',
+    );
     writeExecutable(
       path.join(fakeBin, "docker"),
       `#!/usr/bin/env bash
@@ -474,6 +478,7 @@ fi
       username?: string;
     }) => {
       fs.rmSync(callsPath, { force: true });
+      fs.rmSync(sleepsPath, { force: true });
       fs.rmSync(tokensPath, { force: true });
       fs.rmSync(githubEnv, { force: true });
       return spawnSync(AUTH_HELPER_PATH, [], {
@@ -490,6 +495,7 @@ fi
           GITHUB_JOB: "live",
           PATH: `${fakeBin}:${process.env.PATH}`,
           RUNNER_TEMP: runnerTemp,
+          SLEEP_CALLS: sleepsPath,
         },
       });
     };
@@ -505,36 +511,56 @@ fi
         false,
       );
 
-      const retried = runAuth({
+      const recovered = runAuth({
         authRequired: "1",
-        successAttempt: 3,
+        successAttempt: 4,
         token: "test-docker-token",
         username: "test-user",
       });
-      expect(retried.status, retried.stderr).toBe(0);
+      expect(recovered.status, recovered.stderr).toBe(0);
       const authenticatedConfig = fs.readFileSync(githubEnv, "utf8").trim().split("=")[1];
       const authMarker = path.join(authenticatedConfig, ".nemoclaw-docker-login-attempted");
       expect(fs.existsSync(authMarker)).toBe(true);
       expect(fs.statSync(authMarker).mode & 0o777).toBe(0o600);
-      expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(3);
+      expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(4);
       expect(fs.readFileSync(callsPath, "utf8")).toContain("--password-stdin");
       expect(fs.readFileSync(callsPath, "utf8")).not.toContain("test-docker-token");
       expect(fs.readFileSync(tokensPath, "utf8").trim().split("\n")).toEqual([
         "test-docker-token",
         "test-docker-token",
         "test-docker-token",
+        "test-docker-token",
       ]);
+      expect(fs.readFileSync(sleepsPath, "utf8").trim().split("\n")).toEqual(["5", "5", "5"]);
+
+      const recoveredOnFinalAttempt = runAuth({
+        authRequired: "1",
+        successAttempt: 5,
+        token: "test-docker-token",
+        username: "test-user",
+      });
+      expect(recoveredOnFinalAttempt.status, recoveredOnFinalAttempt.stderr).toBe(0);
+      expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(5);
+      expect(fs.readFileSync(tokensPath, "utf8").trim().split("\n")).toEqual([
+        "test-docker-token",
+        "test-docker-token",
+        "test-docker-token",
+        "test-docker-token",
+        "test-docker-token",
+      ]);
+      expect(fs.readFileSync(sleepsPath, "utf8").trim().split("\n")).toEqual(["5", "5", "5", "5"]);
 
       const exhausted = runAuth({
         authRequired: "1",
-        successAttempt: 4,
+        successAttempt: 6,
         token: "test-docker-token",
         username: "test-user",
       });
       expect(exhausted.status).toBe(1);
-      expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(3);
+      expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(5);
+      expect(fs.readFileSync(sleepsPath, "utf8").trim().split("\n")).toEqual(["5", "5", "5", "5"]);
       expect(`${exhausted.stdout}${exhausted.stderr}`).toContain(
-        "Docker Hub login failed after 3 attempts",
+        "Docker Hub login failed after 5 attempts",
       );
 
       const missing = runAuth({ authRequired: "1", successAttempt: 1 });

test/helpers/vitest-watch-triggers.ts

@@ -133,6 +133,15 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [
     pattern: /(?:^|\/)\.github\/workflows\/e2e\.yaml$/,
     testsToRun: runTests(...E2E_WORKFLOW_CONTRACTS),
   },
+  {
+    pattern:
+      /(?:^|\/)\.github\/(?:actions\/docker-auth-(?:cleanup|setup)\/action\.yaml|scripts\/docker-auth-(?:cleanup|setup)\.sh)$/,
+    testsToRun: runTests("test/e2e/support/dockerhub-auth-workflow-boundary.test.ts"),
+  },
+  {
+    pattern: /(?:^|\/)\.github\/workflows\/sandbox-images-and-e2e\.yaml$/,
+    testsToRun: runTests("test/e2e/support/sandbox-images-workflow-boundary.test.ts"),
+  },
   {
     pattern: /(?:^|\/)\.github\/workflows\/code-scanning\.yaml$/,
     testsToRun: runTests("test/code-scanning-workflow.test.ts"),

test/vitest-watch-triggers.test.ts

@@ -68,6 +68,11 @@ const OPAQUE_INPUTS = [
   "test/e2e/manifests/openclaw-nvidia.yaml",
   "test/e2e/docs/parity-inventory.generated.json",
   ".github/workflows/e2e.yaml",
+  ".github/actions/docker-auth-setup/action.yaml",
+  ".github/actions/docker-auth-cleanup/action.yaml",
+  ".github/scripts/docker-auth-setup.sh",
+  ".github/scripts/docker-auth-cleanup.sh",
+  ".github/workflows/sandbox-images-and-e2e.yaml",
   ".github/workflows/code-scanning.yaml",
   ".github/workflows/pr-review-advisor.yaml",
   "tools/pr-review-advisor/openshell-policy.yaml",
@@ -156,6 +161,19 @@ describe("Vitest opaque-input watch triggers", () => {
       "test/e2e/support/e2e-migration-policy.test.ts",
     ]);
     expect(triggeredBy(".github/workflows/e2e.yaml")).toEqual(E2E_WORKFLOW_CONTRACTS);
+    for (const authPath of [
+      ".github/actions/docker-auth-setup/action.yaml",
+      ".github/actions/docker-auth-cleanup/action.yaml",
+      ".github/scripts/docker-auth-setup.sh",
+      ".github/scripts/docker-auth-cleanup.sh",
+    ]) {
+      expect(triggeredBy(authPath)).toEqual([
+        "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts",
+      ]);
+    }
+    expect(triggeredBy(".github/workflows/sandbox-images-and-e2e.yaml")).toEqual([
+      "test/e2e/support/sandbox-images-workflow-boundary.test.ts",
+    ]);
     expect(triggeredBy(".github/workflows/code-scanning.yaml")).toEqual([
       "test/code-scanning-workflow.test.ts",
     ]);

tools/e2e/cli-artifact-workflow-boundary.mts

@@ -39,7 +39,7 @@ const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance";
 const CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 =
   "3578a053cede863f7aa4814d8399b4ca21ea0b77cee712e6d549c684818f11dd";
 const CLI_ARTIFACT_WORKFLOW_CONTRACT_SHA256 =
-  "cf436f4885b22d05444f78362b9acb0dddd7027bb369ff47bf15c6a0ec66cf55";
+  "c4e838dfa11056b3b47c9769b9a8b16fc422f8ceee518600d703e4a47a717826";
 
 type WorkflowRecord = Record<string, unknown>;
 type WorkflowStep = WorkflowRecord & {

tools/e2e/sandbox-images-workflow-boundary.mts

@@ -253,9 +253,13 @@ function validateCanonicalAuth(errors: string[], auth: SandboxImagesWorkflowStep
     'auth_marker="${DOCKER_CONFIG}/.nemoclaw-docker-login-attempted"',
     ': > "${auth_marker}"',
     'chmod 600 "${auth_marker}"',
-    "for attempt in 1 2 3; do",
+    "login_attempts=5",
+    "retry_seconds=5",
+    "for ((attempt = 1; attempt <= login_attempts; attempt += 1)); do",
     `if printf '%s' "\${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "\${DOCKERHUB_USERNAME}" --password-stdin; then`,
-    "Docker Hub login failed after 3 attempts",
+    "if ((attempt < login_attempts)); then",
+    'sleep "${retry_seconds}"',
+    'Docker Hub login failed after ${login_attempts} attempts',
   ];
   for (const fragment of requiredFragments) {
     if (!run.includes(fragment)) {

tools/e2e/workflow-boundary-policy.mts

@@ -19,9 +19,9 @@ export const E2E_ACTION_PROVENANCE = {
   },
   dockerAuth: {
     reference:
-      "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928",
+      "NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f",
     actionSha256: "cf93dcbd19589a56d1d58225fd6b3f8ad2180705662ff79a3407f340b5dba4c0",
-    scriptSha256: "853a3f742f057c29ed465b63bed1ec8d8f306a1c046877a8556cadf290ef0cb6",
+    scriptSha256: "f4c7ba1d7c3dc5e82bacfdb85c94ed0838251dfaa88a081b4f64fba4f744b6dc",
   },
   dockerCleanup: {
     reference: