ci(e2e): route shared jobs to NVIDIA endpoints

.github/workflows/e2e-script.yaml

@@ -58,10 +58,15 @@ on:
         type: string
         default: ""
       nvidia_api_key:
-        description: Pass the hosted inference source secret as the CI custom endpoint credential.
+        description: Pass the NVIDIA inference credential to the script.
         required: false
         type: boolean
         default: false
+      inference_route:
+        description: "Inference route to export when nvidia_api_key is true: hosted-custom or nvidia-internal."
+        required: false
+        type: string
+        default: hosted-custom
       brave_api_key:
         description: Pass the BRAVE_API_KEY secret to the script.
         required: false
@@ -212,28 +217,48 @@ jobs:
             echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls."
           fi
 
-      - name: Export hosted CI inference environment
+      - name: Export CI inference environment
         if: ${{ inputs.nvidia_api_key }}
         env:
-          NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
+          NEMOCLAW_E2E_INFERENCE_ROUTE: ${{ inputs.inference_route }}
+          HOSTED_NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
         shell: bash
         run: |
           set -euo pipefail
 
-          if [ -z "${NVIDIA_INFERENCE_API_KEY:-}" ]; then
-            echo "::error::NVIDIA_INFERENCE_API_KEY secret is required for hosted CI inference; it is withheld for workflow_dispatch target_ref runs." >&2
-            exit 1
-          fi
+          case "${NEMOCLAW_E2E_INFERENCE_ROUTE:-hosted-custom}" in
+            nvidia-internal|hosted-custom)
+              # Source of truth: both labels intentionally export the same
+              # internal hosted-compatible runtime env. `hosted-custom` remains
+              # the default legacy route; `nvidia-internal` is only an explicit
+              # caller-selection label for jobs moved away from public NVIDIA
+              # Endpoints/Build after HTTP 429s. Remove the split once those jobs
+              # no longer need an auditable opt-in label.
+              if [ -z "${HOSTED_NVIDIA_INFERENCE_API_KEY:-}" ]; then
+                echo "::error::NVIDIA_INFERENCE_API_KEY secret is required for ${NEMOCLAW_E2E_INFERENCE_ROUTE:-hosted-custom} CI inference; it is withheld for workflow_dispatch target_ref runs." >&2
+                exit 1
+              fi
+              if [[ "${HOSTED_NVIDIA_INFERENCE_API_KEY}" == *$'\n'* || "${HOSTED_NVIDIA_INFERENCE_API_KEY}" == *$'\r'* ]]; then
+                echo "::error::NVIDIA_INFERENCE_API_KEY secret must be a single-line value before export to GITHUB_ENV." >&2
+                exit 1
+              fi
 
-          {
-            printf 'NEMOCLAW_E2E_USE_HOSTED_INFERENCE=1\n'
-            printf 'NEMOCLAW_PROVIDER=custom\n'
-            printf 'NEMOCLAW_ENDPOINT_URL=https://inference-api.nvidia.com/v1\n'
-            printf 'NEMOCLAW_MODEL=nvidia/nvidia/nemotron-3-ultra\n'
-            printf 'NEMOCLAW_COMPAT_MODEL=nvidia/nvidia/nemotron-3-ultra\n'
-            printf 'NEMOCLAW_PREFERRED_API=openai-completions\n'
-            printf 'COMPATIBLE_API_KEY=%s\n' "${NVIDIA_INFERENCE_API_KEY}"
-          } >> "$GITHUB_ENV"
+              {
+                printf 'NVIDIA_INFERENCE_API_KEY=%s\n' "${HOSTED_NVIDIA_INFERENCE_API_KEY}"
+                printf 'NEMOCLAW_E2E_USE_HOSTED_INFERENCE=1\n'
+                printf 'NEMOCLAW_PROVIDER=custom\n'
+                printf 'NEMOCLAW_ENDPOINT_URL=https://inference-api.nvidia.com/v1\n'
+                printf 'NEMOCLAW_MODEL=nvidia/nvidia/nemotron-3-ultra\n'
+                printf 'NEMOCLAW_COMPAT_MODEL=nvidia/nvidia/nemotron-3-ultra\n'
+                printf 'NEMOCLAW_PREFERRED_API=openai-completions\n'
+                printf 'COMPATIBLE_API_KEY=%s\n' "${HOSTED_NVIDIA_INFERENCE_API_KEY}"
+              } >> "$GITHUB_ENV"
+              ;;
+            *)
+              echo "::error::Unsupported inference_route value." >&2
+              exit 1
+              ;;
+          esac
 
       - name: Run E2E script
         uses: ./workflow-actions/.github/actions/run-e2e-script
@@ -248,7 +273,6 @@ jobs:
         env:
           BRAVE_API_KEY: ${{ inputs.brave_api_key && secrets.BRAVE_API_KEY || '' }}
           GITHUB_TOKEN: ${{ inputs.github_token && github.token || '' }}
-          NVIDIA_INFERENCE_API_KEY: ${{ inputs.nvidia_api_key && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
           TELEGRAM_BOT_TOKEN_REAL: ${{ inputs.messaging_live_secrets && secrets.TELEGRAM_BOT_TOKEN_REAL || '' }}
           TELEGRAM_CHAT_ID_E2E: ${{ inputs.messaging_live_secrets && secrets.TELEGRAM_CHAT_ID_E2E || '' }}
           DISCORD_BOT_TOKEN_REAL: ${{ inputs.messaging_live_secrets && secrets.DISCORD_BOT_TOKEN_REAL || '' }}

.github/workflows/nightly-e2e.yaml

@@ -376,6 +376,7 @@ jobs:
         /tmp/nemoclaw-e2e-whatsapp-*.log
       env_json: '{"DISCORD_BOT_TOKEN":"test-fake-discord-token-e2e","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-msg-provider","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-e2e","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-e2e"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
       messaging_live_secrets: ${{ github.event_name != 'workflow_dispatch' || inputs.target_ref == '' }}
     secrets:
@@ -764,8 +765,13 @@ jobs:
         /tmp/nc-rebuild-remove.log
       env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-channels-add-remove","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-add-remove-e2e"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
-    secrets: *nightly-e2e-default-secrets
+    secrets: &nightly-e2e-internal-inference-secrets
+      NVIDIA_INFERENCE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
+      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
+      DOCKERHUB_USERNAME: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_USERNAME || '' }}
+      DOCKERHUB_TOKEN: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_TOKEN || '' }}
   # ── Channels stop/start lifecycle E2E (#3462) ───────────────────────
   # Regression coverage for #3453 (stop must disable across rebuild), #3381
   # (start must re-attach from cached credentials). Sharded by agent so the
@@ -787,8 +793,9 @@ jobs:
         /tmp/nc-channels-*.log
       env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"openclaw","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
-    secrets: *nightly-e2e-default-secrets
+    secrets: *nightly-e2e-internal-inference-secrets
   channels-stop-start-hermes-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
@@ -806,8 +813,9 @@ jobs:
         /tmp/nc-channels-*.log
       env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
-    secrets: *nightly-e2e-default-secrets
+    secrets: *nightly-e2e-internal-inference-secrets
   brave-search-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
@@ -1196,8 +1204,9 @@ jobs:
       artifact_path: "/tmp/nemoclaw-e2e-hermes-discord-install.log"
       env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-hermes-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_IDS":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-discord"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
-    secrets: *nightly-e2e-default-secrets
+    secrets: *nightly-e2e-internal-inference-secrets
   hermes-slack-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
@@ -1694,8 +1703,9 @@ jobs:
         /tmp/nemoclaw-e2e-upgrade-install.log
       env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-upgrade-stale"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
-    secrets: *nightly-e2e-default-secrets
+    secrets: *nightly-e2e-internal-inference-secrets
   openshell-gateway-upgrade-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' &&
@@ -1752,8 +1762,9 @@ jobs:
       artifact_path: "/tmp/nemoclaw-e2e-install.log"
       env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-hm"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
-    secrets: *nightly-e2e-default-secrets
+    secrets: *nightly-e2e-internal-inference-secrets
   rebuild-hermes-stale-base-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
@@ -1768,8 +1779,9 @@ jobs:
       artifact_path: "/tmp/nemoclaw-e2e-install.log"
       env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-hm-base"}'
       nvidia_api_key: true
+      inference_route: nvidia-internal
       github_token: true
-    secrets: *nightly-e2e-default-secrets
+    secrets: *nightly-e2e-internal-inference-secrets
   double-onboard-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' &&