fix: pre-timeout watchdog, robust parser import, and always-commit safeguard

.github/actions/setup-api-client/action.yml

@@ -266,7 +266,10 @@ runs:
 
             npm_err=$(cat "$npm_output")
             rm -f "$npm_output"
-            echo "::warning::npm install attempt $attempt/$NPM_MAX_RETRIES failed: $npm_err"
+            echo "::warning::npm install attempt $attempt/$NPM_MAX_RETRIES failed (see logs)"
+            echo "::group::npm stderr (attempt $attempt)"
+            echo "$npm_err"
+            echo "::endgroup::"
 
             # On first failure, also try --legacy-peer-deps in case it's a peer dep conflict
             if [ "$attempt" -eq 1 ]; then
@@ -279,7 +282,10 @@ runs:
               fi
               npm_err_legacy=$(cat "$npm_output")
               rm -f "$npm_output"
-              echo "::warning::npm install with --legacy-peer-deps failed: $npm_err_legacy"
+              echo "::warning::npm install with --legacy-peer-deps failed (see logs)"
+              echo "::group::npm stderr (--legacy-peer-deps)"
+              echo "$npm_err_legacy"
+              echo "::endgroup::"
             fi
 
             if [ "$attempt" -lt "$NPM_MAX_RETRIES" ]; then

.github/workflows/reusable-codex-run.yml

@@ -188,6 +188,7 @@ jobs:
       error-type: ${{ steps.classify_failure.outputs.error_type }}
       error-recovery: ${{ steps.classify_failure.outputs.error_recovery }}
       error-summary: ${{ steps.classify_failure.outputs.error_summary }}
+      watchdog-saved: ${{ steps.run_codex.outputs.watchdog-saved }}
       # LLM analysis outputs
       llm-analysis-run: ${{ steps.llm_analysis.outputs.llm-analysis-run }}
       llm-completed-tasks: ${{ steps.llm_analysis.outputs.completed-tasks }}
@@ -938,6 +939,47 @@ jobs:
             echo "Extra args: provided (${#EXTRA_ARGS[@]} arg(s))"
           fi
 
+          # --- Pre-timeout watchdog ---
+          # When the job approaches the timeout limit, this background process
+          # commits and pushes any uncommitted work so it isn't lost to the
+          # job cancellation.  It fires once, 5 minutes before max_runtime.
+          MAX_RUNTIME_MIN=${{ inputs.max_runtime_minutes }}
+          GRACE_MIN=5
+          WATCHDOG_DELAY=$(( (MAX_RUNTIME_MIN - GRACE_MIN) * 60 ))
+          if [ "$WATCHDOG_DELAY" -gt 60 ]; then
+            (
+              sleep "$WATCHDOG_DELAY"
+              echo "::warning::Pre-timeout watchdog fired (${GRACE_MIN}m before ${MAX_RUNTIME_MIN}m limit)"
+
+              CHANGED=$(git status --porcelain | wc -l)
+              UNPUSHED=0
+              if git rev-parse FETCH_HEAD >/dev/null 2>&1; then
+                UNPUSHED=$(git rev-list FETCH_HEAD..HEAD --count 2>/dev/null || echo 0)
+              fi
+
+              if [ "$CHANGED" -gt 0 ] || [ "$UNPUSHED" -gt 0 ]; then
+                echo "::notice::Watchdog saving ${CHANGED} uncommitted file(s) and ${UNPUSHED} unpushed commit(s)"
+                git config user.name  "github-actions[bot]"
+                git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+                if [ "$CHANGED" -gt 0 ]; then
+                  git add -A
+                  git commit -m "chore(codex-keepalive): pre-timeout checkpoint (PR #${PR_NUM:-})" --no-verify || true
+                fi
+                TARGET_BRANCH="${{ inputs.pr_ref }}"
+                TARGET_BRANCH="${TARGET_BRANCH#refs/heads/}"
+                PUSH_TOKEN="${{ steps.auth_token.outputs.push_token }}"
+                REMOTE_URL="https://x-access-token:${PUSH_TOKEN}@github.com/${{ github.repository }}"
+                git push "${REMOTE_URL}" "HEAD:${TARGET_BRANCH}" || \
+                  echo "::warning::Watchdog push failed"
+                echo "watchdog-saved=true" >> "$GITHUB_OUTPUT"
+              else
+                echo "::notice::Watchdog: no uncommitted or unpushed work to save"
+              fi
+            ) &
+            WATCHDOG_PID=$!
+            echo "::notice::Pre-timeout watchdog started (PID ${WATCHDOG_PID}, fires in $((WATCHDOG_DELAY/60))m)"
+          fi
+
           # Run codex exec with --json to capture rich session data
           # JSONL events stream to stdout, final message still goes to OUTPUT_FILE
           CODEX_EXIT=0
@@ -954,6 +996,12 @@ jobs:
           prompt_content="$(cat "$PROMPT_FILE")"
           "${cmd[@]}" "$prompt_content" > "$SESSION_JSONL" 2>&1 || CODEX_EXIT=$?
 
+          # Kill watchdog if Codex finished before the timer fired
+          if [ -n "${WATCHDOG_PID:-}" ]; then
+            kill "$WATCHDOG_PID" 2>/dev/null || true
+            wait "$WATCHDOG_PID" 2>/dev/null || true
+          fi
+
           echo "exit-code=${CODEX_EXIT}" >> "$GITHUB_OUTPUT"
 
           if [ "$CODEX_EXIT" -ne 0 ]; then
@@ -1008,17 +1056,29 @@ jobs:
 
           # Basic parsing (always available)
           python3 << 'PYEOF'
+          import importlib.util
           import os
           import sys
-          # Add .workflows-lib to path for tools imports
-          sys.path.insert(0, '.workflows-lib')
-          sys.path.insert(0, '.')
 
           session_file = os.environ.get("SESSION_JSONL", "codex-session.jsonl")
           github_output = os.environ.get("GITHUB_OUTPUT", "/dev/null")
 
           try:
-              from tools.codex_jsonl_parser import parse_codex_jsonl_file
+              # Load codex_jsonl_parser from the Workflows checkout by exact path.
+              # Consumer repos (e.g. Counter_Risk) have their own tools/ package with
+              # __init__.py, which shadows the Workflows tools/ on sys.path.  Using
+              # importlib.util.spec_from_file_location bypasses sys.path entirely.
+              _parser_path = os.path.join(
+                  ".workflows-lib", "tools", "codex_jsonl_parser.py"
+              )
+              _spec = importlib.util.spec_from_file_location(
+                  "codex_jsonl_parser", _parser_path
+              )
+              if _spec is None or _spec.loader is None:
+                  raise ImportError(f"Cannot load spec from {_parser_path}")
+              _mod = importlib.util.module_from_spec(_spec)
+              _spec.loader.exec_module(_mod)
+              parse_codex_jsonl_file = _mod.parse_codex_jsonl_file
 
               session = parse_codex_jsonl_file(session_file)
 
@@ -1150,6 +1210,7 @@ jobs:
 
       - name: Commit and push changes
         id: commit
+        if: always()
         env:
           MODE: ${{ inputs.mode }}
           PR_NUMBER: ${{ inputs.pr_number }}

templates/consumer-repo/.github/actions/setup-api-client/action.yml

@@ -239,29 +239,65 @@ runs:
 
           # Install with pinned versions for consistency.
           # lru-cache is an explicit transitive dep of @octokit/auth-app required for
-          # GitHub App token minting; pin it here so npm always hoists it to the top
-          # level even if a prior cached node_modules state is missing it.
-          # Capture stderr for debugging if the command fails
-          npm_output=$(mktemp)
-          npm_cmd=(npm install --no-save --location=project \
-            @octokit/rest@20.0.2 \
-            @octokit/plugin-retry@6.0.1 \
-            @octokit/plugin-paginate-rest@9.1.5 \
-            @octokit/auth-app@6.0.3 \
-            lru-cache@^10.0.0)
-          if "${npm_cmd[@]}" 2>"$npm_output"; then
-            rm -f "$npm_output"
-          else
-            echo "::warning::npm install failed with: $(cat "$npm_output")"
-            echo "::warning::Retrying with --legacy-peer-deps"
+          # GitHub App token minting; pin it here so npm always hoists a specific version
+          # even if a prior cached node_modules state is missing it.
+          #
+          # Retry with exponential backoff to survive transient npm registry errors
+          # (e.g. 403 Forbidden from CDN/rate-limit on safe-buffer, undici, etc.).
+          NPM_PACKAGES=(
+            @octokit/rest@20.0.2
+            @octokit/plugin-retry@6.0.1
+            @octokit/plugin-paginate-rest@9.1.5
+            @octokit/auth-app@6.0.3
+            lru-cache@10.4.3
+          )
+          NPM_MAX_RETRIES=3
+          NPM_BACKOFF=5  # seconds; doubles each retry (5, 10)
+          npm_installed=false
+
+          for (( attempt=1; attempt<=NPM_MAX_RETRIES; attempt++ )); do
+            npm_output=$(mktemp)
+
+            if npm install --no-save --location=project "${NPM_PACKAGES[@]}" 2>"$npm_output"; then
+              rm -f "$npm_output"
+              npm_installed=true
+              break
+            fi
+
+            npm_err=$(cat "$npm_output")
             rm -f "$npm_output"
-            npm_cmd=(npm install --no-save --legacy-peer-deps --location=project \
-              @octokit/rest@20.0.2 \
-              @octokit/plugin-retry@6.0.1 \
-              @octokit/plugin-paginate-rest@9.1.5 \
-              @octokit/auth-app@6.0.3 \
-              lru-cache@^10.0.0)
-            "${npm_cmd[@]}"
+            echo "::warning::npm install attempt $attempt/$NPM_MAX_RETRIES failed (see logs)"
+            echo "::group::npm stderr (attempt $attempt)"
+            echo "$npm_err"
+            echo "::endgroup::"
+
+            # On first failure, also try --legacy-peer-deps in case it's a peer dep conflict
+            if [ "$attempt" -eq 1 ]; then
+              echo "::warning::Retrying with --legacy-peer-deps"
+              npm_output=$(mktemp)
+              if npm install --no-save --legacy-peer-deps --location=project "${NPM_PACKAGES[@]}" 2>"$npm_output"; then
+                rm -f "$npm_output"
+                npm_installed=true
+                break
+              fi
+              npm_err_legacy=$(cat "$npm_output")
+              rm -f "$npm_output"
+              echo "::warning::npm install with --legacy-peer-deps failed (see logs)"
+              echo "::group::npm stderr (--legacy-peer-deps)"
+              echo "$npm_err_legacy"
+              echo "::endgroup::"
+            fi
+
+            if [ "$attempt" -lt "$NPM_MAX_RETRIES" ]; then
+              echo "::notice::Waiting ${NPM_BACKOFF}s before retry..."
+              sleep "$NPM_BACKOFF"
+              NPM_BACKOFF=$((NPM_BACKOFF * 2))
+            fi
+          done
+
+          if [ "$npm_installed" != "true" ]; then
+            echo "::error::npm install failed after $NPM_MAX_RETRIES attempts"
+            exit 1
           fi
 
           # Restore vendored package metadata that npm may have overwritten