From 07abc8a47fb6965cc561228bbd9fbfad1ac7fd44 Mon Sep 17 00:00:00 2001 From: AI DevOps Date: Wed, 11 Mar 2026 10:17:17 -0600 Subject: [PATCH 1/4] fix: auto-update recovery for dirty tree, diverged branch, and API rate limits Port recovery logic from aidevops.sh cmd_update() into auto-update-helper.sh cmd_check() to prevent indefinite update failures: 1. Ensure on main branch before pull (handles detached HEAD) 2. Clean dirty working tree before pull (handles setup.sh leftovers) 3. Fall back to git reset --hard origin/main when ff-only fails 4. Clean working tree after setup.sh to prevent next-cycle failures 5. Use authenticated gh api for version checks (5000/hr vs 60/hr) Fixes 130 pull failures and 106 remote=unknown events observed in logs. Closes #4142 --- .agents/scripts/auto-update-helper.sh | 70 ++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/.agents/scripts/auto-update-helper.sh b/.agents/scripts/auto-update-helper.sh index 2b4595f35..b95d70b88 100755 --- a/.agents/scripts/auto-update-helper.sh +++ b/.agents/scripts/auto-update-helper.sh @@ -303,6 +303,23 @@ get_local_version() { ####################################### get_remote_version() { local version="" + + # Prefer authenticated gh api (higher rate limit: 5000/hr vs 60/hr) + # This avoids the "remote=unknown" failures seen during overnight polling + # when unauthenticated API quota is exhausted. + # See: https://github.com/marcusquinn/aidevops/issues/4142 + if command -v gh &>/dev/null && gh auth status &>/dev/null; then + version=$(gh api repos/marcusquinn/aidevops/contents/VERSION \ + --jq '.content' 2>/dev/null | + base64 -d 2>/dev/null | + tr -d '\n') + if [[ -n "$version" ]]; then + echo "$version" + return 0 + fi + fi + + # Fallback: unauthenticated curl (60 req/hr limit) if command -v jq &>/dev/null; then version=$(curl --proto '=https' -fsSL --max-time 10 \ "https://api.github.com/repos/marcusquinn/aidevops/contents/VERSION" 2>/dev/null | @@ -314,7 +331,8 @@ get_remote_version() { return 0 fi fi - # Fallback to raw (CDN-cached, may be up to 5 min stale) + + # Last resort: raw.githubusercontent.com (CDN-cached, may be up to 5 min stale) curl --proto '=https' -fsSL --max-time 10 \ "https://raw.githubusercontent.com/marcusquinn/aidevops/main/VERSION" 2>/dev/null | tr -d '\n' || echo "unknown" @@ -1064,6 +1082,27 @@ cmd_check() { return 1 fi + # Ensure we're on the main branch (detached HEAD or stale branch blocks pull) + # Mirrors recovery logic from aidevops.sh cmd_update() + # See: https://github.com/marcusquinn/aidevops/issues/4142 + local current_branch + current_branch=$(git -C "$INSTALL_DIR" branch --show-current 2>/dev/null || echo "") + if [[ "$current_branch" != "main" ]]; then + log_info "Not on main branch ($current_branch), switching..." + git -C "$INSTALL_DIR" checkout main --quiet 2>/dev/null || + git -C "$INSTALL_DIR" checkout -b main origin/main --quiet 2>/dev/null || true + fi + + # Clean up any working tree changes left by setup.sh or other processes + # (e.g., chmod on tracked scripts, scan results written to repo) + # This ensures git pull --ff-only won't be blocked by dirty files. + # See: https://github.com/marcusquinn/aidevops/issues/2286 + if ! git -C "$INSTALL_DIR" diff --quiet 2>/dev/null || ! git -C "$INSTALL_DIR" diff --cached --quiet 2>/dev/null; then + log_info "Cleaning up stale working tree changes..." + git -C "$INSTALL_DIR" reset HEAD -- . 2>/dev/null || true + git -C "$INSTALL_DIR" checkout -- . 2>/dev/null || true + fi + # Pull latest changes if ! git -C "$INSTALL_DIR" fetch origin main --quiet 2>>"$LOG_FILE"; then log_error "git fetch failed" @@ -1076,13 +1115,22 @@ cmd_check() { fi if ! git -C "$INSTALL_DIR" pull --ff-only origin main --quiet 2>>"$LOG_FILE"; then - log_error "git pull --ff-only failed (local changes?)" - update_state "update" "$remote" "pull_failed" - check_skill_freshness - check_openclaw_freshness - check_tool_freshness - check_upstream_watch - return 1 + # Fast-forward failed (diverged history or persistent dirty state). + # Since we just fetched origin/main, reset to it — the repo is managed + # by aidevops and should always track origin/main exactly. + # See: https://github.com/marcusquinn/aidevops/issues/2288 + log_warn "git pull --ff-only failed — falling back to reset" + if git -C "$INSTALL_DIR" reset --hard origin/main --quiet 2>>"$LOG_FILE"; then + log_info "Reset to origin/main succeeded" + else + log_error "git reset --hard origin/main also failed" + update_state "update" "$remote" "pull_failed" + check_skill_freshness + check_openclaw_freshness + check_tool_freshness + check_upstream_watch + return 1 + fi fi # Run setup.sh non-interactively to deploy agents @@ -1112,6 +1160,12 @@ cmd_check() { return 1 fi + # Clean up any working tree changes setup.sh may have introduced + # (e.g., chmod on tracked scripts, scan results written to repo) + # Prevents dirty tree from blocking the next update cycle. + # See: https://github.com/marcusquinn/aidevops/issues/2286 + git -C "$INSTALL_DIR" checkout -- . 2>/dev/null || true + # Run daily skill freshness check (24h gate) check_skill_freshness From de4e88fae6edf1a8209102a55f98e05e34d6da51 Mon Sep 17 00:00:00 2001 From: AI DevOps Date: Wed, 11 Mar 2026 12:36:59 -0600 Subject: [PATCH 2/4] refactor: extract run_freshness_checks(), replace silent || true with error logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback from Gemini Code Assist on PR #4144: - Extract duplicated 4-call freshness-check block into run_freshness_checks() helper, reducing 7 call sites to 1 definition + 7 single-line calls - Replace || true on branch checkout with proper error logging and early return on failure (silent failures here block updates indefinitely) - Replace || true on working tree cleanup (reset/checkout) with log_warn so cleanup failures are visible in the update log - Replace || true on post-setup cleanup with log_warn for dirty-state visibility on next update cycle - Redirect git stderr to LOG_FILE instead of /dev/null for debuggability Dismisses Gemini comment on line 319 (gh --jq uses gh's built-in jq engine, not the external jq binary — no portability concern). --- .agents/scripts/auto-update-helper.sh | 76 +++++++++++++-------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/.agents/scripts/auto-update-helper.sh b/.agents/scripts/auto-update-helper.sh index b95d70b88..110207761 100755 --- a/.agents/scripts/auto-update-helper.sh +++ b/.agents/scripts/auto-update-helper.sh @@ -402,6 +402,18 @@ update_state() { return 0 } +####################################### +# Run all periodic freshness checks (skills, OpenClaw, tools, upstream watch). +# Extracted to avoid duplicating the same 4-call block at every exit point +# in cmd_check(). Each check has its own internal time gate. +####################################### +run_freshness_checks() { + check_skill_freshness + check_openclaw_freshness + check_tool_freshness + check_upstream_watch +} + ####################################### # Check skill freshness and auto-update if stale (24h gate) # Called from cmd_check after the main aidevops update logic. @@ -1029,10 +1041,7 @@ cmd_check() { if [[ "$current" == "unknown" || "$remote" == "unknown" ]]; then log_warn "Could not determine versions (local=$current, remote=$remote)" update_state "check" "$current" "version_unknown" - check_skill_freshness - check_openclaw_freshness - check_tool_freshness - check_upstream_watch + run_freshness_checks return 0 fi @@ -1060,10 +1069,7 @@ cmd_check() { fi fi - check_skill_freshness - check_openclaw_freshness - check_tool_freshness - check_upstream_watch + run_freshness_checks return 0 fi @@ -1075,10 +1081,7 @@ cmd_check() { if [[ ! -d "$INSTALL_DIR/.git" ]]; then log_error "Install directory is not a git repo: $INSTALL_DIR" update_state "update" "$remote" "no_git_repo" - check_skill_freshness - check_openclaw_freshness - check_tool_freshness - check_upstream_watch + run_freshness_checks return 1 fi @@ -1089,8 +1092,13 @@ cmd_check() { current_branch=$(git -C "$INSTALL_DIR" branch --show-current 2>/dev/null || echo "") if [[ "$current_branch" != "main" ]]; then log_info "Not on main branch ($current_branch), switching..." - git -C "$INSTALL_DIR" checkout main --quiet 2>/dev/null || - git -C "$INSTALL_DIR" checkout -b main origin/main --quiet 2>/dev/null || true + if ! git -C "$INSTALL_DIR" checkout main --quiet 2>>"$LOG_FILE" && + ! git -C "$INSTALL_DIR" checkout -b main origin/main --quiet 2>>"$LOG_FILE"; then + log_error "Failed to switch to main branch from '$current_branch'" + update_state "update" "$remote" "branch_switch_failed" + run_freshness_checks + return 1 + fi fi # Clean up any working tree changes left by setup.sh or other processes @@ -1099,18 +1107,19 @@ cmd_check() { # See: https://github.com/marcusquinn/aidevops/issues/2286 if ! git -C "$INSTALL_DIR" diff --quiet 2>/dev/null || ! git -C "$INSTALL_DIR" diff --cached --quiet 2>/dev/null; then log_info "Cleaning up stale working tree changes..." - git -C "$INSTALL_DIR" reset HEAD -- . 2>/dev/null || true - git -C "$INSTALL_DIR" checkout -- . 2>/dev/null || true + if ! git -C "$INSTALL_DIR" reset HEAD -- . 2>>"$LOG_FILE"; then + log_warn "git reset HEAD failed during working tree cleanup" + fi + if ! git -C "$INSTALL_DIR" checkout -- . 2>>"$LOG_FILE"; then + log_warn "git checkout -- . failed during working tree cleanup" + fi fi # Pull latest changes if ! git -C "$INSTALL_DIR" fetch origin main --quiet 2>>"$LOG_FILE"; then log_error "git fetch failed" update_state "update" "$remote" "fetch_failed" - check_skill_freshness - check_openclaw_freshness - check_tool_freshness - check_upstream_watch + run_freshness_checks return 1 fi @@ -1125,10 +1134,7 @@ cmd_check() { else log_error "git reset --hard origin/main also failed" update_state "update" "$remote" "pull_failed" - check_skill_freshness - check_openclaw_freshness - check_tool_freshness - check_upstream_watch + run_freshness_checks return 1 fi fi @@ -1153,10 +1159,7 @@ cmd_check() { else log_error "setup.sh failed (exit code: $?)" update_state "update" "$remote" "setup_failed" - check_skill_freshness - check_openclaw_freshness - check_tool_freshness - check_upstream_watch + run_freshness_checks return 1 fi @@ -1164,19 +1167,12 @@ cmd_check() { # (e.g., chmod on tracked scripts, scan results written to repo) # Prevents dirty tree from blocking the next update cycle. # See: https://github.com/marcusquinn/aidevops/issues/2286 - git -C "$INSTALL_DIR" checkout -- . 2>/dev/null || true - - # Run daily skill freshness check (24h gate) - check_skill_freshness - - # Run daily OpenClaw update check (24h gate) - check_openclaw_freshness - - # Run 6-hourly tool freshness check (idle-gated) - check_tool_freshness + if ! git -C "$INSTALL_DIR" checkout -- . 2>>"$LOG_FILE"; then + log_warn "Post-setup working tree cleanup failed — next update cycle may see dirty state" + fi - # Run daily upstream watch check (24h gate) - check_upstream_watch + # Run periodic freshness checks (skills, OpenClaw, tools, upstream watch) + run_freshness_checks return 0 } From 1df47eade2361f3baca805cc8bb78d40cad51155 Mon Sep 17 00:00:00 2001 From: marcusquinn <6428977+marcusquinn@users.noreply.github.com> Date: Wed, 11 Mar 2026 18:59:57 +0000 Subject: [PATCH 3/4] fix: clean working tree before branch-switch recovery --- .agents/scripts/auto-update-helper.sh | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.agents/scripts/auto-update-helper.sh b/.agents/scripts/auto-update-helper.sh index 110207761..c0274bc7e 100755 --- a/.agents/scripts/auto-update-helper.sh +++ b/.agents/scripts/auto-update-helper.sh @@ -1085,6 +1085,20 @@ cmd_check() { return 1 fi + # Clean up any working tree changes left by setup.sh or other processes + # (e.g., chmod on tracked scripts, scan results written to repo) + # This ensures git pull --ff-only won't be blocked by dirty files. + # See: https://github.com/marcusquinn/aidevops/issues/2286 + if ! git -C "$INSTALL_DIR" diff --quiet 2>/dev/null || ! git -C "$INSTALL_DIR" diff --cached --quiet 2>/dev/null; then + log_info "Cleaning up stale working tree changes..." + if ! git -C "$INSTALL_DIR" reset HEAD -- . 2>>"$LOG_FILE"; then + log_warn "git reset HEAD failed during working tree cleanup" + fi + if ! git -C "$INSTALL_DIR" checkout -- . 2>>"$LOG_FILE"; then + log_warn "git checkout -- . failed during working tree cleanup" + fi + fi + # Ensure we're on the main branch (detached HEAD or stale branch blocks pull) # Mirrors recovery logic from aidevops.sh cmd_update() # See: https://github.com/marcusquinn/aidevops/issues/4142 @@ -1101,20 +1115,6 @@ cmd_check() { fi fi - # Clean up any working tree changes left by setup.sh or other processes - # (e.g., chmod on tracked scripts, scan results written to repo) - # This ensures git pull --ff-only won't be blocked by dirty files. - # See: https://github.com/marcusquinn/aidevops/issues/2286 - if ! git -C "$INSTALL_DIR" diff --quiet 2>/dev/null || ! git -C "$INSTALL_DIR" diff --cached --quiet 2>/dev/null; then - log_info "Cleaning up stale working tree changes..." - if ! git -C "$INSTALL_DIR" reset HEAD -- . 2>>"$LOG_FILE"; then - log_warn "git reset HEAD failed during working tree cleanup" - fi - if ! git -C "$INSTALL_DIR" checkout -- . 2>>"$LOG_FILE"; then - log_warn "git checkout -- . failed during working tree cleanup" - fi - fi - # Pull latest changes if ! git -C "$INSTALL_DIR" fetch origin main --quiet 2>>"$LOG_FILE"; then log_error "git fetch failed" From b33af98959605da9acb7acbcf1a850aedd6d9e05 Mon Sep 17 00:00:00 2001 From: marcusquinn <6428977+marcusquinn@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:16:56 +0000 Subject: [PATCH 4/4] fix: include install path in branch-switch error log --- .agents/scripts/auto-update-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/scripts/auto-update-helper.sh b/.agents/scripts/auto-update-helper.sh index c0274bc7e..58a715b47 100755 --- a/.agents/scripts/auto-update-helper.sh +++ b/.agents/scripts/auto-update-helper.sh @@ -1108,7 +1108,7 @@ cmd_check() { log_info "Not on main branch ($current_branch), switching..." if ! git -C "$INSTALL_DIR" checkout main --quiet 2>>"$LOG_FILE" && ! git -C "$INSTALL_DIR" checkout -b main origin/main --quiet 2>>"$LOG_FILE"; then - log_error "Failed to switch to main branch from '$current_branch'" + log_error "Failed to switch to main branch from '$current_branch' in $INSTALL_DIR" update_state "update" "$remote" "branch_switch_failed" run_freshness_checks return 1