From 228b9e5bb7cd30cc45f90c3807c2b605038eea1a Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Thu, 18 Jun 2026 10:04:46 -0400 Subject: [PATCH] feat(scaffold): auto-detect and install pre-commit tool dependencies Add a registry-based system for resolving and installing pre-commit hook tool dependencies at runtime, replacing hardcoded tool installs baked into OpenShell container images. New files: - tools/precommit-tools.yaml: registry mapping hook repos/IDs to system tools with pinned versions and SHA256 checksums - scripts/resolve-precommit-tools.py: standalone Python resolver that parses .pre-commit-config.yaml against the registry - scripts/resolve-precommit-tools.sh: bash wrapper ensuring PyYAML is available before invoking the Python resolver - scripts/install-precommit-tools.sh: installs tools from the JSON manifest (binary/apt/pip/npm) with architecture detection Modified pre/post scripts (pre-code, pre-fix, post-code, post-fix) to call the resolver and installer instead of hardcoding tool versions. Removes LYCHEE_VERSION/UV_VERSION constants from post-code.sh and post-fix.sh. Supply-chain hardening: - Binary downloads use pinned versions + SHA256 checksums - pip installs use --no-deps to prevent transitive dependency attacks - npm installs use --ignore-scripts to prevent install-time RCE - jq architecture lookups use --arg binding (not shell interpolation) - PyYAML pinned to ==6.0.2 - Pre-scripts write to GITHUB_PATH for cross-step persistence Closes #1270 Assisted-by: Claude Signed-off-by: Wayne Sun --- .../scripts/.pre-commit-tools.yaml | 113 +++++++++ .../scripts/install-precommit-tools.sh | 240 ++++++++++++++++++ .../fullsend-repo/scripts/post-code.sh | 62 ++--- .../fullsend-repo/scripts/post-fix.sh | 81 +++--- .../fullsend-repo/scripts/pre-code.sh | 27 ++ .../scaffold/fullsend-repo/scripts/pre-fix.sh | 29 +++ .../scripts/resolve-precommit-tools.py | 151 +++++++++++ internal/scaffold/scaffold.go | 2 + 8 files changed, 614 insertions(+), 91 deletions(-) create mode 100644 internal/scaffold/fullsend-repo/scripts/.pre-commit-tools.yaml create mode 100755 internal/scaffold/fullsend-repo/scripts/install-precommit-tools.sh create mode 100755 internal/scaffold/fullsend-repo/scripts/resolve-precommit-tools.py diff --git a/internal/scaffold/fullsend-repo/scripts/.pre-commit-tools.yaml b/internal/scaffold/fullsend-repo/scripts/.pre-commit-tools.yaml new file mode 100644 index 0000000000..5259f1c901 --- /dev/null +++ b/internal/scaffold/fullsend-repo/scripts/.pre-commit-tools.yaml @@ -0,0 +1,113 @@ +--- +# Known pre-commit hook tool dependencies. +# +# Used by resolve-precommit-tools.py to auto-detect which system tools +# a target repo's .pre-commit-config.yaml requires. The resolver reads +# the target repo's config, matches hook repos and IDs against this +# registry, and produces a JSON manifest that install-precommit-tools.sh +# consumes. +# +# Structure: +# Each entry maps a pre-commit hook repo URL to its hooks and the +# tools they need. Tools are categorized by install method: +# binary — downloaded from a release URL with SHA256 verification +# apt — installed via apt-get on Ubuntu-based runners +# pip — installed via pip +# npm — installed via npm +# +# Binary entries must include pinned versions and per-arch checksums +# for supply-chain safety. Use the same version+checksum pattern as +# images/code/Containerfile and the post-scripts. +# +# Adding a new tool: +# 1. Find the hook repo URL and hook ID in .pre-commit-config.yaml +# 2. Add an entry below with the tools it needs +# 3. For binary downloads: pin version, provide checksums for amd64+arm64 +# 4. Run resolve-precommit-tools.py against a test repo to verify +# +# Only add entries for hooks that pre-commit cannot self-serve: +# language: system → tool must already be on PATH (NEEDS registry entry) +# language: golang → binary download is faster than Go compilation (optional) +# language: python → pre-commit handles via pip/venv (DO NOT add) +# language: node → pre-commit handles via npm (DO NOT add) +# language: docker_image → pre-commit handles via docker pull (DO NOT add) +# +# Customization: +# Per-org: place .pre-commit-tools.yaml in customized/scripts/ +# Per-repo: place .pre-commit-tools.yaml in .fullsend/customized/scripts/ +# The customized file completely replaces these defaults. + +tools: + # ── lychee (markdown link checker) ──────────────────────────────── + - hook_id: lint-md-links + repo: local + match_entry: "lychee" + install: + type: binary + name: lychee + version: "0.24.2" + url_template: "https://github.com/lycheeverse/lychee/releases/download/lychee-v{version}/lychee-{triple}.tar.gz" + checksums: + x86_64: "1f4e0ef7f6554a6ed33dd7ac144fb2e1bbed98598e7af973042fc5cd43951c9a" + aarch64: "91a7bd65685da41b90ccb9bc867a3d649a7818042dae04ff405e55a25bddee4c" + strip_prefix: "lychee-{triple}" + binary_name: lychee + + # ── gitleaks (secret scanning) ──────────────────────────────────── + # Post-scripts install gitleaks independently as a security gate. + # This entry exists only so the resolver recognizes gitleaks hooks + # and does NOT emit a "not in registry" warning. The skip_install + # flag prevents double-installing alongside the post-script copy. + - hook_id: gitleaks + repo: https://github.com/zricethezav/gitleaks + install: + type: binary + name: gitleaks + skip_install: true + + # ── actionlint (GitHub Actions linter) ──────────────────────────── + # The upstream hook uses language: golang, so pre-commit CAN compile + # it from source (~2 min). This entry downloads the pre-built binary + # (~3 sec) to keep the authoritative pre-commit check fast. + # actionlint releases use "amd64"/"arm64" instead of the Rust-style + # triple or gitleaks-style "x64". The goarch_override field lets the + # installer substitute the correct arch string for this tool only. + - hook_id: actionlint + repo: https://github.com/rhysd/actionlint + install: + type: binary + name: actionlint + version: "1.7.11" + url_template: "https://github.com/rhysd/actionlint/releases/download/v{version}/actionlint_{version}_linux_{goarch}.tar.gz" + goarch_override: + x86_64: "amd64" + aarch64: "arm64" + checksums: + x86_64: "900919a84f2229bac68ca9cd4103ea297abc35e9689ebb842c6e34a3d1b01b0a" + aarch64: "21bc0dfb57a913fe175298c2a9e906ee630f747cb66d0a934d0d4b69f4ee1235" + binary_name: actionlint + + # ── uv / uvx (Python package manager, needed for ty check) ─────── + - hook_id: ty + repo: local + match_entry: "uvx" + install: + type: binary + name: uv + version: "0.11.14" + url_template: "https://github.com/astral-sh/uv/releases/download/{version}/uv-{triple}.tar.gz" + checksums: + x86_64: "f3b623eb0e6141a7053d571d59a0bdc341e0f238ea8f5f0b4815ddbec9a2a296" + aarch64: "c4958f729e216f1610632574ed927b8cf0af1bd02cb88cb30d948571727aee43" + strip_prefix: "uv-{triple}" + binary_name: uv + extra_binaries: + - uvx + +# Language fallbacks — when a hook is not in the registry above, +# the resolver uses the hook's `language` field to emit warnings: +# language: system → warns that the tool must be pre-installed +# language: golang → warns that Go toolchain is needed +# language: rust → warns that Rust toolchain is needed +# Hooks using python/node/docker_image/script need no registry +# entry — pre-commit handles them natively. diff --git a/internal/scaffold/fullsend-repo/scripts/install-precommit-tools.sh b/internal/scaffold/fullsend-repo/scripts/install-precommit-tools.sh new file mode 100755 index 0000000000..d3534f75e6 --- /dev/null +++ b/internal/scaffold/fullsend-repo/scripts/install-precommit-tools.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# Install pre-commit hook dependencies on the GitHub Actions runner. +# +# Reads a JSON manifest produced by resolve-precommit-tools.py and +# installs the listed tools. Supports four install types: +# binary — download from release URL with SHA256 verification +# apt — install via apt-get +# pip — install via pip +# npm — install via npm -g +# +# Binary downloads use architecture detection (uname -m) and pinned +# checksums for supply-chain safety. Same pattern as post-code.sh and +# images/code/Containerfile. +# +# Usage: +# install-precommit-tools.sh +# +# The manifest is the JSON output of resolve-precommit-tools.py. +# +# Exit codes: +# 0 — all tools installed (or already present) +# 1 — critical failure (missing required tool, checksum mismatch) +set -euo pipefail + +MANIFEST="${1:?Usage: install-precommit-tools.sh }" + +if [ ! -f "${MANIFEST}" ]; then + echo "::error::Manifest not found: ${MANIFEST}" + exit 1 +fi + +INSTALL_DIR="${HOME}/.local/bin" +mkdir -p "${INSTALL_DIR}" +export PATH="${INSTALL_DIR}:${PATH}" + +# Detect architecture once. +ARCH="$(uname -m)" +case "${ARCH}" in + x86_64) + TRIPLE="x86_64-unknown-linux-gnu" + GOARCH="x64" + ;; + aarch64) + TRIPLE="aarch64-unknown-linux-gnu" + GOARCH="arm64" + ;; + *) + echo "::warning::Unsupported architecture: ${ARCH} — skipping binary installs" + TRIPLE="" + GOARCH="" + ;; +esac + +# Print warnings from the resolver (sanitize to prevent GHA command injection). +WARNINGS="$(jq -r '.warnings[]' "${MANIFEST}" 2>/dev/null || true)" +if [ -n "${WARNINGS}" ]; then + while IFS= read -r w; do + w="${w//::/ }" + w="${w//%0A/ }" + w="${w//%0a/ }" + w="${w//%0D/ }" + w="${w//%0d/ }" + echo "::warning::${w}" + done <<< "${WARNINGS}" +fi + +TOOL_COUNT="$(jq '.tools | length' "${MANIFEST}" 2>/dev/null || echo 0)" +if [ "${TOOL_COUNT}" -eq 0 ]; then + echo "No additional pre-commit tools to install" + exit 0 +fi + +echo "Installing ${TOOL_COUNT} pre-commit tool dependency(ies)..." + +# Process each tool entry. +while IFS= read -r entry; do + TYPE="$(echo "${entry}" | jq -r '.type')" + NAME="$(echo "${entry}" | jq -r '.name')" + + # Skip entries marked as handled elsewhere (e.g., gitleaks in post-scripts). + SKIP="$(echo "${entry}" | jq -r '.skip_install // "false"')" + if [ "${SKIP}" = "true" ]; then + echo " ${NAME}: skipped (managed by post-script)" + continue + fi + + case "${TYPE}" in + binary) + VERSION="$(echo "${entry}" | jq -r '.version')" + if command -v "${NAME}" >/dev/null 2>&1; then + INSTALLED_VERSION="$("${NAME}" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" + if [ "${INSTALLED_VERSION}" = "${VERSION}" ]; then + echo " ${NAME}: already available v${VERSION} ($(command -v "${NAME}"))" + continue + fi + echo " ${NAME}: found v${INSTALLED_VERSION:-unknown}, need v${VERSION} — installing pinned version" + fi + + if [ -z "${TRIPLE}" ]; then + echo "::warning::Cannot install ${NAME} — unsupported architecture" + continue + fi + + URL_TEMPLATE="$(echo "${entry}" | jq -r '.url_template')" + BINARY_NAME="$(echo "${entry}" | jq -r '.binary_name // .name')" + STRIP_PREFIX="$(echo "${entry}" | jq -r '.strip_prefix // ""')" + + # Resolve checksum for current architecture. + CHECKSUM="$(echo "${entry}" | jq -r --arg arch "${ARCH}" '.checksums[$arch] // empty')" + if [ -z "${CHECKSUM}" ]; then + echo "::warning::No checksum for ${NAME} on ${ARCH} — skipping" + continue + fi + + # Resolve per-tool goarch override (e.g., actionlint uses "amd64" not "x64"). + TOOL_GOARCH="$(echo "${entry}" | jq -r --arg arch "${ARCH}" '.goarch_override[$arch] // empty')" + if [ -z "${TOOL_GOARCH}" ]; then + TOOL_GOARCH="${GOARCH}" + fi + + # Resolve URL template. + URL="${URL_TEMPLATE}" + URL="${URL//\{version\}/${VERSION}}" + URL="${URL//\{triple\}/${TRIPLE}}" + URL="${URL//\{goarch\}/${TOOL_GOARCH}}" + + echo " ${NAME} v${VERSION}: downloading..." + DL_TMPDIR="$(mktemp -d)" + TARBALL="${DL_TMPDIR}/${NAME}.tar.gz" + + if ! curl -fsSL "${URL}" -o "${TARBALL}"; then + echo "::warning::Failed to download ${NAME} v${VERSION} — skipping" + rm -rf "${DL_TMPDIR}" + continue + fi + if ! echo "${CHECKSUM} ${TARBALL}" | sha256sum -c -; then + echo "::error::Checksum verification failed for ${NAME} v${VERSION}" + rm -rf "${DL_TMPDIR}" + exit 1 + fi + + if ! tar xzf "${TARBALL}" -C "${DL_TMPDIR}"; then + echo "::warning::Failed to extract ${NAME} archive — skipping" + rm -rf "${DL_TMPDIR}" + continue + fi + + # Find and install the binary. + if [ -n "${STRIP_PREFIX}" ]; then + RESOLVED_PREFIX="${STRIP_PREFIX//\{triple\}/${TRIPLE}}" + RESOLVED_PREFIX="${RESOLVED_PREFIX//\{version\}/${VERSION}}" + BIN_PATH="${DL_TMPDIR}/${RESOLVED_PREFIX}/${BINARY_NAME}" + else + BIN_PATH="${DL_TMPDIR}/${BINARY_NAME}" + fi + + if [ ! -f "${BIN_PATH}" ]; then + echo "::warning::Binary not found at expected path: ${BIN_PATH}" + FOUND="$(find "${DL_TMPDIR}" -name "${BINARY_NAME}" -type f | head -1)" + if [ -n "${FOUND}" ]; then + BIN_PATH="${FOUND}" + else + echo "::error::Cannot find ${BINARY_NAME} in archive" + rm -rf "${DL_TMPDIR}" + continue + fi + fi + + if ! mv "${BIN_PATH}" "${INSTALL_DIR}/${BINARY_NAME}"; then + echo "::warning::Failed to install ${NAME} binary — skipping" + rm -rf "${DL_TMPDIR}" + continue + fi + chmod +x "${INSTALL_DIR}/${BINARY_NAME}" + + # Install extra binaries (e.g., uvx alongside uv). + EXTRAS="$(echo "${entry}" | jq -r '.extra_binaries[]? // empty' 2>/dev/null || true)" + if [ -n "${EXTRAS}" ]; then + while IFS= read -r extra; do + EXTRA_PATH="" + if [ -n "${STRIP_PREFIX}" ]; then + EXTRA_PATH="${DL_TMPDIR}/${RESOLVED_PREFIX}/${extra}" + fi + if [ ! -f "${EXTRA_PATH:-}" ]; then + EXTRA_PATH="$(find "${DL_TMPDIR}" -name "${extra}" -type f | head -1)" + fi + if [ -n "${EXTRA_PATH}" ] && [ -f "${EXTRA_PATH}" ]; then + mv "${EXTRA_PATH}" "${INSTALL_DIR}/${extra}" + chmod +x "${INSTALL_DIR}/${extra}" + echo " ${NAME}: installed extra binary: ${extra}" + fi + done <<< "${EXTRAS}" + fi + + rm -rf "${DL_TMPDIR}" + echo " ${NAME} v${VERSION}: installed to ${INSTALL_DIR}/${BINARY_NAME}" + ;; + + apt) + if command -v "${NAME}" >/dev/null 2>&1; then + echo " ${NAME}: already available" + continue + fi + echo " ${NAME}: installing via apt-get..." + sudo apt-get update -qq && sudo apt-get install -y -qq "${NAME}" 2>/dev/null \ + || echo "::warning::Failed to install ${NAME} via apt-get" + ;; + + pip) + VERSION="$(echo "${entry}" | jq -r '.version // ""')" + if [ -z "${VERSION}" ]; then + echo "::warning::No version pinned for pip package ${NAME} — skipping for supply-chain safety" + continue + fi + PKG="${NAME}==${VERSION}" + echo " ${NAME}: installing via pip..." + pip install --quiet --no-deps --break-system-packages "${PKG}" 2>/dev/null \ + || pip3 install --quiet --no-deps --break-system-packages "${PKG}" 2>/dev/null \ + || echo "::warning::Failed to install ${NAME} via pip" + ;; + + npm) + VERSION="$(echo "${entry}" | jq -r '.version // ""')" + if [ -z "${VERSION}" ]; then + echo "::warning::No version pinned for npm package ${NAME} — skipping for supply-chain safety" + continue + fi + NPM_PKG="${NAME}@${VERSION}" + echo " ${NAME}: installing via npm..." + npm install -g --ignore-scripts "${NPM_PKG}" 2>/dev/null \ + || echo "::warning::Failed to install ${NAME} via npm" + ;; + + *) + echo "::warning::Unknown install type '${TYPE}' for ${NAME}" + ;; + esac +done < <(jq -c '.tools[]' "${MANIFEST}") + +echo "Pre-commit tool installation complete" diff --git a/internal/scaffold/fullsend-repo/scripts/post-code.sh b/internal/scaffold/fullsend-repo/scripts/post-code.sh index 09de04297f..16fe9dd8dc 100755 --- a/internal/scaffold/fullsend-repo/scripts/post-code.sh +++ b/internal/scaffold/fullsend-repo/scripts/post-code.sh @@ -11,6 +11,9 @@ # 3. Branch validation — refuse to push main/master # 4. Token isolation — PUSH_TOKEN never enters the sandbox # +# Pre-commit tool deps are auto-installed from .pre-commit-tools.yaml +# before step 2 to ensure hooks have the binaries they need. +# # Protected-path enforcement lives in post-review.sh: the review agent # cannot approve PRs that touch sensitive paths (e.g. .github/, CODEOWNERS, # agents/). The code agent is free to propose changes to any path. @@ -39,11 +42,6 @@ set -euo pipefail # --------------------------------------------------------------------------- GITLEAKS_VERSION="8.30.1" GITLEAKS_SHA256="551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" -LYCHEE_VERSION="0.24.2" -LYCHEE_SHA256_AMD64="1f4e0ef7f6554a6ed33dd7ac144fb2e1bbed98598e7af973042fc5cd43951c9a" -LYCHEE_SHA256_ARM64="91a7bd65685da41b90ccb9bc867a3d649a7818042dae04ff405e55a25bddee4c" -UV_VERSION="0.11.14" -UV_SHA256="f3b623eb0e6141a7053d571d59a0bdc341e0f238ea8f5f0b4815ddbec9a2a296" # --------------------------------------------------------------------------- # Setup @@ -266,45 +264,29 @@ fi echo "Signed-off-by scan passed — no trailers in agent's commit(s)" # --------------------------------------------------------------------------- -# 4. Install lychee (for pre-commit markdown link checking) +# 4. Auto-install pre-commit tool dependencies # --------------------------------------------------------------------------- -if ! command -v lychee >/dev/null 2>&1; then - echo "Installing lychee v${LYCHEE_VERSION}..." - mkdir -p "${HOME}/.local/bin" - case "$(uname -m)" in - x86_64) LY_TRIPLE="x86_64-unknown-linux-gnu"; LY_SHA="${LYCHEE_SHA256_AMD64}" ;; - aarch64) LY_TRIPLE="aarch64-unknown-linux-gnu"; LY_SHA="${LYCHEE_SHA256_ARM64}" ;; - *) echo "::error::Unsupported architecture for lychee: $(uname -m)" >&2; exit 1 ;; - esac - curl -fsSL \ - "https://github.com/lycheeverse/lychee/releases/download/lychee-v${LYCHEE_VERSION}/lychee-${LY_TRIPLE}.tar.gz" \ - -o /tmp/lychee.tar.gz \ - && echo "${LY_SHA} /tmp/lychee.tar.gz" | sha256sum -c - \ - && tar xzf /tmp/lychee.tar.gz -C /tmp \ - && mv "/tmp/lychee-${LY_TRIPLE}/lychee" "${HOME}/.local/bin/" \ - && rm -rf /tmp/lychee.tar.gz "/tmp/lychee-${LY_TRIPLE}" - export PATH="${HOME}/.local/bin:${PATH}" -fi +SCRIPT_DIR_POST="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESOLVE_SCRIPT="${SCRIPT_DIR_POST}/resolve-precommit-tools.py" +INSTALL_SCRIPT="${SCRIPT_DIR_POST}/install-precommit-tools.sh" -# --------------------------------------------------------------------------- -# 5. Install uv and uvx (for pre-commit Python tooling) -# --------------------------------------------------------------------------- -if ! command -v uvx >/dev/null 2>&1; then - echo "Installing uv v${UV_VERSION} (includes uvx)..." - mkdir -p "${HOME}/.local/bin" - curl -fsSL \ - "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ - -o /tmp/uv.tar.gz \ - && echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \ - && tar xzf /tmp/uv.tar.gz -C /tmp \ - && mv /tmp/uv-x86_64-unknown-linux-gnu/uv "${HOME}/.local/bin/" \ - && mv /tmp/uv-x86_64-unknown-linux-gnu/uvx "${HOME}/.local/bin/" \ - && rm -rf /tmp/uv.tar.gz /tmp/uv-x86_64-unknown-linux-gnu - export PATH="${HOME}/.local/bin:${PATH}" +if [ -f .pre-commit-config.yaml ] \ + && [ -f "${RESOLVE_SCRIPT}" ] \ + && [ -f "${INSTALL_SCRIPT}" ]; then + MANIFEST="$(mktemp)" + if python3 "${RESOLVE_SCRIPT}" "." > "${MANIFEST}"; then + if [ -s "${MANIFEST}" ] && jq -e '.tools | length > 0' "${MANIFEST}" >/dev/null 2>&1; then + bash "${INSTALL_SCRIPT}" "${MANIFEST}" + fi + else + echo "::warning::Pre-commit tool resolution failed — continuing without auto-install" + fi + rm -f "${MANIFEST}" fi +export PATH="${HOME}/.local/bin:${PATH}" # --------------------------------------------------------------------------- -# 6. Authoritative pre-commit check +# 5. Authoritative pre-commit check # --------------------------------------------------------------------------- if [ -f .pre-commit-config.yaml ]; then echo "Running authoritative pre-commit on agent's changed files..." @@ -336,7 +318,7 @@ else fi # --------------------------------------------------------------------------- -# 7. Push branch +# 6. Push branch # --------------------------------------------------------------------------- git remote set-url origin \ "https://x-access-token:${PUSH_TOKEN}@github.com/${REPO_FULL_NAME}.git" diff --git a/internal/scaffold/fullsend-repo/scripts/post-fix.sh b/internal/scaffold/fullsend-repo/scripts/post-fix.sh index dd06332a3f..18b64a9b98 100644 --- a/internal/scaffold/fullsend-repo/scripts/post-fix.sh +++ b/internal/scaffold/fullsend-repo/scripts/post-fix.sh @@ -7,6 +7,7 @@ # # Security layers (defense-in-depth): # - Authoritative secret scan — final gate before any push +# - Auto-install pre-commit tool deps (from .pre-commit-tools.yaml) # - Authoritative pre-commit — run repo hooks on changed files # - Branch validation — refuse to push main/master # - Token isolation — PUSH_TOKEN never enters the sandbox @@ -18,13 +19,12 @@ # Steps: # 0. Check for agent commits # 1. Authoritative secret scan -# 2. Install lychee -# 3. Install uv and uvx -# 4. Authoritative pre-commit check -# 5. Push branch -# 6. Process structured output -# 7. Iteration-cap warning label -# 8. Summary +# 2. Auto-install pre-commit tool deps (from .pre-commit-tools.yaml) +# 3. Authoritative pre-commit check +# 4. Push branch +# 5. Process structured output +# 6. Iteration-cap warning label +# 7. Summary # # After pushing, this script processes fix-result.json to: # - Post a summary comment on the PR documenting fixes and disagreements @@ -59,11 +59,6 @@ is_bot_user() { # --------------------------------------------------------------------------- GITLEAKS_VERSION="8.30.1" GITLEAKS_SHA256="551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" -LYCHEE_VERSION="0.24.2" -LYCHEE_SHA256_AMD64="1f4e0ef7f6554a6ed33dd7ac144fb2e1bbed98598e7af973042fc5cd43951c9a" -LYCHEE_SHA256_ARM64="91a7bd65685da41b90ccb9bc867a3d649a7818042dae04ff405e55a25bddee4c" -UV_VERSION="0.11.14" -UV_SHA256="f3b623eb0e6141a7053d571d59a0bdc341e0f238ea8f5f0b4815ddbec9a2a296" # --------------------------------------------------------------------------- # Setup @@ -181,45 +176,29 @@ if [ "${NO_PUSH}" = "false" ]; then fi # --------------------------------------------------------------------------- -# 2. Install lychee (for pre-commit markdown link checking) +# 2. Auto-install pre-commit tool dependencies # --------------------------------------------------------------------------- -if ! command -v lychee >/dev/null 2>&1; then - echo "Installing lychee v${LYCHEE_VERSION}..." - mkdir -p "${HOME}/.local/bin" - case "$(uname -m)" in - x86_64) LY_TRIPLE="x86_64-unknown-linux-gnu"; LY_SHA="${LYCHEE_SHA256_AMD64}" ;; - aarch64) LY_TRIPLE="aarch64-unknown-linux-gnu"; LY_SHA="${LYCHEE_SHA256_ARM64}" ;; - *) echo "::error::Unsupported architecture for lychee: $(uname -m)" >&2; exit 1 ;; - esac - curl -fsSL \ - "https://github.com/lycheeverse/lychee/releases/download/lychee-v${LYCHEE_VERSION}/lychee-${LY_TRIPLE}.tar.gz" \ - -o /tmp/lychee.tar.gz \ - && echo "${LY_SHA} /tmp/lychee.tar.gz" | sha256sum -c - \ - && tar xzf /tmp/lychee.tar.gz -C /tmp \ - && mv "/tmp/lychee-${LY_TRIPLE}/lychee" "${HOME}/.local/bin/" \ - && rm -rf /tmp/lychee.tar.gz "/tmp/lychee-${LY_TRIPLE}" - export PATH="${HOME}/.local/bin:${PATH}" -fi - -# --------------------------------------------------------------------------- -# 3. Install uv and uvx (for pre-commit Python tooling) -# --------------------------------------------------------------------------- -if ! command -v uvx >/dev/null 2>&1; then - echo "Installing uv v${UV_VERSION} (includes uvx)..." - mkdir -p "${HOME}/.local/bin" - curl -fsSL \ - "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ - -o /tmp/uv.tar.gz \ - && echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \ - && tar xzf /tmp/uv.tar.gz -C /tmp \ - && mv /tmp/uv-x86_64-unknown-linux-gnu/uv "${HOME}/.local/bin/" \ - && mv /tmp/uv-x86_64-unknown-linux-gnu/uvx "${HOME}/.local/bin/" \ - && rm -rf /tmp/uv.tar.gz /tmp/uv-x86_64-unknown-linux-gnu - export PATH="${HOME}/.local/bin:${PATH}" +SCRIPT_DIR_POST="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESOLVE_SCRIPT="${SCRIPT_DIR_POST}/resolve-precommit-tools.py" +INSTALL_SCRIPT="${SCRIPT_DIR_POST}/install-precommit-tools.sh" + +if [ -f .pre-commit-config.yaml ] \ + && [ -f "${RESOLVE_SCRIPT}" ] \ + && [ -f "${INSTALL_SCRIPT}" ]; then + MANIFEST="$(mktemp)" + if python3 "${RESOLVE_SCRIPT}" "." > "${MANIFEST}"; then + if [ -s "${MANIFEST}" ] && jq -e '.tools | length > 0' "${MANIFEST}" >/dev/null 2>&1; then + bash "${INSTALL_SCRIPT}" "${MANIFEST}" + fi + else + echo "::warning::Pre-commit tool resolution failed — continuing without auto-install" + fi + rm -f "${MANIFEST}" fi +export PATH="${HOME}/.local/bin:${PATH}" # --------------------------------------------------------------------------- -# 4. Authoritative pre-commit check (only if pushing) +# 3. Authoritative pre-commit check (only if pushing) # --------------------------------------------------------------------------- if [ "${NO_PUSH}" = "false" ] && [ -f .pre-commit-config.yaml ]; then echo "Running authoritative pre-commit on agent's changed files..." @@ -245,7 +224,7 @@ if [ "${NO_PUSH}" = "false" ] && [ -f .pre-commit-config.yaml ]; then fi # --------------------------------------------------------------------------- -# 5. Push branch (only if we have commits) +# 4. Push branch (only if we have commits) # --------------------------------------------------------------------------- if [ "${NO_PUSH}" = "false" ]; then git remote set-url origin \ @@ -275,7 +254,7 @@ if [ "${NO_PUSH}" = "false" ]; then fi # --------------------------------------------------------------------------- -# 6. Process structured output (fix-result.json) +# 5. Process structured output (fix-result.json) # --------------------------------------------------------------------------- export GH_TOKEN="${PUSH_TOKEN}" @@ -328,7 +307,7 @@ else fi # --------------------------------------------------------------------------- -# 7. Iteration-cap warning label +# 6. Iteration-cap warning label # --------------------------------------------------------------------------- ITERATION="${FIX_ITERATION:-1}" BOT_CAP="${ITERATION_CAP:-5}" @@ -347,7 +326,7 @@ if [ "${ITERATION}" -ge "${WARN_THRESHOLD}" ] && is_bot_user "${TRIGGER_SOURCE}" fi # --------------------------------------------------------------------------- -# 8. Summary +# 7. Summary # --------------------------------------------------------------------------- echo "" echo "Fix post-script complete:" diff --git a/internal/scaffold/fullsend-repo/scripts/pre-code.sh b/internal/scaffold/fullsend-repo/scripts/pre-code.sh index 724156964b..e60273df88 100755 --- a/internal/scaffold/fullsend-repo/scripts/pre-code.sh +++ b/internal/scaffold/fullsend-repo/scripts/pre-code.sh @@ -121,3 +121,30 @@ fi echo "No existing human PRs found — proceeding with code agent" echo "skipped=false" >> "${GITHUB_OUTPUT:-/dev/null}" + +# --------------------------------------------------------------------------- +# Auto-detect and install pre-commit tool dependencies +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TARGET_REPO="${REPO_DIR:-${GITHUB_WORKSPACE:-}/target-repo}" +RESOLVE_SCRIPT="${SCRIPT_DIR}/resolve-precommit-tools.py" +INSTALL_SCRIPT="${SCRIPT_DIR}/install-precommit-tools.sh" + +if [ -f "${TARGET_REPO}/.pre-commit-config.yaml" ] \ + && [ -f "${RESOLVE_SCRIPT}" ] \ + && [ -f "${INSTALL_SCRIPT}" ]; then + echo "Resolving pre-commit tool dependencies..." + MANIFEST="$(mktemp)" + if python3 "${RESOLVE_SCRIPT}" "${TARGET_REPO}" > "${MANIFEST}"; then + if [ -s "${MANIFEST}" ] && jq -e '.tools | length > 0' "${MANIFEST}" >/dev/null 2>&1; then + bash "${INSTALL_SCRIPT}" "${MANIFEST}" + else + echo "No additional pre-commit tools needed" + fi + else + echo "::warning::Pre-commit tool resolution failed — continuing without auto-install" + fi + rm -f "${MANIFEST}" +fi +export PATH="${HOME}/.local/bin:${PATH}" +echo "${HOME}/.local/bin" >> "${GITHUB_PATH:-/dev/null}" diff --git a/internal/scaffold/fullsend-repo/scripts/pre-fix.sh b/internal/scaffold/fullsend-repo/scripts/pre-fix.sh index 1d233bc65d..b2cd70cb9b 100644 --- a/internal/scaffold/fullsend-repo/scripts/pre-fix.sh +++ b/internal/scaffold/fullsend-repo/scripts/pre-fix.sh @@ -99,3 +99,32 @@ if ! is_bot_user "${TRIGGER_SOURCE}" && [[ -n "${HUMAN_INSTRUCTION:-}" ]]; then INSTR_PREVIEW="${HUMAN_INSTRUCTION:0:200}" echo " HUMAN_INSTRUCTION=${INSTR_PREVIEW}..." fi + +# --------------------------------------------------------------------------- +# Auto-detect and install pre-commit tool dependencies +# --------------------------------------------------------------------------- +# Ensures tools required by the target repo's pre-commit hooks are +# available on the runner for the authoritative post-script check. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TARGET_REPO="${REPO_DIR:-${GITHUB_WORKSPACE:-}/target-repo}" +RESOLVE_SCRIPT="${SCRIPT_DIR}/resolve-precommit-tools.py" +INSTALL_SCRIPT="${SCRIPT_DIR}/install-precommit-tools.sh" + +if [ -f "${TARGET_REPO}/.pre-commit-config.yaml" ] \ + && [ -f "${RESOLVE_SCRIPT}" ] \ + && [ -f "${INSTALL_SCRIPT}" ]; then + echo "Resolving pre-commit tool dependencies..." + MANIFEST="$(mktemp)" + if python3 "${RESOLVE_SCRIPT}" "${TARGET_REPO}" > "${MANIFEST}"; then + if [ -s "${MANIFEST}" ] && jq -e '.tools | length > 0' "${MANIFEST}" >/dev/null 2>&1; then + bash "${INSTALL_SCRIPT}" "${MANIFEST}" + else + echo "No additional pre-commit tools needed" + fi + else + echo "::warning::Pre-commit tool resolution failed — continuing without auto-install" + fi + rm -f "${MANIFEST}" +fi +export PATH="${HOME}/.local/bin:${PATH}" +echo "${HOME}/.local/bin" >> "${GITHUB_PATH:-/dev/null}" diff --git a/internal/scaffold/fullsend-repo/scripts/resolve-precommit-tools.py b/internal/scaffold/fullsend-repo/scripts/resolve-precommit-tools.py new file mode 100755 index 0000000000..13b4236477 --- /dev/null +++ b/internal/scaffold/fullsend-repo/scripts/resolve-precommit-tools.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Resolve pre-commit hook tool dependencies for a target repository. + +Reads a target repo's .pre-commit-config.yaml, matches hooks against +the known-tools registry (.pre-commit-tools.yaml), and outputs a JSON +manifest to stdout. + +Usage: + resolve-precommit-tools.py +""" + +import json +import os +import subprocess +import sys + +try: + import yaml +except ImportError: + try: + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "--quiet", + "--no-deps", + "--break-system-packages", + "pyyaml==6.0.2", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + import yaml + except Exception: + print('{"tools":[],"warnings":["failed to install pyyaml — cannot resolve hooks"]}') + sys.exit(0) + + +def resolve(precommit_path: str, registry_path: str) -> dict: + try: + with open(precommit_path) as f: + precommit = yaml.safe_load(f) + except (yaml.YAMLError, OSError) as exc: + return {"tools": [], "warnings": [f"failed to parse .pre-commit-config.yaml: {exc}"]} + try: + with open(registry_path) as f: + registry = yaml.safe_load(f) + except (yaml.YAMLError, OSError) as exc: + return {"tools": [], "warnings": [f"failed to parse tools registry: {exc}"]} + + if not isinstance(precommit, dict) or "repos" not in precommit: + return {"tools": [], "warnings": ["empty or invalid .pre-commit-config.yaml"]} + + repos = precommit["repos"] + if not isinstance(repos, list): + return {"tools": [], "warnings": ["repos field is not a list in .pre-commit-config.yaml"]} + + if not isinstance(registry, dict) or "tools" not in registry: + return {"tools": [], "warnings": ["empty or invalid tools registry"]} + + registry_tools = registry.get("tools") or [] + + repo_hook_map = {} + entry_match_map = {} + for tool in registry_tools: + if not isinstance(tool, dict) or "hook_id" not in tool: + continue + key = (tool.get("repo", ""), tool["hook_id"]) + repo_hook_map[key] = tool + if "match_entry" in tool: + entry_match_map[tool["match_entry"]] = tool + + resolved = [] + seen_names: set[str] = set() + warnings = [] + + for repo_entry in repos: + if not isinstance(repo_entry, dict): + continue + repo_url = repo_entry.get("repo", "") + for hook in repo_entry.get("hooks") or []: + if not isinstance(hook, dict): + continue + hook_id = hook.get("id", "") + entry = hook.get("entry", "") + language = hook.get("language", "") + + tool = repo_hook_map.get((repo_url, hook_id)) + + if tool is None and repo_url == "local": + parts = entry.split() + entry_cmd = parts[0] if parts else "" + for match_str, match_tool in entry_match_map.items(): + if entry_cmd == match_str: + tool = match_tool + break + + if tool is not None: + install = tool.get("install") or {} + name = install.get("name", "") + if name and name not in seen_names: + seen_names.add(name) + resolved.append(install) + else: + if language == "system": + parts = entry.split() + cmd = parts[0] if parts else hook_id + warnings.append( + f"hook '{hook_id}' uses language:system " + f"(command: {cmd}) — not in registry, " + f"must be pre-installed on runner" + ) + elif language in ("golang",): + warnings.append( + f"hook '{hook_id}' requires Go toolchain (language: {language})" + ) + elif language in ("rust",): + warnings.append( + f"hook '{hook_id}' requires Rust toolchain (language: {language})" + ) + + return {"tools": resolved, "warnings": warnings} + + +def main(): + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + target_repo = sys.argv[1] + precommit_config = os.path.join(target_repo, ".pre-commit-config.yaml") + + if not os.path.isfile(precommit_config): + print('{"tools":[],"warnings":["no .pre-commit-config.yaml found"]}') + sys.exit(0) + + script_dir = os.path.dirname(os.path.abspath(__file__)) + registry = os.path.join(script_dir, ".pre-commit-tools.yaml") + + if not os.path.isfile(registry): + print('{"tools":[],"warnings":["tools registry not found"]}') + sys.exit(0) + + result = resolve(precommit_config, registry) + print(json.dumps(result)) + + +if __name__ == "__main__": + main() diff --git a/internal/scaffold/scaffold.go b/internal/scaffold/scaffold.go index dbd44f6438..5d6ceb3ba1 100644 --- a/internal/scaffold/scaffold.go +++ b/internal/scaffold/scaffold.go @@ -42,6 +42,8 @@ var executableFiles = map[string]struct{}{ "scripts/fullsend-check-output": {}, "scripts/validate-output-schema-test.sh": {}, "scripts/validate-source-repo.sh": {}, + "scripts/install-precommit-tools.sh": {}, + "scripts/resolve-precommit-tools.py": {}, } // FileMode returns the Git tree mode for a scaffold file.