diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml new file mode 100644 index 000000000..7e1201a80 --- /dev/null +++ b/.github/workflows/hourly-product-development.yml @@ -0,0 +1,895 @@ +name: Hourly NVIDIA NIM Product Development + +on: + workflow_dispatch: + inputs: + dry_run: + description: Evaluate the gate and task contract without running OpenCode + required: false + default: false + type: boolean + schedule: + - cron: "47 * * * *" + +concurrency: + group: hourly-nim-product-development-${{ github.repository }} + cancel-in-progress: false + +permissions: + contents: read + +env: + DEFAULT_BRANCH: main + OPENCODE_VERSION: "1.17.13" + OPENCODE_SHA256: 157afa289d1a8d9372de0ce19ac726119b937a1f6b201808d46f06e4e59bb348 + OPENCODE_MODEL_CANDIDATES: >- + nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 + nvidia-nim/nvidia/nemotron-3-super-120b-a12b + nvidia-nim/deepseek-ai/deepseek-v4-pro + # Three candidate budgets and two inter-candidate cleanup reinstalls fit in 55 minutes. + OPENCODE_RUN_TIMEOUT_SECONDS: "900" + OPENCODE_KILL_GRACE_SECONDS: "30" + DEPENDENCY_REINSTALL_TIMEOUT_SECONDS: "60" + DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS: "10" + MAX_CHANGED_FILES: "40" + MAX_DIFF_BYTES: "500000" + MAX_PR_TITLE_BYTES: "120" + MAX_PR_BODY_BYTES: "20000" + +jobs: + propose_product_increment: + if: github.repository == 'ContextualWisdomLab/noema' + runs-on: ubuntu-latest + timeout-minutes: 55 + permissions: + contents: read + pull-requests: read + outputs: + dispatch: ${{ steps.gate.outputs.dispatch }} + reason: ${{ steps.gate.outputs.reason }} + has_changes: ${{ steps.proposal.outputs.has_changes }} + base_sha: ${{ steps.base.outputs.base_sha }} + patch_sha256: ${{ steps.proposal.outputs.patch_sha256 }} + changed_files: ${{ steps.proposal.outputs.changed_files }} + diff_bytes: ${{ steps.proposal.outputs.diff_bytes }} + artifact_id: ${{ steps.upload_proposal.outputs.artifact-id }} + artifact_digest: ${{ steps.upload_proposal.outputs.artifact-digest }} + env: + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }} + steps: + - name: Enforce zero-open-PR single-flight gate + id: gate + shell: bash + env: + GH_TOKEN: ${{ github.token }} + NIM_CONFIGURED: ${{ secrets.NVIDIA_NIM_API_KEY != '' }} + MAINTAINER_APP_CLIENT_ID_CONFIGURED: ${{ vars.NOEMA_MAINTAINER_APP_CLIENT_ID != '' }} + MAINTAINER_APP_PRIVATE_KEY_CONFIGURED: ${{ secrets.NOEMA_MAINTAINER_APP_PRIVATE_KEY != '' }} + run: | + set -euo pipefail + + if ! open_prs="$( + gh pr list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --limit 1 \ + --json number,url + )"; then + { + echo "dispatch=false" + echo "reason=pull_request_inventory_unavailable" + } >>"$GITHUB_OUTPUT" + echo "Pull-request inventory was unavailable; development failed closed." \ + >>"$GITHUB_STEP_SUMMARY" + exit 0 + fi + + if [ "$(jq 'length' <<<"$open_prs")" -gt 0 ]; then + { + echo "dispatch=false" + echo "reason=open_pull_request" + } >>"$GITHUB_OUTPUT" + echo "An open pull request exists; exact-head PR governance owns this hour." \ + >>"$GITHUB_STEP_SUMMARY" + exit 0 + fi + + if [ "$NIM_CONFIGURED" != "true" ] && [ "$DRY_RUN" != "true" ]; then + { + echo "dispatch=false" + echo "reason=nim_api_key_unavailable" + } >>"$GITHUB_OUTPUT" + echo "Autonomous development is disabled because the dedicated NVIDIA NIM secret is unavailable." \ + >>"$GITHUB_STEP_SUMMARY" + exit 0 + fi + + if { [ "$MAINTAINER_APP_CLIENT_ID_CONFIGURED" != "true" ] \ + || [ "$MAINTAINER_APP_PRIVATE_KEY_CONFIGURED" != "true" ]; } \ + && [ "$DRY_RUN" != "true" ]; then + { + echo "dispatch=false" + echo "reason=maintainer_app_unavailable" + } >>"$GITHUB_OUTPUT" + echo "Autonomous development is disabled because the repository-scoped Maintainer App cannot publish a proposal." \ + >>"$GITHUB_STEP_SUMMARY" + exit 0 + fi + + { + echo "dispatch=true" + if [ "$NIM_CONFIGURED" = "true" ] \ + && [ "$MAINTAINER_APP_CLIENT_ID_CONFIGURED" = "true" ] \ + && [ "$MAINTAINER_APP_PRIVATE_KEY_CONFIGURED" = "true" ]; then + echo "reason=ready" + else + echo "reason=ready_dry_run_without_credentials" + fi + } >>"$GITHUB_OUTPUT" + + - name: Prepare bounded commercial-quality task + if: steps.gate.outputs.dispatch == 'true' + shell: bash + run: | + set -euo pipefail + cat >"$RUNNER_TEMP/noema-agent-prompt.md" <<'PROMPT' + Continue commercial-quality product development for ContextualWisdomLab/noema. + + Begin by reading AGENTS.md, README.md, CHANGELOG.md, package.json, + the architecture, threat model, operations and doctoring documents, open issues, + recent merged pull requests, current source, tests, workflows, release evidence, + and buyer due-diligence material. Select the single highest-value buyer-visible + product, reliability, interoperability, security, evaluation, accessibility, + supportability, or operations gap that can be completed as exactly one bounded + pull request. Do not create another repository. + + Keep Noema independently deployable and preserve its modular MSA role with + ContextualWisdomLab/.github, naruon, contextual-orchestrator, and other CWL + services. Keep interfaces explicit and replaceable. Use or improve + contextual-orchestrator for every new product-runtime LLM path. Do not alter + the existing reviewer-agent credential names, trust boundary, or provider route. + + Work test-first: add one realistic executable regression or product contract, + run it, and record the expected failure before implementation. Then make the + smallest coherent change that closes the selected gap. Tests must exercise + realistic Noema traffic, operational, security, provenance, or buyer workflows, + including adversarial and failure cases where applicable. Maintain 100% production + statement and branch coverage plus 100% production function coverage. When reviewer + Python changes, maintain 100% line and branch coverage and 100% docstring coverage. + Public modules, classes, functions, methods, properties, inputs, outputs, errors, + and trust boundaries must be understandable to a beginner without reverse-engineering. + + Where behavior is ambiguous, use current authoritative international standards, + primary official documentation, or peer-reviewed research. Record sources and + rationale in docs/doctoring with APA 7 references, and distinguish measured facts, + source-supported claims, project decisions, assumptions, and inference. Use Figma + or Product Design only when a real user-facing visual flow materially benefits. + + Preserve standalone use and modular MSA integration. If a database boundary is + introduced, every new object must use a descriptive two-word-or-longer snake_case + name. Do not add skipped, ignored, quarantined, or weakened release-gate tests. + Run focused tests and npm run release:verify. Update CHANGELOG.md and all affected + product, architecture, security, operations, API, support, and buyer documentation. + + Follow Semantic Versioning. Change the package version only when the integrated + repository is genuinely release-ready and all immutable release, deployment, + production KPI, and governance requirements can be satisfied. Otherwise leave the + version unchanged and explain why in the pull request. + + Do not merge. Do not publish. Do not release. Do not deploy. Do not fabricate + production KPI, customer, paid-pilot, revenue, transfer, attestation, or acquisition + evidence. Do not weaken branch protection, independent review, exact-head Checks, + security scanning, coverage, docstring, sandbox, credential, or fail-closed gates. + Do not make unrelated refactors. + + Leave the working tree with one bounded increment and write PR_MESSAGE.md at the + repository root. Put the pull-request title on the first line, then a body containing + the product gap, design, RED-to-GREEN evidence, complete verification commands and + results, APA 7 sources where applicable, version decision, and explicit residual risk. + A separate uncredentialed verifier will execute the proposal on a fresh runner. + A third credential-bearing publisher will reconstruct the same immutable artifact + without executing proposed code. Existing review → repair → exact-head Checks → merge + governance retains every final decision. + PROMPT + + - name: Record dry-run task contract + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN == 'true' + shell: bash + run: | + set -euo pipefail + { + echo "Dry run: the zero-open-PR gate permits one bounded OpenCode proposal." + echo + cat "$RUNNER_TEMP/noema-agent-prompt.md" + } >>"$GITHUB_STEP_SUMMARY" + + - name: Check out trusted default-branch source without persisted credentials + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN != 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: main + persist-credentials: false + + - name: Capture trusted default-branch head + id: base + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN != 'true' + shell: bash + run: | + set -euo pipefail + base_sha=$(git rev-parse HEAD) + if ! [[ "$base_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::The checked-out default-branch head is not a full commit SHA." + exit 1 + fi + echo "base_sha=$base_sha" >>"$GITHUB_OUTPUT" + + - name: Install repository dependencies without lifecycle scripts + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN != 'true' + shell: bash + run: npm ci --ignore-scripts + + - name: Install checksum-pinned OpenCode CLI + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN != 'true' + shell: bash + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/opencode-linux-x64.tar.gz" + install_dir="${HOME}/.opencode/bin" + mkdir -p "$install_dir" + curl --fail --silent --show-error --location \ + --proto '=https' \ + --connect-timeout 10 \ + --max-time 120 \ + --output "$archive" \ + "https://github.com/anomalyco/opencode/releases/download/v${OPENCODE_VERSION}/opencode-linux-x64.tar.gz" + printf '%s %s\n' "$OPENCODE_SHA256" "$archive" | sha256sum -c - + tar -xzf "$archive" -C "$RUNNER_TEMP" + install -m 0755 "${RUNNER_TEMP}/opencode" "${install_dir}/opencode" + "${install_dir}/opencode" --version + echo "$install_dir" >>"$GITHUB_PATH" + + - name: Configure OpenCode for NVIDIA NIM only + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN != 'true' + shell: bash + run: | + set -euo pipefail + cat >"$RUNNER_TEMP/opencode.json" <<'CONFIG' + { + "$schema": "https://opencode.ai/config.json", + "share": "disabled", + "autoupdate": false, + "lsp": false, + "mcp": {}, + "enabled_providers": ["nvidia-nim"], + "model": "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", + "small_model": "nvidia-nim/meta/llama-3.3-70b-instruct", + "permission": { + "*": "allow", + "external_directory": "deny", + "task": "deny", + "question": "deny", + "webfetch": "deny", + "websearch": "deny", + "bash": { + "*": "allow", + "curl *": "deny", + "wget *": "deny", + "ssh *": "deny", + "scp *": "deny", + "nc *": "deny", + "ncat *": "deny", + "socat *": "deny", + "git clone *": "deny", + "git fetch *": "deny", + "git pull *": "deny", + "git commit *": "deny", + "git push *": "deny", + "git tag *": "deny", + "git remote *": "deny", + "gh *": "deny" + } + }, + "provider": { + "nvidia-nim": { + "npm": "@ai-sdk/openai-compatible", + "name": "NVIDIA NIM", + "options": { + "baseURL": "https://integrate.api.nvidia.com/v1", + "apiKey": "{env:NVIDIA_API_KEY}" + }, + "models": { + "nvidia/llama-3.3-nemotron-super-49b-v1.5": { + "name": "NVIDIA Llama 3.3 Nemotron Super 49B v1.5", + "tool_call": true, + "limit": {"context": 131072, "output": 8192} + }, + "nvidia/nemotron-3-super-120b-a12b": { + "name": "NVIDIA Nemotron 3 Super 120B", + "tool_call": true, + "limit": {"context": 131072, "output": 8192} + }, + "deepseek-ai/deepseek-v4-pro": { + "name": "DeepSeek V4 Pro through NVIDIA NIM", + "tool_call": true, + "limit": {"context": 131072, "output": 8192} + }, + "meta/llama-3.3-70b-instruct": { + "name": "Meta Llama 3.3 70B Instruct through NVIDIA NIM", + "tool_call": true, + "limit": {"context": 131072, "output": 8192} + } + } + } + } + } + CONFIG + chmod 0400 "$RUNNER_TEMP/opencode.json" + + - name: Run bounded NVIDIA NIM model fallback + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN != 'true' + shell: bash + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + OPENCODE_DISABLE_AUTOUPDATE: "true" + OPENCODE_CONFIG: ${{ runner.temp }}/opencode.json + XDG_CONFIG_HOME: ${{ runner.temp }}/opencode-config + XDG_CACHE_HOME: ${{ runner.temp }}/opencode-cache + run: | + set -euo pipefail + prompt="$(cat "$RUNNER_TEMP/noema-agent-prompt.md")" + status=1 + cleanup_failed=false + read -r -a model_candidates <<<"$OPENCODE_MODEL_CANDIDATES" + candidate_count=${#model_candidates[@]} + + for ((candidate_index = 0; candidate_index < candidate_count; candidate_index++)); do + model="${model_candidates[$candidate_index]}" + echo "::group::OpenCode candidate $model" + if timeout --kill-after="${OPENCODE_KILL_GRACE_SECONDS}s" "${OPENCODE_RUN_TIMEOUT_SECONDS}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN \ + -u REPOSITORY_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + -u ACTIONS_RUNTIME_URL \ + -u ACTIONS_RESULTS_URL \ + -u ACTIONS_CACHE_URL \ + -u GITHUB_ENV \ + -u GITHUB_OUTPUT \ + -u GITHUB_PATH \ + -u GITHUB_STATE \ + -u GITHUB_STEP_SUMMARY \ + opencode run "$prompt" --agent build --model "$model"; then + status=0 + echo "::endgroup::" + echo "OpenCode completed one proposal with \`$model\`." \ + >>"$GITHUB_STEP_SUMMARY" + break + fi + + echo "::endgroup::" + if [ "$candidate_index" -eq $((candidate_count - 1)) ]; then + break + fi + + echo "::warning::Candidate $model failed; discarding its partial working tree before the next candidate." + git -C "$GITHUB_WORKSPACE" reset --hard HEAD + git -C "$GITHUB_WORKSPACE" clean -fdx + if ! timeout --kill-after="${DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS}s" "${DEPENDENCY_REINSTALL_TIMEOUT_SECONDS}s" npm ci --ignore-scripts; then + echo "::error::Candidate $model cleanup dependency reinstall failed or timed out; no later candidate will run." + cleanup_failed=true + break + fi + done + + if [ "$status" -ne 0 ]; then + if [ "$cleanup_failed" = "true" ]; then + echo "::error::Every NVIDIA NIM candidate failed or bounded cleanup failed closed; no branch or pull request was created." + else + echo "::error::Every NVIDIA NIM candidate failed; no branch or pull request was created." + fi + exit 1 + fi + + - name: Verify, bound, and export the uncredentialed proposal + id: proposal + if: steps.gate.outputs.dispatch == 'true' && env.DRY_RUN != 'true' + shell: bash + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE" + + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >>"$GITHUB_OUTPUT" + echo "OpenCode produced no working-tree change; this hour is a no-op." \ + >>"$GITHUB_STEP_SUMMARY" + exit 0 + fi + + npm run release:verify + git add -A + git diff --cached --check + + if git diff --cached --raw | awk ' + /^:/ { + old_mode = substr($1, 2) + new_mode = $2 + if (old_mode ~ /^(120000|160000)$/ || new_mode ~ /^(120000|160000)$/) { + found = 1 + } + } + END { exit(found ? 0 : 1) } + '; then + echo "::error::A proposed symlink or gitlink (mode 120000|160000) is outside the bounded agent contract." + exit 1 + fi + + changed_files="$(git diff --cached --name-only -z | node -e ' + const chunks = []; + process.stdin.on("data", (chunk) => chunks.push(chunk)); + process.stdin.on("end", () => { + const names = Buffer.concat(chunks).toString("utf8").split("\0").filter(Boolean); + process.stdout.write(String(names.length)); + }); + ')" + + proposal_dir="$RUNNER_TEMP/noema-product-proposal" + rm -rf "$proposal_dir" + install -d -m 0700 "$proposal_dir" + git diff --cached --binary --full-index >"$proposal_dir/proposal.patch" + chmod 0600 "$proposal_dir/proposal.patch" + diff_bytes="$(wc -c <"$proposal_dir/proposal.patch" | tr -d '[:space:]')" + + if [ "$changed_files" -gt "$MAX_CHANGED_FILES" ]; then + echo "::error::Proposal changes $changed_files files; limit is $MAX_CHANGED_FILES." + exit 1 + fi + if [ "$diff_bytes" -gt "$MAX_DIFF_BYTES" ]; then + echo "::error::Proposal patch is $diff_bytes bytes; limit is $MAX_DIFF_BYTES." + exit 1 + fi + + patch_sha256="$(sha256sum "$proposal_dir/proposal.patch" | awk '{print $1}')" + { + echo "has_changes=true" + echo "patch_sha256=$patch_sha256" + echo "changed_files=$changed_files" + echo "diff_bytes=$diff_bytes" + } >>"$GITHUB_OUTPUT" + + - name: Upload bounded proposal patch + id: upload_proposal + if: steps.proposal.outputs.has_changes == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: noema-product-proposal-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/noema-product-proposal/proposal.patch + if-no-files-found: error + retention-days: 1 + compression-level: 0 + overwrite: false + + package_product_increment: + if: >- + github.repository == 'ContextualWisdomLab/noema' + && needs.propose_product_increment.outputs.has_changes == 'true' + needs: propose_product_increment + runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + actions: read + contents: read + pull-requests: read + steps: + - name: Check out the exact proposal base without persisted credentials + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.propose_product_increment.outputs.base_sha }} + fetch-depth: 1 + persist-credentials: false + + - name: Download immutable proposal artifact for uncredentialed verification + id: download_verified_proposal + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + artifact-ids: ${{ needs.propose_product_increment.outputs.artifact_id }} + path: ${{ runner.temp }}/noema-product-proposal + + - name: Verify immutable artifact identity and apply proposal + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + patch="$RUNNER_TEMP/noema-product-proposal/proposal.patch" + artifact_id="${{ needs.propose_product_increment.outputs.artifact_id }}" + artifact_name="noema-product-proposal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + expected_sha="${{ needs.propose_product_increment.outputs.patch_sha256 }}" + expected_files="${{ needs.propose_product_increment.outputs.changed_files }}" + expected_bytes="${{ needs.propose_product_increment.outputs.diff_bytes }}" + expected_artifact_digest="${{ needs.propose_product_increment.outputs.artifact_digest }}" + + if [ ! -f "$patch" ] || [ -L "$patch" ]; then + echo "::error::Downloaded proposal.patch is not a regular file." + exit 1 + fi + if ! [[ "$artifact_id" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Proposal artifact identity is malformed." + exit 1 + fi + if ! [[ "$expected_artifact_digest" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::Proposal artifact digest output is malformed." + exit 1 + fi + if ! artifact_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}")"; then + echo "::error::Proposal artifact metadata is unavailable." + exit 1 + fi + downloaded_artifact_digest="$(jq -r '.digest // empty' <<<"$artifact_json")" + if ! [[ "$downloaded_artifact_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::Downloaded artifact digest evidence is malformed." + exit 1 + fi + if [ "${downloaded_artifact_digest#sha256:}" != "$expected_artifact_digest" ]; then + echo "::error::Downloaded artifact digest differs from upload evidence." + exit 1 + fi + if ! jq -e \ + --argjson artifact_id "$artifact_id" \ + --arg artifact_name "$artifact_name" \ + --argjson run_id "$GITHUB_RUN_ID" \ + '.id == $artifact_id + and .name == $artifact_name + and .expired == false + and .workflow_run.id == $run_id' \ + <<<"$artifact_json" >/dev/null; then + echo "::error::Downloaded artifact identity is not bound to this workflow run." + exit 1 + fi + if ! [[ "$expected_sha" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::Proposal patch digest output is malformed." + exit 1 + fi + printf '%s %s\n' "$expected_sha" "$patch" | sha256sum -c - + if [ "$(wc -c <"$patch" | tr -d '[:space:]')" != "$expected_bytes" ]; then + echo "::error::Proposal patch byte count changed across jobs." + exit 1 + fi + + git apply --check --binary "$patch" + git apply --index --binary "$patch" + git diff --cached --check + + if git diff --cached --raw | awk ' + /^:/ { + old_mode = substr($1, 2) + new_mode = $2 + if (old_mode ~ /^(120000|160000)$/ || new_mode ~ /^(120000|160000)$/) { + found = 1 + } + } + END { exit(found ? 0 : 1) } + '; then + echo "::error::A proposed symlink or gitlink (mode 120000|160000) is outside the bounded agent contract." + exit 1 + fi + + actual_files="$(git diff --cached --name-only -z | node -e ' + const chunks = []; + process.stdin.on("data", (chunk) => chunks.push(chunk)); + process.stdin.on("end", () => { + const names = Buffer.concat(chunks).toString("utf8").split("\0").filter(Boolean); + process.stdout.write(String(names.length)); + }); + ')" + actual_bytes="$(git diff --cached --binary --full-index | wc -c | tr -d '[:space:]')" + if [ "$actual_files" != "$expected_files" ] || [ "$actual_bytes" != "$expected_bytes" ]; then + echo "::error::Applied proposal differs from the bounded source-job evidence." + exit 1 + fi + + - name: Install exact-base dependencies without lifecycle scripts + shell: bash + run: | + set -euo pipefail + install -d -m 0700 "$RUNNER_TEMP/verify-home" "$RUNNER_TEMP/verify-config" + env \ + -u GH_TOKEN -u GITHUB_TOKEN -u REPOSITORY_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN -u ACTIONS_RUNTIME_URL \ + -u ACTIONS_RESULTS_URL -u ACTIONS_CACHE_URL \ + -u GITHUB_ENV -u GITHUB_OUTPUT -u GITHUB_PATH \ + -u GITHUB_STATE -u GITHUB_STEP_SUMMARY \ + HOME="$RUNNER_TEMP/verify-home" \ + XDG_CONFIG_HOME="$RUNNER_TEMP/verify-config" \ + npm ci --ignore-scripts + + - name: Re-run complete release verification on the fresh runner + shell: bash + run: | + set -euo pipefail + env \ + -u GH_TOKEN -u GITHUB_TOKEN -u REPOSITORY_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN -u ACTIONS_RUNTIME_URL \ + -u ACTIONS_RESULTS_URL -u ACTIONS_CACHE_URL \ + -u GITHUB_ENV -u GITHUB_OUTPUT -u GITHUB_PATH \ + -u GITHUB_STATE -u GITHUB_STEP_SUMMARY \ + HOME="$RUNNER_TEMP/verify-home" \ + XDG_CONFIG_HOME="$RUNNER_TEMP/verify-config" \ + npm run release:verify + + if [ -n "$(git diff --name-only)" ] \ + || [ -n "$(git ls-files --others --exclude-standard)" ]; then + echo "::error::Verification mutated tracked or untracked proposal files." + exit 1 + fi + + post_verify_patch="$RUNNER_TEMP/post-verify-proposal.patch" + git diff --cached --binary --full-index >"$post_verify_patch" + post_verify_sha="$(sha256sum "$post_verify_patch" | awk '{print $1}')" + post_verify_bytes="$(wc -c <"$post_verify_patch" | tr -d '[:space:]')" + if [ "$post_verify_sha" != "${{ needs.propose_product_increment.outputs.patch_sha256 }}" ] \ + || [ "$post_verify_bytes" != "${{ needs.propose_product_increment.outputs.diff_bytes }}" ]; then + echo "::error::Proposal changed during fresh-runner verification." + exit 1 + fi + + publish_product_increment: + if: >- + github.repository == 'ContextualWisdomLab/noema' + && needs.propose_product_increment.outputs.has_changes == 'true' + && needs.package_product_increment.result == 'success' + needs: + - propose_product_increment + - package_product_increment + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: read + contents: read + pull-requests: read + steps: + - name: Check out the exact proposal base for non-executing publication + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.propose_product_increment.outputs.base_sha }} + fetch-depth: 1 + persist-credentials: false + + - name: Preserve trusted metadata parser before applying the proposal + shell: bash + run: | + set -euo pipefail + install -m 0500 \ + scripts/prepare-agent-pr-message.mjs \ + "$RUNNER_TEMP/prepare-agent-pr-message.mjs" + + - name: Download the same immutable proposal artifact + id: download_publication_proposal + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + artifact-ids: ${{ needs.propose_product_increment.outputs.artifact_id }} + path: ${{ runner.temp }}/noema-product-proposal + + - name: Verify and apply immutable proposal without executing it + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + patch="$RUNNER_TEMP/noema-product-proposal/proposal.patch" + artifact_id="${{ needs.propose_product_increment.outputs.artifact_id }}" + artifact_name="noema-product-proposal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + expected_sha="${{ needs.propose_product_increment.outputs.patch_sha256 }}" + expected_files="${{ needs.propose_product_increment.outputs.changed_files }}" + expected_bytes="${{ needs.propose_product_increment.outputs.diff_bytes }}" + expected_artifact_digest="${{ needs.propose_product_increment.outputs.artifact_digest }}" + + if [ ! -f "$patch" ] || [ -L "$patch" ]; then + echo "::error::Downloaded proposal.patch is not a regular file." + exit 1 + fi + if ! [[ "$artifact_id" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Proposal artifact identity is malformed." + exit 1 + fi + if ! [[ "$expected_artifact_digest" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::Proposal artifact digest output is malformed." + exit 1 + fi + if ! artifact_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}")"; then + echo "::error::Proposal artifact metadata is unavailable." + exit 1 + fi + downloaded_artifact_digest="$(jq -r '.digest // empty' <<<"$artifact_json")" + if ! [[ "$downloaded_artifact_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::Downloaded artifact digest evidence is malformed." + exit 1 + fi + if [ "${downloaded_artifact_digest#sha256:}" != "$expected_artifact_digest" ]; then + echo "::error::Downloaded artifact digest differs from upload evidence." + exit 1 + fi + if ! jq -e \ + --argjson artifact_id "$artifact_id" \ + --arg artifact_name "$artifact_name" \ + --argjson run_id "$GITHUB_RUN_ID" \ + '.id == $artifact_id + and .name == $artifact_name + and .expired == false + and .workflow_run.id == $run_id' \ + <<<"$artifact_json" >/dev/null; then + echo "::error::Downloaded artifact identity is not bound to this workflow run." + exit 1 + fi + if ! [[ "$expected_sha" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::Proposal patch digest output is malformed." + exit 1 + fi + printf '%s %s\n' "$expected_sha" "$patch" | sha256sum -c - + if [ "$(wc -c <"$patch" | tr -d '[:space:]')" != "$expected_bytes" ]; then + echo "::error::Proposal patch byte count changed across jobs." + exit 1 + fi + + git apply --check --binary "$patch" + git apply --index --binary "$patch" + git diff --cached --check + + if git diff --cached --raw | awk ' + /^:/ { + old_mode = substr($1, 2) + new_mode = $2 + if (old_mode ~ /^(120000|160000)$/ || new_mode ~ /^(120000|160000)$/) { + found = 1 + } + } + END { exit(found ? 0 : 1) } + '; then + echo "::error::A proposed symlink or gitlink (mode 120000|160000) is outside the bounded agent contract." + exit 1 + fi + + actual_files="$(git diff --cached --name-only -z | node -e ' + const chunks = []; + process.stdin.on("data", (chunk) => chunks.push(chunk)); + process.stdin.on("end", () => { + const names = Buffer.concat(chunks).toString("utf8").split("\0").filter(Boolean); + process.stdout.write(String(names.length)); + }); + ')" + actual_bytes="$(git diff --cached --binary --full-index | wc -c | tr -d '[:space:]')" + if [ "$actual_files" != "$expected_files" ] || [ "$actual_bytes" != "$expected_bytes" ]; then + echo "::error::Publication patch differs from verified source-job evidence." + exit 1 + fi + + - name: Parse bounded untrusted pull-request metadata + shell: bash + run: | + set -euo pipefail + umask 077 + title_file="$RUNNER_TEMP/pr-title.txt" + body_file="$RUNNER_TEMP/pr-body.md" + + if [ -e PR_MESSAGE.md ] || [ -L PR_MESSAGE.md ]; then + MAX_PR_TITLE_BYTES="$MAX_PR_TITLE_BYTES" \ + MAX_PR_BODY_BYTES="$MAX_PR_BODY_BYTES" \ + node "$RUNNER_TEMP/prepare-agent-pr-message.mjs" \ + PR_MESSAGE.md "$title_file" "$body_file" + rm -f PR_MESSAGE.md + else + printf '%s' 'Noema autonomous commercial increment' >"$title_file" + cat >"$body_file" <<'BODY' + OpenCode produced one bounded NVIDIA NIM increment. Review the diff, + CHANGELOG.md, doctoring, and exact-head verification before merge. + BODY + fi + + git add -A + git diff --cached --check + + - name: Mint dedicated maintainer App token only for publication + id: maintainer_app + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 + with: + client-id: ${{ vars.NOEMA_MAINTAINER_APP_CLIENT_ID }} + private-key: ${{ secrets.NOEMA_MAINTAINER_APP_PRIVATE_KEY }} + owner: ContextualWisdomLab + repositories: ${{ github.event.repository.name }} + permission-contents: write + permission-metadata: read + permission-pull-requests: write + + - name: Revalidate queue and default-branch head + shell: bash + env: + GH_TOKEN: ${{ steps.maintainer_app.outputs.token }} + run: | + set -euo pipefail + expected_base="${{ needs.propose_product_increment.outputs.base_sha }}" + current_base="$(git rev-parse HEAD)" + if [ "$current_base" != "$expected_base" ]; then + echo "::error::base_branch_advanced: publication checkout no longer matches the proposal base." + exit 1 + fi + + if ! open_prs="$( + gh pr list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --limit 1 \ + --json number,url + )"; then + echo "::error::pull_request_inventory_unavailable_after_generation" + exit 1 + fi + if [ "$(jq 'length' <<<"$open_prs")" -gt 0 ]; then + echo "::error::open_pull_request_after_generation" + exit 1 + fi + + if ! live_base="$( + gh api \ + "repos/${GITHUB_REPOSITORY}/git/ref/heads/${DEFAULT_BRANCH}" \ + --jq '.object.sha' + )"; then + echo "::error::base_branch_inventory_unavailable_after_generation" + exit 1 + fi + if [ "$live_base" != "$expected_base" ]; then + echo "::error::base_branch_advanced" + exit 1 + fi + + - name: Package exactly one pull request in a trusted step + shell: bash + env: + GH_TOKEN: ${{ steps.maintainer_app.outputs.token }} + run: | + set -euo pipefail + title="$(cat "$RUNNER_TEMP/pr-title.txt")" + body_file="$RUNNER_TEMP/pr-body.md" + branch="nim-agent/product-dev-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$branch" + git -c core.hooksPath=/dev/null commit -m "$title" + gh auth setup-git + + set +e + git ls-remote --exit-code --heads origin "refs/heads/${branch}" >/dev/null 2>&1 + remote_status=$? + set -e + case "$remote_status" in + 0) + echo "::error::proposal_branch_already_exists" + exit 1 + ;; + 2) ;; + *) + echo "::error::proposal_branch_inventory_unavailable" + exit 1 + ;; + esac + + cleanup_remote_branch() { + git push origin --delete "$branch" >/dev/null 2>&1 || true + } + trap cleanup_remote_branch ERR + git push origin "HEAD:refs/heads/${branch}" + pr_url="$( + gh pr create \ + --repo "$GITHUB_REPOSITORY" \ + --base "$DEFAULT_BRANCH" \ + --head "$branch" \ + --title "$title" \ + --body-file "$body_file" + )" + trap - ERR + { + echo "Opened bounded pull request: $pr_url" + echo "hourly-commercial-readiness owns review, repair, exact-head revalidation, and merge." + } >>"$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d2752f55..afc70f70f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## Unreleased +- `hourly-product-development`가 `NVIDIA_NIM_API_KEY`뿐 아니라 `NOEMA_MAINTAINER_APP_CLIENT_ID`와 `NOEMA_MAINTAINER_APP_PRIVATE_KEY` 존재를 checkout·OpenCode 설치·NVIDIA 호출 전에 검증한다. 게시 경로가 준비되지 않았으면 `maintainer_app_unavailable`로 실패 폐쇄하여 알려진 실패에 추론 비용을 쓰지 않으며, `dry_run`은 credential 없이 queue와 task contract를 검토하는 경로로 유지한다. 기존 reviewer App 및 `NOEMA_LLM_API_KEY`·`contextual-orchestrator` reviewer credential 경계는 변경하지 않는다. +- zero open pull requests일 때만 `NVIDIA_NIM_API_KEY` 전용 OpenCode 1.17.13 세션을 실행하는 proposal-only `hourly-product-development` 루프를 추가. minute-47 schedule·non-cancelling single flight·OpenCode binary SHA-256 pin·NVIDIA NIM model fallback·후보 실패 시 clean reset·GitHub/OIDC credential 제거·reviewer key 비참조·full release verification·40-file/500,000-byte proposal budget·trusted one-PR packaging을 강제한다. 각 후보 실행은 900초와 30초 kill grace로 제한하고, 실패 후 `npm ci --ignore-scripts` 재설치는 별도 60초와 10초 kill grace로 제한한다. 재설치가 실패하거나 시간 초과되면 불완전한 dependency tree로 다음 후보를 실행하지 않고 실패 폐쇄한다. 세 후보의 실행·종료 2,790초, 두 번의 후보 간 재설치 140초, 300초 setup/diagnostic reserve를 합친 3,230초가 55분(3,300초) job budget에 들어가며 70초 여유를 남긴다. 마지막 후보가 실패하면 불필요한 reset·clean·재설치를 생략하고 안정적인 전체 후보 실패 진단으로 곧바로 종료한다. 모델 실행, 제안 코드 검증, publication credential을 각각 별도의 GitHub-hosted runner로 분리하고, immutable artifact의 exact ID·workflow-run ID·archive digest와 patch SHA-256·base SHA·file/byte count를 교차 검증하며 symlink(`120000`)와 gitlink(`160000`)를 세 경계 모두에서 차단한다. 제안 코드를 실행한 runner에는 Maintainer App secret/token을 절대 제공하지 않고, 세 번째 non-executing publisher에서만 late-bound repository-scoped App token을 발급한다. merge/release/deploy authority는 기존 `hourly-commercial-readiness` exact-head governance에 유지하며, 운영 Runbook과 OpenCode/NVIDIA/GitHub Actions/NIST SP 800-218 근거를 APA 7th doctoring에 기록했다. package version은 release·deployment·production KPI evidence를 발행하지 않으므로 유지한다. - `/health` liveness와 분리된 unauthenticated `GET`/`HEAD /ready` runtime readiness endpoint를 추가. GitHub Actions OIDC issuer·audience·organization/workflow binding·exact workflow ref·GitHub Cloud API origin·GitHub App identifiers·PKCS#8 private key를 외부 호출 없이 검증하며, 불완전한 설정은 secret/config value를 반사하지 않는 deterministic failure codes와 `503 ERR_SERVICE_NOT_READY`, `Retry-After`, no-store/nosniff/trace/latency headers로 실패-폐쇄한다. exact workflow named ref는 Git `check-ref-format`의 모호성·유효성 경계(`..`, `//`, dot-leading/`.lock` component, revision-expression 문자, trailing dot/slash 등)를 만족해야 하므로 GitHub가 실제로 표현할 수 없는 ref에서 false-ready가 발생하지 않는다. 배포 smoke contract가 liveness·runtime readiness·unauthenticated exchange challenge를 모두 요구하도록 확장하고 Kubernetes probe separation, RFC 9110, NIST SSDF, Git ref-format 근거를 APA 7th doctoring에 기록했다. - 공개 readiness probe의 반복 WebCrypto 비용을 줄이되 binding freshness를 보존하도록 exact unchanged PKCS#8 PEM의 importability decision만 `WeakMap`으로 재사용한다. App id·audience·workflow ref·API boundary 등 비키 binding은 매 요청 재검증하고, Cloudflare가 binding-only 변경 후 isolate를 재사용해도 key rotation은 재import되므로 이전 `ready` 결과가 새 설정을 가리지 않는다. 동일 probe·비키 binding update·key rotation 현실 회귀 테스트와 Cloudflare Workers CPU/binding lifecycle 근거를 APA 7th doctoring에 기록했다. - 배포 smoke evidence를 exact canonical `/exchange` endpoint에 결합하고 userinfo·query·fragment·trailing/alternate path·noncanonical URL은 probe 전에 차단한다. production은 HTTPS만 허용하고 loopback test만 HTTP를 허용하며, 각 probe에 5초 connect·15초 total timeout과 1 MiB response ceiling을 적용한다. 14개 status/schema/header 판단은 `jq`의 structured JSON으로 직렬화하고 canonical endpoint와 timestamp에 결합해 owner-only mode로 보존하며 RFC 3986 근거와 현실 회귀 테스트를 doctoring에 기록했다. diff --git a/README.md b/README.md index ccfbdd91c..8994afa08 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Set `NOEMA_EXCHANGE_URL` in `ContextualWisdomLab/.github` variables to the deplo - [운영 Runbook](./docs/runbook.md) - [Distributed Rate Limiting](./docs/distributed-rate-limiting.md) - [Hourly Commercial-Readiness Loop](./docs/hourly-commercial-readiness-loop.md) +- [Hourly NVIDIA NIM Product Development](./docs/operations/hourly-product-development.md) - [SLA/지원 정책](./docs/sla-and-support.md) - [가격 초안](./docs/pricing-draft.md) - [관측성 KPI](./docs/observability-kpi.md) @@ -110,6 +111,8 @@ Set `NOEMA_EXCHANGE_URL` in `ContextualWisdomLab/.github` variables to the deplo - [Transfer Readiness Plan](./docs/transfer-readiness-plan.md) - [Library Boundary Decision](./docs/library-boundary-decision.md) +`hourly-product-development.yml` runs a proposal-only OpenCode session through the dedicated `NVIDIA_NIM_API_KEY` credential when the PR queue is empty. It cannot review, merge, release, or deploy; the existing hourly commercial-readiness loop retains exact-head governance and SHA-bound merge authority. + ## KPI 계산 ```bash @@ -141,10 +144,10 @@ NOEMA_KPI_REQUIRE_WINDOW_DAYS=30 npm run kpi:verify:strict NOEMA_EXCHANGE_URL=https://.../exchange npm run smoke:check ``` -`npm run smoke:check`는 `/health`와 `/exchange`의 스키마, 추적/지연 헤더, 401 Bearer challenge, no-store/nosniff 보안 헤더를 확인하고 실패 내역을 JSON으로 출력하며, +`npm run smoke:check`는 `/health`, `/ready`, `/exchange`의 스키마, 추적/지연 헤더, runtime readiness, 401 Bearer challenge, no-store/nosniff 보안 헤더를 확인하고 실패 내역을 JSON으로 출력하며, 배포에서 `NOEMA_SMOKE_EVIDENCE_PATH`를 지정하면 `noema-smoke-evidence.json` 형태로 증빙을 저장할 수 있습니다. -CI/CD의 `cd` 워크플로우는 동일 스크립트를 실행해 `/health`/`/exchange` 계약을 검증합니다. +CI/CD의 `cd` 워크플로우는 동일 스크립트를 실행해 `/health`/`/ready`/`/exchange` 계약을 검증합니다. 운영 증빙 수집 전에는 다음 preflight로 production URL과 KPI 로그 수집 입력이 준비됐는지 확인합니다. diff --git a/docs/doctoring/hourly-nim-opencode-development.md b/docs/doctoring/hourly-nim-opencode-development.md new file mode 100644 index 000000000..9198e765e --- /dev/null +++ b/docs/doctoring/hourly-nim-opencode-development.md @@ -0,0 +1,191 @@ +# Hourly NVIDIA NIM OpenCode development: evidence and trust boundaries + +## Documentation standard + +This doctoring note uses APA 7 reference form. It separates externally supported facts from Noema-specific engineering decisions, assumptions, and residual risks. The workflow applies risk-reduction practices from the cited sources but does not claim certification, formal NIST SP 800-218 conformance, or proof that model-generated code is correct. + +## Source-supported facts + +### OpenCode + +OpenCode documents `opencode run` as non-interactive execution for automation. Its configuration supports custom OpenAI-compatible providers, environment-bound API keys, explicit model maps and limits, granular tool permissions, and disabled session sharing. These capabilities support a repository-local NVIDIA NIM provider without GitHub Copilot or the OpenCode GitHub integration. + +Noema pins OpenCode 1.17.13 and the reviewed Linux x64 archive digest rather than following a mutable latest release. The pin is an organization reproducibility decision already used by CWL repositories; it is not represented as the newest available release. Any upgrade requires a newly reviewed exact version, official artifact, digest, compatibility test, and changelog entry. + +### NVIDIA NIM + +NVIDIA documents NIM large-language-model inference through OpenAI-compatible API surfaces. Chat completions, responses, streaming, and tool calling depend on the selected model and deployment profile. The workflow therefore treats each hosted model as a fallible candidate rather than a guaranteed capability. A candidate timeout or provider error triggers clean fallback; every candidate failure produces no PR. + +### GitHub Actions runner lifetime + +GitHub documents a job as a set of steps executed on the same runner. Every step has access to that runner's workspace and filesystem, and files created in one step remain available to subsequent steps in the job. GitHub-hosted jobs, by contrast, run in fresh runner instances. These facts make a job—not a shell step or environment block—the relevant isolation boundary for untrusted execution. + +GitHub also documents that a compromised runner can attempt to access secrets or the job token when those credentials are made available. Removing environment variables after untrusted code has executed cannot prove that the runner has no resident process, modified executable, shell initialization, workspace hook, or other persistence mechanism. Consequently, credentials must not be introduced later on a runner that executed a model-proposed package or test command. + +### GitHub Actions artifacts + +GitHub workflow artifacts are designed to transfer files between jobs. The official `actions/upload-artifact` documentation states that v4 artifacts are immutable and exposes `artifact-id` and SHA-256 `artifact-digest` outputs. GitHub's REST artifact object reports the artifact ID, name, expiry state, `sha256:` digest, and originating workflow-run identity. + +Noema uses those fields as handoff evidence. Exact artifact ID prevents a name-only lookup from selecting a different object; the upload output and REST digest are compared; and workflow-run identity prevents accepting an artifact from another run. Patch SHA-256, file count, byte count, base commit, and Git mode checks remain separate evidence because archive identity alone does not establish the intended Git diff. + +### GitHub App tokens and workflow permissions + +GitHub recommends explicit least-privilege `GITHUB_TOKEN` permissions and full-SHA action pinning. The official `actions/create-github-app-token` action creates a short-lived installation access token and supports explicit repository and permission restriction. Noema keeps all three jobs' repository `GITHUB_TOKEN` read-only and mints the dedicated Maintainer App token only in a fresh publisher that never executes proposed code. + +GitHub documents that most events produced through a repository `GITHUB_TOKEN` do not start new workflow runs. Publishing through the repository-scoped Maintainer App instead preserves the normal pull-request event path into independent Checks. + +### Secure development lifecycle + +NIST SP 800-218 describes a high-level Secure Software Development Framework for reducing vulnerabilities, mitigating undetected defects, addressing root causes, and communicating supply-chain expectations. Noema's test-first contract, complete verification, retained RED-to-GREEN evidence, dependency/security gates, credential separation, and review-only handoff implement those risk-reduction ideas without claiming certification. + +## Noema-specific decisions + +### Proposal and merge are separate control planes + +`hourly-commercial-readiness` is deterministic governance. It evaluates current-head checks, review state, rulesets, and mergeability. Mixing a model call into the same workflow that decides whether to merge would combine proposal generation with the control plane evaluating its own output. + +`hourly-product-development.yml` is therefore proposal-only. The model may edit and verify a local tree, but cannot approve, merge, release, publish a release, or deploy. The resulting PR must pass the existing independent exact-head governance loop. + +### Three-job runner isolation + +The workflow uses three ordered GitHub-hosted jobs: + +1. `propose_product_increment` runs OpenCode and initial verification without repository write credentials. +2. `package_product_increment` reconstructs and executes the proposal on a fresh uncredentialed verifier. +3. `publish_product_increment` reconstructs the same proposal without executing it and only then mints publication credentials on a third fresh runner. + +The original two-job design executed `npm run release:verify` and later minted the Maintainer App token in the same second job. That design removed credentials during the test step but did not establish runner-level isolation. A malicious or compromised proposal could have left a resident process or modified runner state before the later secret-bearing action. CI run `30969603538` retained the RED contract requiring a third runner; the implementation removes the credential action and all remote mutation from the verifier job. + +### Read-only model job + +The model runs in `propose_product_increment`, which has only repository and pull-request read permissions. Its subprocess receives `NVIDIA_API_KEY` but removes GitHub tokens, Actions OIDC credentials, artifact/cache runtime tokens, and runner command-file paths. OpenCode command permissions deny common network and repository-mutation tools. + +The successful working tree becomes a binary full-index `proposal.patch` bound to its exact base SHA, SHA-256, changed-file count, and byte count. Symlink mode `120000` and gitlink mode `160000` are rejected before artifact upload. The artifact expires after one day, cannot be overwritten under the same name, and exports exact ID and archive digest outputs. + +### Fresh uncredentialed verifier + +`package_product_increment` runs on a fresh runner and receives no NIM or Maintainer credential. Its job-level token is read-only. It downloads the artifact by exact ID and verifies the artifact REST object's ID, deterministic name, expiry state, originating workflow run, and digest. It separately verifies patch SHA-256, byte count, changed-file count, exact base, and forbidden Git modes. + +The verifier then applies the patch and executes `npm run release:verify` with GitHub, OIDC, Actions runtime/cache, and runner command-file credentials removed, dependency lifecycle scripts disabled, and an isolated temporary home. It fails if verification mutates tracked or non-ignored untracked files or changes the staged patch digest. + +This job deliberately executes untrusted proposed code, but no publication credential is ever introduced to its runner. + +### Fresh non-executing publisher + +`publish_product_increment` depends on successful proposal and verification jobs. It starts on a third fresh runner with a read-only job token and no NIM credential. It does not install dependencies or run proposed tests, builds, package scripts, binaries, or shell commands. It is the only fresh write-capable runner, and it remains read-only until the late-bound Maintainer App token is minted. + +Before applying the proposal, the publisher copies the trusted base-branch metadata parser into `RUNNER_TEMP`. It downloads the exact same artifact ID and repeats artifact/run/digest and patch/base/file/byte/mode validation. It applies the patch only as data, then uses the preserved parser to transform bounded `PR_MESSAGE.md` input. + +Only after those non-executing operations does the full-SHA-pinned GitHub-maintained action mint a short-lived Maintainer App token restricted to `ContextualWisdomLab/noema` with metadata read, contents write, and pull-request write. Trusted shell steps revalidate the zero-PR queue and exact live `main`, create one unique branch, commit with hooks disabled, push once, and open one PR. + +This three-job boundary provides the intended claim: neither model execution nor proposed-code verification shares a runner with publication authority. + +### Zero-open-PR gate + +The model is invoked only when GitHub returns zero open pull requests. Unreadable inventory fails closed. The publisher repeats the inventory and default-branch checks immediately before remote mutation because another actor can open a PR or advance `main` while the model and verifier run. + +GitHub provides no atomic “create a PR only if none exists” transaction. The remaining race is bounded by revalidation, unique branch names, branch protection, and exact-head governance rather than claimed to be eliminated. + +### Dedicated development and publication credentials + +The workflow maps `secrets.NVIDIA_NIM_API_KEY` to `NVIDIA_API_KEY` only in the model step. It does not use GitHub Copilot, GitHub Models, `NOEMA_LLM_API_KEY`, the reviewer App private key, or production `contextual-orchestrator` reviewer credentials. Reviewer credential names and routing remain unchanged. + +Publication reuses the repository's existing dedicated Maintainer App variables and private-key secret. It does not repurpose the reviewer App identity or key contract. The separation preserves independent review evidence and gives generated PRs a normal event path into `ci`, `reviewer-ci`, and Security Scan. + +### Clean model fallback + +Each model candidate has a timeout of 900 seconds plus a 30-second forced-termination grace period. Cleanup and dependency reinstall occur only between candidates, so three candidate budgets and two bounded reinstalls use `3 × (900 + 30) + 2 × (60 + 10) = 2,930 seconds`. Reserving 300 seconds for setup and the final diagnostic yields 3,230 seconds inside the 3,300-second proposal-job limit, leaving 70 seconds of explicit slack. After the final candidate fails, no later candidate can be protected by cleanup, so the workflow skips reset, clean, and reinstall and emits the stable all-candidates-failed diagnostic directly. Partial output from one model cannot contaminate a later candidate. Fallback improves availability; it is not quality evidence. + +### Executable product contract + +The prompt requires one buyer-visible increment, test-first RED-to-GREEN evidence, realistic Noema-specific tests, 100% production coverage, 100% reviewer coverage/docstrings when touched, APA 7 doctoring, modular MSA compatibility, descriptive database naming, changelog/documentation updates, and Semantic Versioning restraint. Prompt instructions are not treated as enforcement by themselves; static workflow tests, proposal budgets, fresh-runner verification, branch protection, and exact-head review provide executable controls. + +### Bounded proposal and metadata + +The proposal is limited to 40 changed files and 500,000 patch bytes. Symlinks, gitlinks, whitespace errors, malformed patches, artifact identity or digest mismatches, patch digest mismatches, and post-verification mutations fail closed. + +`PR_MESSAGE.md` is untrusted model output. A base-branch parser copied before patch application requires a regular non-symlink file, `O_NOFOLLOW`, stable inode, strict UTF-8, safe control characters, a 120-byte title, and a 20,000-byte body. Trusted output files use owner-only permissions and the source metadata file is excluded from the commit. + +## Residual risks + +### NIM credential exposure within the model process + +The NIM key necessarily exists in the OpenCode process. Command denials are defense in depth, not a microVM egress boundary. A shell-capable process may construct behavior equivalent to a denied command. The security claim is deliberately narrower: the key is development-only, GitHub write credentials are absent, and model output crosses jobs only as a bounded immutable artifact. + +A future stronger design should broker inference through a narrow proxy and keep the upstream NIM credential outside the model process. + +### Repository data processing + +OpenCode can send prompts and selected repository context to NVIDIA NIM. Operators must evaluate confidentiality, retention, regional, contractual, and data-processing requirements before enabling the secret. Production logs, customer evidence, reviewer secrets, deployment credentials, and revenue evidence are not intentionally provided, but committed repository content is readable. + +### Untrusted executable verification + +`npm run release:verify` executes proposed code. The verifier therefore has no Maintainer App secret, App token, NIM secret, GitHub write token, OIDC credential, Actions runtime/cache credential, or runner command-file channel. A fresh publisher starts only after the verifier completes successfully. This is materially stronger than same-job environment cleanup, but the verifier is still not a hostile-code microVM and can affect only its own ephemeral runner and outbound network accessible under GitHub-hosted runner policy. + +### Artifact service trust + +The handoff relies on GitHub Actions artifact storage and API metadata. ID, workflow-run, digest, patch digest, and exact-base validation detect substitution or mismatch within the exposed contract; they do not eliminate compromise of GitHub's platform or prove proposal semantics. Independent PR review and exact-head Checks remain mandatory. + +### Publication credential scope and rotation + +The Maintainer App token is short-lived and repository-scoped, but the App registration and stored private key remain privileged operational assets. Operators must maintain least-privilege App installation scope, rotate compromised keys, audit App ownership, and keep break-glass responsibilities independent from reviewer credentials. Failure to mint the token safely stops publication; it must not trigger fallback to a broader PAT or job-level write `GITHUB_TOKEN`. + +### Model and scheduler instability + +Hosted model availability, quotas, latency, tool behavior, and quality can change. GitHub schedules can be delayed, dropped, or disabled. Candidate success and workflow completion are not semantic quality guarantees. The system safely produces either no PR or one reviewable PR; it never self-approves or self-merges. + +## Verification mapping + +| Requirement or risk | Executable control | +|---|---| +| Existing PR | Read-only inventory gate before model and repeated gate before push | +| Missing NIM key | `nim_api_key_unavailable`, no model call | +| Mutable OpenCode binary | Exact version, official archive, SHA-256 verification | +| Reviewer-key reuse | Negative workflow assertions and dedicated secret mapping | +| Model repository mutation | Read-only job; no GitHub write token; mutation commands denied | +| Runner command-channel poisoning | Command-file and Actions runtime variables removed | +| Candidate contamination | Hard reset, `git clean -fdx`, clean reinstall | +| Artifact substitution | Exact artifact ID, name, workflow-run ID, archive digest, patch digest, exact base, file count, and byte count | +| Unverified proposal | Complete release verification in model job and fresh verifier | +| Verification mutation | Unstaged/untracked check and post-verification digest match | +| Write token during untrusted execution | Verifier contains no Maintainer credentials; publisher is a third fresh runner | +| Proposed-code persistence | Publisher never executes proposed code before or after token minting | +| Overbroad workflow token | All job-level `GITHUB_TOKEN` permission sets remain read-only | +| Generated PR lacks Checks | Dedicated App token, not repository `GITHUB_TOKEN`, authors publication | +| Oversized, symlink, or gitlink proposal | File/byte budget and modes `120000`/`160000` rejection at all three boundaries | +| Malformed PR metadata | Trusted strict parser and byte/control-character limits | +| Stale base or new PR | Live revalidation before push | +| Hooks, orphan branch, or collision | Hooks disabled, unique-branch absence check, and cleanup trap | +| Unauthorized merge/release/deploy | No corresponding workflow command; governance handoff | + +## APA 7 references + +GitHub. (2026). *Compromised runners*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/actions/concepts/security/compromised-runners + +GitHub. (2026). *Events that trigger workflows*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows + +GitHub. (2026). *REST API endpoints for GitHub Actions artifacts*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/rest/actions/artifacts?apiVersion=2026-03-10 + +GitHub. (2026). *Secure use reference*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/actions/reference/security/secure-use + +GitHub. (2026). *Understanding GitHub Actions*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/actions/get-started/understand-github-actions + +GitHub. (2026). *Use GITHUB_TOKEN for authentication in workflows*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/actions/tutorials/authenticate-with-github_token + +GitHub. (2026). *Workflow syntax for GitHub Actions*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +GitHub. (2026). *Create GitHub App token*. GitHub Marketplace. Retrieved August 5, 2026, from https://github.com/actions/create-github-app-token + +GitHub. (2026). *Upload GitHub Actions artifacts*. GitHub. Retrieved August 5, 2026, from https://github.com/actions/upload-artifact + +NVIDIA Corporation. (2026). *API reference—NVIDIA NIM for large language models*. NVIDIA Documentation. Retrieved August 5, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html + +NVIDIA Corporation. (2026). *Tool calling and MCP integration*. NVIDIA Documentation. Retrieved August 5, 2026, from https://docs.nvidia.com/nim/large-language-models/2.0.2/advanced-use-cases/tool-calling-and-mcp.html + +OpenCode. (2026). *CLI*. Retrieved August 5, 2026, from https://opencode.ai/docs/cli/ + +OpenCode. (2026). *Permissions*. Retrieved August 5, 2026, from https://opencode.ai/docs/permissions/ + +OpenCode. (2026). *Providers*. Retrieved August 5, 2026, from https://opencode.ai/docs/providers/ + +Souppaya, M., Scarfone, K., & Dodson, D. (2022). *Secure software development framework (SSDF) version 1.1: Recommendations for mitigating the risk of software vulnerabilities* (NIST Special Publication 800-218). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-218 diff --git a/docs/doctoring/hourly-product-development-prerequisites.md b/docs/doctoring/hourly-product-development-prerequisites.md new file mode 100644 index 000000000..80c698df6 --- /dev/null +++ b/docs/doctoring/hourly-product-development-prerequisites.md @@ -0,0 +1,65 @@ +# Hourly product-development publication prerequisites + +## Documentation standard + +This doctoring note uses APA 7 reference form. It separates source-supported facts from Noema-specific decisions and does not claim formal NIST conformance. + +## Problem statement + +The scheduled development path has two independent credential prerequisites: + +1. `NVIDIA_NIM_API_KEY` permits the read-only OpenCode proposal job to obtain model inference. +2. `NOEMA_MAINTAINER_APP_CLIENT_ID` and `NOEMA_MAINTAINER_APP_PRIVATE_KEY` permit the later non-executing publisher to create one repository-scoped branch and pull request. + +Checking only the inference key can spend model compute on a proposal that the workflow is structurally unable to publish. That is a deterministic configuration failure rather than a model-quality failure and should be rejected before checkout or inference. + +## Source-supported controls + +GitHub documents that a workflow reads a secret only when the workflow explicitly includes it, and recommends granting credentials the minimum possible permissions. GitHub further recommends GitHub Apps as fine-grained, short-lived, non-user-bound credentials when repository automation needs permissions beyond read-only access. These facts support separating the NIM development credential from the repository publication credential and preserving read-only job-level `GITHUB_TOKEN` permissions. This is a least privilege control: model execution never receives publication authority, and publication receives only the repository-scoped permissions required to create one branch and pull request. + +NIST SP 800-218 Version 1.1 recommends integrating secure-development requirements and verification into the software life cycle. NIST SP 800-218A augments that framework with practices specific to generative AI and foundation-model systems. The December 2025 SP 800-218 Revision 1 initial public draft describes updated secure and reliable development practices, but remains a draft; Noema therefore records it as a current informative source while retaining the final Version 1.1 and final AI community profile as the normative published references. + +## Noema-specific decision + +Before OpenCode starts, the proposal gate evaluates only presence booleans: + +- `NVIDIA_NIM_API_KEY != ''` +- `NOEMA_MAINTAINER_APP_CLIENT_ID != ''` +- `NOEMA_MAINTAINER_APP_PRIVATE_KEY != ''` + +The workflow does not reveal values, import the private key, mint an App token, or call a model during this gate. Missing publication configuration returns the stable reason `maintainer_app_unavailable` and stops before checkout, dependency installation, OpenCode download, or NVIDIA inference. + +The App token is still minted only in the third, non-executing publication job. Presence checking does not prove that the key is valid, that the App remains installed, or that permissions are sufficient; those live failures continue to fail closed when `actions/create-github-app-token` runs. This preserves the late-token trust boundary while preventing known-impossible sessions. + +Manual `dry_run` deliberately bypasses credential-presence requirements because it performs no checkout, model call, artifact publication, branch push, or pull-request creation. It remains an operator inspection path rather than evidence that a live proposal can be published. + +## Reviewer credential separation + +The gate does not read, rename, or validate reviewer credentials. In particular, it does not use `NOEMA_LLM_API_KEY`, the reviewer App private key, or `contextual-orchestrator` provider credentials. Development proposal authority, publication authority, and independent review authority remain separate. + +## Verification contract + +Executable tests must prove that: + +- both Maintainer App presence booleans are evaluated in the pre-inference gate; +- either missing value produces `dispatch=false` and `reason=maintainer_app_unavailable`; +- the gate appears before task preparation, checkout, and OpenCode execution; +- `dry_run=true` remains available without production credentials; +- the dedicated NIM secret and reviewer credential boundaries remain unchanged; and +- operations and doctoring documents describe the same failure reason and credential names. + +## Residual risk + +Presence booleans can become stale between the initial gate and publication, and they cannot validate App installation scope or private-key correctness. Exact publication remains protected by fresh token minting, queue and base-head revalidation, repository-scoped permissions, and ordinary pull-request governance. The new gate reduces deterministic cost waste; it is not a substitute for live App readiness evidence under issue #29. + +## APA 7 references + +Booth, H., Souppaya, M., Vassilev, A., Ogata, M., Stanley, M., & Scarfone, K. (2024). *Secure software development practices for generative AI and dual-use foundation models: An SSDF community profile* (NIST Special Publication 800-218A). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-218A + +GitHub. (2026). *Secrets*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/actions/concepts/security/secrets + +GitHub. (2026). *Making authenticated API requests with a GitHub App in a GitHub Actions workflow*. GitHub Docs. Retrieved August 5, 2026, from https://docs.github.com/en/apps/creating-github-apps/writing-code-for-a-github-app/making-authenticated-api-requests-with-a-github-app-in-a-github-actions-workflow + +Souppaya, M., Scarfone, K., & Dodson, D. (2022). *Secure software development framework (SSDF) version 1.1: Recommendations for mitigating the risk of software vulnerabilities* (NIST Special Publication 800-218). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-218 + +Booth, H., Ogata, M., Kent, K., Souppaya, M., & Dodson, D. (2025). *Secure software development framework (SSDF) version 1.2: Recommendations for mitigating the risk of software vulnerabilities* (Initial Public Draft NIST Special Publication 800-218, Revision 1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-218r1.ipd diff --git a/docs/operations/hourly-product-development-prerequisites.md b/docs/operations/hourly-product-development-prerequisites.md new file mode 100644 index 000000000..5b00164d2 --- /dev/null +++ b/docs/operations/hourly-product-development-prerequisites.md @@ -0,0 +1,43 @@ +# 시간 단위 제품 개발 게시 전제조건 + +## 목적 + +`.github/workflows/hourly-product-development.yml`은 OpenCode가 NVIDIA NIM을 호출하기 전에 모델이 만든 제안을 실제 pull request로 게시할 수 있는지 먼저 확인합니다. 게시 경로가 준비되지 않은 상태에서 추론 비용만 소비하고 마지막 단계에서 실패하는 동작을 허용하지 않습니다. + +## 필수 입력 + +일반 실행에는 다음 값이 모두 필요합니다. + +- `NVIDIA_NIM_API_KEY`: OpenCode 개발 세션 전용 조직 또는 저장소 secret +- `NOEMA_MAINTAINER_APP_CLIENT_ID`: `ContextualWisdomLab/noema`에만 설치된 Maintainer GitHub App의 repository variable +- `NOEMA_MAINTAINER_APP_PRIVATE_KEY`: 같은 App의 private-key secret + +기존 reviewer App, `NOEMA_LLM_API_KEY`, `contextual-orchestrator` reviewer credential은 이 전제조건에 사용하지 않으며 이름과 권한 경계를 변경하지 않습니다. + +## 실패 폐쇄 동작 + +열린 pull request가 없더라도 Maintainer App의 client ID 또는 private key가 없으면 gate는 다음 결과를 기록하고 checkout·OpenCode 다운로드·NVIDIA 호출 전에 종료합니다. + +```text +dispatch=false +reason=maintainer_app_unavailable +``` + +`NVIDIA_NIM_API_KEY`가 없으면 기존 `nim_api_key_unavailable` 결과를 유지합니다. pull request inventory를 읽지 못하거나 열린 PR이 있으면 각각 `pull_request_inventory_unavailable`, `open_pull_request`로 종료합니다. + +## dry_run + +`workflow_dispatch`에서 `dry_run=true`를 선택하면 secret이나 App credential이 없어도 queue gate와 전체 task contract를 검토할 수 있습니다. dry run은 checkout, OpenCode 설치, NVIDIA API 호출, artifact 업로드, branch push, pull request 생성 중 어느 것도 수행하지 않습니다. + +## 활성화 확인 + +1. Maintainer App이 `ContextualWisdomLab/noema`에만 설치되어 있는지 확인합니다. +2. App 권한을 Metadata read, Contents write, Pull requests write로 제한합니다. +3. `NOEMA_MAINTAINER_APP_CLIENT_ID`와 `NOEMA_MAINTAINER_APP_PRIVATE_KEY`를 설정합니다. +4. `dry_run=true`로 prompt와 queue 판단을 검토합니다. +5. 임시 검증 PR에서 publication job이 짧은 수명의 repository-scoped token을 생성하고 정확히 한 branch와 한 PR만 만드는지 확인합니다. +6. reviewer credential 이름이나 central review route가 변경되지 않았는지 확인합니다. + +## 운영 복구 + +`maintainer_app_unavailable`이 나타나면 모델 fallback이나 timeout을 조정하지 않습니다. App 설치 범위, client ID variable, private-key secret과 key rotation 상태를 복구한 뒤 다시 실행합니다. 의도적인 중지는 workflow를 비활성화하거나 App credential을 회수하여 수행합니다. diff --git a/docs/operations/hourly-product-development.md b/docs/operations/hourly-product-development.md new file mode 100644 index 000000000..114681605 --- /dev/null +++ b/docs/operations/hourly-product-development.md @@ -0,0 +1,33 @@ +# 시간별 NVIDIA NIM 제품 개발 운영 + +## 목적과 책임 경계 + +`.github/workflows/hourly-product-development.yml`은 **열린 PR 0개** 상태에서만 Noema의 다음 구매자 가시적 제품 증분을 제안합니다. OpenCode 1.17.13과 전용 `NVIDIA_NIM_API_KEY`를 사용하지만 리뷰, 승인, 병합, 릴리스, 배포는 수행하지 않습니다. 정확한 현재 HEAD의 리뷰, 필수 Checks, 미해결 스레드, 저장소 규칙, 병합 가능성 판단은 기존 `hourly-commercial-readiness`가 계속 담당합니다. 자동 개발은 후보 PR을 만드는 역할만 하며 최종 거버넌스 권한을 획득하지 않습니다. + +워크플로는 매시 47분에 실행되고 수동 `dry_run=true`를 지원합니다. 드라이 런은 실제 PR 목록과 작업 계약만 확인하며 checkout, 모델 호출, 아티팩트 업로드, 브랜치 push, PR 생성을 하지 않습니다. GitHub 예약 실행은 정시 SLA가 아니므로 각 실행은 이전 상태를 믿지 않고 열린 PR 목록, 기본 브랜치 SHA, 필요한 자격 증명을 다시 확인합니다. 목록 조회 실패, 기존 PR 발견, 시크릿 부재는 모두 실패 폐쇄 사유입니다. + +## 모델 폴백과 시간 예산 + +공식 OpenCode 아카이브는 고정 버전과 SHA-256으로 검증합니다. 공급자는 NVIDIA NIM 엔드포인트만 허용하고 공유, 자동 업데이트, MCP, LSP, 외부 디렉터리, 하위 에이전트, 질문 도구, 웹 검색과 웹 가져오기를 비활성화합니다. 모델 후보는 Llama Nemotron, Nemotron Super, DeepSeek 순서로 시도합니다. **후보별 900초**와 강제 종료 유예 30초를 적용합니다. 다음 후보가 남아 있을 때만 의존성 재설치를 60초와 강제 종료 유예 10초로 별도 제한합니다. 최악의 경우 세 후보 실행·종료에 `3 × (900 + 30) = 2,790초`, 후보 사이 두 번의 재설치에 `2 × (60 + 10) = 140초`를 사용합니다. 최초 설정과 최종 진단에 300초를 예약하면 총 3,230초이며, 3,300초인 55분 제안 job 예산 안에 70초의 명시적 여유를 남깁니다. + +다음 후보가 남아 있는 상태에서 후보가 실패하면 `git reset --hard HEAD`, `git clean -fdx`를 실행하고 `npm ci --ignore-scripts`를 명시적 60초 제한 안에서 재실행한 뒤에만 폴백합니다. 마지막 후보 실패에는 다음 후보가 없으므로 reset·clean·재설치를 실행하지 않고 곧바로 안정적인 전체 후보 실패 진단으로 종료합니다. 후보 사이 재설치가 실패하거나 제한 시간을 넘기면 의존성 트리가 완전하다고 증명할 수 없으므로 즉시 실패 폐쇄하고 이후 모델 후보를 실행하지 않습니다. 이전 후보의 부분 변경이나 생성 파일이 다음 후보의 입력을 오염시키지 않도록 하며, 폴백은 가용성 제어일 뿐 품질 증거가 아닙니다. 성공한 제안도 독립 검증과 리뷰를 통과해야 합니다. + +## 세 runner의 자격 증명 분리 + +첫 번째 제안 runner는 읽기 권한만 가지며 OpenCode subprocess에는 NVIDIA 키만 전달합니다. GitHub 토큰, OIDC 값, Actions 런타임 토큰, 캐시 토큰, runner 명령 파일 채널을 제거합니다. 변경은 40개 파일과 500,000바이트로 제한하고 공백 오류, 심링크 모드 `120000`, gitlink 모드 `160000`을 원본 모드와 대상 모드 양쪽에서 검사합니다. 결과는 정확한 base SHA, 파일 수, 바이트 수, SHA-256에 결합된 binary full-index `proposal.patch`로 저장합니다. + +두 번째 검증 runner는 NIM 키와 Maintainer App 키가 없는 새 실행기입니다. `actions: read`, `contents: read`, `pull-requests: read`만 사용합니다. artifact ID, 이름, 만료 여부, 원본 workflow run, digest, patch 크기와 해시, base SHA를 독립적으로 확인합니다. 패치를 적용한 뒤 격리된 임시 홈과 제거된 GitHub·OIDC·Actions 채널에서 `npm run release:verify`를 실행하고 검증 전후 staged patch digest가 동일한지 확인합니다. 이 runner는 제안 코드를 실행하지만 게시 권한을 받지 않습니다. + +`publish_product_increment`는 **세 번째 새 게시 runner**입니다. 제안 코드를 실행하지 않고 NIM 키도 받지 않습니다. 기본 브랜치에서 신뢰된 PR 메타데이터 파서를 먼저 복사한 뒤 동일한 artifact ID와 digest-bound patch를 다시 검증합니다. 그 다음에만 full SHA로 고정된 액션이 짧은 수명의 Maintainer App 토큰을 발급합니다. 토큰 범위는 Noema 저장소의 metadata read, contents write, pull-request write로 제한됩니다. App 토큰 발급 후에도 열린 PR 큐와 실제 `main` SHA를 다시 읽고, 새 PR이나 base 전진이 있으면 원격 변경 전에 종료합니다. + +## 신뢰할 수 없는 입력과 게시 + +모델이 만든 `PR_MESSAGE.md`는 신뢰할 수 없는 입력입니다. 파서는 심링크를 거부하고 `O_NOFOLLOW`, inode 안정성, 엄격한 UTF-8, 제어 문자와 양방향 제어 문자 제한, 제목 120바이트, 본문 20,000바이트를 적용합니다. 신뢰된 출력은 mode `0600`으로 기록하고 원본은 commit 전에 삭제합니다. + +게시 단계는 실행별 고유 브랜치를 한 번 만들고 한 번 push한 뒤 PR을 한 번 생성합니다. PR 생성 실패 시 orphan 브랜치를 제거합니다. merge, release, publish, deploy 명령은 없습니다. 생성된 PR은 CodeRabbit, OpenCode review, Noema review, `ci`, `reviewer-ci`, Security Scan, branch protection, unresolved-thread 검사와 exact-head 병합 루프로 인계됩니다. + +## 운영 위험과 롤백 + +NIM 키는 OpenCode 프로세스 안에 존재하므로 명령 거부만으로 microVM egress 경계를 주장하지 않습니다. 지원 가능한 주장은 모델과 쓰기 가능한 저장소 토큰이 공존하지 않고, 신뢰할 수 없는 코드는 게시 자격 증명이 없는 runner에서만 실행되며, 게시 runner는 동일한 immutable patch를 실행 없이 재구성한다는 것입니다. OpenCode는 commit된 저장소 문맥을 외부 모델 서비스로 보낼 수 있으므로 기밀성, 데이터 보존, 지역, 계약 요건을 별도로 평가해야 합니다. + +GitHub에는 다른 PR이 없을 때만 PR을 생성하는 원자적 트랜잭션이 없습니다. 최종 큐와 base 재검증, 고유 브랜치 이름, branch protection, exact-head 리뷰가 남은 경쟁 위험을 통제합니다. 모델 실행을 중지하려면 워크플로를 비활성화하거나 `NVIDIA_NIM_API_KEY`를 폐기합니다. 게시만 중지하려면 Maintainer App 키를 폐기합니다. `main`에서 워크플로를 제거하는 것이 코드 롤백이며 기존 `/exchange`, 리뷰, 릴리스, 배포 경로에는 영향을 주지 않습니다. diff --git a/docs/superpowers/plans/2026-08-05-hourly-nim-opencode-development.md b/docs/superpowers/plans/2026-08-05-hourly-nim-opencode-development.md new file mode 100644 index 000000000..f5298ec31 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-hourly-nim-opencode-development.md @@ -0,0 +1,174 @@ +# Hourly NVIDIA NIM OpenCode Development Implementation Plan + +> **Status:** Implemented and security-revised on PR #64. This plan replaces the initial same-job and two-job packaging sequence with a three-runner design. The retained RED evidence and exact-head CI remain the execution record. + +**Goal:** Add an hourly, proposal-only OpenCode development loop that uses `NVIDIA_NIM_API_KEY`, creates at most one bounded pull request when the queue is empty, and leaves review, release, deployment, and merge decisions to existing exact-head governance. + +**Architecture:** Minute-47 scheduling is separate from the deterministic minute-17 commercial-readiness loop. `propose_product_increment` runs the model and exports one immutable patch from a read-only runner. `package_product_increment` executes and verifies that patch on a fresh runner with no publication credentials. `publish_product_increment` reconstructs the same artifact without executing proposed code on a third fresh runner and only then mints the least-privilege Maintainer App token for one branch and one PR. + +**Tech stack:** GitHub Actions, Bash, GitHub CLI, jq, OpenCode 1.17.13, NVIDIA NIM OpenAI-compatible API, Node.js 24, Vitest, GitHub Actions artifacts, and a repository-scoped Maintainer GitHub App. + +## Global constraints + +- Use `secrets.NVIDIA_NIM_API_KEY` only for the OpenCode proposal job; do not reference GitHub Copilot, GitHub Models, reviewer App credentials, or `NOEMA_LLM_API_KEY`. +- Do not change central reviewer variables, reviewer App identity, or `contextual-orchestrator` reviewer behavior. +- Do not merge, release, publish, deploy, approve, or fabricate production, KPI, customer, revenue, transfer, attestation, or acquisition evidence. +- Invoke the model only when open-PR inventory is readable and the open count is zero. +- Keep every job-level `GITHUB_TOKEN` read-only. Publication authority must be a late-bound repository-scoped Maintainer App token on a runner that never executed proposed code. +- Keep checkout credentials unpersisted and remove GitHub, OIDC, Actions runtime/cache, and runner command-file variables from untrusted execution. +- Bind the proposal handoff to exact base SHA, patch SHA-256, changed-file count, patch byte count, `artifact-id`, `artifact-digest`, artifact name, workflow-run ID, and expiry state. +- Reject symlink mode `120000` and gitlink mode `160000` at proposal, verification, and publication boundaries. +- Require test-first implementation, realistic Noema-specific tests, 100% production coverage, reviewer coverage/docstrings when touched, APA 7 doctoring, modular MSA compatibility, `CHANGELOG.md`, and release restraint. +- Use descriptive two-word-or-longer `snake_case` names for any new database object. +- Figma and Product Design are not used because this increment has no user-facing visual interface. + +--- + +## Task 1 — Define the proposal-only product contract + +**Files** + +- `test/hourly-product-development-workflow.test.ts` +- `docs/superpowers/specs/2026-08-05-hourly-nim-opencode-development-design.md` +- this implementation plan + +- [x] Write the failing workflow contract before implementation. +- [x] Require minute-47 scheduling, manual dry run, non-cancelling concurrency, exact repository binding, readable zero-open-PR gate, NIM-only credentials, checksum-pinned OpenCode, bounded fallback, no model write authority, one PR maximum, no merge/release/deploy command, product-quality prompt requirements, operations documentation, doctoring, and changelog traceability. +- [x] Retain the initial RED workflow run and do not weaken the contract. + +**Evidence:** initial missing-workflow RED run `30966458223`. + +## Task 2 — Implement the read-only proposal runner + +**Job:** `propose_product_increment` + +- [x] Grant only `contents: read` and `pull-requests: read`. +- [x] Query at most one open PR and record stable fail-closed reasons when inventory is unavailable, a PR exists, or the NIM secret is unavailable. +- [x] Check out exact `main` without persisted credentials and record the full 40-character base SHA. +- [x] Install dependencies without lifecycle scripts before model execution. +- [x] Download the official OpenCode 1.17.13 archive and verify SHA-256 `157afa289d1a8d9372de0ce19ac726119b937a1f6b201808d46f06e4e59bb348`. +- [x] Configure only the NVIDIA NIM provider, disable sharing, auto-update, MCP, LSP, web access, external-directory access, task delegation, interactive questions, and repository mutation commands. +- [x] Run ordered NIM fallback with a 900-second candidate timeout and 30-second candidate kill grace. Only between failed candidates, hard reset, remove ignored/untracked files, and run a separately bounded 60-second clean reinstall with 10-second kill grace. Skip cleanup after the final candidate so its failure reaches the stable all-candidates-failed diagnostic; fail closed and stop the chain when an inter-candidate reinstall fails or times out. +- [x] Remove GitHub tokens, Actions OIDC variables, artifact/cache runtime variables, and runner command-file variables from the OpenCode subprocess. +- [x] Run `npm run release:verify` before export. +- [x] Stage the complete proposal, reject whitespace errors, symlinks, gitlinks, more than 40 files, and more than 500,000 patch bytes. +- [x] Serialize `proposal.patch` with `--binary --full-index` and record exact patch SHA-256, file count, byte count, and base SHA. +- [x] Upload with the full-SHA-pinned artifact action, one-day retention, no overwrite, and `id: upload_proposal`. +- [x] Export both `artifact-id` and `artifact-digest` as job outputs. + +**No remote mutation:** this job contains no Maintainer App action, `git push`, or `gh pr create`. + +## Task 3 — Add a fresh uncredentialed verifier + +**Job:** `package_product_increment` + +- [x] Depend on `propose_product_increment` and use a new GitHub-hosted runner. +- [x] Grant read-only repository, pull-request, and Actions permissions. +- [x] Download by exact numeric `artifact-id` rather than mutable name selection. +- [x] Query the artifact REST object and validate ID, deterministic name, non-expired state, originating workflow-run ID, and REST `sha256:` digest against exported `artifact-digest`. +- [x] Validate patch SHA-256, patch byte count, changed-file count, exact base SHA, and forbidden Git modes. +- [x] Apply the patch and install dependencies without lifecycle scripts. +- [x] Execute `npm run release:verify` with GitHub, OIDC, Actions runtime/cache, and runner command-file channels removed and an isolated temporary home. +- [x] Fail when verification mutates tracked or non-ignored untracked files. +- [x] Recompute the staged patch and require the original SHA-256 and byte count after verification. +- [x] Keep `NVIDIA_NIM_API_KEY`, Maintainer App variables, Maintainer App private key, App token, `git push`, and `gh pr create` absent. + +**RED evidence:** run `30969603538` proved the initial architecture failed the required runner-isolation contract because verification and later credential minting shared a job. + +## Task 4 — Add a third fresh non-executing publisher + +**Job:** `publish_product_increment` + +- [x] Depend on both successful proposal and verifier jobs. +- [x] Start on a third fresh GitHub-hosted runner with a read-only job token and no NIM credential. +- [x] Do not install dependencies or execute proposed tests, builds, package scripts, binaries, or shell commands. +- [x] Copy the trusted exact-base PR metadata parser into `RUNNER_TEMP` before patch application. +- [x] Download the same exact `artifact-id` and independently repeat artifact name, workflow-run, expiry, `artifact-digest`, patch digest, base, file, byte, symlink, and gitlink checks. +- [x] Apply the patch only as Git data. +- [x] Parse `PR_MESSAGE.md` with the preserved strict parser, remove the untrusted source file, and stage only bounded outputs. +- [x] Mint the dedicated Maintainer App token only after all non-executing validation. Scope it to `ContextualWisdomLab/noema` with metadata read, contents write, and pull-request write. +- [x] Re-read open-PR inventory and live `main`; fail before push on unreadable inventory, any open PR, checkout mismatch, or advanced base. +- [x] Create a unique branch, disable hooks for commit, verify no remote branch collision, push once, and call `gh pr create` once. +- [x] Delete the orphan branch if PR creation fails. + +The third fresh publisher is the only write-capable runner. Proposed code never executes before or after its Maintainer App token is minted. + +## Task 5 — Treat model-generated PR metadata as untrusted input + +**Files** + +- `scripts/prepare-agent-pr-message.mjs` +- `test/agent-pr-message.test.ts` + +- [x] Require a regular non-symlink file. +- [x] Open with `O_NOFOLLOW` and validate stable inode metadata. +- [x] Decode strict UTF-8 and reject malformed encoding, unsupported control characters, and bidirectional control characters. +- [x] Normalize line endings. +- [x] Enforce a 120-byte title and 20,000-byte body. +- [x] Write trusted title/body files with owner-only permissions. +- [x] Cover realistic Unicode, byte-budget, encoding, control-character, file-mode, and symlink cases. + +**Evidence:** bounded metadata RED run `30967569870` and stale-base/metadata race run `30967373769`. + +## Task 6 — Add immutable artifact and Git object boundary tests + +**Files** + +- `test/hourly-product-development-runner-isolation.test.ts` +- `test/hourly-product-development-git-mode-boundary.test.ts` + +- [x] Prove the verifier contains release verification but no Maintainer App credential or remote mutation. +- [x] Prove the publisher depends on both prior jobs, downloads by `artifact-id`, validates without executing proposed code, parses metadata before token mint, and revalidates queue/base after token mint. +- [x] Prove both fresh jobs use the upload action's `artifact-id` and `artifact-digest` outputs. +- [x] Prove symlink and gitlink modes are rejected at all three boundaries. +- [x] Prove the design specification and this plan remain aligned with the implemented three-runner architecture and contain no superseded same-job write-permission claims. + +## Task 7 — Document operations and scientific/standards rationale + +**Files** + +- `docs/operations/hourly-product-development.md` +- `docs/doctoring/hourly-nim-opencode-development.md` +- `README.md` +- `CHANGELOG.md` + +- [x] Document schedule, dry run, zero-open-PR behavior, model fallback, exact evidence, three runner roles, late-bound App token, generated branch, PR handoff, no-op reasons, enablement, and rollback. +- [x] Record GitHub job/runner lifetime, compromised-runner threat, artifact immutability and digest outputs, GitHub App token scoping, recursive workflow trigger behavior, OpenCode, NVIDIA NIM, and NIST SP 800-218 sources in APA 7 form. +- [x] Separate source-supported facts, Noema decisions, assumptions, and residual risk. +- [x] Update `CHANGELOG.md` without a version bump because this PR does not publish immutable release, production deployment, strict KPI, customer, revenue, or transfer evidence. + +## Task 8 — Exact-head review, repair, and merge + +- [x] Mark the PR ready and inspect every open PR exact head. +- [x] Resolve the stale GitHub Advanced Security token-permission findings by keeping all job-level `GITHUB_TOKEN` permission sets read-only and using the late-bound repository-scoped Maintainer App token. +- [x] Retain and address the three-runner isolation RED contract. +- [x] Retain and address the Git symlink/gitlink boundary contract. +- [x] Run current-head `ci`, `reviewer-ci`, and Security Scan after every implementation change. +- [ ] Obtain substantive CodeRabbit review for the final exact head. +- [ ] Obtain independent OpenCode or Noema `APPROVE` for the final exact head. +- [ ] Confirm zero unresolved threads, current-head required Checks, repository rules, and mergeability. +- [ ] Squash-merge with `expected_head_sha` and verify the open PR count returns to zero. + +Queued, cancelled, stale, predecessor-head, metadata-only, self-review, or model-status evidence is not accepted as success. + +## Final verification commands + +```bash +npx vitest run test/hourly-product-development-workflow.test.ts +npx vitest run test/hourly-product-development-runner-isolation.test.ts +npx vitest run test/hourly-product-development-git-mode-boundary.test.ts +npx vitest run test/agent-pr-message.test.ts +npm run release:verify +git diff --check +``` + +Expected result: every command exits zero; production statements, branches, functions, and lines remain 100%; reviewer line/branch and docstring gates remain 100%; dependency and security scans pass; no reviewer credential name changes; and no merge, release, or deployment authority exists in the product-development workflow. + +## Residual risk and follow-up + +- The NIM key necessarily exists in the OpenCode process. A future narrow inference broker could keep the upstream credential outside the model process. +- Repository source may be processed by NVIDIA NIM and requires operator confidentiality, retention, regional, and contractual review. +- The verifier executes untrusted code on an ephemeral hosted runner with outbound network access under GitHub policy, but it receives no publication, NIM, OIDC, artifact/cache runtime, or command-file credentials. +- The handoff trusts GitHub artifact storage, metadata, hosted runners, and pinned actions. Exact IDs and digests detect exposed mismatch but do not prove semantic correctness. +- GitHub cannot atomically create a PR only when no PR exists. Final queue/base revalidation, unique branches, branch protection, and exact-head governance bound the race. +- A future reusable `ContextualWisdomLab/.github` workflow must preserve this three-runner boundary and allow Noema to remain independently operable. diff --git a/docs/superpowers/specs/2026-08-05-hourly-nim-opencode-development-design.md b/docs/superpowers/specs/2026-08-05-hourly-nim-opencode-development-design.md new file mode 100644 index 000000000..857b70f59 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-hourly-nim-opencode-development-design.md @@ -0,0 +1,136 @@ +# Hourly NVIDIA NIM OpenCode Development Design + +## Status + +Implemented on PR #64 and updated after the runner-isolation security review. This specification supersedes the initial single-job and two-job packaging concepts. The authoritative design uses three ordered GitHub-hosted jobs so no runner that executes model-proposed code later receives publication credentials. + +## Problem + +Noema already has a deterministic commercial-readiness workflow that governs existing pull requests. When the queue is empty, beginning the next bounded commercial increment still depends on a person or an interactive agent. The new scheduler must safely create a reviewable proposal without combining model generation, untrusted execution, repository publication, independent review, or merge authority. + +The principal trust-boundary risk is runner persistence. GitHub Actions steps in one job share a runner and filesystem. Removing credentials from one shell step does not prove that untrusted code cannot leave a resident process or modified runner state that observes credentials introduced later in the same job. Job separation, not environment cleanup alone, is therefore the security boundary. + +## Goals + +1. Start at most one bounded product-development proposal each hour and only when GitHub reports zero open pull requests. +2. Run checksum-pinned OpenCode with `NVIDIA_NIM_API_KEY`, explicit NVIDIA NIM fallback, and no reviewer-credential reuse. +3. Keep model execution unable to commit, push, create a pull request, approve, merge, release, or deploy. +4. Execute the proposal's complete release verification on a fresh runner that never receives publication credentials. +5. Reconstruct the exact verified proposal on a third fresh non-executing publisher before minting a least-privilege Maintainer App token. +6. Bind every cross-job handoff to exact artifact identity, workflow-run identity, archive digest, patch digest, base SHA, changed-file count, and byte count. +7. Preserve Noema standalone operation, modular MSA compatibility, exact-head review governance, and release restraint. + +## Non-goals + +- The workflow does not review, approve, merge, release, publish a release, deploy, or fabricate production or commercial evidence. +- It does not use GitHub Copilot, GitHub Models, the Noema reviewer App key, `NOEMA_LLM_API_KEY`, or production `contextual-orchestrator` reviewer credentials. +- It does not claim hostile-code microVM isolation or eliminate GitHub platform trust. +- It does not create a new repository or prematurely move the Noema-specific prompt into `ContextualWisdomLab/.github`. +- It has no user-facing visual flow; Figma and Product Design are not material to this increment. + +## Control-plane separation + +`.github/workflows/hourly-commercial-readiness.yml` remains authoritative for pull-request inventory, exact-head Checks, review evidence, unresolved threads, repository rules, and SHA-bound merge. + +`.github/workflows/hourly-product-development.yml` is proposal-only. It schedules at minute 47, offset from the minute-17 governance loop, uses a non-cancelling repository concurrency group, and opens at most one pull request. A generated PR enters the normal review → repair → exact-head Checks → merge process. + +## Three-runner architecture + +### 1. `propose_product_increment` + +This read-only job performs the queue gate, runs OpenCode, executes an initial complete release verification, and exports one bounded patch artifact. + +Permissions are `contents: read` and `pull-requests: read`. Checkout uses `persist-credentials: false`. The OpenCode subprocess receives `NVIDIA_API_KEY` only and removes GitHub tokens, Actions OIDC variables, Actions artifact/cache runtime variables, and runner command-file variables. + +A successful proposal is staged and serialized with: + +```bash +git diff --cached --binary --full-index +``` + +The job rejects whitespace errors, symlink mode `120000`, gitlink mode `160000`, more than 40 changed files, or more than 500,000 patch bytes. It records the exact base SHA, patch SHA-256, changed-file count, and byte count. + +The full-SHA-pinned upload action has `id: upload_proposal`. The job exports both: + +- `artifact-id` — the immutable GitHub artifact object identity; +- `artifact-digest` — the upload action's SHA-256 archive digest. + +The artifact name includes workflow run ID and attempt, retention is one day, compression is disabled for predictable handling, and overwrite is false. + +### 2. `package_product_increment` + +This fresh uncredentialed verifier depends on `propose_product_increment`. It has read-only repository, pull-request, and Actions permissions and receives neither the NIM credential nor Maintainer App credentials. + +It downloads the proposal by exact `artifact-id`, queries the GitHub REST artifact object, and verifies: + +- numeric artifact ID; +- deterministic artifact name; +- non-expired state; +- originating workflow-run ID; +- REST `sha256:` digest against the exported `artifact-digest`; +- patch SHA-256, byte count, file count, and exact base SHA; +- absence of symlink and gitlink modes. + +After applying the patch, it installs dependencies without lifecycle scripts and runs `npm run release:verify` with GitHub, OIDC, Actions runtime/cache, and runner command-file channels removed and with an isolated temporary home. Verification must not mutate tracked or non-ignored untracked proposal files, and the post-verification staged patch digest and size must remain identical. + +This job deliberately executes proposed code. It never contains `actions/create-github-app-token`, Maintainer App variables or secrets, `git push`, or `gh pr create`. + +### 3. `publish_product_increment` + +This third fresh runner depends on successful proposal and verification jobs. Its job-level `GITHUB_TOKEN` remains read-only. It receives no NIM credential and does not install dependencies or execute proposed tests, builds, package scripts, binaries, or shell commands. + +Before applying the patch, it copies the trusted exact-base PR metadata parser into `RUNNER_TEMP`. It then downloads the same exact `artifact-id` and independently repeats artifact identity, workflow-run, `artifact-digest`, patch digest, base, file, byte, symlink, and gitlink checks. + +The publisher applies the patch only as Git data. It parses `PR_MESSAGE.md` through the preserved trusted parser, removes the untrusted metadata source from the staged commit, and validates the resulting staged tree without executing proposed code. + +Only after these non-executing checks does the full-SHA-pinned GitHub-maintained token action mint a short-lived repository-scoped Maintainer App token with metadata read, contents write, and pull-request write. Trusted shell steps then re-read the open-PR inventory and live `main` SHA, create a unique branch with hooks disabled, verify the remote branch is absent, push once, and call `gh pr create` once. An error trap removes an orphan branch if PR creation fails. + +The third fresh publisher is the only write-capable runner, and publication authority appears only after the immutable proposal has been validated without executing it. + +## Fail-closed queue and base policy + +The proposal job does nothing when PR inventory is unreadable, an open PR exists, or the NIM secret is absent. Stable reasons are recorded for automation evidence. + +The publisher repeats the queue and live default-branch checks immediately before remote mutation. A new PR, unreadable inventory, mismatched checkout, or advanced `main` fails before push. GitHub does not provide an atomic “create a PR only if none exists” transaction; the remaining narrow race is bounded by unique branches, branch protection, and exact-head governance rather than represented as eliminated. + +## OpenCode and NVIDIA NIM boundary + +The workflow downloads OpenCode 1.17.13 from the official release archive and verifies the reviewed SHA-256 before installation. The generated config enables only the NVIDIA NIM OpenAI-compatible provider, disables sharing, auto-update, MCP, LSP, external-directory access, subagents, interactive questions, web fetch, and web search, and denies common repository and network mutation commands. + +Fallback order is: + +1. `nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5` +2. `nvidia-nim/nvidia/nemotron-3-super-120b-a12b` +3. `nvidia-nim/deepseek-ai/deepseek-v4-pro` + +Each candidate has a 900-second execution timeout and a separate 30-second forced-termination grace period. When another candidate remains, failure causes hard reset and ignored/untracked cleanup, followed by a dependency reinstall with its own 60-second timeout and 10-second kill grace. The fallback proceeds only after that inter-candidate reinstall succeeds. A failed or timed-out reinstall fails closed immediately and prevents later candidates from running with an incomplete dependency tree. After the final candidate fails, the workflow skips reset, cleanup, and reinstall and emits the stable all-candidates-failed diagnostic. The conservative proposer budget is `3 × (900 + 30) + 2 × (60 + 10) + 300 = 3,230 seconds`, which fits inside the 3,300-second job limit with 70 seconds of explicit slack. + +Command restrictions are defense in depth. The decisive repository-security properties are read-only job permissions, absent write credentials, non-persisted checkout credentials, immutable cross-job evidence, a credential-free verifier, and a non-executing publisher. + +## Product-development contract + +The model prompt requires one bounded buyer-visible increment, observed RED-to-GREEN evidence, realistic Noema-specific tests, 100% production statements, branches, functions, and lines, reviewer coverage/docstrings when reviewer Python changes, beginner-readable public documentation, APA 7 doctoring, modular compatibility with `ContextualWisdomLab/.github`, `naruon`, and `contextual-orchestrator`, descriptive two-word-or-longer `snake_case` database objects, `CHANGELOG.md`, and Semantic Versioning restraint. + +It forbids merge, release, deployment, reviewer-key mutation, unrelated refactoring, weakened gates, and fabricated production KPI, customer, revenue, transfer, attestation, or acquisition evidence. + +## Untrusted metadata contract + +`PR_MESSAGE.md` is untrusted model output. The trusted base parser requires a regular non-symlink file, opens with `O_NOFOLLOW`, verifies inode stability, decodes strict UTF-8, rejects unsupported control and bidirectional characters, normalizes line endings, and enforces a 120-byte title and 20,000-byte body. Trusted outputs use owner-only permissions. + +## Verification and evidence + +Static tests prove the three job names and ordering, no credential action in the verifier, no release verification in the publisher, exact `artifact-id` and `artifact-digest` use, symlink/gitlink rejection at every boundary, token mint order, queue/base revalidation order, and alignment of this design and the implementation plan. + +The complete PR head must pass `ci`, `reviewer-ci`, Security Scan, 100% production coverage, reviewer coverage/docstrings, dependency audit, independent exact-head approval, unresolved-thread checks, and repository rules. No predecessor-head or queued result is accepted. + +## Modularity and extraction + +The workflow remains locally owned because the prompt and evidence contract are Noema-specific. Its interfaces are intentionally extractable: repository identity, default branch, NIM secret mapping, model candidates, bounded prompt, patch evidence, Maintainer App identity, and governance handoff. A future central reusable workflow must be full-SHA pinned and preserve the same three-runner boundary; it must not make Noema dependent on central availability for standalone operation. + +## Residual risk + +The NIM credential necessarily exists in the model process, and repository content may be transmitted to NVIDIA NIM. Operators must assess confidentiality, retention, regional, and contractual requirements. + +The verifier executes untrusted proposed code on an ephemeral GitHub-hosted runner with outbound network available under GitHub policy. The design limits the consequence by withholding publication, NIM, OIDC, artifact/cache runtime, and command-file credentials. It is not a hostile-code microVM. + +The handoff trusts GitHub Actions artifact storage, artifact metadata, hosted runners, and pinned actions. Exact IDs, workflow-run binding, archive and patch digests, and independent reconstruction detect exposed mismatches but do not prove semantic correctness or eliminate platform compromise. Independent exact-head review and branch protection remain mandatory. diff --git a/scripts/prepare-agent-pr-message.mjs b/scripts/prepare-agent-pr-message.mjs new file mode 100644 index 000000000..3657e06d2 --- /dev/null +++ b/scripts/prepare-agent-pr-message.mjs @@ -0,0 +1,287 @@ +#!/usr/bin/env node +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const controlCharacterPattern = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f\u0080-\u009f\u202a-\u202e\u2066-\u2069]/u; +const markdownHeadingPattern = /^#{1,6}[\t ]*/u; +const positiveIntegerPattern = /^[1-9][0-9]*$/u; + +/** + * Synchronous filesystem operations used by the trusted metadata adapter. + * + * The object is injectable so tests can deterministically exercise descriptor + * replacement, short-read, and cleanup failures without racing the host + * filesystem. Production always passes this frozen implementation. + */ +export const defaultAgentPrMessageFileSystem = Object.freeze({ + closeSync, + fstatSync, + lstatSync, + openSync, + readFileSync, + writeFileSync, +}); + +/** + * Return the number of UTF-8 wire bytes required for a text value. + * + * @param {string} value Text whose encoded size is required. + * @returns {number} Exact UTF-8 byte length. + */ +export function utf8Length(value) { + return Buffer.byteLength(value, "utf8"); +} + +/** + * Normalize CRLF and legacy CR line endings without changing other content. + * + * @param {string} value Decoded metadata text. + * @returns {string} Text using LF line endings only. + */ +export function normalizeLineEndings(value) { + return value.replace(/\r\n?/gu, "\n"); +} + +/** + * Parse one required positive decimal byte limit from the environment. + * + * @param {string | undefined} raw Untrusted environment-variable value. + * @param {string} name Variable name used in the bounded diagnostic. + * @returns {number} Validated positive safe-integer limit. + * @throws {Error} When the value is absent, unsafe, zero, negative, or non-decimal. + */ +export function parsePositiveLimit(raw, name) { + if (!positiveIntegerPattern.test(raw ?? "")) { + throw new Error(`${name} must be a positive decimal integer`); + } + const value = Number(raw); + if (!Number.isSafeInteger(value)) { + throw new Error(`${name} must be a positive decimal integer`); + } + return value; +} + +/** + * Parse bounded, model-generated pull-request metadata without replacement decoding. + * + * @param {Uint8Array} bytes Raw PR_MESSAGE.md bytes. + * @param {{maxTitleBytes: number, maxBodyBytes: number}} limits UTF-8 byte budgets. + * @returns {{title: string, body: string}} Normalized title and Markdown body. + * @throws {Error} When metadata is malformed, unsafe, empty, or over budget. + */ +export function parseAgentPrMessage(bytes, limits) { + let decoded; + try { + decoded = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error("PR_MESSAGE.md must be valid UTF-8"); + } + + const normalized = normalizeLineEndings(decoded); + if (controlCharacterPattern.test(normalized)) { + throw new Error("PR metadata contains unsupported control characters"); + } + + const [rawTitle = "", ...bodyLines] = normalized.split("\n"); + const title = rawTitle.replace(markdownHeadingPattern, "").trim(); + const body = bodyLines.join("\n").trim(); + + if (!title || utf8Length(title) > limits.maxTitleBytes) { + throw new Error("PR title is empty or exceeds the byte budget"); + } + if (utf8Length(body) > limits.maxBodyBytes) { + throw new Error("PR body exceeds the byte budget"); + } + return { title, body }; +} + +/** + * Read a bounded regular file while rejecting symlinks and replacement races. + * + * @param {string} path Source metadata path. + * @param {number} maximumBytes Maximum accepted file size. + * @param {{lstatSync: Function, openSync: Function, fstatSync: Function, readFileSync: Function, closeSync: Function}} fileSystem Injectable synchronous filesystem operations. + * @returns {Buffer | Uint8Array} Exact bytes read from the validated descriptor. + * @throws {Error} When the path is not a stable bounded regular file. + */ +export function readRegularFileWithoutFollowingSymlinks( + path, + maximumBytes, + fileSystem, +) { + const linkMetadata = fileSystem.lstatSync(path); + if (!linkMetadata.isFile()) { + throw new Error("PR_MESSAGE.md must be a regular non-symlink file"); + } + if (linkMetadata.isSymbolicLink()) { + throw new Error("PR_MESSAGE.md must be a regular non-symlink file"); + } + if (linkMetadata.size > maximumBytes) { + throw new Error("PR_MESSAGE.md exceeds the combined byte budget"); + } + + let descriptor; + try { + descriptor = fileSystem.openSync( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + const openedMetadata = fileSystem.fstatSync(descriptor); + if (!openedMetadata.isFile()) { + throw new Error("PR_MESSAGE.md changed during validation"); + } + if (openedMetadata.dev !== linkMetadata.dev) { + throw new Error("PR_MESSAGE.md changed during validation"); + } + if (openedMetadata.ino !== linkMetadata.ino) { + throw new Error("PR_MESSAGE.md changed during validation"); + } + const bytes = fileSystem.readFileSync(descriptor); + if (bytes.byteLength > maximumBytes) { + throw new Error("PR_MESSAGE.md exceeds the combined byte budget"); + } + return bytes; + } finally { + if (descriptor !== undefined) fileSystem.closeSync(descriptor); + } +} + +/** + * Create one owner-only output file without replacing an existing path. + * + * @param {string} path Trusted output path. + * @param {string} value Validated UTF-8 text to write. + * @param {{writeFileSync: Function}} fileSystem Injectable synchronous filesystem operation. + * @returns {void} + * @throws {Error} When the path already exists or cannot be written safely. + */ +export function writePrivateFile(path, value, fileSystem) { + fileSystem.writeFileSync(path, value, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); +} + +/** + * Run the strict command-line adapter used by the trusted publisher job. + * + * @param {string[] | undefined} argv Three paths: source, title output, and body output. + * @param {NodeJS.ProcessEnv | Record} environment Byte-budget environment values. + * @param {typeof defaultAgentPrMessageFileSystem} fileSystem Injectable filesystem operations. + * @returns {void} + * @throws {Error} When arguments, limits, source metadata, or outputs are invalid. + */ +export function runAgentPrMessageCli(argv, environment, fileSystem) { + if (!Array.isArray(argv) || argv.length !== 3) { + throw new Error( + "Usage: prepare-agent-pr-message.mjs PR_MESSAGE.md pr-title.txt pr-body.md", + ); + } + const [sourcePath, titlePath, bodyPath] = argv; + const maxTitleBytes = parsePositiveLimit( + environment.MAX_PR_TITLE_BYTES, + "MAX_PR_TITLE_BYTES", + ); + const maxBodyBytes = parsePositiveLimit( + environment.MAX_PR_BODY_BYTES, + "MAX_PR_BODY_BYTES", + ); + const bytes = readRegularFileWithoutFollowingSymlinks( + sourcePath, + maxTitleBytes + maxBodyBytes + 16_384, + fileSystem, + ); + const parsed = parseAgentPrMessage(bytes, { maxTitleBytes, maxBodyBytes }); + writePrivateFile(titlePath, parsed.title, fileSystem); + writePrivateFile(bodyPath, parsed.body, fileSystem); +} + +/** + * Execute the production CLI using Node process arguments and environment. + * + * @returns {void} + */ +export function executeDefaultAgentPrMessageCli() { + runAgentPrMessageCli( + process.argv.slice(2), + process.env, + defaultAgentPrMessageFileSystem, + ); +} + +/** + * Write one bounded CLI diagnostic to standard error. + * + * @param {string} message Diagnostic including its trailing newline. + * @returns {void} + */ +export function writeAgentPrMessageCliError(message) { + process.stderr.write(message); +} + +/** + * Set the process exit code without terminating synchronous cleanup. + * + * @param {number} code Non-zero failure code. + * @returns {void} + */ +export function setAgentPrMessageCliExitCode(code) { + process.exitCode = code; +} + +/** + * Resolve Node's optional executable-script argument to a canonical file URL. + * + * @param {string | undefined} entryPath Optional process argument at index one. + * @returns {string} Canonical file URL for a script path, otherwise an empty value. + */ +export function resolveAgentPrMessageInvocationUrl(entryPath) { + return entryPath === undefined + ? "" + : pathToFileURL(resolve(entryPath)).href; +} + +/** + * Execute the CLI only for a direct module invocation and bound any diagnostic. + * + * @param {boolean} invoked Whether the current module is the direct Node entrypoint. + * @param {() => void} cli Trusted CLI operation. + * @param {(message: string) => void} writeError Diagnostic sink. + * @param {(code: number) => void} setExitCode Exit-code sink. + * @returns {void} + */ +export function runAgentPrMessageEntrypoint( + invoked, + cli, + writeError, + setExitCode, +) { + if (!invoked) return; + try { + cli(); + } catch (error) { + const message = error instanceof Error + ? error.message + : "PR metadata parsing failed"; + writeError(`${message}\n`); + setExitCode(1); + } +} + +const invokedPath = resolveAgentPrMessageInvocationUrl(process.argv[1]); +runAgentPrMessageEntrypoint( + invokedPath === import.meta.url, + executeDefaultAgentPrMessageCli, + writeAgentPrMessageCliError, + setAgentPrMessageCliExitCode, +); diff --git a/test/agent-pr-message-entry-path.test.ts b/test/agent-pr-message-entry-path.test.ts new file mode 100644 index 000000000..aa02e37ef --- /dev/null +++ b/test/agent-pr-message-entry-path.test.ts @@ -0,0 +1,18 @@ +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { describe, expect, it } from "vitest"; +import { resolveAgentPrMessageInvocationUrl } from "../scripts/prepare-agent-pr-message.mjs"; + +describe("agent PR-message entry-path resolution", () => { + it("returns an empty URL when Node has no script argument", () => { + expect(resolveAgentPrMessageInvocationUrl(undefined)).toBe(""); + }); + + it("resolves the script argument to the same canonical file URL used by the entrypoint", () => { + const entryPath = "scripts/prepare-agent-pr-message.mjs"; + + expect(resolveAgentPrMessageInvocationUrl(entryPath)).toBe( + pathToFileURL(resolve(entryPath)).href, + ); + }); +}); diff --git a/test/agent-pr-message-internals.test.ts b/test/agent-pr-message-internals.test.ts new file mode 100644 index 000000000..5fc045d45 --- /dev/null +++ b/test/agent-pr-message-internals.test.ts @@ -0,0 +1,340 @@ +import { execFileSync } from "node:child_process"; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + afterEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { + executeDefaultAgentPrMessageCli, + normalizeLineEndings, + parsePositiveLimit, + readRegularFileWithoutFollowingSymlinks, + runAgentPrMessageCli, + runAgentPrMessageEntrypoint, + setAgentPrMessageCliExitCode, + utf8Length, + writeAgentPrMessageCliError, + writePrivateFile, +} from "../scripts/prepare-agent-pr-message.mjs"; + +interface SyntheticMetadata { + dev: number; + ino: number; + size: number; + isFile(): boolean; + isSymbolicLink(): boolean; +} + +const temporaryDirectories: string[] = []; + +/** Create and register one private temporary directory for filesystem tests. */ +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), "noema-agent-pr-internals-")); + temporaryDirectories.push(directory); + return directory; +} + +/** Build deterministic file metadata for descriptor and replacement-boundary tests. */ +function metadata( + overrides: Partial<{ + dev: number; + ino: number; + size: number; + file: boolean; + symlink: boolean; + }> = {}, +): SyntheticMetadata { + return { + dev: overrides.dev ?? 11, + ino: overrides.ino ?? 13, + size: overrides.size ?? 4, + isFile: () => overrides.file ?? true, + isSymbolicLink: () => overrides.symlink ?? false, + }; +} + +/** Build an injectable synchronous filesystem double with overridable operations. */ +function fileSystem(overrides: Record = {}) { + const linked = metadata(); + return { + lstatSync: vi.fn(() => linked), + openSync: vi.fn(() => 7), + fstatSync: vi.fn(() => linked), + readFileSync: vi.fn(() => Buffer.from("safe")), + closeSync: vi.fn(), + writeFileSync: vi.fn(), + ...overrides, + }; +} + +describe("agent PR metadata internal safety contracts", () => { + afterEach(() => { + vi.restoreAllMocks(); + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("measures UTF-8 bytes and normalizes legacy line endings", () => { + expect(utf8Length("가a")).toBe(4); + expect(normalizeLineEndings("a\r\nb\rc\n")).toBe("a\nb\nc\n"); + }); + + it("accepts only finite positive safe decimal limits", () => { + expect(parsePositiveLimit("120", "LIMIT")).toBe(120); + + for (const value of [undefined, "", "0", "-1", "1.5", "01", "9".repeat(32)]) { + expect(() => parsePositiveLimit(value, "LIMIT")).toThrow( + "LIMIT must be a positive decimal integer", + ); + } + }); + + it.each([ + { linked: metadata({ file: false }), label: "non-file" }, + { linked: metadata({ symlink: true }), label: "symlink" }, + ])("rejects a $label before opening it", ({ linked }) => { + const fs = fileSystem({ lstatSync: vi.fn(() => linked) }); + + expect(() => readRegularFileWithoutFollowingSymlinks("input", 10, fs)).toThrow( + "PR_MESSAGE.md must be a regular non-symlink file", + ); + expect(fs.openSync).not.toHaveBeenCalled(); + }); + + it("rejects an oversized path before opening it", () => { + const fs = fileSystem({ lstatSync: vi.fn(() => metadata({ size: 11 })) }); + + expect(() => readRegularFileWithoutFollowingSymlinks("input", 10, fs)).toThrow( + "PR_MESSAGE.md exceeds the combined byte budget", + ); + expect(fs.openSync).not.toHaveBeenCalled(); + }); + + it.each([ + { opened: metadata({ file: false }), label: "non-file descriptor" }, + { opened: metadata({ dev: 99 }), label: "device replacement" }, + { opened: metadata({ ino: 99 }), label: "inode replacement" }, + ])("rejects a $label and closes the descriptor", ({ opened }) => { + const fs = fileSystem({ fstatSync: vi.fn(() => opened) }); + + expect(() => readRegularFileWithoutFollowingSymlinks("input", 10, fs)).toThrow( + "PR_MESSAGE.md changed during validation", + ); + expect(fs.closeSync).toHaveBeenCalledWith(7); + }); + + it("rejects bytes that grow beyond the post-open budget", () => { + const fs = fileSystem({ + readFileSync: vi.fn(() => Buffer.from("too-large")), + }); + + expect(() => readRegularFileWithoutFollowingSymlinks("input", 4, fs)).toThrow( + "PR_MESSAGE.md exceeds the combined byte budget", + ); + expect(fs.closeSync).toHaveBeenCalledWith(7); + }); + + it("returns exact bytes and closes the validated descriptor", () => { + const expected = Buffer.from("safe"); + const fs = fileSystem({ readFileSync: vi.fn(() => expected) }); + + expect(readRegularFileWithoutFollowingSymlinks("input", 4, fs)).toBe(expected); + expect(fs.closeSync).toHaveBeenCalledWith(7); + }); + + it("does not close an unassigned descriptor when opening fails", () => { + const fs = fileSystem({ + openSync: vi.fn(() => { + throw new Error("open failed"); + }), + }); + + expect(() => readRegularFileWithoutFollowingSymlinks("input", 10, fs)).toThrow( + "open failed", + ); + expect(fs.closeSync).not.toHaveBeenCalled(); + }); + + it("writes owner-only output without replacement", () => { + const fs = fileSystem(); + + writePrivateFile("output", "validated", fs); + + expect(fs.writeFileSync).toHaveBeenCalledWith("output", "validated", { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + }); + + it.each([ + { args: undefined }, + { args: "source title body" }, + { args: [] }, + { args: ["source"] }, + { args: ["source", "title"] }, + { args: ["source", "title", "body", "extra"] }, + ] as Array<{ args: unknown }>)( + "rejects invalid CLI arguments $args", + ({ args }) => { + const fs = fileSystem(); + + expect(() => runAgentPrMessageCli(args as string[], {}, fs)).toThrow( + "Usage: prepare-agent-pr-message.mjs PR_MESSAGE.md pr-title.txt pr-body.md", + ); + }, + ); + + it("can be imported when Node has no script argv entry", () => { + const moduleUrl = new URL( + "../scripts/prepare-agent-pr-message.mjs", + import.meta.url, + ).href; + + expect(() => execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `process.argv.length = 1; await import(${JSON.stringify(moduleUrl)});`, + ], + { encoding: "utf8" }, + )).not.toThrow(); + }); + + it("validates limits, reads once, and writes both CLI outputs", () => { + const writes: Array<[string, string, Record]> = []; + const fs = fileSystem({ + lstatSync: vi.fn(() => metadata({ size: 31 })), + fstatSync: vi.fn(() => metadata({ size: 31 })), + readFileSync: vi.fn(() => Buffer.from("feat: direct contract\nBody")), + writeFileSync: vi.fn((path, value, options) => { + writes.push([String(path), String(value), options]); + }), + }); + + runAgentPrMessageCli( + ["source", "title-output", "body-output"], + { MAX_PR_TITLE_BYTES: "120", MAX_PR_BODY_BYTES: "20000" }, + fs, + ); + + expect(writes).toEqual([ + ["title-output", "feat: direct contract", { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }], + ["body-output", "Body", { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }], + ]); + }); + + it("executes the default CLI against the process arguments and environment", () => { + const directory = temporaryDirectory(); + const source = join(directory, "PR_MESSAGE.md"); + const titlePath = join(directory, "title.txt"); + const bodyPath = join(directory, "body.md"); + writeFileSync(source, "feat: default adapter\nDefault body", { mode: 0o600 }); + const previousArgv = process.argv; + const previousTitleLimit = process.env.MAX_PR_TITLE_BYTES; + const previousBodyLimit = process.env.MAX_PR_BODY_BYTES; + + try { + process.argv = [process.execPath, "prepare-agent-pr-message.mjs", source, titlePath, bodyPath]; + process.env.MAX_PR_TITLE_BYTES = "120"; + process.env.MAX_PR_BODY_BYTES = "20000"; + + executeDefaultAgentPrMessageCli(); + + expect(readFileSync(titlePath, "utf8")).toBe("feat: default adapter"); + expect(readFileSync(bodyPath, "utf8")).toBe("Default body"); + } finally { + process.argv = previousArgv; + if (previousTitleLimit === undefined) delete process.env.MAX_PR_TITLE_BYTES; + else process.env.MAX_PR_TITLE_BYTES = previousTitleLimit; + if (previousBodyLimit === undefined) delete process.env.MAX_PR_BODY_BYTES; + else process.env.MAX_PR_BODY_BYTES = previousBodyLimit; + } + }); + + it("writes the bounded production diagnostic", () => { + const write = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + writeAgentPrMessageCliError("bounded failure\n"); + + expect(write).toHaveBeenCalledWith("bounded failure\n"); + }); + + it("sets the production process exit code", () => { + const previousExitCode = process.exitCode; + try { + setAgentPrMessageCliExitCode(1); + expect(process.exitCode).toBe(1); + } finally { + process.exitCode = previousExitCode; + } + }); + + it("does nothing for an imported module", () => { + const cli = vi.fn(); + const writeError = vi.fn(); + const setExitCode = vi.fn(); + + runAgentPrMessageEntrypoint(false, cli, writeError, setExitCode); + + expect(cli).not.toHaveBeenCalled(); + expect(writeError).not.toHaveBeenCalled(); + expect(setExitCode).not.toHaveBeenCalled(); + }); + + it("runs a direct invocation without emitting an error", () => { + const cli = vi.fn(); + const writeError = vi.fn(); + const setExitCode = vi.fn(); + + runAgentPrMessageEntrypoint(true, cli, writeError, setExitCode); + + expect(cli).toHaveBeenCalledOnce(); + expect(writeError).not.toHaveBeenCalled(); + expect(setExitCode).not.toHaveBeenCalled(); + }); + + it("reports Error diagnostics without a stack trace", () => { + const writeError = vi.fn(); + const setExitCode = vi.fn(); + + runAgentPrMessageEntrypoint(true, () => { + throw new Error("bounded failure"); + }, writeError, setExitCode); + + expect(writeError).toHaveBeenCalledWith("bounded failure\n"); + expect(setExitCode).toHaveBeenCalledWith(1); + }); + + it("uses a stable diagnostic for non-Error failures", () => { + const writeError = vi.fn(); + const setExitCode = vi.fn(); + + runAgentPrMessageEntrypoint(true, () => { + throw "untrusted failure"; + }, writeError, setExitCode); + + expect(writeError).toHaveBeenCalledWith("PR metadata parsing failed\n"); + expect(setExitCode).toHaveBeenCalledWith(1); + }); +}); diff --git a/test/agent-pr-message-quality-policy.test.ts b/test/agent-pr-message-quality-policy.test.ts new file mode 100644 index 000000000..efc36d1d8 --- /dev/null +++ b/test/agent-pr-message-quality-policy.test.ts @@ -0,0 +1,41 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const sourcePath = "scripts/prepare-agent-pr-message.mjs"; + +function functionNames(source: string): string[] { + return Array.from( + source.matchAll(/^(?:export )?function ([A-Za-z_$][\w$]*)\s*\(/gmu), + (match) => match[1], + ); +} + +function hasLeadingJsDoc(source: string, functionName: string): boolean { + const declaration = new RegExp( + `^(?:export )?function ${functionName.replace(/[$]/gu, "\\$")}\\s*\\(`, + "mu", + ).exec(source); + if (declaration?.index === undefined) return false; + + const prefix = source.slice(0, declaration.index).trimEnd(); + if (!prefix.endsWith("*/")) return false; + const commentStart = prefix.lastIndexOf("/**"); + const previousCommentEnd = prefix.lastIndexOf("*/", prefix.length - 3); + return commentStart > previousCommentEnd; +} + +describe("agent PR metadata production-quality policy", () => { + it("includes the production parser in the enforced 100 percent coverage set", () => { + const configuration = readFileSync("vitest.config.ts", "utf8"); + + expect(configuration).toContain(`"${sourcePath}"`); + }); + + it("documents every production function with a JSDoc contract", () => { + const source = readFileSync(sourcePath, "utf8"); + const names = functionNames(source); + + expect(names).not.toHaveLength(0); + expect(names.filter((name) => !hasLeadingJsDoc(source, name))).toEqual([]); + }); +}); diff --git a/test/agent-pr-message.test.ts b/test/agent-pr-message.test.ts new file mode 100644 index 000000000..e8f4399ef --- /dev/null +++ b/test/agent-pr-message.test.ts @@ -0,0 +1,143 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + lstatSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parseAgentPrMessage } from "../scripts/prepare-agent-pr-message.mjs"; + +const temporaryDirectories: string[] = []; +const encoder = new TextEncoder(); + +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), "noema-agent-pr-message-")); + temporaryDirectories.push(directory); + return directory; +} + +function parse(text: string, limits = { maxTitleBytes: 120, maxBodyBytes: 20_000 }) { + return parseAgentPrMessage(encoder.encode(text), limits); +} + +describe("model-generated pull-request metadata", () => { + afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("normalizes a realistic Markdown message with Unicode and CRLF", () => { + expect(parse( + "# feat(operations): harden buyer evidence ✅\r\n\r\n" + + "## Summary\r\n- Binds evidence to the exact release.\r\n", + )).toEqual({ + title: "feat(operations): harden buyer evidence ✅", + body: "## Summary\n- Binds evidence to the exact release.", + }); + }); + + it("rejects an empty title after removing a Markdown heading", () => { + expect(() => parse("### \nBody without a title")).toThrow( + "PR title is empty or exceeds the byte budget", + ); + }); + + it("enforces the title budget in UTF-8 bytes rather than JavaScript characters", () => { + expect(() => parse(`${"가".repeat(41)}\nBody`)).toThrow( + "PR title is empty or exceeds the byte budget", + ); + }); + + it("rejects a body that exceeds its UTF-8 byte budget", () => { + expect(() => parse(`feat: bounded title\n${"a".repeat(20_001)}`)).toThrow( + "PR body exceeds the byte budget", + ); + }); + + it.each([ + "feat: bell\u0007 injection\nBody", + "feat: valid title\nBody with null\u0000 injection", + "feat: valid title\nBody with escape\u001b injection", + ])("rejects unsupported control characters in %j", (message) => { + expect(() => parse(message)).toThrow( + "PR metadata contains unsupported control characters", + ); + }); + + it("rejects malformed UTF-8 instead of replacing invalid bytes", () => { + expect(() => parseAgentPrMessage( + new Uint8Array([0xc3, 0x28]), + { maxTitleBytes: 120, maxBodyBytes: 20_000 }, + )).toThrow("PR_MESSAGE.md must be valid UTF-8"); + }); + + it("writes bounded metadata files with owner-only permissions", () => { + const directory = temporaryDirectory(); + const source = join(directory, "PR_MESSAGE.md"); + const titlePath = join(directory, "pr-title.txt"); + const bodyPath = join(directory, "pr-body.md"); + writeFileSync(source, "feat: safe metadata\n\nVerified body\n", { mode: 0o600 }); + + const result = spawnSync( + process.execPath, + ["scripts/prepare-agent-pr-message.mjs", source, titlePath, bodyPath], + { + cwd: process.cwd(), + env: { + ...process.env, + MAX_PR_TITLE_BYTES: "120", + MAX_PR_BODY_BYTES: "20000", + }, + encoding: "utf8", + }, + ); + + expect(result.status).toBe(0); + expect(readFileSync(titlePath, "utf8")).toBe("feat: safe metadata"); + expect(readFileSync(bodyPath, "utf8")).toBe("Verified body"); + expect(statSync(titlePath).mode & 0o777).toBe(0o600); + expect(statSync(bodyPath).mode & 0o777).toBe(0o600); + }); + + it("rejects a symlink before reading model-generated metadata", () => { + const directory = temporaryDirectory(); + const target = join(directory, "target.md"); + const source = join(directory, "PR_MESSAGE.md"); + writeFileSync(target, "feat: unsafe indirection\nBody", { mode: 0o600 }); + symlinkSync(target, source); + chmodSync(target, 0o600); + + const result = spawnSync( + process.execPath, + [ + "scripts/prepare-agent-pr-message.mjs", + source, + join(directory, "pr-title.txt"), + join(directory, "pr-body.md"), + ], + { + cwd: process.cwd(), + env: { + ...process.env, + MAX_PR_TITLE_BYTES: "120", + MAX_PR_BODY_BYTES: "20000", + }, + encoding: "utf8", + }, + ); + + expect(lstatSync(source).isSymbolicLink()).toBe(true); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "PR_MESSAGE.md must be a regular non-symlink file", + ); + }); +}); diff --git a/test/helpers/hourly-workflow.ts b/test/helpers/hourly-workflow.ts new file mode 100644 index 000000000..698aae18d --- /dev/null +++ b/test/helpers/hourly-workflow.ts @@ -0,0 +1,200 @@ +const candidateModelPattern = /^ nvidia-nim\/.+$/gm; +const fallbackStepName = "- name: Run bounded NVIDIA NIM model fallback"; + +/** Seconds reserved for setup work and the stable terminal diagnostic. */ +export const SETUP_AND_DIAGNOSTIC_RESERVE_SECONDS = 300; + +/** Parsed candidate and cleanup budgets from the production workflow. */ +export interface CandidateBudget { + candidateCount: number; + candidateSeconds: number; + candidateGraceSeconds: number; + reinstallSeconds: number; + reinstallGraceSeconds: number; + interCandidateCleanupCount: number; + jobSeconds: number; + totalSeconds: number; +} + +/** Ordered offsets for the production final-candidate cleanup control flow. */ +export interface CandidateControlFlow { + candidateListIndex: number; + candidateLoopIndex: number; + finalCandidateGuardIndex: number; + resetIndex: number; + reinstallIndex: number; +} + +/** + * Return one complete job block from the workflow text. + * + * @param workflow Complete workflow YAML. + * @param jobName Job whose two-space-indented declaration starts the slice. + * @param nextJobName Optional following job that terminates the slice. + * @returns Exact job block, excluding the following job declaration. + * @throws {Error} When either requested boundary is absent or out of order. + */ +export function readJobSlice( + workflow: string, + jobName: string, + nextJobName?: string, +): string { + const start = workflow.indexOf(` ${jobName}:`); + if (start < 0) { + throw new Error(`Workflow job '${jobName}' is missing.`); + } + + if (nextJobName === undefined) { + return workflow.slice(start); + } + + const end = workflow.indexOf(` ${nextJobName}:`, start + 1); + if (end <= start) { + throw new Error( + `Workflow job '${nextJobName}' is missing after '${jobName}'.`, + ); + } + return workflow.slice(start, end); +} + +/** + * Parse one required positive integer capture from workflow text. + * + * @param text Workflow fragment to inspect. + * @param pattern Pattern whose first capture is the decimal value. + * @param label Human-readable contract name for diagnostics. + * @returns Parsed positive safe integer. + * @throws {Error} When the contract is absent or not a positive safe integer. + */ +function readPositiveCapture( + text: string, + pattern: RegExp, + label: string, +): number { + const match = text.match(pattern); + if (match === null) { + throw new Error(`Workflow ${label} is missing.`); + } + const value = Number(match[1]); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`Workflow ${label} is not a positive safe integer.`); + } + return value; +} + +/** + * Read the configured candidate, cleanup, and proposer-job budgets. + * + * @param workflow Complete workflow YAML. + * @returns Parsed budget values and their enforced worst-case total. + */ +export function readCandidateBudget(workflow: string): CandidateBudget { + const proposer = readJobSlice( + workflow, + "propose_product_increment", + "package_product_increment", + ); + const candidateSeconds = readPositiveCapture( + workflow, + /OPENCODE_RUN_TIMEOUT_SECONDS: "(\d+)"/, + "candidate timeout", + ); + const candidateGraceSeconds = readPositiveCapture( + workflow, + /OPENCODE_KILL_GRACE_SECONDS: "(\d+)"/, + "candidate kill grace", + ); + const reinstallSeconds = readPositiveCapture( + workflow, + /DEPENDENCY_REINSTALL_TIMEOUT_SECONDS: "(\d+)"/, + "dependency reinstall timeout", + ); + const reinstallGraceSeconds = readPositiveCapture( + workflow, + /DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS: "(\d+)"/, + "dependency reinstall kill grace", + ); + const jobMinutes = readPositiveCapture( + proposer, + /timeout-minutes: (\d+)/, + "proposal-job timeout", + ); + const candidateCount = workflow.match(candidateModelPattern)?.length ?? 0; + const interCandidateCleanupCount = Math.max(candidateCount - 1, 0); + const jobSeconds = jobMinutes * 60; + const totalSeconds = candidateCount * ( + candidateSeconds + candidateGraceSeconds + ) + interCandidateCleanupCount * ( + reinstallSeconds + reinstallGraceSeconds + ) + SETUP_AND_DIAGNOSTIC_RESERVE_SECONDS; + + return { + candidateCount, + candidateSeconds, + candidateGraceSeconds, + reinstallSeconds, + reinstallGraceSeconds, + interCandidateCleanupCount, + jobSeconds, + totalSeconds, + }; +} + +/** + * Validate and return the ordered final-candidate cleanup control flow. + * + * @param workflow Complete workflow YAML. + * @returns Ordered offsets within the fallback step. + * @throws {Error} When a required anchor is absent or cleanup can precede the + * final-candidate guard. + */ +export function readCandidateControlFlow( + workflow: string, +): CandidateControlFlow { + const proposer = readJobSlice( + workflow, + "propose_product_increment", + "package_product_increment", + ); + const fallbackStart = proposer.indexOf(fallbackStepName); + if (fallbackStart < 0) { + throw new Error("Workflow bounded fallback step is missing."); + } + const fallback = proposer.slice(fallbackStart); + const anchors = { + candidateListIndex: fallback.indexOf( + 'read -r -a model_candidates <<<"$OPENCODE_MODEL_CANDIDATES"', + ), + candidateLoopIndex: fallback.indexOf( + "for ((candidate_index = 0; candidate_index < candidate_count; candidate_index++)); do", + ), + finalCandidateGuardIndex: fallback.indexOf( + 'if [ "$candidate_index" -eq $((candidate_count - 1)) ]; then', + ), + resetIndex: fallback.indexOf( + 'git -C "$GITHUB_WORKSPACE" reset --hard HEAD', + ), + reinstallIndex: fallback.indexOf( + 'timeout --kill-after="${DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS}s" "${DEPENDENCY_REINSTALL_TIMEOUT_SECONDS}s" npm ci --ignore-scripts', + ), + }; + + for (const [label, index] of Object.entries(anchors)) { + if (index < 0) { + throw new Error(`Workflow fallback anchor '${label}' is missing.`); + } + } + + if (!( + anchors.candidateListIndex < anchors.candidateLoopIndex + && anchors.candidateLoopIndex < anchors.finalCandidateGuardIndex + && anchors.finalCandidateGuardIndex < anchors.resetIndex + && anchors.resetIndex < anchors.reinstallIndex + )) { + throw new Error( + "Workflow final-candidate guard must precede inter-candidate cleanup.", + ); + } + + return anchors; +} diff --git a/test/hourly-product-development-final-candidate-cleanup.test.ts b/test/hourly-product-development-final-candidate-cleanup.test.ts new file mode 100644 index 000000000..191e94c41 --- /dev/null +++ b/test/hourly-product-development-final-candidate-cleanup.test.ts @@ -0,0 +1,38 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { + readCandidateBudget, + readCandidateControlFlow, +} from "./helpers/hourly-workflow"; + +function workflowText(): string { + return readFileSync( + ".github/workflows/hourly-product-development.yml", + "utf8", + ); +} + +describe("hourly product-development final-candidate cleanup", () => { + it("runs bounded cleanup only between failed model candidates", () => { + const workflow = workflowText(); + const budget = readCandidateBudget(workflow); + const controlFlow = readCandidateControlFlow(workflow); + + expect(budget.candidateCount).toBe(3); + expect(budget.interCandidateCleanupCount).toBe(2); + expect(budget.totalSeconds).toBeLessThanOrEqual(budget.jobSeconds); + expect(controlFlow.candidateLoopIndex).toBeGreaterThan( + controlFlow.candidateListIndex, + ); + expect(controlFlow.finalCandidateGuardIndex).toBeGreaterThan( + controlFlow.candidateLoopIndex, + ); + expect(controlFlow.finalCandidateGuardIndex).toBeLessThan( + controlFlow.resetIndex, + ); + expect(controlFlow.resetIndex).toBeLessThan(controlFlow.reinstallIndex); + expect(workflow).not.toContain( + "for model in $OPENCODE_MODEL_CANDIDATES; do", + ); + }); +}); diff --git a/test/hourly-product-development-git-mode-boundary.test.ts b/test/hourly-product-development-git-mode-boundary.test.ts new file mode 100644 index 000000000..f5fdb4452 --- /dev/null +++ b/test/hourly-product-development-git-mode-boundary.test.ts @@ -0,0 +1,172 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const workflowPath = ".github/workflows/hourly-product-development.yml"; +const temporaryRepositories: string[] = []; + +const rawModeGate = `git diff --cached --raw | awk ' + /^:/ { + old_mode = substr($1, 2) + new_mode = $2 + if (old_mode ~ /^(120000|160000)$/ || new_mode ~ /^(120000|160000)$/) { + found = 1 + } + } + END { exit(found ? 0 : 1) } + '`; + +type GitObjectMode = "regular" | "symlink" | "gitlink"; + +interface ModeGateProcessResult { + error?: Error; + status: number | null; + signal: NodeJS.Signals | null; +} + +/** Run one Git command in the isolated fixture repository and return trimmed output. */ +function runGit(repository: string, args: string[]): string { + return execFileSync("git", args, { + cwd: repository, + encoding: "utf8", + }).trim(); +} + +/** Stage one regular file, symbolic link, or gitlink at the shared fixture path. */ +function stageObject(repository: string, mode: GitObjectMode): void { + const itemPath = join(repository, "item"); + rmSync(itemPath, { force: true, recursive: true }); + + if (mode === "regular") { + writeFileSync(itemPath, "bounded content\n", "utf8"); + runGit(repository, ["add", "item"]); + return; + } + if (mode === "symlink") { + symlinkSync("bounded-target", itemPath); + runGit(repository, ["add", "item"]); + return; + } + + const targetCommit = runGit(repository, ["rev-parse", "HEAD"]); + runGit(repository, [ + "update-index", + "--add", + "--cacheinfo", + `160000,${targetCommit},item`, + ]); +} + +/** Create a committed temporary repository whose item begins in the requested mode. */ +function createRepository(initialMode: GitObjectMode): string { + const repository = mkdtempSync(join(tmpdir(), "noema-git-mode-boundary-")); + temporaryRepositories.push(repository); + runGit(repository, ["init", "-q"]); + runGit(repository, ["config", "user.email", "test@example.invalid"]); + runGit(repository, ["config", "user.name", "Noema Boundary Test"]); + writeFileSync(join(repository, "seed"), "seed\n", "utf8"); + runGit(repository, ["add", "seed"]); + runGit(repository, ["commit", "-qm", "seed"]); + stageObject(repository, initialMode); + runGit(repository, ["commit", "-qm", `baseline ${initialMode}`]); + return repository; +} + +/** Replace the committed item with another staged Git object mode. */ +function replaceStagedObject(repository: string, mode: GitObjectMode): void { + const itemPath = join(repository, "item"); + try { + unlinkSync(itemPath); + } catch (error) { + const code = error instanceof Error && "code" in error + ? String(error.code) + : ""; + if (code !== "ENOENT") throw error; + } + stageObject(repository, mode); +} + +/** Interpret one gate process result without mistaking abnormal termination for acceptance. */ +function interpretModeGateResult(result: ModeGateProcessResult): boolean { + if (result.error) throw result.error; + if (result.status === null || result.signal !== null) { + const termination = result.signal === null + ? "without an exit status" + : `after signal ${result.signal}`; + throw new Error(`Git mode gate terminated ${termination}`); + } + return result.status === 0; +} + +/** Execute the production AWK gate unchanged and report whether it rejects the diff. */ +function modeGateRejects(repository: string): boolean { + const result = spawnSync("bash", ["-c", rawModeGate], { + cwd: repository, + encoding: "utf8", + }); + return interpretModeGateResult(result); +} + +describe("hourly product-development Git object boundary", () => { + afterEach(() => { + for (const repository of temporaryRepositories.splice(0)) { + rmSync(repository, { recursive: true, force: true }); + } + }); + + it("checks both source and target modes for symlinks and gitlinks at every boundary", () => { + const workflow = readFileSync(workflowPath, "utf8"); + + expect(workflow.match(/git diff --cached --raw/g)).toHaveLength(3); + expect(workflow.split(rawModeGate)).toHaveLength(4); + expect(workflow).not.toContain("git diff --cached --summary"); + expect(workflow.match(/symlink or gitlink/g)).toHaveLength(3); + }); + + it.each([ + { + status: null, + signal: null, + diagnostic: "Git mode gate terminated without an exit status", + }, + { + status: null, + signal: "SIGTERM" as NodeJS.Signals, + diagnostic: "Git mode gate terminated after signal SIGTERM", + }, + ])( + "fails closed when the gate exits abnormally: $diagnostic", + ({ status, signal, diagnostic }) => { + expect(() => interpretModeGateResult({ status, signal })).toThrow( + diagnostic, + ); + }, + ); + + it.each([ + { from: "regular", to: "symlink", boundary: "new 120000 mode" }, + { from: "symlink", to: "regular", boundary: "old 120000 mode" }, + { from: "regular", to: "gitlink", boundary: "new 160000 mode" }, + { from: "gitlink", to: "regular", boundary: "old 160000 mode" }, + ] as const)( + "rejects $from to $to staged transitions through the $boundary", + ({ from, to }) => { + const repository = createRepository(from); + replaceStagedObject(repository, to); + + expect(runGit(repository, ["diff", "--cached", "--raw"])).toMatch( + /(?:120000|160000)/, + ); + expect(modeGateRejects(repository)).toBe(true); + }, + ); +}); diff --git a/test/hourly-product-development-publication-prerequisite.test.ts b/test/hourly-product-development-publication-prerequisite.test.ts new file mode 100644 index 000000000..e7f9f98db --- /dev/null +++ b/test/hourly-product-development-publication-prerequisite.test.ts @@ -0,0 +1,69 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +function workflowText(): string { + return readFileSync( + ".github/workflows/hourly-product-development.yml", + "utf8", + ); +} + +describe("hourly product-development publication prerequisites", () => { + it("fails closed before OpenCode when the Maintainer App cannot publish a proposal", () => { + const workflow = workflowText(); + const prerequisiteIndex = workflow.indexOf("maintainer_app_unavailable"); + const promptIndex = workflow.indexOf("Prepare bounded commercial-quality task"); + const checkoutIndex = workflow.indexOf( + "Check out trusted default-branch source without persisted credentials", + ); + const modelIndex = workflow.indexOf("Run bounded NVIDIA NIM model fallback"); + + expect(workflow).toContain( + "MAINTAINER_APP_CLIENT_ID_CONFIGURED: ${{ vars.NOEMA_MAINTAINER_APP_CLIENT_ID != '' }}", + ); + expect(workflow).toContain( + "MAINTAINER_APP_PRIVATE_KEY_CONFIGURED: ${{ secrets.NOEMA_MAINTAINER_APP_PRIVATE_KEY != '' }}", + ); + expect(workflow).toContain( + '[ "$MAINTAINER_APP_CLIENT_ID_CONFIGURED" != "true" ]', + ); + expect(workflow).toContain( + '[ "$MAINTAINER_APP_PRIVATE_KEY_CONFIGURED" != "true" ]', + ); + expect(workflow).toContain("reason=maintainer_app_unavailable"); + expect(workflow).toContain( + '&& [ "$DRY_RUN" != "true" ]; then', + ); + expect(prerequisiteIndex).toBeGreaterThan(-1); + expect(prerequisiteIndex).toBeLessThan(promptIndex); + expect(prerequisiteIndex).toBeLessThan(checkoutIndex); + expect(prerequisiteIndex).toBeLessThan(modelIndex); + }); + + it("documents the fail-closed publication-readiness boundary", () => { + const operations = readFileSync( + "docs/operations/hourly-product-development-prerequisites.md", + "utf8", + ); + const doctoring = readFileSync( + "docs/doctoring/hourly-product-development-prerequisites.md", + "utf8", + ); + const changelog = readFileSync("CHANGELOG.md", "utf8"); + + for (const requiredText of [ + "NOEMA_MAINTAINER_APP_CLIENT_ID", + "NOEMA_MAINTAINER_APP_PRIVATE_KEY", + "maintainer_app_unavailable", + "OpenCode", + "NVIDIA_NIM_API_KEY", + "dry_run", + ]) { + expect(operations).toContain(requiredText); + } + expect(doctoring).toContain("NIST SP 800-218"); + expect(doctoring).toContain("APA 7"); + expect(doctoring).toContain("least privilege"); + expect(changelog).toContain("maintainer_app_unavailable"); + }); +}); diff --git a/test/hourly-product-development-runner-isolation.test.ts b/test/hourly-product-development-runner-isolation.test.ts new file mode 100644 index 000000000..4dc9bb77f --- /dev/null +++ b/test/hourly-product-development-runner-isolation.test.ts @@ -0,0 +1,113 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +function workflowText(): string { + return readFileSync( + ".github/workflows/hourly-product-development.yml", + "utf8", + ); +} + +function architectureDocument(path: string): string { + return readFileSync(path, "utf8"); +} + +describe("hourly product-development runner isolation", () => { + it("never mints publication credentials on a runner that executed proposed code", () => { + const workflow = workflowText(); + const verifierIndex = workflow.indexOf(" package_product_increment:"); + const publisherIndex = workflow.indexOf(" publish_product_increment:"); + + expect(verifierIndex).toBeGreaterThan(-1); + expect(publisherIndex).toBeGreaterThan(verifierIndex); + + const verifier = workflow.slice(verifierIndex, publisherIndex); + const publisher = workflow.slice(publisherIndex); + + expect(verifier).toContain("npm run release:verify"); + expect(verifier).not.toContain("actions/create-github-app-token"); + expect(verifier).not.toContain("NOEMA_MAINTAINER_APP_CLIENT_ID"); + expect(verifier).not.toContain("NOEMA_MAINTAINER_APP_PRIVATE_KEY"); + expect(verifier).not.toContain("gh pr create"); + expect(verifier).not.toContain("git push origin"); + + expect(publisher).toMatch( + /needs:\s*\n\s*- propose_product_increment\s*\n\s*- package_product_increment/, + ); + expect(publisher).toContain( + "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093", + ); + expect(publisher).toContain("artifact-ids:"); + expect(publisher).toContain( + "Verify and apply immutable proposal without executing it", + ); + expect(publisher).toContain( + "Parse bounded untrusted pull-request metadata", + ); + expect(publisher).toContain( + "Mint dedicated maintainer App token only for publication", + ); + expect(publisher).not.toContain("npm run release:verify"); + expect(publisher).not.toContain("NVIDIA_API_KEY"); + + const applyIndex = publisher.indexOf( + "Verify and apply immutable proposal without executing it", + ); + const parserIndex = publisher.indexOf( + "Parse bounded untrusted pull-request metadata", + ); + const tokenIndex = publisher.indexOf( + "Mint dedicated maintainer App token only for publication", + ); + const revalidationIndex = publisher.indexOf( + "Revalidate queue and default-branch head", + ); + + expect(applyIndex).toBeGreaterThan(-1); + expect(parserIndex).toBeGreaterThan(applyIndex); + expect(tokenIndex).toBeGreaterThan(parserIndex); + expect(revalidationIndex).toBeGreaterThan(tokenIndex); + }); + + it("binds both fresh jobs to the same immutable upload artifact identity", () => { + const workflow = workflowText(); + + expect(workflow).toContain("id: upload_proposal"); + expect(workflow).toContain( + "artifact_id: ${{ steps.upload_proposal.outputs.artifact-id }}", + ); + expect(workflow).toContain( + "artifact_digest: ${{ steps.upload_proposal.outputs.artifact-digest }}", + ); + expect(workflow.match(/artifact-ids:/g)).toHaveLength(2); + expect(workflow).toContain("expected_artifact_digest="); + expect(workflow).toContain("downloaded_artifact_digest="); + }); + + it("keeps implementation plans aligned with the three-runner security boundary", () => { + const documents = [ + architectureDocument( + "docs/superpowers/specs/2026-08-05-hourly-nim-opencode-development-design.md", + ), + architectureDocument( + "docs/superpowers/plans/2026-08-05-hourly-nim-opencode-development.md", + ), + ]; + + for (const document of documents) { + expect(document).toContain("propose_product_increment"); + expect(document).toContain("package_product_increment"); + expect(document).toContain("publish_product_increment"); + expect(document).toContain("artifact-id"); + expect(document).toContain("artifact-digest"); + expect(document).toContain("third fresh"); + expect(document).not.toContain("The same trusted job gates"); + expect(document).not.toContain( + "The repository job has only `contents: write` and `pull-requests: write`", + ); + expect(document).not.toContain( + "Create one repository-pinned job with `contents: write`, `pull-requests: write`", + ); + } + }); +}); diff --git a/test/hourly-product-development-workflow.test.ts b/test/hourly-product-development-workflow.test.ts new file mode 100644 index 000000000..a234e35ab --- /dev/null +++ b/test/hourly-product-development-workflow.test.ts @@ -0,0 +1,457 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const workflowPath = ".github/workflows/hourly-product-development.yml"; + +function workflowText(): string { + return readFileSync(workflowPath, "utf8"); +} + +function metadataParserText(): string { + return readFileSync("scripts/prepare-agent-pr-message.mjs", "utf8"); +} + +function jobSlice( + workflow: string, + jobName: string, + nextJobName?: string, +): string { + const start = workflow.indexOf(` ${jobName}:`); + expect(start).toBeGreaterThan(-1); + const end = nextJobName === undefined + ? workflow.length + : workflow.indexOf(` ${nextJobName}:`, start + 1); + if (nextJobName !== undefined) expect(end).toBeGreaterThan(start); + return workflow.slice(start, end); +} + +describe("hourly NVIDIA NIM OpenCode product-development workflow", () => { + it("runs hourly without overlapping deterministic commercial-readiness governance", () => { + const workflow = workflowText(); + + expect(workflow).toContain("workflow_dispatch:"); + expect(workflow).toContain("dry_run:"); + expect(workflow).toContain('cron: "47 * * * *"'); + expect(workflow).toContain( + "group: hourly-nim-product-development-${{ github.repository }}", + ); + expect(workflow).toContain("cancel-in-progress: false"); + expect(workflow).toContain( + "github.repository == 'ContextualWisdomLab/noema'", + ); + expect(workflow).not.toContain('cron: "17 * * * *"'); + expect(workflow).not.toContain("pull_request_target:"); + }); + + it("separates model execution, untrusted verification, and publication authority by job", () => { + const workflow = workflowText(); + const proposer = jobSlice( + workflow, + "propose_product_increment", + "package_product_increment", + ); + const verifier = jobSlice( + workflow, + "package_product_increment", + "publish_product_increment", + ); + const publisher = jobSlice(workflow, "publish_product_increment"); + + expect(proposer).toContain( + "permissions:\n contents: read\n pull-requests: read", + ); + expect(proposer).not.toContain("contents: write"); + expect(proposer).not.toContain("pull-requests: write"); + expect(proposer).not.toContain("actions/create-github-app-token"); + + expect(verifier).toContain("needs: propose_product_increment"); + expect(verifier).toContain( + "permissions:\n actions: read\n contents: read\n pull-requests: read", + ); + expect(verifier).toContain("Re-run complete release verification"); + expect(verifier).not.toContain("NVIDIA_API_KEY"); + expect(verifier).not.toContain("NOEMA_MAINTAINER_APP_CLIENT_ID"); + expect(verifier).not.toContain("NOEMA_MAINTAINER_APP_PRIVATE_KEY"); + expect(verifier).not.toContain("actions/create-github-app-token"); + expect(verifier).not.toContain("git push origin"); + expect(verifier).not.toContain("gh pr create"); + + expect(publisher).toContain("- propose_product_increment"); + expect(publisher).toContain("- package_product_increment"); + expect(publisher).toContain( + "permissions:\n actions: read\n contents: read\n pull-requests: read", + ); + expect(publisher).not.toContain("NVIDIA_API_KEY"); + expect(publisher).toContain( + "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1", + ); + expect(publisher).toContain( + "client-id: ${{ vars.NOEMA_MAINTAINER_APP_CLIENT_ID }}", + ); + expect(publisher).toContain( + "private-key: ${{ secrets.NOEMA_MAINTAINER_APP_PRIVATE_KEY }}", + ); + expect(publisher).toContain("permission-contents: write"); + expect(publisher).toContain("permission-pull-requests: write"); + expect(publisher).toContain( + "GH_TOKEN: ${{ steps.maintainer_app.outputs.token }}", + ); + + const metadataIndex = publisher.indexOf( + "Parse bounded untrusted pull-request metadata", + ); + const tokenIndex = publisher.indexOf( + "Mint dedicated maintainer App token only for publication", + ); + const revalidationIndex = publisher.indexOf( + "Revalidate queue and default-branch head", + ); + expect(metadataIndex).toBeGreaterThan(-1); + expect(tokenIndex).toBeGreaterThan(metadataIndex); + expect(tokenIndex).toBeLessThan(revalidationIndex); + + expect(workflow).toContain( + "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", + ); + expect(workflow).toContain( + "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093", + ); + expect(workflow).toContain("retention-days: 1"); + expect(workflow).toContain("proposal.patch"); + expect(workflow).toContain("patch_sha256"); + expect(verifier).toContain("sha256sum -c -"); + expect(publisher).toContain("sha256sum -c -"); + }); + + it("fails closed before model execution when PR inventory or NIM credentials are unavailable", () => { + const workflow = workflowText(); + + expect(workflow).toContain("gh pr list"); + expect(workflow).toContain("--state open"); + expect(workflow).toContain("--limit 1"); + expect(workflow).toContain("pull_request_inventory_unavailable"); + expect(workflow).toContain("open_pull_request"); + expect(workflow).toContain("nim_api_key_unavailable"); + expect(workflow).toContain( + "NIM_CONFIGURED: ${{ secrets.NVIDIA_NIM_API_KEY != '' }}", + ); + expect(workflow).toContain("dispatch=false"); + expect(workflow).toContain("dispatch=true"); + }); + + it("uses only the dedicated NVIDIA NIM development credential", () => { + const workflow = workflowText(); + + expect(workflow).toContain( + "NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}", + ); + expect(workflow).not.toContain("secrets.NVIDIA_API_KEY"); + expect(workflow).not.toContain("NOEMA_LLM_API_KEY"); + expect(workflow).not.toContain("NOEMA_GITHUB_APP_PRIVATE_KEY"); + expect(workflow.toLowerCase()).not.toContain("copilot"); + expect(workflow).not.toContain("id-token: write"); + }); + + it("installs checksum-pinned OpenCode and configures NVIDIA NIM only", () => { + const workflow = workflowText(); + + expect(workflow).toContain('OPENCODE_VERSION: "1.17.13"'); + expect(workflow).toContain( + "157afa289d1a8d9372de0ce19ac726119b937a1f6b201808d46f06e4e59bb348", + ); + expect(workflow).toContain( + "https://github.com/anomalyco/opencode/releases/download/v${OPENCODE_VERSION}/opencode-linux-x64.tar.gz", + ); + expect(workflow).toContain('"share": "disabled"'); + expect(workflow).toContain('"lsp": false'); + expect(workflow).toContain('"mcp": {}'); + expect(workflow).toContain('"enabled_providers": ["nvidia-nim"]'); + expect(workflow).toContain( + '"baseURL": "https://integrate.api.nvidia.com/v1"', + ); + expect(workflow).toContain('"apiKey": "{env:NVIDIA_API_KEY}"'); + expect(workflow).toContain( + "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", + ); + expect(workflow).toContain( + "nvidia-nim/nvidia/nemotron-3-super-120b-a12b", + ); + expect(workflow).toContain("nvidia-nim/deepseek-ai/deepseek-v4-pro"); + expect(workflow).toContain("nvidia-nim/meta/llama-3.3-70b-instruct"); + expect(workflow).not.toContain("github-models/"); + expect(workflow).not.toContain("opencode-free/"); + }); + + it("keeps GitHub credentials and runner command files out of untrusted subprocesses", () => { + const workflow = workflowText(); + + expect(workflow).toContain("persist-credentials: false"); + expect(workflow).toContain("env -u GH_TOKEN -u GITHUB_TOKEN"); + for (const variable of [ + "REPOSITORY_TOKEN", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN", + "ACTIONS_ID_TOKEN_REQUEST_URL", + "ACTIONS_RUNTIME_TOKEN", + "ACTIONS_RUNTIME_URL", + "ACTIONS_RESULTS_URL", + "ACTIONS_CACHE_URL", + "GITHUB_ENV", + "GITHUB_OUTPUT", + "GITHUB_PATH", + "GITHUB_STATE", + "GITHUB_STEP_SUMMARY", + ]) { + expect(workflow).toContain(`-u ${variable}`); + } + expect(workflow).toContain('"external_directory": "deny"'); + expect(workflow).toContain('"task": "deny"'); + expect(workflow).toContain('"webfetch": "deny"'); + expect(workflow).toContain('"websearch": "deny"'); + expect(workflow).toContain('"git commit *": "deny"'); + expect(workflow).toContain('"git push *": "deny"'); + expect(workflow).toContain('"git tag *": "deny"'); + expect(workflow).toContain('"git remote *": "deny"'); + expect(workflow).toContain('"gh *": "deny"'); + }); + + it("fits every candidate, termination grace, cleanup, and final diagnostic inside the proposal-job budget", () => { + const workflow = workflowText(); + const proposer = jobSlice( + workflow, + "propose_product_increment", + "package_product_increment", + ); + const candidateTimeoutMatch = workflow.match( + /OPENCODE_RUN_TIMEOUT_SECONDS: "(\d+)"/, + ); + const candidateGraceMatch = workflow.match( + /OPENCODE_KILL_GRACE_SECONDS: "(\d+)"/, + ); + const reinstallTimeoutMatch = workflow.match( + /DEPENDENCY_REINSTALL_TIMEOUT_SECONDS: "(\d+)"/, + ); + const reinstallGraceMatch = workflow.match( + /DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS: "(\d+)"/, + ); + const jobTimeoutMatch = proposer.match(/timeout-minutes: (\d+)/); + const candidateCount = workflow.match(/^ nvidia-nim\/.+$/gm)?.length ?? 0; + + expect(candidateTimeoutMatch).not.toBeNull(); + expect(candidateGraceMatch).not.toBeNull(); + expect(reinstallTimeoutMatch).not.toBeNull(); + expect(reinstallGraceMatch).not.toBeNull(); + expect(jobTimeoutMatch).not.toBeNull(); + expect(candidateCount).toBe(3); + const candidateSeconds = Number(candidateTimeoutMatch?.[1]); + const candidateGraceSeconds = Number(candidateGraceMatch?.[1]); + const reinstallSeconds = Number(reinstallTimeoutMatch?.[1]); + const reinstallGraceSeconds = Number(reinstallGraceMatch?.[1]); + const jobSeconds = Number(jobTimeoutMatch?.[1]) * 60; + const boundedSetupAndDiagnosticReserve = 300; + + const interCandidateCleanupCount = Math.max(candidateCount - 1, 0); + + expect(interCandidateCleanupCount).toBe(2); + expect( + candidateCount * (candidateSeconds + candidateGraceSeconds) + + interCandidateCleanupCount * ( + reinstallSeconds + reinstallGraceSeconds + ) + + boundedSetupAndDiagnosticReserve, + ).toBeLessThanOrEqual(jobSeconds); + expect(workflow).toContain( + 'timeout --kill-after="${DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS}s" "${DEPENDENCY_REINSTALL_TIMEOUT_SECONDS}s" npm ci --ignore-scripts', + ); + expect(workflow).toContain( + "cleanup dependency reinstall failed or timed out", + ); + expect(workflow).toContain("Every NVIDIA NIM candidate failed"); + + const fallbackStart = proposer.indexOf( + "- name: Run bounded NVIDIA NIM model fallback", + ); + const fallback = proposer.slice(fallbackStart); + const candidateListIndex = fallback.indexOf( + 'read -r -a model_candidates <<<"$OPENCODE_MODEL_CANDIDATES"', + ); + const candidateLoopIndex = fallback.indexOf( + "for ((candidate_index = 0; candidate_index < candidate_count; candidate_index++)); do", + ); + const finalCandidateGuardIndex = fallback.indexOf( + 'if [ "$candidate_index" -eq $((candidate_count - 1)) ]; then', + ); + const resetIndex = fallback.indexOf( + 'git -C "$GITHUB_WORKSPACE" reset --hard HEAD', + ); + const reinstallIndex = fallback.indexOf( + 'timeout --kill-after="${DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS}s" "${DEPENDENCY_REINSTALL_TIMEOUT_SECONDS}s" npm ci --ignore-scripts', + ); + + expect(fallbackStart).toBeGreaterThan(-1); + expect( + candidateListIndex, + "fallback must materialize indexed model candidates", + ).toBeGreaterThan(-1); + expect(candidateLoopIndex).toBeGreaterThan(candidateListIndex); + expect(finalCandidateGuardIndex).toBeGreaterThan(candidateLoopIndex); + expect(finalCandidateGuardIndex).toBeLessThan(resetIndex); + expect(resetIndex).toBeLessThan(reinstallIndex); + }); + + it("cleans failed candidates, verifies twice, and packages at most one bounded pull request", () => { + const workflow = workflowText(); + + expect(workflow).toContain( + 'timeout --kill-after="${OPENCODE_KILL_GRACE_SECONDS}s" "${OPENCODE_RUN_TIMEOUT_SECONDS}s"', + ); + expect(workflow).toContain( + 'timeout --kill-after="${DEPENDENCY_REINSTALL_KILL_GRACE_SECONDS}s" "${DEPENDENCY_REINSTALL_TIMEOUT_SECONDS}s" npm ci --ignore-scripts', + ); + expect(workflow).toMatch(/git -C "\$GITHUB_WORKSPACE" reset --hard HEAD/); + expect(workflow).toMatch(/git -C "\$GITHUB_WORKSPACE" clean -fdx/); + expect(workflow).toContain('MAX_CHANGED_FILES: "40"'); + expect(workflow).toContain('MAX_DIFF_BYTES: "500000"'); + expect(workflow.match(/npm run release:verify/g)?.length).toBeGreaterThanOrEqual(2); + expect(workflow).toContain("git diff --cached --check"); + expect(workflow).toContain( + 'branch="nim-agent/product-dev-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + ); + expect(workflow.match(/gh pr create/g)).toHaveLength(1); + expect(workflow).toContain('--base "$DEFAULT_BRANCH"'); + expect(workflow).toContain('--head "$branch"'); + expect(workflow).toContain("PR_MESSAGE.md"); + expect(workflow).toContain("git status --porcelain"); + expect(workflow).toContain( + "Verification mutated tracked or untracked proposal files", + ); + expect(workflow).toContain( + "Proposal changed during fresh-runner verification", + ); + expect(workflow).toContain("core.hooksPath=/dev/null"); + expect(workflow).toContain("cleanup_remote_branch"); + expect(workflow.indexOf("npm run release:verify")).toBeLessThan( + workflow.indexOf("gh pr create"), + ); + expect(workflow).not.toMatch(/gh pr merge|gh release create|wrangler deploy/); + }); + + it("revalidates queue and base head before remote proposal mutation", () => { + const workflow = workflowText(); + const publisher = jobSlice(workflow, "publish_product_increment"); + const revalidationIndex = publisher.indexOf( + "Revalidate queue and default-branch head", + ); + const pushIndex = publisher.indexOf("git push origin"); + const createIndex = publisher.indexOf("gh pr create"); + + expect(workflow).toContain("id: base"); + expect(workflow).toContain("base_sha=$(git rev-parse HEAD)"); + expect(workflow).toContain( + "needs.propose_product_increment.outputs.base_sha", + ); + expect(workflow).toContain( + '"repos/${GITHUB_REPOSITORY}/git/ref/heads/${DEFAULT_BRANCH}"', + ); + expect(workflow).toContain( + "pull_request_inventory_unavailable_after_generation", + ); + expect(workflow).toContain("open_pull_request_after_generation"); + expect(workflow).toContain("base_branch_advanced"); + expect(revalidationIndex).toBeGreaterThan(-1); + expect(revalidationIndex).toBeLessThan(pushIndex); + expect(revalidationIndex).toBeLessThan(createIndex); + }); + + it("treats model-generated pull-request metadata as bounded untrusted input", () => { + const workflow = workflowText(); + const parser = metadataParserText(); + + expect(workflow).toContain('MAX_PR_TITLE_BYTES: "120"'); + expect(workflow).toContain('MAX_PR_BODY_BYTES: "20000"'); + expect(workflow).toContain("prepare-agent-pr-message.mjs"); + expect(workflow).toContain("pr-title.txt"); + expect(workflow).toContain("pr-body.md"); + expect(parser).toContain("lstatSync"); + expect(parser).toContain('TextDecoder("utf-8", { fatal: true })'); + expect(parser).toContain( + "PR_MESSAGE.md must be a regular non-symlink file", + ); + expect(parser).toContain("PR title is empty or exceeds the byte budget"); + expect(parser).toContain("PR body exceeds the byte budget"); + expect(parser).toContain( + "PR metadata contains unsupported control characters", + ); + }); + + it("requires a commercial-quality modular test-first Noema increment", () => { + const workflow = workflowText(); + + for (const requiredPattern of [ + /AGENTS\.md/, + /single highest-value buyer-visible/, + /test-first/, + /realistic/, + /100% production\s+statement/, + /100% production function/, + /docstring coverage/, + /APA 7/, + /docs\/doctoring/, + /contextual-orchestrator/, + /ContextualWisdomLab\/\.github/, + /naruon/, + /modular MSA/, + /two-word-or-longer snake_case/, + /CHANGELOG\.md/, + /Semantic Versioning/, + /PR_MESSAGE\.md/, + /Do not merge/, + /Do not publish/, + /Do not release/, + /Do not deploy/, + /Do not fabricate/, + ]) { + expect(workflow).toMatch(requiredPattern); + } + }); + + it("documents Korean operations, security boundaries, and review-loop ownership", () => { + const operations = readFileSync( + "docs/operations/hourly-product-development.md", + "utf8", + ); + const doctoring = readFileSync( + "docs/doctoring/hourly-nim-opencode-development.md", + "utf8", + ); + const readme = readFileSync("README.md", "utf8"); + const changelog = readFileSync("CHANGELOG.md", "utf8"); + + expect(operations.match(/[가-힣]/g)?.length ?? 0).toBeGreaterThan(1000); + for (const requiredText of [ + "hourly-product-development.yml", + "NVIDIA_NIM_API_KEY", + "OpenCode 1.17.13", + "열린 PR 0개", + "자격 증명", + "폴백", + "hourly-commercial-readiness", + "proposal.patch", + "세 번째 새 게시 runner", + "Maintainer App", + "후보별 900초", + ]) { + expect(operations).toContain(requiredText); + } + expect(doctoring).toContain("APA 7"); + expect(doctoring).toContain("OpenCode"); + expect(doctoring).toContain("NVIDIA NIM"); + expect(doctoring).toContain("GitHub Actions"); + expect(doctoring).toContain("NIST SP 800-218"); + expect(doctoring).toContain("write-capable runner"); + expect(doctoring).toContain("Maintainer App"); + expect(doctoring).toContain("900 seconds"); + expect(readme).toContain("hourly-product-development"); + expect(changelog).toContain("NVIDIA_NIM_API_KEY"); + expect(changelog).toContain("OpenCode"); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index d14e56998..258654b41 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ include: [ "src/**/*.ts", "scripts/normalize-commercial-readiness-evidence.mjs", + "scripts/prepare-agent-pr-message.mjs", ], thresholds: { lines: 100,