From f9465be85fa39f138da6289577516881609fd945 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:06:42 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20implement=20issue=20#817=20?= =?UTF-8?q?=E2=80=94=20Clean=20up=20upsert=20output=20before=20fleet=20app?= =?UTF-8?q?ly:=20conditional=20negation=20tail=20+=20migrated-comment=20de?= =?UTF-8?q?dup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/lib/gitignore-baseline.sh | 144 ++++++++++++------ .../gitignore-baseline.bats | 9 +- test/scripts/lib/gitignore-baseline.bats | 99 +++++++++++- .../sync-gitignore-baseline/dry-run.bats | 9 +- 4 files changed, 198 insertions(+), 63 deletions(-) diff --git a/scripts/lib/gitignore-baseline.sh b/scripts/lib/gitignore-baseline.sh index dd8c10f55..eb07a888f 100644 --- a/scripts/lib/gitignore-baseline.sh +++ b/scripts/lib/gitignore-baseline.sh @@ -57,47 +57,54 @@ _gib_has_markers() { # _gib_neutralize_l2 # Read candidate L2 content on stdin; print it with every line that would fight -# the L1 block removed. A line is dropped when — after CRLF and trailing-whitespace -# trimming — it exactly matches a *pattern* line of (comment and blank -# lines of the block are ignored, so they never strip a repo's own comments). This -# does double duty: +# the L1 block removed, then append a *minimal* re-allow tail. Two passes: # -# • Negations win (identical patterns): a bare re-hiding pattern the target kept -# in L2 (e.g. `*.pem` with no `!public.pem`) is identical to the block's own -# broad pattern, so it is stripped. -# • Negations win (non-identical globs): a glob-variant pattern (e.g. `**/*.pem` -# or `.env*`) that is NOT an exact match for any block line but would still -# re-hide a baseline-negated path (e.g. `!public.pem` or `!.env.example`) is -# handled by the *re-allow tail*: all baseline negation lines are re-emitted at -# the very end of the L2 output so they always evaluate after any L2 pattern — -# identical or not — and the negated paths remain re-allowed. -# • Migrate, don't duplicate: an old *unmarkered* baseline pasted into L2 has its -# pattern lines folded away (they all live in the block now), so upsert stops -# shipping a second full copy. The genuine ecosystem/OS entries the repo added -# stay put. +# Pass 1 (pure awk) drops L2 lines that duplicate the block: +# • Pattern lines: a line is dropped when — after CRLF and trailing-whitespace +# trimming — it exactly matches a *pattern* line of . This strips a +# bare re-hider the target kept in L2 (e.g. `*.pem` with no `!public.pem`, +# identical to the block's own broad pattern) and folds away an old +# *unmarkered* baseline pasted into L2 (all its pattern lines now live in the +# block, so upsert stops shipping a second copy). +# • Comment lines (#817 Part B): a comment line is dropped only when it exactly +# matches a *comment* line of the block AND is not a bare `#` separator. This +# removes the orphaned section comments a migrated unmarkered baseline leaves +# behind, while never clobbering a repo's own comments — bare `#` dividers +# (which the block also uses) are always preserved. +# Genuine per-repo ecosystem/OS entries are untouched. Runs of blank lines left +# behind by removed lines are squeezed (leading and consecutive blanks dropped) +# so migration output stays tidy. # -# Only pattern lines are matched for dropping — never comments — because block -# comments include bare `#` separators that would otherwise clobber a repo's own -# comment lines. Runs of blank lines left behind by the removed patterns are -# squeezed (leading and consecutive blanks dropped) so migration output stays tidy. -# Idempotent: exact-match lines are gone after one pass; the re-allow tail lines -# match exactly on re-run and are dropped before being re-appended. The tail is -# only appended when at least one real (non-comment, non-blank) pattern survives -# in L2 — a comment-only L2 cannot re-hide anything, so no tail is emitted and -# an already-current file stays byte-for-byte identical on re-run. +# Pass 2 (#817 Part A) appends a *conditional* re-allow tail. A glob-variant L2 +# pattern that is NOT an exact match for any block line (e.g. `**/*.pem` or +# `.env*`) can still re-hide a baseline-negated path (`!public.pem`, `!.env.example`). +# Rather than re-emit ALL ~32 baseline negations unconditionally (noise on every +# repo), we compose `block + neutralized-L2` and ask git — the authority on +# gitignore semantics — which negated paths are STILL ignored. Only those +# negations are re-emitted, in block order, so they win over the surviving L2 +# pattern. A clean L2 that re-hides nothing gets NO tail. See _gib_negation_tail. +# +# Idempotent: exact-match lines are gone after pass 1; any re-emitted tail lines +# are themselves block negation patterns, so a second pass strips them before +# pass 2 recomputes them identically. An already-current file stays byte-for-byte +# identical on re-run — and, with the conditional tail, on the FIRST application. _gib_neutralize_l2() { local block="$1" block="$(tr -d '\r' <<< "$block")" - tr -d '\r' | awk -v block="$block" ' + + local body + body="$(tr -d '\r' | awk -v block="$block" ' BEGIN { n = split(block, barr, "\n") for (i = 1; i <= n; i++) { bl = barr[i] sub(/[ \t]+$/, "", bl) - if (bl ~ /^[ \t]*$/) continue # skip blank block lines - if (bl ~ /^[ \t]*#/) continue # skip comment block lines (incl. bare #) + if (bl ~ /^[ \t]*$/) continue # skip blank block lines + if (bl ~ /^[ \t]*#/) { # block comment line + if (bl !~ /^[ \t]*#[ \t]*$/) bcomment[bl] = 1 # …dedup target unless bare # + continue + } drop[bl] = 1 - if (bl ~ /^!/) neg[++neg_count] = bl # collect negation anchors for tail } prev_blank = 1 # squeeze any blank lines that would lead the output } @@ -106,27 +113,72 @@ _gib_neutralize_l2() { key = line sub(/[ \t]+$/, "", key) if (key in drop) next + if ((key ~ /^[ \t]*#/) && (key in bcomment)) next # #817B: drop orphaned block comment is_blank = (key ~ /^[ \t]*$/) if (is_blank && prev_blank) next print line prev_blank = is_blank - if (!is_blank && !(key ~ /^[ \t]*#/)) had_pattern = 1 - } - END { - # Re-emit baseline negation anchors last so they win over any L2 pattern - # (identical or non-identical glob) that would otherwise re-hide a - # baseline-negated path (e.g. `.env*` re-hides `!.env.example`, or - # `**/*.pem` re-hides `!public.pem`). Idempotent: the appended negations - # are in `drop`, so a second pass strips them before re-appending here. - # Guard: only emit when a real pattern survived — comment-only L2 cannot - # re-hide anything, and emitting the tail would make an already-current - # file appear changed (triggering spurious REFRESH PRs). - if (had_pattern && neg_count > 0) { - if (!prev_blank) print "" - for (i = 1; i <= neg_count; i++) print neg[i] - } } - ' + ')" + + local tail + tail="$(_gib_negation_tail "$block" "$body")" + + [ -n "$body" ] && printf '%s\n' "$body" + if [ -n "$tail" ]; then + [ -n "$body" ] && printf '\n' + printf '%s\n' "$tail" + fi +} + +# _gib_negation_tail +# Print the baseline negation lines (in block order) that MUST be re-emitted +# because a surviving L2 pattern would otherwise re-hide their re-allowed path. +# Decided by real gitignore semantics: compose `block + neutralized-L2` into a +# throwaway git repo and ask `git check-ignore` which negated paths remain +# ignored; those (and only those) are re-emitted. A representative concrete path +# is derived from each negation by replacing `*` globs with a literal token. +# If git is unavailable, fall back to re-emitting every negation (the safe, +# pre-#817 behavior) so a non-identical L2 glob can never silently re-hide a +# baseline-negated path. +_gib_negation_tail() { + local block="$1" body="$2" + local negs=() paths=() bl key rep + while IFS= read -r bl; do + key="$bl" + key="${key%"${key##*[![:space:]]}"}" # rstrip trailing whitespace + case "$key" in + '!'?*) + negs+=("$key") + rep="${key#!}" + rep="${rep//\*/X}" # representative concrete path + paths+=("$rep") + ;; + esac + done <<< "$block" + [ "${#negs[@]}" -eq 0 ] && return 0 + + if ! command -v git >/dev/null 2>&1; then + printf '%s\n' "${negs[@]}" + return 0 + fi + + local d; d="$(mktemp -d)" + { + printf '%s\n' "$block" + [ -n "$body" ] && printf '%s\n' "$body" + } > "$d/.gitignore" + ( cd "$d" && git init -q 2>/dev/null ) + local ignored + ignored="$(cd "$d" && git -c core.excludesfile=/dev/null check-ignore -- "${paths[@]}" 2>/dev/null)" + rm -rf "$d" + + local i + for ((i = 0; i < ${#negs[@]}; i++)); do + if printf '%s\n' "$ignored" | grep -qxF -- "${paths[$i]}"; then + printf '%s\n' "${negs[$i]}" + fi + done } # upsert_gitignore_baseline [existing_file] diff --git a/test/scripts/compliance-remediate/gitignore-baseline.bats b/test/scripts/compliance-remediate/gitignore-baseline.bats index 2744d81fb..16d661e84 100644 --- a/test/scripts/compliance-remediate/gitignore-baseline.bats +++ b/test/scripts/compliance-remediate/gitignore-baseline.bats @@ -182,11 +182,10 @@ run_remediate() { # no mutating call, and it is recorded as a skip (not a failure). # --------------------------------------------------------------------------- @test "already-current baseline is an idempotent no-op (no PR opened)" { - # "Current" = the upsert steady state (block + neutralized L2 + negation tail - # per #809), i.e. a repo already synced — not the raw canonical. - source "${TT_REPO_ROOT}/scripts/lib/gitignore-baseline.sh" - _blk="$(mktemp "$TT_TMP/block.XXXXXX")"; gib_extract_baseline_block "$CANONICAL" > "$_blk" - GH_GITIGNORE_B64="$(upsert_gitignore_baseline "$_blk" "$CANONICAL" | b64)"; export GH_GITIGNORE_B64 + # "Current" = the raw canonical .gitignore. Per #817 the negation tail is now + # conditional, so upsert(canonical) == canonical again — the raw canonical is + # the steady state and must be detected as an already-current no-op. + GH_GITIGNORE_B64="$(b64 < "$CANONICAL")"; export GH_GITIGNORE_B64 findings="$(tt_write_finding "markets" "push-protection" "gitignore_baseline")" run_remediate "$findings" diff --git a/test/scripts/lib/gitignore-baseline.bats b/test/scripts/lib/gitignore-baseline.bats index 954527802..8dea790d7 100644 --- a/test/scripts/lib/gitignore-baseline.bats +++ b/test/scripts/lib/gitignore-baseline.bats @@ -277,8 +277,8 @@ _gib_check_ignored() { grep -qxF 'node_modules/' <<< "$gi" } -# ── #809: migrate an existing unmarkered baseline, don't duplicate it ────────── -@test "#809 TalkTerm-like: unmarkered baseline in L2 is folded into one block, not duplicated" { +# ── #809/#817: migrate an existing unmarkered baseline, don't duplicate it ───── +@test "#809/#817 TalkTerm-like: unmarkered baseline in L2 is folded into one block, no tail, genuine L2 kept" { # L2 IS a copy of the old unmarkered baseline (same secret lines, no markers), # plus genuine ecosystem/OS entries the repo legitimately added. local existing="${TT_TMP}/talkterm" @@ -300,13 +300,14 @@ EOF # Exactly one marker-wrapped block. [ "$(printf '%s\n' "$output" | grep -cxF "$GIB_BEGIN_MARKER")" -eq 1 ] [ "$(printf '%s\n' "$output" | grep -cxF "$GIB_END_MARKER")" -eq 1 ] - # Broad patterns appear exactly once (the L2 copies are folded away; they are - # not negations so they are not re-emitted in the re-allow tail). + # Broad patterns are folded into the block and appear exactly once. [ "$(printf '%s\n' "$output" | grep -cxF '.env')" -eq 1 ] [ "$(printf '%s\n' "$output" | grep -cxF '*.pem')" -eq 1 ] - # Negation anchors appear twice: once inside the block and once in the - # re-allow tail that ensures they win over any surviving L2 pattern. - [ "$(printf '%s\n' "$output" | grep -cxF '!.env.example')" -eq 2 ] + # #817: the genuine L2 (node_modules/, dist/, *.log, .DS_Store) re-hides none + # of the baseline-negated paths, so NO negation tail is appended — each + # negation appears exactly once (inside the block only). + [ "$(printf '%s\n' "$output" | grep -cxF '!.env.example')" -eq 1 ] + [ "$(printf '%s\n' "$output" | grep -cxF '!*.pub')" -eq 1 ] # Genuine ecosystem/OS L2 entries are preserved. grep -qxF 'node_modules/' <<< "$output" grep -qxF 'dist/' <<< "$output" @@ -314,6 +315,90 @@ EOF grep -qxF '.DS_Store' <<< "$output" } +# ── #817: negation tail is CONDITIONAL — only re-emit what L2 actually re-hides ─ +@test "#817 clean L2 (no re-hider) appends NO negation tail; upsert is a first-application fixed point" { + # A marker-wrapped file whose L2 is genuine ecosystem/OS cruft that re-hides + # nothing the block negates. The steady state must be block + L2 verbatim (no + # negation tail), and re-upserting it must be a byte-for-byte no-op — on the + # FIRST application, not only from the second pass on. + local existing="${TT_TMP}/clean" + { + cat "$BLOCK" + printf 'node_modules/\n.DS_Store\n' + } > "$existing" + + local out + out="$(upsert_gitignore_baseline "$BLOCK" "$existing")" + + # No negation tail: !.env.example / !*.pub appear ONLY inside the block (once). + [ "$(printf '%s\n' "$out" | grep -cxF '!.env.example')" -eq 1 ] + [ "$(printf '%s\n' "$out" | grep -cxF '!*.pub')" -eq 1 ] + # Genuine L2 preserved. + grep -qxF 'node_modules/' <<< "$out" + grep -qxF '.DS_Store' <<< "$out" + # Fixed point on the FIRST application: upsert(block + clean-L2) == input. + [ "$out" = "$(cat "$existing")" ] +} + +# ── #817 Part B: dedup migrated baseline comments (never a bare-# or repo one) ── +@test "#817 migrated baseline section comment (block-identical) is dropped; repo comments kept" { + # Simulate migrating an OLD unmarkered baseline whose section comment is + # byte-identical to the current block's comment — it must fold away, while + # the repo's own comment stays put. + local existing="${TT_TMP}/migrated" + cat > "$existing" < "$realblock" + + local existing="${TT_TMP}/withbarehash" + printf '#\n# my section\nnode_modules/\n' > "$existing" + + local out l2 + out="$(upsert_gitignore_baseline "$realblock" "$existing")" + l2="$(printf '%s\n' "$out" | awk -v e="$GIB_END_MARKER" 'seen{print} $0==e{seen=1}')" + + grep -qxF '#' <<< "$l2" # bare # kept + grep -qxF '# my section' <<< "$l2" + grep -qxF 'node_modules/' <<< "$l2" +} + +@test "#817 already-current canonical .gitignore is a byte-for-byte fixed point" { + # With the conditional tail, upsert(block, canonical) == canonical again + # (the canonical L2 re-hides nothing and its bare-# separator is preserved). + local realblock="${TT_TMP}/realblock" + gib_extract_baseline_block "${TT_REPO_ROOT}/.gitignore" > "$realblock" + + local out + out="$(upsert_gitignore_baseline "$realblock" "${TT_REPO_ROOT}/.gitignore")" + [ "$out" = "$(cat "${TT_REPO_ROOT}/.gitignore")" ] +} + @test "#809 negation-win + migration is idempotent" { local realblock="${TT_TMP}/realblock" gib_extract_baseline_block "${TT_REPO_ROOT}/.gitignore" > "$realblock" diff --git a/test/scripts/sync-gitignore-baseline/dry-run.bats b/test/scripts/sync-gitignore-baseline/dry-run.bats index 197412ef2..d081333f3 100644 --- a/test/scripts/sync-gitignore-baseline/dry-run.bats +++ b/test/scripts/sync-gitignore-baseline/dry-run.bats @@ -61,11 +61,10 @@ b64() { base64 -w 0 2>/dev/null || base64 -b 0; } } @test "dry-run skips a repo whose .gitignore already carries the current baseline" { - # "Current" = the upsert steady state (block + neutralized L2 + negation tail - # per #809), i.e. a repo already synced by this tool — not the raw canonical. - source "${REPO_ROOT}/scripts/lib/gitignore-baseline.sh" - _blk="$(mktemp "$TT_TMP/block.XXXXXX")"; gib_extract_baseline_block "$CANONICAL" > "$_blk" - GH_CONTENT_B64="$(upsert_gitignore_baseline "$_blk" "$CANONICAL" | b64)"; export GH_CONTENT_B64 + # "Current" = the raw canonical .gitignore. Per #817 the negation tail is now + # conditional, so upsert(canonical) == canonical again (its L2 re-hides nothing + # and its bare-# separator is preserved) — the raw canonical is the steady state. + GH_CONTENT_B64="$(b64 < "$CANONICAL")"; export GH_CONTENT_B64 install_gh_stub run env GH_TOKEN=x bash "$SCRIPT" --dry-run --repo markets [ "$status" -eq 0 ] From 72af4d70a992a9114482f51501498c1f768c113b Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:16:26 +0000 Subject: [PATCH 2/2] fix(bot): address bot feedback [skip ci-relay] --- scripts/lib/gitignore-baseline.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/lib/gitignore-baseline.sh b/scripts/lib/gitignore-baseline.sh index eb07a888f..1e2c84e35 100644 --- a/scripts/lib/gitignore-baseline.sh +++ b/scripts/lib/gitignore-baseline.sh @@ -163,14 +163,15 @@ _gib_negation_tail() { return 0 fi - local d; d="$(mktemp -d)" + local d; d="$(mktemp -d -t gitignore-baseline.XXXXXX)" { printf '%s\n' "$block" [ -n "$body" ] && printf '%s\n' "$body" } > "$d/.gitignore" ( cd "$d" && git init -q 2>/dev/null ) - local ignored - ignored="$(cd "$d" && git -c core.excludesfile=/dev/null check-ignore -- "${paths[@]}" 2>/dev/null)" + local ignored exit_code + ignored="$(cd "$d" && git -c core.excludesfile=/dev/null check-ignore -- "${paths[@]}" 2>/dev/null)" || exit_code=$? + [ "${exit_code:-0}" -gt 1 ] && ignored="" rm -rf "$d" local i