diff --git a/tools/498-phase13-apply.py b/tools/498-phase13-apply.py new file mode 100644 index 00000000..3ef3aa90 --- /dev/null +++ b/tools/498-phase13-apply.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""#498 phase 1-3 AIF serialization APPLY — fill the 2 AIF columns for the +scaleup phase 1-3 generative leaves (GATED execution — dry-run by default). + +Companion to `tools/498-phase13-serialization-audit.py` (#756, merged), which +proved the 50-leaf map is byte-faithful to the 3 machine-readable annotation +CSVs (50/50 match). This script does the actual byte-exact CELL FILL into +`Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv` (the 2 columns already +exist post-#753; this is cell-fill, not column insertion). + +GATE (per ai-01 dispatch `hlqdw4`, 2026-07-09): jsboige content nod for the 50 +generative leaves is the prod-write gate. #756 surfaced the decision (D1); +ai-01 escalated D1 to jsboige. This script is READY TO RUN but `--write` is +GATED until ai-01 relays the jsboige verdict. Running dry-run (default) proves +byte-preservation without touching prod. + +D2 coupling (834/847): ai-01 lean = override → undercut (ASPIC+ supersedes +#753's strict-conservateur FAIL-LOUD), but bundles it in the D1 escalation. +This script is PARAMETERIZED so a single flag flips at the verdict: + python tools/498-phase13-apply.py # dry-run, 45 NEW only (834/847 stay FAIL-LOUD) + python tools/498-phase13-apply.py --write # APPLY 45 NEW (GATED — jsboige nod) + python tools/498-phase13-apply.py --write --with-overrides # APPLY 47 (45 NEW + 834/847 undercut) + +Source: PHASE_MAP + FAIL_LOUD_753 are byte-copied from +`tools/498-phase13-serialization-audit.py` (#756), which cross-checks them 50/50 +against the annotation CSVs. NODE is the deterministic ASPIC+ map (ratified +#707§4 Option a): undercut→RA-node, undermine→I-node, rebut→CA-node. +""" +import csv, io, sys +from collections import Counter + +PATH = "Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv" +BACKUP = "tmp/Fallacies-backup-pre-phase13.csv" # saved before --write (for independent verify) +NODE = {"undercut": "RA-node", "undermine": "I-node", "rebut": "CA-node"} +WRITE = "--write" in sys.argv +WITH_OVERRIDES = "--with-overrides" in sys.argv + +# ── 50-leaf phase 1-3 map (copied from #756 audit; cross-checked 50/50 vs +# annotation CSVs there — re-verified below as a load-bearing assertion) ──── +PHASE_MAP = { + # phase 1 (Abus langage + Erreur raisonnement) — 11 + "826":"undercut","834":"undercut","844":"undercut","847":"undercut","855":"undercut", + "698":"undermine","707":"undercut","727":"undercut","735":"undercut","750":"undercut","784":"undercut", + # phase 2 (Erreur math + Insuffisance) — 12 + "55":"undermine","96":"undercut","112":"undercut","134":"undercut","153":"undercut", + "165":"undermine","596":"undermine","644":"undermine","658":"undercut","667":"undermine", + "681":"undermine","690":"undercut", + # phase 3 (Influence + Tricherie + Obstruction) — 27 + "177":"undermine","219":"undermine","247":"undermine","300":"undermine","322":"undermine", + "340":"undermine","357":"undermine","420":"undermine","511":"undermine", + "889":"undermine","942":"undermine","953":"undermine", + "974":"undercut","992":"undercut","1011":"undercut", + "1024":"undermine","1174":"undermine","1242":"undermine", + "1282":"rebut","1287":"undermine","1297":"undermine","1313":"rebut", + "1345":"undercut","1352":"undermine","1361":"rebut", + "1371":"undermine","1398":"undermine", +} +# #753 deliberate FAIL-LOUD set (strict-conservateur verdict "garde tel quel"). +# 834/847 are in this set → they are the D2 CONFLICT (override vs keep empty). +FAIL_LOUD_753 = {"829","840","847","848","853","832","834","835","837", + "861","868","869","870","871","872","873","874"} +assert len(PHASE_MAP) == 50 +assert Counter(PHASE_MAP.values()) == {"undercut":20, "undermine":27, "rebut":3} + +# ── byte-exact splitters (CSV-aware: doubled quotes + embedded LF) ───────────── +def split_logical_rows(text): + rows, cur, in_q = [], [], False + i, n = 0, len(text) + while i < n: + ch = text[i] + if ch == '"': + if in_q and i+1 < n and text[i+1] == '"': + cur.append('""'); i += 2 + else: + in_q = not in_q; cur.append(ch); i += 1 + elif ch == '\r' and not in_q and i+1 < n and text[i+1] == '\n': + rows.append(''.join(cur)); cur = []; i += 2 + else: + cur.append(ch); i += 1 + if cur: rows.append(''.join(cur)) + return rows + +def split_fields(row): + segs, cur, in_q = [], [], False + i, n = 0, len(row) + while i < n: + ch = row[i] + if ch == '"': + if in_q and i+1 < n and row[i+1] == '"': + cur.append('""'); i += 2 + else: + in_q = not in_q; cur.append(ch); i += 1 + elif ch == ',' and not in_q: + segs.append(''.join(cur)); cur = []; i += 1 + else: + cur.append(ch); i += 1 + segs.append(''.join(cur)) + return segs + +# ── load-bearing: re-verify PHASE_MAP vs the annotation CSVs (independent of #756) +import glob +ANNOT_CSVS = sorted(glob.glob("docs/taxonomy/498-scaleup-phase*-annotations.csv")) +annot_map = {} +for c in ANNOT_CSVS: + with open(c, encoding="utf-8-sig") as fh: + for row in csv.DictReader(fh): + annot_map[row["fallacy_pk"].strip()] = row["attack_type"].strip() +assert set(annot_map) == set(PHASE_MAP), ( + f"PHASE_MAP vs annotation-CSV PK mismatch\n" + f" only CSVs: {set(annot_map)-set(PHASE_MAP)}\n only MAP: {set(PHASE_MAP)-set(annot_map)}") +assert all(annot_map[pk] == PHASE_MAP[pk] for pk in PHASE_MAP), "attack_type mismatch" +# (50/50 re-verified — extraction is byte-faithful) + +# ── read current CSV ─────────────────────────────────────────────────────────── +raw = open(PATH, "rb").read() +bom = raw[:3] == b'\xef\xbb\xbf' +text = (raw[3:] if bom else raw).decode('utf-8') +ended_crlf = text.endswith('\r\n') +rows = split_logical_rows(text) +header = split_fields(rows[0]) +ATI = header.index('AIF_attackType') +ANI = header.index('AIF_attackedNode') +PKI = 0 # uppercase 'PK' col +assert header[ATI-1] == 'AIF_skosMappingType', "AIF col block moved?" + +# ── determine the apply_set per the D2 flag (based on CURRENT CSV state) ────── +# Snapshot which PHASE_MAP PKs are currently empty (= fillable) vs filled. +# The 3 CONFIRM PKs (826/844/855) are already filled undercut by #753 → not +# fillable → excluded from apply_set (no-op, no overwrite). +pre_state = {} +for r in rows[1:]: + s = split_fields(r) + pk = s[PKI].strip() + if pk in PHASE_MAP: + pre_state[pk] = s[ATI].strip() +empty_in_phase = {pk for pk, v in pre_state.items() if not v} # fillable +filled_in_phase = {pk for pk, v in pre_state.items() if v} # CONFIRM (no-op) +# D2 CONFLICT = fillable phase PKs that #753 deliberately left FAIL-LOUD. +conflict_pks = empty_in_phase & FAIL_LOUD_753 # {834, 847} +new_pks = empty_in_phase - FAIL_LOUD_753 # 45 NEW (fillable, not fail-loud) +if WITH_OVERRIDES: + apply_set = empty_in_phase # 47 (45 NEW + 834/847) + d2_note = "834/847 OVERRIDE → undercut (per jsboige nod)" +else: + apply_set = new_pks # 45 NEW only (834/847 stay FAIL-LOUD) + d2_note = "834/847 KEPT FAIL-LOUD (#753 strict-conservateur)" + +# safety: every apply_set PK must be empty (fill, not overwrite). +non_empty = {pk for pk in apply_set if pre_state.get(pk)} +assert not non_empty, ( + f"ABORT: apply_set PKs not empty (would overwrite, not fill): {non_empty}") + +# ── apply: cell-fill byte-exact (only ATI/ANI of apply_set PKs change) ──────── +new_rows = [rows[0]] +filled = Counter() +for rtext in rows[1:]: + s = split_fields(rtext) + pk = s[PKI].strip() + if pk in apply_set: + at = PHASE_MAP[pk] + s[ATI] = at + s[ANI] = NODE[at] + filled[at] += 1 + new_rows.append(",".join(s)) +new_text = "\r\n".join(new_rows) + ("\r\n" if ended_crlf else "") + +# ── byte-preservation proof (only ATI/ANI of apply_set may differ) ──────────── +new_rows2 = split_logical_rows(new_text) +mismatches = 0 +for i in range(len(rows)): + o = split_fields(rows[i]); n = split_fields(new_rows2[i]) + assert len(o) == len(n) == 104, f"row {i} col count drift" + for j in range(104): + if o[j] != n[j] and not (i > 0 and j in (ATI, ANI) and o[PKI].strip() in apply_set): + mismatches += 1 + if mismatches <= 3: + print(f" MISMATCH row {i} pk {o[PKI].strip()!r} col {j}: {o[j]!r} -> {n[j]!r}") + +# re-parse well-formedness +chk = list(csv.reader(io.StringIO(new_text))) +assert len(chk) == len(rows) and all(len(r) == 104 for r in chk), "well-formedness" + +# ── report ───────────────────────────────────────────────────────────────────── +total_now = sum(1 for r in rows[1:] if split_fields(r)[ATI].strip()) +total_after = total_now + len(apply_set) +print("="*72) +print(f"#498 PHASE 1-3 APPLY (write={WRITE}, with-overrides={WITH_OVERRIDES})") +print("="*72) +print(f"D2 handling: {d2_note}") +print(f"apply_set: {len(apply_set)} PKs -> distribution: {dict(filled)}") +print(f" (NEW={len(new_pks)}, conflict-overridden={len(apply_set-new_pks)}, CONFIRM no-op={len(filled_in_phase)})") +print(f"PHASE_MAP re-verified 50/50 vs annotation CSVs: OK") +print(f"pre-state check: all {len(apply_set)} apply_set PKs empty (fill, no overwrite): OK") +print(f"byte-preservation mismatches: {mismatches} (must be 0)") +print(f"well-formedness: {len(chk)} rows × 104 cols, CRLF({ended_crlf})+BOM({bom}) preserved") +print(f"delta if written: {len(new_text)-len(text)} bytes") +print(f"CSV filled total: {total_now} -> {total_after}") +if WRITE: + import os + os.makedirs("tmp", exist_ok=True) + open(BACKUP, "wb").write(raw) # save the ORIGINAL bytes (pre-write) for independent verify + open(PATH, "wb").write((b'\xef\xbb\xbf' if bom else b'') + new_text.encode('utf-8')) + print(f">>> WRITTEN ({len(apply_set)} cells filled). Backup saved to {BACKUP} for independent verify.") + print(f" GATE was lifted by ai-01 relay of jsboige nod. Run: python tools/498-phase13-verify.py [--with-overrides]") +else: + print(">>> DRY-RUN (pass --write to APPLY; GATED until ai-01 relays jsboige nod. [--with-overrides] for 834/847)") diff --git a/tools/498-phase13-verify.py b/tools/498-phase13-verify.py new file mode 100644 index 00000000..4f7ba8a6 --- /dev/null +++ b/tools/498-phase13-verify.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""#498 phase 1-3 INDEPENDENT byte-preservation verifier (run AFTER apply --write). + +This is the INDEPENDENT verifier (backup-vs-result), NOT the apply script's +dry-run self-check. Same discipline as #753/#755 (ai-01's independent byte-check): +reads `tmp/Fallacies-backup-pre-phase13.csv` (saved by the apply script) vs the +current prod CSV and proves: + 1. 0 byte-preservation mismatch — every field except AIF_attackType/ + AIF_attackedNode of the apply_set PKs is byte-identical. + 2. Independent re-derive of the distribution (count + assert vs expected). + 3. Spot-checks: the 3 rebut PKs → rebut/CA-node; a few undermine → I-node; + a few undercut → RA-node; the 3 CONFIRM PKs unchanged; 834/847 per the flag. + +The expected apply_set is re-derived INDEPENDENTLY here from the current CSV +state + PHASE_MAP + FAIL_LOUD_753 + the --with-overrides flag (not imported from +the apply script — re-declaration is the independence guarantee). + +Usage (after `python tools/498-phase13-apply.py --write [--with-overrides]`): + python tools/498-phase13-verify.py # verify MODE 1 (45 NEW) + python tools/498-phase13-verify.py --with-overrides # verify MODE 2 (47) + +If the backup doesn't exist (no --write yet), this prints the expected +post-state without comparison (dry verification of the re-derive logic). +""" +import csv, os, sys +from collections import Counter + +PATH = "Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv" +BACKUP = "tmp/Fallacies-backup-pre-phase13.csv" +NODE = {"undercut": "RA-node", "undermine": "I-node", "rebut": "CA-node"} +WITH_OVERRIDES = "--with-overrides" in sys.argv + +# ── re-declared independently (NOT imported from apply script) ───────────────── +PHASE_MAP = { + "826":"undercut","834":"undercut","844":"undercut","847":"undercut","855":"undercut", + "698":"undermine","707":"undercut","727":"undercut","735":"undercut","750":"undercut","784":"undercut", + "55":"undermine","96":"undercut","112":"undercut","134":"undercut","153":"undercut", + "165":"undermine","596":"undermine","644":"undermine","658":"undercut","667":"undermine", + "681":"undermine","690":"undercut", + "177":"undermine","219":"undermine","247":"undermine","300":"undermine","322":"undermine", + "340":"undermine","357":"undermine","420":"undermine","511":"undermine", + "889":"undermine","942":"undermine","953":"undermine", + "974":"undercut","992":"undercut","1011":"undercut", + "1024":"undermine","1174":"undermine","1242":"undermine", + "1282":"rebut","1287":"undermine","1297":"undermine","1313":"rebut", + "1345":"undercut","1352":"undermine","1361":"rebut", + "1371":"undermine","1398":"undermine", +} +FAIL_LOUD_753 = {"829","840","847","848","853","832","834","835","837", + "861","868","869","870","871","872","873","874"} + +# ── byte-exact splitters (doubled quotes + embedded LF aware) ────────────────── +def split_logical_rows(text): + rows, cur, in_q = [], [], False + i, n = 0, len(text) + while i < n: + ch = text[i] + if ch == '"': + if in_q and i+1 < n and text[i+1] == '"': + cur.append('""'); i += 2 + else: + in_q = not in_q; cur.append(ch); i += 1 + elif ch == '\r' and not in_q and i+1 < n and text[i+1] == '\n': + rows.append(''.join(cur)); cur = []; i += 2 + else: + cur.append(ch); i += 1 + if cur: rows.append(''.join(cur)) + return rows + +def split_fields(row): + segs, cur, in_q = [], [], False + i, n = 0, len(row) + while i < n: + ch = row[i] + if ch == '"': + if in_q and i+1 < n and row[i+1] == '"': + cur.append('""'); i += 2 + else: + in_q = not in_q; cur.append(ch); i += 1 + elif ch == ',' and not in_q: + segs.append(''.join(cur)); cur = []; i += 1 + else: + cur.append(ch); i += 1 + segs.append(''.join(cur)) + return segs + +def load(path): + raw = open(path, "rb").read() + bom = raw[:3] == b'\xef\xbb\xbf' + return split_logical_rows((raw[3:] if bom else raw).decode('utf-8')), bom + +# ── read current prod CSV ────────────────────────────────────────────────────── +new_rows, nbom = load(PATH) +header = split_fields(new_rows[0]) +ATI = header.index('AIF_attackType') +ANI = header.index('AIF_attackedNode') +PKI = 0 + +# ── re-derive the EXPECTED apply_set independently (per the flag) ────────────── +# Build from CURRENT CSV state: which PHASE_MAP PKs were empty before the write? +# After a successful write, those PKs are now FILLED; CONFIRM were already filled; +# the apply_set = empty-before ∩ (new ∪ overrides-if-flag). +# Independently, we determine: which PKs SHOULD now be filled with what. +expected = {} # pk -> attack_type +for r in new_rows[1:]: + s = split_fields(r); pk = s[PKI].strip() + if pk in PHASE_MAP: + at = s[ATI].strip() + if at: # filled (either CONFIRM already, or just-written) + expected[pk] = at + +# Sanity: all PHASE_MAP PKs present in CSV +present = {split_fields(r)[PKI].strip(): True for r in new_rows[1:] if split_fields(r)[PKI].strip() in PHASE_MAP} +assert len(present) == len(PHASE_MAP), f"missing PHASE_MAP PKs: {set(PHASE_MAP)-set(present)}" + +# spot-check: every filled PHASE_MAP PK's attack_type matches PHASE_MAP +filled_in_csv = {} +for r in new_rows[1:]: + s = split_fields(r); pk = s[PKI].strip() + if pk in PHASE_MAP and s[ATI].strip(): + filled_in_csv[pk] = s[ATI].strip() +assert set(filled_in_csv) == set(expected) +bad = {pk for pk, at in filled_in_csv.items() if at != PHASE_MAP[pk]} +assert not bad, f"filled PK attack_type != PHASE_MAP: {bad}" + +# the apply_set that was written = filled now AND would-have-been-empty-before. +# We can't see "before" without the backup, so recompute the EXPECTED apply_set: +empty_now = {pk for pk in PHASE_MAP if pk not in filled_in_csv} +# Determine which empty_now PKs SHOULD have been written (depends on flag + FAIL-LOUD) +# If MODE 1 (no overrides): apply_set was (empty - FAIL_LOUD); 834/847 stay empty. +# If MODE 2 (overrides): apply_set was all empty. +# So the FILLED set we see must match: filled = PHASE_MAP - (empty-if-not-written) +# ── header + re-derive summary ──────────────────────────────────────────────── +CRLF = b'\r\n' +raw_new = open(PATH, "rb").read() +print("="*72) +print(f"#498 PHASE 1-3 INDEPENDENT VERIFY (with-overrides={WITH_OVERRIDES})") +print("="*72) +print(f"prod CSV: {len(new_rows)-1} data rows, AIF cols at idx {ATI}/{ANI}") +print(f"PHASE_MAP PKs filled in CSV: {len(filled_in_csv)} -> distribution: {dict(Counter(filled_in_csv.values()))}") +print(f"empty PHASE_MAP PKs: {sorted(empty_now) or 'none'}") + +if not os.path.exists(BACKUP): + # PRE-write state (no --write yet): dry verification of the re-derive only. + # empty_now here = all PHASE_MAP PKs still empty (45 NEW + 834/847 = 47). + # Report the apply_set --write WOULD fill (per flag); skip POST-write asserts. + would_fill = {pk for pk in empty_now if WITH_OVERRIDES or pk not in FAIL_LOUD_753} + print(f"\n⚠ no backup at {BACKUP} — apply --write has not run (PRE-write state).") + print(f" dry verification of re-derive only (no byte-compare yet).") + print(f" --write would fill {len(would_fill)} PKs (MODE {'2 +834/847 override' if WITH_OVERRIDES else '1 NEW only'}): " + f"{dict(Counter(PHASE_MAP[p] for p in would_fill))}") + print(f" POST-write emptiness asserts + byte-compare run AFTER apply --write.") + sys.exit(0) + +# ── POST-write state (backup exists): assert expected emptiness per MODE ─────── +if WITH_OVERRIDES: + # MODE 2: all 50 PHASE_MAP PKs filled (45 written + 3 CONFIRM + 2 overrides) + assert not empty_now, f"MODE 2 (overrides) but PKs still empty: {empty_now}" +else: + # MODE 1: 834/847 stay empty (FAIL-LOUD); everything else filled. + assert empty_now == {"834","847"}, f"MODE 1 expected only 834/847 empty, got: {empty_now}" + +orig_rows, obom = load(BACKUP) +assert obom == nbom, f"BOM changed {obom}->{nbom}" +assert len(orig_rows) == len(new_rows) == 1409, f"row count {len(orig_rows)} vs {len(new_rows)}" + +# determine apply_set from backup: PHASE_MAP PKs that were empty in backup +apply_set = set() +for i in range(1, 1409): + o = split_fields(orig_rows[i]); opk = o[PKI].strip() + if opk in PHASE_MAP and not o[ATI].strip(): + # was empty in backup → was it written? depends on flag + if WITH_OVERRIDES or opk not in FAIL_LOUD_753: + apply_set.add(opk) + +mismatches = 0 +for i in range(1409): + o = split_fields(orig_rows[i]); n = split_fields(new_rows[i]) + assert len(o) == len(n) == 104, f"row {i} col count" + for j in range(104): + if o[j] != n[j]: + ok = (i > 0 and j in (ATI, ANI) and o[PKI].strip() in apply_set) + if not ok: + mismatches += 1 + if mismatches <= 3: + print(f" MISMATCH row {i} pk {o[PKI].strip()!r} col {j}: {o[j]!r} -> {n[j]!r}") + +raw_old = open(BACKUP, "rb").read() +print(f"\nbackup: {BACKUP} ({len(raw_old)} bytes) vs prod ({len(raw_new)} bytes)") +print(f"byte-preservation mismatches: {mismatches} (must be 0)") +assert mismatches == 0 +print(f"CRLF: backup={raw_old.count(CRLF)}, prod={raw_new.count(CRLF)}") +print(f"BOM: backup={raw_old[:3]==b'\xef\xbb\xbf'}, prod={raw_new[:3]==b'\xef\xbb\xbf'}") +print(f"delta: {len(raw_new)-len(raw_old)} bytes") + +# ── spot-checks (independent of apply script) ────────────────────────────────── +by_pk = {split_fields(r)[PKI].strip(): split_fields(r) for r in new_rows[1:]} +spot = { + "1282":("rebut","CA-node"), "1313":("rebut","CA-node"), "1361":("rebut","CA-node"), # the 3 rebut + "698":("undermine","I-node"), "889":("undermine","I-node"), "340":("undermine","I-node"), + "707":("undercut","RA-node"), "974":("undercut","RA-node"), "1345":("undercut","RA-node"), + # CONFIRM (already #753) — must be unchanged + "826":("undercut","RA-node"), "844":("undercut","RA-node"), "855":("undercut","RA-node"), +} +for pk,(at,node) in spot.items(): + r = by_pk[pk]; ok = r[ATI]==at and r[ANI]==node + print(f" spot pk {pk}: [{r[ATI]},{r[ANI]}] expect [{at},{node}] -> {'OK' if ok else 'FAIL'}") + assert ok +# D2: 834/847 per flag +for pk in ("834","847"): + r = by_pk[pk] + if WITH_OVERRIDES: + ok = r[ATI]=="undercut" and r[ANI]=="RA-node"; exp="[undercut,RA-node]" + else: + ok = r[ATI]=="" and r[ANI]==""; exp="[empty] (FAIL-LOUD)" + print(f" D2 pk {pk}: [{r[ATI]!r},{r[ANI]!r}] expect {exp} -> {'OK' if ok else 'FAIL'}") + assert ok +# a random non-phase PK must be untouched +r = by_pk.get("2") or by_pk.get("801") +print(f" non-phase pk {r[PKI].strip()}: unchanged (not in apply_set) -> OK") + +print("\n>>> VERIFY PASSED: byte-exact preservation + correct values + correct apply_set + spot-checks OK")