diff --git a/.github/workflows/butler-reconcile.yml b/.github/workflows/butler-reconcile.yml index 239645d..d30ff5e 100644 --- a/.github/workflows/butler-reconcile.yml +++ b/.github/workflows/butler-reconcile.yml @@ -48,3 +48,33 @@ jobs: STALE_DAYS_OVERRIDE: ${{ inputs.stale_days_override }} BUTLER_DRY_RUN: ${{ inputs.dry_run }} run: bash governance/butler-reconcile.sh + - name: 影子账本落盘（butler-ledger 分支，IR-0006 W1-B2 / BEH-03） + # butler 源影子持久化：本地 shadow-evidence.jsonl relink 续接 butler-ledger + # 基链（双侧验链，防覆盖掩盖篡改）后写回。always()：扫描有发现（exit 1） + # 时审计事件同样必须落账（append-only 纪律优先于本轮结论）。 + if: ${{ always() }} + env: + BUTLER_TOKEN: ${{ secrets.GOVERNANCE_TOKEN }} + run: | + set -euo pipefail + SHADOW="governance/butler/shadow-evidence.jsonl" + if [[ ! -s "$SHADOW" ]]; then + echo "OK 本轮无影子记录（audit_emit 未触发或 dry-run）——跳过（幂等）" + exit 0 + fi + git config --global user.name butler-ledger-bot && git config --global user.email butler-bot@users.noreply.github.com + if ! git clone --depth 1 "https://x-access-token:${BUTLER_TOKEN}@github.com/Cloudbird-Software/.github.git" ledger -b butler-ledger 2>/dev/null; then + git clone --depth 1 "https://x-access-token:${BUTLER_TOKEN}@github.com/Cloudbird-Software/.github.git" ledger + git -C ledger checkout -b butler-ledger + fi + BASE="ledger/$SHADOW" + [[ -f "$BASE" ]] || : > "$BASE" + python3 governance/evidence_shadow.py relink --base "$BASE" --local "$SHADOW" --out merged.jsonl + python3 governance/evidence_shadow.py verify --file merged.jsonl + mkdir -p "ledger/governance/butler" + cp merged.jsonl "$BASE" + git -C ledger add "$SHADOW" + git -C ledger diff --cached --quiet && { echo "OK 影子无新增——不提交（幂等）"; exit 0; } + git -C ledger commit -m "butler: 影子账本追加（IR-0006 W1-B2 双写，链验通过）" + for i in 1 2 3; do git -C ledger push "https://x-access-token:${BUTLER_TOKEN}@github.com/Cloudbird-Software/.github.git" HEAD:refs/heads/butler-ledger && break + git -C ledger pull --rebase "https://x-access-token:${BUTLER_TOKEN}@github.com/Cloudbird-Software/.github.git" butler-ledger || true; sleep 5; done diff --git a/.github/workflows/seed-drill.yml b/.github/workflows/seed-drill.yml index 314ebd0..89793a9 100644 --- a/.github/workflows/seed-drill.yml +++ b/.github/workflows/seed-drill.yml @@ -253,8 +253,8 @@ jobs: EOF cat rec.json python3 governance/drill/drill.py record --history ledger/governance/drill/history.jsonl --json "$(cat rec.json)" - git -C ledger add governance/drill/history.jsonl - git -C ledger commit -m "drill(seed): 台账追加 $VERDICT（ADR-0069 周演习 run ${RUN_ID}）" + git -C ledger add governance/drill/history.jsonl governance/drill/shadow-evidence.jsonl + git -C ledger commit -m "drill(seed): 台账追加 $VERDICT（ADR-0069 周演习 run ${RUN_ID}；影子双写 IR-0006 W1-B2）" for i in 1 2 3; do git -C ledger push "https://x-access-token:${DRILL_TOKEN}@github.com/Cloudbird-Software/.github.git" HEAD:refs/heads/drill-ledger && break git -C ledger pull --rebase "https://x-access-token:${DRILL_TOKEN}@github.com/Cloudbird-Software/.github.git" drill-ledger || true; sleep 5; done python3 governance/drill/drill.py redrate --history ledger/governance/drill/history.jsonl diff --git a/governance/butler-audit.sh b/governance/butler-audit.sh index 953050d..21e9c90 100644 --- a/governance/butler-audit.sh +++ b/governance/butler-audit.sh @@ -25,6 +25,16 @@ # - actions JSON 为 SLI 字段（#98 口径：auto_merge_rate / check_latency / revert_count） # 预留键位——账本 JSON 状态块由 W1-C3 dashboard 脚本负责，本行结构已兼容（机器可 # grep '^AUDIT' 提取后 json.loads 尾段）。 +# +# 影子双写（IR-0006 W1-B2 / BEH-03，ADR-0103）：audit_emit 在输出 AUDIT 行（原层， +# 只增不改）的同时，按证据 schema v1 追加影子记录到 +# ${BUTLER_SHADOW_FILE:-<本脚本同目录>/butler/shadow-evidence.jsonl} +# （kind=gate / action=butler-<名> / verdict=；card 哨兵 .github#0，tenant +# 缺省 cloudbird-internal——env BUTLER_CARD/BUTLER_TENANT 可注入）。影子写入失败 +# =fail-closed（return 2，BEH-01：双写不一致必须当场可见）。影子账本由 +# butler-reconcile 每 6h 落盘 butler-ledger 分支（governance/evidence_shadow.py +# relink 同款链执法）；其他 workflow 的本地影子随 runner 销毁（丢弃层友海—— +# 持久化优先级在 reconcile 主循环）。 _butler_audit_cli=0 if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then _butler_audit_cli=1; fi @@ -104,6 +114,33 @@ audit_emit() { fi printf '%s\n' "$line" >> "$GITHUB_STEP_SUMMARY" || return 0 fi + _shadow_emit "$butler" "$outcome" "$actions" || return 2 +} + +# 影子双写（BEH-03）：schema v1 判定记录落本地影子账本（链式 hash，写入器独占） +_shadow_emit() { + [[ -n "$_BUTLER_PY" ]] || return 0 # 无 python 环境：影子无法成链——原层照常（极端降级） + local butler="$1" outcome="$2" actions="$3" + local here shadow evf + here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + shadow="${BUTLER_SHADOW_FILE:-$here/butler/shadow-evidence.jsonl}" + evf="$(mktemp)" + trap 'rm -f "$evf"' RETURN + "$_BUTLER_PY" - "$evf" "$butler" "$outcome" <<'PYEOF' || { echo "FATAL: 影子事件构造失败" >&2; return 2; } +import datetime, json, sys +ev = { + "ts": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "kind": "gate", "action": f"butler-{sys.argv[2]}", "verdict": sys.argv[3], + "subject": {"card": __import__("os").environ.get("BUTLER_CARD", "Cloudbird-Software/.github#0"), + "tenant": __import__("os").environ.get("BUTLER_TENANT", "cloudbird-internal")}, + "actor": {"identity": sys.argv[2], "role": "bot", "model": None}, +} +open(sys.argv[1], "w", encoding="utf-8").write(json.dumps(ev, ensure_ascii=False)) +PYEOF + if ! "$_BUTLER_PY" "$here/evidence_shadow.py" append --file "$shadow" --event-file "$evf" >/dev/null; then + echo "FATAL: 影子账本写入失败（$shadow）——fail-closed（BEH-01 双写不一致当场可见）" >&2 + return 2 + fi } # ---------- CLI 模式（bash butler-audit.sh ...；source 时不执行） ---------- diff --git a/governance/drill/drill.py b/governance/drill/drill.py index b50c1d1..e973dae 100644 --- a/governance/drill/drill.py +++ b/governance/drill/drill.py @@ -14,6 +14,7 @@ """ import argparse import base64 +import hashlib import json import os import random @@ -227,7 +228,23 @@ def cmd_record(a): die(f"同一 run 的 {rec['kind']} 记录已存在（run_id={rec['run_id']}）") with open(a.history, "a", encoding="utf-8", newline="\n") as f: f.write(json.dumps(rec, ensure_ascii=False, sort_keys=True) + "\n") - print(f"OK append 1 行（现有 {len(lines) + 1} 行）") + # ---- 影子双写（IR-0006 W1-B2 / BEH-03）：同一判定按证据 schema v1 落影子账本 ---- + # 原台账只增不改（AC-4b 平移不搬移）；影子事件 kind=gate（演习裁决）， + # card 哨兵 .github#0（基建事件未绑卡——#0 不参与卡聚合） + import sys as _sys + _sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import evidence_shadow + shadow = os.path.join(os.path.dirname(os.path.abspath(a.history)), "shadow-evidence.jsonl") + shadow_ev = { + "ts": rec["ts"], "kind": "gate", "action": f"drill-{rec['kind']}", + "verdict": str(rec.get("verdict") or "recorded"), + "subject": {"card": "Cloudbird-Software/.github#0", "tenant": "cloudbird-internal"}, + "actor": {"identity": "drill-seed-bot", "role": "bot", "model": None}, + "inputs_digest": "sha256:" + hashlib.sha256( + json.dumps(rec, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest(), + } + evidence_shadow.append(shadow, shadow_ev) + print(f"OK append 1 行（现有 {len(lines) + 1} 行；影子 → {shadow}）") def cmd_redrate(a): @@ -294,7 +311,7 @@ def main(): p = sub.add_parser("redrate", help="红率+难度趋势聚合（AC-4）") p.add_argument("--history", default=os.path.join(here, "history.jsonl")) p.add_argument("--fail-unhealthy", action="store_true", - help="红率<100% 时非零退出（workflow 告警用）") + help="红率<100%% 时非零退出（workflow 告警用）") p.set_defaults(func=cmd_redrate) a = ap.parse_args() diff --git a/governance/evidence-query.sh b/governance/evidence-query.sh new file mode 100644 index 0000000..c0f092b --- /dev/null +++ b/governance/evidence-query.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# evidence-query.sh —— 三源统一证据查询（IR-0006 W1-B2 / BEH-03 / ADR-0103，AC-4a） +# +# 一条命令跨三源拉取 schema v1 影子账本、逐源验链（fail-closed：链断=红）、 +# 按时间归并输出统一 JSONL（stdout）+ 分源统计（stderr）： +# 源 1 metering Cloudbird-Software/CI-Workflows @ metering-ledger shadow-evidence-*.jsonl（根） +# 源 2 drill Cloudbird-Software/.github @ drill-ledger governance/drill/shadow-evidence.jsonl +# 源 3 butler Cloudbird-Software/.github @ butler-ledger governance/butler/shadow-evidence.jsonl +# +# 用法: +# bash governance/evidence-query.sh [--card owner/repo#n] [--json] # --json=汇总行也走 stdout +# env: +# GH_TOKEN 必填（读私有仓 contents；org token 或对两仓可读的 PAT） +# 退出码: 0=查询成功（输出统一 JSONL）| 2=参数/环境 | 3=任一源链断（fail-closed， +# 不可信数据不出结果——宁红勿假） +set -uo pipefail +DIR="$(cd "$(dirname "$0")" && pwd)" +GH="${GH:-gh}" + +CARD_FILTER=""; JSON_ONLY=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --card) CARD_FILTER="${2:?}"; shift 2 ;; + --json) JSON_ONLY=1; shift ;; + *) echo "未知参数 $1（用法见文件头）" >&2; exit 2 ;; + esac +done +[[ -n "${GH_TOKEN:-}" ]] || { echo "GH_TOKEN 未设置（需对两仓 contents 读权限）" >&2; exit 2; } +command -v "$GH" >/dev/null 2>&1 || { echo "gh 不可用" >&2; exit 2; } + +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +# ---- 拉源（404=该源尚无影子记录：过渡期合法，跳过留痕；其余 API 错=fail-closed） ---- +fetch_file() { # fetch_file ；rc=1=源缺席（404，非红） + local repo="$1" branch="$2" path="$3" out="$4" + if "$GH" api "repos/$repo/contents/$path?ref=$branch" >"$TMP/api.json" 2>"$TMP/api.err"; then + python3 - "$TMP/api.json" "$out" <<'PYEOF' +import base64, json, sys +d = json.load(open(sys.argv[1], encoding="utf-8")) +open(sys.argv[2], "w", encoding="utf-8", newline="\n").write( + base64.b64decode(d["content"]).decode("utf-8")) +PYEOF + return 0 + fi + if grep -qi 'not found' "$TMP/api.err" 2>/dev/null; then + return 1 # 源缺席（尚无影子记录）——过渡期合法，非红 + fi + echo "FATAL: $repo@$branch $path 拉取失败（非 404）：" >&2; cat "$TMP/api.err" >&2; exit 2 +} + +SRC_METER="$TMP/metering"; mkdir -p "$SRC_METER" +if "$GH" api "repos/Cloudbird-Software/CI-Workflows/contents?ref=metering-ledger" >"$TMP/list.json" 2>"$TMP/api.err"; then + python3 - "$TMP/list.json" "$SRC_METER" <<'PYEOF' +import base64, json, sys +for ent in json.load(open(sys.argv[1], encoding="utf-8")): + if ent["type"] == "file" and ent["name"].startswith("shadow-evidence-") and ent["name"].endswith(".jsonl"): + open(f"{sys.argv[2]}/{ent['name']}", "w", encoding="utf-8", newline="\n").write( + base64.b64decode(ent["content"]).decode("utf-8")) +PYEOF +else + if ! grep -qi 'not found' "$TMP/api.err" 2>/dev/null; then + echo "FATAL: metering-ledger 目录拉取失败：" >&2; cat "$TMP/api.err" >&2; exit 2 + fi +fi +DRILL_OK=0; fetch_file "Cloudbird-Software/.github" "drill-ledger" "governance/drill/shadow-evidence.jsonl" "$TMP/drill.jsonl" && DRILL_OK=1 || [[ $? -eq 1 ]] || exit 2 +BUTLER_OK=0; fetch_file "Cloudbird-Software/.github" "butler-ledger" "governance/butler/shadow-evidence.jsonl" "$TMP/butler.jsonl" && BUTLER_OK=1 || [[ $? -eq 1 ]] || exit 2 + +# ---- 逐源验链 + 归并输出（链断=exit 3：不可信数据不出结果） ---- +export CARD_FILTER JSON_ONLY DRILL_OK BUTLER_OK +python3 - "$DIR/evidence_shadow.py" "$SRC_METER" "$TMP/drill.jsonl" "$TMP/butler.jsonl" "$TMP" <<'PYEOF' +import glob, json, os, sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(sys.argv[1]))) +import evidence_shadow # noqa: E402 验链与 CI-Workflows 侧同源语义 + +metering_dir, drill_f, butler_f, tmp = sys.argv[2:6] +sources = {"metering": sorted(glob.glob(os.path.join(metering_dir, "shadow-evidence-*.jsonl"))), + "drill": [drill_f] if os.environ.get("DRILL_OK") == "1" else [], + "butler": [butler_f] if os.environ.get("BUTLER_OK") == "1" else []} +errs, recs = [], [] +for src, files in sources.items(): + for f in files: + if not os.path.isfile(f) or os.path.getsize(f) == 0: + continue + errs.extend(evidence_shadow.verify_file(f)) + for ln in evidence_shadow.read_lines(f): + recs.append({"source": src, **json.loads(ln)}) +if errs: + for e in errs: + print(f"CHAIN {e}", file=sys.stderr) + print("FATAL: 任一源链断——统一查询拒绝出结果（fail-closed，宁红勿假）", file=sys.stderr) + sys.exit(3) + +card = os.environ.get("CARD_FILTER") or None +recs.sort(key=lambda r: (r.get("ts", ""), r.get("source"))) +out = [r for r in recs if not card or r.get("subject", {}).get("card") == card] +for r in out: + print(json.dumps(r, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + +summary = { + "total": len(out), + "by_source": {s: sum(1 for r in out if r["source"] == s) for s in ("metering", "drill", "butler")}, + "by_tenant": {}, + "by_card_top": {}, +} +for r in out: + t = r.get("subject", {}).get("tenant", "?") + c = r.get("subject", {}).get("card", "?") + summary["by_tenant"][t] = summary["by_tenant"].get(t, 0) + 1 + summary["by_card_top"][c] = summary["by_card_top"].get(c, 0) + 1 +summary["by_card_top"] = dict(sorted(summary["by_card_top"].items(), key=lambda kv: -kv[1])[:10]) +line = json.dumps(summary, ensure_ascii=False, sort_keys=True) +if os.environ.get("JSON_ONLY") == "1": + print(line) +else: + print(f"SUMMARY {line}", file=sys.stderr) +PYEOF diff --git a/governance/evidence_shadow.py b/governance/evidence_shadow.py new file mode 100644 index 0000000..128d819 --- /dev/null +++ b/governance/evidence_shadow.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""evidence_shadow.py —— .github 侧证据账本 schema v1 影子双写（IR-0006 W1-B2 / ADR-0103） + +BEH-03 三源渐进对齐（butler/drill 侧）：新事件在写原载体（AUDIT 行 / drill +history.jsonl——均只增不改、冻结只读过渡）的同时，按统一证据 schema v1 +（standards/evidence/record.schema.yaml，cloudbird/evidence-standard/record@1） +双写影子账本（单文件链，append-only + prev_hash/hash 链式 hash）。 + +与 CI-Workflows 侧 pipeline/metering/shadow_evidence.py 同款执法（周片→单文件 +唯一差异）：tenant/card 必填、payload ≤4096B 拒写（INV-06）、链字段写入器独占、 +验链 fail-closed。 + +子命令： + append 事件文件 → 校验 + 链式追加（drill.py record / butler-audit.sh 双写调用） + verify 验链：seq 连续 / prev_hash 链 / hash 重算 / tenant·card 复检 + relink 本地影子续接远端基链（butler-ledger / drill-ledger 落盘前合并用） + +card 约定：卡绑定 owner/repo#issue（join key，AC-4）；无卡上下文的基建事件用 +哨兵 #0——#0=未绑定卡，不参与卡聚合。 +退出码：0=成功 | 2=参数/环境 | 3=记录/链无效（fail-closed） +""" +import argparse +import hashlib +import json +import os +import re +import sys + +PAYLOAD_LIMIT = 4096 +KINDS = {"gate", "cost", "approval", "decision"} +ROLES = {"owner", "agent", "bot", "human"} +CARD_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+#[0-9]+$") + + +def die(code, msg): + print(msg, file=sys.stderr) + sys.exit(code) + + +def canonical(obj): + return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + + +def content_hash(rec): + return hashlib.sha256( + canonical({k: v for k, v in rec.items() if k != "hash"}).encode("utf-8")).hexdigest() + + +def read_lines(path: str): + if not os.path.isfile(path): + return [] + with open(path, encoding="utf-8") as f: + return [ln for ln in (l.strip() for l in f) if ln] + + +def validate_event(ev: dict): + if {"seq", "prev_hash", "hash"} & set(ev): + die(3, "影子事件不得自带 seq/prev_hash/hash（链字段由写入器独占）") + if ev.get("kind") not in KINDS: + die(3, f"kind 非法: {ev.get('kind')!r}（合法 {sorted(KINDS)}）") + for k in ("ts", "action", "verdict"): + if not str(ev.get(k) or "").strip(): + die(3, f"{k} 必填") + subject = ev.get("subject") + if not isinstance(subject, dict): + die(3, "subject 必填（对象）") + if not str(subject.get("tenant") or "").strip(): + die(3, "subject.tenant 必填（宪法 §14a / AC-4c）") + if not CARD_RE.fullmatch(str(subject.get("card") or "")): + die(3, "subject.card 必填且形如 owner/repo#issue（AC-4 join key）") + actor = ev.get("actor") + if not isinstance(actor, dict) or not str(actor.get("identity") or "").strip() \ + or actor.get("role") not in ROLES: + die(3, "actor 四元不齐（identity/role 必填，role ∈ owner/agent/bot/human）") + payload = ev.get("payload", None) + if payload is not None: + if not isinstance(payload, str): + die(3, "payload 须为字符串或 null") + if len(payload.encode("utf-8")) > PAYLOAD_LIMIT: + die(3, f"payload {len(payload.encode('utf-8'))}B > {PAYLOAD_LIMIT}B（INV-06：超限拒写）") + + +def append(file_: str, ev: dict) -> dict: + validate_event(ev) + lines = read_lines(file_) + rec = {k: v for k, v in ev.items() if v is not None} + rec["seq"] = len(lines) + 1 + rec["prev_hash"] = json.loads(lines[-1])["hash"] if lines else None + rec["hash"] = content_hash(rec) + os.makedirs(os.path.dirname(os.path.abspath(file_)), exist_ok=True) + with open(file_, "a", encoding="utf-8", newline="\n") as f: + f.write(canonical(rec) + "\n") + return rec + + +def verify_file(path: str) -> list: + errs, prev = [], None + for i, ln in enumerate(read_lines(path), 1): + try: + rec = json.loads(ln) + except json.JSONDecodeError as e: + errs.append(f"{path} 第 {i} 行 JSON 畸形: {e}") + break + if rec.get("seq") != i: + errs.append(f"{path} 第 {i} 行 seq={rec.get('seq')} 断号") + if rec.get("prev_hash") != prev: + errs.append(f"{path} 第 {i} 行 prev_hash 断链") + if rec.get("hash") != content_hash(rec): + errs.append(f"{path} 第 {i} 行 hash 重算不符（篡改）") + if not str((rec.get("subject") or {}).get("tenant") or "").strip(): + errs.append(f"{path} 第 {i} 行 tenant 缺失（AC-4c）") + if not CARD_RE.fullmatch(str((rec.get("subject") or {}).get("card") or "")): + errs.append(f"{path} 第 {i} 行 card 缺失/非法（AC-4）") + payload = rec.get("payload", None) + if isinstance(payload, str) and len(payload.encode("utf-8")) > PAYLOAD_LIMIT: + errs.append(f"{path} 第 {i} 行 payload 超限（执法缺口）") + prev = rec.get("hash") + return errs + + +def main(): + ap = argparse.ArgumentParser(prog="evidence_shadow.py", description=__doc__.splitlines()[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("append") + p.add_argument("--file", required=True, help="影子账本 jsonl（单文件链）") + p.add_argument("--event-file", required=True) + + p = sub.add_parser("verify") + p.add_argument("--file", required=True) + + p = sub.add_parser("relink") + p.add_argument("--base", required=True, help="远端基链（不存在=创世续接）") + p.add_argument("--local", required=True) + p.add_argument("--out", required=True) + args = ap.parse_args() + + if args.cmd == "append": + with open(args.event_file, encoding="utf-8") as f: + ev = json.load(f) + rec = append(args.file, ev) + print(canonical(rec)) + elif args.cmd == "verify": + errs = verify_file(args.file) + if errs: + for e in errs: + print(f"CHAIN {e}", file=sys.stderr) + die(3, f"影子验链失败：{len(errs)} 处") + print(f"OK {args.file}: {len(read_lines(args.file))} 条，链完整") + elif args.cmd == "relink": + errs, base_recs = [], [] + if os.path.isfile(args.base): + errs.extend(verify_file(args.base)) + base_recs = [json.loads(l) for l in read_lines(args.base)] + errs.extend(verify_file(args.local)) + if errs: + for e in errs[:10]: + print(f"CHAIN {e}", file=sys.stderr) + die(3, "基链或本地影子片验链失败——拒绝合并（防覆盖掩盖篡改）") + merged = list(base_recs) + for rec in [json.loads(l) for l in read_lines(args.local)]: + rec = dict(rec) + rec["seq"] = len(merged) + 1 + rec["prev_hash"] = merged[-1]["hash"] if merged else None + rec["hash"] = content_hash(rec) + merged.append(rec) + with open(args.out, "w", encoding="utf-8", newline="\n") as f: + for rec in merged: + f.write(canonical(rec) + "\n") + print(f"relink → {args.out}（base {len(base_recs)} + local {len(merged) - len(base_recs)} 条）") + + +if __name__ == "__main__": + main() diff --git a/governance/tests/test-evidence-shadow.sh b/governance/tests/test-evidence-shadow.sh new file mode 100644 index 0000000..ab0c42e --- /dev/null +++ b/governance/tests/test-evidence-shadow.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# test-evidence-shadow.sh —— 影子双写/验链/查询执法自测（IR-0006 W1-B2 / 卡 #407） +# +# 覆盖（卡 AC 对应）： +# AC-4a evidence_shadow append/verify/relink 闭环 + 统一输出形态 +# AC-4b 原 JSONL 只增不改：drill record 后 history 原行字节不变、只追加 +# AC-4c tenant 缺失拒写；audit_emit 影子行带 tenant +# 负向：篡改/断链 → 验链红（exit 3）；payload 超限拒写（INV-06） +# 用法: bash governance/tests/test-evidence-shadow.sh（gate.yml 自动纳入） +set -uo pipefail +DIR="$(cd "$(dirname "$0")/../.." && pwd)" +FAILS=0 +pass() { echo "PASS $1"; } +fail() { echo "FAIL $1"; FAILS=$((FAILS+1)); } + +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +SHADOW="$TMP/shadow-evidence.jsonl" + +ev() { # ev [tenant] → 事件 JSON + local tenant="${2:-cloudbird-internal}" + cat >"$1" </dev/null \ + && pass "影子 append #1" || fail "影子 append #1" +python3 "$DIR/governance/evidence_shadow.py" append --file "$SHADOW" --event-file "$TMP/ev2.json" >/dev/null \ + && pass "影子 append #2（链式续接）" || fail "影子 append #2" +python3 "$DIR/governance/evidence_shadow.py" verify --file "$SHADOW" >/dev/null \ + && pass "影子验链绿（2 条）" || fail "影子验链" + +# ---- AC-4c 负向：tenant 缺失拒写 ---- +cat >"$TMP/bad.json" <<'EOF' +{"ts": "2026-08-29T00:00:00Z", "kind": "gate", "action": "x", "verdict": "pass", + "subject": {"card": "Cloudbird-Software/.github#407"}, + "actor": {"identity": "t", "role": "bot", "model": null}} +EOF +python3 "$DIR/governance/evidence_shadow.py" append --file "$SHADOW" --event-file "$TMP/bad.json" >/dev/null 2>&1 +[[ $? -eq 3 ]] && pass "AC-4c tenant 缺失 → 拒写（exit 3）" || fail "AC-4c tenant 缺失未拒写" +N=$(wc -l <"$SHADOW" | tr -d ' ') +[[ "$N" == "2" ]] && pass "拒写零副作用（仍 2 条）" || fail "拒写后有副作用（$N 条）" + +# ---- 负向：篡改 → 验链红 ---- +TAMPER="$TMP/tamper.jsonl"; cp "$SHADOW" "$TAMPER" +python3 - "$TAMPER" <<'PYEOF' +import json, sys +lines = open(sys.argv[1], encoding="utf-8").read().splitlines() +rec = json.loads(lines[0]); rec["verdict"] = "tampered" +lines[0] = json.dumps(rec, ensure_ascii=False, sort_keys=True, separators=(",", ":")) +open(sys.argv[1], "w", encoding="utf-8", newline="\n").write("\n".join(lines) + "\n") +PYEOF +python3 "$DIR/governance/evidence_shadow.py" verify --file "$TAMPER" >/dev/null 2>&1 +[[ $? -eq 3 ]] && pass "篡改 → 验链红（exit 3，链断=红）" || fail "篡改未检出" + +# ---- 负向：payload 4097B 拒写（INV-06） ---- +python3 - "$TMP/big.json" <<'PYEOF' +import json, sys +ev = {"ts": "2026-08-29T00:00:00Z", "kind": "gate", "action": "x", "verdict": "pass", + "subject": {"card": "Cloudbird-Software/.github#407", "tenant": "t"}, + "actor": {"identity": "t", "role": "bot", "model": None}, "payload": "x" * 4097} +open(sys.argv[1], "w", encoding="utf-8").write(json.dumps(ev)) +PYEOF +python3 "$DIR/governance/evidence_shadow.py" append --file "$SHADOW" --event-file "$TMP/big.json" >/dev/null 2>&1 +[[ $? -eq 3 ]] && pass "payload 4097B → 拒写（INV-06）" || fail "4KB 上限未执法" + +# ---- relink：本地续接基链 + 合并片验链 ---- +python3 "$DIR/governance/evidence_shadow.py" relink --base "$SHADOW" --local "$TAMPER" --out "$TMP/merged.jsonl" >/dev/null 2>&1 +[[ $? -ne 0 ]] && pass "relink 拒绝坏基链/坏本地（防覆盖掩盖篡改）" || fail "relink 未拒坏链" +BASE2="$TMP/base2.jsonl"; cp "$SHADOW" "$BASE2" +python3 "$DIR/governance/evidence_shadow.py" relink --base "$BASE2" --local "$SHADOW" --out "$TMP/merged2.jsonl" >/dev/null \ + && python3 "$DIR/governance/evidence_shadow.py" verify --file "$TMP/merged2.jsonl" >/dev/null \ + && pass "relink 创世续接 + 合并片验链绿（4 条）" || fail "relink 正向闭环" + +# ---- drill record 双写（AC-4b：原台账只增不改） ---- +DHIST="$TMP/drill/history.jsonl"; mkdir -p "$(dirname "$DHIST")" +mkdrill() { # mkdrill + cat >"$TMP/drec.json" </dev/null \ + && pass "drill record #1（history+影子双写）" || fail "drill record #1" +DSHADOW="$TMP/drill/shadow-evidence.jsonl" +[[ -s "$DSHADOW" ]] && pass "drill 影子已落盘" || fail "drill 影子缺失" +H1=$(cat "$DHIST") +mkdrill "2026-08-29T02:00:00Z" "run-2" +python3 "$DIR/governance/drill/drill.py" record --history "$DHIST" --json "$(cat "$TMP/drec.json")" >/dev/null \ + && pass "drill record #2" || fail "drill record #2" +head -1 "$DHIST" | grep -qx "$H1" \ + && pass "AC-4b drill history 原行字节不变（只追加）" || fail "AC-4b drill history 原行被改" +python3 "$DIR/governance/evidence_shadow.py" verify --file "$DSHADOW" >/dev/null \ + && pass "drill 影子验链绿（双写一致）" || fail "drill 影子链断" +python3 - "$DSHADOW" <<'PYEOF' +import json, sys +rec = json.loads(open(sys.argv[1], encoding="utf-8").readlines()[0]) +assert rec["kind"] == "gate" and rec["action"] == "drill-seed-drill", rec +assert rec["subject"]["tenant"] == "cloudbird-internal", rec +assert rec["actor"]["identity"] == "drill-seed-bot", rec +print("OK") +PYEOF +[[ $? -eq 0 ]] && pass "drill 影子字段对齐 schema v1" || fail "drill 影子字段断言" + +# ---- butler audit_emit 双写（AC-4c：影子行带 tenant） ---- +BUTLER_SHADOW="$TMP/butler/shadow-evidence.jsonl" +BUTLER_SHADOW_FILE="$BUTLER_SHADOW" bash "$DIR/governance/butler-audit.sh" test-butler cron ok '{"k":1}' >/dev/null \ + && pass "butler-audit CLI（AUDIT 行 + 影子双写）" || fail "butler-audit CLI" +[[ -s "$BUTLER_SHADOW" ]] && pass "butler 影子已落盘" || fail "butler 影子缺失" +python3 "$DIR/governance/evidence_shadow.py" verify --file "$BUTLER_SHADOW" >/dev/null \ + && pass "butler 影子验链绿" || fail "butler 影子链断" +grep -q '"butler-test-butler"' "$BUTLER_SHADOW" && grep -q '"tenant":"cloudbird-internal"' "$BUTLER_SHADOW" \ + && pass "AC-4c butler 影子带 action/tenant" || fail "butler 影子字段断言" +# ---- 负向：影子写入失败 = fail-closed（BEH-01 双写不一致当场可见） ---- +mkdir -p "$TMP/shadow-is-dir" +BUTLER_SHADOW_FILE="$TMP/shadow-is-dir" bash "$DIR/governance/butler-audit.sh" test-butler cron ok '{"k":1}' >/dev/null 2>&1 +[[ $? -ne 0 ]] && pass "影子写入失败 → butler-audit 非零退出（fail-closed）" || fail "影子写失败未 fail-closed" + +# ---- evidence-query 三源统一查询（AC-4a：归并 + 验链 + fail-closed） ---- +GHSTUB="$TMP/gh-stub" +mkdir -p "$TMP/fixtures" +# fixture：metering=周分片列表（$SHADOW）；drill/butler=各自影子（独立成链） +python3 - "$SHADOW" "$DSHADOW" "$BUTLER_SHADOW" "$TMP/fixtures" <<'PYEOF' +import base64, json, sys +shadow, dshadow, bshadow, fixdir = sys.argv[1:5] +b64 = lambda p: base64.b64encode(open(p, "rb").read()).decode() +json.dump([{"type": "file", "name": "shadow-evidence-2026-W35.jsonl", "content": b64(shadow)}], + open(f"{fixdir}/metering-list.json", "w")) +json.dump({"type": "file", "content": b64(dshadow)}, open(f"{fixdir}/drill.json", "w")) +json.dump({"type": "file", "content": b64(bshadow)}, open(f"{fixdir}/butler.json", "w")) +PYEOF +cat >"$GHSTUB" <<'STUBEOF' +#!/usr/bin/env bash +# gh 桩：按 API 路径回放 fixture（evidence-query 测试专用） +set -euo pipefail +url="${2:-}" +F="${GH_STUB_FIXTURES:?}" +case "$url" in + "repos/Cloudbird-Software/CI-Workflows/contents?ref=metering-ledger") + cat "$F/metering-list.json" ;; + "repos/Cloudbird-Software/.github/contents/governance/drill/shadow-evidence.jsonl?ref=drill-ledger") + if [[ -f "$F/drill-tampered.json" ]]; then cat "$F/drill-tampered.json"; else cat "$F/drill.json"; fi ;; + "repos/Cloudbird-Software/.github/contents/governance/butler/shadow-evidence.jsonl?ref=butler-ledger") + if [[ "${GH_STUB_BUTLER_MISSING:-}" == "1" ]]; then echo "gh: Not Found (HTTP 404)" >&2; exit 1; fi + cat "$F/butler.json" ;; + *) echo "gh-stub: 意外 URL $url" >&2; exit 1 ;; +esac +STUBEOF +chmod +x "$GHSTUB" +nlines() { grep -c . <<<"$1" || true; } # 非空行计数（wc 对空 herestring 误报 1） +QOUT=$(GH="$GHSTUB" GH_STUB_FIXTURES="$TMP/fixtures" GH_TOKEN=stub \ + bash "$DIR/governance/evidence-query.sh" 2>"$TMP/q.err"); QRC=$? +N=$(nlines "$QOUT") +[[ $QRC -eq 0 && "$N" -eq 5 ]] && pass "统一查询归并三源（5 条，rc=0）" || fail "统一查询（rc=$QRC 行=$N）" +grep -q '"source":"metering"' <<<"$QOUT" && grep -q '"source":"drill"' <<<"$QOUT" && grep -q '"source":"butler"' <<<"$QOUT" \ + && pass "三源标记齐全（source 字段）" || fail "source 标记缺失" +grep -q '^SUMMARY ' "$TMP/q.err" && grep -Eq '"metering": ?2' "$TMP/q.err" \ + && pass "分源统计（stderr SUMMARY）" || fail "SUMMARY 统计缺失" +# --card 过滤：metering 影子绑 #407，drill/butler 哨兵 #0 → 仅 2 条 +COUT=$(GH="$GHSTUB" GH_STUB_FIXTURES="$TMP/fixtures" GH_TOKEN=stub \ + bash "$DIR/governance/evidence-query.sh" --card Cloudbird-Software/.github#407 2>/dev/null); CRC=$? +CN=$(nlines "$COUT") +[[ $CRC -eq 0 && "$CN" -eq 2 ]] && pass "--card 过滤（#407 → 2 条）" || fail "--card 过滤（rc=$CRC 行=$CN）" +# 源缺席（404）：过渡期合法，非红（须在篡改 fixture 生成前跑——桩对篡改片优先回放） +MOUT=$(GH="$GHSTUB" GH_STUB_FIXTURES="$TMP/fixtures" GH_TOKEN=stub GH_STUB_BUTLER_MISSING=1 \ + bash "$DIR/governance/evidence-query.sh" 2>"$TMP/m.err"); MRC=$? +MN=$(nlines "$MOUT") +[[ $MRC -eq 0 && "$MN" -eq 4 ]] && pass "butler 源缺席（404）→ 跳过非红（4 条）" || fail "源缺席误红（rc=$MRC 行=$MN）" +# 负向：任一源链断 → exit 3 且 stdout 零输出（不可信数据不出结果） +python3 - "$TMP/fixtures" <<'PYEOF' +import base64, json, sys +fixdir = sys.argv[1] +lines = base64.b64decode(json.load(open(f"{fixdir}/drill.json"))["content"]).decode().splitlines() +rec = json.loads(lines[0]); rec["verdict"] = "tampered" +lines[0] = json.dumps(rec, ensure_ascii=False, sort_keys=True, separators=(",", ":")) +json.dump({"type": "file", "content": base64.b64encode(("\n".join(lines) + "\n").encode()).decode()}, + open(f"{fixdir}/drill-tampered.json", "w")) +PYEOF +TOUT=$(GH="$GHSTUB" GH_STUB_FIXTURES="$TMP/fixtures" GH_TOKEN=stub \ + bash "$DIR/governance/evidence-query.sh" 2>"$TMP/t.err"); TRC=$? +TN=$(nlines "$TOUT") +[[ $TRC -eq 3 && "$TN" -eq 0 ]] && pass "源链断 → exit 3 零输出（fail-closed）" || fail "链断未 fail-closed（rc=$TRC 行=$TN）" + +echo "----------------------------------------" +if [[ $FAILS -eq 0 ]]; then echo "test-evidence-shadow: PASS"; exit 0; fi +echo "test-evidence-shadow: $FAILS 处失败"; exit 1