From 14d2149a997f1b3838ec79f00a311a5e1a720885 Mon Sep 17 00:00:00 2001 From: randypanding Date: Sat, 22 Aug 2026 04:02:58 +0800 Subject: [PATCH] =?UTF-8?q?feat(dashboard):=20=E5=9B=9B=E7=B1=BB=E6=8C=87?= =?UTF-8?q?=E6=A0=87=E8=81=9A=E5=90=88+human-brief=20=E6=B8=B2=E6=9F=93?= =?UTF-8?q?=EF=BC=88W5-C4=20.github#227=EF=BC=8CADR-0073=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 注意力会计(签署 p50/p90/可疑快速签署/needs-human p90 停摆线)/安全正确性 (误放行窗过滤/演习分母口径)/成本(声明价折算/零 IR 不除零)/用户结果 (产品读取位 pending 不造数+季度配额记录位)+ eval CLI,fixture 自测 9 例。PR 3/5。Card: Cloudbird-Software/.github#227 --- governance/metrics.py | 187 +++++++++++++++++++++++- governance/tests/test-metrics-groups.sh | 155 ++++++++++++++++++++ 2 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 governance/tests/test-metrics-groups.sh diff --git a/governance/metrics.py b/governance/metrics.py index c76c1c9..35dad0b 100644 --- a/governance/metrics.py +++ b/governance/metrics.py @@ -17,6 +17,7 @@ 阈值唯一来源 governance/policy/metrics.yaml(本库不内嵌任何阈值)。 """ import argparse +import datetime as _dt import json import math import os @@ -109,20 +110,198 @@ def north_star(data, policy): } +# ---------- 四类指标(AC-2/AC-4,宪法 §8;缺数据=pending 不造数) ---------- + +def _fmt_s(sec): + """秒→人读时长(签署/停留呈现用)。""" + if sec is None: + return "pending" + if sec < 90: + return f"{sec:.0f}s" + if sec < 5400: + return f"{sec / 60:.1f}min" + return f"{sec / 3600:.1f}h" + + +def attention_stats(data, policy): + """注意力会计:签署耗时/needs-human p90/超时默认触发数/可疑快速签署数(宪法 §7)。""" + at = policy["attention"] + durations = data.get("sign_durations_seconds") or [] + dwell = data.get("needs_human_dwell_hours") or [] + p90 = percentile(dwell, 0.90) + stop_h = at["needs_human_p90_stop_hours"] + return { + "sign_count": len(durations), + "sign_p50_seconds": percentile(durations, 0.50), + "sign_p90_seconds": percentile(durations, 0.90), + "sign_in_flight": data.get("sign_in_flight", 0), # 仍 ir-draft 未签(不计耗时统计) + "suspicious_fast_signs": sum(1 for s in durations if s < at["suspicious_fast_sign_seconds"]), + "suspicious_fast_sign_seconds": at["suspicious_fast_sign_seconds"], + "needs_human_count": len(dwell), + "needs_human_p90_hours": p90, + "needs_human_p90_stop": (None if p90 is None else bool(p90 > stop_h)), + "needs_human_stop_hours": stop_h, + # 数据源未落(决策卡/审计包基建在后续波次)——pending 诚实显示,不冒充 0 + "timeout_defaults": "pending:决策卡超时默认触发数(数据源未落)", + "owner_minutes_per_merge": "pending:每合并 owner 分钟(评审事件流未落)", + "audit_overtime_rate": "pending:周审计超时率(审计包组装未落)", + } + + +def security_stats(data, policy): + """安全正确性:误放行/误拒(arbiter 台账窗内)、泄漏、演习红率、陷阱拦截率。""" + win_days = policy["security"]["false_decision_window_days"] + now = _parse_iso(data.get("now")) + allow = deny = 0 + for rec in data.get("false_decision_lines") or []: + ts = _parse_iso(rec.get("date")) + if ts and now and (now - ts).days <= win_days: + if rec.get("kind") == "false-allow": + allow += 1 + elif rec.get("kind") == "false-deny": + deny += 1 + drills = data.get("drill_records") or [] + reds = sum(1 for r in drills if r.get("verdict") == "red") + denom = sum(1 for r in drills if r.get("verdict") in ("red", "green")) # no-surface 不入分母 + return { + "false_allow_window": allow, + "false_deny_window": deny, + "false_decision_window_days": win_days, + "drill_red": reds, "drill_denom": denom, + "state_change_leaks": "pending:未经仲裁的状态变更泄漏检测面未建", + "trap_intercept_rate": "pending:陷阱拦截率(ADR-0071 W5-C2 信任门未落)", + } + + +def cost_stats(data, policy): + """成本:单 IR 美元(声明价折算——公开仓计费净额 $0,防失控速率的虚拟口径)。""" + co = policy["cost"] + minutes = data.get("actions_minutes_month") + tokens = data.get("llm_tokens_month") + irs = data.get("ir_count_month") + per_ir = None + if minutes is not None and tokens is not None and irs: + per_ir = round((minutes * co["actions_price_per_minute_usd"] + + tokens / 1000 * co["llm_price_per_1k_tokens_usd"]) / irs, 4) + return { + "actions_minutes_month": minutes, + "llm_tokens_month": tokens, + "ir_count_month": irs, + "per_ir_usd": per_ir, + "per_ir_usd_note": ("pending:当月零 IR——不除零(#98 T2)" if irs == 0 + else "声明价折算(actions $/min × 分钟 + LLM $/1k × token)/ 当月 IR 数"), + "actions_price_per_minute_usd": co["actions_price_per_minute_usd"], + "llm_price_per_1k_tokens_usd": co["llm_price_per_1k_tokens_usd"], + "snapshot_age_minutes": data.get("cost_snapshot_age_minutes"), + "butler_usd_week": "pending:管家美元/周(按 workflow 分钟拆账未落)", + "patrol_yield": "pending:patrol yield(W3-C2 demo-probe 期,真实 bug 计数为 0 分母)", + } + + +def user_results_stats(data, policy): + """用户结果指标:各产品仓声明读取位(文件缺失=pending)+ 季度难测配额记录位。""" + ur = policy["user_results"] + files = data.get("user_metric_files") or {} + products = {} + for p in ur.get("products") or []: + repo = p["repo"] + m = files.get(repo) + if isinstance(m, dict) and m.get("metric_key") and "value" in m: + products[repo] = {"status": "ok", **m} + else: + products[repo] = {"status": "pending", + "detail": f"产品仓未声明用户结果指标({ur['read_path']} 缺失——埋点滞后,ADR-0073 后果节)"} + quota = (ur.get("quarterly_hard_quota") or {}).get("entries") or [] + return {"products": products, + "quarterly_hard_quota": {"entries": quota, + "note": "配额制:每季度刻意做一个难测的——entries 空=本季未立(记录位,owner 回填)"}} + + +def build_payload(data, policy): + """schema v2 组装:north_star + metrics 四组(generated_at 由采集层注入)。""" + return { + "generated_at": data.get("generated_at"), + "north_star": north_star(data, policy), + "metrics": { + "attention": attention_stats(data, policy), + "security": security_stats(data, policy), + "cost": cost_stats(data, policy), + "user_results": user_results_stats(data, policy), + }, + } + + +# ---------- human-brief 渲染(宪法 §8"人 30 秒读懂";正文顶部=北极星对) ---------- + +def render_brief(payload): + ns, m = payload["north_star"], payload["metrics"] + z = ns["zero_touch_merges_7d"] + if z["zeroed"]: + head = (f"零接触合并数(近 7 天):**0(显示归零——护栏破线:" + f"{'、'.join(z['zeroed_reasons'])})**;原始计数 {z['raw']} 保留在 JSON raw" + "(呈现层归零,非数据删除)") + elif z["raw"] == 0: + head = "零接触合并数(近 7 天):**0**(零接触合并周——如实 0,非归零)" + else: + head = f"零接触合并数(近 7 天):**{z['raw']}**(护栏全绿——如实显示)" + gtxt = " · ".join(f"{n} {ns['guardrails'][n]['status']}" for n in GUARD_ORDER) + glines = "\n".join(f" - {n}: **{g['status']}**({g['detail']})" + for n, g in ns["guardrails"].items()) + at, se, co, ur = m["attention"], m["security"], m["cost"], m["user_results"] + p90h = "pending" if at["needs_human_p90_hours"] is None else f"{at['needs_human_p90_hours']:.0f}h" + per_ir = "pending" if co["per_ir_usd"] is None else f"${co['per_ir_usd']}" + prod_txt = " · ".join(f"{r}: {v['status']}" + (f"({v['metric_key']}={v['value']}{v.get('unit', '')})" if v["status"] == "ok" else "") + for r, v in sorted(ur["products"].items())) + quota = "、".join(f"{e['quarter']} {e['product']}({e.get('status', 'planned')})" + for e in ur["quarterly_hard_quota"]["entries"]) or "本季未立(记录位空——诚实显示)" + return f"""## 北极星对(同屏互锁 · 宪法 §8 / ADR-0073 决策 1) + +{head} +质量护栏:{gtxt} +{glines} +盲区(pending,不参与归零判定——缺数据≠劣化):{'、'.join(ns['pending_blind_zones']) or '无'} + +## 四类指标(宪法 §8 全景) + +- 注意力会计:签署 {at['sign_count']} 例(p50 {_fmt_s(at['sign_p50_seconds'])} / p90 {_fmt_s(at['sign_p90_seconds'])},在途 {at['sign_in_flight']})· 可疑快速签署(<{at['suspicious_fast_sign_seconds']}s){at['suspicious_fast_signs']} 例 · needs-human {at['needs_human_count']} 张 p90 停留 {p90h}(>{at['needs_human_stop_hours']}h=整机停摆线)· 超时默认触发 pending +- 安全正确性({se['false_decision_window_days']} 天窗):误放行 {se['false_allow_window']} · 误拒 {se['false_deny_window']} · 演习红率 {se['drill_red']}/{se['drill_denom']} · 泄漏 pending · 陷阱拦截率 pending +- 成本:单 IR {per_ir}(Actions {co['actions_minutes_month'] if co['actions_minutes_month'] is not None else 'pending'} 分钟 + LLM {co['llm_tokens_month'] if co['llm_tokens_month'] is not None else 'pending'} token,声明价)· 管家美元/周 pending · patrol yield pending +- 用户结果:{prod_txt} +- 季度难测配额:{quota} +""" + + +def _parse_iso(s): + if not s: + return None + try: + return _dt.datetime.fromisoformat(str(s).replace("Z", "+00:00")) + except ValueError: + return None + + def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = ap.add_subparsers(dest="cmd", required=True) a = sub.add_parser("northstar", help="北极星互锁判定(fixture 输入→JSON 输出)") a.add_argument("--input", required=True, help="JSON 文件:{zero_touch_merges_7d, escape_rate_sustained, ...}") a.add_argument("--policy", default=None) - a.add_argument("--now", default=None, help="注入时钟(ISO)——离线复算用") + a = sub.add_parser("eval", help="全量指标计算(fixture/采集层输入→payload JSON + human-brief)") + a.add_argument("--input", required=True, help="JSON 文件(dashboard 数据结构,见各 *_stats docstring)") + a.add_argument("--policy", default=None) + a.add_argument("--render", action="store_true", help="附加 human-brief markdown(分节符 ==== 后输出)") args = ap.parse_args(argv) policy = load_policy(args.policy) with open(args.input, encoding="utf-8") as f: data = json.load(f) - if args.now: # 显式注入优先(owner 复算可复现;缺省取系统钟) - os.environ["METRICS_NOW"] = args.now - print(json.dumps(north_star(data, policy), ensure_ascii=False, indent=2)) + if args.cmd == "northstar": + print(json.dumps(north_star(data, policy), ensure_ascii=False, indent=2)) + return 0 + payload = build_payload(data, policy) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + if args.render: + print("\n==== human-brief ====\n") + print(render_brief(payload), end="") return 0 diff --git a/governance/tests/test-metrics-groups.sh b/governance/tests/test-metrics-groups.sh new file mode 100644 index 0000000..32bf837 --- /dev/null +++ b/governance/tests/test-metrics-groups.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# test-metrics-groups.sh —— 四类指标聚合 + human-brief 渲染自测(W5-C4 AC-2/AC-4,ADR-0073) +# +# fixture 时间序列→p90/签署/率值/成本折算断言(零网络;实现=governance/metrics.py +# eval 子命令)。锁定口径: +# p90 最近邻秩(10 样本 1..10h → 9h)· 可疑快速签署按 policy 阈值计数 +# 误放行/误拒窗过滤(窗内外分离)· 演习 no-surface 不入分母 +# 成本声明价折算 + 零 IR 不除零 · 用户结果 pending 不造数 · 渲染含归零标注 +# 用法:bash governance/tests/test-metrics-groups.sh +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GOV="$(cd "$HERE/.." && pwd)" + +PY="" +for c in "${PYTHON:-}" python3 python py -3; do + [[ -n "$c" ]] || continue + "$c" -c 'import sys, yaml; print("ok")' >/dev/null 2>&1 || continue + PY="$c"; break +done +[[ -n "$PY" ]] || { echo "::error::无可用 python(含 pyyaml)"; exit 2; } + +PASS=0; FAIL=0 +pass() { PASS=$((PASS+1)); echo "PASS $1"; } +fail() { FAIL=$((FAIL+1)); echo "FAIL $1"; } +jget() { "$PY" -c "import json,sys;d=json.loads(sys.argv[1]);print(eval(sys.argv[2]))" "$1" "$2" 2>/dev/null | tr -d '\r'; } + +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +M="$GOV/metrics.py" + +cat >"$TMP/in.json" <<'EOF' +{ + "now": "2026-08-22T00:00:00Z", + "generated_at": "2026-08-22T00:00:00Z", + "zero_touch_merges_7d": 8, + "escape_rate_sustained": {"current": 0, "previous": 0}, + "revert_rate": {"num": 0, "denom": 8}, + "drill_red_rate": {"red": 3, "denom": 3}, + "false_allow": 0, + "sign_durations_seconds": [30, 120, 3600, 7200], + "sign_in_flight": 2, + "needs_human_dwell_hours": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "false_decision_lines": [ + {"date": "2026-08-10T00:00:00Z", "kind": "false-allow"}, + {"date": "2026-08-15T00:00:00Z", "kind": "false-deny"}, + {"date": "2026-05-01T00:00:00Z", "kind": "false-allow"}, + {"date": "2026-08-20T00:00:00Z", "kind": "infra"} + ], + "drill_records": [ + {"verdict": "red"}, {"verdict": "red"}, {"verdict": "green"}, {"verdict": "no-surface"} + ], + "actions_minutes_month": 100, + "llm_tokens_month": 50000, + "ir_count_month": 5, + "user_metric_files": {"mutual": {"metric_key": "ndcg@5", "value": 0.41, "unit": "", "updated_at": "2026-08-21T00:00:00Z"}} +} +EOF + +if ! OUT=$("$PY" "$M" eval --input "$TMP/in.json" --render 2>"$TMP/err.txt"); then + echo "::error::metrics.py eval 失败:$(cat "$TMP/err.txt")"; exit 2 +fi +JSON="${OUT%%====*}" + +# --- 注意力会计:p50/p90 最近邻秩 + 可疑快速签署(阈值来自 policy=60s) --- +[[ $(jget "$JSON" "d['metrics']['attention']['sign_p50_seconds']") == 120 \ + && $(jget "$JSON" "d['metrics']['attention']['sign_p90_seconds']") == 7200 \ + && $(jget "$JSON" "d['metrics']['attention']['suspicious_fast_signs']") == 1 \ + && $(jget "$JSON" "d['metrics']['attention']['sign_in_flight']") == 2 ]] \ + && pass "签署 p50=120s/p90=7200s(最近邻秩),可疑快速签署 1 例(30s<60s 阈)" \ + || fail "签署统计断言失败" + +# --- needs-human p90:10 样本 1..10h → p90=9h(未破 24h 停摆线) --- +[[ $(jget "$JSON" "d['metrics']['attention']['needs_human_p90_hours']") == 9 \ + && $(jget "$JSON" "d['metrics']['attention']['needs_human_p90_stop']") == False ]] \ + && pass "needs-human p90=9h(最近邻秩),停摆线未破" \ + || fail "needs-human p90 断言失败" + +# --- 安全正确性:窗过滤(30 天窗:1 误放行+1 误拒;窗外与 infra 不计)+ 演习分母 --- +[[ $(jget "$JSON" "d['metrics']['security']['false_allow_window']") == 1 \ + && $(jget "$JSON" "d['metrics']['security']['false_deny_window']") == 1 \ + && $(jget "$JSON" "d['metrics']['security']['drill_red']") == 2 \ + && $(jget "$JSON" "d['metrics']['security']['drill_denom']") == 3 ]] \ + && pass "误放行/误拒窗内各 1(窗外与 infra 不计);演习 2 红/3 可判定(no-surface 不入分母)" \ + || fail "安全正确性断言失败" + +# --- 成本:声明价折算 (100*0.008 + 50*0.002)/5 = 0.18 美元/IR --- +[[ $(jget "$JSON" "d['metrics']['cost']['per_ir_usd']") == 0.18 ]] \ + && pass '单 IR $0.18 =(100min×$0.008 + 50k token×$0.002)/ 5 IR' \ + || fail "成本折算断言失败(期望 0.18)" + +# --- 用户结果:声明仓 ok、其余 pending;配额空=未立 --- +[[ $(jget "$JSON" "d['metrics']['user_results']['products']['mutual']['status']") == ok \ + && $(jget "$JSON" "d['metrics']['user_results']['products']['mutual']['value']") == 0.41 \ + && $(jget "$JSON" "d['metrics']['user_results']['products']['Shorts_Director']['status']") == pending \ + && $(jget "$JSON" "len(d['metrics']['user_results']['quarterly_hard_quota']['entries'])") == 0 ]] \ + && pass "用户结果:mutual ok(0.41)、未声明仓 pending、配额记录位空=未立" \ + || fail "用户结果断言失败" + +# --- 渲染(AC-1 同屏 + §8 人 30 秒):北极星对在最顶,归零时含标注与 raw 保留 --- +BRIEF="${OUT#*====}" +if grep -q "^## 北极星对" <<<"$OUT" && grep -q "零接触合并数(近 7 天):\*\*8\*\*" <<<"$BRIEF" \ + && grep -q "## 四类指标" <<<"$BRIEF" && grep -q "可疑快速签署(<60s)1 例" <<<"$BRIEF" \ + && grep -q "needs-human 10 张 p90 停留 9h" <<<"$BRIEF" && grep -q "误放行 1 · 误拒 1" <<<"$BRIEF" \ + && grep -q "单 IR \$0.18" <<<"$BRIEF" && grep -q "季度难测配额:本季未立" <<<"$BRIEF"; then + pass "human-brief:北极星对置顶+四类指标一行一类+30 秒可读" +else fail "human-brief 渲染断言失败"; fi + +# --- 渲染归零形态:护栏破线 → 顶部显示 0+原因+raw 保留(AC-1 呈现层断言) --- +# 路径经 argv 传递(MSYS 自动转换——嵌入代码字符串的 /tmp 路径原生 python 不识别) +"$PY" - "$TMP/in.json" "$TMP/red.json" <<'PYEOF' +import json, sys +d = json.load(open(sys.argv[1], encoding="utf-8")) +d["drill_red_rate"] = {"red": 1, "denom": 3} # 0.33 < 1.0 破线 +d["drill_records"] = [{"verdict": "red"}, {"verdict": "green"}, {"verdict": "green"}] +json.dump(d, open(sys.argv[2], "w", encoding="utf-8"), ensure_ascii=False) +PYEOF +OUT2=$("$PY" "$M" eval --input "$TMP/red.json" --render 2>/dev/null) || { echo "::error::red eval 失败"; exit 2; } +JSON2="${OUT2%%====*}"; BRIEF2="${OUT2#*====}" +[[ $(jget "$JSON2" "d['north_star']['zero_touch_merges_7d']['display']") == 0 \ + && $(jget "$JSON2" "d['north_star']['zero_touch_merges_7d']['raw']") == 8 ]] \ + && grep -q "显示归零——护栏破线:drill_red_rate" <<<"$BRIEF2" \ + && grep -q "原始计数 8 保留" <<<"$BRIEF2" \ + && pass "渲染归零形态:0+破线路由标注+raw=8 保留(非数据删除)" \ + || fail "渲染归零形态断言失败" + +# --- 诚实口径:零 IR 不除零 · 零签署样本 pending · 零 needs-human p90=None --- +"$PY" - "$TMP/in.json" "$TMP/zero.json" <<'PYEOF' +import json, sys +d = json.load(open(sys.argv[1], encoding="utf-8")) +d.update({"ir_count_month": 0, "sign_durations_seconds": [], "needs_human_dwell_hours": []}) +json.dump(d, open(sys.argv[2], "w", encoding="utf-8"), ensure_ascii=False) +PYEOF +OUT3=$("$PY" "$M" eval --input "$TMP/zero.json" 2>/dev/null) || { echo "::error::zero eval 失败"; exit 2; } +[[ $(jget "$OUT3" "d['metrics']['cost']['per_ir_usd']") == None \ + && $(jget "$OUT3" "d['metrics']['attention']['sign_p90_seconds']") == None \ + && $(jget "$OUT3" "d['metrics']['attention']['needs_human_p90_hours']") == None \ + && $(jget "$OUT3" "d['metrics']['attention']['suspicious_fast_signs']") == 0 ]] \ + && pass "零 IR/零样本:null 不除零不出假值(可疑签署 0=真 0 非缺数据)" \ + || fail "零分母诚实口径断言失败" + +# --- JSON schema:metrics 四组键全集(agent 消费契约) --- +"$PY" -c " +import json, sys +d = json.loads(sys.argv[1]) +ms = d['metrics'] +assert set(ms) == {'attention', 'security', 'cost', 'user_results'}, ms.keys() +assert {'sign_count','sign_p50_seconds','sign_p90_seconds','suspicious_fast_signs', + 'needs_human_count','needs_human_p90_hours','needs_human_p90_stop'} <= set(ms['attention']) +assert {'false_allow_window','false_deny_window','drill_red','drill_denom'} <= set(ms['security']) +assert {'per_ir_usd','actions_minutes_month','llm_tokens_month','butler_usd_week','patrol_yield'} <= set(ms['cost']) +assert {'products','quarterly_hard_quota'} == set(ms['user_results']) +assert d['generated_at'] == '2026-08-22T00:00:00Z' +" "$JSON" 2>/dev/null && pass "payload schema v2 键全集锁定" || fail "schema 断言失败" + +echo "== test-metrics-groups: pass=$PASS fail=$FAIL ==" +[[ $FAIL -eq 0 ]]