fix(pilot): reject placeholder or ambiguous readiness evidence

scripts/lib/pilot-readiness.mjs

@@ -16,28 +16,49 @@ function metricValue(entry, name) {
   return match ? Number(match[1]) : null;
 }
 
+function metricCount(entry, name) {
+  const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return [...entry.matchAll(new RegExp(`^-\\s*\`?${escaped}\`?\\s*:`, "gm"))].length;
+}
+
 function fieldValue(entry, label) {
   const escaped = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
   const match = entry.match(new RegExp(`^-\\s*${escaped}:\\s*(.+)\\s*$`, "m"));
   return match ? match[1].trim() : "";
 }
 
+function fieldCount(entry, label) {
+  const escaped = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return [...entry.matchAll(new RegExp(`^-\\s*${escaped}\\s*:`, "gm"))].length;
+}
+
 function hasCheckedLine(entry, labelPattern) {
   return new RegExp(`^-\\s*\\[x\\]\\s*${labelPattern}\\s*$`, "m").test(entry);
 }
 
+function isLocalOnlyHostname(host) {
+  const normalized = host.startsWith("[") && host.endsWith("]")
+    ? host.slice(1, -1)
+    : host;
+  if (normalized === "::" || normalized === "::1" || normalized === "0.0.0.0") return true;
+  if (/^::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}$/i.test(normalized)) return true;
+  return /^127(?:\.\d{1,3}){3}$/.test(normalized);
+}
+
 function isUsableProductionUrl(value) {
   if (!value) return false;
   try {
     const url = new URL(value.replace(/`/g, ""));
     const host = url.hostname.toLowerCase();
+    const canonicalHost = host.endsWith(".") ? host.slice(0, -1) : host;
     return url.protocol === "https:"
       && url.username === ""
       && url.password === ""
-      && host !== "localhost"
-      && host !== "127.0.0.1"
-      && !host.endsWith(".local")
-      && !host.includes("example");
+      && canonicalHost !== "localhost"
+      && !canonicalHost.endsWith(".localhost")
+      && !isLocalOnlyHostname(canonicalHost)
+      && !canonicalHost.endsWith(".local")
+      && !canonicalHost.includes("example");
   } catch {
     return false;
   }
@@ -46,11 +67,19 @@ function isUsableProductionUrl(value) {
 function isUsableSupportChannel(value) {
   const normalized = value.toLowerCase();
   return normalized.length > 0
-    && !normalized.includes("@noema.local")
+    && !normalized.includes(".local")
     && !normalized.includes("example")
     && !normalized.includes("localhost");
 }
 
+function isUsableEvidenceReference(value) {
+  const normalized = value.toLowerCase();
+  return normalized.length > 0
+    && !normalized.includes("example")
+    && !normalized.includes("localhost")
+    && !normalized.includes(".local");
+}
+
 function evaluatePilotEntry(entry) {
   const customerName = fieldValue(entry, "고객명");
   const noemaUrl = fieldValue(entry, "NOEMA URL");
@@ -65,8 +94,22 @@ function evaluatePilotEntry(entry) {
   const p95 = metricValue(entry, "exchange_p95_latency_ms");
   const onboardingDateStatus = dateStatus(onboardingDate);
   const handoverDateStatus = dateStatus(handoverDate);
+  const duplicateAuthorities = [
+    ["고객명", fieldCount(entry, "고객명")],
+    ["NOEMA URL", fieldCount(entry, "NOEMA URL")],
+    ["지원 채널 합의", fieldCount(entry, "지원 채널 합의")],
+    ["온보딩 완료일", fieldCount(entry, "온보딩 완료일")],
+    ["운영 전환 승인일", fieldCount(entry, "운영 전환 승인일")],
+    ["증빙 출처", fieldCount(entry, "증빙 출처") + fieldCount(entry, "evidence_source_kind")],
+    ["계약/매출 증빙 경로", fieldCount(entry, "계약/매출 증빙 경로")],
+    ["분석 데이터 경로", fieldCount(entry, "분석 데이터 경로")],
+    ["trace_id 샘플", fieldCount(entry, "trace_id 샘플")],
+    ["exchange_failure_rate", metricCount(entry, "exchange_failure_rate")],
+    ["exchange_p95_latency_ms", metricCount(entry, "exchange_p95_latency_ms")],
+  ].filter(([, count]) => count > 1);
 
   const failures = [];
+  for (const [label] of duplicateAuthorities) failures.push(`${label} must appear exactly once`);
   if (!customerName) failures.push("고객명 required");
   if (!isUsableProductionUrl(noemaUrl)) failures.push("NOEMA URL must be a non-example HTTPS production URL");
   if (!isUsableSupportChannel(supportChannel)) failures.push("지원 채널 합의 must be a real non-local channel");
@@ -80,9 +123,12 @@ function evaluatePilotEntry(entry) {
   if (failureRate === null || failureRate > 0.02) failures.push("exchange_failure_rate must be <= 0.02");
   if (p95 === null || p95 >= 300) failures.push("exchange_p95_latency_ms must be < 300");
   if (!evidencePath) failures.push("분석 데이터 경로 required");
+  else if (!isUsableEvidenceReference(evidencePath)) failures.push("분석 데이터 경로 must be a non-example evidence reference");
   if (!traceId) failures.push("trace_id 샘플 required");
+  else if (!isUsableEvidenceReference(traceId)) failures.push("trace_id 샘플 must be a non-example evidence reference");
   if (evidenceSourceKind !== "production") failures.push("증빙 출처 must be production");
   if (!contractEvidencePath) failures.push("계약/매출 증빙 경로 required");
+  else if (!isUsableEvidenceReference(contractEvidencePath)) failures.push("계약/매출 증빙 경로 must be a non-example evidence reference");
 
   return {
     customerName,

test/pilot-readiness-placeholder-evidence.test.ts

@@ -0,0 +1,150 @@
+import { describe, expect, it } from "vitest";
+import { evaluatePilotReadinessText } from "../scripts/lib/pilot-readiness.mjs";
+
+function completedPilot(contractEvidencePath: string, evidencePath: string, traceId = "trace-2f4c9a77-1e8a-4f3b-9b9a-a8c1e6f0b5d1") {
+  return `# 파일럿 온보딩 진행 기록
+
+## 항목 1
+- 고객명: Acme Security
+- NOEMA URL: https://noema.acme-security.com/exchange
+- 지원 채널 합의: Slack acme-noema-ops
+- 증빙 출처: production
+- 계약/매출 증빙 경로: ${contractEvidencePath}
+- 분석 데이터 경로: ${evidencePath}
+- exchange_failure_rate: 0
+- exchange_p95_latency_ms: 157
+- [x] 실패율 <= 0.02
+- [x] p95 < 300
+- [x] 운영 이관 승인
+- 운영 전환 승인일: 2026-06-30
+- 온보딩 완료일: 2026-07-01
+- trace_id 샘플: ${traceId}
+`;
+}
+
+describe("pilot readiness evidence references", () => {
+  it.each([
+    ["example/contracts/demo-paid-pilot.pdf", "artifacts/example/noema-kpi-evidence.json"],
+    ["localhost/contracts/demo-paid-pilot.pdf", "artifacts/localhost/noema-kpi-evidence.json"],
+    ["fixtures.local/contracts/demo-paid-pilot.pdf", "artifacts/fixtures.local/noema-kpi-evidence.json"],
+  ])("rejects documented sample markers in commercial evidence references", (contractEvidencePath, evidencePath) => {
+    const result = evaluatePilotReadinessText(completedPilot(contractEvidencePath, evidencePath));
+
+    expect(result.passed).toBe(false);
+    expect(result.entries[0].failures).toContain("계약/매출 증빙 경로 must be a non-example evidence reference");
+    expect(result.entries[0].failures).toContain("분석 데이터 경로 must be a non-example evidence reference");
+  });
+
+  it("rejects an example trace key as completion evidence", () => {
+    const result = evaluatePilotReadinessText(completedPilot(
+      "contracts/acme-paid-pilot.pdf",
+      "artifacts/saleable-readiness/noema-kpi-evidence.json",
+      "example-trace-id",
+    ));
+
+    expect(result.passed).toBe(false);
+    expect(result.entries[0].failures).toContain("trace_id 샘플 must be a non-example evidence reference");
+  });
+
+  it.each([
+    "https://127.0.0.2/exchange",
+    "https://[::1]/exchange",
+    "https://[::ffff:127.0.0.2]/exchange",
+    "https://tenant.localhost/exchange",
+    "https://0.0.0.0/exchange",
+    "https://[::]/exchange",
+  ])("rejects local-only or non-routable listener identities as production URLs (%s)", (url) => {
+    const text = completedPilot(
+      "contracts/acme-paid-pilot.pdf",
+      "artifacts/saleable-readiness/noema-kpi-evidence.json",
+    ).replace(
+      "- NOEMA URL: https://noema.acme-security.com/exchange",
+      `- NOEMA URL: ${url}`,
+    );
+
+    const result = evaluatePilotReadinessText(text);
+
+    expect(result.passed).toBe(false);
+    expect(result.entries[0].failures).toContain("NOEMA URL must be a non-example HTTPS production URL");
+  });
+
+  it.each([
+    "https://localhost./exchange",
+    "https://fixtures.local./exchange",
+  ])("rejects absolute local DNS names as production URLs (%s)", (url) => {
+    const text = completedPilot(
+      "contracts/acme-paid-pilot.pdf",
+      "artifacts/saleable-readiness/noema-kpi-evidence.json",
+    ).replace(
+      "- NOEMA URL: https://noema.acme-security.com/exchange",
+      `- NOEMA URL: ${url}`,
+    );
+
+    const result = evaluatePilotReadinessText(text);
+
+    expect(result.passed).toBe(false);
+    expect(result.entries[0].failures).toContain("NOEMA URL must be a non-example HTTPS production URL");
+  });
+
+  it("rejects a .local support channel as sample evidence", () => {
+    const text = completedPilot(
+      "contracts/acme-paid-pilot.pdf",
+      "artifacts/saleable-readiness/noema-kpi-evidence.json",
+    ).replace(
+      "- 지원 채널 합의: Slack acme-noema-ops",
+      "- 지원 채널 합의: support@acme.local",
+    );
+
+    const result = evaluatePilotReadinessText(text);
+
+    expect(result.passed).toBe(false);
+    expect(result.entries[0].failures).toContain("지원 채널 합의 must be a real non-local channel");
+  });
+
+  it("rejects duplicate authoritative evidence instead of trusting the first matching line", () => {
+    const text = completedPilot(
+      "contracts/acme-paid-pilot.pdf",
+      "artifacts/saleable-readiness/noema-kpi-evidence.json",
+    )
+      .replace(
+        "- 증빙 출처: production",
+        "- 증빙 출처: production\n- evidence_source_kind: fixture",
+      )
+      .replace(
+        "- 계약/매출 증빙 경로: contracts/acme-paid-pilot.pdf",
+        "- 계약/매출 증빙 경로: contracts/acme-paid-pilot.pdf\n- 계약/매출 증빙 경로: example/contracts/forged.pdf",
+      )
+      .replace(
+        "- exchange_failure_rate: 0",
+        "- exchange_failure_rate: 0\n- exchange_failure_rate: 0.9",
+      );
+
+    const result = evaluatePilotReadinessText(text);
+
+    expect(result.passed).toBe(false);
+    expect(result.entries[0].failures).toContain("증빙 출처 must appear exactly once");
+    expect(result.entries[0].failures).toContain("계약/매출 증빙 경로 must appear exactly once");
+    expect(result.entries[0].failures).toContain("exchange_failure_rate must appear exactly once");
+  });
+
+  it("rejects malformed or blank duplicate authority instead of ignoring the second label", () => {
+    const text = completedPilot(
+      "contracts/acme-paid-pilot.pdf",
+      "artifacts/saleable-readiness/noema-kpi-evidence.json",
+    )
+      .replace(
+        "- 계약/매출 증빙 경로: contracts/acme-paid-pilot.pdf",
+        "- 계약/매출 증빙 경로: contracts/acme-paid-pilot.pdf\n- 계약/매출 증빙 경로:",
+      )
+      .replace(
+        "- exchange_failure_rate: 0",
+        "- exchange_failure_rate: 0\n- exchange_failure_rate: forged",
+      );
+
+    const result = evaluatePilotReadinessText(text);
+
+    expect(result.passed).toBe(false);
+    expect(result.entries[0].failures).toContain("계약/매출 증빙 경로 must appear exactly once");
+    expect(result.entries[0].failures).toContain("exchange_failure_rate must appear exactly once");
+  });
+});