fix(#727): mask ISO-8601 dates as dates, not phone numbers

middleware/packages/harness-orchestrator/src/datasetImport.ts

@@ -12,16 +12,31 @@
  *
  * Scanning uses the SAME `createBaselineDetector()` (C0 regex) pass that
  * protects free-text user prompts today (`@omadia/plugin-privacy-guard`).
- * Only `string`/`date`-typed columns are scanned: a `number`/`boolean`
- * column is, by construction, a cell that parsed cleanly as a number/bool
- * for EVERY row — there is no free-text surface left for the regex to
- * match, and running it anyway risks corrupting legitimate numeric data on
- * a false-positive hit (e.g. a 7-digit id that happens to start with a
- * leading `0`, which the phone-number pattern would flag). This is a v1
+ * Only `string`-typed columns are scanned. A `number`/`boolean`/`date`
+ * column is, by construction, a cell that parsed cleanly as a
+ * number/bool/date for EVERY row — there is no free-text surface left for
+ * the regex to match, and running it anyway risks corrupting legitimate
+ * data on a false-positive hit (e.g. a 7-digit id that happens to start
+ * with a leading `0`, which the phone-number pattern would flag).
+ *
+ * `date` is skipped for the same structural reason AND a correctness one
+ * (#727): masking runs over a *persisted* value here — no pseudonym map is
+ * retained after import, so the substitution is irreversible, and a masked
+ * date would make the stored value contradict the column's declared `date`
+ * type (a `query_dataset` gt/lt/min/max over it would then compare against a
+ * surrogate string, returning confidently-wrong answers). A pure date carries
+ * no name/email/phone/address on its own, so there is nothing to redact; the
+ * real date is stored as-is and the schema stays honest. This is a v1
  * scoping call, not a bypass: every ROW still goes through the pipeline,
  * exactly as the issue requires — only cells the pipeline could not
  * possibly find PII in are skipped.
  *
+ * Known residual (documented, not glossed): a column that is a *bare* PII
+ * date — e.g. a `birth_date` of ISO dates — is persisted un-redacted, the
+ * same class of trade as a national-ID column inferred as `number`. Reversible
+ * masking of date columns needs a retained per-dataset pseudonym map, a #430
+ * design question tracked as a follow-up, not a blocker for the masker fix.
+ *
  * Cost note: this is O(rows × string-columns) baseline-detector calls,
  * each a handful of regex passes over one cell's text — CPU-bound, not
  * network-bound (`createBaselineDetector` never makes an HTTP call), so a
@@ -175,7 +190,7 @@ function inferColumnType(values: readonly string[]): DatasetColumnType {
 
 export interface PrivacyScanStats {
   /** Total cells (across every row) that were passed through the baseline
-   *  detector — string/date-typed columns only, see module doc. */
+   *  detector — string-typed columns only, see module doc. */
   scannedCells: number;
   /** Cells where at least one span was masked. */
   maskedCells: number;
@@ -226,8 +241,15 @@ export async function buildDatasetFromCsv(bytes: Buffer): Promise<
         outRow[header] = raw.trim() === '' ? null : /^true$/i.test(raw.trim());
         continue;
       }
-      // 'string' | 'date' — the only cells that can carry free text, so the
-      // only ones that go through the privacy scan (see module doc).
+      if (type === 'date') {
+        // Skipped like number/boolean: no free-text surface, and masking a
+        // persisted date is irreversible + contradicts the declared type
+        // (#727). Store the real date so the schema and the data agree.
+        outRow[header] = raw.trim() === '' ? null : raw;
+        continue;
+      }
+      // 'string' — the only cells that can carry free text, so the only ones
+      // that go through the privacy scan (see module doc).
       scannedCells += 1;
       if (raw.length === 0) {
         outRow[header] = raw;

middleware/packages/harness-plugin-privacy-guard/src/promptMask.ts

@@ -209,6 +209,11 @@ interface ExtendedSpan {
   readonly type: string;
   readonly detector: string;
   readonly confidence: number;
+  /** Length of the detector's OWN match, BEFORE word-boundary extension. A
+   *  span that matched its whole value natively is more self-contained than
+   *  one that only reached the same range by growing across a shared
+   *  separator — used as a tie-break key below (#727). */
+  readonly nativeLen: number;
 }
 
 /** The parts of `[candidate.start, candidate.end)` not covered by any of
@@ -240,8 +245,17 @@ function hasWordChar(text: string, start: number, end: number): boolean {
 
 /**
  * Merge detector outputs: extend to word boundaries, then resolve overlaps
- * by letting the higher-confidence span (ties → the longer span) own the
- * contested characters. A losing span is NOT discarded wholesale: the parts
+ * by letting the higher-confidence span own the contested characters. Ties
+ * are broken by a documented, order-independent rule so the outcome never
+ * depends on detector/pattern declaration order (#727): (1) higher
+ * confidence, then (2) longer extended span, then (3) larger NATIVE match —
+ * a span that matched its value directly beats one that only grew into the
+ * same range (this is what makes the ISO date `2026-07-02` beat the phone
+ * pattern that grabbed its `-07-02` tail and extended back over the `-`),
+ * then — only for two spans still identical on all three — (4) a fixed
+ * lexical order of the type name (present purely for determinism, NOT
+ * semantic priority), then (5) earliest start.
+ * A losing span is NOT discarded wholesale: the parts
  * of it no winning span covers are kept as masking spans of their own —
  * otherwise a long low-confidence C1 span (e.g. a free-form address at
  * score 0.8) that merely brushes a short confidence-1 C0 hit (the postal
@@ -257,11 +271,30 @@ export function dedupSpans(
     .filter(({ span }) => span.end > span.start && span.start >= 0 && span.end <= text.length)
     .map(({ span, detector }) => {
       const { start, end } = extendToWordBoundaries(text, span.start, span.end);
-      return { start, end, type: span.type, detector, confidence: span.confidence };
+      return {
+        start,
+        end,
+        type: span.type,
+        detector,
+        confidence: span.confidence,
+        nativeLen: span.end - span.start,
+      };
     })
     .sort(
       (a, b) =>
-        b.confidence - a.confidence || b.end - b.start - (a.end - a.start) || a.start - b.start,
+        b.confidence - a.confidence ||
+        b.end - b.start - (a.end - a.start) ||
+        // Native (pre-extension) match: the span that matched its whole value
+        // beats one that only grew into the range across a shared separator.
+        b.nativeLen - a.nativeLen ||
+        // Deterministic last resort: a fixed lexical order of the type NAME.
+        // The point is determinism — never array/pattern order (#727) — not
+        // semantic priority: this only fires for two identical-range,
+        // identical-native-length spans of different types, where no type is
+        // "more right", so a fixed arbitrary order is the honest choice. Plain
+        // code-unit compare, not localeCompare (which varies by locale).
+        (a.type < b.type ? -1 : a.type > b.type ? 1 : 0) ||
+        a.start - b.start,
     );
 
   const kept: ExtendedSpan[] = [];

middleware/packages/harness-plugin-privacy-guard/src/validation/RESULTS.md

@@ -8,6 +8,22 @@ before `mask_user_prompt` flips on for it.
 
 Two runs are recorded, newest first:
 
+- **#727 re-run — 2026-08-19 · overlap tie-break (no pattern change).** #727
+  makes the `dedupSpans` overlap tie-break a documented, order-independent
+  rule (native match length, then a fixed lexical order of the type name,
+  present for determinism only — not semantic priority) so an ISO-8601 date
+  (`2026-07-02`) no longer loses its exact-tie to the general phone pattern
+  and mask as a phone surrogate. This changes only **which surrogate type**
+  replaces an already-masked span — never **whether** a span is masked. The
+  harness scores span *coverage* (was the labelled region masked?), not
+  surrogate type, so the ISO date was always counted as covered (it was
+  masked, just as the wrong type) — which is exactly why the harness never
+  surfaced the bug, and why a fresh `c0` run reproduces Run 2's tables
+  **byte-for-byte**: de/en 100%, es/fr/it 99.1% (PASS), nl 89.0% (FAIL,
+  address-only, C1-carried), precision proxy 32/32 clean in every locale.
+  No recall/precision/latency number moves. Verified 2026-08-19 with
+  `node --import tsx …/promptDetectorEval.ts --markdown` (c0 only, no
+  sidecar). C1 is untouched, so the derived `c0+c1` projection stands.
 - **Run 2 — 2026-08-05 · locale-aware C0 patterns (#482).** Extends the C0
   regex baseline with the recorded es/fr/nl miss classes: separator-less
   (`899 €`) and space-grouped (`2 400 €`) amounts, dashed (`30-06-2027`) and

middleware/test/datasetImport.test.ts

@@ -210,6 +210,45 @@ describe('buildDatasetFromCsv — privacy scan', () => {
       'expected the phone number to be masked by the baseline detector',
     );
   });
+
+  it('#727 — an ISO-8601 date column is stored as its real dates, not masked to a phone surrogate at rest', async () => {
+    // The reported corruption: a `date`-typed column of ISO dates was run
+    // through the masker, which mis-typed `2026-07-02` as a phone and
+    // persisted `+49 30 55590000` — destroying the column irreversibly (no
+    // map retained) and contradicting the declared `date` type. Date columns
+    // are now excluded from the scan (like number/boolean), so the real,
+    // DISTINCT dates are stored and the schema stays honest.
+    const csv =
+      'customer,order_date\n' +
+      'Meier GmbH,2026-07-02\n' +
+      'Weber AG,2026-08-01\n';
+    const built = await buildDatasetFromCsv(Buffer.from(csv, 'utf8'));
+    assert.equal(built.ok, true);
+    if (!built.ok) return;
+
+    const byName = new Map(built.columns.map((c) => [c.name, c]));
+    assert.equal(byName.get('order_date')?.type, 'date');
+
+    // Read the values back OUT of the produced rows (the persisted artifact),
+    // not the renderer input: real dates, distinct per row, no phone leak.
+    assert.equal(built.rows[0]?.['order_date'], '2026-07-02');
+    assert.equal(built.rows[1]?.['order_date'], '2026-08-01');
+    assert.notEqual(built.rows[0]?.['order_date'], built.rows[1]?.['order_date']);
+    for (const row of built.rows) {
+      assert.ok(!String(row['order_date']).includes('+49'), 'a date must never become a phone surrogate');
+    }
+
+    // The schema sample no longer advertises a type the stored value contradicts.
+    assert.equal(byName.get('order_date')?.sample, '2026-07-02');
+
+    // The date column was NOT scanned — only the two `customer` (string) cells
+    // were. This pins the "skip date columns" decision, not just its effect.
+    assert.equal(
+      built.privacyScan.scannedCells,
+      2,
+      'only the string column may be scanned; the date column must be skipped',
+    );
+  });
 });
 
 describe('importCsvDataset', () => {

middleware/test/privacyPromptMask.test.ts

@@ -125,6 +125,132 @@ describe('dedupSpans', () => {
       assert.ok(resolved[i - 1]!.end <= resolved[i]!.start);
     }
   });
+
+  it('#727 — the ISO-date exact tie resolves to date, in either input order', () => {
+    // The ISO-date bug in miniature: `2026-07-02` yields a native `date` span
+    // over the whole token [0,10] and a native `phone` span over the `07-02`
+    // tail [5,10] that word-boundary extension grows back to [0,10]. Both are
+    // confidence 1 and end up the same length at the same start — a total tie
+    // on (confidence, length, start). The tie MUST resolve to `date` because
+    // its NATIVE match is larger (it matched the value directly; phone only
+    // grew into the range) — and it must do so regardless of the order the
+    // spans arrive in (previously the winner was decided by C0_PATTERNS
+    // insertion order via stable-sort fall-through).
+    const text = '2026-07-02';
+    const dateSpan = { span: { start: 0, end: 10, type: 'date', confidence: 1 }, detector: 'c0-regex' };
+    const phoneSpan = { span: { start: 5, end: 10, type: 'phone', confidence: 1 }, detector: 'c0-regex' };
+
+    for (const order of [
+      [phoneSpan, dateSpan],
+      [dateSpan, phoneSpan],
+    ]) {
+      const resolved = dedupSpans(text, order);
+      assert.equal(resolved.length, 1);
+      assert.equal(
+        resolved[0]!.type,
+        'date',
+        'the native full-token date span must win the tie in either input order',
+      );
+      assert.equal(resolved[0]!.value, '2026-07-02');
+    }
+  });
+
+  it('#727 — native match length is the decider, ABOVE the lexical fallback', () => {
+    // Isolate the nativeLen key from the lexical last-resort. For date/phone
+    // both keys happen to favour `date`, so that pair alone cannot prove
+    // WHICH key decided. Here the full-token span is typed `phone` and the
+    // grown-in tail is typed `date`: the lexical fallback (`date` < `phone`)
+    // would pick `date`, but nativeLen (full 10 > tail 5) must pick `phone`.
+    // `phone` winning proves nativeLen outranks the lexical key AND input
+    // order. (Types are assigned synthetically to exercise the comparator —
+    // no real detector emits this arrangement.)
+    const text = '2026-07-02';
+    const fullSpan = { span: { start: 0, end: 10, type: 'phone', confidence: 1 }, detector: 'c0-regex' };
+    const tailSpan = { span: { start: 5, end: 10, type: 'date', confidence: 1 }, detector: 'c0-regex' };
+    for (const order of [
+      [fullSpan, tailSpan],
+      [tailSpan, fullSpan],
+    ]) {
+      const resolved = dedupSpans(text, order);
+      assert.equal(resolved.length, 1);
+      assert.equal(
+        resolved[0]!.type,
+        'phone',
+        'the larger native match must win regardless of the lexical fallback or input order',
+      );
+    }
+  });
+
+  it('#727 — when native length also ties, a fixed LEXICAL order (not array order) decides', () => {
+    // Force the last-resort key: two spans with the SAME native range but
+    // different types. The fall-through is a fixed lexical compare of the type
+    // NAME (determinism, not priority), so the lexically-smaller type wins in
+    // either input order — proving the tiebreak is the documented rule, never
+    // C0_PATTERNS insertion order. Checked with two pairs so a single
+    // `date`-always-wins coincidence cannot pass it.
+    const text = '2026-07-02';
+    const mk = (type: string) => ({ span: { start: 0, end: 10, type, confidence: 1 }, detector: 'c0-regex' });
+    // 'date' < 'phone'  and  'amount' < 'phone'  (code-unit order)
+    const pairs: readonly (readonly [lo: string, hi: string])[] = [
+      ['date', 'phone'],
+      ['amount', 'phone'],
+    ];
+    for (const [lo, hi] of pairs) {
+      for (const order of [
+        [mk(hi), mk(lo)],
+        [mk(lo), mk(hi)],
+      ]) {
+        const resolved = dedupSpans(text, order);
+        assert.equal(resolved.length, 1);
+        assert.equal(
+          resolved[0]!.type,
+          lo,
+          `the lexically-smaller type '${lo}' must win over '${hi}' in either input order`,
+        );
+      }
+    }
+  });
+});
+
+describe('#727 — ISO-8601 dates mask as dates, not phone numbers', () => {
+  const PHONE_SHAPE = /\+49\s/; // the phone surrogate pool is `+49 30 5559xxxx`
+  const DATE_SHAPE = /^\d{2}\.\d{2}\.\d{4}$/; // the date surrogate pool is `dd.mm.yyyy`
+
+  it('substitutes a bare ISO date from the DATE surrogate pool', async () => {
+    const result = await maskPrompt('2026-07-02', [createBaselineDetector()]);
+    // The winning span is typed `date`, not `phone`.
+    assert.equal(result.spans.length, 1);
+    assert.equal(result.spans[0]!.type, 'date');
+    assert.equal(result.spans[0]!.value, '2026-07-02');
+    // And the wire value is a date surrogate, never a phone number.
+    assert.ok(DATE_SHAPE.test(result.maskedText), `expected a date surrogate, got '${result.maskedText}'`);
+    assert.ok(!PHONE_SHAPE.test(result.maskedText), `ISO date leaked as a phone surrogate: '${result.maskedText}'`);
+    // Round-trips like any masked value.
+    assert.equal(resolvePseudonyms(result.maskedText, result.map), '2026-07-02');
+  });
+
+  it('the ISO and the dotted shape both mask as dates, and a real phone still masks as a phone (regression: the two must not drift)', async () => {
+    // The dotted form was already correct; pin it beside the ISO form so a
+    // future change cannot fix one and re-break the other. A genuine phone
+    // must still take the phone surrogate — the fix narrows nothing.
+    //
+    // `30-06-2027` (dashed dd-mm-yyyy, added for nl/fr by #482) is here
+    // deliberately: it was broken by the SAME mechanism the issue reported for
+    // the ISO shape — the phone pattern's `\b0` branch grabs its `06-2027`
+    // tail and word-boundary extension grows it over the whole token — but the
+    // issue only named the ISO form, and the C0 eval could not surface either,
+    // because it scores span COVERAGE, not surrogate type. Any date whose
+    // month/day segment starts with `0` after a `-` is in this class, so all
+    // three separator styles are pinned together.
+    for (const isoOrDotted of ['2026-07-02', '02.07.2026', '1990-06-15', '30-06-2027']) {
+      const r = await maskPrompt(isoOrDotted, [createBaselineDetector()]);
+      assert.equal(r.spans[0]?.type, 'date', `${isoOrDotted} must be typed date`);
+      assert.ok(DATE_SHAPE.test(r.maskedText), `${isoOrDotted} -> non-date surrogate '${r.maskedText}'`);
+    }
+    const phone = await maskPrompt('+49 171 2345678', [createBaselineDetector()]);
+    assert.equal(phone.spans[0]?.type, 'phone');
+    assert.ok(PHONE_SHAPE.test(phone.maskedText), `phone -> non-phone surrogate '${phone.maskedText}'`);
+  });
 });
 
 describe('createPromptPseudonymMap', () => {