diff --git a/middleware/packages/harness-orchestrator/src/datasetImport.ts b/middleware/packages/harness-orchestrator/src/datasetImport.ts index df475a06..87526ee3 100644 --- a/middleware/packages/harness-orchestrator/src/datasetImport.ts +++ b/middleware/packages/harness-orchestrator/src/datasetImport.ts @@ -12,16 +12,31 @@ * * Scanning uses the SAME `createBaselineDetector()` (C0 regex) pass that * protects free-text user prompts today (`@omadia/plugin-privacy-guard`). - * Only `string`/`date`-typed columns are scanned: a `number`/`boolean` - * column is, by construction, a cell that parsed cleanly as a number/bool - * for EVERY row — there is no free-text surface left for the regex to - * match, and running it anyway risks corrupting legitimate numeric data on - * a false-positive hit (e.g. a 7-digit id that happens to start with a - * leading `0`, which the phone-number pattern would flag). This is a v1 + * Only `string`-typed columns are scanned. A `number`/`boolean`/`date` + * column is, by construction, a cell that parsed cleanly as a + * number/bool/date for EVERY row — there is no free-text surface left for + * the regex to match, and running it anyway risks corrupting legitimate + * data on a false-positive hit (e.g. a 7-digit id that happens to start + * with a leading `0`, which the phone-number pattern would flag). + * + * `date` is skipped for the same structural reason AND a correctness one + * (#727): masking runs over a *persisted* value here — no pseudonym map is + * retained after import, so the substitution is irreversible, and a masked + * date would make the stored value contradict the column's declared `date` + * type (a `query_dataset` gt/lt/min/max over it would then compare against a + * surrogate string, returning confidently-wrong answers). A pure date carries + * no name/email/phone/address on its own, so there is nothing to redact; the + * real date is stored as-is and the schema stays honest. This is a v1 * scoping call, not a bypass: every ROW still goes through the pipeline, * exactly as the issue requires — only cells the pipeline could not * possibly find PII in are skipped. * + * Known residual (documented, not glossed): a column that is a *bare* PII + * date — e.g. a `birth_date` of ISO dates — is persisted un-redacted, the + * same class of trade as a national-ID column inferred as `number`. Reversible + * masking of date columns needs a retained per-dataset pseudonym map, a #430 + * design question tracked as a follow-up, not a blocker for the masker fix. + * * Cost note: this is O(rows × string-columns) baseline-detector calls, * each a handful of regex passes over one cell's text — CPU-bound, not * network-bound (`createBaselineDetector` never makes an HTTP call), so a @@ -175,7 +190,7 @@ function inferColumnType(values: readonly string[]): DatasetColumnType { export interface PrivacyScanStats { /** Total cells (across every row) that were passed through the baseline - * detector — string/date-typed columns only, see module doc. */ + * detector — string-typed columns only, see module doc. */ scannedCells: number; /** Cells where at least one span was masked. */ maskedCells: number; @@ -226,8 +241,15 @@ export async function buildDatasetFromCsv(bytes: Buffer): Promise< outRow[header] = raw.trim() === '' ? null : /^true$/i.test(raw.trim()); continue; } - // 'string' | 'date' — the only cells that can carry free text, so the - // only ones that go through the privacy scan (see module doc). + if (type === 'date') { + // Skipped like number/boolean: no free-text surface, and masking a + // persisted date is irreversible + contradicts the declared type + // (#727). Store the real date so the schema and the data agree. + outRow[header] = raw.trim() === '' ? null : raw; + continue; + } + // 'string' — the only cells that can carry free text, so the only ones + // that go through the privacy scan (see module doc). scannedCells += 1; if (raw.length === 0) { outRow[header] = raw; diff --git a/middleware/packages/harness-plugin-privacy-guard/src/promptMask.ts b/middleware/packages/harness-plugin-privacy-guard/src/promptMask.ts index dc334af6..cd4d9ea6 100644 --- a/middleware/packages/harness-plugin-privacy-guard/src/promptMask.ts +++ b/middleware/packages/harness-plugin-privacy-guard/src/promptMask.ts @@ -209,6 +209,11 @@ interface ExtendedSpan { readonly type: string; readonly detector: string; readonly confidence: number; + /** Length of the detector's OWN match, BEFORE word-boundary extension. A + * span that matched its whole value natively is more self-contained than + * one that only reached the same range by growing across a shared + * separator — used as a tie-break key below (#727). */ + readonly nativeLen: number; } /** The parts of `[candidate.start, candidate.end)` not covered by any of @@ -240,8 +245,17 @@ function hasWordChar(text: string, start: number, end: number): boolean { /** * Merge detector outputs: extend to word boundaries, then resolve overlaps - * by letting the higher-confidence span (ties → the longer span) own the - * contested characters. A losing span is NOT discarded wholesale: the parts + * by letting the higher-confidence span own the contested characters. Ties + * are broken by a documented, order-independent rule so the outcome never + * depends on detector/pattern declaration order (#727): (1) higher + * confidence, then (2) longer extended span, then (3) larger NATIVE match — + * a span that matched its value directly beats one that only grew into the + * same range (this is what makes the ISO date `2026-07-02` beat the phone + * pattern that grabbed its `-07-02` tail and extended back over the `-`), + * then — only for two spans still identical on all three — (4) a fixed + * lexical order of the type name (present purely for determinism, NOT + * semantic priority), then (5) earliest start. + * A losing span is NOT discarded wholesale: the parts * of it no winning span covers are kept as masking spans of their own — * otherwise a long low-confidence C1 span (e.g. a free-form address at * score 0.8) that merely brushes a short confidence-1 C0 hit (the postal @@ -257,11 +271,30 @@ export function dedupSpans( .filter(({ span }) => span.end > span.start && span.start >= 0 && span.end <= text.length) .map(({ span, detector }) => { const { start, end } = extendToWordBoundaries(text, span.start, span.end); - return { start, end, type: span.type, detector, confidence: span.confidence }; + return { + start, + end, + type: span.type, + detector, + confidence: span.confidence, + nativeLen: span.end - span.start, + }; }) .sort( (a, b) => - b.confidence - a.confidence || b.end - b.start - (a.end - a.start) || a.start - b.start, + b.confidence - a.confidence || + b.end - b.start - (a.end - a.start) || + // Native (pre-extension) match: the span that matched its whole value + // beats one that only grew into the range across a shared separator. + b.nativeLen - a.nativeLen || + // Deterministic last resort: a fixed lexical order of the type NAME. + // The point is determinism — never array/pattern order (#727) — not + // semantic priority: this only fires for two identical-range, + // identical-native-length spans of different types, where no type is + // "more right", so a fixed arbitrary order is the honest choice. Plain + // code-unit compare, not localeCompare (which varies by locale). + (a.type < b.type ? -1 : a.type > b.type ? 1 : 0) || + a.start - b.start, ); const kept: ExtendedSpan[] = []; diff --git a/middleware/packages/harness-plugin-privacy-guard/src/validation/RESULTS.md b/middleware/packages/harness-plugin-privacy-guard/src/validation/RESULTS.md index 2c1b9a44..9a6564a4 100644 --- a/middleware/packages/harness-plugin-privacy-guard/src/validation/RESULTS.md +++ b/middleware/packages/harness-plugin-privacy-guard/src/validation/RESULTS.md @@ -8,6 +8,22 @@ before `mask_user_prompt` flips on for it. Two runs are recorded, newest first: +- **#727 re-run — 2026-08-19 · overlap tie-break (no pattern change).** #727 + makes the `dedupSpans` overlap tie-break a documented, order-independent + rule (native match length, then a fixed lexical order of the type name, + present for determinism only — not semantic priority) so an ISO-8601 date + (`2026-07-02`) no longer loses its exact-tie to the general phone pattern + and mask as a phone surrogate. This changes only **which surrogate type** + replaces an already-masked span — never **whether** a span is masked. The + harness scores span *coverage* (was the labelled region masked?), not + surrogate type, so the ISO date was always counted as covered (it was + masked, just as the wrong type) — which is exactly why the harness never + surfaced the bug, and why a fresh `c0` run reproduces Run 2's tables + **byte-for-byte**: de/en 100%, es/fr/it 99.1% (PASS), nl 89.0% (FAIL, + address-only, C1-carried), precision proxy 32/32 clean in every locale. + No recall/precision/latency number moves. Verified 2026-08-19 with + `node --import tsx …/promptDetectorEval.ts --markdown` (c0 only, no + sidecar). C1 is untouched, so the derived `c0+c1` projection stands. - **Run 2 — 2026-08-05 · locale-aware C0 patterns (#482).** Extends the C0 regex baseline with the recorded es/fr/nl miss classes: separator-less (`899 €`) and space-grouped (`2 400 €`) amounts, dashed (`30-06-2027`) and diff --git a/middleware/test/datasetImport.test.ts b/middleware/test/datasetImport.test.ts index e9364f88..5cf2bc1e 100644 --- a/middleware/test/datasetImport.test.ts +++ b/middleware/test/datasetImport.test.ts @@ -210,6 +210,45 @@ describe('buildDatasetFromCsv — privacy scan', () => { 'expected the phone number to be masked by the baseline detector', ); }); + + it('#727 — an ISO-8601 date column is stored as its real dates, not masked to a phone surrogate at rest', async () => { + // The reported corruption: a `date`-typed column of ISO dates was run + // through the masker, which mis-typed `2026-07-02` as a phone and + // persisted `+49 30 55590000` — destroying the column irreversibly (no + // map retained) and contradicting the declared `date` type. Date columns + // are now excluded from the scan (like number/boolean), so the real, + // DISTINCT dates are stored and the schema stays honest. + const csv = + 'customer,order_date\n' + + 'Meier GmbH,2026-07-02\n' + + 'Weber AG,2026-08-01\n'; + const built = await buildDatasetFromCsv(Buffer.from(csv, 'utf8')); + assert.equal(built.ok, true); + if (!built.ok) return; + + const byName = new Map(built.columns.map((c) => [c.name, c])); + assert.equal(byName.get('order_date')?.type, 'date'); + + // Read the values back OUT of the produced rows (the persisted artifact), + // not the renderer input: real dates, distinct per row, no phone leak. + assert.equal(built.rows[0]?.['order_date'], '2026-07-02'); + assert.equal(built.rows[1]?.['order_date'], '2026-08-01'); + assert.notEqual(built.rows[0]?.['order_date'], built.rows[1]?.['order_date']); + for (const row of built.rows) { + assert.ok(!String(row['order_date']).includes('+49'), 'a date must never become a phone surrogate'); + } + + // The schema sample no longer advertises a type the stored value contradicts. + assert.equal(byName.get('order_date')?.sample, '2026-07-02'); + + // The date column was NOT scanned — only the two `customer` (string) cells + // were. This pins the "skip date columns" decision, not just its effect. + assert.equal( + built.privacyScan.scannedCells, + 2, + 'only the string column may be scanned; the date column must be skipped', + ); + }); }); describe('importCsvDataset', () => { diff --git a/middleware/test/privacyPromptMask.test.ts b/middleware/test/privacyPromptMask.test.ts index d280ad0e..2ca91cef 100644 --- a/middleware/test/privacyPromptMask.test.ts +++ b/middleware/test/privacyPromptMask.test.ts @@ -125,6 +125,132 @@ describe('dedupSpans', () => { assert.ok(resolved[i - 1]!.end <= resolved[i]!.start); } }); + + it('#727 — the ISO-date exact tie resolves to date, in either input order', () => { + // The ISO-date bug in miniature: `2026-07-02` yields a native `date` span + // over the whole token [0,10] and a native `phone` span over the `07-02` + // tail [5,10] that word-boundary extension grows back to [0,10]. Both are + // confidence 1 and end up the same length at the same start — a total tie + // on (confidence, length, start). The tie MUST resolve to `date` because + // its NATIVE match is larger (it matched the value directly; phone only + // grew into the range) — and it must do so regardless of the order the + // spans arrive in (previously the winner was decided by C0_PATTERNS + // insertion order via stable-sort fall-through). + const text = '2026-07-02'; + const dateSpan = { span: { start: 0, end: 10, type: 'date', confidence: 1 }, detector: 'c0-regex' }; + const phoneSpan = { span: { start: 5, end: 10, type: 'phone', confidence: 1 }, detector: 'c0-regex' }; + + for (const order of [ + [phoneSpan, dateSpan], + [dateSpan, phoneSpan], + ]) { + const resolved = dedupSpans(text, order); + assert.equal(resolved.length, 1); + assert.equal( + resolved[0]!.type, + 'date', + 'the native full-token date span must win the tie in either input order', + ); + assert.equal(resolved[0]!.value, '2026-07-02'); + } + }); + + it('#727 — native match length is the decider, ABOVE the lexical fallback', () => { + // Isolate the nativeLen key from the lexical last-resort. For date/phone + // both keys happen to favour `date`, so that pair alone cannot prove + // WHICH key decided. Here the full-token span is typed `phone` and the + // grown-in tail is typed `date`: the lexical fallback (`date` < `phone`) + // would pick `date`, but nativeLen (full 10 > tail 5) must pick `phone`. + // `phone` winning proves nativeLen outranks the lexical key AND input + // order. (Types are assigned synthetically to exercise the comparator — + // no real detector emits this arrangement.) + const text = '2026-07-02'; + const fullSpan = { span: { start: 0, end: 10, type: 'phone', confidence: 1 }, detector: 'c0-regex' }; + const tailSpan = { span: { start: 5, end: 10, type: 'date', confidence: 1 }, detector: 'c0-regex' }; + for (const order of [ + [fullSpan, tailSpan], + [tailSpan, fullSpan], + ]) { + const resolved = dedupSpans(text, order); + assert.equal(resolved.length, 1); + assert.equal( + resolved[0]!.type, + 'phone', + 'the larger native match must win regardless of the lexical fallback or input order', + ); + } + }); + + it('#727 — when native length also ties, a fixed LEXICAL order (not array order) decides', () => { + // Force the last-resort key: two spans with the SAME native range but + // different types. The fall-through is a fixed lexical compare of the type + // NAME (determinism, not priority), so the lexically-smaller type wins in + // either input order — proving the tiebreak is the documented rule, never + // C0_PATTERNS insertion order. Checked with two pairs so a single + // `date`-always-wins coincidence cannot pass it. + const text = '2026-07-02'; + const mk = (type: string) => ({ span: { start: 0, end: 10, type, confidence: 1 }, detector: 'c0-regex' }); + // 'date' < 'phone' and 'amount' < 'phone' (code-unit order) + const pairs: readonly (readonly [lo: string, hi: string])[] = [ + ['date', 'phone'], + ['amount', 'phone'], + ]; + for (const [lo, hi] of pairs) { + for (const order of [ + [mk(hi), mk(lo)], + [mk(lo), mk(hi)], + ]) { + const resolved = dedupSpans(text, order); + assert.equal(resolved.length, 1); + assert.equal( + resolved[0]!.type, + lo, + `the lexically-smaller type '${lo}' must win over '${hi}' in either input order`, + ); + } + } + }); +}); + +describe('#727 — ISO-8601 dates mask as dates, not phone numbers', () => { + const PHONE_SHAPE = /\+49\s/; // the phone surrogate pool is `+49 30 5559xxxx` + const DATE_SHAPE = /^\d{2}\.\d{2}\.\d{4}$/; // the date surrogate pool is `dd.mm.yyyy` + + it('substitutes a bare ISO date from the DATE surrogate pool', async () => { + const result = await maskPrompt('2026-07-02', [createBaselineDetector()]); + // The winning span is typed `date`, not `phone`. + assert.equal(result.spans.length, 1); + assert.equal(result.spans[0]!.type, 'date'); + assert.equal(result.spans[0]!.value, '2026-07-02'); + // And the wire value is a date surrogate, never a phone number. + assert.ok(DATE_SHAPE.test(result.maskedText), `expected a date surrogate, got '${result.maskedText}'`); + assert.ok(!PHONE_SHAPE.test(result.maskedText), `ISO date leaked as a phone surrogate: '${result.maskedText}'`); + // Round-trips like any masked value. + assert.equal(resolvePseudonyms(result.maskedText, result.map), '2026-07-02'); + }); + + it('the ISO and the dotted shape both mask as dates, and a real phone still masks as a phone (regression: the two must not drift)', async () => { + // The dotted form was already correct; pin it beside the ISO form so a + // future change cannot fix one and re-break the other. A genuine phone + // must still take the phone surrogate — the fix narrows nothing. + // + // `30-06-2027` (dashed dd-mm-yyyy, added for nl/fr by #482) is here + // deliberately: it was broken by the SAME mechanism the issue reported for + // the ISO shape — the phone pattern's `\b0` branch grabs its `06-2027` + // tail and word-boundary extension grows it over the whole token — but the + // issue only named the ISO form, and the C0 eval could not surface either, + // because it scores span COVERAGE, not surrogate type. Any date whose + // month/day segment starts with `0` after a `-` is in this class, so all + // three separator styles are pinned together. + for (const isoOrDotted of ['2026-07-02', '02.07.2026', '1990-06-15', '30-06-2027']) { + const r = await maskPrompt(isoOrDotted, [createBaselineDetector()]); + assert.equal(r.spans[0]?.type, 'date', `${isoOrDotted} must be typed date`); + assert.ok(DATE_SHAPE.test(r.maskedText), `${isoOrDotted} -> non-date surrogate '${r.maskedText}'`); + } + const phone = await maskPrompt('+49 171 2345678', [createBaselineDetector()]); + assert.equal(phone.spans[0]?.type, 'phone'); + assert.ok(PHONE_SHAPE.test(phone.maskedText), `phone -> non-phone surrogate '${phone.maskedText}'`); + }); }); describe('createPromptPseudonymMap', () => {