Repository navigation
feat(compression): risk-gate pre-pass — shield sensitive spans from lossy compression #5243
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 9 commits
4dbef70
eb966dc
92be57c
4f80f30
56b0340
02217c4
1dbb88a
be84b06
1cffa13
323c7e7
0605721
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| export { detectRiskSpans, type RiskGateConfig, type RiskSpan } from "./riskGate.ts"; | ||
| export { applyRiskMask, restoreRiskBlocks, type RiskGateStats, type RiskMaskResult } from "./riskGateStep.ts"; | ||
| export type { RiskCategory } from "./riskPatterns.ts"; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| import { RISK_PATTERNS, type RiskCategory } from "./riskPatterns.ts"; | ||
|
|
||
| export interface RiskGateConfig { | ||
| enabled: boolean; | ||
| /** Subset of categories to scan for. Absent/empty ⇒ all categories. */ | ||
| categories?: RiskCategory[]; | ||
| } | ||
|
|
||
| export interface RiskSpan { | ||
| start: number; // inclusive char offset | ||
| end: number; // exclusive | ||
| category: RiskCategory; | ||
| } | ||
|
|
||
| interface Hit { | ||
| start: number; | ||
| end: number; | ||
| category: RiskCategory; | ||
| } | ||
|
|
||
| const SHORT_SECTION = 200; | ||
| const MIN_DDL = 2; | ||
| const VCS_LINE = /^(?:commit [0-9a-f]{7,40}|diff --git |@@ |[+-]{3} )/m; | ||
| const DIFF_HUNK_LINE = /^[+-]/; | ||
|
|
||
| function isLikelyVcsContext(text: string): boolean { | ||
| return VCS_LINE.test(text); | ||
| } | ||
|
|
||
| /** Collect raw regex hits for enabled categories (no guards yet). */ | ||
| function collectRegexHits(text: string, enabled: Set<RiskCategory>): Hit[] { | ||
| const hits: Hit[] = []; | ||
| for (const { category, regex } of RISK_PATTERNS) { | ||
| if (!enabled.has(category)) continue; | ||
| regex.lastIndex = 0; | ||
| let m: RegExpExecArray | null; | ||
| while ((m = regex.exec(text)) !== null) { | ||
| if (m[0].length === 0) { | ||
| regex.lastIndex++; | ||
| continue; | ||
| } | ||
| hits.push({ start: m.index, end: m.index + m[0].length, category }); | ||
| } | ||
| } | ||
| return hits; | ||
| } | ||
|
|
||
| /** True when the hit's line starts with a diff marker (`+`/`-`). */ | ||
| function inDiffHunk(text: string, start: number): boolean { | ||
| const lineStart = text.lastIndexOf("\n", start - 1) + 1; | ||
| return DIFF_HUNK_LINE.test(text.slice(lineStart, lineStart + 1)); | ||
| } | ||
|
|
||
| /** Structural k8s Secret detector: a YAML doc carrying `kind: Secret` + `data:`. */ | ||
| function detectK8sSecret(text: string): Hit[] { | ||
| const hits: Hit[] = []; | ||
| const kindRe = /^kind:[ \t]*Secret\b/gm; | ||
| let m: RegExpExecArray | null; | ||
| while ((m = kindRe.exec(text)) !== null) { | ||
| // Document boundaries: nearest `---`/start before, nearest `---`/end after. | ||
| const prevSep = text.lastIndexOf("\n---", m.index); | ||
| const docStart = prevSep === -1 ? 0 : prevSep + 1; | ||
| const nextSep = text.indexOf("\n---", m.index); | ||
| const docEnd = nextSep === -1 ? text.length : nextSep + 1; | ||
| const doc = text.slice(docStart, docEnd); | ||
| if (/^\s*(?:data|stringData):/m.test(doc)) { | ||
| hits.push({ start: docStart, end: docEnd, category: "k8s_secret" }); | ||
| } | ||
| } | ||
| return hits; | ||
| } | ||
|
|
||
| function mergeSpans(spans: RiskSpan[]): RiskSpan[] { | ||
| if (spans.length <= 1) return spans; | ||
| const sorted = [...spans].sort((a, b) => a.start - b.start); | ||
| const out: RiskSpan[] = [sorted[0]]; | ||
| for (let i = 1; i < sorted.length; i++) { | ||
| const last = out[out.length - 1]; | ||
| const cur = sorted[i]; | ||
| if (cur.start <= last.end) { | ||
| last.end = Math.max(last.end, cur.end); | ||
| } else { | ||
| out.push(cur); | ||
| } | ||
| } | ||
| return out; | ||
| } | ||
|
|
||
| /** | ||
| * Detect spans that should be shielded from compression. Pure and fail-open: | ||
| * any internal error yields an empty result (never throws). | ||
| */ | ||
| export function detectRiskSpans(text: string, cfg: RiskGateConfig): RiskSpan[] { | ||
| try { | ||
| if (!cfg.enabled) return []; | ||
| if (!text) return []; | ||
| const enabled = new Set<RiskCategory>( | ||
| cfg.categories?.length | ||
| ? cfg.categories | ||
| : (["stack_trace", "private_key", "secret_assignment", "k8s_secret", "db_migration", "legal"] as RiskCategory[]) | ||
| ); | ||
| const vcs = isLikelyVcsContext(text); | ||
|
|
||
| const regexHits = collectRegexHits(text, enabled); | ||
| const k8sHits = enabled.has("k8s_secret") ? detectK8sSecret(text) : []; | ||
|
|
||
| // db_migration: require >=MIN_DDL hits; drop those inside a diff hunk. | ||
| const ddl = regexHits.filter((h) => h.category === "db_migration" && !(vcs && inDiffHunk(text, h.start))); | ||
| const ddlPromoted: RiskSpan[] = | ||
| ddl.length >= MIN_DDL ? [{ start: ddl[0].start, end: ddl[ddl.length - 1].end, category: "db_migration" }] : []; | ||
|
|
||
| // Guarded categories: secret_assignment, stack_trace, legal. | ||
| const guarded = regexHits.filter( | ||
| (h) => h.category === "secret_assignment" || h.category === "stack_trace" || h.category === "legal" | ||
| ); | ||
| // private_key is the only regex-hit self-evident category; k8s_secret and | ||
| // db_migration self-promote via their own structural paths below. | ||
| const selfEvident = regexHits.filter((h) => h.category === "private_key"); | ||
|
|
||
| // Count corroborating signals (self-evident + promoted-ddl + k8s + guarded). | ||
| const signalCount = selfEvident.length + (ddlPromoted.length ? 1 : 0) + k8sHits.length + guarded.length; | ||
| const shortSection = !vcs && text.length < SHORT_SECTION; | ||
| const guardedPromoted = signalCount >= 2 || shortSection ? guarded : []; | ||
|
|
||
| const promoted: RiskSpan[] = [ | ||
| ...selfEvident.map((h) => ({ start: h.start, end: h.end, category: h.category })), | ||
| ...k8sHits.map((h) => ({ start: h.start, end: h.end, category: h.category })), | ||
| ...ddlPromoted, | ||
| ...guardedPromoted.map((h) => ({ start: h.start, end: h.end, category: h.category })), | ||
| ]; | ||
|
|
||
| return mergeSpans(promoted); | ||
| } catch { | ||
| return []; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,115 @@ | ||||||||||||||||
| import { preserveSpans, restorePreservedBlocks, type PreservedBlock } from "../preservation.ts"; | ||||||||||||||||
| import { detectRiskSpans, type RiskGateConfig } from "./riskGate.ts"; | ||||||||||||||||
| import type { RiskCategory } from "./riskPatterns.ts"; | ||||||||||||||||
|
|
||||||||||||||||
| export interface RiskGateStats { | ||||||||||||||||
| spansProtected: number; | ||||||||||||||||
| categories: Partial<Record<RiskCategory, number>>; | ||||||||||||||||
| } | ||||||||||||||||
|
|
||||||||||||||||
| export interface RiskMaskResult { | ||||||||||||||||
| maskedBody: Record<string, unknown>; | ||||||||||||||||
| blocks: PreservedBlock[]; | ||||||||||||||||
| stats: RiskGateStats; | ||||||||||||||||
| } | ||||||||||||||||
|
|
||||||||||||||||
| interface TextPart { | ||||||||||||||||
| type?: string; | ||||||||||||||||
| text?: string; | ||||||||||||||||
| } | ||||||||||||||||
|
|
||||||||||||||||
| /** Mask one content string; returns masked text + blocks + per-category counts. */ | ||||||||||||||||
| function maskString( | ||||||||||||||||
| text: string, | ||||||||||||||||
| cfg: RiskGateConfig, | ||||||||||||||||
| tally: Partial<Record<RiskCategory, number>> | ||||||||||||||||
| ): { masked: string; blocks: PreservedBlock[] } { | ||||||||||||||||
| const spans = detectRiskSpans(text, cfg); | ||||||||||||||||
| if (!spans.length) return { masked: text, blocks: [] }; | ||||||||||||||||
| for (const s of spans) tally[s.category] = (tally[s.category] ?? 0) + 1; | ||||||||||||||||
| const { text: masked, blocks } = preserveSpans( | ||||||||||||||||
| text, | ||||||||||||||||
| spans.map((s) => ({ start: s.start, end: s.end, kind: `risk_${s.category}` })) | ||||||||||||||||
| ); | ||||||||||||||||
| return { masked, blocks }; | ||||||||||||||||
| } | ||||||||||||||||
|
|
||||||||||||||||
| /** | ||||||||||||||||
| * Mask risky spans in every message content (string or `{type:"text"}` parts). | ||||||||||||||||
| * Pure: clones touched messages, leaves the original body unmutated. Fail-open. | ||||||||||||||||
| */ | ||||||||||||||||
| export function applyRiskMask(body: Record<string, unknown>, cfg: RiskGateConfig): RiskMaskResult { | ||||||||||||||||
| const tally: Partial<Record<RiskCategory, number>> = {}; | ||||||||||||||||
| const allBlocks: PreservedBlock[] = []; | ||||||||||||||||
| const messages = body.messages; | ||||||||||||||||
| if (!Array.isArray(messages)) { | ||||||||||||||||
| return { maskedBody: body, blocks: [], stats: { spansProtected: 0, categories: {} } }; | ||||||||||||||||
| } | ||||||||||||||||
|
|
||||||||||||||||
| let changed = false; | ||||||||||||||||
| const maskedMessages = messages.map((msg) => { | ||||||||||||||||
| const m = msg as { role?: unknown; content?: unknown }; | ||||||||||||||||
| if (typeof m.content === "string") { | ||||||||||||||||
|
Comment on lines
+50
to
+52
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If
Suggested change
|
||||||||||||||||
| const { masked, blocks } = maskString(m.content, cfg, tally); | ||||||||||||||||
| if (!blocks.length) return msg; | ||||||||||||||||
| changed = true; | ||||||||||||||||
| allBlocks.push(...blocks); | ||||||||||||||||
| return { ...m, content: masked }; | ||||||||||||||||
| } | ||||||||||||||||
| if (Array.isArray(m.content)) { | ||||||||||||||||
| let partChanged = false; | ||||||||||||||||
| const parts = (m.content as TextPart[]).map((p) => { | ||||||||||||||||
| if (p && p.type === "text" && typeof p.text === "string") { | ||||||||||||||||
| const { masked, blocks } = maskString(p.text, cfg, tally); | ||||||||||||||||
| if (!blocks.length) return p; | ||||||||||||||||
| partChanged = true; | ||||||||||||||||
| allBlocks.push(...blocks); | ||||||||||||||||
| return { ...p, text: masked }; | ||||||||||||||||
| } | ||||||||||||||||
| return p; | ||||||||||||||||
| }); | ||||||||||||||||
| if (!partChanged) return msg; | ||||||||||||||||
| changed = true; | ||||||||||||||||
| return { ...m, content: parts }; | ||||||||||||||||
| } | ||||||||||||||||
| return msg; | ||||||||||||||||
| }); | ||||||||||||||||
|
|
||||||||||||||||
| const maskedBody = changed ? { ...body, messages: maskedMessages } : body; | ||||||||||||||||
| return { | ||||||||||||||||
| maskedBody, | ||||||||||||||||
| blocks: allBlocks, | ||||||||||||||||
| stats: { spansProtected: allBlocks.length, categories: tally }, | ||||||||||||||||
| }; | ||||||||||||||||
| } | ||||||||||||||||
|
|
||||||||||||||||
| /** Restore every masked span in the (possibly compressed) body. Fail-open. */ | ||||||||||||||||
| export function restoreRiskBlocks( | ||||||||||||||||
| body: Record<string, unknown>, | ||||||||||||||||
| blocks: PreservedBlock[] | ||||||||||||||||
| ): Record<string, unknown> { | ||||||||||||||||
| if (!blocks.length) return body; | ||||||||||||||||
| const messages = body.messages; | ||||||||||||||||
| if (!Array.isArray(messages)) return body; | ||||||||||||||||
| const restoreParts = (content: unknown): unknown => { | ||||||||||||||||
| if (typeof content === "string") return restorePreservedBlocks(content, blocks); | ||||||||||||||||
| if (Array.isArray(content)) { | ||||||||||||||||
| return (content as TextPart[]).map((p) => | ||||||||||||||||
| p && p.type === "text" && typeof p.text === "string" | ||||||||||||||||
| ? { ...p, text: restorePreservedBlocks(p.text, blocks) } | ||||||||||||||||
| : p | ||||||||||||||||
| ); | ||||||||||||||||
| } | ||||||||||||||||
| return content; | ||||||||||||||||
| }; | ||||||||||||||||
| return { | ||||||||||||||||
| ...body, | ||||||||||||||||
| messages: messages.map((msg) => { | ||||||||||||||||
| const m = msg as { content?: unknown }; | ||||||||||||||||
| if (typeof m.content === "string" || Array.isArray(m.content)) { | ||||||||||||||||
|
Comment on lines
+107
to
+109
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If
Suggested change
|
||||||||||||||||
| return { ...m, content: restoreParts(m.content) }; | ||||||||||||||||
| } | ||||||||||||||||
| return msg; | ||||||||||||||||
| }), | ||||||||||||||||
| }; | ||||||||||||||||
| } | ||||||||||||||||
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,61 @@ | ||||||||||
| /** | ||||||||||
| * Risk-gate pattern catalog. Every variable-length pattern uses bounded | ||||||||||
| * quantifiers (`{0,N}`) to prevent catastrophic backtracking (ReDoS) on | ||||||||||
| * untrusted input. Patterns are ours (not agent-supplied), so safe-regex is not | ||||||||||
| * required — boundedness is verified by an adversarial-input test. | ||||||||||
| */ | ||||||||||
| export type RiskCategory = | ||||||||||
| | "stack_trace" | ||||||||||
| | "private_key" | ||||||||||
| | "secret_assignment" | ||||||||||
| | "k8s_secret" | ||||||||||
| | "db_migration" | ||||||||||
| | "legal"; | ||||||||||
|
|
||||||||||
| export const MAX_PEM_LEN = 4096; | ||||||||||
|
|
||||||||||
| export interface RiskPattern { | ||||||||||
| category: RiskCategory; | ||||||||||
| regex: RegExp; | ||||||||||
| } | ||||||||||
|
|
||||||||||
| /** | ||||||||||
| * Categories whose single match is strong enough evidence on its own (no | ||||||||||
| * corroborating second signal / short-section required). | ||||||||||
| * `k8s_secret` and `db_migration` are promoted structurally in riskGate.ts. | ||||||||||
| */ | ||||||||||
| export const SELF_EVIDENT: ReadonlySet<RiskCategory> = new Set<RiskCategory>([ | ||||||||||
| "private_key", | ||||||||||
| "k8s_secret", | ||||||||||
| "db_migration", | ||||||||||
| ]); | ||||||||||
|
|
||||||||||
| export const RISK_PATTERNS: RiskPattern[] = [ | ||||||||||
| { | ||||||||||
| category: "private_key", | ||||||||||
| regex: new RegExp( | ||||||||||
| `-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----[\\s\\S]{1,${MAX_PEM_LEN}}?-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----`, | ||||||||||
| "g" | ||||||||||
| ), | ||||||||||
| }, | ||||||||||
| { | ||||||||||
| category: "secret_assignment", | ||||||||||
| regex: | ||||||||||
| /\b(?:api[_-]?key|secret|token|password|passwd|bearer|authorization|client[_-]?secret)\b[ \t]{0,20}[:=][ \t]{0,20}["']?[A-Za-z0-9._\-+/]{8,200}/gi, | ||||||||||
|
Comment on lines
+43
to
+44
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The
Suggested change
|
||||||||||
| }, | ||||||||||
| { | ||||||||||
| category: "stack_trace", | ||||||||||
| regex: | ||||||||||
| /^\s{0,8}(?:at\s+\S.{0,300}|File ".{1,300}", line \d{1,9}|Traceback \(most recent call last\):|[A-Za-z_.]{1,80}(?:Error|Exception):.{0,300})$/gm, | ||||||||||
| }, | ||||||||||
| { | ||||||||||
| // Single DDL hit; the ">=2 DDL" promotion rule is enforced in riskGate.ts. | ||||||||||
| category: "db_migration", | ||||||||||
| regex: /\b(?:CREATE|ALTER|DROP)\s+(?:TABLE|INDEX|SCHEMA|DATABASE|COLUMN)\b/gi, | ||||||||||
| }, | ||||||||||
| { | ||||||||||
| category: "legal", | ||||||||||
| regex: | ||||||||||
| /\bWITHOUT WARRANTY\b|\bPermission is hereby granted\b|^SPDX-License-Identifier:.{0,200}$|\bCopyright \(c\)\b|\bAll rights reserved\b/gim, | ||||||||||
| }, | ||||||||||
| ]; | ||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| import type { CompressionResult } from "../types.ts"; | ||
| import type { CompressionConfig } from "../types.ts"; | ||
| import { applyRiskMask, restoreRiskBlocks } from "./riskGateStep.ts"; | ||
| import type { RiskGateConfig } from "./riskGate.ts"; | ||
|
|
||
| /** Resolve the effective risk-gate config (explicit option wins over config); enabled-gated. */ | ||
| export function resolveRiskGate(options?: { | ||
| riskGate?: RiskGateConfig; | ||
| config?: CompressionConfig; | ||
| }): RiskGateConfig | undefined { | ||
| const rg = options?.riskGate ?? options?.config?.riskGate; | ||
| return rg?.enabled ? rg : undefined; | ||
| } | ||
|
|
||
| function attach( | ||
| result: CompressionResult, | ||
| mask: ReturnType<typeof applyRiskMask> | ||
| ): CompressionResult { | ||
| if (mask.blocks.length) result.body = restoreRiskBlocks(result.body, mask.blocks); | ||
| if (result.stats) result.stats.riskGate = mask.stats; | ||
| return result; | ||
| } | ||
|
|
||
| /** Outer mask→run→restore wrapper for a sync compression entry point. Byte-identical when gate absent. */ | ||
| export function withRiskGate( | ||
| body: Record<string, unknown>, | ||
| riskGate: RiskGateConfig | undefined, | ||
| run: (b: Record<string, unknown>) => CompressionResult | ||
| ): CompressionResult { | ||
| if (!riskGate) return run(body); | ||
| const mask = applyRiskMask(body, riskGate); | ||
| return attach(run(mask.maskedBody), mask); | ||
| } | ||
|
|
||
| /** Async variant of withRiskGate. */ | ||
| export async function withRiskGateAsync( | ||
| body: Record<string, unknown>, | ||
| riskGate: RiskGateConfig | undefined, | ||
| run: (b: Record<string, unknown>) => Promise<CompressionResult> | ||
| ): Promise<CompressionResult> { | ||
| if (!riskGate) return run(body); | ||
| const mask = applyRiskMask(body, riskGate); | ||
| return attach(await run(mask.maskedBody), mask); | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Promoting a single giant span from the first DDL statement to the last DDL statement (
ddl[0].starttoddl[ddl.length - 1].end) is extremely blunt. If a document contains two DDL statements separated by a large amount of normal prose, the entire intermediate text will be shielded from compression. Instead, we should promote each DDL hit individually as its own span.mergeSpanswill automatically merge them if they actually overlap or are adjacent.