Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified open-sse/services/compression/preservation.ts
Binary file not shown.
3 changes: 3 additions & 0 deletions open-sse/services/compression/riskGate/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export { detectRiskSpans, type RiskGateConfig, type RiskSpan } from "./riskGate.ts";
export { applyRiskMask, restoreRiskBlocks, type RiskGateStats, type RiskMaskResult } from "./riskGateStep.ts";
export type { RiskCategory } from "./riskPatterns.ts";
136 changes: 136 additions & 0 deletions open-sse/services/compression/riskGate/riskGate.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
import { RISK_PATTERNS, type RiskCategory } from "./riskPatterns.ts";

export interface RiskGateConfig {
enabled: boolean;
/** Subset of categories to scan for. Absent/empty ⇒ all categories. */
categories?: RiskCategory[];
}

export interface RiskSpan {
start: number; // inclusive char offset
end: number; // exclusive
category: RiskCategory;
}

interface Hit {
start: number;
end: number;
category: RiskCategory;
}

const SHORT_SECTION = 200;
const MIN_DDL = 2;
const VCS_LINE = /^(?:commit [0-9a-f]{7,40}|diff --git |@@ |[+-]{3} )/m;
const DIFF_HUNK_LINE = /^[+-]/;

function isLikelyVcsContext(text: string): boolean {
return VCS_LINE.test(text);
}

/** Collect raw regex hits for enabled categories (no guards yet). */
function collectRegexHits(text: string, enabled: Set<RiskCategory>): Hit[] {
const hits: Hit[] = [];
for (const { category, regex } of RISK_PATTERNS) {
if (!enabled.has(category)) continue;
regex.lastIndex = 0;
let m: RegExpExecArray | null;
while ((m = regex.exec(text)) !== null) {
if (m[0].length === 0) {
regex.lastIndex++;
continue;
}
hits.push({ start: m.index, end: m.index + m[0].length, category });
}
}
return hits;
}

/** True when the hit's line starts with a diff marker (`+`/`-`). */
function inDiffHunk(text: string, start: number): boolean {
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
return DIFF_HUNK_LINE.test(text.slice(lineStart, lineStart + 1));
}

/** Structural k8s Secret detector: a YAML doc carrying `kind: Secret` + `data:`. */
function detectK8sSecret(text: string): Hit[] {
const hits: Hit[] = [];
const kindRe = /^kind:[ \t]*Secret\b/gm;
let m: RegExpExecArray | null;
while ((m = kindRe.exec(text)) !== null) {
// Document boundaries: nearest `---`/start before, nearest `---`/end after.
const prevSep = text.lastIndexOf("\n---", m.index);
const docStart = prevSep === -1 ? 0 : prevSep + 1;
const nextSep = text.indexOf("\n---", m.index);
const docEnd = nextSep === -1 ? text.length : nextSep + 1;
const doc = text.slice(docStart, docEnd);
if (/^\s*(?:data|stringData):/m.test(doc)) {
hits.push({ start: docStart, end: docEnd, category: "k8s_secret" });
}
}
return hits;
}

function mergeSpans(spans: RiskSpan[]): RiskSpan[] {
if (spans.length <= 1) return spans;
const sorted = [...spans].sort((a, b) => a.start - b.start);
const out: RiskSpan[] = [sorted[0]];
for (let i = 1; i < sorted.length; i++) {
const last = out[out.length - 1];
const cur = sorted[i];
if (cur.start <= last.end) {
last.end = Math.max(last.end, cur.end);
} else {
out.push(cur);
}
}
return out;
}

/**
* Detect spans that should be shielded from compression. Pure and fail-open:
* any internal error yields an empty result (never throws).
*/
export function detectRiskSpans(text: string, cfg: RiskGateConfig): RiskSpan[] {
try {
if (!cfg.enabled) return [];
if (!text) return [];
const enabled = new Set<RiskCategory>(
cfg.categories?.length
? cfg.categories
: (["stack_trace", "private_key", "secret_assignment", "k8s_secret", "db_migration", "legal"] as RiskCategory[])
);
const vcs = isLikelyVcsContext(text);

const regexHits = collectRegexHits(text, enabled);
const k8sHits = enabled.has("k8s_secret") ? detectK8sSecret(text) : [];

// db_migration: require >=MIN_DDL hits; drop those inside a diff hunk.
const ddl = regexHits.filter((h) => h.category === "db_migration" && !(vcs && inDiffHunk(text, h.start)));
const ddlPromoted: RiskSpan[] =
ddl.length >= MIN_DDL ? [{ start: ddl[0].start, end: ddl[ddl.length - 1].end, category: "db_migration" }] : [];
Comment on lines +109 to +110

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Promoting a single giant span from the first DDL statement to the last DDL statement (ddl[0].start to ddl[ddl.length - 1].end) is extremely blunt. If a document contains two DDL statements separated by a large amount of normal prose, the entire intermediate text will be shielded from compression. Instead, we should promote each DDL hit individually as its own span. mergeSpans will automatically merge them if they actually overlap or are adjacent.

Suggested change
const ddlPromoted: RiskSpan[] =
ddl.length >= MIN_DDL ? [{ start: ddl[0].start, end: ddl[ddl.length - 1].end, category: "db_migration" }] : [];
const ddlPromoted: RiskSpan[] =
ddl.length >= MIN_DDL ? ddl.map((h) => ({ start: h.start, end: h.end, category: "db_migration" })) : [];


// Guarded categories: secret_assignment, stack_trace, legal.
const guarded = regexHits.filter(
(h) => h.category === "secret_assignment" || h.category === "stack_trace" || h.category === "legal"
);
// private_key is the only regex-hit self-evident category; k8s_secret and
// db_migration self-promote via their own structural paths below.
const selfEvident = regexHits.filter((h) => h.category === "private_key");

// Count corroborating signals (self-evident + promoted-ddl + k8s + guarded).
const signalCount = selfEvident.length + (ddlPromoted.length ? 1 : 0) + k8sHits.length + guarded.length;
const shortSection = !vcs && text.length < SHORT_SECTION;
const guardedPromoted = signalCount >= 2 || shortSection ? guarded : [];

const promoted: RiskSpan[] = [
...selfEvident.map((h) => ({ start: h.start, end: h.end, category: h.category })),
...k8sHits.map((h) => ({ start: h.start, end: h.end, category: h.category })),
...ddlPromoted,
...guardedPromoted.map((h) => ({ start: h.start, end: h.end, category: h.category })),
];

return mergeSpans(promoted);
} catch {
return [];
}
}
115 changes: 115 additions & 0 deletions open-sse/services/compression/riskGate/riskGateStep.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import { preserveSpans, restorePreservedBlocks, type PreservedBlock } from "../preservation.ts";
import { detectRiskSpans, type RiskGateConfig } from "./riskGate.ts";
import type { RiskCategory } from "./riskPatterns.ts";

export interface RiskGateStats {
spansProtected: number;
categories: Partial<Record<RiskCategory, number>>;
}

export interface RiskMaskResult {
maskedBody: Record<string, unknown>;
blocks: PreservedBlock[];
stats: RiskGateStats;
}

interface TextPart {
type?: string;
text?: string;
}

/** Mask one content string; returns masked text + blocks + per-category counts. */
function maskString(
text: string,
cfg: RiskGateConfig,
tally: Partial<Record<RiskCategory, number>>
): { masked: string; blocks: PreservedBlock[] } {
const spans = detectRiskSpans(text, cfg);
if (!spans.length) return { masked: text, blocks: [] };
for (const s of spans) tally[s.category] = (tally[s.category] ?? 0) + 1;
const { text: masked, blocks } = preserveSpans(
text,
spans.map((s) => ({ start: s.start, end: s.end, kind: `risk_${s.category}` }))
);
return { masked, blocks };
}

/**
* Mask risky spans in every message content (string or `{type:"text"}` parts).
* Pure: clones touched messages, leaves the original body unmutated. Fail-open.
*/
export function applyRiskMask(body: Record<string, unknown>, cfg: RiskGateConfig): RiskMaskResult {
const tally: Partial<Record<RiskCategory, number>> = {};
const allBlocks: PreservedBlock[] = [];
const messages = body.messages;
if (!Array.isArray(messages)) {
return { maskedBody: body, blocks: [], stats: { spansProtected: 0, categories: {} } };
}

let changed = false;
const maskedMessages = messages.map((msg) => {
const m = msg as { role?: unknown; content?: unknown };
if (typeof m.content === "string") {
Comment on lines +50 to +52

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

If msg is null or not an object at runtime, accessing m.content will throw a TypeError. We should add a defensive guard to ensure msg is a valid object before accessing its properties.

Suggested change
const maskedMessages = messages.map((msg) => {
const m = msg as { role?: unknown; content?: unknown };
if (typeof m.content === "string") {
const maskedMessages = messages.map((msg) => {
if (!msg || typeof msg !== "object") return msg;
const m = msg as { role?: unknown; content?: unknown };
if (typeof m.content === "string") {

const { masked, blocks } = maskString(m.content, cfg, tally);
if (!blocks.length) return msg;
changed = true;
allBlocks.push(...blocks);
return { ...m, content: masked };
}
if (Array.isArray(m.content)) {
let partChanged = false;
const parts = (m.content as TextPart[]).map((p) => {
if (p && p.type === "text" && typeof p.text === "string") {
const { masked, blocks } = maskString(p.text, cfg, tally);
if (!blocks.length) return p;
partChanged = true;
allBlocks.push(...blocks);
return { ...p, text: masked };
}
return p;
});
if (!partChanged) return msg;
changed = true;
return { ...m, content: parts };
}
return msg;
});

const maskedBody = changed ? { ...body, messages: maskedMessages } : body;
return {
maskedBody,
blocks: allBlocks,
stats: { spansProtected: allBlocks.length, categories: tally },
};
}

/** Restore every masked span in the (possibly compressed) body. Fail-open. */
export function restoreRiskBlocks(
body: Record<string, unknown>,
blocks: PreservedBlock[]
): Record<string, unknown> {
if (!blocks.length) return body;
const messages = body.messages;
if (!Array.isArray(messages)) return body;
const restoreParts = (content: unknown): unknown => {
if (typeof content === "string") return restorePreservedBlocks(content, blocks);
if (Array.isArray(content)) {
return (content as TextPart[]).map((p) =>
p && p.type === "text" && typeof p.text === "string"
? { ...p, text: restorePreservedBlocks(p.text, blocks) }
: p
);
}
return content;
};
return {
...body,
messages: messages.map((msg) => {
const m = msg as { content?: unknown };
if (typeof m.content === "string" || Array.isArray(m.content)) {
Comment on lines +107 to +109

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

If msg is null or not an object at runtime, accessing m.content will throw a TypeError. We should add a defensive guard to ensure msg is a valid object before accessing its properties.

Suggested change
messages: messages.map((msg) => {
const m = msg as { content?: unknown };
if (typeof m.content === "string" || Array.isArray(m.content)) {
messages: messages.map((msg) => {
if (!msg || typeof msg !== "object") return msg;
const m = msg as { content?: unknown };
if (typeof m.content === "string" || Array.isArray(m.content)) {

return { ...m, content: restoreParts(m.content) };
}
return msg;
}),
};
}
61 changes: 61 additions & 0 deletions open-sse/services/compression/riskGate/riskPatterns.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/**
* Risk-gate pattern catalog. Every variable-length pattern uses bounded
* quantifiers (`{0,N}`) to prevent catastrophic backtracking (ReDoS) on
* untrusted input. Patterns are ours (not agent-supplied), so safe-regex is not
* required — boundedness is verified by an adversarial-input test.
*/
export type RiskCategory =
| "stack_trace"
| "private_key"
| "secret_assignment"
| "k8s_secret"
| "db_migration"
| "legal";

export const MAX_PEM_LEN = 4096;

export interface RiskPattern {
category: RiskCategory;
regex: RegExp;
}

/**
* Categories whose single match is strong enough evidence on its own (no
* corroborating second signal / short-section required).
* `k8s_secret` and `db_migration` are promoted structurally in riskGate.ts.
*/
export const SELF_EVIDENT: ReadonlySet<RiskCategory> = new Set<RiskCategory>([
"private_key",
"k8s_secret",
"db_migration",
]);

export const RISK_PATTERNS: RiskPattern[] = [
{
category: "private_key",
regex: new RegExp(
`-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----[\\s\\S]{1,${MAX_PEM_LEN}}?-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----`,
"g"
),
},
{
category: "secret_assignment",
regex:
/\b(?:api[_-]?key|secret|token|password|passwd|bearer|authorization|client[_-]?secret)\b[ \t]{0,20}[:=][ \t]{0,20}["']?[A-Za-z0-9._\-+/]{8,200}/gi,
Comment on lines +43 to +44

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

security-medium medium

The secret_assignment regex matches an optional opening quote ["']? but does not match the corresponding closing quote. This leaves the closing quote exposed to lossy compression, which can lead to syntax errors or broken quotes in the restored text if the lossy compressor decides to strip or modify the unmatched trailing quote. We should use a capture group and backreference to match matching quotes cleanly.

Suggested change
regex:
/\b(?:api[_-]?key|secret|token|password|passwd|bearer|authorization|client[_-]?secret)\b[ \t]{0,20}[:=][ \t]{0,20}["']?[A-Za-z0-9._\-+/]{8,200}/gi,
regex:
/\b(?:api[_-]?key|secret|token|password|passwd|bearer|authorization|client[_-]?secret)\b[ \t]{0,20}[:=][ \t]{0,20}(["']?)[A-Za-z0-9._\-+/]{8,200}\1/gi,

},
{
category: "stack_trace",
regex:
/^\s{0,8}(?:at\s+\S.{0,300}|File ".{1,300}", line \d{1,9}|Traceback \(most recent call last\):|[A-Za-z_.]{1,80}(?:Error|Exception):.{0,300})$/gm,
},
{
// Single DDL hit; the ">=2 DDL" promotion rule is enforced in riskGate.ts.
category: "db_migration",
regex: /\b(?:CREATE|ALTER|DROP)\s+(?:TABLE|INDEX|SCHEMA|DATABASE|COLUMN)\b/gi,
},
{
category: "legal",
regex:
/\bWITHOUT WARRANTY\b|\bPermission is hereby granted\b|^SPDX-License-Identifier:.{0,200}$|\bCopyright \(c\)\b|\bAll rights reserved\b/gim,
},
];
44 changes: 44 additions & 0 deletions open-sse/services/compression/riskGate/strategyWrap.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import type { CompressionResult } from "../types.ts";
import type { CompressionConfig } from "../types.ts";
import { applyRiskMask, restoreRiskBlocks } from "./riskGateStep.ts";
import type { RiskGateConfig } from "./riskGate.ts";

/** Resolve the effective risk-gate config (explicit option wins over config); enabled-gated. */
export function resolveRiskGate(options?: {
riskGate?: RiskGateConfig;
config?: CompressionConfig;
}): RiskGateConfig | undefined {
const rg = options?.riskGate ?? options?.config?.riskGate;
return rg?.enabled ? rg : undefined;
}

function attach(
result: CompressionResult,
mask: ReturnType<typeof applyRiskMask>
): CompressionResult {
if (mask.blocks.length) result.body = restoreRiskBlocks(result.body, mask.blocks);
if (result.stats) result.stats.riskGate = mask.stats;
return result;
}

/** Outer mask→run→restore wrapper for a sync compression entry point. Byte-identical when gate absent. */
export function withRiskGate(
body: Record<string, unknown>,
riskGate: RiskGateConfig | undefined,
run: (b: Record<string, unknown>) => CompressionResult
): CompressionResult {
if (!riskGate) return run(body);
const mask = applyRiskMask(body, riskGate);
return attach(run(mask.maskedBody), mask);
}

/** Async variant of withRiskGate. */
export async function withRiskGateAsync(
body: Record<string, unknown>,
riskGate: RiskGateConfig | undefined,
run: (b: Record<string, unknown>) => Promise<CompressionResult>
): Promise<CompressionResult> {
if (!riskGate) return run(body);
const mask = applyRiskMask(body, riskGate);
return attach(await run(mask.maskedBody), mask);
}
Loading
Loading