merge queue: checking release/v3.8.49 (7123236), #7100 and #7102 together

changelog.d/fixes/1556-openai-regex-lookaround.md

@@ -0,0 +1 @@
+- **fix(codex):** strip regex `pattern` lookaround (lookahead/lookbehind) from tool JSON Schemas on the Codex/OpenAI native passthrough path — previously only the translated-request path coerced tool schemas, so a `pattern` like `^(?=.*@).+$` reached OpenAI unmodified and was rejected with `regex lookaround is not supported`. (thanks @evinjohnn) (#7100)

changelog.d/fixes/port-2132-headroom-developer-role.md

@@ -0,0 +1 @@
+- **fix(compression):** the Headroom SmartCrusher tabular-compaction guard now also skips `role: "developer"` messages, not just `role: "system"` — Codex CLI sends its instructions/tool-schema turn as `developer` (the Responses-API equivalent of `system`), so an embedded JSON array (e.g. an `update_plan` example) could get tabular-compacted, corrupting the model's tool-calling instructions and breaking Codex CLI plan mode. (thanks @SingCJ)

open-sse/executors/codex/tools.ts

@@ -1,6 +1,8 @@
 // Codex Responses-API tool normalization (hosted-tool passthrough + free-plan gating).
 // Extracted verbatim from codex.ts. Self-contained (console.debug only).
 
+import { stripUnsupportedRegexPatterns } from "../../translator/helpers/schemaCoercion.ts";
+
 // Responses-API hosted tool types that OpenAI/Codex executes server-side.
 // These arrive shaped as `{ type, ...params }` with no `function` object and no `name` —
 // e.g. Codex CLI injects `{ type: "image_generation", output_format: "png" }` or
@@ -133,14 +135,19 @@ export function normalizeCodexTools(
           ? functionObject.strict
           : undefined;
 
+    // Codex/OpenAI Responses API rejects `pattern` fields using regex lookaround
+    // (e.g. `^(?=.*@).+$`) with a 400 "regex lookaround is not supported" error.
+    // Strip those before the schema reaches upstream (9router#1556).
+    const sanitizedParameters = stripUnsupportedRegexPatterns(parameters);
+
     // Rewrite in-place to Responses format
     for (const key of Object.keys(tool)) {
       delete tool[key];
     }
     tool.type = "function";
     tool.name = name.slice(0, 128);
     if (description) tool.description = description;
-    tool.parameters = parameters;
+    tool.parameters = sanitizedParameters;
     if (strict !== undefined) tool.strict = strict;
 
     validToolNames.add(name);

open-sse/services/compression/engines/headroom/smartcrusher.ts

@@ -160,7 +160,7 @@ export function collectCompactableArrays(
     while ((m = regex.exec(text)) !== null) pushIfCompactable(m[1].trim());
   };
   for (const msg of messages) {
-    if (msg.role === "system") continue;
+    if (msg.role === "system" || msg.role === "developer") continue;
     if (typeof msg.content === "string") scanText(msg.content);
     else if (Array.isArray(msg.content)) {
       for (const part of msg.content) {
@@ -218,8 +218,12 @@ export function crushMessages(
   let changed = false;
 
   const result = messages.map((msg): MessageLike => {
-    // Guard: never touch system messages
-    if (msg.role === "system") return { ...msg };
+    // Guard: never touch system messages. "developer" is the Responses-API equivalent of
+    // "system" used by newer models (e.g. Codex CLI, see open-sse/executors/codex.ts) and
+    // carries the same kind of instructions/tool-schema content — compacting a JSON array
+    // embedded there (e.g. an update_plan example) can corrupt the model's tool-calling
+    // instructions (9router#2132: broke Codex CLI plan mode).
+    if (msg.role === "system" || msg.role === "developer") return { ...msg };
 
     if (typeof msg.content === "string") {
       const crushed = crushText(msg.content, minRows);

open-sse/translator/helpers/schemaCoercion.ts

@@ -24,6 +24,18 @@ const NUMERIC_SCHEMA_FIELDS = [
   "multipleOf",
 ] as const;
 
+// Fix (9router#1556): OpenAI/Codex's Responses API rejects JSON Schema `pattern`
+// values that use regex lookaround (lookahead/lookbehind) with
+// "Invalid JSON schema: regex lookaround is not supported.". IDE/SDK agent
+// harnesses commonly emit lookahead patterns (e.g. `^(?=.*@).+$`), so any
+// `pattern` field containing `(?=`, `(?!`, `(?<=`, or `(?<!` must be dropped
+// before the schema reaches the Codex/OpenAI upstream.
+const REGEX_LOOKAROUND_PATTERN = /\(\?<?[=!]/;
+
+function hasUnsupportedRegexLookaround(pattern: unknown): boolean {
+  return typeof pattern === "string" && REGEX_LOOKAROUND_PATTERN.test(pattern);
+}
+
 function isPlainObject(value: unknown): value is JsonRecord {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value);
 }
@@ -81,6 +93,11 @@ export function coerceSchemaNumericFields(schema: unknown): unknown {
     delete result.default;
   }
 
+  // Fix (9router#1556): drop unsupported regex lookaround from `pattern`.
+  if (hasUnsupportedRegexLookaround(result.pattern)) {
+    delete result.pattern;
+  }
+
   for (const field of NUMERIC_SCHEMA_FIELDS) {
     if (field in result) {
       result[field] = coerceNumericString(result[field]);
@@ -142,6 +159,70 @@ export function coerceSchemaNumericFields(schema: unknown): unknown {
   return result;
 }
 
+// Sub-schema maps keyed by property name (each value is itself walked recursively).
+const REGEX_STRIP_OBJECT_MAP_FIELDS = [
+  "properties",
+  "patternProperties",
+  "definitions",
+  "$defs",
+] as const;
+
+// Sub-schema lists (each entry is itself walked recursively).
+const REGEX_STRIP_ARRAY_MAP_FIELDS = ["prefixItems", "anyOf", "oneOf", "allOf"] as const;
+
+/** Recursively strips unsupported regex lookaround from every value of an object map field. */
+function stripRegexFromObjectMap(record: JsonRecord): JsonRecord {
+  return Object.fromEntries(
+    Object.entries(record).map(([key, value]) => [key, stripUnsupportedRegexPatterns(value)])
+  );
+}
+
+/**
+ * Strip regex `pattern` constraints that use lookaround (lookahead/lookbehind),
+ * which OpenAI/Codex's Responses API rejects outright with a 400
+ * ("Invalid JSON schema: regex lookaround is not supported."). Walks the same
+ * JSON Schema shape as `coerceSchemaNumericFields` (properties, items,
+ * anyOf/oneOf/allOf, $defs/definitions, etc). See 9router#1556.
+ */
+export function stripUnsupportedRegexPatterns(schema: unknown): unknown {
+  if (Array.isArray(schema)) {
+    return schema.map((entry) => stripUnsupportedRegexPatterns(entry));
+  }
+  if (!isPlainObject(schema)) return schema;
+
+  const result: JsonRecord = { ...schema };
+
+  if (hasUnsupportedRegexLookaround(result.pattern)) {
+    delete result.pattern;
+  }
+
+  for (const field of REGEX_STRIP_OBJECT_MAP_FIELDS) {
+    if (isPlainObject(result[field])) {
+      result[field] = stripRegexFromObjectMap(result[field]);
+    }
+  }
+
+  for (const field of REGEX_STRIP_ARRAY_MAP_FIELDS) {
+    if (Array.isArray(result[field])) {
+      result[field] = (result[field] as unknown[]).map((entry) =>
+        stripUnsupportedRegexPatterns(entry)
+      );
+    }
+  }
+
+  if (result.items !== undefined) {
+    result.items = stripUnsupportedRegexPatterns(result.items);
+  }
+  if (result.additionalProperties && typeof result.additionalProperties === "object") {
+    result.additionalProperties = stripUnsupportedRegexPatterns(result.additionalProperties);
+  }
+  if (isPlainObject(result.not)) {
+    result.not = stripUnsupportedRegexPatterns(result.not);
+  }
+
+  return result;
+}
+
 export function sanitizeToolDescription(tool: unknown): unknown {
   if (!isPlainObject(tool)) return tool;
 

tests/unit/codex-tools-regex-lookaround.test.ts

@@ -0,0 +1,74 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+import { normalizeCodexTools } from "../../open-sse/executors/codex/tools.ts";
+
+// Port of 9router#1556: OpenAI/Codex Responses API rejects JSON Schema `pattern`
+// fields containing regex lookaround (lookahead/lookbehind) with:
+//   "Invalid JSON schema: regex lookaround is not supported. Found at $.properties.email.pattern."
+// Clients (e.g. IDE agent harnesses) commonly emit lookahead patterns such as
+// `^(?=.*@).+$` for "must contain an @". These must be stripped before the
+// tool schema reaches the Codex/OpenAI Responses API.
+test("normalizeCodexTools strips regex lookaround from function tool parameter patterns", () => {
+  const body: Record<string, unknown> = {
+    tools: [
+      {
+        type: "function",
+        function: {
+          name: "send_email",
+          description: "Send an email",
+          parameters: {
+            type: "object",
+            properties: {
+              email: {
+                type: "string",
+                pattern: "^(?=.*@).+$",
+              },
+            },
+          },
+        },
+      },
+    ],
+  };
+
+  normalizeCodexTools(body);
+
+  const tools = body.tools as Array<Record<string, unknown>>;
+  const parameters = tools[0].parameters as Record<string, unknown>;
+  const properties = parameters.properties as Record<string, unknown>;
+  const emailSchema = properties.email as Record<string, unknown>;
+
+  assert.equal(
+    emailSchema.pattern,
+    undefined,
+    "lookaround pattern must be stripped, not forwarded upstream"
+  );
+});
+
+test("normalizeCodexTools preserves plain (non-lookaround) regex patterns", () => {
+  const body: Record<string, unknown> = {
+    tools: [
+      {
+        type: "function",
+        function: {
+          name: "send_email",
+          parameters: {
+            type: "object",
+            properties: {
+              zip: { type: "string", pattern: "^[0-9]{5}$" },
+            },
+          },
+        },
+      },
+    ],
+  };
+
+  normalizeCodexTools(body);
+
+  const tools = body.tools as Array<Record<string, unknown>>;
+  const parameters = tools[0].parameters as Record<string, unknown>;
+  const properties = parameters.properties as Record<string, unknown>;
+  const zipSchema = properties.zip as Record<string, unknown>;
+
+  assert.equal(zipSchema.pattern, "^[0-9]{5}$");
+});

tests/unit/compression/headroom-developer-role-2132.test.ts

@@ -0,0 +1,91 @@
+/**
+ * Regression test for upstream 9router#2132 (ported): "Token saver Headroom ruins plan mode
+ * in Codex CLI".
+ *
+ * Root cause: SmartCrusher's system-message guard only checked `role === "system"`. Codex CLI
+ * (open-sse/executors/codex.ts) sends its instructions/tool-schema turn with role "developer"
+ * (the Responses-API equivalent of "system" used by newer models). Every other guard in this
+ * codebase that excludes "system" also excludes "developer" (see roleNormalizer.ts,
+ * contextManager.ts, claudeUpstreamMessages.ts, etc.) — SmartCrusher was the exception, so it
+ * happily tabular-compacted JSON arrays (e.g. the update_plan tool schema/examples) embedded in
+ * the developer-role turn, corrupting the instructions the model needs to call the plan tool.
+ */
+
+import { describe, it, before } from "node:test";
+import assert from "node:assert/strict";
+
+let crushMessages: typeof import("../../../open-sse/services/compression/engines/headroom/smartcrusher.ts").crushMessages;
+let collectCompactableArrays: typeof import("../../../open-sse/services/compression/engines/headroom/smartcrusher.ts").collectCompactableArrays;
+let headroomEngine: import("../../../open-sse/services/compression/engines/headroom/index.ts").headroomEngine;
+
+before(async () => {
+  const mod = await import("../../../open-sse/services/compression/engines/headroom/smartcrusher.ts");
+  crushMessages = mod.crushMessages;
+  collectCompactableArrays = mod.collectCompactableArrays;
+
+  const engineMod = await import("../../../open-sse/services/compression/engines/headroom/index.ts");
+  headroomEngine = engineMod.headroomEngine;
+});
+
+/** A homogeneous array big enough (>= default minRows=8) to trigger compaction. */
+function makePlanSchemaExample(): Record<string, unknown>[] {
+  return Array.from({ length: 10 }, (_, i) => ({
+    step: `step-${i + 1}`,
+    status: i === 0 ? "in_progress" : "pending",
+  }));
+}
+
+describe("headroom SmartCrusher — developer-role guard (9router#2132)", () => {
+  it("does NOT compact JSON arrays embedded in a developer-role message (crushMessages)", () => {
+    const json = JSON.stringify(makePlanSchemaExample());
+    const messages = [
+      {
+        role: "developer",
+        content: `Use the update_plan tool. Example plan:\n\`\`\`json\n${json}\n\`\`\``,
+      },
+      { role: "user", content: "Refactor the auth module." },
+    ];
+
+    const { messages: result, changed } = crushMessages(messages, 8);
+
+    assert.equal(changed, false, "developer-role content must not be touched");
+    assert.equal(result[0].content, messages[0].content);
+  });
+
+  it("still compacts the same payload when placed under role: system (control case)", () => {
+    // Sanity check: this proves the array itself WOULD be compactable — the guard, not the
+    // shape of the payload, is what must change.
+    const json = JSON.stringify(makePlanSchemaExample());
+    const messages = [{ role: "user", content: `\`\`\`json\n${json}\n\`\`\`` }];
+
+    const { changed } = crushMessages(messages, 8);
+    assert.equal(changed, true, "control case: user-role content of the same shape IS compacted");
+  });
+
+  it("collectCompactableArrays does not surface arrays from developer-role messages", () => {
+    const json = JSON.stringify(makePlanSchemaExample());
+    const messages = [
+      { role: "developer", content: `\`\`\`json\n${json}\n\`\`\`` },
+    ];
+    const found = collectCompactableArrays(messages, 8);
+    assert.equal(found.length, 0);
+  });
+
+  it("headroomEngine.apply leaves a Codex-CLI-shaped developer turn untouched end-to-end", () => {
+    const json = JSON.stringify(makePlanSchemaExample());
+    const body: Record<string, unknown> = {
+      model: "gpt-5-codex",
+      messages: [
+        {
+          role: "developer",
+          content: `Instructions with an embedded schema example:\n\`\`\`json\n${json}\n\`\`\``,
+        },
+        { role: "user", content: "Implement the feature." },
+      ],
+    };
+
+    const result = headroomEngine.apply(body);
+    assert.equal(result.compressed, false);
+    assert.deepEqual(result.body, body);
+  });
+});