diff --git a/src/tools/BashTool/sedEditParser.test.ts b/src/tools/BashTool/sedEditParser.test.ts index f9c8732648..95523d37d5 100644 --- a/src/tools/BashTool/sedEditParser.test.ts +++ b/src/tools/BashTool/sedEditParser.test.ts @@ -1,6 +1,26 @@ -import { expect, test } from 'bun:test' +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' -import { applySedSubstitution, type SedEditInfo } from './sedEditParser.js' +import { + applySedSubstitution, + parseSedEditCommand, + sedLocaleCountsCharacters, + type SedEditInfo, +} from './sedEditParser.js' + +// Simulation is only claimed in a UTF-8 locale (see sedLocaleCountsCharacters), +// and CI machines do not reliably set one. Pin it so these cases exercise the +// translation rather than the locale gate; the gate has its own tests below. +const savedLcAll = process.env.LC_ALL +beforeAll(() => { + process.env.LC_ALL = 'en_US.UTF-8' +}) +afterAll(() => { + if (savedLcAll === undefined) { + delete process.env.LC_ALL + } else { + process.env.LC_ALL = savedLcAll + } +}) function sedInfo(pattern: string, replacement: string, extendedRegex = false): SedEditInfo { return { @@ -21,13 +41,16 @@ test('BRE mode keeps unescaped plus literal', () => { expect(result).toBe('literal-plus and aaab') }) -test('BRE mode treats escaped plus as one-or-more', () => { - const result = applySedSubstitution( - 'abbb and a+b', - sedInfo('ab\\+', 'one-or-more'), - ) - - expect(result).toBe('one-or-more and a+b') +test('BRE mode declines the GNU-only escaped plus and question mark', () => { + // `\+` and `\?` are GNU extensions: BSD/macOS sed matches them as literal + // `+`/`?`, so one platform's operator is the other's literal and a single + // preview cannot be right for both. + expect( + parseSedEditCommand("sed -i '' 's/ab\\+/X/' example.txt"), + ).toBeNull() + expect( + parseSedEditCommand("sed -i '' 's/ab\\?c/X/' example.txt"), + ).toBeNull() }) test('BRE mode preserves escaped backslashes', () => { @@ -38,3 +61,471 @@ test('BRE mode preserves escaped backslashes', () => { expect(result).toBe('backslash-match foo/bar') }) + +test('BRE mode treats escaped braces as an interval quantifier', () => { + // `a\{2\}` is the BRE interval quantifier (exactly two a's). Before the fix + // braces were omitted from the metacharacter set, so it was emitted as the + // JS literal `a\{2\}` and matched nothing — the preview showed no change + // while real sed rewrote the file. + const result = applySedSubstitution('aa and a', sedInfo('a\\{2\\}', 'X')) + expect(result).toBe('X and a') +}) + +test('BRE mode treats bare braces as literals', () => { + // A bare `{2}` is literal in BRE, so it must not become a JS quantifier. + const result = applySedSubstitution('a{2} and aa', sedInfo('a{2}', 'X')) + expect(result).toBe('X and aa') +}) + +test('BRE mode supports escaped interval ranges', () => { + // `b\{1,3\}` matches one-to-three b's; on "bbbc" it consumes the three b's + // (the upper bound) and leaves the trailing "c". + const result = applySedSubstitution('bbbc', sedInfo('b\\{1,3\\}', 'X')) + expect(result).toBe('Xc') +}) + +test('BRE mode declines the GNU-only open lower-bound interval \\{,m\\}', () => { + // `\{,3\}` is a GNU extension: BSD/macOS sed rejects it and leaves the file + // untouched, and this parser explicitly supports macOS via its `-i ''` + // handling. Previewing a {0,3} result would show an edit on platforms where + // the real command fails, so no sed edit is claimed even without `g`. + expect( + parseSedEditCommand("sed -i '' 's/a\\{,3\\}/X/' example.txt"), + ).toBeNull() +}) + +test('BRE mode supports the open upper-bound interval \\{n,\\}', () => { + const result = applySedSubstitution('aaab', sedInfo('a\\{2,\\}', 'X')) + expect(result).toBe('Xb') +}) + +test('BRE mode declines malformed intervals rather than guessing', () => { + // GNU sed rejects both of these outright ("Invalid content of \{\}"), so the + // command aborts and the file is untouched. They are not literal braces, and + // there is no edit to render. + expect( + parseSedEditCommand("sed -i '' 's/a\\{\\}/X/g' example.txt"), + ).toBeNull() + expect( + parseSedEditCommand("sed -i '' 's/a\\{1,2,3\\}/X/g' example.txt"), + ).toBeNull() +}) + +test('BRE mode declines alternation rather than mis-selecting a branch', () => { + // POSIX regex picks the leftmost-longest alternative while JavaScript picks + // the first that matches: sed writes `\(a\|aa\)` on "aa" as "X", a JS + // preview as "Xa". Selection cannot be reproduced, so no edit is claimed. + expect( + parseSedEditCommand("sed -i '' 's/cat\\|dog/X/g' example.txt"), + ).toBeNull() + expect( + parseSedEditCommand("sed -i '' 's/\\(a\\|aa\\)\\{1\\}/X/' example.txt"), + ).toBeNull() +}) + +test('BRE mode keeps escaped groups with portable interval quantifiers', () => { + // `\(...\)` and `\{n\}` are POSIX BRE, supported identically by GNU and + // BSD sed, so groups repeated via an interval still simulate. + const result = applySedSubstitution('abab cd', sedInfo('\\(ab\\)\\{2\\}', 'X')) + expect(result).toBe('X cd') +}) + +test('BRE mode applies g across multiple interval quantifiers', () => { + const result = applySedSubstitution( + 'aabb aabb', + sedInfo('a\\{2\\}b\\{2\\}', 'X'), + ) + expect(result).toBe('X X') +}) + +test('ERE mode uses native brace intervals', () => { + // Under -E the braces are already the JS quantifier form and must pass + // through untouched. + const result = applySedSubstitution('aaab', sedInfo('a{2}', 'X', true)) + expect(result).toBe('Xab') +}) + +describe('declines to simulate what it cannot reproduce faithfully', () => { + const cmd = (expr: string) => `sed -i '' '${expr}' example.txt` + + test('rejects zero-minimum intervals under g', () => { + // sed and JS advance differently past an empty match, so a global + // zero-minimum quantifier genuinely diverges: GNU sed turns "aaaab" into + // "XXbX" for both of these, while a JS global replace yields "XXXbX". + // Rendering that as an approved file diff would show the user a change that + // is not what sed writes, so no sed edit is claimed at all. + expect(parseSedEditCommand(cmd('s/a\\{0,3\\}/X/g'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a\\{,3\\}/X/g'))).toBeNull() + // Same class, pre-dating interval support: `*` also matches empty. + expect(parseSedEditCommand(cmd('s/a*/X/g'))).toBeNull() + }) + + test('still simulates zero-minimum intervals without g', () => { + // Only one substitution happens, so the empty-match advance never matters. + expect(parseSedEditCommand(cmd('s/a\\{0,3\\}/X/'))).not.toBeNull() + }) + + test('rejects interval syntax inside a bracket expression', () => { + // `[\{,3\}]` is a bracket expression whose members are ordinary characters; + // GNU sed turns "0,{3}" into "0XXXX". A backslash is a literal member in + // POSIX brackets but an escape in a JS character class, so this cannot be + // mapped across and must not be parsed as an interval. + expect(parseSedEditCommand(cmd('s/[\\{,3\\}]/X/g'))).toBeNull() + }) + + test('still simulates ordinary bracket expressions', () => { + const info = parseSedEditCommand(cmd('s/[abc]/X/g')) + expect(info).not.toBeNull() + expect(applySedSubstitution('abcd', info!)).toBe('XXXd') + }) + + test('rejects a pattern whose translation is not a valid regex', () => { + // Previously `new RegExp` threw and the catch returned the original + // content, rendering an invalid pattern as a silent "no change" diff. + expect(parseSedEditCommand(cmd('s/a*\\{2\\}/X/g'))).toBeNull() + }) + + test('rejects bracket expressions with a leading ] member', () => { + // POSIX treats the first `]` as an ordinary member, so GNU sed rewrites + // "a]b" to "aXb"; JavaScript reads it as the class terminator and matches + // nothing, leaving the text untouched. + expect(parseSedEditCommand(cmd('s/[]]/X/g'))).toBeNull() + expect(parseSedEditCommand(cmd('s/[^]]/X/g'))).toBeNull() + expect( + parseSedEditCommand("sed -i '' -E 's/[]]/X/g' example.txt"), + ).toBeNull() + }) + + test('rejects an unterminated bracket expression', () => { + // GNU sed rejects `s/[/X/g` with an unterminated-address error and leaves + // the file untouched; rendering `[` as a literal would persist an edit the + // command cannot perform. + expect(parseSedEditCommand(cmd('s/[/X/g'))).toBeNull() + }) + + test('rejects POSIX character classes JavaScript cannot interpret', () => { + // `[[:digit:]]` is a digit class to sed but a plain character set to a JS + // regex, so the two produce different files ("1a2": sed XaX, JS unchanged). + expect(parseSedEditCommand(cmd('s/[[:digit:]]/X/g'))).toBeNull() + expect( + parseSedEditCommand("sed -i '' -E 's/[[:alpha:]]+/X/g' example.txt"), + ).toBeNull() + }) + + test('rejects ERE alternation for the same POSIX-selection reason', () => { + expect( + parseSedEditCommand("sed -i '' -E 's/(a|aa)/X/' example.txt"), + ).toBeNull() + }) + + test('rejects ERE (?...) group extensions GNU sed does not implement', () => { + // `sed -E` supports only plain capturing `(...)`. `(?` opens JS-only syntax + // -- non-capturing, lookaround, named groups -- that JavaScript compiles + // but GNU sed rejects, so a preview would edit a file the real command + // leaves untouched. Every `(?` form must decline. + for (const pattern of [ + '(?:a)b', + '(?=a)', + '(?!a)b', + '(?<=a)b', + '(?a)', + ]) { + expect( + parseSedEditCommand(`sed -i '' -E 's/${pattern}/X/' example.txt`), + ).toBeNull() + } + // A plain capturing group is faithful and still parses. + expect( + parseSedEditCommand("sed -i '' -E 's/(a)b/X/' example.txt"), + ).not.toBeNull() + }) + + test('rejects numeric occurrence flags it does not model', () => { + // `2` selects the second match on each line, but the simulator always + // rewrites the first: sed turns "aaaa" into "aaX", a preview into "Xaa". + expect(parseSedEditCommand(cmd('s/a\\{2\\}/X/2'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a/X/9'))).toBeNull() + // `p` prints and `m`/`M` redefine ^ and $ inside the pattern space. + expect(parseSedEditCommand(cmd('s/a/X/p'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a/X/m'))).toBeNull() + // A repeated `g` is not a global rewrite: GNU sed rejects it outright, so a + // preview of a successful edit would diverge from the failing command. + expect(parseSedEditCommand(cmd('s/a/X/gg'))).toBeNull() + // A single `g` still parses. + expect(parseSedEditCommand(cmd('s/a/X/g'))).not.toBeNull() + }) + + test('rejects replacements carrying sed-specific syntax', () => { + // `\1` is a backreference to sed but two literal characters to the + // simulator: sed rewrites "aa" as "a", the preview as "\1". + expect(parseSedEditCommand(cmd('s/\\(a\\)\\{2\\}/\\1/'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a/\\n/'))).toBeNull() + // A plain literal replacement still simulates. + expect(parseSedEditCommand(cmd('s/a/X/'))).not.toBeNull() + }) + + test('rejects escapes whose sed meaning is not the JavaScript meaning', () => { + // GNU sed reads `\<`/`\>` as word boundaries; JS reads literal angle + // brackets, so the preview shows no change while sed rewrites the file. + expect(parseSedEditCommand(cmd('s/\\/X/g'))).toBeNull() + // The converse: `\d` is a digit class in JS but a literal `d` in BRE. + expect(parseSedEditCommand(cmd('s/\\d/X/g'))).toBeNull() + expect(parseSedEditCommand(cmd('s/\\w/X/g'))).toBeNull() + }) + + test('rejects ^ and $ where BRE treats them as literals', () => { + // Bare `^`/`$` only anchor at a BRE boundary; elsewhere sed matches them + // literally while JS always reads an anchor. + expect(parseSedEditCommand(cmd('s/a^b/X/'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a$b/X/'))).toBeNull() + // Genuine anchors still simulate. + expect(parseSedEditCommand(cmd('s/^a/X/'))).not.toBeNull() + expect(parseSedEditCommand(cmd('s/a$/X/'))).not.toBeNull() + }) + + test('validates interval bodies on the ERE path too', () => { + // JS reads `a{,3}` as literal braces; GNU sed -E applies its extension and + // rewrites "aaaab" to "XXbX", and BSD has no portable behavior. + expect( + parseSedEditCommand("sed -i '' -E 's/a{,3}/X/g' example.txt"), + ).toBeNull() + expect( + parseSedEditCommand("sed -i '' -E 's/a{1,2,3}/X/g' example.txt"), + ).toBeNull() + }) + + test('counts characters like sed, not UTF-16 code units', () => { + // Verified against GNU sed 4.10: `s/.\{2\}/X/` on the emoji + "a" writes a + // bare "X". Without a unicode-aware matcher the quantifier consumes only + // the emoji's surrogate pair and leaves the "a". + const info = parseSedEditCommand(cmd('s/.\\{2\\}/X/')) + expect(info).not.toBeNull() + expect(applySedSubstitution('\u{1F600}a', info!)).toBe('X') + }) + + test('substitutes every character on a line, matching sed under LF content', () => { + // The permission path (SedEditPermissionRequest) normalizes CRLF to LF + // before it ever calls the simulator, so the preview it approves only ever + // sees `\n`-terminated lines. On that normalized content `s/./X/g` rewrites + // each character exactly as GNU sed does. Raw-CR fidelity is deliberately + // out of scope: the simulator never receives a `\r`, so this asserts the + // behavior the approval gate actually exercises rather than a raw-byte case + // production cannot reach. + const info = parseSedEditCommand(cmd('s/./X/g')) + expect(info).not.toBeNull() + expect(applySedSubstitution('ab', info!)).toBe('XX') + }) + + test('rejects a standalone empty pattern', () => { + // `s//X/` has no previous regular expression to reuse, so sed errors and + // leaves the file untouched; an empty JS regex would prefix every line. + expect(parseSedEditCommand(cmd('s//X/'))).toBeNull() + }) + + test('applies the substitution once per line like sed, not once per file', () => { + // sed substitutes the first match on EVERY line even without `g`. + const info = parseSedEditCommand(cmd('s/a\\{2\\}/X/')) + expect(info).not.toBeNull() + expect(applySedSubstitution('aa\naa\n', info!)).toBe('X\nX\n') + // With `g`, all matches on every line. + const g = parseSedEditCommand(cmd('s/a\\{2\\}/X/g')) + expect(applySedSubstitution('aaaa\naa b aa\n', g!)).toBe('XX\nX b X\n') + // A trailing newline is preserved and never treated as an extra empty line. + expect(applySedSubstitution('aa', info!)).toBe('X') + }) + + test('leaves an empty file empty like sed', () => { + // An empty file has no lines: sed never runs the substitution, so even an + // anchored pattern that matches the empty string writes nothing. + const info = parseSedEditCommand(cmd('s/^/X/')) + expect(info).not.toBeNull() + expect(applySedSubstitution('', info!)).toBe('') + }) +}) + +describe('replacement text is written literally, as sed writes it', () => { + const cmd = (expr: string) => `sed -i '' '${expr}' example.txt` + + test('writes dollar tokens literally instead of expanding them', () => { + // `$` is an ordinary character in a sed replacement but a substitution + // token to String.replace. GNU sed 4.10 writes the two characters `$1` + // here; before the fix the preview expanded it to the matched text, so an + // approved diff differed from what the command performs. + const one = parseSedEditCommand(cmd('s/\\(a\\)/$1/')) + expect(one).not.toBeNull() + expect(applySedSubstitution('a', one!)).toBe('$1') + + const dollars = parseSedEditCommand(cmd('s/a/$$/')) + expect(dollars).not.toBeNull() + expect(applySedSubstitution('a', dollars!)).toBe('$$') + + // $` and $' expand to the text before/after the match in JS. + const before = parseSedEditCommand(cmd('s/b/$`/')) + expect(before).not.toBeNull() + expect(applySedSubstitution('abc', before!)).toBe('a$`c') + + // `$'` cannot go through the single-quoted fixture above, so drive it + // directly: in JS it expands to the text following the match. + expect(applySedSubstitution('abc', sedInfo('b', "$'"))).toBe("a$'c") + + // `&` is sed's own whole-match token and is declined outright, so a + // replacement spelling JS's `$&` never reaches the simulator. + expect(parseSedEditCommand(cmd('s/b/$&/'))).toBeNull() + }) + + test('a plain replacement is unaffected', () => { + const info = parseSedEditCommand(cmd('s/a/USD 5/g')) + expect(info).not.toBeNull() + expect(applySedSubstitution('a b a', info!)).toBe('USD 5 b USD 5') + }) +}) + +describe('ERE patterns are screened for JS-only escapes', () => { + const ere = (expr: string) => `sed -i '' -E '${expr}' example.txt` + + test('declines escapes that mean a character class only in JS', () => { + // GNU sed reads `\d` as a literal `d`, so `-E 's/\d/X/g'` on "1d2" writes + // "1X2"; a JS regex reads a digit class and would preview "XdX". The same + // divergence applies to the other JS-only escapes, and several are not + // valid sed at all. + for (const expr of [ + 's/\\d/X/g', + 's/\\w/X/g', + 's/\\s/X/g', + 's/\\S/X/g', + 's/\\b/X/g', + 's/\\u{41}/X/g', + 's/\\p{L}/X/g', + 's/\\x41/X/g', + ]) { + expect(parseSedEditCommand(ere(expr))).toBeNull() + } + }) + + test('still accepts escapes that are the same literal in both dialects', () => { + const dot = parseSedEditCommand(ere('s/a\\.b/X/g')) + expect(dot).not.toBeNull() + expect(applySedSubstitution('a.b axb', dot!)).toBe('X axb') + + const plus = parseSedEditCommand(ere('s/a\\+/X/g')) + expect(plus).not.toBeNull() + expect(applySedSubstitution('a+ aa', plus!)).toBe('X aa') + }) + + test('declines a trailing lone backslash', () => { + expect(parseSedEditCommand(ere('s/a\\/X/g'))).toBeNull() + }) +}) + +describe('case-insensitive matching is declined', () => { + test('declines the i and I flags rather than folding by Unicode rules', () => { + // The emitted regex needs `u` for the quantifier fix, and `u` + `i` is + // ECMAScript Unicode case folding, not sed's locale matching: `s/k/X/I` + // would rewrite a Kelvin sign that GNU sed under C.UTF-8 leaves alone. + for (const expr of ["s/k/X/I", "s/k/X/i", "s/k/X/gI", "s/k/X/gi"]) { + expect( + parseSedEditCommand(`sed -i '' '${expr}' example.txt`), + ).toBeNull() + } + // The g flag on its own is still simulated. + expect( + parseSedEditCommand("sed -i '' 's/k/X/g' example.txt"), + ).not.toBeNull() + }) +}) + +describe('locale-sensitive matching is declined outside UTF-8', () => { + test('declines when sed would count bytes rather than characters', () => { + // The emitted regex carries `u` so a quantifier counts characters. That is + // sed's behavior in a UTF-8 locale only: under LC_ALL=C, `s/.\{2\}/X/` on + // "😀a" consumes two bytes of the emoji and leaves the rest in the file. + for (const locale of ['C', 'POSIX', 'en_US.ISO-8859-1']) { + expect(sedLocaleCountsCharacters({ LC_ALL: locale })).toBe(false) + } + // POSIX resolves an unset or empty locale to C. + expect(sedLocaleCountsCharacters({})).toBe(false) + expect(sedLocaleCountsCharacters({ LC_ALL: '' , LANG: ''})).toBe(false) + }) + + test('accepts the UTF-8 spellings that actually occur', () => { + expect(sedLocaleCountsCharacters({ LC_ALL: 'en_US.UTF-8' })).toBe(true) + expect(sedLocaleCountsCharacters({ LANG: 'de_DE.utf8' })).toBe(true) + // macOS sets the bare codeset for LC_CTYPE. + expect(sedLocaleCountsCharacters({ LC_CTYPE: 'UTF-8' })).toBe(true) + expect(sedLocaleCountsCharacters({ LANG: 'fr_FR.UTF-8@euro' })).toBe(true) + }) + + test('honours the POSIX precedence order', () => { + // LC_ALL overrides everything; LC_CTYPE overrides LANG. + expect( + sedLocaleCountsCharacters({ LC_ALL: 'C', LC_CTYPE: 'en_US.UTF-8' }), + ).toBe(false) + expect(sedLocaleCountsCharacters({ LC_CTYPE: 'C', LANG: 'en_US.UTF-8' })).toBe( + false, + ) + expect(sedLocaleCountsCharacters({ LANG: 'en_US.UTF-8' })).toBe(true) + }) + + test('claims no sed edit at all under a byte locale', () => { + const saved = process.env.LC_ALL + process.env.LC_ALL = 'C' + try { + expect( + parseSedEditCommand("sed -i '' 's/a\\{2\\}/X/g' example.txt"), + ).toBeNull() + } finally { + process.env.LC_ALL = saved + } + }) +}) + +describe('patterns that can stall the approval UI are declined', () => { + const cmd = (expr: string) => `sed -i '' '${expr}' example.txt` + + test('declines a quantified group that already contains a quantifier', () => { + // `(a{1,}){1,}b` backtracks exponentially on a run of `a`s with no `b`, and + // applySedSubstitution runs synchronously while the permission request is + // rendered -- so this freezes the approval UI before the user can decide. + expect(parseSedEditCommand(cmd('s/\\(a\\{1,\\}\\)\\{1,\\}b/X/'))).toBeNull() + expect(parseSedEditCommand(cmd('s/\\(a*\\)\\{2\\}b/X/'))).toBeNull() + expect( + parseSedEditCommand("sed -i '' -E 's/(a+)+b/X/' example.txt"), + ).toBeNull() + }) + + test('still accepts a quantified group with no inner quantifier', () => { + const info = parseSedEditCommand(cmd('s/\\(ab\\)\\{2\\}/X/g')) + expect(info).not.toBeNull() + expect(applySedSubstitution('abab ab', info!)).toBe('X ab') + }) + + test('still accepts an inner quantifier when the group is not quantified', () => { + const info = parseSedEditCommand(cmd('s/\\(a\\{2\\}\\)b/X/g')) + expect(info).not.toBeNull() + expect(applySedSubstitution('aab ab', info!)).toBe('X ab') + }) +}) + +describe('literal replacement escapes keep their preview', () => { + const cmd = (expr: string) => `sed -i '' '${expr}' example.txt` + + test('accepts the two escapes the translation already handles', () => { + // Rejecting these sent ordinary commands back to the generic bash approval + // even though applySedSubstitution translates both faithfully. + const slash = parseSedEditCommand(cmd('s/foo/path\\/to/')) + expect(slash).not.toBeNull() + expect(applySedSubstitution('foo\n', slash!)).toBe('path/to\n') + + const amp = parseSedEditCommand(cmd('s/foo/a\\&b/')) + expect(amp).not.toBeNull() + expect(applySedSubstitution('foo\n', amp!)).toBe('a&b\n') + }) + + test('still declines backreferences, case folding and a bare &', () => { + expect(parseSedEditCommand(cmd('s/\\(a\\)/\\1/'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a/\\U&/'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a/\\n/'))).toBeNull() + expect(parseSedEditCommand(cmd('s/a/x&y/'))).toBeNull() + }) +}) diff --git a/src/tools/BashTool/sedEditParser.ts b/src/tools/BashTool/sedEditParser.ts index 02be50a742..75fed8bc11 100644 --- a/src/tools/BashTool/sedEditParser.ts +++ b/src/tools/BashTool/sedEditParser.ts @@ -21,30 +21,154 @@ export type SedEditInfo = { extendedRegex: boolean } -function convertBrePatternToJs(pattern: string): string { +// Escaped forms that are portable POSIX BRE operators: `\(` `\)` (grouping). +// The escaped forms of the characters below are NOT portable — `\+` `\?` `\|` +// are GNU extensions that BSD/macOS sed matches literally — so they decline +// instead (see the branch handling them). +const BRE_PORTABLE_OPERATOR_ESCAPES = '()' + +// JS regex metacharacters that are literal when bare in BRE (the reverse of +// JS), so they must be escaped in the translation. Braces are handled +// separately because they only form an operator when escaped around a valid +// count. +const BRE_BARE_LITERAL_METACHARS = '+?|()' + +/** + * Translate the body of a BRE interval `\{...\}` to its JS quantifier form, or + * null when it cannot be previewed faithfully. Only the POSIX-portable forms + * `n`, `n,` and `n,m` are accepted: the GNU-only `,m` extension is rejected by + * BSD/macOS sed (which this parser explicitly supports via its `-i ''` + * handling), so previewing it would show an edit on platforms where the real + * command fails and changes nothing. Anything else — empty, extra commas, + * non-numeric — is rejected by sed itself ("Invalid content of \{\}"), which + * aborts the command and leaves the file untouched. + */ +function breIntervalBodyToJs(body: string): string | null { + if (/^[0-9]+(,[0-9]*)?$/.test(body)) { + return `{${body}}` + } + return null +} + +/** + * Find the index of the `]` closing the BRE bracket expression that starts at + * `open`, or -1 when it is never closed. A `]` in the first position (after an + * optional leading `^`) is an ordinary member, not the terminator. + */ +function findBracketEnd(pattern: string, open: number): number { + let i = open + 1 + if (pattern[i] === '^') i++ + if (pattern[i] === ']') i++ + for (; i < pattern.length; i++) { + if (pattern[i] === ']') return i + } + return -1 +} + +/** + * True when a bracket expression opens with a `]` member (`[]]`, `[^]]`). + * + * POSIX treats that first `]` as an ordinary member, so `s/[]]/X/g` rewrites + * "a]b" to "aXb". JavaScript reads it as the class terminator instead, leaving + * the text untouched, so the two dialects disagree and the pattern declines. + */ +function bracketHasLeadingCloseMember(pattern: string, open: number): boolean { + let i = open + 1 + if (pattern[i] === '^') i++ + return pattern[i] === ']' +} + +/** + * Convert a BRE pattern to its JS equivalent, or null when it cannot be + * translated faithfully and the caller must decline to simulate the edit. + */ +function convertBrePatternToJs(pattern: string): string | null { + if (!breAnchorsAreUnambiguous(pattern)) return null let result = '' for (let i = 0; i < pattern.length; i++) { const char = pattern[i]! + if (char === '[') { + // Inside a bracket expression `\{`/`\}` are ordinary members rather than + // an interval, so the interval scan below must not see them. Several + // bracket constructs cannot be previewed faithfully and must decline: + // - an unterminated `[` is an error in sed (file untouched), not a + // literal; + // - a backslash is a literal member in a POSIX bracket expression but an + // escape in a JS character class; + // - `[:class:]`, `[=equiv=]` and `[.collate.]` constructs mean nothing + // to JS, which would read them as a plain set of characters. + const end = findBracketEnd(pattern, i) + if (end === -1) return null + if (bracketHasLeadingCloseMember(pattern, i)) return null + const body = pattern.slice(i, end + 1) + if (body.includes('\\') || body.slice(1).includes('[')) return null + // Remaining members are literal in both dialects; carry the body as-is. + result += body + i = end + continue + } + if (char === '\\') { const next = pattern[i + 1] if (next === undefined) { result += '\\\\' continue } + if (next === '{') { + // `\{...\}` is the BRE interval quantifier. An unterminated or + // illegal-bodied interval is an error in sed rather than a literal, so + // decline instead of emitting braces that would match something else. + const close = pattern.indexOf('\\}', i + 2) + if (close === -1) return null + const js = breIntervalBodyToJs(pattern.slice(i + 2, close)) + if (js === null) return null + result += js + i = close + 1 // consume through the closing `\}` + continue + } + if (next === '}') { + // A stray escaped closing brace with no matching interval open: literal. + result += '\\}' + i++ + continue + } + if (next === '|') { + // GNU alternation extension. Doubly unfaithful: BSD/macOS sed matches + // `\|` as a literal pipe, and even on GNU, POSIX regex selects the + // leftmost-longest alternative while JavaScript takes the first one + // that matches (`\(a\|aa\)` previews `aa` as `Xa` where GNU sed writes + // `X`). Decline. + return null + } + if (next === '+' || next === '?') { + // GNU extensions: BSD/macOS sed matches `\+` and `\?` as literal + // `+`/`?`, so one platform's operator is the other's literal and a + // single preview cannot be right for both. Decline. + return null + } if (next === '\\') { result += '\\\\' - } else if ('+?|()'.includes(next)) { + } else if (BRE_PORTABLE_OPERATOR_ESCAPES.includes(next)) { result += next - } else { + } else if (BRE_PORTABLE_LITERAL_ESCAPES.includes(next)) { result += `\\${next}` + } else { + // Not a portable escape: its sed meaning is not the JS meaning. + return null } i++ continue } - if ('+?|()'.includes(char)) { + if (BRE_BARE_LITERAL_METACHARS.includes(char)) { + result += `\\${char}` + continue + } + + // Bare braces are literal in BRE (the reverse of JS), so escape them. + if (char === '{' || char === '}') { result += `\\${char}` continue } @@ -55,6 +179,85 @@ function convertBrePatternToJs(pattern: string): string { return result } +/** + * Only replacements that are pure literal text can be previewed. + * + * sed's replacement syntax has its own meanings that this module does not + * translate: `\1`-`\9` are backreferences, `&` is the whole match, `\n`/`\t` + * are escapes, and `\U`/`\L` (GNU) case-fold. The simulator passes the + * replacement to String.replace as-is, so `s/\(a\)\{2\}/\1/` writes the two + * literal characters `\1` where sed writes `a`. + * + * `\/` and `\&` are the exceptions: both denote a literal character, and the + * translation below already handles them, so rejecting them would send + * ordinary commands like `s/foo/path\/to/` back to the generic bash approval + * for no reason. Every other backslash escape, and a bare `&` (sed's + * whole-match token), declines. + */ +function isFaithfullyLiteralReplacement(replacement: string): boolean { + for (let i = 0; i < replacement.length; i++) { + const char = replacement[i]! + if (char === '\\') { + const escaped = replacement[i + 1] + if (escaped !== '/' && escaped !== '&') return false + i++ + continue + } + if (char === '&') return false + } + return true +} + +/** + * `$` is an ordinary character in a sed replacement but a substitution token in + * a JS one: `$1`, `$$`, `` $` `` and `$'` all expand. `s/\(a\)/$1/` is a + * literal replacement by the rule above, yet String.replace would write the + * matched text where sed writes the two characters `$1`. Doubling each `$` is + * the documented way to emit one literally. + */ +function escapeJsReplacement(replacement: string): string { + return replacement.replaceAll('$', '$$$$') +} + +/** + * Escapes that mean the same thing in a POSIX BRE and in the emitted JS regex. + * + * Everything outside this set declines. GNU sed reads `\<`/`\>` as word + * boundaries while JS reads them as literal angle brackets, so `s/\/X/g` + * previews no change while sed rewrites the file; `\d` has the converse problem + * (a digit class in JS, a literal `d` in BRE). Carrying unhandled escapes + * through means the emitted regex is not the pattern sed was given. + */ +const BRE_PORTABLE_LITERAL_ESCAPES = '.*[]^$/' + +/** + * `^` and `$` are only anchors at the start/end of a BRE or of a `\( \)` + * subexpression; anywhere else sed matches them literally, while JS always + * reads them as anchors. `s/a^b/X/` would therefore be accepted and preview no + * edit while sed rewrites the literal text. Rather than model every + * subexpression boundary, decline when either character appears outside the + * positions where both dialects agree it anchors. + */ +function breAnchorsAreUnambiguous(pattern: string): boolean { + for (let i = 0; i < pattern.length; i++) { + const char = pattern[i]! + if (char === '\\') { + i++ + continue + } + if (char === '[') { + const end = findBracketEnd(pattern, i) + if (end === -1) return false + i = end + continue + } + // A leading `^` and a trailing `$` are anchors in both dialects. + if (char === '^' && i !== 0) return false + if (char === '$' && i !== pattern.length - 1) return false + } + return true +} + /** * Check if a command is a sed in-place edit command * Returns true only for simple sed -i 's/pattern/replacement/flags' file commands @@ -244,12 +447,44 @@ export function parseSedEditCommand(command: string): SedEditInfo | null { return null } - // Validate flags - only allow safe substitution flags - const validFlags = /^[gpimIM1-9]*$/ + // Only the flags this module actually models. `1`-`9` select the Nth match on + // each line and `p` prints, neither of which the simulator implements — it + // always rewrites the first match (or every match under `g`), so + // `s/a\{2\}/X/2` on "aaaa" would be previewed as "Xaa" where sed writes + // "aaX". `m`/`M` redefine `^`/`$` per line inside the pattern space. Decline + // all of them rather than approve a write that differs from the command. + // + // `i`/`I` are declined for a subtler reason: the emitted regex needs `u` for + // the quantifier fix, and `u` + `i` selects ECMAScript Unicode case folding + // rather than sed's locale-based matching. `s/k/X/I` would then rewrite a + // Kelvin sign, which GNU sed under C.UTF-8 leaves alone. Until that folding + // can be modeled, an approved preview would not be the edit sed performs. + // At most one `g`: GNU sed rejects a repeated flag ("multiple `g' options to + // `s' command"), so a `gg` that this module previewed as a successful global + // rewrite would diverge from the command sed actually refuses to run. + const validFlags = /^g?$/ if (!validFlags.test(flags)) { return null } + // The emitted regex counts characters, which is only what sed does in a + // UTF-8 locale; under a byte locale it counts bytes and writes a different + // file. + if (!sedLocaleCountsCharacters()) { + return null + } + + // Only claim this is a renderable sed edit if we can reproduce it faithfully. + // Declining falls back to ordinary bash rendering, which is far better than + // showing the user a diff that does not match what sed will write. + if (!canSimulateFaithfully(pattern, flags, extendedRegex)) { + return null + } + + if (!isFaithfullyLiteralReplacement(replacement)) { + return null + } + return { filePath, pattern, @@ -260,42 +495,249 @@ export function parseSedEditCommand(command: string): SedEditInfo | null { } /** - * Apply a sed substitution to file content - * Returns the new content after applying the substitution + * Escapes that denote the same literal character in a POSIX ERE and in the + * emitted JS regex: the ERE metacharacters, plus the delimiter and a literal + * backslash. + * + * ERE patterns are handed to JS verbatim, so an escape outside this set is not + * the pattern sed was given. `\d` is the clearest case -- GNU sed reads it as a + * literal `d`, JS as a digit class, so `sed -E 's/\d/X/g'` on "1d2" previews + * "XdX" while sed writes "1X2". `\w`, `\s`, `\b` and `\u{...}` diverge the same + * way, and several are not valid sed at all. */ -export function applySedSubstitution( - content: string, - sedInfo: SedEditInfo, -): string { - // Convert sed pattern to JavaScript regex - let regexFlags = '' +const ERE_PORTABLE_LITERAL_ESCAPES = '.*[]^$/+?(){}|\\' - // Handle global flag - if (sedInfo.flags.includes('g')) { - regexFlags += 'g' +/** + * Reject ERE constructs whose behavior in a JS regex is not the POSIX behavior. + * The BRE converter declines these itself; ERE patterns carry over verbatim, so + * they need the same screening: + * - alternation: POSIX picks the leftmost-longest alternative, JS the first + * that matches; + * - bracket expressions holding a backslash (literal member in POSIX, escape + * in JS), a POSIX `[:class:]`-style construct, or no terminator at all + * (an error in sed, not a literal); + * - escapes outside the set that is literal in both dialects (see + * ERE_PORTABLE_LITERAL_ESCAPES). + */ +function ereHasUnfaithfulConstructs(pattern: string): boolean { + for (let i = 0; i < pattern.length; i++) { + const char = pattern[i]! + if (char === '\\') { + const escaped = pattern[i + 1] + if (escaped === undefined) return true + if (!ERE_PORTABLE_LITERAL_ESCAPES.includes(escaped)) return true + i++ + continue + } + if (char === '|') return true + if (char === '^' && i !== 0) return true + if (char === '$' && i !== pattern.length - 1) return true + if (char === '(' && pattern[i + 1] === '?') { + // POSIX/GNU `sed -E` supports only plain capturing groups `(...)`. A `(?` + // opens JavaScript-only syntax -- non-capturing `(?:)`, lookaround + // `(?=)`/`(?!)`/`(?<=)`/`(?)` groups -- which JS + // compiles but GNU sed rejects ("Invalid preceding regular expression"). + // The simulator would render a concrete edit for a command sed refuses to + // run, so decline. + return true + } + if (char === '{') { + // JS reads an illegal interval body as literal braces, but GNU sed either + // errors or applies its own extension: `sed -E 's/a{,3}/X/g'` rewrites + // "aaaab" to "XXbX" while JS matches the literal text "a{,3}". BSD has no + // portable behavior here either, so anything outside the POSIX forms + // declines. + const close = pattern.indexOf('}', i + 1) + if (close === -1) return true + if (breIntervalBodyToJs(pattern.slice(i + 1, close)) === null) return true + i = close + continue + } + if (char === '[') { + const end = findBracketEnd(pattern, i) + if (end === -1) return true + if (bracketHasLeadingCloseMember(pattern, i)) return true + const body = pattern.slice(i + 1, end) + if (body.includes('\\') || body.includes('[')) return true + i = end + } } + return false +} - // Handle case-insensitive flag (i or I in sed) - if (sedInfo.flags.includes('i') || sedInfo.flags.includes('I')) { - regexFlags += 'i' +/** + * Convert the sed pattern to the JS regex source this module would run, or null + * if it cannot be translated faithfully. + */ +function toJsPatternSource( + pattern: string, + extendedRegex: boolean, +): string | null { + const unescaped = pattern.replace(/\\\//g, '/') + if (extendedRegex) { + return ereHasUnfaithfulConstructs(unescaped) ? null : unescaped } + return convertBrePatternToJs(unescaped) +} - // Handle multiline flag (m or M in sed) - if (sedInfo.flags.includes('m') || sedInfo.flags.includes('M')) { - regexFlags += 'm' +/** + * Whether the simulated substitution is guaranteed to match what sed does. + * + * Declined here: + * - the pattern does not translate — GNU-only interval forms, alternation, + * untranslatable or unterminated bracket expressions (see + * convertBrePatternToJs / ereHasUnfaithfulConstructs) — or the translation + * is not a valid JS regex (previously this threw and was swallowed into a + * "no change" preview); + * - the pattern can match the empty string under `g`. sed and JS advance + * differently after an empty match, so the results genuinely differ: + * `s/a*​/X/g` on "aaaab" is "XbX" in sed but "XXbX" in JS, and + * `s/a\{0,3\}/X/g` is "XXbX" in sed but "XXXbX" in JS. + */ +function canSimulateFaithfully( + pattern: string, + flags: string, + extendedRegex: boolean, +): boolean { + // A standalone `s//X/` has no previous regular expression to reuse, so sed + // errors and leaves the file untouched. JS would compile an empty regex that + // matches at every position and prefix every line. + if (pattern === '') return false + + const jsPattern = toJsPatternSource(pattern, extendedRegex) + if (jsPattern === null) return false + + let regex: RegExp + try { + // Same flags the simulator will use, so a source that only compiles without + // them cannot slip through the gate. + regex = new RegExp(jsPattern, jsRegexFlags(flags)) + } catch { + return false } - // Convert sed pattern to JavaScript regex pattern - let jsPattern = sedInfo.pattern - // Unescape \/ to / - .replace(/\\\//g, '/') + if (flags.includes('g') && regex.test('')) return false + + if (hasNestedQuantifier(jsPattern)) return false + + return true +} - // In BRE mode (no -E flag), metacharacters have opposite escaping: - // BRE: \+ means "one or more", + is literal - // ERE/JS: + means "one or more", \+ is literal - // We need to convert BRE escaping to ERE for JavaScript regex - if (!sedInfo.extendedRegex) { - jsPattern = convertBrePatternToJs(jsPattern) +/** + * Whether a quantifier is applied to a group that already contains one. + * + * Interval support means `\(a\{1,\}\)\{1,\}b` translates to `(a{1,}){1,}b` and + * otherwise passes this gate. On a run of `a`s with no `b`, matching that + * backtracks exponentially -- and applySedSubstitution runs synchronously while + * the permission request is being rendered, so a command plus a repository file + * can stall the approval UI before the user gets to decide. There is no useful + * preview to salvage here, so decline. + */ +function hasNestedQuantifier(jsSource: string): boolean { + const isQuantifierStart = (char: string | undefined): boolean => + char === '*' || char === '+' || char === '?' || char === '{' + + for (let i = 0; i < jsSource.length; i++) { + const char = jsSource[i]! + if (char === '\\') { + i++ + continue + } + if (char !== ')' || !isQuantifierStart(jsSource[i + 1])) continue + + // Walk back to this group's opening paren, then look for a quantifier + // inside it. Escaped parens are literals and do not open or close a group. + let depth = 0 + for (let j = i; j >= 0; j--) { + if (j > 0 && jsSource[j - 1] === '\\') continue + const inner = jsSource[j]! + if (inner === ')') depth++ + else if (inner === '(') { + depth-- + if (depth === 0) { + const body = jsSource.slice(j + 1, i) + for (let k = 0; k < body.length; k++) { + if (body[k] === '\\') { + k++ + continue + } + if (isQuantifierStart(body[k])) return true + } + break + } + } + } + } + return false +} + +/** + * The JS flags that reproduce sed's matching for the accepted flag subset. + * + * `u` is not optional: in the UTF-8 locales sed runs in, a quantifier counts + * characters, while a non-unicode JS regex counts UTF-16 code units — without + * it `s/.\{2\}/X/` turns "\u{1F600}a" into "Xa" (consuming only the emoji's + * surrogate pair) where sed writes "X". + * + * `s` makes `.` match every character in the line. sed's pattern space holds a + * lone `\r` on CRLF input and `.` matches it, but a JS `.` excludes carriage + * returns, so `sed 's/./X/g'` on "a\r\n" writes "XX" where an unflagged + * simulation writes "X\r". + * + * There is deliberately no `i` here: `u` + `i` is Unicode case folding, not + * sed's locale matching, so `i`/`I` are declined at the flag gate instead. + */ +function jsRegexFlags(sedFlags: string): string { + let flags = 'us' + if (sedFlags.includes('g')) flags += 'g' + return flags +} + +/** + * Whether the locale sed will inherit makes character-wise matching correct. + * + * The emitted regex always carries `u`, which is required for a quantifier to + * count characters the way sed does in a UTF-8 locale. But the command inherits + * the process locale, and in a byte locale (`LC_ALL=C`) sed counts bytes + * instead: `s/.\{2\}/X/` on "😀a" consumes two bytes of the emoji and leaves + * the rest of it in the file, where the simulation consumes the whole + * character. Since an approved preview is written directly, that is a different + * file. + * + * POSIX resolves the locale as LC_ALL, then LC_CTYPE, then LANG, and an unset + * or empty locale means the C locale -- so require an explicit UTF-8 codeset + * rather than assuming one. Declining costs only the specialized diff; the + * command still renders as an ordinary bash approval. + * + * Exported for testing. + */ +export function sedLocaleCountsCharacters( + env: Record = process.env, +): boolean { + const locale = env.LC_ALL || env.LC_CTYPE || env.LANG || '' + // Either a codeset suffix ("en_US.UTF-8", "de_DE.utf8@euro") or the bare + // codeset, which is what macOS sets for LC_CTYPE. + return /(^|[.@])utf-?8($|@)/i.test(locale) +} + +/** + * Apply a sed substitution to file content + * Returns the new content after applying the substitution + */ +export function applySedSubstitution( + content: string, + sedInfo: SedEditInfo, +): string { + const regexFlags = jsRegexFlags(sedInfo.flags) + + // Convert sed pattern to JavaScript regex pattern. In BRE mode (no -E flag) + // metacharacters have opposite escaping: BRE `\+` means "one or more" and `+` + // is literal, the reverse of ERE/JS. + const jsPattern = toJsPatternSource(sedInfo.pattern, sedInfo.extendedRegex) + if (jsPattern === null) { + // Not translatable. parseSedEditCommand rejects these up front, so this is + // only reachable for a hand-built SedEditInfo; leave the content untouched. + return content } // Unescape sed-specific escapes in replacement @@ -303,7 +745,7 @@ export function applySedSubstitution( // Use a unique placeholder with random salt to prevent injection attacks const salt = randomBytes(8).toString('hex') const ESCAPED_AMP_PLACEHOLDER = `___ESCAPED_AMPERSAND_${salt}___` - const jsReplacement = sedInfo.replacement + const jsReplacement = escapeJsReplacement(sedInfo.replacement) // Unescape \/ to / .replace(/\\\//g, '/') // First escape \& to a placeholder @@ -313,11 +755,32 @@ export function applySedSubstitution( // Convert placeholder back to literal & .replace(new RegExp(ESCAPED_AMP_PLACEHOLDER, 'g'), '&') + let regex: RegExp try { - const regex = new RegExp(jsPattern, regexFlags) - return content.replace(regex, jsReplacement) + regex = new RegExp(jsPattern, regexFlags) } catch { // If regex is invalid, return original content return content } + + // An empty file has no lines, so sed never runs the substitution and the + // output stays empty; splitting '' would fabricate one empty line and let + // anchored patterns like s/^/X/ preview an edit sed does not make. + if (content.length === 0) { + return content + } + + // sed applies s/// to each line of the pattern space independently: without + // `g` it substitutes the first match on EVERY line, not the first match in + // the file. A single whole-buffer replace previewed `sed 's/a\{2\}/X/'` on + // "aa\naa\n" as "X\naa\n" where sed writes "X\nX\n". Apply per line. A + // trailing newline produces a final empty split element that corresponds to + // no input line, so it is carried over untouched. + const endsWithNewline = content.endsWith('\n') + const body = endsWithNewline ? content.slice(0, -1) : content + const result = body + .split('\n') + .map(line => line.replace(regex, jsReplacement)) + .join('\n') + return endsWithNewline ? result + '\n' : result }