shell: Allow duplicating output fds (e.g. 2>&1)

docs/runtime/shell.md

@@ -67,9 +67,20 @@ console.log(exitCode); // 0
 
 ## Redirection
 
-Bun Shell supports redirection with `<`, `>`, and `|` operators.
+A command's _input_ or _output_ may be _redirected_ using the typical Bash operators:
+- `<`           redirect stdin 
+- `>` or `1>`   redirect stdout
+- `2>`          redirect stderr
+- `&>`          redirect both stdout and stderr
+- `>>` or `1>>` redirect stdout, _appending_ to the destination, instead of overwriting
+- `2>>`         redirect stderr, _appending_ to the destination, instead of overwriting
+- `&>>`         redirect both stdout and stderr, _appending_ to the destination, instead of overwriting
+- `1>&2`        redirect stdout to stderr (all writes to stdout will instead be in stderr)
+- `2>&1`        redirect stderr to stdout (all writes to stderr will instead be in stdout)
 
-### To JavaScript objects (`>`)
+Bun Shell also supports redirecting from and to JavaScript objects.
+
+### Example: Redirect output to JavaScript objects (`>`)
 
 To redirect stdout to a JavaScript object, use the `>` operator:
 
@@ -88,7 +99,7 @@ The following JavaScript objects are supported for redirection to:
 - `Buffer`, `Uint8Array`, `Uint16Array`, `Uint32Array`, `Int8Array`, `Int16Array`, `Int32Array`, `Float32Array`, `Float64Array`, `ArrayBuffer`, `SharedArrayBuffer` (writes to the underlying buffer)
 - `Bun.file(path)`, `Bun.file(fd)` (writes to the file)
 
-### From JavaScript objects (`<`)
+### Example: Redirect input from JavaScript objects (`<`)
 
 To redirect the output from JavaScript objects to stdin, use the `<` operator:
 
@@ -108,7 +119,52 @@ The following JavaScript objects are supported for redirection from:
 - `Bun.file(path)`, `Bun.file(fd)` (reads from the file)
 - `Response` (reads from the body)
 
-### Piping (`|`)
+### Example: Redirect stdin -> file
+
+```js
+import { $ } from "bun"
+
+await $`cat < myfile.txt`
+```
+
+### Example: Redirect stdout -> file
+
+```js
+import { $ } from "bun"
+
+await $`echo bun! > greeting.txt`
+```
+
+### Example: Redirect stderr -> file
+
+```js
+import { $ } from "bun"
+
+await $`bun run index.ts 2> errors.txt`
+```
+
+
+### Example: Redirect stdout -> stderr
+
+```js
+import { $ } from "bun"
+
+// redirects stderr to stdout, so all output
+// will be available on stdout
+await $`bun run ./index.ts 2>&1`
+```
+
+### Example: Redirect stderr -> stdout
+
+```js
+import { $ } from "bun"
+
+// redirects stdout to stderr, so all output
+// will be available on stderr
+await $`bun run ./index.ts 1>&2`
+```
+
+## Piping (`|`)
 
 Like in bash, you can pipe the output of one command to another:
 

src/shell/interpreter.zig

@@ -158,7 +158,7 @@ pub const IO = struct {
         std: struct { captured: ?*bun.ByteList = null },
         /// Write/Read to/from file descriptor
         fd: bun.FileDescriptor,
-        /// Buffers the output
+        /// Buffers the output (handled in Cmd.BufferedIoClosed.close())
         pipe,
         /// Discards output
         ignore,
@@ -2919,6 +2919,7 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
             exec: Exec = .none,
             exit_code: ?ExitCode = null,
             io: IO,
+            // duplicate_out: enum { none, stdout, stderr } = .none,
             freed: bool = false,
 
             state: union(enum) {
@@ -3004,7 +3005,7 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
                                 const readable = io.stdout;
 
                                 // If the shell state is piped (inside a cmd substitution) aggregate the output of this command
-                                if (cmd.base.shell.io.stdout == .pipe and cmd.io.stdout == .pipe and !cmd.node.redirect.stdout) {
+                                if (cmd.base.shell.io.stdout == .pipe and cmd.io.stdout == .pipe and !cmd.node.redirect.redirectsElsewhere(.stdout)) {
                                     cmd.base.shell.buffered_stdout().append(bun.default_allocator, readable.pipe.buffer.internal_buffer.slice()) catch bun.outOfMemory();
                                 }
 
@@ -3017,7 +3018,7 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
                                 const readable = io.stderr;
 
                                 // If the shell state is piped (inside a cmd substitution) aggregate the output of this command
-                                if (cmd.base.shell.io.stderr == .pipe and cmd.io.stderr == .pipe and !cmd.node.redirect.stdout) {
+                                if (cmd.base.shell.io.stderr == .pipe and cmd.io.stderr == .pipe and !cmd.node.redirect.redirectsElsewhere(.stderr)) {
                                     cmd.base.shell.buffered_stderr().append(bun.default_allocator, readable.pipe.buffer.internal_buffer.slice()) catch bun.outOfMemory();
                                 }
 
@@ -3489,6 +3490,14 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
                             setStdioFromRedirect(&spawn_args.stdio, this.node.redirect, .{ .fd = redirfd });
                         },
                     }
+                } else if (this.node.redirect.duplicate_out) {
+                    if (this.node.redirect.stdout) {
+                        spawn_args.stdio[stderr_no] = .{ .dup2 = .{ .out = .stderr, .to = .stdout } };
+                    }
+
+                    if (this.node.redirect.stderr) {
+                        spawn_args.stdio[stdout_no] = .{ .dup2 = .{ .out = .stdout, .to = .stderr } };
+                    }
                 }
 
                 const buffered_closed = BufferedIoClosed.fromStdio(&spawn_args.stdio);
@@ -3519,12 +3528,17 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
                     stdio.*[stdin_no] = val;
                 }
 
-                if (flags.stdout) {
+                if (flags.duplicate_out) {
                     stdio.*[stdout_no] = val;
-                }
-
-                if (flags.stderr) {
                     stdio.*[stderr_no] = val;
+                } else {
+                    if (flags.stdout) {
+                        stdio.*[stdout_no] = val;
+                    }
+
+                    if (flags.stderr) {
+                        stdio.*[stderr_no] = val;
+                    }
                 }
             }
 
@@ -3649,7 +3663,7 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
                     std.debug.assert(this.exec == .subproc);
                 }
                 log("cmd ({x}) close buffered stdout", .{@intFromPtr(this)});
-                if (this.io.stdout == .std and this.io.stdout.std.captured != null and !this.node.redirect.stdout) {
+                if (this.io.stdout == .std and this.io.stdout.std.captured != null and !this.node.redirect.redirectsElsewhere(.stdout)) {
                     var buf = this.io.stdout.std.captured.?;
                     buf.append(bun.default_allocator, this.exec.subproc.child.stdout.pipe.buffer.internal_buffer.slice()) catch bun.outOfMemory();
                 }
@@ -3662,7 +3676,7 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
                     std.debug.assert(this.exec == .subproc);
                 }
                 log("cmd ({x}) close buffered stderr", .{@intFromPtr(this)});
-                if (this.io.stderr == .std and this.io.stderr.std.captured != null and !this.node.redirect.stderr) {
+                if (this.io.stderr == .std and this.io.stderr.std.captured != null and !this.node.redirect.redirectsElsewhere(.stderr)) {
                     var buf = this.io.stderr.std.captured.?;
                     buf.append(bun.default_allocator, this.exec.subproc.child.stderr.pipe.buffer.internal_buffer.slice()) catch bun.outOfMemory();
                 }
@@ -4073,6 +4087,14 @@ pub fn NewInterpreter(comptime EventLoopKind: JSC.EventLoopKind) type {
                             }
                         },
                     }
+                } else if (node.redirect.duplicate_out) {
+                    if (node.redirect.stdout) {
+                        cmd.exec.bltn.stderr = cmd.exec.bltn.stdout;
+                    }
+
+                    if (node.redirect.stderr) {
+                        cmd.exec.bltn.stdout = cmd.exec.bltn.stderr;
+                    }
                 }
 
                 return .cont;

src/shell/shell.zig

@@ -442,7 +442,26 @@ pub const AST = struct {
             stdout: bool = false,
             stderr: bool = false,
             append: bool = false,
-            __unused: u4 = 0,
+            /// 1>&2 === stdout=true and duplicate_out=true
+            /// 2>&1 === stderr=true and duplicate_out=true
+            duplicate_out: bool = false,
+            __unused: u3 = 0,
+
+            pub fn redirectsElsewhere(this: RedirectFlags, io_kind: enum { stdin, stdout, stderr }) bool {
+                return switch (io_kind) {
+                    .stdin => this.stdin,
+                    .stdout => if (this.duplicate_out) !this.stdout else this.stdout,
+                    .stderr => if (this.duplicate_out) !this.stderr else this.stderr,
+                };
+            }
+
+            pub fn @"2>&1"() RedirectFlags {
+                return .{ .stderr = true, .duplicate = true };
+            }
+
+            pub fn @"1>&2"() RedirectFlags {
+                return .{ .stdout = true, .duplicate = true };
+            }
 
             pub fn toFlags(this: RedirectFlags) bun.Mode {
                 const read_write_flags: bun.Mode = if (this.stdin) std.os.O.RDONLY else std.os.O.WRONLY | std.os.O.CREAT;
@@ -792,6 +811,7 @@ pub const Parser = struct {
                 }
 
                 const redirect_file = try self.parse_atom() orelse {
+                    if (redirect.duplicate_out) break :redirect_file null;
                     try self.add_error("Redirection with no file", .{});
                     return ParseError.Expected;
                 };
@@ -809,6 +829,8 @@ pub const Parser = struct {
         } };
     }
 
+    const ParsedRedirect = struct { flags: AST.Cmd.RedirectFlags, redirect: AST.Cmd.Redirect };
+
     /// Try to parse an assignment. If no assignment could be parsed then return
     /// null and backtrack the parser state
     fn parse_assign(self: *Parser) !?AST.Assign {
@@ -1812,9 +1834,67 @@ pub fn NewLexer(comptime encoding: StringEncoding) type {
             return false;
         }
 
+        // TODO Arbitrary file descriptor redirect
         fn eat_redirect(self: *@This(), first: InputChar) ?AST.Cmd.RedirectFlags {
             var flags: AST.Cmd.RedirectFlags = .{};
             switch (first.char) {
+                '0' => flags.stdin = true,
+                '1' => flags.stdout = true,
+                '2' => flags.stderr = true,
+                // Just allow the std file descriptors for now
+                else => return null,
+            }
+            var dir: RedirectDirection = .out;
+            if (self.peek()) |input| {
+                if (input.escaped) return null;
+                switch (input.char) {
+                    '>' => {
+                        _ = self.eat();
+                        dir = .out;
+                        const is_double = self.eat_simple_redirect_operator(dir);
+                        if (is_double) flags.append = true;
+                        if (self.peek()) |peeked| {
+                            if (!peeked.escaped and peeked.char == '&') {
+                                _ = self.eat();
+                                if (self.peek()) |peeked2| {
+                                    switch (peeked2.char) {
+                                        '1' => {
+                                            _ = self.eat();
+                                            if (!flags.stdout and flags.stderr) {
+                                                flags.duplicate_out = true;
+                                                flags.stdout = true;
+                                                flags.stderr = false;
+                                            } else return null;
+                                        },
+                                        '2' => {
+                                            _ = self.eat();
+                                            if (!flags.stderr and flags.stdout) {
+                                                flags.duplicate_out = true;
+                                                flags.stderr = true;
+                                                flags.stdout = false;
+                                            } else return null;
+                                        },
+                                        else => return null,
+                                    }
+                                }
+                            }
+                        }
+                        return flags;
+                    },
+                    '<' => {
+                        dir = .in;
+                        const is_double = self.eat_simple_redirect_operator(dir);
+                        if (is_double) flags.append = true;
+                        return flags;
+                    },
+                    else => return null,
+                }
+            } else return null;
+        }
+
+        fn eat_redirect_old(self: *@This(), first: InputChar) ?AST.Cmd.RedirectFlags {
+            var flags: AST.Cmd.RedirectFlags = .{};
+            if (self.matchesAsciiLiteral("2>&1")) {} else if (self.matchesAsciiLiteral("1>&2")) {} else switch (first.char) {
                 '0'...'9' => {
                     // Codepoint int casts are safe here because the digits are in the ASCII range
                     var count: usize = 1;
@@ -1988,10 +2068,33 @@ pub fn NewLexer(comptime encoding: StringEncoding) type {
             return std.mem.eql(u8, bytes[0 .. LEX_JS_STRING_PREFIX.len - 1], LEX_JS_STRING_PREFIX[1..]);
         }
 
-        fn eatJSSubstitutionIdx(self: *@This(), comptime literal: []const u8, comptime name: []const u8, comptime validate: *const fn (*@This(), usize) bool) ?usize {
+        fn bumpCursorAscii(self: *@This(), new_idx: usize, prev_ascii_char: ?u7, cur_ascii_char: u7) void {
+            if (comptime encoding == .ascii) {
+                self.chars.src.i = new_idx;
+                if (prev_ascii_char) |pc| self.chars.prev = .{ .char = pc };
+                self.chars.current = .{ .char = cur_ascii_char };
+                return;
+            }
+            self.chars.src.cursor = CodepointIterator.Cursor{
+                .i = @intCast(new_idx),
+                .c = cur_ascii_char,
+                .width = 1,
+            };
+            self.chars.src.next_cursor = self.chars.src.cursor;
+            SrcUnicode.nextCursor(&self.chars.src.iter, &self.chars.src.next_cursor);
+            if (prev_ascii_char) |pc| self.chars.prev = .{ .char = pc };
+            self.chars.current = .{ .char = cur_ascii_char };
+        }
+
+        fn matchesAsciiLiteral(self: *@This(), literal: []const u8) bool {
             const bytes = self.chars.srcBytesAtCursor();
-            if (literal.len - 1 >= bytes.len) return null;
-            if (std.mem.eql(u8, bytes[0 .. literal.len - 1], literal[1..])) {
+            if (literal.len >= bytes.len) return false;
+            return std.mem.eql(u8, bytes[0..literal.len], literal[0..]);
+        }
+
+        fn eatJSSubstitutionIdx(self: *@This(), comptime literal: []const u8, comptime name: []const u8, comptime validate: *const fn (*@This(), usize) bool) ?usize {
+            if (self.matchesAsciiLiteral(literal[1..literal.len])) {
+                const bytes = self.chars.srcBytesAtCursor();
                 var i: usize = 0;
                 var digit_buf: [32]u8 = undefined;
                 var digit_buf_count: u8 = 0;
@@ -2032,26 +2135,10 @@ pub fn NewLexer(comptime encoding: StringEncoding) type {
                 // }
 
                 // Bump the cursor
-                brk: {
-                    const new_idx = self.chars.cursorPos() + i;
-                    const prev_ascii_char: ?u7 = if (digit_buf_count == 1) null else @truncate(digit_buf[digit_buf_count - 2]);
-                    const cur_ascii_char: u7 = @truncate(digit_buf[digit_buf_count - 1]);
-                    if (comptime encoding == .ascii) {
-                        self.chars.src.i = new_idx;
-                        if (prev_ascii_char) |pc| self.chars.prev = .{ .char = pc };
-                        self.chars.current = .{ .char = cur_ascii_char };
-                        break :brk;
-                    }
-                    self.chars.src.cursor = CodepointIterator.Cursor{
-                        .i = @intCast(new_idx),
-                        .c = cur_ascii_char,
-                        .width = 1,
-                    };
-                    self.chars.src.next_cursor = self.chars.src.cursor;
-                    SrcUnicode.nextCursor(&self.chars.src.iter, &self.chars.src.next_cursor);
-                    if (prev_ascii_char) |pc| self.chars.prev = .{ .char = pc };
-                    self.chars.current = .{ .char = cur_ascii_char };
-                }
+                const new_idx = self.chars.cursorPos() + i;
+                const prev_ascii_char: ?u7 = if (digit_buf_count == 1) null else @truncate(digit_buf[digit_buf_count - 2]);
+                const cur_ascii_char: u7 = @truncate(digit_buf[digit_buf_count - 1]);
+                self.bumpCursorAscii(new_idx, prev_ascii_char, cur_ascii_char);
 
                 // return self.string_refs[idx];
                 return idx;