From b9e00166e9dfb46271adbc4037f597645229c2b5 Mon Sep 17 00:00:00 2001 From: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> Date: Sat, 24 Feb 2024 19:56:37 -0800 Subject: [PATCH 01/10] Support unix domain sockets in fetch() --- src/bun.js/webcore/response.zig | 31 ++++++++++++++++++++++++++++++- src/http.zig | 27 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig index 55cf79afb401..468a803f8bbf 100644 --- a/src/bun.js/webcore/response.zig +++ b/src/bun.js/webcore/response.zig @@ -798,6 +798,7 @@ pub const Fetch = struct { this.request_headers = Headers{ .allocator = undefined }; if (this.http != null) { + allocator.free(this.http.?.client.unix_socket_path); this.http.?.clearData(); } @@ -1513,6 +1514,7 @@ pub const Fetch = struct { fetch_tasklet.http.?.client.remaining_redirect_count = 0; } + fetch_tasklet.http.?.client.unix_socket_path = fetch_options.unix_socket_path; fetch_tasklet.http.?.client.disable_timeout = fetch_options.disable_timeout; fetch_tasklet.http.?.client.verbose = fetch_options.verbose; fetch_tasklet.http.?.client.disable_keepalive = fetch_options.disable_keepalive; @@ -1567,6 +1569,7 @@ pub const Fetch = struct { hostname: ?[]u8 = null, memory_reporter: *JSC.MemoryReportingAllocator, check_server_identity: JSC.Strong = .{}, + unix_socket_path: []const u8 = "", }; pub fn queue( @@ -1720,11 +1723,19 @@ pub const Fetch = struct { var signal: ?*JSC.WebCore.AbortSignal = null; // Custom Hostname var hostname: ?[]u8 = null; + var unix_socket_path: ZigString.Slice = ZigString.Slice.empty; - var url_proxy_buffer: []const u8 = undefined; + var url_proxy_buffer: []const u8 = ""; var is_file_url = false; var reject_unauthorized = script_ctx.bundler.env.getTLSRejectUnauthorized(); var check_server_identity: JSValue = .zero; + + defer { + if (free_memory_reporter) { + unix_socket_path.deinit(); + } + } + // TODO: move this into a DRYer implementation // The status quo is very repetitive and very bug prone if (first_arg.as(Request)) |request| { @@ -1917,6 +1928,14 @@ pub const Fetch = struct { url_proxy_buffer = buffer; } } + + if (options.get(globalThis, "socket")) |socket_path| { + if (socket_path.isString() and socket_path.getLength(ctx) > 0) { + if (socket_path.toSliceCloneWithAllocator(globalThis, allocator)) |slice| { + unix_socket_path = slice; + } + } + } } } else { method = request.method; @@ -2125,6 +2144,14 @@ pub const Fetch = struct { url_proxy_buffer = buffer; } } + + if (options.get(globalThis, "socket")) |socket_path| { + if (socket_path.isString() and socket_path.getLength(ctx) > 0) { + if (socket_path.toSliceCloneWithAllocator(globalThis, allocator)) |slice| { + unix_socket_path = slice; + } + } + } } } } @@ -2146,6 +2173,7 @@ pub const Fetch = struct { // But it's better than status quo. if (is_file_url) { defer allocator.free(url_proxy_buffer); + defer unix_socket_path.deinit(); var path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; const PercentEncoding = @import("../../url.zig").PercentEncoding; var path_buf2: [bun.MAX_PATH_BYTES]u8 = undefined; @@ -2395,6 +2423,7 @@ pub const Fetch = struct { .hostname = hostname, .memory_reporter = memory_reporter, .check_server_identity = if (check_server_identity.isEmptyOrUndefinedOrNull()) .{} else JSC.Strong.create(check_server_identity, globalThis), + .unix_socket_path = unix_socket_path.slice(), }, // Pass the Strong value instead of creating a new one, or else we // will leak it diff --git a/src/http.zig b/src/http.zig index a59a06f6f40e..d52313f308bc 100644 --- a/src/http.zig +++ b/src/http.zig @@ -632,6 +632,22 @@ fn NewHTTPContext(comptime ssl: bool) type { return null; } + pub fn connectSocket(this: *@This(), client: *HTTPClient, socket_path: []const u8) !HTTPSocket { + client.connected_url = if (client.http_proxy) |proxy| proxy else client.url; + + if (HTTPSocket.connectUnixAnon( + socket_path, + this.us_socket_context, + undefined, + )) |socket| { + client.allow_retry = false; + socket.ext(**anyopaque).?.* = bun.cast(**anyopaque, ActiveSocket.init(client).ptr()); + return socket; + } + + return error.FailedToOpenSocket; + } + pub fn connect(this: *@This(), client: *HTTPClient, hostname_: []const u8, port: u16) !HTTPSocket { const hostname = if (FeatureFlags.hardcode_localhost_to_127_0_0_1 and strings.eqlComptime(hostname_, "localhost")) "127.0.0.1" @@ -747,6 +763,10 @@ pub const HTTPThread = struct { } pub fn connect(this: *@This(), client: *HTTPClient, comptime is_ssl: bool) !NewHTTPContext(is_ssl).HTTPSocket { + if (client.unix_socket_path.len > 0) { + return try this.context(is_ssl).connectSocket(client, client.unix_socket_path); + } + if (client.http_proxy) |url| { return try this.context(is_ssl).connect(client, url.hostname, url.getPortAuto()); } @@ -1509,6 +1529,9 @@ async_http_id: u32 = 0, hostname: ?[]u8 = null, reject_unauthorized: bool = true, +/// if empty, that means we are not using a unix socket +unix_socket_path: []const u8 = "", + pub fn init( allocator: std.mem.Allocator, method: Method, @@ -1546,6 +1569,10 @@ pub fn deinit(this: *HTTPClient) void { pub fn isKeepAlivePossible(this: *HTTPClient) bool { if (comptime FeatureFlags.enable_keepalive) { + if (this.unix_socket_path.len > 0) + // TODO: + return false; + // is not possible to reuse Proxy with TSL, so disable keepalive if url is tunneling HTTPS if (this.http_proxy != null and this.url.isHTTPS()) { return false; From 6b20bae9f40d971562bc61e1a48ae62eec1edb6b Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Tue, 5 Mar 2024 11:21:54 +0100 Subject: [PATCH 02/10] rename to 'socket' to 'unix', add test --- src/bun.js/webcore/response.zig | 4 +-- test/js/web/fetch/fetch.unix.test.ts | 38 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 test/js/web/fetch/fetch.unix.test.ts diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig index 468a803f8bbf..6771e50a23f5 100644 --- a/src/bun.js/webcore/response.zig +++ b/src/bun.js/webcore/response.zig @@ -1929,7 +1929,7 @@ pub const Fetch = struct { } } - if (options.get(globalThis, "socket")) |socket_path| { + if (options.get(globalThis, "unix")) |socket_path| { if (socket_path.isString() and socket_path.getLength(ctx) > 0) { if (socket_path.toSliceCloneWithAllocator(globalThis, allocator)) |slice| { unix_socket_path = slice; @@ -2145,7 +2145,7 @@ pub const Fetch = struct { } } - if (options.get(globalThis, "socket")) |socket_path| { + if (options.get(globalThis, "unix")) |socket_path| { if (socket_path.isString() and socket_path.getLength(ctx) > 0) { if (socket_path.toSliceCloneWithAllocator(globalThis, allocator)) |slice| { unix_socket_path = slice; diff --git a/test/js/web/fetch/fetch.unix.test.ts b/test/js/web/fetch/fetch.unix.test.ts new file mode 100644 index 000000000000..279d1eca6881 --- /dev/null +++ b/test/js/web/fetch/fetch.unix.test.ts @@ -0,0 +1,38 @@ +import { serve, ServeOptions, Server } from "bun"; +import { afterAll, afterEach, expect, it } from "bun:test"; +import { mkdtempSync, realpathSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +const tmp_dir = mkdtempSync(join(realpathSync(tmpdir()), "fetch.unix.test")); + +let server: Server; +function startServerUnix({ fetch, ...options }: ServeOptions): string { + const socketPath = join(tmp_dir, `socket-${Math.random().toString(36).slice(2)}`); + server = serve({ + ...options, + fetch, + unix: socketPath, + }); + return socketPath; +} + +afterEach(() => { + server?.stop?.(true); +}); + +afterAll(() => { + rmSync(tmp_dir, { force: true, recursive: true }); +}); + +it("provide body", async () => { + const path = startServerUnix({ + fetch(req) { + return new Response(req.body); + }, + }); + // POST with body + const response = await fetch("http://localhost/hello", { method: "POST", body: "buntastic", unix: path }); + expect(response.status).toBe(200); + expect(await response.text()).toBe("buntastic"); +}); From 832391fcf541d81c337ff929ac522b7982c02f92 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Tue, 5 Mar 2024 12:17:09 +0100 Subject: [PATCH 03/10] enable keepalive on unix socket fetch --- src/http.zig | 4 ---- test/js/web/fetch/fetch.unix.test.ts | 8 +++++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/http.zig b/src/http.zig index d52313f308bc..4c63a6515074 100644 --- a/src/http.zig +++ b/src/http.zig @@ -1569,10 +1569,6 @@ pub fn deinit(this: *HTTPClient) void { pub fn isKeepAlivePossible(this: *HTTPClient) bool { if (comptime FeatureFlags.enable_keepalive) { - if (this.unix_socket_path.len > 0) - // TODO: - return false; - // is not possible to reuse Proxy with TSL, so disable keepalive if url is tunneling HTTPS if (this.http_proxy != null and this.url.isHTTPS()) { return false; diff --git a/test/js/web/fetch/fetch.unix.test.ts b/test/js/web/fetch/fetch.unix.test.ts index 279d1eca6881..05142863295b 100644 --- a/test/js/web/fetch/fetch.unix.test.ts +++ b/test/js/web/fetch/fetch.unix.test.ts @@ -32,7 +32,9 @@ it("provide body", async () => { }, }); // POST with body - const response = await fetch("http://localhost/hello", { method: "POST", body: "buntastic", unix: path }); - expect(response.status).toBe(200); - expect(await response.text()).toBe("buntastic"); + for (let i = 0; i < 20; i++) { + const response = await fetch("http://localhost/hello", { method: "POST", body: String(i), unix: path }); + expect(response.status).toBe(200); + expect(await response.text()).toBe(String(i)); + } }); From e4b2af5b8af08eb899aca0c992c0f43785c8f150 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 6 Mar 2024 13:09:51 +0100 Subject: [PATCH 04/10] make ownership of unix_socket_path clear, refactor http.zig a bit --- src/bun.js/webcore/response.zig | 66 ++++++++++-------- src/http.zig | 114 +++++++++++++++++--------------- src/install/install.zig | 13 ++-- 3 files changed, 106 insertions(+), 87 deletions(-) diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig index 6771e50a23f5..725c7c8e6677 100644 --- a/src/bun.js/webcore/response.zig +++ b/src/bun.js/webcore/response.zig @@ -632,6 +632,7 @@ pub const Fetch = struct { pub const fetch_error_no_args = "fetch() expects a string but received no arguments."; pub const fetch_error_blank_url = "fetch() URL must not be a blank string."; pub const fetch_error_unexpected_body = "fetch() request with GET/HEAD/OPTIONS method cannot have body."; + pub const fetch_error_proxy_unix = "fetch() cannot use a proxy with a unix socket."; const JSTypeErrorEnum = std.enums.EnumArray(JSType, string); pub const fetch_type_error_names: JSTypeErrorEnum = brk: { var errors = JSTypeErrorEnum.initUndefined(); @@ -798,7 +799,6 @@ pub const Fetch = struct { this.request_headers = Headers{ .allocator = undefined }; if (this.http != null) { - allocator.free(this.http.?.client.unix_socket_path); this.http.?.clearData(); } @@ -1503,25 +1503,27 @@ pub const Fetch = struct { ).init( fetch_tasklet, ), - proxy, - - fetch_options.hostname, fetch_options.redirect_type, - fetch_tasklet.signals, + .{ + .http_proxy = proxy, + .hostname = fetch_options.hostname, + .signals = fetch_tasklet.signals, + .unix_socket_path = fetch_options.unix_socket_path, + .disable_timeout = fetch_options.disable_timeout, + .disable_keepalive = fetch_options.disable_keepalive, + .disable_decompression = fetch_options.disable_decompression, + .reject_unauthorized = fetch_options.reject_unauthorized, + .verbose = fetch_options.verbose, + }, ); + // TODO is this necessary? the http client already sets the redirect type, + // so manually setting it here seems redundant if (fetch_options.redirect_type != FetchRedirect.follow) { fetch_tasklet.http.?.client.remaining_redirect_count = 0; } - fetch_tasklet.http.?.client.unix_socket_path = fetch_options.unix_socket_path; - fetch_tasklet.http.?.client.disable_timeout = fetch_options.disable_timeout; - fetch_tasklet.http.?.client.verbose = fetch_options.verbose; - fetch_tasklet.http.?.client.disable_keepalive = fetch_options.disable_keepalive; - fetch_tasklet.http.?.client.disable_decompression = fetch_options.disable_decompression; - fetch_tasklet.http.?.client.reject_unauthorized = fetch_options.reject_unauthorized; - - // we wanna to return after headers are received + // we want to return after headers are received fetch_tasklet.signal_store.header_progress.store(true, .Monotonic); if (fetch_tasklet.request_body == .Sendfile) { @@ -1569,7 +1571,7 @@ pub const Fetch = struct { hostname: ?[]u8 = null, memory_reporter: *JSC.MemoryReportingAllocator, check_server_identity: JSC.Strong = .{}, - unix_socket_path: []const u8 = "", + unix_socket_path: ZigString.Slice, }; pub fn queue( @@ -1677,17 +1679,18 @@ pub const Fetch = struct { var exception_val = [_]JSC.C.JSValueRef{null}; const exception: JSC.C.ExceptionRef = &exception_val; var memory_reporter = bun.default_allocator.create(JSC.MemoryReportingAllocator) catch @panic("out of memory"); - var free_memory_reporter = false; + // used to clean up dynamically allocated memory on error (a poor man's errdefer) + var is_error = false; var allocator = memory_reporter.wrap(bun.default_allocator); defer { if (exception.* != null) { - free_memory_reporter = true; + is_error = true; ctx.throwValue(JSC.JSValue.c(exception.*)); } memory_reporter.report(globalThis.vm()); - if (free_memory_reporter) bun.default_allocator.destroy(memory_reporter); + if (is_error) bun.default_allocator.destroy(memory_reporter); } if (arguments.len == 0) { @@ -1731,7 +1734,7 @@ pub const Fetch = struct { var check_server_identity: JSValue = .zero; defer { - if (free_memory_reporter) { + if (is_error) { unix_socket_path.deinit(); } } @@ -1748,7 +1751,7 @@ pub const Fetch = struct { allocator.free(host); hostname = null; } - free_memory_reporter = true; + is_error = true; return JSPromise.rejectedPromiseValue(globalThis, err); } @@ -1772,7 +1775,7 @@ pub const Fetch = struct { allocator.free(host); hostname = null; } - free_memory_reporter = true; + is_error = true; return JSPromise.rejectedPromiseValue( globalThis, err, @@ -2130,7 +2133,7 @@ pub const Fetch = struct { hostname = null; } allocator.free(url_proxy_buffer); - free_memory_reporter = true; + is_error = true; return JSPromise.rejectedPromiseValue(globalThis, err); } defer href.deref(); @@ -2163,11 +2166,17 @@ pub const Fetch = struct { } if (url.isEmpty()) { - free_memory_reporter = true; + is_error = true; const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, fetch_error_blank_url, .{}, ctx); return JSPromise.rejectedPromiseValue(globalThis, err); } + if (proxy != null and unix_socket_path.length() > 0) { + is_error = true; + const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, fetch_error_proxy_unix, .{}, ctx); + return JSPromise.rejectedPromiseValue(globalThis, err); + } + // This is not 100% correct. // We don't pass along headers, we ignore method, we ignore status code... // But it's better than status quo. @@ -2261,7 +2270,7 @@ pub const Fetch = struct { if (!(url.isHTTP() or url.isHTTPS())) { defer allocator.free(url_proxy_buffer); const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, "protocol must be http: or https:", .{}, ctx); - free_memory_reporter = true; + is_error = true; return JSPromise.rejectedPromiseValue(globalThis, err); } } @@ -2269,7 +2278,7 @@ pub const Fetch = struct { if (!method.hasRequestBody() and body.size() > 0) { defer allocator.free(url_proxy_buffer); const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, fetch_error_unexpected_body, .{}, ctx); - free_memory_reporter = true; + is_error = true; return JSPromise.rejectedPromiseValue(globalThis, err); } @@ -2302,7 +2311,7 @@ pub const Fetch = struct { headers_.buf.deinit(allocator); headers_.entries.deinit(allocator); } - free_memory_reporter = true; + is_error = true; return rejected_value; }, .result => |fd| fd, @@ -2373,7 +2382,7 @@ pub const Fetch = struct { switch (res) { .err => |err| { allocator.free(url_proxy_buffer); - free_memory_reporter = true; + is_error = true; const rejected_value = JSPromise.rejectedPromiseValue(globalThis, err.toJSC(globalThis)); body.detach(); if (headers) |*headers_| { @@ -2398,7 +2407,6 @@ pub const Fetch = struct { const promise_val = promise.value(); - // var resolve = FetchTasklet.FetchResolver.Class.make(ctx: js.JSContextRef, ptr: *ZigType) _ = FetchTasklet.queue( allocator, globalThis, @@ -2423,13 +2431,13 @@ pub const Fetch = struct { .hostname = hostname, .memory_reporter = memory_reporter, .check_server_identity = if (check_server_identity.isEmptyOrUndefinedOrNull()) .{} else JSC.Strong.create(check_server_identity, globalThis), - .unix_socket_path = unix_socket_path.slice(), + .unix_socket_path = unix_socket_path, }, // Pass the Strong value instead of creating a new one, or else we // will leak it // see https://github.com/oven-sh/bun/issues/2985 promise, - ) catch unreachable; + ) catch bun.outOfMemory(); return promise_val; } diff --git a/src/http.zig b/src/http.zig index 4c63a6515074..b77405ddb13e 100644 --- a/src/http.zig +++ b/src/http.zig @@ -763,8 +763,8 @@ pub const HTTPThread = struct { } pub fn connect(this: *@This(), client: *HTTPClient, comptime is_ssl: bool) !NewHTTPContext(is_ssl).HTTPSocket { - if (client.unix_socket_path.len > 0) { - return try this.context(is_ssl).connectSocket(client, client.unix_socket_path); + if (client.unix_socket_path.length() > 0) { + return try this.context(is_ssl).connectSocket(client, client.unix_socket_path.slice()); } if (client.http_proxy) |url| { @@ -1529,28 +1529,7 @@ async_http_id: u32 = 0, hostname: ?[]u8 = null, reject_unauthorized: bool = true, -/// if empty, that means we are not using a unix socket -unix_socket_path: []const u8 = "", - -pub fn init( - allocator: std.mem.Allocator, - method: Method, - url: URL, - header_entries: Headers.Entries, - header_buf: string, - hostname: ?[]u8, - signals: Signals, -) HTTPClient { - return HTTPClient{ - .allocator = allocator, - .method = method, - .url = url, - .header_entries = header_entries, - .header_buf = header_buf, - .hostname = hostname, - .signals = signals, - }; -} +unix_socket_path: JSC.ZigString.Slice = JSC.ZigString.Slice.empty, pub fn deinit(this: *HTTPClient) void { if (this.redirect.len > 0) { @@ -1565,6 +1544,7 @@ pub fn deinit(this: *HTTPClient) void { tunnel.deinit(); this.proxy_tunnel = null; } + this.unix_socket_path.deinit(); } pub fn isKeepAlivePossible(this: *HTTPClient) bool { @@ -1778,6 +1758,18 @@ pub const AsyncHTTP = struct { }; const AtomicState = std.atomic.Value(State); + pub const Options = struct { + http_proxy: ?URL = null, + hostname: ?[]u8 = null, + signals: ?Signals = null, + unix_socket_path: ?JSC.ZigString.Slice = null, + disable_timeout: ?bool = null, + verbose: ?bool = null, + disable_keepalive: ?bool = null, + disable_decompression: ?bool = null, + reject_unauthorized: ?bool = null, + }; + pub fn init( allocator: std.mem.Allocator, method: Method, @@ -1788,10 +1780,8 @@ pub const AsyncHTTP = struct { request_body: []const u8, timeout: usize, callback: HTTPClientResult.Callback, - http_proxy: ?URL, - hostname: ?[]u8, redirect_type: FetchRedirect, - signals: ?Signals, + options: Options, ) AsyncHTTP { var this = AsyncHTTP{ .allocator = allocator, @@ -1802,19 +1792,46 @@ pub const AsyncHTTP = struct { .request_body = .{ .bytes = request_body }, .response_buffer = response_buffer, .result_callback = callback, - .http_proxy = http_proxy, - .signals = signals orelse .{}, - .async_http_id = if (signals != null and signals.?.aborted != null) async_http_id.fetchAdd(1, .Monotonic) else 0, + .http_proxy = options.http_proxy, + .signals = options.signals orelse .{}, + .async_http_id = if (options.signals != null and options.signals.?.aborted != null) async_http_id.fetchAdd(1, .Monotonic) else 0, + .timeout = timeout, }; - this.client = HTTPClient.init(allocator, method, url, headers, headers_buf, hostname, signals orelse this.signals); - this.client.async_http_id = this.async_http_id; - this.client.timeout = timeout; - this.client.http_proxy = this.http_proxy; - this.client.redirect_type = redirect_type; - this.timeout = timeout; + this.client = .{ + .allocator = allocator, + .method = method, + .url = url, + .header_entries = headers, + .header_buf = headers_buf, + .hostname = options.hostname, + .signals = options.signals orelse this.signals, + .async_http_id = this.async_http_id, + .timeout = timeout, + .http_proxy = this.http_proxy, + .redirect_type = redirect_type, + }; + if (options.unix_socket_path) |val| { + std.debug.assert(this.client.unix_socket_path.length() == 0); + this.client.unix_socket_path = val; + } + if (options.disable_timeout) |val| { + this.client.disable_timeout = val; + } + if (options.verbose) |val| { + this.client.verbose = val; + } + if (options.disable_decompression) |val| { + this.client.disable_decompression = val; + } + if (options.disable_keepalive) |val| { + this.client.disable_keepalive = val; + } + if (options.reject_unauthorized) |val| { + this.client.reject_unauthorized = val; + } - if (http_proxy) |proxy| { + if (options.http_proxy) |proxy| { // Username between 0 and 4096 chars if (proxy.username.len > 0 and proxy.username.len < 4096) { // Password between 0 and 4096 chars @@ -1874,21 +1891,10 @@ pub const AsyncHTTP = struct { } pub fn initSync(allocator: std.mem.Allocator, method: Method, url: URL, headers: Headers.Entries, headers_buf: string, response_buffer: *MutableString, request_body: []const u8, timeout: usize, http_proxy: ?URL, hostname: ?[]u8, redirect_type: FetchRedirect) AsyncHTTP { - return @This().init( - allocator, - method, - url, - headers, - headers_buf, - response_buffer, - request_body, - timeout, - undefined, - http_proxy, - hostname, - redirect_type, - null, - ); + return @This().init(allocator, method, url, headers, headers_buf, response_buffer, request_body, timeout, undefined, redirect_type, .{ + .http_proxy = http_proxy, + .hostname = hostname, + }); } fn reset(this: *AsyncHTTP) !void { @@ -2001,6 +2007,8 @@ pub const AsyncHTTP = struct { var callback = this.result_callback; this.elapsed = http_thread.timer.read() -| this.elapsed; + // TODO: this condition seems wrong: if we started with a non-default value, we might + // report a redirect even if none happened this.redirected = this.client.remaining_redirect_count != default_redirect_count; if (result.isSuccess()) { this.err = null; @@ -2189,6 +2197,8 @@ pub fn doRedirect(this: *HTTPClient, comptime is_ssl: bool, ctx: *NewHTTPContext this.remaining_redirect_count -|= 1; std.debug.assert(this.redirect_type == FetchRedirect.follow); + // TODO: should this check be before decrementing the redirect count? + // the current logic will allow one less redirect than requested if (this.remaining_redirect_count == 0) { this.fail(error.TooManyRedirects); return; diff --git a/src/install/install.zig b/src/install/install.zig index 3ee5fdb62ffb..ebd0dece290a 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -404,10 +404,10 @@ const NetworkTask = struct { "", 0, this.getCompletionCallback(), - this.package_manager.httpProxy(url), - null, HTTP.FetchRedirect.follow, - null, + .{ + .http_proxy = this.package_manager.httpProxy(url), + } ); this.http.client.reject_unauthorized = this.package_manager.tlsRejectUnauthorized(); @@ -488,10 +488,11 @@ const NetworkTask = struct { "", 0, this.getCompletionCallback(), - this.package_manager.httpProxy(url), - null, HTTP.FetchRedirect.follow, - null, + .{ + + .http_proxy = this.package_manager.httpProxy(url), + } ); this.http.client.reject_unauthorized = this.package_manager.tlsRejectUnauthorized(); if (PackageManager.verbose_install) { From dd346c948f1b2dcbeb736ffb24d26981ca859da3 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:11:26 +0000 Subject: [PATCH 05/10] [autofix.ci] apply automated fixes --- src/install/install.zig | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index ebd0dece290a..79f87ab9b276 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -394,21 +394,9 @@ const NetworkTask = struct { this.allocator = allocator; const url = URL.parse(this.url_buf); - this.http = AsyncHTTP.init( - allocator, - .GET, - url, - header_builder.entries, - header_builder.content.ptr.?[0..header_builder.content.len], - &this.response_buffer, - "", - 0, - this.getCompletionCallback(), - HTTP.FetchRedirect.follow, - .{ - .http_proxy = this.package_manager.httpProxy(url), - } - ); + this.http = AsyncHTTP.init(allocator, .GET, url, header_builder.entries, header_builder.content.ptr.?[0..header_builder.content.len], &this.response_buffer, "", 0, this.getCompletionCallback(), HTTP.FetchRedirect.follow, .{ + .http_proxy = this.package_manager.httpProxy(url), + }); this.http.client.reject_unauthorized = this.package_manager.tlsRejectUnauthorized(); if (PackageManager.verbose_install) { @@ -478,22 +466,9 @@ const NetworkTask = struct { const url = URL.parse(this.url_buf); - this.http = AsyncHTTP.init( - allocator, - .GET, - url, - header_builder.entries, - header_buf, - &this.response_buffer, - "", - 0, - this.getCompletionCallback(), - HTTP.FetchRedirect.follow, - .{ - - .http_proxy = this.package_manager.httpProxy(url), - } - ); + this.http = AsyncHTTP.init(allocator, .GET, url, header_builder.entries, header_buf, &this.response_buffer, "", 0, this.getCompletionCallback(), HTTP.FetchRedirect.follow, .{ + .http_proxy = this.package_manager.httpProxy(url), + }); this.http.client.reject_unauthorized = this.package_manager.tlsRejectUnauthorized(); if (PackageManager.verbose_install) { this.http.client.verbose = true; From 184488e2ebb26a848a610f8ab0cc68374c1b0e7d Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 6 Mar 2024 14:15:21 +0100 Subject: [PATCH 06/10] add fetch unix redirect test, fix crash --- src/http.zig | 3 ++ test/js/web/fetch/fetch.unix.test.ts | 42 ++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/http.zig b/src/http.zig index b77405ddb13e..2dac5e090211 100644 --- a/src/http.zig +++ b/src/http.zig @@ -1545,6 +1545,7 @@ pub fn deinit(this: *HTTPClient) void { this.proxy_tunnel = null; } this.unix_socket_path.deinit(); + this.unix_socket_path = JSC.ZigString.Slice.empty; } pub fn isKeepAlivePossible(this: *HTTPClient) bool { @@ -1747,6 +1748,8 @@ pub const AsyncHTTP = struct { this.response_headers = .{}; this.request = null; this.response = null; + this.client.unix_socket_path.deinit(); + this.client.unix_socket_path = JSC.ZigString.Slice.empty; } pub const State = enum(u32) { diff --git a/test/js/web/fetch/fetch.unix.test.ts b/test/js/web/fetch/fetch.unix.test.ts index 05142863295b..bf71566f56a0 100644 --- a/test/js/web/fetch/fetch.unix.test.ts +++ b/test/js/web/fetch/fetch.unix.test.ts @@ -6,10 +6,10 @@ import { join } from "path"; const tmp_dir = mkdtempSync(join(realpathSync(tmpdir()), "fetch.unix.test")); -let server: Server; +let server_unix: Server; function startServerUnix({ fetch, ...options }: ServeOptions): string { const socketPath = join(tmp_dir, `socket-${Math.random().toString(36).slice(2)}`); - server = serve({ + server_unix = serve({ ...options, fetch, unix: socketPath, @@ -17,7 +17,17 @@ function startServerUnix({ fetch, ...options }: ServeOptions): string { return socketPath; } +let server: Server; +function startServer({ fetch, ...options }: ServeOptions) { + server = serve({ + ...options, + fetch, + port: 0, + }); +} + afterEach(() => { + server_unix?.stop?.(true); server?.stop?.(true); }); @@ -38,3 +48,31 @@ it("provide body", async () => { expect(await response.text()).toBe(String(i)); } }); + +it("handle redirect to non-unix", async () => { + startServer({ + fetch(req) { + console.log('ip', req.url); + if (req.url.endsWith("/world")) { + return new Response(req.body); + } + return new Response(null, { status: 404 }); + } + }) + const path = startServerUnix({ + fetch(req) { + console.log('unix', req.url); + if (req.url.endsWith("/hello")) { + return new Response(null, { status: 302, headers: { Location: `http://${server.hostname}:${server.port}/world` } }); + } + return new Response(null, { status: 404 }); + }, + }); + // POST with body + for (let i = 0; i < 20; i++) { + const response = await fetch("http://localhost/hello", { method: "POST", body: String(i), unix: path, redirect: "follow", verbose: true }); + expect(response.status).toBe(200); + expect(await response.text()).toBe(String(i)); + } + +}); From 0fe77035920a00c14b50ca1cb793e28c63fe844d Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:16:37 +0000 Subject: [PATCH 07/10] [autofix.ci] apply automated fixes --- test/js/web/fetch/fetch.unix.test.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/test/js/web/fetch/fetch.unix.test.ts b/test/js/web/fetch/fetch.unix.test.ts index bf71566f56a0..634a590b1e6b 100644 --- a/test/js/web/fetch/fetch.unix.test.ts +++ b/test/js/web/fetch/fetch.unix.test.ts @@ -52,27 +52,35 @@ it("provide body", async () => { it("handle redirect to non-unix", async () => { startServer({ fetch(req) { - console.log('ip', req.url); + console.log("ip", req.url); if (req.url.endsWith("/world")) { return new Response(req.body); } return new Response(null, { status: 404 }); - } - }) + }, + }); const path = startServerUnix({ fetch(req) { - console.log('unix', req.url); + console.log("unix", req.url); if (req.url.endsWith("/hello")) { - return new Response(null, { status: 302, headers: { Location: `http://${server.hostname}:${server.port}/world` } }); + return new Response(null, { + status: 302, + headers: { Location: `http://${server.hostname}:${server.port}/world` }, + }); } return new Response(null, { status: 404 }); }, }); // POST with body for (let i = 0; i < 20; i++) { - const response = await fetch("http://localhost/hello", { method: "POST", body: String(i), unix: path, redirect: "follow", verbose: true }); + const response = await fetch("http://localhost/hello", { + method: "POST", + body: String(i), + unix: path, + redirect: "follow", + verbose: true, + }); expect(response.status).toBe(200); expect(await response.text()).toBe(String(i)); } - }); From e3f397fc88c2128e847010e9be89af51127ab5e8 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 6 Mar 2024 14:43:51 +0100 Subject: [PATCH 08/10] fix fetch redirect from unix to non-unix --- src/http.zig | 3 +++ test/js/web/fetch/fetch.unix.test.ts | 16 ++++------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/http.zig b/src/http.zig index 2dac5e090211..5630685b934c 100644 --- a/src/http.zig +++ b/src/http.zig @@ -2178,6 +2178,9 @@ pub fn buildRequest(this: *HTTPClient, body_len: usize) picohttp.Request { } pub fn doRedirect(this: *HTTPClient, comptime is_ssl: bool, ctx: *NewHTTPContext(is_ssl), socket: NewHTTPContext(is_ssl).HTTPSocket) void { + this.unix_socket_path.deinit(); + this.unix_socket_path = JSC.ZigString.Slice.empty; + this.state.response_message_buffer.deinit(); // we need to clean the client reference before closing the socket because we are going to reuse the same ref in a another request socket.ext(**anyopaque).?.* = bun.cast( diff --git a/test/js/web/fetch/fetch.unix.test.ts b/test/js/web/fetch/fetch.unix.test.ts index 634a590b1e6b..9a4a73cd5d5e 100644 --- a/test/js/web/fetch/fetch.unix.test.ts +++ b/test/js/web/fetch/fetch.unix.test.ts @@ -51,17 +51,15 @@ it("provide body", async () => { it("handle redirect to non-unix", async () => { startServer({ - fetch(req) { - console.log("ip", req.url); + async fetch(req) { if (req.url.endsWith("/world")) { - return new Response(req.body); + return new Response("world"); } return new Response(null, { status: 404 }); }, }); const path = startServerUnix({ fetch(req) { - console.log("unix", req.url); if (req.url.endsWith("/hello")) { return new Response(null, { status: 302, @@ -73,14 +71,8 @@ it("handle redirect to non-unix", async () => { }); // POST with body for (let i = 0; i < 20; i++) { - const response = await fetch("http://localhost/hello", { - method: "POST", - body: String(i), - unix: path, - redirect: "follow", - verbose: true, - }); + const response = await fetch("http://localhost/hello", { unix: path }); expect(response.status).toBe(200); - expect(await response.text()).toBe(String(i)); + expect(await response.text()).toBe("world"); } }); From 7afffd8174f6fa1fa28763d02df9bf1b6e857035 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 6 Mar 2024 14:48:31 +0100 Subject: [PATCH 09/10] disable keepalive for unix sockets (for now) so we don't leak them --- src/http.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/http.zig b/src/http.zig index 5630685b934c..5b54751c849f 100644 --- a/src/http.zig +++ b/src/http.zig @@ -1550,6 +1550,8 @@ pub fn deinit(this: *HTTPClient) void { pub fn isKeepAlivePossible(this: *HTTPClient) bool { if (comptime FeatureFlags.enable_keepalive) { + // TODO keepalive for unix sockets + if (this.unix_socket_path.length() > 0) return false; // is not possible to reuse Proxy with TSL, so disable keepalive if url is tunneling HTTPS if (this.http_proxy != null and this.url.isHTTPS()) { return false; From c0f6e345aef9883767c4d62dbfad4ebe38768625 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 6 Mar 2024 14:11:43 -0800 Subject: [PATCH 10/10] Update test/js/web/fetch/fetch.unix.test.ts Co-authored-by: Ciro Spaciari --- test/js/web/fetch/fetch.unix.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/js/web/fetch/fetch.unix.test.ts b/test/js/web/fetch/fetch.unix.test.ts index 9a4a73cd5d5e..5e42a3a3e3be 100644 --- a/test/js/web/fetch/fetch.unix.test.ts +++ b/test/js/web/fetch/fetch.unix.test.ts @@ -63,7 +63,7 @@ it("handle redirect to non-unix", async () => { if (req.url.endsWith("/hello")) { return new Response(null, { status: 302, - headers: { Location: `http://${server.hostname}:${server.port}/world` }, + headers: { Location: `${server.url.origin}/world` }, }); } return new Response(null, { status: 404 });