From 5ac10221a64a6a49d29f3f09851c17f869e1807d Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Thu, 12 Feb 2026 21:45:34 +0000 Subject: [PATCH 1/2] fix(libstore/filetransfer): enable TCP keep-alive on curl handles Idle connections in libcurl's connection pool can be silently dropped by the OS or intermediate firewalls/NATs before they can be reused, forcing new TCP connections to be created. This is especially problematic for HTTP/1.1 endpoints where multiplexing is unavailable. Enable TCP keep-alive with a 60-second idle/interval on all curl easy handles to prevent idle connection drops and improve connection reuse. (cherry picked from commit 736abd50ff5ed6e41b20091371662c6581a8a7cd) --- src/libstore/filetransfer.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8a7577146ef..9aafc1c0d14 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -479,6 +479,10 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L); + curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 60L); + curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 60L); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); From 695501815b0f8fb6e51ccf6598701fc459f9328f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Thu, 12 Feb 2026 21:51:01 +0000 Subject: [PATCH 2/2] feat(libstore/s3): use virtual-hosted-style URLs and add addressing-style option S3 binary caches now use virtual-hosted-style URLs by default for standard AWS endpoints. Path-style endpoints (s3.region.amazonaws.com) only serve HTTP/1.1, preventing HTTP/2 multiplexing and causing TCP TIME_WAIT socket exhaustion under high concurrency. Virtual-hosted-style endpoints (bucket.s3.region.amazonaws.com) support HTTP/2, enabling multiplexing with the existing CURLPIPE_MULTIPLEX configuration. Add a new `addressing-style` store option (auto/path/virtual) to control this behavior. `auto` (default) uses virtual-hosted-style for standard AWS endpoints and path-style for custom endpoints. `path` forces path-style for backwards compatibility. `virtual` forces virtual-hosted- style for all endpoints including custom ones. Fixes: https://github.com/NixOS/nix/issues/15208 (cherry picked from commit 759f6c856b5668da66ce7619aa8eed3096e6ed70) --- doc/manual/rl-next/s3-virtual-hosted-style.md | 32 +++ src/libstore-tests/s3-url.cc | 252 ++++++++++++++++-- src/libstore/filetransfer.cc | 2 + .../nix/store/s3-binary-cache-store.hh | 24 +- src/libstore/include/nix/store/s3-url.hh | 29 +- src/libstore/s3-url.cc | 146 +++++++++- tests/nixos/s3-binary-cache-store.nix | 98 +++++++ 7 files changed, 551 insertions(+), 32 deletions(-) create mode 100644 doc/manual/rl-next/s3-virtual-hosted-style.md diff --git a/doc/manual/rl-next/s3-virtual-hosted-style.md b/doc/manual/rl-next/s3-virtual-hosted-style.md new file mode 100644 index 00000000000..e8c2d9766d4 --- /dev/null +++ b/doc/manual/rl-next/s3-virtual-hosted-style.md @@ -0,0 +1,32 @@ +--- +synopsis: S3 binary caches now use virtual-hosted-style addressing by default +issues: [15208] +--- + +S3 binary caches now use virtual-hosted-style URLs +(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs +(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS +S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection +exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads. + +A new `addressing-style` store option controls this behavior: + +- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style + for custom endpoints. +- `path`: forces path-style addressing (deprecated by AWS). +- `virtual`: forces virtual-hosted-style addressing (bucket names must not + contain dots). + +Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back +to path-style addressing in `auto` mode, because dotted names create +multi-level subdomains that break TLS wildcard certificate validation. + +Example using path-style for backwards compatibility: + +``` +s3://my-bucket/key?region=us-east-1&addressing-style=path +``` + +Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing +idle connections from being silently dropped by intermediate network devices +(NATs, firewalls, load balancers). diff --git a/src/libstore-tests/s3-url.cc b/src/libstore-tests/s3-url.cc index 9fa625fd6c7..cd68b0437a2 100644 --- a/src/libstore-tests/s3-url.cc +++ b/src/libstore-tests/s3-url.cc @@ -104,6 +104,33 @@ INSTANTIATE_TEST_SUITE_P( }, }, "with_absolute_endpoint_uri", + }, + ParsedS3URLTestCase{ + "s3://bucket/key?addressing-style=virtual", + { + .bucket = "bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + }, + "with_addressing_style_virtual", + }, + ParsedS3URLTestCase{ + "s3://bucket/key?addressing-style=path", + { + .bucket = "bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Path, + }, + "with_addressing_style_path", + }, + ParsedS3URLTestCase{ + "s3://bucket/key?addressing-style=auto", + { + .bucket = "bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Auto, + }, + "with_addressing_style_auto", }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); @@ -138,6 +165,26 @@ INSTANTIATE_TEST_SUITE_P( InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}), [](const ::testing::TestParamInfo & info) { return info.param.description; }); +TEST(ParsedS3URLTest, invalidAddressingStyleThrows) +{ + ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle); +} + +TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows) +{ + ParsedS3URL input{ + .bucket = "bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + .endpoint = + ParsedURL{ + .scheme = "file", + .path = {"", "some", "path"}, + }, + }; + ASSERT_THROW(input.toHttpsUrl(), nix::Error); +} + // ============================================================================= // S3 URL to HTTPS Conversion Tests // ============================================================================= @@ -166,6 +213,7 @@ INSTANTIATE_TEST_SUITE_P( S3ToHttpsConversion, S3ToHttpsConversionTest, ::testing::Values( + // Default (auto) addressing style: virtual-hosted for standard AWS endpoints S3ToHttpsConversionTestCase{ ParsedS3URL{ .bucket = "my-bucket", @@ -173,10 +221,10 @@ INSTANTIATE_TEST_SUITE_P( }, ParsedURL{ .scheme = "https", - .authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"}, - .path = {"", "my-bucket", "my-key.txt"}, + .authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"}, + .path = {"", "my-key.txt"}, }, - "https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt", + "https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt", "basic_s3_default_region", }, S3ToHttpsConversionTestCase{ @@ -187,12 +235,13 @@ INSTANTIATE_TEST_SUITE_P( }, ParsedURL{ .scheme = "https", - .authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"}, - .path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"}, + .authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"}, + .path = {"", "nix", "store", "abc123.nar.xz"}, }, - "https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz", + "https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz", "with_eu_west_1_region", }, + // Custom endpoint authority: path-style by default S3ToHttpsConversionTestCase{ ParsedS3URL{ .bucket = "bucket", @@ -208,6 +257,7 @@ INSTANTIATE_TEST_SUITE_P( "http://custom.s3.com/bucket/key", "custom_endpoint_authority", }, + // Custom endpoint URL: path-style by default S3ToHttpsConversionTestCase{ ParsedS3URL{ .bucket = "bucket", @@ -236,10 +286,10 @@ INSTANTIATE_TEST_SUITE_P( }, ParsedURL{ .scheme = "https", - .authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"}, - .path = {"", "bucket", "path", "to", "file.txt"}, + .authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"}, + .path = {"", "path", "to", "file.txt"}, }, - "https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt", + "https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt", "complex_path_and_region", }, S3ToHttpsConversionTestCase{ @@ -250,11 +300,11 @@ INSTANTIATE_TEST_SUITE_P( }, ParsedURL{ .scheme = "https", - .authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"}, - .path = {"", "my-bucket", "my-key.txt"}, + .authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"}, + .path = {"", "my-key.txt"}, .query = {{"versionId", "abc123xyz"}}, }, - "https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz", + "https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz", "with_versionId", }, S3ToHttpsConversionTestCase{ @@ -266,13 +316,185 @@ INSTANTIATE_TEST_SUITE_P( }, ParsedURL{ .scheme = "https", - .authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"}, - .path = {"", "versioned-bucket", "path", "to", "object"}, + .authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"}, + .path = {"", "path", "to", "object"}, .query = {{"versionId", "version456"}}, }, - "https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456", + "https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456", "with_region_and_versionId", + }, + // Explicit addressing-style=path forces path-style on standard AWS endpoints + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "my-bucket", + .key = {"my-key.txt"}, + .region = "us-west-2", + .addressingStyle = S3AddressingStyle::Path, + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"}, + .path = {"", "my-bucket", "my-key.txt"}, + }, + "https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt", + "explicit_path_style", + }, + // Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "bucket", + .key = {"key"}, + .scheme = "http", + .addressingStyle = S3AddressingStyle::Virtual, + .endpoint = ParsedURL::Authority{.host = "custom.s3.com"}, + }, + ParsedURL{ + .scheme = "http", + .authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"}, + .path = {"", "key"}, + }, + "http://bucket.custom.s3.com/key", + "explicit_virtual_style_custom_endpoint", + }, + // Explicit addressing-style=virtual with full endpoint URL + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + .endpoint = + ParsedURL{ + .scheme = "http", + .authority = ParsedURL::Authority{.host = "server", .port = 9000}, + .path = {""}, + }, + }, + ParsedURL{ + .scheme = "http", + .authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000}, + .path = {"", "key"}, + }, + "http://bucket.server:9000/key", + "explicit_virtual_style_full_endpoint_url", + }, + // Dotted bucket names work normally with explicit path-style + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "my.bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Path, + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"}, + .path = {"", "my.bucket", "key"}, + }, + "https://s3.us-east-1.amazonaws.com/my.bucket/key", + "dotted_bucket_with_path_style", + }, + // Dotted bucket names fall back to path-style with auto on standard AWS endpoints + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "my.bucket.name", + .key = {"key"}, + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"}, + .path = {"", "my.bucket.name", "key"}, + }, + "https://s3.us-east-1.amazonaws.com/my.bucket.name/key", + "dotted_bucket_with_auto_style_on_aws", + }, + // Dotted bucket names work with auto style on custom endpoints (auto = path-style) + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "my.bucket", + .key = {"key"}, + .endpoint = ParsedURL::Authority{.host = "minio.local"}, + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "minio.local"}, + .path = {"", "my.bucket", "key"}, + }, + "https://minio.local/my.bucket/key", + "dotted_bucket_with_auto_style_custom_endpoint", }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); +// ============================================================================= +// S3 URL to HTTPS Conversion Error Tests +// ============================================================================= + +struct S3ToHttpsConversionErrorTestCase +{ + ParsedS3URL input; + std::string description; +}; + +class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface, + public ::testing::Test +{}; + +TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion) +{ + auto & [input, description] = GetParam(); + ASSERT_THROW(input.toHttpsUrl(), nix::Error); +} + +INSTANTIATE_TEST_SUITE_P( + S3ToHttpsConversionErrors, + S3ToHttpsConversionErrorTest, + ::testing::Values( + S3ToHttpsConversionErrorTestCase{ + ParsedS3URL{ + .bucket = "bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + .endpoint = ParsedURL::Authority{.host = ""}, + }, + "virtual_style_with_empty_host_authority", + }, + S3ToHttpsConversionErrorTestCase{ + ParsedS3URL{ + .bucket = "my.bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + }, + "dotted_bucket_with_explicit_virtual_style", + }, + S3ToHttpsConversionErrorTestCase{ + ParsedS3URL{ + .bucket = "my.bucket.name", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + }, + "dotted_bucket_with_explicit_virtual_style_multi_dot", + }, + S3ToHttpsConversionErrorTestCase{ + ParsedS3URL{ + .bucket = "my.bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + .endpoint = ParsedURL::Authority{.host = "minio.local"}, + }, + "dotted_bucket_with_explicit_virtual_style_custom_authority", + }, + S3ToHttpsConversionErrorTestCase{ + ParsedS3URL{ + .bucket = "my.bucket", + .key = {"key"}, + .addressingStyle = S3AddressingStyle::Virtual, + .endpoint = + ParsedURL{ + .scheme = "http", + .authority = ParsedURL::Authority{.host = "minio.local", .port = 9000}, + .path = {""}, + }, + }, + "dotted_bucket_with_explicit_virtual_style_full_endpoint_url", + }), + [](const ::testing::TestParamInfo & info) { return info.param.description; }); + } // namespace nix diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 9aafc1c0d14..a08468002bb 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -479,6 +479,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); + // Enable TCP keep-alive so that idle connections in curl's reuse pool + // are not silently dropped by NATs, firewalls, or load balancers. curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L); curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 60L); curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 60L); diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 5896293f1c4..e679035e461 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -3,6 +3,7 @@ #include "nix/store/config.hh" #include "nix/store/http-binary-cache-store.hh" +#include "nix/store/s3-url.hh" namespace nix { @@ -52,13 +53,22 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig "endpoint", R"( The S3 endpoint to use. When empty (default), uses AWS S3 with - region-specific endpoints (e.g., s3.us-east-1.amazonaws.com). - For S3-compatible services such as MinIO, set this to your service's endpoint. + region-specific endpoints. For S3-compatible services such as + MinIO, set this to your service's endpoint. + )"}; - > **Note** - > - > Custom endpoints must support HTTPS and use path-based - > addressing instead of virtual host based addressing. + Setting addressingStyle{ + this, + S3AddressingStyle::Auto, + "addressing-style", + R"( + The S3 addressing style to use. `auto` (default) uses + virtual-hosted-style for standard AWS endpoints and path-style + for custom endpoints; bucket names containing dots automatically + fall back to path-style to avoid TLS certificate errors. `path` + forces path-style addressing (deprecated by AWS). `virtual` + forces virtual-hosted-style addressing (bucket names must not + contain dots). )"}; const Setting multipartUpload{ @@ -117,7 +127,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig * Set of settings that are part of the S3 URI itself. * These are needed for region specification and other S3-specific settings. */ - const std::set s3UriSettings = {&profile, ®ion, &scheme, &endpoint}; + const std::set s3UriSettings = {&profile, ®ion, &scheme, &endpoint, &addressingStyle}; static const std::string name() { diff --git a/src/libstore/include/nix/store/s3-url.hh b/src/libstore/include/nix/store/s3-url.hh index cf59dbea86a..53371074522 100644 --- a/src/libstore/include/nix/store/s3-url.hh +++ b/src/libstore/include/nix/store/s3-url.hh @@ -1,16 +1,41 @@ #pragma once ///@file #include "nix/store/config.hh" +#include "nix/util/error.hh" #include "nix/util/url.hh" #include "nix/util/util.hh" #include #include +#include #include #include namespace nix { +/** + * S3 addressing style for bucket access. + * - Auto: virtual-hosted-style for standard AWS endpoints, path-style for custom endpoints. + * - Path: always use path-style (bucket in URL path). + * - Virtual: always use virtual-hosted-style (bucket as hostname prefix; bucket name must not contain dots). + */ +enum class S3AddressingStyle { + Auto, + Path, + Virtual, +}; + +MakeError(InvalidS3AddressingStyle, Error); + +S3AddressingStyle parseS3AddressingStyle(std::string_view style); +std::string_view showS3AddressingStyle(S3AddressingStyle style); + +template<> +S3AddressingStyle BaseSetting::parse(const std::string & str) const; + +template<> +std::string BaseSetting::to_string() const; + /** * Parsed S3 URL. */ @@ -27,6 +52,7 @@ struct ParsedS3URL std::optional region; std::optional scheme; std::optional versionId; + std::optional addressingStyle; /** * The endpoint can be either missing, be an absolute URI (with a scheme like `http:`) * or an authority (so an IP address or a registered name). @@ -46,7 +72,8 @@ struct ParsedS3URL static ParsedS3URL parse(const ParsedURL & uri); /** - * Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication + * Convert this ParsedS3URL to an HTTP(S) ParsedURL for use with curl's AWS SigV4 authentication. + * The scheme defaults to HTTPS but respects the 'scheme' setting and custom endpoint schemes. */ ParsedURL toHttpsUrl() const; diff --git a/src/libstore/s3-url.cc b/src/libstore/s3-url.cc index 503c0cd9105..e6b5553661a 100644 --- a/src/libstore/s3-url.cc +++ b/src/libstore/s3-url.cc @@ -1,8 +1,14 @@ #include "nix/store/s3-url.hh" +#include "nix/util/abstract-setting-to-json.hh" +#include "nix/util/config-impl.hh" #include "nix/util/error.hh" +#include "nix/util/logging.hh" +#include "nix/util/json-impls.hh" #include "nix/util/split.hh" #include "nix/util/strings-inline.hh" +#include +#include #include #include @@ -10,6 +16,30 @@ using namespace std::string_view_literals; namespace nix { +S3AddressingStyle parseS3AddressingStyle(std::string_view style) +{ + if (style == "auto") + return S3AddressingStyle::Auto; + if (style == "path") + return S3AddressingStyle::Path; + if (style == "virtual") + return S3AddressingStyle::Virtual; + throw InvalidS3AddressingStyle("unknown S3 addressing style '%s', expected 'auto', 'path', or 'virtual'", style); +} + +std::string_view showS3AddressingStyle(S3AddressingStyle style) +{ + switch (style) { + case S3AddressingStyle::Auto: + return "auto"; + case S3AddressingStyle::Path: + return "path"; + case S3AddressingStyle::Virtual: + return "virtual"; + } + unreachable(); +} + ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed) try { if (parsed.scheme != "s3"sv) @@ -49,6 +79,9 @@ try { .region = getOptionalParam("region"), .scheme = getOptionalParam("scheme"), .versionId = getOptionalParam("versionId"), + .addressingStyle = getOptionalParam("addressing-style").transform([](const std::string & s) { + return parseS3AddressingStyle(s); + }), .endpoint = [&]() -> decltype(ParsedS3URL::endpoint) { if (!endpoint) return std::monostate(); @@ -65,6 +98,9 @@ try { } catch (BadURL & e) { e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string()); throw; +} catch (InvalidS3AddressingStyle & e) { + e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string()); + throw; } ParsedURL ParsedS3URL::toHttpsUrl() const @@ -80,41 +116,95 @@ ParsedURL ParsedS3URL::toHttpsUrl() const queryParams["versionId"] = *versionId; } + auto style = addressingStyle.value_or(S3AddressingStyle::Auto); + + // Virtual-hosted-style prepends the bucket name to the hostname, so bucket + // names containing dots produce multi-level subdomains (e.g. + // my.bucket.s3.amazonaws.com) that break TLS wildcard certificate validation. + // In auto mode, fall back to path-style; only error on explicit virtual. + auto hasDottedBucket = bucket.find('.') != std::string::npos; + auto useVirtualForEndpoint = [&](bool defaultVirtual) { + auto useVirtual = defaultVirtual ? style != S3AddressingStyle::Path : style == S3AddressingStyle::Virtual; + if (useVirtual && hasDottedBucket) { + if (style == S3AddressingStyle::Virtual) + throw Error( + "bucket name '%s' contains a dot, which is incompatible with " + "virtual-hosted-style addressing (causes TLS certificate errors); " + "use 'addressing-style=path' or 'addressing-style=auto' instead", + bucket); + static std::atomic warnedDottedBucket{false}; + warnOnce( + warnedDottedBucket, + "bucket name '%s' contains a dot; falling back to path-style addressing " + "(virtual-hosted-style requires non-dotted bucket names for TLS certificate validity); " + "set 'addressing-style=path' to silence this warning", + bucket); + return false; + } + return useVirtual; + }; + // Handle endpoint configuration using std::visit return std::visit( overloaded{ [&](const std::monostate &) { - // No custom endpoint, use standard AWS S3 endpoint + // No custom endpoint: use virtual-hosted-style by default (auto), + // path-style when explicitly requested or for dotted bucket names. + auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ true); std::vector path{""}; - path.push_back(bucket); + if (!useVirtual) + path.push_back(bucket); path.insert(path.end(), key.begin(), key.end()); return ParsedURL{ .scheme = std::string{schemeStr}, - .authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"}, + .authority = + ParsedURL::Authority{ + .host = useVirtual ? bucket + ".s3." + regionStr + ".amazonaws.com" + : "s3." + regionStr + ".amazonaws.com"}, .path = std::move(path), .query = std::move(queryParams), }; }, [&](const ParsedURL::Authority & auth) { - // Endpoint is just an authority (hostname/port) + // Custom endpoint authority: use path-style by default (auto), + // virtual-hosted-style only when explicitly requested (not for dotted buckets). + auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false); + if (useVirtual && auth.host.empty()) + throw Error( + "cannot use virtual-hosted-style addressing with endpoint '%s' " + "because it has no hostname; use 'addressing-style=path' instead", + auth.to_string()); std::vector path{""}; - path.push_back(bucket); + if (!useVirtual) + path.push_back(bucket); path.insert(path.end(), key.begin(), key.end()); return ParsedURL{ .scheme = std::string{schemeStr}, - .authority = auth, + .authority = + useVirtual ? ParsedURL::Authority{.host = bucket + "." + auth.host, .port = auth.port} : auth, .path = std::move(path), .query = std::move(queryParams), }; }, [&](const ParsedURL & endpointUrl) { - // Endpoint is already a ParsedURL (e.g., http://server:9000) + // Full endpoint URL: use path-style by default (auto), + // virtual-hosted-style only when explicitly requested (not for dotted buckets). + auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false); + if (useVirtual && (!endpointUrl.authority || endpointUrl.authority->host.empty())) + throw Error( + "cannot use virtual-hosted-style addressing with endpoint '%s' " + "because it has no authority (hostname)", + endpointUrl.to_string()); auto path = endpointUrl.path; - path.push_back(bucket); + if (!useVirtual) + path.push_back(bucket); path.insert(path.end(), key.begin(), key.end()); return ParsedURL{ .scheme = endpointUrl.scheme, - .authority = endpointUrl.authority, + .authority = useVirtual ? std::optional{ParsedURL::Authority{ + .host = bucket + "." + endpointUrl.authority->host, + .port = endpointUrl.authority->port}} + : endpointUrl.authority, .path = std::move(path), .query = std::move(queryParams), }; @@ -123,4 +213,42 @@ ParsedURL ParsedS3URL::toHttpsUrl() const endpoint); } +void to_json(nlohmann::json & j, const S3AddressingStyle & e) +{ + j = std::string{showS3AddressingStyle(e)}; +} + +void from_json(const nlohmann::json & j, S3AddressingStyle & e) +{ + e = parseS3AddressingStyle(j.get()); +} + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +S3AddressingStyle BaseSetting::parse(const std::string & str) const +{ + try { + return parseS3AddressingStyle(str); + } catch (InvalidS3AddressingStyle &) { + throw UsageError("option '%s' has invalid value '%s', expected 'auto', 'path', or 'virtual'", name, str); + } +} + +template<> +std::string BaseSetting::to_string() const +{ + return std::string{showS3AddressingStyle(value)}; +} + +template<> +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template class BaseSetting; + } // namespace nix diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 5804057487d..57f04aa5cf6 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -48,9 +48,14 @@ in rootCredentialsFile = pkgs.writeText "minio-credentials-full" '' MINIO_ROOT_USER=${accessKey} MINIO_ROOT_PASSWORD=${secretKey} + MINIO_DOMAIN=minio.local ''; }; networking.firewall.allowedTCPPorts = [ 9000 ]; + # Static hosts for virtual-hosted-style S3 tests. + # MinIO with MINIO_DOMAIN=minio.local accepts virtual-hosted requests + # where the bucket name is a hostname prefix. + networking.extraHosts = "127.0.0.1 vhost-test.minio.local minio.local"; }; client = @@ -62,6 +67,7 @@ in experimental-features = nix-command substituters = ''; + networking.extraHosts = "192.168.1.2 vhost-test.minio.local minio.local"; }; }; @@ -83,6 +89,10 @@ in ENDPOINT = 'http://server:9000' REGION = 'eu-west-1' + # Virtual-hosted-style configuration (requires MINIO_DOMAIN and static host entries) + VHOST_DOMAIN = 'minio.local' + VHOST_ENDPOINT = f'http://{VHOST_DOMAIN}:9000' + PKGS = { 'A': '${pkgA}', 'B': '${pkgB}', @@ -859,6 +869,92 @@ in print(output) raise Exception("Expected SSO provider to be skipped") + def test_virtual_hosted_copy(): + """Test nix copy with virtual-hosted-style addressing on custom endpoint""" + print("\n=== Testing Virtual-Hosted-Style Addressing ===") + + # Use a fixed bucket name matching the static /etc/hosts entries + bucket = 'vhost-test' + server.succeed(f"mc mb minio/{bucket}") + try: + store_url = make_s3_url( + bucket, + endpoint=VHOST_ENDPOINT, + **{'addressing-style': 'virtual'} + ) + + # Upload with virtual-hosted-style, capture debug output + output = server.succeed( + f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1" + ) + + # Verify virtual-hosted-style URL was used (bucket in hostname) + vhost_url_prefix = f"http://{bucket}.{VHOST_DOMAIN}:9000/" + if vhost_url_prefix not in output: + print("Debug output:") + print(output) + raise Exception( + f"Expected virtual-hosted-style URL containing '{vhost_url_prefix}'" + ) + + # Verify path-style URL was NOT used (bucket should not be in the path) + path_style_pattern = f"{VHOST_ENDPOINT}/{bucket}/" + if path_style_pattern in output: + print("Debug output:") + print(output) + raise Exception("Found path-style URL when virtual-hosted-style was expected") + + # Download with virtual-hosted-style + verify_packages_in_store(client, PKGS['A'], should_exist=False) + output = client.succeed( + f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs " + f"--from '{store_url}' {PKGS['A']} 2>&1" + ) + + if vhost_url_prefix not in output: + print("Debug output:") + print(output) + raise Exception( + f"Expected virtual-hosted-style URL in download containing '{vhost_url_prefix}'" + ) + + verify_packages_in_store(client, PKGS['A']) + finally: + server.succeed(f"mc rb --force minio/{bucket}") + for pkg in PKGS.values(): + client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}") + + @setup_s3() + def test_explicit_path_style(bucket): + """Test that addressing-style=path works as backwards-compatible fallback""" + print("\n=== Testing Explicit Path-Style Addressing ===") + + store_url = make_s3_url( + bucket, + **{'addressing-style': 'path'} + ) + + # Upload with explicit path-style + output = server.succeed( + f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1" + ) + + # Verify path-style URL was used (bucket in path, not hostname) + path_style_pattern = f"{ENDPOINT}/{bucket}/" + if path_style_pattern not in output: + print("Debug output:") + print(output) + raise Exception( + f"Expected path-style URL containing '{path_style_pattern}'" + ) + + # Download + verify_packages_in_store(client, PKGS['A'], should_exist=False) + client.succeed( + f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}" + ) + verify_packages_in_store(client, PKGS['A']) + # ============================================================================ # Main Test Execution # ============================================================================ @@ -896,5 +992,7 @@ in test_profile_credentials() test_env_vars_precedence() test_credential_provider_chain() + test_virtual_hosted_copy() + test_explicit_path_style() ''; }