From 2ca7f251bf7c48650373494ecb9050d3b100fa6b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 22 Jun 2026 09:37:49 -0700 Subject: [PATCH 1/7] Add APIs for inferring remote endpoint type from URL. In https://github.com/rapidsai/cudf/pull/22739, cudf / cudf-polars needed to infer the remote endpoint type kvikio would use for some URL. This adds public APIs to kvikio to do that. --- cpp/include/kvikio/remote_handle.hpp | 12 ++ cpp/src/remote_handle.cpp | 169 ++++++++++-------- cpp/tests/test_remote_handle.cpp | 24 +++ python/kvikio/kvikio/__init__.py | 8 +- python/kvikio/kvikio/_lib/remote_handle.pyx | 14 +- python/kvikio/kvikio/remote_file.py | 8 +- .../tests/test_remote_endpoint_utils.py | 52 ++++++ 7 files changed, 213 insertions(+), 74 deletions(-) create mode 100644 python/kvikio/tests/test_remote_endpoint_utils.py diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 2e73323d5f..4831b5dc4c 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -330,6 +331,17 @@ class S3EndpointWithPresignedUrl : public RemoteEndpoint { static bool is_url_valid(std::string const& url) noexcept; }; +/** + * @brief Infer remote endpoint type from URL. + * + * This function follows the same endpoint-selection order as `RemoteHandle::open()` in + * `RemoteEndpointType::AUTO` mode, but only infers the endpoint type and does not create a handle. + * + * @param url The URL of the remote file. + * @return The inferred endpoint type. + */ +RemoteEndpointType infer_remote_endpoint_type(std::string url); + /** * @brief Handle of remote file. */ diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 271bf4eb89..10c33938fe 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -231,6 +232,89 @@ std::string encode_special_chars_in_path(std::string const& url) components.path = detail::UrlEncoder::encode_path(components.path.value()); return detail::UrlBuilder::build_manually(components); } + +std::vector get_default_allow_list() +{ + return {RemoteEndpointType::S3, + RemoteEndpointType::S3_PUBLIC, + RemoteEndpointType::S3_PRESIGNED_URL, + RemoteEndpointType::WEBHDFS, + RemoteEndpointType::HTTP}; +} + +std::unique_ptr create_endpoint_from_type(std::string const& url, + std::string const& scheme, + RemoteEndpointType type) +{ + switch (type) { + case RemoteEndpointType::S3: + if (!S3Endpoint::is_url_valid(url)) { return nullptr; } + if (scheme == "s3") { + auto const [bucket, object] = S3Endpoint::parse_s3_url(url); + return std::make_unique(std::pair{bucket, object}); + } + return std::make_unique(url); + + case RemoteEndpointType::S3_PUBLIC: + if (!S3PublicEndpoint::is_url_valid(url)) { return nullptr; } + return std::make_unique(url); + + case RemoteEndpointType::S3_PRESIGNED_URL: + if (!S3EndpointWithPresignedUrl::is_url_valid(url)) { return nullptr; } + return std::make_unique(url); + + case RemoteEndpointType::WEBHDFS: + if (!WebHdfsEndpoint::is_url_valid(url)) { return nullptr; } + return std::make_unique(url); + + case RemoteEndpointType::HTTP: + if (!HttpEndpoint::is_url_valid(url)) { return nullptr; } + return std::make_unique(url); + + default: return nullptr; + } +} + +struct InferredEndpoint { + RemoteEndpointType remote_endpoint_type{}; + std::unique_ptr endpoint; + std::optional probed_nbytes; +}; + +InferredEndpoint infer_endpoint_impl(std::string const& url, + std::vector const& allow_list, + bool probe_s3_connectivity) +{ + auto const scheme = + detail::UrlParser::extract_component(url, CURLUPART_SCHEME, CURLU_NON_SUPPORT_SCHEME); + KVIKIO_EXPECT(scheme.has_value(), "Missing scheme in URL."); + + for (auto const& type : allow_list) { + try { + auto endpoint = create_endpoint_from_type(url, scheme.value(), type); + if (endpoint == nullptr) { continue; } + + std::optional probed_nbytes = std::nullopt; + if (probe_s3_connectivity && type == RemoteEndpointType::S3) { + // Check connectivity for the credential-based S3 endpoint and reuse this size in + // RemoteHandle::open to avoid a second HEAD request. + probed_nbytes = endpoint->get_file_size(); + } + return InferredEndpoint{type, std::move(endpoint), probed_nbytes}; + } catch (...) { + // If the credential-based S3 endpoint cannot be used to access the URL, try using + // S3 public endpoint instead when it is in the allowlist. + if (type == RemoteEndpointType::S3 && + std::find(allow_list.begin(), allow_list.end(), RemoteEndpointType::S3_PUBLIC) != + allow_list.end()) { + return InferredEndpoint{ + RemoteEndpointType::S3_PUBLIC, std::make_unique(url), std::nullopt}; + } + throw; + } + } + KVIKIO_FAIL("Unsupported endpoint URL.", std::runtime_error); +} } // namespace RemoteEndpoint::RemoteEndpoint(RemoteEndpointType remote_endpoint_type) @@ -575,87 +659,27 @@ bool S3EndpointWithPresignedUrl::is_url_valid(std::string const& url) noexcept } } +RemoteEndpointType infer_remote_endpoint_type(std::string url) +{ + KVIKIO_NVTX_FUNC_RANGE(); + return infer_endpoint_impl(url, get_default_allow_list(), false).remote_endpoint_type; +} + RemoteHandle RemoteHandle::open(std::string url, RemoteEndpointType remote_endpoint_type, std::optional> allow_list, std::optional nbytes) { KVIKIO_NVTX_FUNC_RANGE(); - if (!allow_list.has_value()) { - allow_list = {RemoteEndpointType::S3, - RemoteEndpointType::S3_PUBLIC, - RemoteEndpointType::S3_PRESIGNED_URL, - RemoteEndpointType::WEBHDFS, - RemoteEndpointType::HTTP}; - } - - auto const scheme = - detail::UrlParser::extract_component(url, CURLUPART_SCHEME, CURLU_NON_SUPPORT_SCHEME); - KVIKIO_EXPECT(scheme.has_value(), "Missing scheme in URL."); - - // Helper to create endpoint based on type - auto create_endpoint = - [&url = url, &scheme = scheme](RemoteEndpointType type) -> std::unique_ptr { - switch (type) { - case RemoteEndpointType::S3: - if (!S3Endpoint::is_url_valid(url)) { return nullptr; } - if (scheme.value() == "s3") { - auto const [bucket, object] = S3Endpoint::parse_s3_url(url); - return std::make_unique(std::pair{bucket, object}); - } - return std::make_unique(url); - - case RemoteEndpointType::S3_PUBLIC: - if (!S3PublicEndpoint::is_url_valid(url)) { return nullptr; } - return std::make_unique(url); - - case RemoteEndpointType::S3_PRESIGNED_URL: - if (!S3EndpointWithPresignedUrl::is_url_valid(url)) { return nullptr; } - return std::make_unique(url); - - case RemoteEndpointType::WEBHDFS: - if (!WebHdfsEndpoint::is_url_valid(url)) { return nullptr; } - return std::make_unique(url); - - case RemoteEndpointType::HTTP: - if (!HttpEndpoint::is_url_valid(url)) { return nullptr; } - return std::make_unique(url); - - default: return nullptr; - } - }; + if (!allow_list.has_value()) { allow_list = get_default_allow_list(); } std::unique_ptr endpoint; std::optional probed_nbytes; if (remote_endpoint_type == RemoteEndpointType::AUTO) { - // Try each allowed type in the order of allowlist - for (auto const& type : allow_list.value()) { - try { - endpoint = create_endpoint(type); - if (endpoint == nullptr) { continue; } - if (type == RemoteEndpointType::S3) { - // Check connectivity for the credential-based S3 endpoint, and throw an exception if - // failed. Reuse this size when constructing the handle to avoid a second HEAD request. - probed_nbytes = endpoint->get_file_size(); - } - } catch (...) { - // If the credential-based S3 endpoint cannot be used to access the URL, try using S3 public - // endpoint instead if it is in the allowlist - if (type == RemoteEndpointType::S3 && - std::find(allow_list->begin(), allow_list->end(), RemoteEndpointType::S3_PUBLIC) != - allow_list->end()) { - endpoint = std::make_unique(url); - probed_nbytes = std::nullopt; - } else { - throw; - } - } - - // At this point, a matching endpoint has been found - break; - } - KVIKIO_EXPECT(endpoint.get() != nullptr, "Unsupported endpoint URL.", std::runtime_error); + auto inferred = infer_endpoint_impl(url, allow_list.value(), true); + endpoint = std::move(inferred.endpoint); + probed_nbytes = inferred.probed_nbytes; } else { // Validate it is in the allow list KVIKIO_EXPECT( @@ -665,7 +689,10 @@ RemoteHandle RemoteHandle::open(std::string url, std::runtime_error); // Create the specific type - endpoint = create_endpoint(remote_endpoint_type); + auto const scheme = + detail::UrlParser::extract_component(url, CURLUPART_SCHEME, CURLU_NON_SUPPORT_SCHEME); + KVIKIO_EXPECT(scheme.has_value(), "Missing scheme in URL."); + endpoint = create_endpoint_from_type(url, scheme.value(), remote_endpoint_type); KVIKIO_EXPECT(endpoint.get() != nullptr, std::string{"Invalid URL for "} + get_remote_endpoint_type_name(remote_endpoint_type) + " endpoint", diff --git a/cpp/tests/test_remote_handle.cpp b/cpp/tests/test_remote_handle.cpp index 1ecacad025..7aebacc683 100644 --- a/cpp/tests/test_remote_handle.cpp +++ b/cpp/tests/test_remote_handle.cpp @@ -312,3 +312,27 @@ TEST_F(RemoteHandleTest, test_open) } } } + +TEST_F(RemoteHandleTest, test_infer_remote_endpoint_type) +{ + kvikio::test::EnvVarContext env_var_ctx{{"AWS_DEFAULT_REGION", "my_aws_default_region"}, + {"AWS_ACCESS_KEY_ID", "my_aws_access_key_id"}, + {"AWS_SECRET_ACCESS_KEY", "my_aws_secrete_access_key"}}; + + EXPECT_EQ(kvikio::infer_remote_endpoint_type("s3://bucket-name/object-key-name"), + kvikio::RemoteEndpointType::S3); + EXPECT_EQ(kvikio::infer_remote_endpoint_type("https://host:1234/webhdfs/v1/data.bin"), + kvikio::RemoteEndpointType::WEBHDFS); + EXPECT_EQ(kvikio::infer_remote_endpoint_type("https://example.com/path/file.bin"), + kvikio::RemoteEndpointType::HTTP); + EXPECT_EQ(kvikio::infer_remote_endpoint_type( + "https://bucket-name.s3.region-code.amazonaws.com/" + "object-key-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=sig&" + "X-Amz-Credential=cred&X-Amz-SignedHeaders=host"), + kvikio::RemoteEndpointType::S3_PRESIGNED_URL); + + EXPECT_THAT([&] { kvikio::infer_remote_endpoint_type("unsupported://example.com/path"); }, + ThrowsMessage(HasSubstr("Unsupported endpoint URL"))); + EXPECT_THAT([&] { kvikio::infer_remote_endpoint_type("example.com/path"); }, + ThrowsMessage(HasSubstr("Bad scheme"))); +} diff --git a/python/kvikio/kvikio/__init__.py b/python/kvikio/kvikio/__init__.py index 24ab1fbe6f..ee9d43d197 100644 --- a/python/kvikio/kvikio/__init__.py +++ b/python/kvikio/kvikio/__init__.py @@ -23,7 +23,12 @@ get_page_cache_info, ) from kvikio.mmap import Mmap -from kvikio.remote_file import RemoteEndpointType, RemoteFile, is_remote_file_available +from kvikio.remote_file import ( + RemoteEndpointType, + RemoteFile, + infer_remote_endpoint_type, + is_remote_file_available, +) from kvikio.stream import stream_deregister, stream_register from kvikio.utils import kvikio_deprecation_notice @@ -36,6 +41,7 @@ "drop_system_page_cache", "Mmap", "get_page_cache_info", + "infer_remote_endpoint_type", "is_remote_file_available", "kvikio_deprecation_notice", "RemoteEndpointType", diff --git a/python/kvikio/kvikio/_lib/remote_handle.pyx b/python/kvikio/kvikio/_lib/remote_handle.pyx index 2f7031a7c3..49ca8d7bd7 100644 --- a/python/kvikio/kvikio/_lib/remote_handle.pyx +++ b/python/kvikio/kvikio/_lib/remote_handle.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # distutils: language = c++ @@ -60,6 +60,10 @@ cdef extern from "" namespace "kvikio" nogil: (cpp_RemoteEndpoint): cpp_S3EndpointWithPresignedUrl(string presigned_url) except + + RemoteEndpointType cpp_infer_remote_endpoint_type "kvikio::infer_remote_endpoint_type"( + string url + ) except + + cdef cppclass cpp_RemoteHandle "kvikio::RemoteHandle": cpp_RemoteHandle( unique_ptr[cpp_RemoteEndpoint] endpoint, size_t nbytes @@ -442,3 +446,11 @@ cdef class RemoteFile: ) return _wrap_io_future(fut) + + +def infer_remote_endpoint_type(url: str) -> RemoteEndpointType: + cdef string cpp_url = _to_string(url) + cdef RemoteEndpointType result + with nogil: + result = cpp_infer_remote_endpoint_type(cpp_url) + return result diff --git a/python/kvikio/kvikio/remote_file.py b/python/kvikio/kvikio/remote_file.py index 1faf010c58..36ed05814a 100644 --- a/python/kvikio/kvikio/remote_file.py +++ b/python/kvikio/kvikio/remote_file.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -84,6 +84,12 @@ def _get_remote_module(): return kvikio._lib.remote_handle +def infer_remote_endpoint_type(url: str) -> RemoteEndpointType: + """Infer endpoint type from URL using AUTO endpoint resolution rules.""" + result = _get_remote_module().infer_remote_endpoint_type(url) + return RemoteEndpointType[result.name] + + class RemoteFile: """File handle of a remote file.""" diff --git a/python/kvikio/tests/test_remote_endpoint_utils.py b/python/kvikio/tests/test_remote_endpoint_utils.py new file mode 100644 index 0000000000..fa5d30a645 --- /dev/null +++ b/python/kvikio/tests/test_remote_endpoint_utils.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +import kvikio + +pytestmark = pytest.mark.skipif( + not kvikio.is_remote_file_available(), + reason=( + "RemoteFile not available, please build KvikIO " + "with libcurl (-DKvikIO_REMOTE_SUPPORT=ON)" + ), +) + + +@pytest.fixture +def aws_env(monkeypatch): + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-access-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key") + + +def test_infer_remote_endpoint_type(aws_env): + assert ( + kvikio.infer_remote_endpoint_type("s3://bucket-name/object-key-name") + == kvikio.RemoteEndpointType.S3 + ) + assert ( + kvikio.infer_remote_endpoint_type("https://host:1234/webhdfs/v1/data.bin") + == kvikio.RemoteEndpointType.WEBHDFS + ) + assert ( + kvikio.infer_remote_endpoint_type("https://example.com/path/file.bin") + == kvikio.RemoteEndpointType.HTTP + ) + assert ( + kvikio.infer_remote_endpoint_type( + "https://bucket-name.s3.region-code.amazonaws.com/" + "object-key-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=sig&" + "X-Amz-Credential=cred&X-Amz-SignedHeaders=host" + ) + == kvikio.RemoteEndpointType.S3_PRESIGNED_URL + ) + + +def test_infer_remote_endpoint_type_invalid_url(): + with pytest.raises(RuntimeError, match="Bad scheme"): + kvikio.infer_remote_endpoint_type("example.com/path") + + with pytest.raises(RuntimeError, match="Unsupported endpoint URL"): + kvikio.infer_remote_endpoint_type("unsupported://example.com/path") From cbffb90b997bf9c83bfd8980e50e578b003713cc Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 22 Jun 2026 10:24:22 -0700 Subject: [PATCH 2/7] docs --- docs/source/api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/api.rst b/docs/source/api.rst index d7100afcd1..71897a0769 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -63,6 +63,8 @@ RemoteFile ---------- .. currentmodule:: kvikio.remote_file +.. autofunction:: infer_remote_endpoint_type + .. autoclass:: RemoteEndpointType .. autoclass:: RemoteFile From d81377878bc501f03b95458c003fe5d2e7298400 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 23 Jun 2026 06:40:50 -0700 Subject: [PATCH 3/7] const --- cpp/include/kvikio/remote_handle.hpp | 4 ++-- cpp/src/remote_handle.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 4831b5dc4c..792398f3ff 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -340,7 +340,7 @@ class S3EndpointWithPresignedUrl : public RemoteEndpoint { * @param url The URL of the remote file. * @return The inferred endpoint type. */ -RemoteEndpointType infer_remote_endpoint_type(std::string url); +RemoteEndpointType infer_remote_endpoint_type(std::string const& url); /** * @brief Handle of remote file. @@ -428,7 +428,7 @@ class RemoteHandle { * ); * @endcode */ - static RemoteHandle open(std::string url, + static RemoteHandle open(std::string const& url, RemoteEndpointType remote_endpoint_type = RemoteEndpointType::AUTO, std::optional> allow_list = std::nullopt, std::optional nbytes = std::nullopt); diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 10c33938fe..3f49720abc 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -659,13 +659,13 @@ bool S3EndpointWithPresignedUrl::is_url_valid(std::string const& url) noexcept } } -RemoteEndpointType infer_remote_endpoint_type(std::string url) +RemoteEndpointType infer_remote_endpoint_type(std::string const& url) { KVIKIO_NVTX_FUNC_RANGE(); return infer_endpoint_impl(url, get_default_allow_list(), false).remote_endpoint_type; } -RemoteHandle RemoteHandle::open(std::string url, +RemoteHandle RemoteHandle::open(std::string const& url, RemoteEndpointType remote_endpoint_type, std::optional> allow_list, std::optional nbytes) From 112ab8142ff6cac0f6fc0c97099e99f855b58fd9 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 23 Jun 2026 14:23:47 -0700 Subject: [PATCH 4/7] Review --- cpp/src/remote_handle.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 3f49720abc..d8720d67cc 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -233,13 +233,14 @@ std::string encode_special_chars_in_path(std::string const& url) return detail::UrlBuilder::build_manually(components); } -std::vector get_default_allow_list() -{ - return {RemoteEndpointType::S3, - RemoteEndpointType::S3_PUBLIC, - RemoteEndpointType::S3_PRESIGNED_URL, - RemoteEndpointType::WEBHDFS, - RemoteEndpointType::HTTP}; +std::vector const& get_default_allow_list() +{ + static std::vector const res{RemoteEndpointType::S3, + RemoteEndpointType::S3_PUBLIC, + RemoteEndpointType::S3_PRESIGNED_URL, + RemoteEndpointType::WEBHDFS, + RemoteEndpointType::HTTP}; + return res; } std::unique_ptr create_endpoint_from_type(std::string const& url, From 37b9ddbdf2cd690bcfd63ec8bfff5fffb354e39d Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 23 Jun 2026 17:33:43 -0700 Subject: [PATCH 5/7] Remove the struct --- cpp/src/remote_handle.cpp | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index d8720d67cc..9339681c30 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -276,15 +277,10 @@ std::unique_ptr create_endpoint_from_type(std::string const& url } } -struct InferredEndpoint { - RemoteEndpointType remote_endpoint_type{}; - std::unique_ptr endpoint; - std::optional probed_nbytes; -}; - -InferredEndpoint infer_endpoint_impl(std::string const& url, - std::vector const& allow_list, - bool probe_s3_connectivity) +std::pair, std::optional> infer_endpoint_impl( + std::string const& url, + std::vector const& allow_list, + bool probe_s3_connectivity) { auto const scheme = detail::UrlParser::extract_component(url, CURLUPART_SCHEME, CURLU_NON_SUPPORT_SCHEME); @@ -301,15 +297,14 @@ InferredEndpoint infer_endpoint_impl(std::string const& url, // RemoteHandle::open to avoid a second HEAD request. probed_nbytes = endpoint->get_file_size(); } - return InferredEndpoint{type, std::move(endpoint), probed_nbytes}; + return {std::move(endpoint), probed_nbytes}; } catch (...) { // If the credential-based S3 endpoint cannot be used to access the URL, try using // S3 public endpoint instead when it is in the allowlist. if (type == RemoteEndpointType::S3 && std::find(allow_list.begin(), allow_list.end(), RemoteEndpointType::S3_PUBLIC) != allow_list.end()) { - return InferredEndpoint{ - RemoteEndpointType::S3_PUBLIC, std::make_unique(url), std::nullopt}; + return {std::make_unique(url), std::nullopt}; } throw; } @@ -663,7 +658,9 @@ bool S3EndpointWithPresignedUrl::is_url_valid(std::string const& url) noexcept RemoteEndpointType infer_remote_endpoint_type(std::string const& url) { KVIKIO_NVTX_FUNC_RANGE(); - return infer_endpoint_impl(url, get_default_allow_list(), false).remote_endpoint_type; + std::unique_ptr endpoint; + std::tie(endpoint, std::ignore) = infer_endpoint_impl(url, get_default_allow_list(), false); + return endpoint->remote_endpoint_type(); } RemoteHandle RemoteHandle::open(std::string const& url, @@ -679,8 +676,8 @@ RemoteHandle RemoteHandle::open(std::string const& url, if (remote_endpoint_type == RemoteEndpointType::AUTO) { auto inferred = infer_endpoint_impl(url, allow_list.value(), true); - endpoint = std::move(inferred.endpoint); - probed_nbytes = inferred.probed_nbytes; + endpoint = std::move(inferred.first); + probed_nbytes = inferred.second; } else { // Validate it is in the allow list KVIKIO_EXPECT( From b752f5abe158ac792328d9d828053f44d7a7c552 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 23 Jun 2026 17:36:59 -0700 Subject: [PATCH 6/7] Document public / private --- cpp/include/kvikio/remote_handle.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 792398f3ff..a14d3a1415 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -336,6 +336,9 @@ class S3EndpointWithPresignedUrl : public RemoteEndpoint { * * This function follows the same endpoint-selection order as `RemoteHandle::open()` in * `RemoteEndpointType::AUTO` mode, but only infers the endpoint type and does not create a handle. + * Note that this function will not return `RemoteEndpointType::S3_PUBLIC`, because disambiguating + * between a URL that's accessible only with authorization or only anonymously is not possible + * without making an HTTP request. * * @param url The URL of the remote file. * @return The inferred endpoint type. From e91aed6bbe31ed0a599582262410cbb3fd725394 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 24 Jun 2026 06:12:17 -0700 Subject: [PATCH 7/7] Try with _ --- cpp/src/remote_handle.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 9339681c30..335cf700b5 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -658,7 +658,7 @@ bool S3EndpointWithPresignedUrl::is_url_valid(std::string const& url) noexcept RemoteEndpointType infer_remote_endpoint_type(std::string const& url) { KVIKIO_NVTX_FUNC_RANGE(); - std::unique_ptr endpoint; + auto [endpoint, _] = infer_endpoint_impl(url, get_default_allow_list(), false); std::tie(endpoint, std::ignore) = infer_endpoint_impl(url, get_default_allow_list(), false); return endpoint->remote_endpoint_type(); }