diff --git a/duckdb b/duckdb index b8a06e4..24c295a 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit b8a06e4a22672e254cd0baa68a3dbed2eb51c56e +Subproject commit 24c295a9dd3f793222fc965f2f41ade148854526 diff --git a/src/httpfs.cpp b/src/httpfs.cpp index 802581e..a11af95 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -729,7 +729,7 @@ void HTTPFileHandle::LoadFileInfo() { return; } else { // HEAD request fail, use Range request for another try (read only one byte) - if (flags.OpenForReading() && res->status != HTTPStatusCode::NotFound_404) { + if (flags.OpenForReading() && res->status != HTTPStatusCode::NotFound_404 && res->status != HTTPStatusCode::MovedPermanently_301) { auto range_res = hfs.GetRangeRequest(*this, path, {}, 0, nullptr, 2); if (range_res->status != HTTPStatusCode::PartialContent_206 && range_res->status != HTTPStatusCode::Accepted_202 && range_res->status != HTTPStatusCode::OK_200) { diff --git a/src/httpfs_extension.cpp b/src/httpfs_extension.cpp index 79d9923..9070621 100644 --- a/src/httpfs_extension.cpp +++ b/src/httpfs_extension.cpp @@ -70,7 +70,7 @@ static void LoadInternal(ExtensionLoader &loader) { config.AddExtensionOption("ca_cert_file", "Path to a custom certificate file for self-signed certificates.", LogicalType::VARCHAR, Value("")); // Global S3 config - config.AddExtensionOption("s3_region", "S3 Region", LogicalType::VARCHAR, Value("us-east-1")); + config.AddExtensionOption("s3_region", "S3 Region", LogicalType::VARCHAR); config.AddExtensionOption("s3_access_key_id", "S3 Access Key ID", LogicalType::VARCHAR); config.AddExtensionOption("s3_secret_access_key", "S3 Access Key", LogicalType::VARCHAR); config.AddExtensionOption("s3_session_token", "S3 Session Token", LogicalType::VARCHAR); diff --git a/src/include/s3fs.hpp b/src/include/s3fs.hpp index 525e0dd..a7e933e 100644 --- a/src/include/s3fs.hpp +++ b/src/include/s3fs.hpp @@ -231,7 +231,7 @@ class S3FileSystem : public HTTPFileSystem { return true; } - static string GetS3BadRequestError(S3AuthParams &s3_auth_params); + static string GetS3BadRequestError(S3AuthParams &s3_auth_params, string correct_region = ""); static string GetS3AuthError(S3AuthParams &s3_auth_params); static string GetGCSAuthError(S3AuthParams &s3_auth_params); static HTTPException GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index cbdecba..72eddc3 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -872,6 +872,7 @@ void S3FileHandle::Initialize(optional_ptr opener) { ErrorData error(ex); bool refreshed_secret = false; if (error.Type() == ExceptionType::IO || error.Type() == ExceptionType::HTTP) { + // legacy endpoint (no region) returns 400 auto context = opener->TryGetClientContext(); if (context) { auto transaction = CatalogTransaction::GetSystemCatalogTransaction(*context); @@ -887,9 +888,13 @@ void S3FileHandle::Initialize(optional_ptr opener) { auto &extra_info = error.ExtraInfo(); auto entry = extra_info.find("status_code"); if (entry != extra_info.end()) { - if (entry->second == "400") { - // 400: BAD REQUEST - auto extra_text = S3FileSystem::GetS3BadRequestError(auth_params); + if (entry->second == "301" || entry->second == "400") { + auto new_region = extra_info.find("header_x-amz-bucket-region"); + string correct_region = ""; + if (new_region != extra_info.end()) { + correct_region = new_region->second; + } + auto extra_text = S3FileSystem::GetS3BadRequestError(auth_params, correct_region); throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info); } if (entry->second == "403") { @@ -1138,12 +1143,15 @@ bool S3FileSystem::ListFiles(const string &directory, const std::function:.*Unknown error for HTTP HEAD to 'http://test-bucket.s3.eu-west-1.amazonaws.com/whatever.parquet'.* +:.*HTTP Error: Unable to connect to URL .*http://test-bucket.s3.eu-west-1.amazonaws.com/whatever.parquet.*: 301 .Moved Permanently..* +.* +.*Bad Request - this can be caused by the S3 region being set incorrectly.* +.*Provided region is: .eu-west-1.* +.*Correct region is: .us-east-1.* statement error SELECT * FROM 'r2://test-bucket/whatever.parquet'; ---- -:.*Unknown error for HTTP HEAD to 'http://test-bucket.s3.eu-west-1.amazonaws.com/whatever.parquet'.* +:.*HTTP Error: Unable to connect to URL .*http://test-bucket.s3.eu-west-1.amazonaws.com/whatever.parquet.*: 301 .Moved Permanently..* +.* +.*Bad Request - this can be caused by the S3 region being set incorrectly.* +.*Provided region is: .eu-west-1.* +.*Correct region is: .us-east-1.* statement error SELECT * FROM 'gcs://test-bucket/whatever.parquet';