diff --git a/Cargo.lock b/Cargo.lock index ce437599471b2..15c1d86cfa0a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -503,7 +503,7 @@ dependencies = [ "base64 0.21.4", "bytes", "futures", - "http", + "http 0.2.9", "memchr", "nkeys", "nuid", @@ -684,9 +684,9 @@ dependencies = [ [[package]] name = "aws-config" -version = "0.57.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7489a72550db3712fe3a0a92068f832d6270ff82f518b84a800af131f99570d7" +checksum = "8e245d7c741a8e4b23133f36750c4bcb3938a66ac49510caaf8b83afd52db1ec" dependencies = [ "aws-credential-types", "aws-http", @@ -701,7 +701,7 @@ dependencies = [ "aws-types", "bytes", "fastrand 2.0.0", - "http", + "http 0.2.9", "hyper", "time", "tokio", @@ -710,9 +710,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80009846d61a0a4f9070d789cf0e64db284cba6984fae3871050d044e6569cd2" +checksum = "8c1317e1a3514b103cf7d5828bbab3b4d30f56bd22d684f8568bc51b6cfbbb1c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -722,16 +722,15 @@ dependencies = [ [[package]] name = "aws-http" -version = "0.57.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e65730b741a5f6422fd338bf6f76b7956b090affeaa045e78fca8c4186e0fd5" +checksum = "361c4310fdce94328cc2d1ca0c8a48c13f43009c61d3367585685a50ca8c66b6" dependencies = [ - "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http", + "http 0.2.9", "http-body", "pin-project-lite", "tracing", @@ -739,9 +738,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2414b96071ae840b97c0cc1d44b248d5607d648593cdf474f3fb5465572898" +checksum = "1ed7ef604a15fd0d4d9e43701295161ea6b504b63c44990ead352afea2bc15e9" dependencies = [ "aws-credential-types", "aws-http", @@ -753,7 +752,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "fastrand 2.0.0", - "http", + "http 0.2.9", "percent-encoding", "tracing", "uuid", @@ -761,9 +760,9 @@ dependencies = [ [[package]] name = "aws-sdk-ec2" -version = "0.35.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3492d8b17bc47532abee2d83a3f861997a7017ca548abb0ab4a26d5e6a691b" +checksum = "88111ec31c6a3b9ed890f345b063bc2c76b49897e0c4601f6b1350c95d8d96d6" dependencies = [ "aws-credential-types", "aws-http", @@ -778,16 +777,16 @@ dependencies = [ "aws-smithy-xml", "aws-types", "fastrand 2.0.0", - "http", + "http 0.2.9", "regex", "tracing", ] [[package]] name = "aws-sdk-kinesis" -version = "0.35.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acef0ae3beb145ba41e640fe33b2b5ac021218ee0b42199779393d2b7e4e2cc" +checksum = "dfaddefaf8724180293b248dd6977b271800d79a47de23d3a1589b3fc1466a31" dependencies = [ "aws-credential-types", "aws-http", @@ -800,16 +799,16 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "http", + "http 0.2.9", "regex", "tracing", ] [[package]] name = "aws-sdk-s3" -version = "0.35.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84022763485483ea17d417f9832d5da198bc36829b59f086c0d35ecd2ce59991" +checksum = "62d240c8c3e3663cd278e47702bbd2566203362d93b51d95575d7b2e0c265e99" dependencies = [ "aws-credential-types", "aws-http", @@ -826,7 +825,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", - "http", + "http 0.2.9", "http-body", "once_cell", "percent-encoding", @@ -837,9 +836,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.35.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b1a8ae5c7098502a3e6d4130dbee1e1d3fcb8dc5d65cecab39e01d595f90f6" +checksum = "9b1d955bacd8c3637908a40a4af2f7a732461ee7d95ec02599a3610ee55781d7" dependencies = [ "aws-credential-types", "aws-http", @@ -853,33 +852,35 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "http", + "http 0.2.9", "regex", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3347c738e0a8449020877d319cda56da74d6e8aba9fff210720fac66cae3c7f4" +checksum = "380adcc8134ad8bbdfeb2ace7626a869914ee266322965276cbc54066186d236" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac", - "http", - "num-bigint", + "http 0.2.9", "once_cell", "p256 0.11.1", "percent-encoding", "regex", "ring 0.17.5", "sha2", + "subtle", "time", "tracing", "zeroize", @@ -887,9 +888,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b65a284265d3eec6cc9f1daef2d0cc3b78684b712cb6c7f1d0f665456b7604" +checksum = "1fbfa248f7f966d73e325dbc85851a5500042b6d96e3c3b535a8527707f36fe4" dependencies = [ "futures-util", "pin-project-lite", @@ -898,9 +899,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.57.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40f1d5a222ba11ac7d6b20f3668ae282970e50615fa5ee1dd8ac8180c0c1803" +checksum = "c5a373ec01aede3dd066ec018c1bc4e8f5dd11b2c11c59c8eef1a5c68101f397" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -908,7 +909,7 @@ dependencies = [ "crc32c", "crc32fast", "hex", - "http", + "http 0.2.9", "http-body", "md-5", "pin-project-lite", @@ -919,9 +920,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.57.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16e7ecebc2b083a1b138868a46a343204a6097f343c4830a8b22b3a0d30013e" +checksum = "1c669e1e5fc0d79561bf7a122b118bd50c898758354fe2c53eb8f2d31507cbc3" dependencies = [ "aws-smithy-types", "bytes", @@ -930,9 +931,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.57.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715aeb61fb743848d5d398ce6fb1259f5eba5e13dceec5d5064cada1a181d38d" +checksum = "5b1de8aee22f67de467b2e3d0dd0fb30859dc53f579a63bd5381766b987db644" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -940,7 +941,7 @@ dependencies = [ "bytes", "bytes-utils", "futures-core", - "http", + "http 0.2.9", "http-body", "once_cell", "percent-encoding", @@ -951,18 +952,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.57.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de21d368dcd5cab17033406ea6e7351b091164b208381de837510bd7558c0f30" +checksum = "6a46dd338dc9576d6a6a5b5a19bd678dcad018ececee11cf28ecd7588bd1a55c" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.57.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5ace389c7e4def130bff7275647481c8d49b867909ca61d5dc9a809b3632f3" +checksum = "feb5b8c7a86d4b6399169670723b7e6f21a39fc833a30f5c5a2f997608178129" dependencies = [ "aws-smithy-types", "urlencoding", @@ -970,9 +971,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4395310662d10f1847324af5fe43e621922cba03b1aa6d26c21096e18a4e79" +checksum = "064b808143d80b50744b1b22cce801238a545b84859c6cf8e275997252dd1d25" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -980,7 +981,7 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand 2.0.0", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-rustls", @@ -994,14 +995,14 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27594c06f5b36e97d18ef26ed693f1d4c7167b9bbb544b3a9bb653f9f7035" +checksum = "4d27c3235d4972ed976b5c1a82286e7c4457f618f3c2ae6d4ae44f081dd24575" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", - "http", + "http 0.2.9", "pin-project-lite", "tokio", "tracing", @@ -1009,15 +1010,15 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d36f1723ed61e82094498e7283510fe21484b73c215c33874c81a84411b5bdc" +checksum = "d2fc32035dc0636a8583cf0c6dd7f1e6d5404103b836d26228b8730907a88d9f" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", - "http", + "http 0.2.9", "http-body", "hyper", "itoa", @@ -1033,24 +1034,24 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.57.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68225c8d3e3e6c565a3cf764aa82440837ef15c33d1dd7205e15715444e4b4ad" +checksum = "0ec40d74a67fd395bc3f6b4ccbdf1543672622d905ef3f979689aea5b730cb95" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.57.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc27aac60f715bab25f5d758ba5651b80aae791c48e9871ffe298683f00a2b" +checksum = "8403fc56b1f3761e8efe45771ddc1165e47ec3417c68e68a4519b5cb030159ca" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "http", + "http 0.2.9", "rustc_version", "tracing", ] @@ -1066,7 +1067,7 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", + "http 0.2.9", "http-body", "hyper", "itoa", @@ -1096,7 +1097,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", + "http 0.2.9", "http-body", "mime", "rustversion", @@ -2194,9 +2195,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740fe28e594155f10cfc383984cbefd529d7396050557148f79cb0f621204124" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", "rand_core", @@ -2767,7 +2768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d97ca172ae9dc9f9b779a6e3a65d308f2af74e5b8c921299075bdb4a0370e914" dependencies = [ "base16ct 0.2.0", - "crypto-bigint 0.5.3", + "crypto-bigint 0.5.5", "digest", "ff 0.13.0", "generic-array", @@ -2935,7 +2936,7 @@ name = "etcd-client" version = "0.12.1" source = "git+https://github.com/risingwavelabs/etcd-client.git?rev=4e84d40#4e84d40a84b35718d814cc2afccc9274c9d78e1e" dependencies = [ - "http", + "http 0.2.9", "prost 0.12.1", "tokio", "tokio-stream", @@ -3644,7 +3645,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8bdaaa4bc036e8318274d1b25f0f2265b3e95418b765fd1ea1c7ef938fd69bd" dependencies = [ "google-cloud-token", - "http", + "http 0.2.9", "thiserror", "tokio", "tokio-retry", @@ -3753,7 +3754,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.9", "indexmap 1.9.3", "slab", "tokio", @@ -3918,6 +3919,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -3925,7 +3937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.9", "pin-project-lite", ] @@ -3964,7 +3976,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "httparse", "httpdate", @@ -3984,7 +3996,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", + "http 0.2.9", "hyper", "log", "rustls", @@ -4761,9 +4773,9 @@ dependencies = [ [[package]] name = "madsim-aws-sdk-s3" -version = "0.3.0+0.35" +version = "0.5.0+1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f484435d52fc8f9af55a687b7777fcdd8fd9acc01901526367137fc39473c56" +checksum = "0b442215f0047e7e7df7b98991129c43ee5fb186ce899e8f7e27515bc99de508" dependencies = [ "aws-sdk-s3", "aws-smithy-http", @@ -4771,7 +4783,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "http", + "http 1.0.0", "madsim", "spin 0.9.8", "tracing", @@ -4785,7 +4797,7 @@ checksum = "02b4b5de48bb7f3f7eae0bca62b3ed0b7d714b1b273d7347329b92c3a2eef113" dependencies = [ "etcd-client", "futures-util", - "http", + "http 0.2.9", "madsim", "serde", "serde_with", @@ -5468,7 +5480,7 @@ dependencies = [ "base64 0.13.1", "chrono", "getrandom", - "http", + "http 0.2.9", "rand", "reqwest", "serde", @@ -5515,7 +5527,7 @@ dependencies = [ "chrono", "flagset", "futures", - "http", + "http 0.2.9", "hyper", "log", "md-5", @@ -5545,7 +5557,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "hmac", - "http", + "http 0.2.9", "itertools 0.10.5", "log", "oauth2", @@ -5638,7 +5650,7 @@ checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" dependencies = [ "async-trait", "futures-core", - "http", + "http 0.2.9", "opentelemetry-proto", "opentelemetry-semantic-conventions", "opentelemetry_api", @@ -7083,7 +7095,7 @@ dependencies = [ "hex", "hmac", "home", - "http", + "http 0.2.9", "jsonwebtoken", "log", "once_cell", @@ -7112,7 +7124,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-rustls", @@ -7434,7 +7446,7 @@ dependencies = [ "fs-err", "futures", "hex", - "http", + "http 0.2.9", "http-body", "humantime", "hyper", @@ -7673,7 +7685,7 @@ dependencies = [ "gcp-bigquery-client", "glob", "google-cloud-pubsub", - "http", + "http 0.2.9", "hyper", "hyper-tls", "icelake", @@ -10590,7 +10602,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-timeout", @@ -10620,7 +10632,7 @@ dependencies = [ "base64 0.21.4", "bytes", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-timeout", @@ -10678,7 +10690,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", + "http 0.2.9", "http-body", "http-range-header", "httpdate", @@ -11468,6 +11480,7 @@ dependencies = [ "combine", "crossbeam-epoch", "crossbeam-utils", + "crypto-bigint 0.5.5", "deranged", "digest", "either", @@ -11642,7 +11655,7 @@ dependencies = [ "async-trait", "base64 0.13.1", "futures", - "http", + "http 0.2.9", "hyper", "hyper-rustls", "itertools 0.10.5", diff --git a/Cargo.toml b/Cargo.toml index e99ba6030ebb2..59fd689d7be32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,35 +68,38 @@ repository = "https://github.com/risingwavelabs/risingwave" [workspace.dependencies] await-tree = "0.1.1" -aws-config = { version = "0.57", default-features = false, features = [ +aws-config = { version = "1", default-features = false, features = [ + "behavior-version-latest", "rt-tokio", "rustls", ] } -aws-credential-types = { version = "0.57", default-features = false, features = [ +aws-credential-types = { version = "1", default-features = false, features = [ "hardcoded-credentials", ] } -aws-sdk-kinesis = { version = "0.35", default-features = false, features = [ +aws-sdk-kinesis = { version = "1", default-features = false, features = [ "rt-tokio", "rustls", ] } -aws-sdk-s3 = { version = "0.35", default-features = false, features = [ +aws-sdk-s3 = { version = "1", default-features = false, features = [ "rt-tokio", "rustls", ] } -aws-sdk-ec2 = { version = "0.35", default-features = false, features = [ +aws-sdk-ec2 = { version = "1", default-features = false, features = [ "rt-tokio", "rustls", ] } -aws-sdk-sqs = { version = "0.35", default-features = false, features = [ +aws-sdk-sqs = { version = "1", default-features = false, features = [ "rt-tokio", "rustls", ] } -aws-smithy-http = "0.57" -aws-smithy-runtime-api = "0.57" -aws-smithy-runtime = "0.57" -aws-smithy-types = { version = "0.57", default-features = false, features = ["hyper-0-14-x"] } -aws-endpoint = "0.57" -aws-types = "0.57" +aws-smithy-http = "0.60" +aws-smithy-runtime-api = "1" +aws-smithy-runtime = "1" +aws-smithy-types = { version = "1", default-features = false, features = [ + "hyper-0-14-x", +] } +aws-endpoint = "0.60" +aws-types = "1" etcd-client = { package = "madsim-etcd-client", version = "0.4" } futures-async-stream = "0.2.9" hytra = "0.1" diff --git a/src/connector/src/source/filesystem/file_common.rs b/src/connector/src/source/filesystem/file_common.rs index 85d65f40a3479..ef47724e92233 100644 --- a/src/connector/src/source/filesystem/file_common.rs +++ b/src/connector/src/source/filesystem/file_common.rs @@ -32,7 +32,7 @@ impl From<&Object> for FsSplit { Self { name: value.key().unwrap().to_owned(), offset: 0, - size: value.size() as usize, + size: value.size().unwrap_or_default() as usize, } } } @@ -81,7 +81,7 @@ impl From<&Object> for FsPageItem { let aws_ts = value.last_modified().unwrap(); Self { name: value.key().unwrap().to_owned(), - size: value.size(), + size: value.size().unwrap_or_default(), timestamp: Timestamp::from_timestamp_uncheck(aws_ts.secs(), aws_ts.subsec_nanos()), } } diff --git a/src/connector/src/source/filesystem/s3/source/reader.rs b/src/connector/src/source/filesystem/s3/source/reader.rs index 68cdc4b6cfa71..e4132ee615d3f 100644 --- a/src/connector/src/source/filesystem/s3/source/reader.rs +++ b/src/connector/src/source/filesystem/s3/source/reader.rs @@ -24,7 +24,6 @@ use aws_smithy_runtime_api::client::result::SdkError; use aws_smithy_types::body::SdkBody; use aws_smithy_types::byte_stream::ByteStream; use futures_async_stream::try_stream; -use hyper::Response; use io::StreamReader; use risingwave_common::error::RwError; use tokio::io::BufReader; @@ -145,7 +144,10 @@ impl S3FileReader { bucket_name: &str, object_name: &str, start: usize, - ) -> std::result::Result>> { + ) -> std::result::Result< + ByteStream, + SdkError>, + > { let range = if start == 0 { None } else { diff --git a/src/connector/src/source/filesystem/s3_v2/lister.rs b/src/connector/src/source/filesystem/s3_v2/lister.rs index 3c55f23d37f67..2454037514377 100644 --- a/src/connector/src/source/filesystem/s3_v2/lister.rs +++ b/src/connector/src/source/filesystem/s3_v2/lister.rs @@ -39,7 +39,7 @@ impl FsListInner for S3SplitEnumerator { .send() .await .map_err(|e| anyhow!(DisplayErrorContext(e)))?; - if res.is_truncated() { + if res.is_truncated().unwrap_or_default() { self.next_continuation_token = res.next_continuation_token.clone(); } else { has_finished = true; diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml index a3f77e101bd4f..9be44a74252d6 100644 --- a/src/object_store/Cargo.toml +++ b/src/object_store/Cargo.toml @@ -12,7 +12,7 @@ repository = { workspace = true } async-trait = "0.1" await-tree = { workspace = true } aws-config = { workspace = true } -aws-sdk-s3 = { version = "0.3", package = "madsim-aws-sdk-s3" } +aws-sdk-s3 = { version = "0.5", package = "madsim-aws-sdk-s3" } aws-smithy-http = { workspace = true } aws-smithy-runtime = { workspace = true } aws-smithy-runtime-api = { workspace = true } @@ -22,7 +22,7 @@ crc32fast = "1.3.2" either = "1" fail = "0.5" futures = { version = "0.3", default-features = false, features = ["alloc"] } -hyper = "0.14" +hyper = { version = "0.14", features = ["tcp", "client"] } hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] } hyper-tls = "0.5.0" itertools = "0.12" diff --git a/src/object_store/src/object/error.rs b/src/object_store/src/object/error.rs index 7a432a45fdc57..80cc85ae75f95 100644 --- a/src/object_store/src/object/error.rs +++ b/src/object_store/src/object/error.rs @@ -21,7 +21,6 @@ use aws_sdk_s3::operation::get_object::GetObjectError; use aws_sdk_s3::operation::head_object::HeadObjectError; use aws_sdk_s3::primitives::ByteStreamError; use aws_smithy_types::body::SdkBody; -use hyper::Response; use risingwave_common::error::BoxedError; use thiserror::Error; use tokio::sync::oneshot::error::RecvError; @@ -84,7 +83,7 @@ impl ObjectError { if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) = e .downcast_ref::, + aws_smithy_runtime_api::http::Response, >>() { return matches!(err.err(), GetObjectError::NoSuchKey(_)); @@ -92,7 +91,7 @@ impl ObjectError { if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) = e .downcast_ref::, + aws_smithy_runtime_api::http::Response, >>() { return matches!(err.err(), HeadObjectError::NotFound(_)); @@ -113,11 +112,12 @@ impl ObjectError { } } -impl From> for ObjectError +impl From> for ObjectError where E: std::error::Error + Sync + Send + 'static, + R: Send + Sync + 'static + std::fmt::Debug, { - fn from(e: aws_sdk_s3::error::SdkError) -> Self { + fn from(e: aws_smithy_runtime_api::client::result::SdkError) -> Self { ObjectErrorInner::S3(e.into()).into() } } diff --git a/src/object_store/src/object/s3.rs b/src/object_store/src/object/s3.rs index f2158ca3f1cf4..d150b4178f964 100644 --- a/src/object_store/src/object/s3.rs +++ b/src/object_store/src/object/s3.rs @@ -41,7 +41,7 @@ use either::Either; use fail::fail_point; use futures::future::{try_join_all, BoxFuture, FutureExt}; use futures::{stream, Stream, StreamExt, TryStreamExt}; -use hyper::{Body, Response}; +use hyper::Body; use itertools::Itertools; use risingwave_common::config::ObjectStoreConfig; use risingwave_common::monitor::connection::monitor_connector; @@ -420,7 +420,7 @@ impl ObjectStore for S3ObjectStore { .last_modified() .expect("last_modified required") .as_secs_f64(), - total_size: resp.content_length as usize, + total_size: resp.content_length.unwrap_or_default() as usize, }) } @@ -645,20 +645,27 @@ impl S3ObjectStore { let s3_object_store_config = ObjectStoreConfig::default(); #[cfg(madsim)] - let builder = aws_sdk_s3::config::Builder::new(); + let builder = aws_sdk_s3::config::Builder::new().credentials_provider( + Credentials::from_keys(access_key_id, secret_access_key, None), + ); #[cfg(not(madsim))] - let builder: aws_sdk_s3::config::Builder = - aws_sdk_s3::config::Builder::from(&aws_config::ConfigLoader::default().load().await) - .force_path_style(true) - .http_client(Self::new_http_client(&s3_object_store_config)); + let builder = aws_sdk_s3::config::Builder::from( + &aws_config::ConfigLoader::default() + // FIXME: https://github.com/awslabs/aws-sdk-rust/issues/973 + .credentials_provider(Credentials::from_keys( + access_key_id, + secret_access_key, + None, + )) + .load() + .await, + ) + .force_path_style(true) + .http_client(Self::new_http_client(&s3_object_store_config)) + .behavior_version_latest(); let config = builder .region(Region::new("custom")) .endpoint_url(format!("{}{}", endpoint_prefix, address)) - .credentials_provider(Credentials::from_keys( - access_key_id, - secret_access_key, - None, - )) .build(); let client = Client::from_conf(config); @@ -808,7 +815,10 @@ impl S3ObjectStore { #[inline(always)] fn should_retry( - err: &Either>, ByteStreamError>, + err: &Either< + SdkError>, + ByteStreamError, + >, ) -> bool { match err { Either::Left(err) => { @@ -845,14 +855,14 @@ struct S3ObjectIter { bucket: String, prefix: String, next_continuation_token: Option, - is_truncated: bool, + is_truncated: Option, #[allow(clippy::type_complexity)] send_future: Option< BoxFuture< 'static, Result< - (Vec, Option, bool), - SdkError>, + (Vec, Option, Option), + SdkError>, >, >, >, @@ -866,7 +876,7 @@ impl S3ObjectIter { bucket, prefix, next_continuation_token: None, - is_truncated: true, + is_truncated: Some(true), send_future: None, } } @@ -894,7 +904,7 @@ impl Stream for S3ObjectIter { } }; } - if !self.is_truncated { + if !self.is_truncated.unwrap_or_default() { return Poll::Ready(None); } let mut request = self @@ -917,7 +927,7 @@ impl Stream for S3ObjectIter { .last_modified() .map(|l| l.as_secs_f64()) .unwrap_or(0f64), - total_size: obj.size() as usize, + total_size: obj.size().unwrap_or_default() as usize, }) .collect_vec(); let is_truncated = r.is_truncated; @@ -932,8 +942,20 @@ impl Stream for S3ObjectIter { } } -impl From>, ByteStreamError>> for ObjectError { - fn from(e: Either>, ByteStreamError>) -> Self { +impl + From< + Either< + SdkError>, + ByteStreamError, + >, + > for ObjectError +{ + fn from( + e: Either< + SdkError>, + ByteStreamError, + >, + ) -> Self { match e { Either::Left(e) => e.into(), Either::Right(e) => e.into(), diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index 1f420d371622f..72eeca9837bf0 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -13,7 +13,7 @@ normal = ["serde"] [dependencies] anyhow = "1.0" async-trait = "0.1" -aws-sdk-s3 = { version = "0.3", package = "madsim-aws-sdk-s3" } +aws-sdk-s3 = { version = "0.5", package = "madsim-aws-sdk-s3" } cfg-or-panic = "0.2" clap = { version = "4", features = ["derive"] } console = "0.15" diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index b263a9c5bcefb..e26d614800d5d 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -22,12 +22,12 @@ ahash = { version = "0.8" } allocator-api2 = { version = "0.2", default-features = false, features = ["alloc", "nightly"] } anyhow = { version = "1", features = ["backtrace"] } async-std = { version = "1", features = ["attributes", "tokio1"] } -aws-credential-types = { version = "0.57", default-features = false, features = ["hardcoded-credentials"] } -aws-runtime = { version = "0.57", default-features = false, features = ["event-stream", "sigv4a"] } -aws-sdk-s3 = { version = "0.35" } -aws-sigv4 = { version = "0.57", features = ["http0-compat", "sign-eventstream", "sigv4a"] } -aws-smithy-runtime = { version = "0.57", default-features = false, features = ["client", "rt-tokio", "tls-rustls"] } -aws-smithy-types = { version = "0.57", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "hyper-0-14-x", "rt-tokio"] } +aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials"] } +aws-runtime = { version = "1", default-features = false, features = ["event-stream", "sigv4a"] } +aws-sdk-s3 = { version = "1" } +aws-sigv4 = { version = "1", features = ["http0-compat", "sign-eventstream", "sigv4a"] } +aws-smithy-runtime = { version = "1", default-features = false, features = ["client", "rt-tokio", "tls-rustls"] } +aws-smithy-types = { version = "1", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "hyper-0-14-x", "rt-tokio"] } base64 = { version = "0.21", features = ["alloc"] } bit-vec = { version = "0.6" } bitflags = { version = "2", default-features = false, features = ["serde", "std"] } @@ -39,6 +39,7 @@ clap_builder = { version = "4", default-features = false, features = ["cargo", " combine = { version = "4", features = ["tokio"] } crossbeam-epoch = { version = "0.9" } crossbeam-utils = { version = "0.8" } +crypto-bigint = { version = "0.5", features = ["generic-array", "zeroize"] } deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] } digest = { version = "0.10", features = ["mac", "oid", "std"] } either = { version = "1", features = ["serde"] }