From 066e6f35396b0b2da33d76880e1cde230789e811 Mon Sep 17 00:00:00 2001 From: Yves Peter Date: Wed, 6 Jul 2022 13:01:18 +0200 Subject: [PATCH 1/3] enhancement(azure_blob sink): update to latest azure SDK Signed-off-by: Yves Peter Co-authored-by: David Schneider --- Cargo.lock | 76 ++++++++++++----------- Cargo.toml | 30 ++++++--- src/sinks/azure_blob/config.rs | 4 +- src/sinks/azure_blob/integration_tests.rs | 47 ++++++++------ src/sinks/azure_common/config.rs | 65 +++++++++---------- src/sinks/azure_common/service.rs | 7 ++- 6 files changed, 126 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d92a827f7c182..c6acab28decc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -516,18 +516,6 @@ version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" -[[package]] -name = "async-timer" -version = "1.0.0-beta.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faacdfdccd10db54656717fddcd1a2ab6cd1ab16c0d6e7d89ec365b885fc9844" -dependencies = [ - "error-code", - "libc", - "wasm-bindgen", - "winapi", -] - [[package]] name = "async-trait" version = "0.1.57" @@ -1088,67 +1076,62 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.2.2" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b7171eb40909f7f2805f4622e076f8a6dbbe2d98#b7171eb40909f7f2805f4622e076f8a6dbbe2d98" +version = "0.5.0" +source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" dependencies = [ "async-trait", "base64", "bytes 1.2.1", - "chrono", "dyn-clone", "futures 0.3.24", "getrandom 0.2.6", - "http", + "http-types", "log", - "oauth2", + "paste", "pin-project", "rand 0.8.5", "reqwest", "rustc_version 0.4.0", "serde", - "serde_derive", + "serde-xml-rs", "serde_json", - "thiserror", + "time", "url", "uuid 1.1.2", ] [[package]] name = "azure_identity" -version = "0.3.0" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b7171eb40909f7f2805f4622e076f8a6dbbe2d98#b7171eb40909f7f2805f4622e076f8a6dbbe2d98" +version = "0.6.0" +source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" dependencies = [ "async-lock", - "async-timer", "async-trait", "azure_core", "base64", - "chrono", + "fix-hidden-lifetime-bug", "futures 0.3.24", "log", "oauth2", - "reqwest", "serde", "serde_json", - "thiserror", + "time", "url", "uuid 1.1.2", ] [[package]] name = "azure_storage" -version = "0.2.0" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b7171eb40909f7f2805f4622e076f8a6dbbe2d98#b7171eb40909f7f2805f4622e076f8a6dbbe2d98" +version = "0.6.0" +source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" dependencies = [ "RustyXML", "async-trait", "azure_core", "base64", "bytes 1.2.1", - "chrono", "futures 0.3.24", "hmac", - "http", "log", "once_cell", "serde", @@ -1156,31 +1139,29 @@ dependencies = [ "serde_derive", "serde_json", "sha2 0.10.6", - "thiserror", + "time", "url", "uuid 1.1.2", ] [[package]] name = "azure_storage_blobs" -version = "0.2.0" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b7171eb40909f7f2805f4622e076f8a6dbbe2d98#b7171eb40909f7f2805f4622e076f8a6dbbe2d98" +version = "0.6.0" +source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" dependencies = [ "RustyXML", "azure_core", "azure_storage", "base64", "bytes 1.2.1", - "chrono", "futures 0.3.24", - "http", "log", "md5", "serde", "serde-xml-rs", "serde_derive", "serde_json", - "thiserror", + "time", "url", "uuid 1.1.2", ] @@ -2785,6 +2766,26 @@ dependencies = [ "winapi", ] +[[package]] +name = "fix-hidden-lifetime-bug" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ae9c2016a663983d4e40a9ff967d6dcac59819672f0b47f2b17574e99c33c8" +dependencies = [ + "fix-hidden-lifetime-bug-proc_macros", +] + +[[package]] +name = "fix-hidden-lifetime-bug-proc_macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c81935e123ab0741c4c4f0d9b8377e5fb21d3de7e062fa4b1263b1fbcba1ea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fixedbitset" version = "0.4.1" @@ -6930,9 +6931,9 @@ dependencies = [ [[package]] name = "serde-xml-rs" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa" +checksum = "fb3aa78ecda1ebc9ec9847d5d3aba7d618823446a049ba2491940506da6e2782" dependencies = [ "log", "serde", @@ -8789,6 +8790,7 @@ dependencies = [ "syslog", "tempfile", "tikv-jemallocator", + "time", "tokio", "tokio-openssl", "tokio-postgres", diff --git a/Cargo.toml b/Cargo.toml index 338392c122069..95e5eeadc2486 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,10 +171,24 @@ aws-smithy-http-tower = { version = "0.49.0", default-features = false, optional aws-smithy-types = { version = "0.49.0", default-features = false, optional = true } # Azure -azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, features = ["enable_reqwest"], optional = true } -azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, features = ["enable_reqwest"], optional = true } -azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, optional = true } -azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, optional = true } +azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"], optional = true } +azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"], optional = true } +azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, optional = true } +azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, optional = true } + +# fix for: +#error: failed to select a version for `time`. +# ... required by package `azure_core v0.5.0 (https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d49)` +# ... which satisfies git dependency `azure_core` of package `azure_identity v0.6.0 (https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d49)` +# ... which satisfies git dependency `azure_identity` of package `vector v0.25.0 (/home/u112324/git/vector)` +#versions that meet the requirements `^0.3.10` are: 0.3.14, 0.3.13, 0.3.12, 0.3.11, 0.3.10 +# +#all possible versions conflict with previously selected packages. +# +# previously selected package `time v0.3.9` +# ... which satisfies dependency `time = "^0.3.4"` (locked to 0.3.9) of package `aws-config v0.49.0` +# ... which satisfies dependency `aws-config = "^0.49.0"` (locked to 0.49.0) of package `vector v0.25.0 (/home/u112324/git/vector)` +time = { version = "0.3.10" } # Tower tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } @@ -332,10 +346,10 @@ tonic-build = { version = "0.8", default-features = false, features = ["transpor [dev-dependencies] approx = "0.5.1" assert_cmd = { version = "2.0.4", default-features = false } -azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } -azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, features = ["enable_reqwest"] } -azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, features = ["azurite_workaround"] } -azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b7171eb40909f7f2805f4622e076f8a6dbbe2d98", default-features = false, features = ["azurite_workaround"] } +azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"] } +azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"] } +azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false } +azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false } base64 = "0.13.0" criterion = { version = "0.4.0", features = ["html_reports", "async_tokio"] } libc = "0.2.133" diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 04b0937c87779..e4a28bc1a001d 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -48,7 +48,7 @@ pub struct AzureBlobSinkConfig { /// [env_cred_docs]: https://docs.rs/azure_identity/latest/azure_identity/struct.EnvironmentCredential.html /// [managed_ident_docs]: https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview /// [az_cli_docs]: https://docs.microsoft.com/en-us/cli/azure/account?view=azure-cli-latest#az-account-get-access-token - pub storage_account: Option, + pub storage_account: Option, /// The Azure Blob Storage Account container name. pub(super) container_name: String, @@ -117,7 +117,7 @@ impl GenerateConfig for AzureBlobSinkConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { connection_string: Some(String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into()), - storage_account: Some(String::from("some-account-name").into()), + storage_account: Some(String::from("some-account-name")), container_name: String::from("logs"), blob_prefix: Some(String::from("blob")), blob_time_format: Some(String::from("%s")), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 6a597c7bd8963..23bfc2ba5d7af 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -3,7 +3,7 @@ use std::{ num::NonZeroU32, }; -use azure_core::{prelude::Range, HttpError}; +use azure_core::{error::HttpError, prelude::Range}; use azure_storage_blobs::prelude::*; use bytes::{Buf, BytesMut}; use codecs::{ @@ -80,7 +80,7 @@ async fn azure_blob_insert_lines_into_blob() { config.run_assert(input).await; - let blobs = config.list_blobs(blob_prefix.as_str()).await; + let blobs = config.list_blobs(blob_prefix).await; assert_eq!(blobs.len(), 1); assert!(blobs[0].clone().ends_with(".log")); let (blob, blob_lines) = config.get_blob(blobs[0].clone()).await; @@ -105,7 +105,7 @@ async fn azure_blob_insert_json_into_blob() { config.run_assert(input).await; - let blobs = config.list_blobs(blob_prefix.as_str()).await; + let blobs = config.list_blobs(blob_prefix).await; assert_eq!(blobs.len(), 1); assert!(blobs[0].clone().ends_with(".log")); let (blob, blob_lines) = config.get_blob(blobs[0].clone()).await; @@ -132,7 +132,7 @@ async fn azure_blob_insert_lines_into_blob_gzip() { config.run_assert(events).await; - let blobs = config.list_blobs(blob_prefix.as_str()).await; + let blobs = config.list_blobs(blob_prefix).await; assert_eq!(blobs.len(), 1); assert!(blobs[0].clone().ends_with(".log.gz")); let (blob, blob_lines) = config.get_blob(blobs[0].clone()).await; @@ -164,7 +164,7 @@ async fn azure_blob_insert_json_into_blob_gzip() { config.run_assert(input).await; - let blobs = config.list_blobs(blob_prefix.as_str()).await; + let blobs = config.list_blobs(blob_prefix).await; assert_eq!(blobs.len(), 1); assert!(blobs[0].clone().ends_with(".log.gz")); let (blob, blob_lines) = config.get_blob(blobs[0].clone()).await; @@ -198,7 +198,7 @@ async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { config.run_assert(input).await; - let blobs = config.list_blobs(blob_prefix.as_str()).await; + let blobs = config.list_blobs(blob_prefix).await; assert_eq!(blobs.len(), 3); let response = stream::iter(blobs) .fold(Vec::new(), |mut acc, blob| async { @@ -253,7 +253,7 @@ impl AzureBlobSinkConfig { .expect("Running sink failed"); } - pub async fn list_blobs(&self, prefix: &str) -> Vec { + pub async fn list_blobs(&self, prefix: String) -> Vec { let client = azure_common::config::build_client( self.connection_string.clone().map(Into::into), self.storage_account.clone().map(Into::into), @@ -266,14 +266,15 @@ impl AzureBlobSinkConfig { .max_results(NonZeroU32::new(1000).unwrap()) .delimiter("/") .include_metadata(true) - .execute() + .into_stream() + .next() .await - .expect("Failed to fetch blobs"); + .expect("Failed to fetch blobs") + .unwrap(); let blobs = response .blobs - .blobs - .iter() + .blobs() .map(|blob| blob.name.clone()) .collect::>(); @@ -288,14 +289,19 @@ impl AzureBlobSinkConfig { ) .unwrap(); let response = client - .as_blob_client(blob.as_str()) + .blob_client(blob) .get() .range(Range::new(0, 1024 * 1024)) - .execute() + .into_stream() + .next() .await - .expect("Failed to get blob"); + .expect("Failed to get blob") + .unwrap(); - (response.blob, self.get_blob_content(response.data.to_vec())) + ( + response.blob, + self.get_blob_content(response.data.collect().await.unwrap().to_vec()), + ) } fn get_blob_content(&self, data: Vec) -> Vec { @@ -318,14 +324,17 @@ impl AzureBlobSinkConfig { self.container_name.clone(), ) .unwrap(); - let request = client.create().public_access(PublicAccess::None).execute(); + let request = client + .create() + .public_access(PublicAccess::None) + .into_future(); let response = match request.await { Ok(_) => Ok(()), Err(reason) => match reason.downcast_ref::() { - Some(HttpError::StatusCode { status, .. }) => match *status { - StatusCode::CONFLICT => Ok(()), - status => Err(format!("Unexpected status code {}", status)), + Some(err) => match StatusCode::from_u16(err.status().into()) { + Ok(StatusCode::CONFLICT) => Ok(()), + _ => Err(format!("Unexpected status code {}", err.status())), }, _ => Err(format!("Unexpected error {}", reason)), }, diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 4266a3f64f06e..73245aab4c5e2 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,9 +1,9 @@ use std::sync::Arc; -use azure_core::{new_http_client, HttpError}; +use azure_core::{error::HttpError, RetryOptions}; use azure_identity::{AutoRefreshingTokenCredential, DefaultAzureCredential}; -use azure_storage::prelude::*; -use azure_storage_blobs::{blob::responses::PutBlockBlobResponse, prelude::*}; +use azure_storage::{prelude::*, ConnectionString}; +use azure_storage_blobs::{blob::operations::PutBlockBlobResponse, prelude::*}; use bytes::Bytes; use futures::FutureExt; use http::StatusCode; @@ -45,12 +45,8 @@ impl RetryLogic for AzureBlobRetryLogic { type Response = AzureBlobResponse; fn is_retriable_error(&self, error: &Self::Error) -> bool { - match error { - HttpError::StatusCode { status, .. } => { - status.is_server_error() || status == &StatusCode::TOO_MANY_REQUESTS - } - _ => false, - } + error.status().is_server_error() + || StatusCode::TOO_MANY_REQUESTS.as_u16() == Into::::into(error.status()) } } @@ -91,22 +87,23 @@ pub fn build_healthcheck( client: Arc, ) -> crate::Result { let healthcheck = async move { - let request = client.get_properties().execute().await; + let response = client.get_properties().into_future().await; - match request { + let resp: crate::Result<()> = match response { Ok(_) => Ok(()), Err(reason) => Err(match reason.downcast_ref::() { - Some(HttpError::StatusCode { status, .. }) => match *status { - StatusCode::FORBIDDEN => HealthcheckError::InvalidCredentials.into(), - StatusCode::NOT_FOUND => HealthcheckError::UnknownContainer { + Some(err) => match StatusCode::from_u16(err.status().into()) { + Ok(StatusCode::FORBIDDEN) => Box::new(HealthcheckError::InvalidCredentials), + Ok(StatusCode::NOT_FOUND) => Box::new(HealthcheckError::UnknownContainer { container: container_name, - } - .into(), - status => HealthcheckError::Unknown { status }.into(), + }), + Ok(status) => Box::new(HealthcheckError::Unknown { status }), + Err(_) => "unknown status code".into(), }, - _ => reason, + _ => reason.into(), }), - } + }; + resp }; Ok(healthcheck.boxed()) @@ -120,24 +117,24 @@ pub fn build_client( let client; match (connection_string, storage_account) { (Some(connection_string_p), None) => { - client = StorageAccountClient::new_connection_string( - new_http_client(), - &connection_string_p, - )? - .as_storage_client() - .as_container_client(container_name); + let connection_string = ConnectionString::new(&connection_string_p)?; + client = ClientBuilder::new( + connection_string + .account_name + .ok_or("Account name missing in connection string")?, + connection_string.storage_credentials()?, + ) + .retry(RetryOptions::none()) + .container_client(container_name); } (None, Some(storage_account_p)) => { let creds = std::sync::Arc::new(DefaultAzureCredential::default()); - let auto_creds = Box::new(AutoRefreshingTokenCredential::new(creds)); + let auto_creds = std::sync::Arc::new(AutoRefreshingTokenCredential::new(creds)); + let storage_credentials = StorageCredentials::TokenCredential(auto_creds); - client = StorageAccountClient::new_token_credential( - new_http_client(), - storage_account_p, - auto_creds, - ) - .as_storage_client() - .as_container_client(container_name); + client = ClientBuilder::new(storage_account_p, storage_credentials) + .retry(RetryOptions::none()) + .container_client(container_name) } (None, None) => { return Err("Either `connection_string` or `storage_account` has to be provided".into()) @@ -149,5 +146,5 @@ pub fn build_client( ) } } - Ok(client) + Ok(std::sync::Arc::new(client)) } diff --git a/src/sinks/azure_common/service.rs b/src/sinks/azure_common/service.rs index 66dd546426490..ee21611d10375 100644 --- a/src/sinks/azure_common/service.rs +++ b/src/sinks/azure_common/service.rs @@ -39,7 +39,7 @@ impl Service for AzureBlobService { Box::pin(async move { let client = this .client - .as_blob_client(request.metadata.partition_key.as_str()); + .blob_client(request.metadata.partition_key.as_str()); let byte_size = request.blob_data.len(); let blob = client .put_block_blob(request.blob_data) @@ -50,9 +50,10 @@ impl Service for AzureBlobService { }; let result = blob - .execute() + .into_future() .instrument(info_span!("request").or_current()) - .await; + .await + .map_err(|err| err.into()); result.map(|inner| AzureBlobResponse { inner, From c46153eae962bbaa4ff060b3e538447e69306c04 Mon Sep 17 00:00:00 2001 From: yvespp Date: Wed, 28 Sep 2022 22:18:42 +0200 Subject: [PATCH 2/3] fix(azure_blob_sink): fix integration test Signed-off-by: yvespp --- Cargo.toml | 6 +++--- src/sinks/azure_common/config.rs | 14 ++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 95e5eeadc2486..273b14b1d173d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,10 +346,10 @@ tonic-build = { version = "0.8", default-features = false, features = ["transpor [dev-dependencies] approx = "0.5.1" assert_cmd = { version = "2.0.4", default-features = false } -azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"] } +azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"] } -azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false } -azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false } +azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["azurite_workaround"] } +azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["azurite_workaround"] } base64 = "0.13.0" criterion = { version = "0.4.0", features = ["html_reports", "async_tokio"] } libc = "0.2.133" diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 73245aab4c5e2..7567980eeb877 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use azure_core::{error::HttpError, RetryOptions}; use azure_identity::{AutoRefreshingTokenCredential, DefaultAzureCredential}; -use azure_storage::{prelude::*, ConnectionString}; +use azure_storage::{prelude::*, CloudLocation, ConnectionString}; use azure_storage_blobs::{blob::operations::PutBlockBlobResponse, prelude::*}; use bytes::Bytes; use futures::FutureExt; @@ -118,14 +118,20 @@ pub fn build_client( match (connection_string, storage_account) { (Some(connection_string_p), None) => { let connection_string = ConnectionString::new(&connection_string_p)?; - client = ClientBuilder::new( + let mut builder = ClientBuilder::new( connection_string .account_name .ok_or("Account name missing in connection string")?, connection_string.storage_credentials()?, ) - .retry(RetryOptions::none()) - .container_client(container_name); + .retry(RetryOptions::none()); + if let Some(uri) = connection_string.blob_endpoint { + builder = builder.cloud_location(CloudLocation::Custom { + uri: uri.to_string(), + credentials: connection_string.storage_credentials()?, + }); + } + client = builder.container_client(container_name); } (None, Some(storage_account_p)) => { let creds = std::sync::Arc::new(DefaultAzureCredential::default()); From 39c2d49292a165327cddada4448547b5cc101599 Mon Sep 17 00:00:00 2001 From: Yves Peter Date: Fri, 30 Sep 2022 08:18:54 +0200 Subject: [PATCH 3/3] fix(azure_blob_sink): remove time dep Signed-off-by: Yves Peter --- Cargo.lock | 1 - Cargo.toml | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6acab28decc8..8e08e2b9e49a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8790,7 +8790,6 @@ dependencies = [ "syslog", "tempfile", "tikv-jemallocator", - "time", "tokio", "tokio-openssl", "tokio-postgres", diff --git a/Cargo.toml b/Cargo.toml index 273b14b1d173d..6a0d268c3eae9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,20 +176,6 @@ azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, optional = true } azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, optional = true } -# fix for: -#error: failed to select a version for `time`. -# ... required by package `azure_core v0.5.0 (https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d49)` -# ... which satisfies git dependency `azure_core` of package `azure_identity v0.6.0 (https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d49)` -# ... which satisfies git dependency `azure_identity` of package `vector v0.25.0 (/home/u112324/git/vector)` -#versions that meet the requirements `^0.3.10` are: 0.3.14, 0.3.13, 0.3.12, 0.3.11, 0.3.10 -# -#all possible versions conflict with previously selected packages. -# -# previously selected package `time v0.3.9` -# ... which satisfies dependency `time = "^0.3.4"` (locked to 0.3.9) of package `aws-config v0.49.0` -# ... which satisfies dependency `aws-config = "^0.49.0"` (locked to 0.49.0) of package `vector v0.25.0 (/home/u112324/git/vector)` -time = { version = "0.3.10" } - # Tower tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } # Serde