From 8fbea5e718930160b7860bd46cbe7971d8677f05 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Sun, 22 Feb 2026 23:52:28 +0000 Subject: [PATCH 01/26] start by moving auth config to azure_common --- src/sinks/azure_common/config.rs | 200 ++++++++++++++++++++++- src/sinks/azure_logs_ingestion/config.rs | 199 +--------------------- src/sinks/azure_logs_ingestion/tests.rs | 12 +- 3 files changed, 207 insertions(+), 204 deletions(-) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 139608d9847ab..af61971588c7e 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -4,16 +4,27 @@ use azure_core::error::Error as AzureCoreError; use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; use crate::sinks::azure_common::shared_key_policy::SharedKeyAuthorizationPolicy; -use azure_core::http::Url; +use azure_core::http::{ClientMethodOptions, StatusCode, Url}; + +use azure_core::credentials::{TokenCredential, TokenRequestOptions}; +use azure_core::{Error, error::ErrorKind}; + +use azure_identity::{ + AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientSecretCredential, + ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, + WorkloadIdentityCredential, +}; + use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; -use azure_core::http::StatusCode; use bytes::Bytes; use futures::FutureExt; use snafu::Snafu; use vector_lib::{ + configurable::configurable_component, json_size::JsonSize, request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, + sensitive_string::SensitiveString, stream::DriverResponse, }; @@ -22,6 +33,191 @@ use crate::{ sinks::{Healthcheck, util::retries::RetryLogic}, }; +/// Configuration of the authentication strategy for interacting with Azure services. +#[configurable_component] +#[derive(Clone, Debug, Derivative, Eq, PartialEq)] +#[derivative(Default)] +#[serde(deny_unknown_fields, untagged)] +pub enum AzureAuthentication { + /// Use client credentials + #[derivative(Default)] + ClientSecretCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + azure_client_id: String, + + /// The [Azure Client Secret][azure_client_secret]. + /// + /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] + azure_client_secret: SensitiveString, + }, + + /// Use credentials from environment variables + #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] + Specific(SpecificAzureCredential), +} + +/// Specific Azure credential types. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde( + tag = "azure_credential_kind", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum SpecificAzureCredential { + /// Use Azure CLI credentials + #[cfg(not(target_arch = "wasm32"))] + AzureCli {}, + + /// Use Managed Identity credentials + ManagedIdentity { + /// The User Assigned Managed Identity (Client ID) to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + }, + + /// Use Managed Identity with Client Assertion credentials + ManagedIdentityClientAssertion { + /// The User Assigned Managed Identity (Client ID) to use for the managed identity. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The target Tenant ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_tenant_id: String, + + /// The target Client ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_client_id: String, + }, + + /// Use Workload Identity credentials + WorkloadIdentity {}, +} + +#[derive(Debug)] +struct ManagedIdentityClientAssertion { + credential: Arc, + scope: String, +} + +#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] +impl ClientAssertion for ManagedIdentityClientAssertion { + async fn secret(&self, options: Option>) -> azure_core::Result { + Ok(self + .credential + .get_token( + &[&self.scope], + Some(TokenRequestOptions { + method_options: options.unwrap_or_default(), + }), + ) + .await? + .token + .secret() + .to_string()) + } +} + +impl AzureAuthentication { + /// Returns the provider for the credentials based on the authentication mechanism chosen. + pub async fn credential(&self) -> azure_core::Result> { + match self { + Self::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + } => { + if azure_tenant_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_secret.inner().is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + let secret: String = azure_client_secret.inner().into(); + let credential: Arc = ClientSecretCredential::new( + &azure_tenant_id.clone(), + azure_client_id.clone(), + secret.into(), + None, + )?; + Ok(credential) + } + + Self::Specific(specific) => specific.credential().await, + } + } +} + +impl SpecificAzureCredential { + /// Returns the provider for the credentials based on the specific credential type. + pub async fn credential(&self) -> azure_core::Result> { + let credential: Arc = match self { + #[cfg(not(target_arch = "wasm32"))] + Self::AzureCli {} => AzureCliCredential::new(None)?, + + Self::ManagedIdentity { + user_assigned_managed_identity_id, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); + } + ManagedIdentityCredential::new(Some(options))? + } + + Self::ManagedIdentityClientAssertion { + user_assigned_managed_identity_id, + client_assertion_tenant_id, + client_assertion_client_id, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); + } + let msi: Arc = ManagedIdentityCredential::new(Some(options))?; + let assertion = ManagedIdentityClientAssertion { + credential: msi, + // Future: make this configurable for sovereign clouds? (no way to test...) + scope: "api://AzureADTokenExchange/.default".to_string(), + }; + + ClientAssertionCredential::new( + client_assertion_tenant_id.clone(), + client_assertion_client_id.clone(), + assertion, + None, + )? + } + + Self::WorkloadIdentity {} => WorkloadIdentityCredential::new(None)?, + }; + Ok(credential) + } +} + #[derive(Debug, Clone)] pub struct AzureBlobRequest { pub blob_data: Bytes, diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 4e876a18a26bd..2649edc320a89 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -1,20 +1,14 @@ use std::sync::Arc; -use azure_core::credentials::{TokenCredential, TokenRequestOptions}; -use azure_core::http::ClientMethodOptions; -use azure_core::{Error, error::ErrorKind}; - -use azure_identity::{ - AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientSecretCredential, - ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, - WorkloadIdentityCredential, -}; -use vector_lib::{configurable::configurable_component, schema, sensitive_string::SensitiveString}; +use azure_core::credentials::TokenCredential; + +use vector_lib::{configurable::configurable_component, schema}; use vrl::value::Kind; use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ + azure_common::config::AzureAuthentication, prelude::*, util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, }, @@ -129,191 +123,6 @@ impl Default for AzureLogsIngestionConfig { } } -/// Configuration of the authentication strategy for interacting with Azure services. -#[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] -#[serde(deny_unknown_fields, untagged)] -pub enum AzureAuthentication { - /// Use client credentials - #[derivative(Default)] - ClientSecretCredential { - /// The [Azure Tenant ID][azure_tenant_id]. - /// - /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_tenant_id: String, - - /// The [Azure Client ID][azure_client_id]. - /// - /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_client_id: String, - - /// The [Azure Client Secret][azure_client_secret]. - /// - /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] - azure_client_secret: SensitiveString, - }, - - /// Use credentials from environment variables - #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] - Specific(SpecificAzureCredential), -} - -/// Specific Azure credential types. -#[configurable_component] -#[derive(Clone, Debug, Eq, PartialEq)] -#[serde( - tag = "azure_credential_kind", - rename_all = "snake_case", - deny_unknown_fields -)] -pub enum SpecificAzureCredential { - /// Use Azure CLI credentials - #[cfg(not(target_arch = "wasm32"))] - AzureCli {}, - - /// Use Managed Identity credentials - ManagedIdentity { - /// The User Assigned Managed Identity (Client ID) to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - }, - - /// Use Managed Identity with Client Assertion credentials - ManagedIdentityClientAssertion { - /// The User Assigned Managed Identity (Client ID) to use for the managed identity. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - - /// The target Tenant ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_tenant_id: String, - - /// The target Client ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_client_id: String, - }, - - /// Use Workload Identity credentials - WorkloadIdentity {}, -} - -#[derive(Debug)] -struct ManagedIdentityClientAssertion { - credential: Arc, - scope: String, -} - -#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] -impl ClientAssertion for ManagedIdentityClientAssertion { - async fn secret(&self, options: Option>) -> azure_core::Result { - Ok(self - .credential - .get_token( - &[&self.scope], - Some(TokenRequestOptions { - method_options: options.unwrap_or_default(), - }), - ) - .await? - .token - .secret() - .to_string()) - } -} - -impl AzureAuthentication { - /// Returns the provider for the credentials based on the authentication mechanism chosen. - pub async fn credential(&self) -> azure_core::Result> { - match self { - Self::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - if azure_tenant_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_secret.inner().is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - let secret: String = azure_client_secret.inner().into(); - let credential: Arc = ClientSecretCredential::new( - &azure_tenant_id.clone(), - azure_client_id.clone(), - secret.into(), - None, - )?; - Ok(credential) - } - - Self::Specific(specific) => specific.credential().await, - } - } -} - -impl SpecificAzureCredential { - /// Returns the provider for the credentials based on the specific credential type. - pub async fn credential(&self) -> azure_core::Result> { - let credential: Arc = match self { - #[cfg(not(target_arch = "wasm32"))] - Self::AzureCli {} => AzureCliCredential::new(None)?, - - Self::ManagedIdentity { - user_assigned_managed_identity_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - ManagedIdentityCredential::new(Some(options))? - } - - Self::ManagedIdentityClientAssertion { - user_assigned_managed_identity_id, - client_assertion_tenant_id, - client_assertion_client_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - let msi: Arc = ManagedIdentityCredential::new(Some(options))?; - let assertion = ManagedIdentityClientAssertion { - credential: msi, - // Future: make this configurable for sovereign clouds? (no way to test...) - scope: "api://AzureADTokenExchange/.default".to_string(), - }; - - ClientAssertionCredential::new( - client_assertion_tenant_id.clone(), - client_assertion_client_id.clone(), - assertion, - None, - )? - } - - Self::WorkloadIdentity {} => WorkloadIdentityCredential::new(None)?, - }; - Ok(credential) - } -} - impl AzureLogsIngestionConfig { #[allow(clippy::too_many_arguments)] pub(super) async fn build_inner( diff --git a/src/sinks/azure_logs_ingestion/tests.rs b/src/sinks/azure_logs_ingestion/tests.rs index 26eda91338836..fd9e01835db2b 100644 --- a/src/sinks/azure_logs_ingestion/tests.rs +++ b/src/sinks/azure_logs_ingestion/tests.rs @@ -8,6 +8,8 @@ use vector_lib::config::log_schema; use azure_core::credentials::{AccessToken, TokenCredential}; use azure_core::time::OffsetDateTime; +use crate::sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}; + use super::config::AzureLogsIngestionConfig; use crate::{ @@ -48,7 +50,7 @@ async fn basic_config_error_with_no_auth() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { + crate::sinks::azure_common::config::AzureAuthentication::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, @@ -105,7 +107,7 @@ fn basic_config_with_client_credentials() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { + AzureAuthentication::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, @@ -146,11 +148,7 @@ fn basic_config_with_managed_identity() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::Specific( - crate::sinks::azure_logs_ingestion::config::SpecificAzureCredential::ManagedIdentity { - .. - }, - ) => { + AzureAuthentication::Specific(SpecificAzureCredential::ManagedIdentity { .. }) => { // Expected variant } _ => panic!("Expected Specific(ManagedIdentity) variant"), From cf02890f81a3832b8ae951f92b9976df3b6e025d Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 24 Feb 2026 20:08:07 +0000 Subject: [PATCH 02/26] add auth to azure_blob Signed-off-by: Jed Laundry --- Cargo.toml | 2 +- src/sinks/azure_blob/config.rs | 8 ++++++- src/sinks/azure_blob/test.rs | 44 ++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 40cbe17479b3d..479beb235b8a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -857,7 +857,7 @@ sinks-aws_s3 = ["dep:base64", "dep:md-5", "aws-core", "dep:aws-sdk-s3"] sinks-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] sinks-aws_sns = ["aws-core", "dep:aws-sdk-sns"] sinks-axiom = ["sinks-http"] -sinks-azure_blob = ["dep:azure_core", "dep:azure_storage_blob"] +sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob"] sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity"] sinks-azure_monitor_logs = [] sinks-blackhole = [] diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index db0521b4fa137..ff9ace574a9ee 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -16,7 +16,8 @@ use crate::{ sinks::{ Healthcheck, VectorSink, azure_common::{ - self, config::AzureBlobRetryLogic, service::AzureBlobService, sink::AzureBlobSink, + self, config::AzureAuthentication, config::AzureBlobRetryLogic, + service::AzureBlobService, sink::AzureBlobSink, }, util::{ BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt, @@ -41,6 +42,10 @@ impl TowerRequestConfigDefaults for AzureBlobTowerRequestConfigDefaults { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct AzureBlobSinkConfig { + #[configurable(derived)] + #[serde(default)] + pub auth: Option, + /// The Azure Blob Storage Account connection string. /// /// Authentication with an access key or shared access signature (SAS) @@ -147,6 +152,7 @@ pub fn default_blob_prefix() -> Template { impl GenerateConfig for AzureBlobSinkConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { + auth: None, connection_string: String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into(), container_name: String::from("logs"), blob_prefix: default_blob_prefix(), diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index b7f1a9689229f..d5cf4e830c16d 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -14,6 +14,8 @@ use super::{config::AzureBlobSinkConfig, request_builder::AzureBlobRequestOption use crate::{ codecs::{Encoder, EncodingConfigWithFraming}, event::{Event, LogEvent}, + sinks::azure_common::config::AzureAuthentication, + sinks::prelude::*, sinks::util::{ Compression, request_builder::{EncodeResult, RequestBuilder}, @@ -22,6 +24,7 @@ use crate::{ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { AzureBlobSinkConfig { + auth: Default::default(), connection_string: Default::default(), container_name: Default::default(), blob_prefix: Default::default(), @@ -234,3 +237,44 @@ fn azure_blob_build_request_with_uuid() { assert_eq!(request.content_encoding, None); assert_eq!(request.content_type, "text/plain"); } + +#[tokio::test] +async fn azure_blob_build_config_with_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + } => { + assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); + assert_eq!(azure_client_id, "mock-client-id"); + let secret: String = azure_client_secret.inner().into(); + assert_eq!(secret, "mock-client-secret"); + } + _ => panic!("Expected ClientSecretCredential variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} From f560854c1ff9e6240f8d3b4d16273bf4d13bd174 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Wed, 25 Feb 2026 07:54:54 +0000 Subject: [PATCH 03/26] initial auth switch Signed-off-by: Jed Laundry --- src/sinks/azure_common/config.rs | 55 ++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index af61971588c7e..2f4a5ddafd244 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,6 +1,8 @@ use std::sync::Arc; use azure_core::error::Error as AzureCoreError; +use tokio::runtime::Handle; +use tokio::task; use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; use crate::sinks::azure_common::shared_key_policy::SharedKeyAuthorizationPolicy; @@ -323,6 +325,7 @@ pub fn build_healthcheck( } pub fn build_client( + auth: Option, connection_string: String, container_name: String, proxy: &crate::config::ProxyConfig, @@ -336,16 +339,26 @@ pub fn build_client( .map_err(|e| format!("Failed to build container URL: {e}"))?; let url = Url::parse(&container_url).map_err(|e| format!("Invalid container URL: {e}"))?; + let mut credential: Option> = None; + // Prepare options; attach Shared Key policy if needed let mut options = BlobContainerClientOptions::default(); - match parsed.auth() { - Auth::Sas { .. } | Auth::None => { - // No extra policy; SAS is in the URL already (or anonymous) + match (parsed.auth(), &auth) { + (Auth::None { .. }, None) => { + warn!("No authentication method provided, requests will be anonymous"); + } + (Auth::Sas { .. }, None) => { + info!("Using SAS token authentication"); } + ( Auth::SharedKey { account_name, account_key, - } => { + }, + None, + ) => { + info!("Using Shared Key authentication"); + let policy = SharedKeyAuthorizationPolicy::new( account_name, account_key, @@ -358,6 +371,36 @@ pub fn build_client( .per_call_policies .push(Arc::new(policy)); } + (Auth::None { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { + info!("Using Client Secret authentication"); + let async_credential_result = task::block_in_place(|| { + Handle::current().block_on(async { auth.unwrap().credential().await.unwrap() }) + }); + credential = Some(async_credential_result); + } + (Auth::None { .. }, Some(AzureAuthentication::Specific(..))) => { + info!("Using specific Azure Authentication method"); + let async_credential_result = task::block_in_place(|| { + Handle::current().block_on(async { auth.unwrap().credential().await.unwrap() }) + }); + credential = Some(async_credential_result); + } + (Auth::Sas { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { + panic!("Cannot use both SAS token and Client ID/Secret at the same time"); + } + (Auth::SharedKey { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { + panic!("Cannot use both Shared Key and Client ID/Secret at the same time"); + } + (Auth::Sas { .. }, Some(AzureAuthentication::Specific(..))) => { + panic!( + "Cannot use both SAS token and another Azure Authentication method at the same time" + ); + } + (Auth::SharedKey { .. }, Some(AzureAuthentication::Specific(..))) => { + panic!( + "Cannot use both Shared Key and another Azure Authentication method at the same time" + ); + } } // Use reqwest v0.12 since Azure SDK only implements HttpClient for reqwest::Client v0.12 @@ -392,7 +435,7 @@ pub fn build_client( .build() .map_err(|e| format!("Failed to build reqwest client: {e}"))?, ))); - let client = - BlobContainerClient::from_url(url, None, Some(options)).map_err(|e| format!("{e}"))?; + let client = BlobContainerClient::from_url(url, credential, Some(options)) + .map_err(|e| format!("{e}"))?; Ok(Arc::new(client)) } From 2903a28e195b10491dd0c05a558bdc7427bc7361 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Wed, 25 Feb 2026 07:57:00 +0000 Subject: [PATCH 04/26] integration test WIP Signed-off-by: Jed Laundry --- src/sinks/azure_blob/config.rs | 8 ++ src/sinks/azure_blob/integration_tests.rs | 70 ++++++++++++- src/sinks/azure_blob/test.rs | 1 + src/sinks/azure_common/config.rs | 22 ++++- tests/data/Makefile | 18 ++++ .../certs/azurite-chain.cert.pem | 98 +++++++++++++++++++ .../certs/azurite.cert.pem | 32 ++++++ .../intermediate_server/csr/azurite.csr.pem | 17 ++++ tests/data/ca/intermediate_server/index.txt | 1 + .../data/ca/intermediate_server/index.txt.old | 1 + .../ca/intermediate_server/newcerts/1009.pem | 32 ++++++ .../private/azurite.key.pem | 28 ++++++ tests/data/ca/intermediate_server/serial | 2 +- tests/data/ca/intermediate_server/serial.old | 2 +- tests/integration/azure/config/compose.yaml | 8 ++ tests/integration/azure/config/test.yaml | 3 +- 16 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem create mode 100644 tests/data/ca/intermediate_server/certs/azurite.cert.pem create mode 100644 tests/data/ca/intermediate_server/csr/azurite.csr.pem create mode 100644 tests/data/ca/intermediate_server/newcerts/1009.pem create mode 100644 tests/data/ca/intermediate_server/private/azurite.key.pem diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index ff9ace574a9ee..f15a8af40359e 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -25,6 +25,7 @@ use crate::{ }, }, template::Template, + tls::TlsConfig, }; #[derive(Clone, Copy, Debug)] @@ -143,6 +144,10 @@ pub struct AzureBlobSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[serde(default)] + #[configurable(derived)] + pub tls: Option, } pub fn default_blob_prefix() -> Template { @@ -163,6 +168,7 @@ impl GenerateConfig for AzureBlobSinkConfig { batch: BatchConfig::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), + tls: None, }) .unwrap() } @@ -173,9 +179,11 @@ impl GenerateConfig for AzureBlobSinkConfig { impl SinkConfig for AzureBlobSinkConfig { async fn build(&self, cx: SinkContext) -> Result<(VectorSink, Healthcheck)> { let client = azure_common::config::build_client( + self.auth.clone(), self.connection_string.clone().into(), self.container_name.clone(), cx.proxy(), + self.tls.clone(), )?; let healthcheck = azure_common::config::build_healthcheck( diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 63943312fe66d..9014ac47929ea 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -5,6 +5,7 @@ use azure_core::http::StatusCode; use bytes::{Buf, BytesMut}; use flate2::read::GzDecoder; use futures::{Stream, StreamExt, stream}; +use vector_common::sensitive_string::SensitiveString; use vector_lib::{ ByteSizeOf, codecs::{ @@ -24,15 +25,18 @@ use crate::{ components::{SINK_TAGS, assert_sink_compliance}, random_events_with_stream, random_lines, random_lines_with_stream, random_string, }, + tls::{self, TlsConfig}, }; #[tokio::test] async fn azure_blob_healthcheck_passed() { let config = AzureBlobSinkConfig::new_emulator().await; let client = azure_common::config::build_client( + None, config.connection_string.clone().into(), config.container_name.clone(), &crate::config::ProxyConfig::default(), + None, ) .expect("Failed to create client"); @@ -50,9 +54,11 @@ async fn azure_blob_healthcheck_unknown_container() { ..config }; let client = azure_common::config::build_client( + None, config.connection_string.clone().into(), config.container_name.clone(), &crate::config::ProxyConfig::default(), + None, ) .expect("Failed to create client"); @@ -211,10 +217,33 @@ async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { } } +#[tokio::test] +async fn azure_blob_insert_lines_into_blob_with_oauth() { + let blob_prefix = format!("lines/into/blob/{}", random_string(10)); + let config = AzureBlobSinkConfig::new_emulator_with_oauth().await; + let config = AzureBlobSinkConfig { + blob_prefix: blob_prefix.clone().try_into().unwrap(), + ..config + }; + let (lines, input) = random_lines_with_stream(100, 10, None); + + config.run_assert(input).await; + + let blobs = config.list_blobs(blob_prefix).await; + assert_eq!(blobs.len(), 1); + assert!(blobs[0].clone().ends_with(".log")); + let (content_type, content_encoding, blob_lines) = config.get_blob(blobs[0].clone()).await; + assert_eq!(content_type, Some(String::from("text/plain"))); + assert_eq!(content_encoding, None); + assert_eq!(lines, blob_lines); +} + impl AzureBlobSinkConfig { pub async fn new_emulator() -> AzureBlobSinkConfig { - let address = std::env::var("AZURE_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { + auth: None, + tls: None, connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into(), container_name: "logs".to_string(), blob_prefix: Default::default(), @@ -232,11 +261,44 @@ impl AzureBlobSinkConfig { config } + pub async fn new_emulator_with_oauth() -> AzureBlobSinkConfig { + let client_secret_credential = + azure_common::config::AzureAuthentication::ClientSecretCredential { + azure_tenant_id: "00000000-0000-0000-0000-000000000000".to_string(), + azure_client_id: "00000000-0000-0000-0000-000000000000".to_string(), + azure_client_secret: SensitiveString::from("mock-secret".to_string()), + }; + let address = std::env::var("AZURITE_OAUTH_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let config = AzureBlobSinkConfig { + auth: Some(client_secret_credential), + tls: Some(TlsConfig { + ca_file: Some(tls::TEST_PEM_CA_PATH.into()), + ..Default::default() + }), + connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into(), + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + }; + + config.ensure_container().await; + + config + } + fn to_sink(&self) -> VectorSink { let client = azure_common::config::build_client( + None, self.connection_string.clone().into(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + None, ) .expect("Failed to create client"); @@ -252,9 +314,11 @@ impl AzureBlobSinkConfig { pub async fn list_blobs(&self, prefix: String) -> Vec { let client = azure_common::config::build_client( + None, self.connection_string.clone().into(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + None, ) .unwrap(); @@ -277,9 +341,11 @@ impl AzureBlobSinkConfig { pub async fn get_blob(&self, blob: String) -> (Option, Option, Vec) { let client = azure_common::config::build_client( + None, self.connection_string.clone().into(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + None, ) .unwrap(); @@ -338,9 +404,11 @@ impl AzureBlobSinkConfig { async fn ensure_container(&self) { let client = azure_common::config::build_client( + None, self.connection_string.clone().into(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + None, ) .unwrap(); let result = client.create_container(None).await; diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index d5cf4e830c16d..190d0f4ed9292 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -35,6 +35,7 @@ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { batch: Default::default(), request: Default::default(), acknowledgements: Default::default(), + tls: Default::default(), } } diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 2f4a5ddafd244..74e7c0de347fe 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,3 +1,5 @@ +use std::fs::File; +use std::io::Read; use std::sync::Arc; use azure_core::error::Error as AzureCoreError; @@ -33,6 +35,7 @@ use vector_lib::{ use crate::{ event::{EventFinalizers, EventStatus, Finalizable}, sinks::{Healthcheck, util::retries::RetryLogic}, + tls::TlsConfig, }; /// Configuration of the authentication strategy for interacting with Azure services. @@ -329,6 +332,7 @@ pub fn build_client( connection_string: String, container_name: String, proxy: &crate::config::ProxyConfig, + tls: Option, ) -> crate::Result> { // Parse connection string without legacy SDK let parsed = ParsedConnectionString::parse(&connection_string) @@ -351,9 +355,9 @@ pub fn build_client( info!("Using SAS token authentication"); } ( - Auth::SharedKey { - account_name, - account_key, + Auth::SharedKey { + account_name, + account_key, }, None, ) => { @@ -430,6 +434,18 @@ pub fn build_client( reqwest_builder = reqwest_builder.proxy(p); } } + + if let Some(tls_config) = tls { + if let Some(ca_file) = tls_config.ca_file { + let mut buf = Vec::new(); + File::open(&ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_12::Certificate::from_pem(&buf)?; + + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); + } + } + options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( reqwest_builder .build() diff --git a/tests/data/Makefile b/tests/data/Makefile index f3550bf8bf100..32a70afa57db6 100644 --- a/tests/data/Makefile +++ b/tests/data/Makefile @@ -61,6 +61,24 @@ ca/intermediate_server/certs/localhost.cert.pem: ca/intermediate_server/csr/loca ca/intermediate_server/certs/localhost-chain.cert.pem: ca/intermediate_server/certs/ca-chain.cert.pem ca/intermediate_server/certs/localhost.cert.pem cat ca/intermediate_server/certs/localhost.cert.pem ca/intermediate_server/certs/ca-chain.cert.pem > ca/intermediate_server/certs/localhost-chain.cert.pem +ca/intermediate_server/private/azurite.key.pem: + openssl genrsa -out ca/intermediate_server/private/azurite.key.pem 2048 + +ca/intermediate_server/csr/azurite.csr.pem: ca/intermediate_server/private/azurite.key.pem + openssl req -config ca/intermediate_server/openssl.cnf \ + -key ca/intermediate_server/private/azurite.key.pem \ + -subj '/CN=azurite/OU=Vector/O=Datadog/ST=New York/L=New York/C=US' \ + -new -sha256 -out ca/intermediate_server/csr/azurite.csr.pem + +ca/intermediate_server/certs/azurite.cert.pem: ca/intermediate_server/csr/azurite.csr.pem + openssl ca -batch -config ca/intermediate_server/openssl.cnf \ + -extensions server_cert -days 3650 -notext -md sha256 \ + -in ca/intermediate_server/csr/azurite.csr.pem \ + -out ca/intermediate_server/certs/azurite.cert.pem + +ca/intermediate_server/certs/azurite-chain.cert.pem: ca/intermediate_server/certs/ca-chain.cert.pem ca/intermediate_server/certs/azurite.cert.pem + cat ca/intermediate_server/certs/azurite.cert.pem ca/intermediate_server/certs/ca-chain.cert.pem > ca/intermediate_server/certs/azurite-chain.cert.pem + ca/intermediate_server/private/elasticsearch-secure.key.pem: openssl genrsa -out ca/intermediate_server/private/elasticsearch-secure.key.pem 2048 diff --git a/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem new file mode 100644 index 0000000000000..934f433785fe7 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem @@ -0,0 +1,98 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDIyNDIxNDY0MFoXDTM2MDIyMjIxNDY0MFowaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAArqDt0rNQNa +eu18+YgML201hSK9AWatk4ew9HogDhpIiqqwfDY1okTHBTWKdMRIOzD7EOvlbEMl +1fqPb09DVTuOrV1vWpBkbvHIvHYn/v28mslUIRbIw1qczdWLGPhzNdXnbVI0D47T +B3vzrXlSJlZbaDtvjoVIm2Jhq/i+1fVO4adbuWDXPhjzZq1eKWC5wQOYyXyeZAQo +0RwbqXnuGeJqUeo1/6sr/ft5v+1W62yHHaKZWiUIcrSqiv3bFIBexvzIZV34VHwN +p2JgVlmP6TXtyTouwFdx3FTBmS0kUViYW1AY8nGV3ZMmOIlqYCs3y78lVhxdjXnG +q/9U1xaAaXfEaPOu/19J14HAXJ9jnVlq/civCDWTbJvIv1AxblGpa8rz/0HlI0Pd +/T1OfAUD/e5FAqn8w2psnJGwGyMp+Mr32Ip+fNt5IHm1dhMv0WAglSLJb0L1EQe9 +GnGDqLOED1MfE5rLukJ+yXp2PWTkLgMmCXBMf4WdH1sUNRGaOzkNf6BB0c3eBJDP +gMIwmI13yBWw141BTICKmYCF+W928BQBKVFTpbI/buRs44eGWbwDx7R8qX9S3f0y +VN3WxS0kB5NYsKTbuoeuzWVtDz/zRxvod7l/fpYWbonZqp6Pa9EHuCkJF+5DSQ8P +7+iB0vOdu7mvLW4IrJnd7WXphAB16r+B +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFtzCCA5+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwajESMBAGA1UEAwwJVmVj +dG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNV +BAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMwHhcN +MjIwNjA3MjIyNzUzWhcNMzIwNjA0MjIyNzUzWjBrMQswCQYDVQQGEwJVUzERMA8G +A1UECAwITmV3IFlvcmsxEDAOBgNVBAoMB0RhdGFkb2cxDzANBgNVBAsMBlZlY3Rv +cjEmMCQGA1UEAwwdVmVjdG9yIEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCy/mB0/ZwfgKrSZPQIFaGPtRA9xL2N +o2SsHndZ8k2XOCV225Hb2fzNH+o2WGNSjwmGjLP/uXb47KH0cHCAyFGzSjp++8/O +zoZaFiO0P5El02hQxmoabO3Cqu/N62EFsLfpSM828JM6YOn9p+WXUDn1+YPNoOOE +H142p4/RjFnXNHkzR3geXU4Pfi3KXDrMi8vK42lDqXPLPs6rhreBAfQ2dsYyqhz6 +tg6FzZuXxxzEYyYtNgGh+zTji99WCBMLbCmRcDurRjdTDO7m4O3PrwbGUy0xdLeb +HJiNGvUDCPH4bfwLiNqwVIZY38RBCAqbCnrqRhDaZIfAUev4mq3Kqh6KUeO/U7Vx +/5J5rL5ApREKOfWPATHMprBuEU2rs3N+MPBA04HoiFlu311urCxVEA1qsZCTkoCg +GHuDIVSU4E4hT4co95/J0to4zWgPlfPg1+cXyU8lAIMe7JdCGkG9cDe7Umw/GSbt +ZdoCMQZ6WyyiW2Hw+7sFD3V3VzYa5YA/rjKZRduPmGWKrs+mAa5J5pM2M22rrjbd +EpfTHWLS9s6cPN3/jxpCxn6Hv/KhIYRAcIterugag1+clvS1ajVjxBRavOxPBsf+ +hYdh7S5NTZnT98gjkc3yOuGQm7BPtXau+IYZRlWcB0dJ4/E2P69hmWQezSo9VVWh +5/K1RkbPvqTGZQIDAQABo2YwZDAdBgNVHQ4EFgQUPD06L8zVggN9mcRY8eHbNu+t +DUEwHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wEgYDVR0TAQH/BAgw +BgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAGqaGBuL +2J6Na8RHx/GmSeuZFiVcWhmd/I9bVpeMSYHSZujA2nay6OGaUYs0Lq/G5OKgsuT9 +AIHnsn7VUE1zqoDfXac/K8sXlOig8if7rTb+06jgymaP1YSELg3R+pBsdkZnXVil +izh/9FvzoyV+QQlIhojqCIybVFgxa1XFHq4QCPhDfwkg+tp9RctfwNmWgsJ63H19 +RmxN+H2xIrySvObwXnB4j6D4wvgu468QXQMEuSsnLcIQFg6Zteqe8fixbqTiOTBf +Dk1k+EpB9VMEkIPvMdfa48vseXdBEe6Ma9zGuJC76q4q1ZapVLTvOUP5Y24khlgd +cj5tfP7o7yc6HqymfXAcD1lzP2JQhqaRxA4I18Nrd+aHi+G1EM2c3cicuD3n6Iw9 +9oqdCwmMfS25fv5cyA5B6hRusIZ9wRopTi7at+JHl0GIt/FelaTYI7kRmAqgakQe +oEKLpXcH8lRJW802DmXm7ka4eQzwxa7Ngyf8O+JOFtGO0+EshuLJovxiPl6IyLyG +NJ/dHq3ad+46YVManbHdyjHxgT5PSvJFkq0Yluvf44NIyP5QRTCAvfH76bu7hXgS +QoQj5t5ILn6meQRTR79r2iwpQTanPLTEdoZvmrE4TeUBev9BA5KpiPPA3i3ZF/oV +0EYorXCNri7M/jylGW7AuWvNUyaVR6xgxAn6 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFujCCA6KgAwIBAgIJAKhPL9BkNaFGMA0GCSqGSIb3DQEBCwUAMGoxEjAQBgNV +BAMMCVZlY3RvciBDQTEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9n +MREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxCzAJBgNVBAYT +AlVTMB4XDTIyMDYwNzIyMjc1MloXDTQyMDYwMjIyMjc1MlowajESMBAGA1UEAwwJ +VmVjdG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9c1T+NXTNmqiiV36NSEJt +7mo0cyv8Byk2ZGdC85vHBm45QDY5USoh0vgonzPpWgSMggPn1WbR0f1y+LBwXdlM ++ZyZh2RVVeUrSjJ88lLHVn4DfywpdDkwQaFj1VmOsj2I9rMMrgc5x5n1Hj7lwZ+t +uPVSAGmgKp4iFfzLph9r/rjP1TUAnVUComfTUVS+Gd7zoGPOc14cMJXG6g2P2aAU +P6dg5uQlTxRmagnlx7bwm3lRwv6LMtnAdnjwBDBxr933nucAnk21GgE92GejiO3Z +OwlzIdzBI23lPcWi5pq+vCTgAArNq24W1Ha+7Jn5QewNTGKFyyYAJetZAwCUR8QS +Ip++2GE2pNhaGqcV5u1Tbwl02eD6p2qRqjfgLxmb+aC6xfl0n9kiFGPZppjCqDEW +sw+gX66nf+qxZVRWpJon2kWcFvhTnLqoa3T3+9+KIeamz2lW6wxMnki/Co2EA1Wa +mmedaUUcRPCgMx9aCktRkMyH6bEY8/vfJ07juxUsszOc46T00Scmn6Vkuo9Uc3Kf +2Q2N6Wo4jtyAiMO4gAwq5kzzpBAhNgRfLHOb83r2gAUj2Y4Vln/UUR/KR8ZbJi4i +r1BjX16Lz3yblJXXb1lp4uZynlbHNaAevXyGlRqHddM2ykKtAX/vgJcZRGSvms11 +uce/cqzrzx60AhpLRma5CwIDAQABo2MwYTAdBgNVHQ4EFgQURTWK6ARqnZkz8rkt +Uc5PrtasIh8wHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAEf5 +TR3hq/DtSAmsYotu1lAWz/OlTpG+7AdqSOHB878X4ETN3xaQ+KWvSwvf0K70ZDTV +tFOTh/r43cpzPifPKd1P+2ctnQEzrBtAacvyETLq1ABRK9VJOtfJ6Xk5KZXPhKdY +t353PQgBgW8YzQ2adq2B7FtgIlX7f1DIndjcMZBbolETR6xt9QwB/UnPI7Mwt01T ++bCBhr1fWAbZ4YAMlQ0xRam4qUOTjxgfmePrmSrv4HO7cXHMsRMLiXk+BLcx959/ +K/B6xzpzn6366Eqnqlo/uDiMpo5ud2I/Snz5PduB6oLztPMEf/8RmkG5tpHXYdWr +tM64WqNGO+ikluIrrtYvtyZS4DfsLAMfMYZcxX/Uw56gHo0i2c8I6+6JvGWdvOJ0 +FjrsKeIQoRlV77z025kI4V9jKi3XNMEsAIH+W7KNSut0X80yX7SugvQGoe0GDkXu +0fy8hMC3uTN2LEycYFRRfoIeKPLi6OZFK0PdS2E15d8PEU3n3W4eBCPgMtmiOKLY +d8QNBC8XLAuBoK9R8luCJpOJWUcFXjLpjcDab4V2hKTuAs+GQyDh/Xx4wF1yHX0r +zIkyN0EkOD/SvD8X4uFaM4mdsAh+ucn4ryUV7i5PgvDM9z4InHAMAee1ebBl0U+h ++NzMWF5c5OwxD5o6/Wh1HopmzJiVNT2v9u0kHT/f +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/certs/azurite.cert.pem b/tests/data/ca/intermediate_server/certs/azurite.cert.pem new file mode 100644 index 0000000000000..6db5de3454389 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDIyNDIxNDY0MFoXDTM2MDIyMjIxNDY0MFowaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAArqDt0rNQNa +eu18+YgML201hSK9AWatk4ew9HogDhpIiqqwfDY1okTHBTWKdMRIOzD7EOvlbEMl +1fqPb09DVTuOrV1vWpBkbvHIvHYn/v28mslUIRbIw1qczdWLGPhzNdXnbVI0D47T +B3vzrXlSJlZbaDtvjoVIm2Jhq/i+1fVO4adbuWDXPhjzZq1eKWC5wQOYyXyeZAQo +0RwbqXnuGeJqUeo1/6sr/ft5v+1W62yHHaKZWiUIcrSqiv3bFIBexvzIZV34VHwN +p2JgVlmP6TXtyTouwFdx3FTBmS0kUViYW1AY8nGV3ZMmOIlqYCs3y78lVhxdjXnG +q/9U1xaAaXfEaPOu/19J14HAXJ9jnVlq/civCDWTbJvIv1AxblGpa8rz/0HlI0Pd +/T1OfAUD/e5FAqn8w2psnJGwGyMp+Mr32Ip+fNt5IHm1dhMv0WAglSLJb0L1EQe9 +GnGDqLOED1MfE5rLukJ+yXp2PWTkLgMmCXBMf4WdH1sUNRGaOzkNf6BB0c3eBJDP +gMIwmI13yBWw141BTICKmYCF+W928BQBKVFTpbI/buRs44eGWbwDx7R8qX9S3f0y +VN3WxS0kB5NYsKTbuoeuzWVtDz/zRxvod7l/fpYWbonZqp6Pa9EHuCkJF+5DSQ8P +7+iB0vOdu7mvLW4IrJnd7WXphAB16r+B +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/csr/azurite.csr.pem b/tests/data/ca/intermediate_server/csr/azurite.csr.pem new file mode 100644 index 0000000000000..8de7cd20cda66 --- /dev/null +++ b/tests/data/ca/intermediate_server/csr/azurite.csr.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICrTCCAZUCAQAwaDEQMA4GA1UEAwwHYXp1cml0ZTEPMA0GA1UECwwGVmVjdG9y +MRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwI +TmV3IFlvcmsxCzAJBgNVBAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVPbGW0LqD1WO42dSmUMQ2iRGsF +eHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bRRInmuyocDTvm4t7VU+ybnPD7 +MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3QZ4loS61BR5TxrCdnpE++gxh +7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ63PaXXvGpGiDwPqUqi0WhIYp +73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFnVr8lMVavcDcoTrHdJ8PXN4Ei +pgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABoAAwDQYJKoZIhvcNAQELBQADggEB +AANtDoLqI/uQEn3u/xe9R539MjjzP2t7y8HWwaSJBJrNwkiHIub7NfMHOdyIPhLt +jPV6QqHWaWFhj0cW5YeBifcO7t/F523VGcH1GawaTNBIfemmpeCXDEkWOSDnM9Fp +N/J3kEgUUg9/Lw+0ygzR5oXUGx9srBK9M8gqgeWF4kpDlYTiTsekmAU+Td9GcxE7 +IrqK8kTx25CIcRrA00iKTrQLLFEtVED3F+AFJ2mi03iYIVuu3t6ZNurSZZyOAd6H +I9mSC51Yife3GuabkzSW38FUFdJmE7pElmvD2WkU/5XpPhjxajkxJxAs5W54tesL +3TJv/xhzHzH1g4FikkDjZnw= +-----END CERTIFICATE REQUEST----- diff --git a/tests/data/ca/intermediate_server/index.txt b/tests/data/ca/intermediate_server/index.txt index 0402ad0946130..dfa1286cc5b99 100644 --- a/tests/data/ca/intermediate_server/index.txt +++ b/tests/data/ca/intermediate_server/index.txt @@ -6,3 +6,4 @@ V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar +V 360222214640Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite diff --git a/tests/data/ca/intermediate_server/index.txt.old b/tests/data/ca/intermediate_server/index.txt.old index ab4efe2c2f6bd..0402ad0946130 100644 --- a/tests/data/ca/intermediate_server/index.txt.old +++ b/tests/data/ca/intermediate_server/index.txt.old @@ -5,3 +5,4 @@ V 320613195026Z 1004 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=kafka V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq +V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar diff --git a/tests/data/ca/intermediate_server/newcerts/1009.pem b/tests/data/ca/intermediate_server/newcerts/1009.pem new file mode 100644 index 0000000000000..6db5de3454389 --- /dev/null +++ b/tests/data/ca/intermediate_server/newcerts/1009.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDIyNDIxNDY0MFoXDTM2MDIyMjIxNDY0MFowaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAArqDt0rNQNa +eu18+YgML201hSK9AWatk4ew9HogDhpIiqqwfDY1okTHBTWKdMRIOzD7EOvlbEMl +1fqPb09DVTuOrV1vWpBkbvHIvHYn/v28mslUIRbIw1qczdWLGPhzNdXnbVI0D47T +B3vzrXlSJlZbaDtvjoVIm2Jhq/i+1fVO4adbuWDXPhjzZq1eKWC5wQOYyXyeZAQo +0RwbqXnuGeJqUeo1/6sr/ft5v+1W62yHHaKZWiUIcrSqiv3bFIBexvzIZV34VHwN +p2JgVlmP6TXtyTouwFdx3FTBmS0kUViYW1AY8nGV3ZMmOIlqYCs3y78lVhxdjXnG +q/9U1xaAaXfEaPOu/19J14HAXJ9jnVlq/civCDWTbJvIv1AxblGpa8rz/0HlI0Pd +/T1OfAUD/e5FAqn8w2psnJGwGyMp+Mr32Ip+fNt5IHm1dhMv0WAglSLJb0L1EQe9 +GnGDqLOED1MfE5rLukJ+yXp2PWTkLgMmCXBMf4WdH1sUNRGaOzkNf6BB0c3eBJDP +gMIwmI13yBWw141BTICKmYCF+W928BQBKVFTpbI/buRs44eGWbwDx7R8qX9S3f0y +VN3WxS0kB5NYsKTbuoeuzWVtDz/zRxvod7l/fpYWbonZqp6Pa9EHuCkJF+5DSQ8P +7+iB0vOdu7mvLW4IrJnd7WXphAB16r+B +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/private/azurite.key.pem b/tests/data/ca/intermediate_server/private/azurite.key.pem new file mode 100644 index 0000000000000..545d8904a3d73 --- /dev/null +++ b/tests/data/ca/intermediate_server/private/azurite.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC+cDIKkK8OivJI +I1qy1Vb3Hf7fVNrhJU9sZbQuoPVY7jZ1KZQxDaJEawV4e8ziC5euFeWgFV/fkvU7 +JZGn9uW0eWV4uuvnxtFEiea7KhwNO+bi3tVT7Juc8PsyF02xsyajYcGfXZwRjs+z +PFU7BxnyPRc4e0j697dBniWhLrUFHlPGsJ2ekT76DGHsqFQbXJOIqYQS3nzu9Ei5 +UFY2ssVWFhA3qZz7Yknrc9pde8akaIPA+pSqLRaEhinvddXIipseK8jbtOK6/+f2 +Bwc8C3ZTBVu/UCqoUWdWvyUxVq9wNyhOsd0nw9c3gSKmC+ywkXXiAUWDsvRTgA6P +jG7fV76JAgMBAAECggEAEXeC73sjw0a1QC6V9A8jQdkrdlp3FO1yInQVma1Ds5tt +vUNKB1HDz1itkMZyHU2I5Pu3Kv6q43u8KGeiu3Am023LA87JMmIG8a1gT0xmdERJ +QgfEM0VhZHyp3YdLpf/TjGq78p0IYofhvwPKoXZeR9yYk6KjJ/mugkM6GlWJXuWn +sE2ZYL4Wl6d//1/9D6xWfNVU1zCH7ScbFFvQKkBxURLpnQlgkfhr8HMrCQyV4Lps +6n166zcQKJO4mwGrWyu75HU8vcBYv1+6WbJ1Rgke6lL3dzjIhzE7C61i3dI636Aq +l0wZ3NVeysyrn5Xzu5vfq0D9/pt+Of1PE0FmmlRogQKBgQDhaY8OyUafMYhGnike +7K1ho60tFHE5vgxMpoUTinLyARgW8bA9VRtQ++PW90Z1sHzXAHjb30YZcYEuKaCW +SvIWg49/L4bORX2jY4UPZvESemJtnIXZkuuiGoYaXjIwNudorT1siCD/GYM24Uko +ax0QbFsNRtq89QE/ibKdFLRfUQKBgQDYR7JfauKmsY6OQjB8RvFe50AnnPfiJSZ+ +rD7H7gXWlt3q9zpsU3xBK+fwI9QsvTlUNsIkFhJpu6znCzK26nLIJCqGcm0W+9Ah +rJR+0TXyafQn1d1dTrtn7XVIL6QQEuUY9jggsAXWBejdPCeaVxplEiJa1i9x3YOp +OQWmMTvNuQKBgCiGF6fq231nJD691Fqw5gK1sD54fFqLJh7pmOcIbt2/AJuvW6XL +FRwcDLvqvIoP7oGgnhm5LBsK4tRvu2UJmDgf8r5ExxFyQMIM9DDuqsxNoEBgcVfK +J/5+kjlPUeqFFFknO/G1D2mNJp/JJKPVjeYT9NKQOGbcDRtlH+1JeZvhAoGAZkOg +Z9WWTdNu4H0Th+/TeVhG0XQ7EUcXqJWxKb+2Kv0y+ULk8QuYmQg1pyqJzI28acFq +kr2M/0mqO6Tj2fGJTHEtWl0Ij/GJPCLqI/ywUWsf8yYAgXoUytNQvU0peiA1C1SA +vZP9bnFk5hbncub0qA2nCOR1kpV3B7DapvZonKECgYEAjzpfOhaULZgwnkYOkoFw +VS+tDOVNVfjDOjuV82B/0zCgCW8dv2EeTCobEkIbSl8GYMzukNZQR8r3ZAxbs1By +Ieliad2FD/s0MtsPiGcxJN4twbDQO/SvLFtjQ+jCysgOVON6KiqpGVRcEL+6U4ZD +NiqT36xmXpmkZ05bMxOXPLM= +-----END PRIVATE KEY----- diff --git a/tests/data/ca/intermediate_server/serial b/tests/data/ca/intermediate_server/serial index 6cb3869343bf8..4e75b247d2247 100644 --- a/tests/data/ca/intermediate_server/serial +++ b/tests/data/ca/intermediate_server/serial @@ -1 +1 @@ -1009 +100A diff --git a/tests/data/ca/intermediate_server/serial.old b/tests/data/ca/intermediate_server/serial.old index 617ba1c154075..6cb3869343bf8 100644 --- a/tests/data/ca/intermediate_server/serial.old +++ b/tests/data/ca/intermediate_server/serial.old @@ -1 +1 @@ -1008 +1009 diff --git a/tests/integration/azure/config/compose.yaml b/tests/integration/azure/config/compose.yaml index 933a72d235a24..3db399d5f78ca 100644 --- a/tests/integration/azure/config/compose.yaml +++ b/tests/integration/azure/config/compose.yaml @@ -7,6 +7,14 @@ services: volumes: - /var/run:/var/run + azurite: + image: mcr.microsoft.com/azure-storage/azurite:${CONFIG_VERSION} + command: azurite --blobHost 0.0.0.0 --blobPort 14430 --queuePort 14431 --tablePort 14432 --loose --oauth basic --cert /certs/azurite-chain.cert.pem --key /private/azurite.key.pem + volumes: + - /var/run:/var/run + - ../../../data/ca/intermediate_server/certs:/certs + - ../../../data/ca/intermediate_server/private/azurite.key.pem:/private/azurite.key.pem + networks: default: name: ${VECTOR_NETWORK} diff --git a/tests/integration/azure/config/test.yaml b/tests/integration/azure/config/test.yaml index 191bea7256cea..88f46966896ee 100644 --- a/tests/integration/azure/config/test.yaml +++ b/tests/integration/azure/config/test.yaml @@ -4,7 +4,8 @@ features: test_filter: ::azure_ env: - AZURE_ADDRESS: local-azure-blob + AZURITE_ADDRESS: local-azure-blob + AZURITE_OAUTH_ADDRESS: azurite HEARTBEAT_ADDRESS: 0.0.0.0:8080 LOGSTASH_ADDRESS: 0.0.0.0:8081 From 45d9e74ffe26ffff179cbc3343d46a6ccdb8b60d Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Wed, 25 Feb 2026 08:20:08 +0000 Subject: [PATCH 05/26] doc updates Signed-off-by: Jed Laundry --- .../components/sinks/generated/azure_blob.cue | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 5ea0c1dd20221..22b7bb60bc566 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -27,6 +27,67 @@ generated: components: sinks: azure_blob: configuration: { type: bool: {} } } + auth: { + description: "Configuration of the authentication strategy for interacting with Azure services." + required: false + type: object: options: { + azure_client_id: { + description: """ + The [Azure Client ID][azure_client_id]. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + azure_client_secret: { + description: """ + The [Azure Client Secret][azure_client_secret]. + + [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000"] + } + azure_credential_kind: { + description: "The kind of Azure credential to use." + required: true + type: string: enum: { + azure_cli: "Use Azure CLI credentials" + managed_identity: "Use Managed Identity credentials" + managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" + workload_identity: "Use Workload Identity credentials" + } + } + azure_tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_assertion_client_id: { + description: "The target Client ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_assertion_tenant_id: { + description: "The target Tenant ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + user_assigned_managed_identity_id: { + description: "The User Assigned Managed Identity (Client ID) to use." + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + } + } batch: { description: "Event batching behavior." required: false @@ -862,4 +923,98 @@ generated: components: sinks: azure_blob: configuration: { } } } + tls: { + description: "TLS configuration." + required: false + type: object: options: { + alpn_protocols: { + description: """ + Sets the list of supported ALPN protocols. + + Declare the supported ALPN protocols, which are used during negotiation with a peer. They are prioritized in the order + that they are defined. + """ + required: false + type: array: items: type: string: examples: ["h2"] + } + ca_file: { + description: """ + Absolute path to an additional CA certificate file. + + The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + crt_file: { + description: """ + Absolute path to a certificate file used to identify this server. + + The certificate must be in DER, PEM (X.509), or PKCS#12 format. Additionally, the certificate can be provided as + an inline string in PEM format. + + If this is set _and_ is not a PKCS#12 archive, `key_file` must also be set. + """ + required: false + type: string: examples: ["/path/to/host_certificate.crt"] + } + key_file: { + description: """ + Absolute path to a private key file used to identify this server. + + The key must be in DER or PEM (PKCS#8) format. Additionally, the key can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/host_certificate.key"] + } + key_pass: { + description: """ + Passphrase used to unlock the encrypted key file. + + This has no effect unless `key_file` is set. + """ + required: false + type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"] + } + server_name: { + description: """ + Server name to use when using Server Name Indication (SNI). + + Only relevant for outgoing connections. + """ + required: false + type: string: examples: ["www.example.com"] + } + verify_certificate: { + description: """ + Enables certificate verification. For components that create a server, this requires that the + client connections have a valid client certificate. For components that initiate requests, + this validates that the upstream has a valid certificate. + + If enabled, certificates must not be expired and must be issued by a trusted + issuer. This verification operates in a hierarchical manner, checking that the leaf certificate (the + certificate presented by the client/server) is not only valid, but that the issuer of that certificate is also valid, and + so on, until the verification process reaches a root certificate. + + Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + """ + required: false + type: bool: {} + } + verify_hostname: { + description: """ + Enables hostname verification. + + If enabled, the hostname used to connect to the remote host must be present in the TLS certificate presented by + the remote host, either as the Common Name or as an entry in the Subject Alternative Name extension. + + Only relevant for outgoing connections. + + Do NOT set this to `false` unless you understand the risks of not verifying the remote hostname. + """ + required: false + type: bool: {} + } + } + } } From 206692f666db7206d3af8cfb04ae433e834791e7 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Wed, 25 Feb 2026 08:22:35 +0000 Subject: [PATCH 06/26] add feature changelog Signed-off-by: Jed Laundry --- changelog.d/24729_azure_blob_authentication.feature.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/24729_azure_blob_authentication.feature.md diff --git a/changelog.d/24729_azure_blob_authentication.feature.md b/changelog.d/24729_azure_blob_authentication.feature.md new file mode 100644 index 0000000000000..7b224172563b7 --- /dev/null +++ b/changelog.d/24729_azure_blob_authentication.feature.md @@ -0,0 +1,3 @@ +Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. + +authors: jlaundry From 30e73b5120a32b78cbdbd0329164fd753c4134d6 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Wed, 25 Feb 2026 08:41:27 +0000 Subject: [PATCH 07/26] clippy changes Signed-off-by: Jed Laundry --- src/sinks/azure_common/config.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 74e7c0de347fe..a0901e9b14c8e 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -348,7 +348,7 @@ pub fn build_client( // Prepare options; attach Shared Key policy if needed let mut options = BlobContainerClientOptions::default(); match (parsed.auth(), &auth) { - (Auth::None { .. }, None) => { + (Auth::None, None) => { warn!("No authentication method provided, requests will be anonymous"); } (Auth::Sas { .. }, None) => { @@ -375,14 +375,14 @@ pub fn build_client( .per_call_policies .push(Arc::new(policy)); } - (Auth::None { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { + (Auth::None, Some(AzureAuthentication::ClientSecretCredential { .. })) => { info!("Using Client Secret authentication"); let async_credential_result = task::block_in_place(|| { Handle::current().block_on(async { auth.unwrap().credential().await.unwrap() }) }); credential = Some(async_credential_result); } - (Auth::None { .. }, Some(AzureAuthentication::Specific(..))) => { + (Auth::None, Some(AzureAuthentication::Specific(..))) => { info!("Using specific Azure Authentication method"); let async_credential_result = task::block_in_place(|| { Handle::current().block_on(async { auth.unwrap().credential().await.unwrap() }) @@ -435,15 +435,15 @@ pub fn build_client( } } - if let Some(tls_config) = tls { - if let Some(ca_file) = tls_config.ca_file { - let mut buf = Vec::new(); - File::open(&ca_file)?.read_to_end(&mut buf)?; - let cert = reqwest_12::Certificate::from_pem(&buf)?; + if let Some(tls_config) = tls + && let Some(ca_file) = tls_config.ca_file + { + let mut buf = Vec::new(); + File::open(&ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_12::Certificate::from_pem(&buf)?; - warn!("Adding TLS root certificate from {}", ca_file.display()); - reqwest_builder = reqwest_builder.add_root_certificate(cert); - } + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); } options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( From 5a0b5d2c6f4c7f4012eed34e3b23324f127504bd Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Fri, 27 Feb 2026 23:39:18 +0000 Subject: [PATCH 08/26] add account_name and blob_endpoint options Signed-off-by: Jed Laundry --- src/sinks/azure_blob/config.rs | 55 +++++++++++- src/sinks/azure_blob/integration_tests.rs | 86 ++++++++++++------ src/sinks/azure_blob/test.rs | 90 +++++++++++++++++++ src/sinks/azure_common/config.rs | 24 +++-- .../components/sinks/generated/azure_blob.cue | 24 ++++- 5 files changed, 238 insertions(+), 41 deletions(-) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index f15a8af40359e..f97ca40946ea3 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -67,7 +67,22 @@ pub struct AzureBlobSinkConfig { #[configurable(metadata( docs::examples = "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken" ))] - pub connection_string: SensitiveString, + #[configurable(metadata(docs::examples = "AccountName=mylogstorage"))] + pub connection_string: Option, + + /// The Azure Blob Storage Account name. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "mylogstorage"))] + pub(super) account_name: Option, + + /// The Azure Blob Storage endpoint. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "https://mylogstorage.blob.core.windows.net/"))] + pub(super) blob_endpoint: Option, /// The Azure Blob Storage Account container name. #[configurable(metadata(docs::examples = "my-logs"))] @@ -158,7 +173,9 @@ impl GenerateConfig for AzureBlobSinkConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { auth: None, - connection_string: String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into(), + connection_string: Some(String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into()), + account_name: None, + blob_endpoint: None, container_name: String::from("logs"), blob_prefix: default_blob_prefix(), blob_time_format: Some(String::from("%s")), @@ -178,9 +195,41 @@ impl GenerateConfig for AzureBlobSinkConfig { #[typetag::serde(name = "azure_blob")] impl SinkConfig for AzureBlobSinkConfig { async fn build(&self, cx: SinkContext) -> Result<(VectorSink, Healthcheck)> { + let connection_string: String = match ( + &self.connection_string, + &self.account_name, + &self.blob_endpoint, + ) { + (Some(connstr), None, None) => connstr.inner().into(), + (None, Some(account_name), None) => { + format!("AccountName={}", account_name) + } + (None, None, Some(blob_endpoint)) => { + // BlobEndpoint must always end in a trailing slash + let blob_endpoint = if blob_endpoint.ends_with('/') { + blob_endpoint.clone() + } else { + format!("{}/", blob_endpoint) + }; + format!("BlobEndpoint={}", blob_endpoint) + } + (None, None, None) => { + return Err("One of `connection_string`, `account_name`, or `blob_endpoint` must be provided".into()); + } + (Some(_), Some(_), _) => { + return Err("Cannot provide both `connection_string` and `account_name`".into()); + } + (Some(_), _, Some(_)) => { + return Err("Cannot provide both `connection_string` and `blob_endpoint`".into()); + } + (_, Some(_), Some(_)) => { + return Err("Cannot provide both `account_name` and `blob_endpoint`".into()); + } + }; + let client = azure_common::config::build_client( self.auth.clone(), - self.connection_string.clone().into(), + connection_string.clone(), self.container_name.clone(), cx.proxy(), self.tls.clone(), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 9014ac47929ea..94e1de006e5ee 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -33,7 +33,12 @@ async fn azure_blob_healthcheck_passed() { let config = AzureBlobSinkConfig::new_emulator().await; let client = azure_common::config::build_client( None, - config.connection_string.clone().into(), + config + .connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), config.container_name.clone(), &crate::config::ProxyConfig::default(), None, @@ -55,7 +60,12 @@ async fn azure_blob_healthcheck_unknown_container() { }; let client = azure_common::config::build_client( None, - config.connection_string.clone().into(), + config + .connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), config.container_name.clone(), &crate::config::ProxyConfig::default(), None, @@ -244,17 +254,19 @@ impl AzureBlobSinkConfig { let config = AzureBlobSinkConfig { auth: None, tls: None, - connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into(), - container_name: "logs".to_string(), - blob_prefix: Default::default(), - blob_time_format: None, - blob_append_uuid: None, - encoding: (None::, TextSerializerConfig::default()).into(), - compression: Compression::None, - batch: Default::default(), - request: TowerRequestConfig::default(), - acknowledgements: Default::default(), - }; + connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + }; config.ensure_container().await; @@ -275,17 +287,19 @@ impl AzureBlobSinkConfig { ca_file: Some(tls::TEST_PEM_CA_PATH.into()), ..Default::default() }), - connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into(), - container_name: "logs".to_string(), - blob_prefix: Default::default(), - blob_time_format: None, - blob_append_uuid: None, - encoding: (None::, TextSerializerConfig::default()).into(), - compression: Compression::None, - batch: Default::default(), - request: TowerRequestConfig::default(), - acknowledgements: Default::default(), - }; + connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + }; config.ensure_container().await; @@ -295,7 +309,11 @@ impl AzureBlobSinkConfig { fn to_sink(&self) -> VectorSink { let client = azure_common::config::build_client( None, - self.connection_string.clone().into(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), None, @@ -315,7 +333,11 @@ impl AzureBlobSinkConfig { pub async fn list_blobs(&self, prefix: String) -> Vec { let client = azure_common::config::build_client( None, - self.connection_string.clone().into(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), None, @@ -342,7 +364,11 @@ impl AzureBlobSinkConfig { pub async fn get_blob(&self, blob: String) -> (Option, Option, Vec) { let client = azure_common::config::build_client( None, - self.connection_string.clone().into(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), None, @@ -405,7 +431,11 @@ impl AzureBlobSinkConfig { async fn ensure_container(&self) { let client = azure_common::config::build_client( None, - self.connection_string.clone().into(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), None, diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index 190d0f4ed9292..fbb36f627d948 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -26,6 +26,8 @@ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { AzureBlobSinkConfig { auth: Default::default(), connection_string: Default::default(), + account_name: Default::default(), + blob_endpoint: Default::default(), container_name: Default::default(), blob_prefix: Default::default(), blob_time_format: Default::default(), @@ -279,3 +281,91 @@ async fn azure_blob_build_config_with_client_id_and_secret() { .await .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); } + +#[tokio::test] +async fn azure_blob_build_config_with_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!( + "Config build should have errored due to conflicting connection_string and account_name" + ), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`connection_string` and `account_name`"), + "Config build did not complain about conflicting connection_string and account_name: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage;AccountKey=mockkey" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => { + panic!("Config build should have errored due to conflicting Shared Key and Client ID") + } + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("Cannot use both Shared Key and Client ID"), + "Config build did not complain about conflicting Shared Key and Client ID: {}", + err_str + ); + } + } +} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index a0901e9b14c8e..400a80232a7ba 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -390,20 +390,28 @@ pub fn build_client( credential = Some(async_credential_result); } (Auth::Sas { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { - panic!("Cannot use both SAS token and Client ID/Secret at the same time"); + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both SAS token and Client ID/Secret at the same time", + ))); } (Auth::SharedKey { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { - panic!("Cannot use both Shared Key and Client ID/Secret at the same time"); + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both Shared Key and Client ID/Secret at the same time", + ))); } (Auth::Sas { .. }, Some(AzureAuthentication::Specific(..))) => { - panic!( - "Cannot use both SAS token and another Azure Authentication method at the same time" - ); + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both SAS token and another Azure Authentication method at the same time", + ))); } (Auth::SharedKey { .. }, Some(AzureAuthentication::Specific(..))) => { - panic!( - "Cannot use both Shared Key and another Azure Authentication method at the same time" - ); + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both Shared Key and another Azure Authentication method at the same time", + ))); } } diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 22b7bb60bc566..5641dd6b6f3d5 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -1,6 +1,16 @@ package metadata generated: components: sinks: azure_blob: configuration: { + account_name: { + description: """ + The Azure Blob Storage Account name. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["mylogstorage"] + } acknowledgements: { description: """ Controls how acknowledgements are handled for this sink. @@ -135,6 +145,16 @@ generated: components: sinks: azure_blob: configuration: { required: false type: bool: {} } + blob_endpoint: { + description: """ + The Azure Blob Storage endpoint. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["https://mylogstorage.blob.core.windows.net/"] + } blob_prefix: { description: """ A prefix to apply to all blob keys. @@ -226,8 +246,8 @@ generated: components: sinks: azure_blob: configuration: { | Allowed resource types | Container & Object | | Allowed permissions | Read & Create | """ - required: true - type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken"] + required: false + type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken", "AccountName=mylogstorage"] } container_name: { description: "The Azure Blob Storage Account container name." From c611d4d713c50552ea628e1dd9beb8091a08f33d Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Sat, 28 Feb 2026 21:56:22 +0000 Subject: [PATCH 09/26] remove block_in_place --- src/sinks/azure_blob/config.rs | 2 +- src/sinks/azure_blob/integration_tests.rs | 20 +++++++++++++------- src/sinks/azure_common/config.rs | 16 +++++----------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index f97ca40946ea3..74c9e81dc0fea 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -233,7 +233,7 @@ impl SinkConfig for AzureBlobSinkConfig { self.container_name.clone(), cx.proxy(), self.tls.clone(), - )?; + ).await?; let healthcheck = azure_common::config::build_healthcheck( self.container_name.clone(), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 94e1de006e5ee..4e3f8efe2dd81 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -43,6 +43,7 @@ async fn azure_blob_healthcheck_passed() { &crate::config::ProxyConfig::default(), None, ) + .await .expect("Failed to create client"); azure_common::config::build_healthcheck(config.container_name, client) @@ -70,6 +71,7 @@ async fn azure_blob_healthcheck_unknown_container() { &crate::config::ProxyConfig::default(), None, ) + .await .expect("Failed to create client"); assert_eq!( @@ -253,7 +255,6 @@ impl AzureBlobSinkConfig { let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { auth: None, - tls: None, connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into()), account_name: None, blob_endpoint: None, @@ -266,6 +267,7 @@ impl AzureBlobSinkConfig { batch: Default::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), + tls: None, }; config.ensure_container().await; @@ -283,10 +285,6 @@ impl AzureBlobSinkConfig { let address = std::env::var("AZURITE_OAUTH_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { auth: Some(client_secret_credential), - tls: Some(TlsConfig { - ca_file: Some(tls::TEST_PEM_CA_PATH.into()), - ..Default::default() - }), connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), account_name: None, blob_endpoint: None, @@ -299,6 +297,10 @@ impl AzureBlobSinkConfig { batch: Default::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), + tls: Some(TlsConfig { + ca_file: Some(tls::TEST_PEM_CA_PATH.into()), + ..Default::default() + }), }; config.ensure_container().await; @@ -306,7 +308,7 @@ impl AzureBlobSinkConfig { config } - fn to_sink(&self) -> VectorSink { + async fn to_sink(&self) -> VectorSink { let client = azure_common::config::build_client( None, self.connection_string @@ -318,6 +320,7 @@ impl AzureBlobSinkConfig { &crate::config::ProxyConfig::default(), None, ) + .await .expect("Failed to create client"); self.build_processor(client).expect("Failed to create sink") @@ -325,7 +328,7 @@ impl AzureBlobSinkConfig { async fn run_assert(&self, input: impl Stream + Send) { // `to_sink` needs to be inside the assertion check - assert_sink_compliance(&SINK_TAGS, async move { self.to_sink().run(input).await }) + assert_sink_compliance(&SINK_TAGS, async move { self.to_sink().await.run(input).await }) .await .expect("Running sink failed"); } @@ -342,6 +345,7 @@ impl AzureBlobSinkConfig { &crate::config::ProxyConfig::default(), None, ) + .await .unwrap(); // Iterate pager results and collect blob names. Filter by prefix server-side. @@ -373,6 +377,7 @@ impl AzureBlobSinkConfig { &crate::config::ProxyConfig::default(), None, ) + .await .unwrap(); let blob_client = client.blob_client(&blob); @@ -440,6 +445,7 @@ impl AzureBlobSinkConfig { &crate::config::ProxyConfig::default(), None, ) + .await .unwrap(); let result = client.create_container(None).await; diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 400a80232a7ba..b3ebbbaeee008 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -3,8 +3,6 @@ use std::io::Read; use std::sync::Arc; use azure_core::error::Error as AzureCoreError; -use tokio::runtime::Handle; -use tokio::task; use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; use crate::sinks::azure_common::shared_key_policy::SharedKeyAuthorizationPolicy; @@ -327,7 +325,7 @@ pub fn build_healthcheck( Ok(healthcheck.boxed()) } -pub fn build_client( +pub async fn build_client( auth: Option, connection_string: String, container_name: String, @@ -377,17 +375,13 @@ pub fn build_client( } (Auth::None, Some(AzureAuthentication::ClientSecretCredential { .. })) => { info!("Using Client Secret authentication"); - let async_credential_result = task::block_in_place(|| { - Handle::current().block_on(async { auth.unwrap().credential().await.unwrap() }) - }); - credential = Some(async_credential_result); + let credential_result: Arc = auth.unwrap().credential().await.unwrap(); + credential = Some(credential_result); } (Auth::None, Some(AzureAuthentication::Specific(..))) => { info!("Using specific Azure Authentication method"); - let async_credential_result = task::block_in_place(|| { - Handle::current().block_on(async { auth.unwrap().credential().await.unwrap() }) - }); - credential = Some(async_credential_result); + let credential_result: Arc = auth.unwrap().credential().await.unwrap(); + credential = Some(credential_result); } (Auth::Sas { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { return Err(Box::new(Error::with_message( From 1ad3097bd0e371bc4b461f196a20647ef18f5795 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Wed, 4 Mar 2026 18:27:05 +0000 Subject: [PATCH 10/26] add ClientCertificateCredential Signed-off-by: Jed Laundry --- Cargo.lock | 1 + Cargo.toml | 2 +- src/sinks/azure_blob/config.rs | 3 +- src/sinks/azure_blob/integration_tests.rs | 9 ++- src/sinks/azure_blob/test.rs | 40 ++++++++++++- src/sinks/azure_common/config.rs | 70 ++++++++++++++++++++-- tests/data/ClientCertificateAuth.pfx | Bin 0 -> 2467 bytes 7 files changed, 114 insertions(+), 11 deletions(-) create mode 100644 tests/data/ClientCertificateAuth.pfx diff --git a/Cargo.lock b/Cargo.lock index eaa88b9180219..a2103642e2fd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1724,6 +1724,7 @@ dependencies = [ "async-trait", "azure_core", "futures 0.3.31", + "openssl", "pin-project", "serde", "time", diff --git a/Cargo.toml b/Cargo.toml index 479beb235b8a1..e77e0c15ce302 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -303,7 +303,7 @@ aws-smithy-types = { version = "1.2.11", default-features = false, features = [" # Azure azure_core = { version = "0.30", features = ["reqwest", "hmac_openssl"], optional = true } -azure_identity = { version = "0.30", optional = true } +azure_identity = { version = "0.30", features = ["client_certificate"], optional = true } # Azure Storage azure_storage_blob = { version = "0.7", optional = true } diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 74c9e81dc0fea..6646a4981aa8c 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -233,7 +233,8 @@ impl SinkConfig for AzureBlobSinkConfig { self.container_name.clone(), cx.proxy(), self.tls.clone(), - ).await?; + ) + .await?; let healthcheck = azure_common::config::build_healthcheck( self.container_name.clone(), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 4e3f8efe2dd81..b0a79cc247828 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -328,9 +328,12 @@ impl AzureBlobSinkConfig { async fn run_assert(&self, input: impl Stream + Send) { // `to_sink` needs to be inside the assertion check - assert_sink_compliance(&SINK_TAGS, async move { self.to_sink().await.run(input).await }) - .await - .expect("Running sink failed"); + assert_sink_compliance( + &SINK_TAGS, + async move { self.to_sink().await.run(input).await }, + ) + .await + .expect("Running sink failed"); } pub async fn list_blobs(&self, prefix: String) -> Vec { diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index fbb36f627d948..ff58e9034dccc 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -14,7 +14,7 @@ use super::{config::AzureBlobSinkConfig, request_builder::AzureBlobRequestOption use crate::{ codecs::{Encoder, EncodingConfigWithFraming}, event::{Event, LogEvent}, - sinks::azure_common::config::AzureAuthentication, + sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}, sinks::prelude::*, sinks::util::{ Compression, @@ -282,6 +282,44 @@ async fn azure_blob_build_config_with_client_id_and_secret() { .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); } +#[tokio::test] +async fn azure_blob_build_config_with_client_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_certificate_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + certificate_file = "tests/data/ClientCertificateAuth.pfx" + certificate_password = "MockPassword123" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientCertificateCredential { + .. + }) => { + // Expected variant + } + _ => panic!("Expected Specific(ClientCertificateCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + #[tokio::test] async fn azure_blob_build_config_with_account_name() { let config: AzureBlobSinkConfig = toml::from_str::( diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index b3ebbbaeee008..a1e04506a21a0 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,7 +1,10 @@ use std::fs::File; use std::io::Read; +use std::path::PathBuf; use std::sync::Arc; +use base64::prelude::*; + use azure_core::error::Error as AzureCoreError; use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; @@ -12,9 +15,9 @@ use azure_core::credentials::{TokenCredential, TokenRequestOptions}; use azure_core::{Error, error::ErrorKind}; use azure_identity::{ - AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientSecretCredential, - ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, - WorkloadIdentityCredential, + AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientCertificateCredential, + ClientCertificateCredentialOptions, ClientSecretCredential, ManagedIdentityCredential, + ManagedIdentityCredentialOptions, UserAssignedId, WorkloadIdentityCredential, }; use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; @@ -82,6 +85,26 @@ pub enum SpecificAzureCredential { #[cfg(not(target_arch = "wasm32"))] AzureCli {}, + /// Use certificate credentials + ClientCertificateCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + azure_client_id: String, + + /// PKCS12 certificate with RSA private key. + #[configurable(metadata(docs::examples = "path/to/certificate.pfx"))] + certificate_file: PathBuf, + + /// The password for the client certificate, if applicable. + certificate_password: Option, + }, + /// Use Managed Identity credentials ManagedIdentity { /// The User Assigned Managed Identity (Client ID) to use. @@ -181,6 +204,41 @@ impl SpecificAzureCredential { #[cfg(not(target_arch = "wasm32"))] Self::AzureCli {} => AzureCliCredential::new(None)?, + // requires azure_identity feature 'client_certificate' + Self::ClientCertificateCredential { + azure_tenant_id, + azure_client_id, + certificate_file, + certificate_password, + } => { + let certificate_bytes: Vec = std::fs::read(certificate_file).map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!( + "Failed to read certificate file {}: {e}", + certificate_file.display() + ), + ) + })?; + + // Note: in azure_identity 0.33.0+, this changes to SecretBytes, and the base64 encoding is no longer needed + let certificate_base64: azure_core::credentials::Secret = + BASE64_STANDARD.encode(&certificate_bytes).into(); + + let mut options: ClientCertificateCredentialOptions = + ClientCertificateCredentialOptions::default(); + if let Some(password) = certificate_password { + options.password = Some(password.inner().to_string().into()); + } + + ClientCertificateCredential::new( + azure_tenant_id.clone(), + azure_client_id.clone(), + certificate_base64, + Some(options), + )? + } + Self::ManagedIdentity { user_assigned_managed_identity_id, } => { @@ -375,12 +433,14 @@ pub async fn build_client( } (Auth::None, Some(AzureAuthentication::ClientSecretCredential { .. })) => { info!("Using Client Secret authentication"); - let credential_result: Arc = auth.unwrap().credential().await.unwrap(); + let credential_result: Arc = + auth.unwrap().credential().await.unwrap(); credential = Some(credential_result); } (Auth::None, Some(AzureAuthentication::Specific(..))) => { info!("Using specific Azure Authentication method"); - let credential_result: Arc = auth.unwrap().credential().await.unwrap(); + let credential_result: Arc = + auth.unwrap().credential().await.unwrap(); credential = Some(credential_result); } (Auth::Sas { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { diff --git a/tests/data/ClientCertificateAuth.pfx b/tests/data/ClientCertificateAuth.pfx new file mode 100644 index 0000000000000000000000000000000000000000..e289b55375a4ac01132352e54813ffd76405bcc0 GIT binary patch literal 2467 zcmai$X*3iH8^_JYjO|(m*)GNs8rRtOeN7nIn_)7@zGSB`jBHaP(Y2&TlOjuK24hcl zDceP|zPhL+GMI$>y61gQ_v8KWoaa3M|9Q^y>GwYmiU@lM1b|RPmgra;t>(+DF#`Fr0R=8`-8d_*|TW-N^ zbl*q?#EcG5mTh$AF9my5!#w>BN%mvzvYCMHjMl9IEW7mY*aR|^%vM|mk z>D$T_eLDX7@~?mtF+1NH>3DzL7G&!OEmFHc3r%yrn8ag zH8OA8F}bG=p>N?#M!pyvyu4cQ#Qn?L7u)AQZ+B7!Q>u$)TiA@enyp?*wo7RZbgbNr z&9_oG>09(IIZv_)5y+kBwFf*c{#`)C3Juaoj0>HebqYigAvb>V_czdx7%&kMaVm$N zRv;_K|5@i`2LMk^l~Yma-vGXH&|ttURI>b6fZ?ozpeNh!`Rl5i<{r?J`@ux;7!I5> zk~oIIy(wTih5{n}gchSthP}|EWtvrAb1wwuF6w4`MtP(6ZzlOy;D-whH6j*woj-bk z)v4+nR`TDo8;<5`qfGI)l#GY2W8vLL+qKq31G%0>Uz1m#-SPChsR%41Y%^EtCv>o!YV;iZjxy@}h{{#bOtDMheWo zw(4F>xS3U(D%6LzC{1e0{q;x_u^s9p{A(?5C~05GqJXvpEEI}q(C|}I>7$HpPTa;( zrebEj=W6qY%t~uLkEblEA|;}Cy=kE5_?bzyuZr-)^J(I+G-#qWHO=2a1)KPk4-x38 z>=FAgP}b@({qg}lYNQf-c}`5X>Fr+!zr!Jbn*O(J*30vA{65;tEAe7zed!atQ<2X3 z#17Kh&D-g=(PI1y>@p-1C(HK!inEb}_oaA>{NkD#QW%M@eom+@KXV%@A#!?&T}rD< zd0yKg;gPnNgH3^^q=a#VR}ih@({t}!#?1yiEy;M0UXBJT9$34IMNi(SJ;TYLnr+tk z*c(E97tZ-w>DIX5PPWM^{Zd9y^_Z4iD?Fk7*@xJ;EWY?T^6!?b!bNqx<> zC%b<-DHuB|936ZiiCucg_3C?4^B>+chJ0pT;DP3e+L7ry8H^|a7^7h7O5T{@p4eRK zdXJ3Kb?FhAD+sCCqY7rE5>n&Kc7x_9>@w+kq)nE)XnBIt9mRn{-n-@f!Rwo%8sAp; z`G7ryL63GLEZ?Q$bVhwSc+|UTM%u5xBI7fmr&&j7eD#MYwMWopHu>PmHzWgMvdW7! zTeBA{-vzd?b+Y#GBWkjPq4owL1{?ak(~%HdqsX(sVK_!hiGkb$FvLH8pi|->j?aRt zs+J{|Z@0&nhxZgax?n>9Zo{>iq`l6pl zESYILy?h-=+#aO1HPU*Di&}%zsdiGF+oKSstNb?mjUSkxHeCO+WGgXJK4Rzdob#%S zo?N4OC+z_dCiFoKYwe?Ek-1%UmYwfp+*QKsb%~(F&Z`K1u8>Sg)r)PTx3td8i&(1F zvzK4x@G5O7&%dzVRJmnWgVdFg)8wg8DMaeWaeWut0Qw&nV@ZrO%V$}4UIx81Gk@E* z;39mM#(E!rt_hFo7t+-=p4R^`KKh%MSVeS38RG9F! zYl0t@52PjH7ZfO3M~=Hg4Pu^`L)OV>4BM%Ed!SMfMzq&s|D?1q`Z!vw|a*UpR2qV1*M+9 zLztw&9}G=BTDO3B;VLfa6)@9bE?Qz51B!p-smP+Fe+Dc}Kt&J?NABjzJC%MW3RI`X za2d$ig=@Blp-fOJD6XIH4G;if0ZaN8uILs-Ob3Hud6PjPmwKTu_?mrd<`jRwu)MFc Tg&i2m^(Z6qB2fO!-% Date: Wed, 4 Mar 2026 20:54:19 +0000 Subject: [PATCH 11/26] remake certificate with required attributes --- src/sinks/azure_blob/config.rs | 2 +- src/sinks/azure_blob/test.rs | 22 ++++++++++++++++ src/sinks/azure_common/config.rs | 4 +-- tests/data/Makefile | 3 +++ .../certs/azurite-chain.cert.pem | 26 +++++++++---------- .../certs/azurite.cert.pem | 26 +++++++++---------- .../intermediate_server/csr/azurite.csr.pem | 17 ++++++------ tests/data/ca/intermediate_server/index.txt | 2 +- .../ca/intermediate_server/newcerts/1009.pem | 26 +++++++++---------- 9 files changed, 77 insertions(+), 51 deletions(-) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 6646a4981aa8c..010b77fc2209b 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -160,8 +160,8 @@ pub struct AzureBlobSinkConfig { )] pub(super) acknowledgements: AcknowledgementsConfig, - #[serde(default)] #[configurable(derived)] + #[serde(default)] pub tls: Option, } diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index ff58e9034dccc..d89cbdc04020c 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -407,3 +407,25 @@ async fn azure_blob_build_config_with_conflicting_connection_string_and_client_i } } } +#[tokio::test] +async fn azure_blob_build_config_with_custom_ca_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [tls] + ca_file = "tests/data/ca/certs/ca.cert.pem" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} \ No newline at end of file diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index a1e04506a21a0..26e759fa7ab51 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -497,8 +497,8 @@ pub async fn build_client( } } - if let Some(tls_config) = tls - && let Some(ca_file) = tls_config.ca_file + if let Some(tls_config) = &tls + && let Some(ca_file) = &tls_config.ca_file { let mut buf = Vec::new(); File::open(&ca_file)?.read_to_end(&mut buf)?; diff --git a/tests/data/Makefile b/tests/data/Makefile index 32a70afa57db6..6d4a17ab0d54c 100644 --- a/tests/data/Makefile +++ b/tests/data/Makefile @@ -68,6 +68,9 @@ ca/intermediate_server/csr/azurite.csr.pem: ca/intermediate_server/private/azuri openssl req -config ca/intermediate_server/openssl.cnf \ -key ca/intermediate_server/private/azurite.key.pem \ -subj '/CN=azurite/OU=Vector/O=Datadog/ST=New York/L=New York/C=US' \ + -addext 'subjectAltName=DNS:azurite' \ + -addext 'basicConstraints=critical,CA:FALSE' \ + -addext 'extendedKeyUsage=serverAuth' \ -new -sha256 -out ca/intermediate_server/csr/azurite.csr.pem ca/intermediate_server/certs/azurite.cert.pem: ca/intermediate_server/csr/azurite.csr.pem diff --git a/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem index 934f433785fe7..f646a0fe3c20a 100644 --- a/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem +++ b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem @@ -2,7 +2,7 @@ MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X -DTI2MDIyNDIxNDY0MFoXDTM2MDIyMjIxNDY0MFowaDELMAkGA1UEBhMCVVMxETAP +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP @@ -17,18 +17,18 @@ FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw -EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAArqDt0rNQNa -eu18+YgML201hSK9AWatk4ew9HogDhpIiqqwfDY1okTHBTWKdMRIOzD7EOvlbEMl -1fqPb09DVTuOrV1vWpBkbvHIvHYn/v28mslUIRbIw1qczdWLGPhzNdXnbVI0D47T -B3vzrXlSJlZbaDtvjoVIm2Jhq/i+1fVO4adbuWDXPhjzZq1eKWC5wQOYyXyeZAQo -0RwbqXnuGeJqUeo1/6sr/ft5v+1W62yHHaKZWiUIcrSqiv3bFIBexvzIZV34VHwN -p2JgVlmP6TXtyTouwFdx3FTBmS0kUViYW1AY8nGV3ZMmOIlqYCs3y78lVhxdjXnG -q/9U1xaAaXfEaPOu/19J14HAXJ9jnVlq/civCDWTbJvIv1AxblGpa8rz/0HlI0Pd -/T1OfAUD/e5FAqn8w2psnJGwGyMp+Mr32Ip+fNt5IHm1dhMv0WAglSLJb0L1EQe9 -GnGDqLOED1MfE5rLukJ+yXp2PWTkLgMmCXBMf4WdH1sUNRGaOzkNf6BB0c3eBJDP -gMIwmI13yBWw141BTICKmYCF+W928BQBKVFTpbI/buRs44eGWbwDx7R8qX9S3f0y -VN3WxS0kB5NYsKTbuoeuzWVtDz/zRxvod7l/fpYWbonZqp6Pa9EHuCkJF+5DSQ8P -7+iB0vOdu7mvLW4IrJnd7WXphAB16r+B +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIFtzCCA5+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwajESMBAGA1UEAwwJVmVj diff --git a/tests/data/ca/intermediate_server/certs/azurite.cert.pem b/tests/data/ca/intermediate_server/certs/azurite.cert.pem index 6db5de3454389..3e2f680ef37c0 100644 --- a/tests/data/ca/intermediate_server/certs/azurite.cert.pem +++ b/tests/data/ca/intermediate_server/certs/azurite.cert.pem @@ -2,7 +2,7 @@ MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X -DTI2MDIyNDIxNDY0MFoXDTM2MDIyMjIxNDY0MFowaDELMAkGA1UEBhMCVVMxETAP +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP @@ -17,16 +17,16 @@ FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw -EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAArqDt0rNQNa -eu18+YgML201hSK9AWatk4ew9HogDhpIiqqwfDY1okTHBTWKdMRIOzD7EOvlbEMl -1fqPb09DVTuOrV1vWpBkbvHIvHYn/v28mslUIRbIw1qczdWLGPhzNdXnbVI0D47T -B3vzrXlSJlZbaDtvjoVIm2Jhq/i+1fVO4adbuWDXPhjzZq1eKWC5wQOYyXyeZAQo -0RwbqXnuGeJqUeo1/6sr/ft5v+1W62yHHaKZWiUIcrSqiv3bFIBexvzIZV34VHwN -p2JgVlmP6TXtyTouwFdx3FTBmS0kUViYW1AY8nGV3ZMmOIlqYCs3y78lVhxdjXnG -q/9U1xaAaXfEaPOu/19J14HAXJ9jnVlq/civCDWTbJvIv1AxblGpa8rz/0HlI0Pd -/T1OfAUD/e5FAqn8w2psnJGwGyMp+Mr32Ip+fNt5IHm1dhMv0WAglSLJb0L1EQe9 -GnGDqLOED1MfE5rLukJ+yXp2PWTkLgMmCXBMf4WdH1sUNRGaOzkNf6BB0c3eBJDP -gMIwmI13yBWw141BTICKmYCF+W928BQBKVFTpbI/buRs44eGWbwDx7R8qX9S3f0y -VN3WxS0kB5NYsKTbuoeuzWVtDz/zRxvod7l/fpYWbonZqp6Pa9EHuCkJF+5DSQ8P -7+iB0vOdu7mvLW4IrJnd7WXphAB16r+B +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P -----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/csr/azurite.csr.pem b/tests/data/ca/intermediate_server/csr/azurite.csr.pem index 8de7cd20cda66..09cf2dd0bc3f7 100644 --- a/tests/data/ca/intermediate_server/csr/azurite.csr.pem +++ b/tests/data/ca/intermediate_server/csr/azurite.csr.pem @@ -1,5 +1,5 @@ -----BEGIN CERTIFICATE REQUEST----- -MIICrTCCAZUCAQAwaDEQMA4GA1UEAwwHYXp1cml0ZTEPMA0GA1UECwwGVmVjdG9y +MIIC9TCCAd0CAQAwaDEQMA4GA1UEAwwHYXp1cml0ZTEPMA0GA1UECwwGVmVjdG9y MRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwI TmV3IFlvcmsxCzAJBgNVBAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB CgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVPbGW0LqD1WO42dSmUMQ2iRGsF @@ -7,11 +7,12 @@ eHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bRRInmuyocDTvm4t7VU+ybnPD7 MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3QZ4loS61BR5TxrCdnpE++gxh 7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ63PaXXvGpGiDwPqUqi0WhIYp 73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFnVr8lMVavcDcoTrHdJ8PXN4Ei -pgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABoAAwDQYJKoZIhvcNAQELBQADggEB -AANtDoLqI/uQEn3u/xe9R539MjjzP2t7y8HWwaSJBJrNwkiHIub7NfMHOdyIPhLt -jPV6QqHWaWFhj0cW5YeBifcO7t/F523VGcH1GawaTNBIfemmpeCXDEkWOSDnM9Fp -N/J3kEgUUg9/Lw+0ygzR5oXUGx9srBK9M8gqgeWF4kpDlYTiTsekmAU+Td9GcxE7 -IrqK8kTx25CIcRrA00iKTrQLLFEtVED3F+AFJ2mi03iYIVuu3t6ZNurSZZyOAd6H -I9mSC51Yife3GuabkzSW38FUFdJmE7pElmvD2WkU/5XpPhjxajkxJxAs5W54tesL -3TJv/xhzHzH1g4FikkDjZnw= +pgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABoEgwRgYJKoZIhvcNAQkOMTkwNzAS +BgNVHREECzAJggdhenVyaXRlMAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwDQYJKoZIhvcNAQELBQADggEBAIbILz+IbvX9126CRLqmtlp4s1q2L+A2 +jYz+qk1G1UhImumfEcLMBfj6X/ZQ5IRIrH8BZZtW9ungXUS/9eYKUCQP52GXPPZJ +WkS3sERjvktEP7orNrTLo7v36f9P0J9eMjN8XYhcqHmZB5OyoQ4S3ndqgE4+HV8j +OyP9lZC1Urjrre/S7pyxxnkVftxXtVHXzB0MgoQa77U/ukqeaMYIw3qWRqGwwl44 +JnZm05mukTpYnDVmo+J2Ra1uNOJf9/SIEnawyL1ROYCINxgIcdodrpuxz+dnYT0K +zvsbMGQ1MTacJK61Pw0EBbxERB9ZhWnApRdeuhrF5oq9uGEzD/76wTg= -----END CERTIFICATE REQUEST----- diff --git a/tests/data/ca/intermediate_server/index.txt b/tests/data/ca/intermediate_server/index.txt index dfa1286cc5b99..b2a0481603b78 100644 --- a/tests/data/ca/intermediate_server/index.txt +++ b/tests/data/ca/intermediate_server/index.txt @@ -6,4 +6,4 @@ V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar -V 360222214640Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite +V 360301205253Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite diff --git a/tests/data/ca/intermediate_server/newcerts/1009.pem b/tests/data/ca/intermediate_server/newcerts/1009.pem index 6db5de3454389..3e2f680ef37c0 100644 --- a/tests/data/ca/intermediate_server/newcerts/1009.pem +++ b/tests/data/ca/intermediate_server/newcerts/1009.pem @@ -2,7 +2,7 @@ MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X -DTI2MDIyNDIxNDY0MFoXDTM2MDIyMjIxNDY0MFowaDELMAkGA1UEBhMCVVMxETAP +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP @@ -17,16 +17,16 @@ FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw -EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAArqDt0rNQNa -eu18+YgML201hSK9AWatk4ew9HogDhpIiqqwfDY1okTHBTWKdMRIOzD7EOvlbEMl -1fqPb09DVTuOrV1vWpBkbvHIvHYn/v28mslUIRbIw1qczdWLGPhzNdXnbVI0D47T -B3vzrXlSJlZbaDtvjoVIm2Jhq/i+1fVO4adbuWDXPhjzZq1eKWC5wQOYyXyeZAQo -0RwbqXnuGeJqUeo1/6sr/ft5v+1W62yHHaKZWiUIcrSqiv3bFIBexvzIZV34VHwN -p2JgVlmP6TXtyTouwFdx3FTBmS0kUViYW1AY8nGV3ZMmOIlqYCs3y78lVhxdjXnG -q/9U1xaAaXfEaPOu/19J14HAXJ9jnVlq/civCDWTbJvIv1AxblGpa8rz/0HlI0Pd -/T1OfAUD/e5FAqn8w2psnJGwGyMp+Mr32Ip+fNt5IHm1dhMv0WAglSLJb0L1EQe9 -GnGDqLOED1MfE5rLukJ+yXp2PWTkLgMmCXBMf4WdH1sUNRGaOzkNf6BB0c3eBJDP -gMIwmI13yBWw141BTICKmYCF+W928BQBKVFTpbI/buRs44eGWbwDx7R8qX9S3f0y -VN3WxS0kB5NYsKTbuoeuzWVtDz/zRxvod7l/fpYWbonZqp6Pa9EHuCkJF+5DSQ8P -7+iB0vOdu7mvLW4IrJnd7WXphAB16r+B +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P -----END CERTIFICATE----- From f3dc448ed52da76212e0ac9bf16ed6a7a8522b92 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 5 Mar 2026 18:26:14 +0000 Subject: [PATCH 12/26] refactor integration tests to support both connection string and oauth Signed-off-by: Jed Laundry --- src/sinks/azure_blob/integration_tests.rs | 166 ++++++++-------------- src/sinks/azure_blob/test.rs | 2 +- src/sinks/azure_common/config.rs | 84 +++++++++++ 3 files changed, 145 insertions(+), 107 deletions(-) diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index b0a79cc247828..d32cab88afaa3 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -1,11 +1,12 @@ use std::io::{BufRead, BufReader}; +use std::sync::Arc; use azure_core::http::StatusCode; +use azure_storage_blob::BlobContainerClient; use bytes::{Buf, BytesMut}; use flate2::read::GzDecoder; use futures::{Stream, StreamExt, stream}; -use vector_common::sensitive_string::SensitiveString; use vector_lib::{ ByteSizeOf, codecs::{ @@ -31,20 +32,18 @@ use crate::{ #[tokio::test] async fn azure_blob_healthcheck_passed() { let config = AzureBlobSinkConfig::new_emulator().await; - let client = azure_common::config::build_client( - None, - config - .connection_string - .clone() - .expect("failed to unwrap connection_string") - .inner() - .to_string(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - None, - ) - .await - .expect("Failed to create client"); + let client = config.build_test_client().await; + + azure_common::config::build_healthcheck(config.container_name, client) + .expect("Failed to build healthcheck") + .await + .expect("Failed to pass healthcheck"); +} + +#[tokio::test] +async fn azure_blob_healthcheck_passed_with_oauth() { + let config = AzureBlobSinkConfig::new_emulator_with_oauth().await; + let client = config.build_test_client().await; azure_common::config::build_healthcheck(config.container_name, client) .expect("Failed to build healthcheck") @@ -59,20 +58,7 @@ async fn azure_blob_healthcheck_unknown_container() { container_name: String::from("other-container-name"), ..config }; - let client = azure_common::config::build_client( - None, - config - .connection_string - .clone() - .expect("failed to unwrap connection_string") - .inner() - .to_string(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - None, - ) - .await - .expect("Failed to create client"); + let client = config.build_test_client().await; assert_eq!( azure_common::config::build_healthcheck(config.container_name, client) @@ -84,10 +70,8 @@ async fn azure_blob_healthcheck_unknown_container() { ); } -#[tokio::test] -async fn azure_blob_insert_lines_into_blob() { +async fn assert_insert_lines_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("lines/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), ..config @@ -106,9 +90,17 @@ async fn azure_blob_insert_lines_into_blob() { } #[tokio::test] -async fn azure_blob_insert_json_into_blob() { +async fn azure_blob_insert_lines_into_blob() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_lines_into_blob_with_oauth() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + +async fn assert_insert_json_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("json/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), encoding: ( @@ -135,6 +127,16 @@ async fn azure_blob_insert_json_into_blob() { assert_eq!(expected, blob_lines); } +#[tokio::test] +async fn azure_blob_insert_json_into_blob() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_json_into_blob_with_oauth() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + #[ignore] #[tokio::test] // This test fails to get the posted blob with "header not found content-length". @@ -195,14 +197,12 @@ async fn azure_blob_insert_json_into_blob_gzip() { assert_eq!(expected, blob_lines); } -#[tokio::test] -async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { +async fn assert_rotate_files_after_the_buffer_size_is_reached(mut config: AzureBlobSinkConfig) { let groups = 3; let (lines, size, input) = random_lines_with_stream_with_group_key(100, 30, groups); let size_per_group = (size / groups) + 10; let blob_prefix = format!("lines-rotate/into/blob/{}", random_string(10)); - let mut config = AzureBlobSinkConfig::new_emulator().await; config.batch.max_bytes = Some(size_per_group); let config = AzureBlobSinkConfig { @@ -230,24 +230,17 @@ async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { } #[tokio::test] -async fn azure_blob_insert_lines_into_blob_with_oauth() { - let blob_prefix = format!("lines/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator_with_oauth().await; - let config = AzureBlobSinkConfig { - blob_prefix: blob_prefix.clone().try_into().unwrap(), - ..config - }; - let (lines, input) = random_lines_with_stream(100, 10, None); - - config.run_assert(input).await; +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { + assert_rotate_files_after_the_buffer_size_is_reached(AzureBlobSinkConfig::new_emulator().await) + .await; +} - let blobs = config.list_blobs(blob_prefix).await; - assert_eq!(blobs.len(), 1); - assert!(blobs[0].clone().ends_with(".log")); - let (content_type, content_encoding, blob_lines) = config.get_blob(blobs[0].clone()).await; - assert_eq!(content_type, Some(String::from("text/plain"))); - assert_eq!(content_encoding, None); - assert_eq!(lines, blob_lines); +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached_with_oauth() { + assert_rotate_files_after_the_buffer_size_is_reached( + AzureBlobSinkConfig::new_emulator_with_oauth().await, + ) + .await; } impl AzureBlobSinkConfig { @@ -276,16 +269,10 @@ impl AzureBlobSinkConfig { } pub async fn new_emulator_with_oauth() -> AzureBlobSinkConfig { - let client_secret_credential = - azure_common::config::AzureAuthentication::ClientSecretCredential { - azure_tenant_id: "00000000-0000-0000-0000-000000000000".to_string(), - azure_client_id: "00000000-0000-0000-0000-000000000000".to_string(), - azure_client_secret: SensitiveString::from("mock-secret".to_string()), - }; let address = std::env::var("AZURITE_OAUTH_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { - auth: Some(client_secret_credential), - connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), + auth: Some(azure_common::config::AzureAuthentication::MockCredential), + connection_string: Some(format!("DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), account_name: None, blob_endpoint: None, container_name: "logs".to_string(), @@ -308,9 +295,9 @@ impl AzureBlobSinkConfig { config } - async fn to_sink(&self) -> VectorSink { - let client = azure_common::config::build_client( - None, + async fn build_test_client(&self) -> Arc { + azure_common::config::build_client( + self.auth.clone(), self.connection_string .clone() .expect("failed to unwrap connection_string") @@ -318,11 +305,14 @@ impl AzureBlobSinkConfig { .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), - None, + self.tls.clone(), ) .await - .expect("Failed to create client"); + .expect("Failed to create client") + } + async fn to_sink(&self) -> VectorSink { + let client = self.build_test_client().await; self.build_processor(client).expect("Failed to create sink") } @@ -337,19 +327,7 @@ impl AzureBlobSinkConfig { } pub async fn list_blobs(&self, prefix: String) -> Vec { - let client = azure_common::config::build_client( - None, - self.connection_string - .clone() - .expect("failed to unwrap connection_string") - .inner() - .to_string(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - None, - ) - .await - .unwrap(); + let client = self.build_test_client().await; // Iterate pager results and collect blob names. Filter by prefix server-side. let mut pager = client @@ -369,19 +347,7 @@ impl AzureBlobSinkConfig { } pub async fn get_blob(&self, blob: String) -> (Option, Option, Vec) { - let client = azure_common::config::build_client( - None, - self.connection_string - .clone() - .expect("failed to unwrap connection_string") - .inner() - .to_string(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - None, - ) - .await - .unwrap(); + let client = self.build_test_client().await; let blob_client = client.blob_client(&blob); @@ -437,19 +403,7 @@ impl AzureBlobSinkConfig { } async fn ensure_container(&self) { - let client = azure_common::config::build_client( - None, - self.connection_string - .clone() - .expect("failed to unwrap connection_string") - .inner() - .to_string(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - None, - ) - .await - .unwrap(); + let client = self.build_test_client().await; let result = client.create_container(None).await; let response = match result { diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index d89cbdc04020c..dcb96ee32f78f 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -428,4 +428,4 @@ async fn azure_blob_build_config_with_custom_ca_certificate() { .build(cx) .await .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); -} \ No newline at end of file +} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 26e759fa7ab51..0bf72f56ee50f 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -70,6 +70,11 @@ pub enum AzureAuthentication { /// Use credentials from environment variables #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] Specific(SpecificAzureCredential), + + /// Mock credential for testing — returns a static fake token + #[cfg(test)] + #[serde(skip)] + MockCredential, } /// Specific Azure credential types. @@ -193,6 +198,9 @@ impl AzureAuthentication { } Self::Specific(specific) => specific.credential().await, + + #[cfg(test)] + Self::MockCredential => Ok(Arc::new(MockTokenCredential) as Arc), } } } @@ -467,6 +475,18 @@ pub async fn build_client( "Cannot use both Shared Key and another Azure Authentication method at the same time", ))); } + #[cfg(test)] + (Auth::None, Some(AzureAuthentication::MockCredential)) => { + warn!("Using mock token credential authentication"); + credential = Some(auth.unwrap().credential().await.unwrap()); + } + #[cfg(test)] + (_, Some(AzureAuthentication::MockCredential)) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both connection string auth and mock credential at the same time", + ))); + } } // Use reqwest v0.12 since Azure SDK only implements HttpClient for reqwest::Client v0.12 @@ -517,3 +537,67 @@ pub async fn build_client( .map_err(|e| format!("{e}"))?; Ok(Arc::new(client)) } + +#[cfg(test)] +#[derive(Debug)] +struct MockTokenCredential; + +#[cfg(test)] +#[async_trait::async_trait] +impl TokenCredential for MockTokenCredential { + async fn get_token( + &self, + scopes: &[&str], + _options: Option>, + ) -> azure_core::Result { + let Some(scope) = scopes.first() else { + return Err(Error::with_message( + ErrorKind::Credential, + "no scopes were provided", + )); + }; + + let jwt = serde_json::json!({ + "aud": scope.strip_suffix("/.default").unwrap_or(*scope), + "iat": 0, + "exp": 2147483647, + "iss": "https://sts.windows.net/", + "nbf": 0 + }); + + // JWTs do not include standard base64 padding. + // this seemed cleaner than importing a new crates just for this function + let jwt_base64 = format!( + "e30.{}.", + BASE64_STANDARD + .encode(serde_json::to_string(&jwt).unwrap()) + .trim_end_matches("=") + ) + .to_string(); + + warn!( + "Using mock token credential, JWT: {}, base64: {}", + serde_json::to_string(&jwt).unwrap(), + jwt_base64 + ); + + Ok(azure_core::credentials::AccessToken::new( + jwt_base64, + azure_core::time::OffsetDateTime::now_utc() + std::time::Duration::from_secs(3600), + )) + } +} + +#[cfg(test)] +#[tokio::test] +async fn azure_mock_token_credential_test() { + let credential = MockTokenCredential; + let access_token = credential + .get_token(&["https://example.com/.default"], None) + .await + .expect("valid credential should return a token"); + assert_eq!( + access_token.token.secret(), + "e30.eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiaWF0IjowLCJleHAiOjIxNDc0ODM2NDcsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LyIsIm5iZiI6MH0." + ); +} From 2de30d84aef2827ef207d886fddadd5960250b4f Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 5 Mar 2026 18:39:50 +0000 Subject: [PATCH 13/26] update doc examples Signed-off-by: Jed Laundry --- src/sinks/azure_common/config.rs | 17 ++++++++++--- .../components/sinks/generated/azure_blob.cue | 25 +++++++++++++++---- .../sinks/generated/azure_logs_ingestion.cue | 25 +++++++++++++++---- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 0bf72f56ee50f..609b98273e36f 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -52,18 +52,21 @@ pub enum AzureAuthentication { /// /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] azure_tenant_id: String, /// The [Azure Client ID][azure_client_id]. /// /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] azure_client_id: String, /// The [Azure Client Secret][azure_client_secret]. /// /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_SECRET:?err}"))] azure_client_secret: SensitiveString, }, @@ -95,18 +98,24 @@ pub enum SpecificAzureCredential { /// The [Azure Tenant ID][azure_tenant_id]. /// /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] azure_tenant_id: String, /// The [Azure Client ID][azure_client_id]. /// /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] azure_client_id: String, /// PKCS12 certificate with RSA private key. #[configurable(metadata(docs::examples = "path/to/certificate.pfx"))] - certificate_file: PathBuf, + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"))] + certificate_path: PathBuf, /// The password for the client certificate, if applicable. + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PASSWORD}"))] certificate_password: Option, }, @@ -216,15 +225,15 @@ impl SpecificAzureCredential { Self::ClientCertificateCredential { azure_tenant_id, azure_client_id, - certificate_file, + certificate_path, certificate_password, } => { - let certificate_bytes: Vec = std::fs::read(certificate_file).map_err(|e| { + let certificate_bytes: Vec = std::fs::read(certificate_path).map_err(|e| { Error::with_message( ErrorKind::Credential, format!( "Failed to read certificate file {}: {e}", - certificate_file.display() + certificate_path.display() ), ) })?; diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 5641dd6b6f3d5..e076472b91c66 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -47,8 +47,9 @@ generated: components: sinks: azure_blob: configuration: { [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: true - type: string: examples: ["00000000-0000-0000-0000-000000000000"] + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] } azure_client_secret: { description: """ @@ -57,13 +58,14 @@ generated: components: sinks: azure_blob: configuration: { [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ required: true - type: string: examples: ["00-00~000000-0000000~0000000000000000000"] + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] } azure_credential_kind: { description: "The kind of Azure credential to use." required: true type: string: enum: { azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" managed_identity: "Use Managed Identity credentials" managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" workload_identity: "Use Workload Identity credentials" @@ -75,8 +77,21 @@ generated: components: sinks: azure_blob: configuration: { [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: true - type: string: examples: ["00000000-0000-0000-0000-000000000000"] + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] } client_assertion_client_id: { description: "The target Client ID to use." diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index ec03373e5354e..06c9a39630105 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -37,10 +37,11 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false type: string: { default: "" - examples: ["00000000-0000-0000-0000-000000000000"] + examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] } } azure_client_secret: { @@ -52,7 +53,7 @@ generated: components: sinks: azure_logs_ingestion: configuration: { required: false type: string: { default: "" - examples: ["00-00~000000-0000000~0000000000000000000"] + examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] } } azure_credential_kind: { @@ -60,6 +61,7 @@ generated: components: sinks: azure_logs_ingestion: configuration: { required: true type: string: enum: { azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" managed_identity: "Use Managed Identity credentials" managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" workload_identity: "Use Workload Identity credentials" @@ -71,12 +73,25 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false type: string: { default: "" - examples: ["00000000-0000-0000-0000-000000000000"] + examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] } } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] + } client_assertion_client_id: { description: "The target Client ID to use." relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" From 6f94501d92e138425cfb05e98b6e2fdaf5f0d061 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 5 Mar 2026 18:45:28 +0000 Subject: [PATCH 14/26] make clippy happy Signed-off-by: Jed Laundry --- .github/actions/spelling/expect.txt | 1 + src/sinks/azure_common/config.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index edc77d73e5612..0b00b839e9484 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -377,6 +377,7 @@ mycie mycorp mydatabase mylabel +mylogstorage mypod myvalue Namazu diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 609b98273e36f..e0b581a57aa9c 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -530,7 +530,7 @@ pub async fn build_client( && let Some(ca_file) = &tls_config.ca_file { let mut buf = Vec::new(); - File::open(&ca_file)?.read_to_end(&mut buf)?; + File::open(ca_file)?.read_to_end(&mut buf)?; let cert = reqwest_12::Certificate::from_pem(&buf)?; warn!("Adding TLS root certificate from {}", ca_file.display()); From 50e4ff0a8601b02a03b5b552a14d8c25e94ee03c Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 5 Mar 2026 18:51:58 +0000 Subject: [PATCH 15/26] fix tests Signed-off-by: Jed Laundry --- src/sinks/azure_blob/test.rs | 2 +- src/sinks/azure_common/config.rs | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index dcb96ee32f78f..5f0861c92f4fa 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -296,7 +296,7 @@ async fn azure_blob_build_config_with_client_certificate() { azure_credential_kind = "client_certificate_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" - certificate_file = "tests/data/ClientCertificateAuth.pfx" + certificate_path = "tests/data/ClientCertificateAuth.pfx" certificate_password = "MockPassword123" "#, ) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index e0b581a57aa9c..8761903362c9b 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -566,12 +566,14 @@ impl TokenCredential for MockTokenCredential { )); }; + // serde_json sometimes does and sometimes doesn't preserve order, be careful to sort + // the claims in alphabetical order to ensure a consistent base64 encoding for testing let jwt = serde_json::json!({ "aud": scope.strip_suffix("/.default").unwrap_or(*scope), - "iat": 0, "exp": 2147483647, + "iat": 0, "iss": "https://sts.windows.net/", - "nbf": 0 + "nbf": 0, }); // JWTs do not include standard base64 padding. @@ -607,6 +609,6 @@ async fn azure_mock_token_credential_test() { .expect("valid credential should return a token"); assert_eq!( access_token.token.secret(), - "e30.eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiaWF0IjowLCJleHAiOjIxNDc0ODM2NDcsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LyIsIm5iZiI6MH0." + "e30.eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiZXhwIjoyMTQ3NDgzNjQ3LCJpYXQiOjAsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LyIsIm5iZiI6MH0." ); } From bc44aea92e4a20339b47fdf8ead340968ab815fc Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 10 Mar 2026 07:12:27 +0000 Subject: [PATCH 16/26] make "client_secret_credential" explicitly configured Signed-off-by: Jed Laundry --- src/sinks/azure_blob/test.rs | 38 ++++- src/sinks/azure_common/config.rs | 146 ++++++++---------- src/sinks/azure_logs_ingestion/config.rs | 1 - src/sinks/azure_logs_ingestion/tests.rs | 54 ++----- .../components/sinks/generated/azure_blob.cue | 10 +- .../sinks/generated/azure_logs_ingestion.cue | 31 ++-- 6 files changed, 135 insertions(+), 145 deletions(-) diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index 5f0861c92f4fa..024cbc15af1a0 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -241,6 +241,33 @@ fn azure_blob_build_request_with_uuid() { assert_eq!(request.content_type, "text/plain"); } +#[tokio::test] +async fn azure_blob_build_config_with_null_auth() { + let config: Result = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + "#, + ); + + match config { + Ok(_) => panic!("Config parsing should have failed due to invalid auth config"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("data did not match any variant of untagged enum"), + "Config parsing did not complain about invalid auth config: {}", + err_str + ); + } + } +} + #[tokio::test] async fn azure_blob_build_config_with_client_id_and_secret() { let config: AzureBlobSinkConfig = toml::from_str::( @@ -252,6 +279,7 @@ async fn azure_blob_build_config_with_client_id_and_secret() { codec = "json" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -262,17 +290,17 @@ async fn azure_blob_build_config_with_client_id_and_secret() { assert!(&config.auth.is_some()); match &config.auth.clone().unwrap() { - AzureAuthentication::ClientSecretCredential { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, - } => { + }) => { assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); assert_eq!(azure_client_id, "mock-client-id"); let secret: String = azure_client_secret.inner().into(); assert_eq!(secret, "mock-client-secret"); } - _ => panic!("Expected ClientSecretCredential variant"), + _ => panic!("Expected Specific(ClientSecretCredential) variant"), } let cx = SinkContext::default(); @@ -382,6 +410,7 @@ async fn azure_blob_build_config_with_conflicting_connection_string_and_client_i codec = "json" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -400,7 +429,8 @@ async fn azure_blob_build_config_with_conflicting_connection_string_and_client_i Err(e) => { let err_str = e.to_string(); assert!( - err_str.contains("Cannot use both Shared Key and Client ID"), + err_str + .contains("Cannot use both Shared Key and another Azure Authentication method"), "Config build did not complain about conflicting Shared Key and Client ID: {}", err_str ); diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 8761903362c9b..95d5d43408c15 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -39,38 +39,11 @@ use crate::{ tls::TlsConfig, }; -/// Configuration of the authentication strategy for interacting with Azure services. +/// Azure service principal authentication. #[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Debug, Eq, PartialEq)] #[serde(deny_unknown_fields, untagged)] pub enum AzureAuthentication { - /// Use client credentials - #[derivative(Default)] - ClientSecretCredential { - /// The [Azure Tenant ID][azure_tenant_id]. - /// - /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] - azure_tenant_id: String, - - /// The [Azure Client ID][azure_client_id]. - /// - /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] - azure_client_id: String, - - /// The [Azure Client Secret][azure_client_secret]. - /// - /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] - #[configurable(metadata(docs::examples = "${AZURE_CLIENT_SECRET:?err}"))] - azure_client_secret: SensitiveString, - }, - - /// Use credentials from environment variables #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] Specific(SpecificAzureCredential), @@ -80,6 +53,17 @@ pub enum AzureAuthentication { MockCredential, } +impl Default for AzureAuthentication { + // This should never be actually used. + // This is only needed when using Default::default() (such as unit tests), + // as serde requires `azure_credential_kind` to be specified. + fn default() -> Self { + Self::Specific(SpecificAzureCredential::ManagedIdentity { + user_assigned_managed_identity_id: None, + }) + } +} + /// Specific Azure credential types. #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] @@ -119,6 +103,30 @@ pub enum SpecificAzureCredential { certificate_password: Option, }, + /// Use client ID/secret credentials + ClientSecretCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// The [Azure Client Secret][azure_client_secret]. + /// + /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_SECRET:?err}"))] + azure_client_secret: SensitiveString, + }, + /// Use Managed Identity credentials ManagedIdentity { /// The User Assigned Managed Identity (Client ID) to use. @@ -176,36 +184,6 @@ impl AzureAuthentication { /// Returns the provider for the credentials based on the authentication mechanism chosen. pub async fn credential(&self) -> azure_core::Result> { match self { - Self::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - if azure_tenant_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_secret.inner().is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - let secret: String = azure_client_secret.inner().into(); - let credential: Arc = ClientSecretCredential::new( - &azure_tenant_id.clone(), - azure_client_id.clone(), - secret.into(), - None, - )?; - Ok(credential) - } - Self::Specific(specific) => specific.credential().await, #[cfg(test)] @@ -256,6 +234,36 @@ impl SpecificAzureCredential { )? } + Self::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + } => { + if azure_tenant_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_secret.inner().is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + + let secret: String = azure_client_secret.inner().into(); + ClientSecretCredential::new( + &azure_tenant_id.clone(), + azure_client_id.clone(), + secret.into(), + None, + )? + } + Self::ManagedIdentity { user_assigned_managed_identity_id, } => { @@ -448,30 +456,12 @@ pub async fn build_client( .per_call_policies .push(Arc::new(policy)); } - (Auth::None, Some(AzureAuthentication::ClientSecretCredential { .. })) => { - info!("Using Client Secret authentication"); - let credential_result: Arc = - auth.unwrap().credential().await.unwrap(); - credential = Some(credential_result); - } (Auth::None, Some(AzureAuthentication::Specific(..))) => { - info!("Using specific Azure Authentication method"); + info!("Using Azure Authentication method"); let credential_result: Arc = auth.unwrap().credential().await.unwrap(); credential = Some(credential_result); } - (Auth::Sas { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { - return Err(Box::new(Error::with_message( - ErrorKind::Credential, - "Cannot use both SAS token and Client ID/Secret at the same time", - ))); - } - (Auth::SharedKey { .. }, Some(AzureAuthentication::ClientSecretCredential { .. })) => { - return Err(Box::new(Error::with_message( - ErrorKind::Credential, - "Cannot use both Shared Key and Client ID/Secret at the same time", - ))); - } (Auth::Sas { .. }, Some(AzureAuthentication::Specific(..))) => { return Err(Box::new(Error::with_message( ErrorKind::Credential, diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 2649edc320a89..3b105d549d0c1 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -59,7 +59,6 @@ pub struct AzureLogsIngestionConfig { pub stream_name: String, #[configurable(derived)] - #[serde(default)] pub auth: AzureAuthentication, /// [Token scope][token_scope] for dedicated Azure regions. diff --git a/src/sinks/azure_logs_ingestion/tests.rs b/src/sinks/azure_logs_ingestion/tests.rs index fd9e01835db2b..46095071bd28c 100644 --- a/src/sinks/azure_logs_ingestion/tests.rs +++ b/src/sinks/azure_logs_ingestion/tests.rs @@ -28,50 +28,22 @@ fn generate_config() { #[tokio::test] async fn basic_config_error_with_no_auth() { - let config: AzureLogsIngestionConfig = toml::from_str::( - r#" + let config: Result = + toml::from_str::( + r#" endpoint = "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" dcr_immutable_id = "dcr-00000000000000000000000000000000" stream_name = "Custom-UnitTest" "#, - ) - .expect("Config parsing failed"); - - assert_eq!( - config.endpoint, - "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" - ); - assert_eq!( - config.dcr_immutable_id, - "dcr-00000000000000000000000000000000" - ); - assert_eq!(config.stream_name, "Custom-UnitTest"); - assert_eq!(config.token_scope, "https://monitor.azure.com/.default"); - assert_eq!(config.timestamp_field, "TimeGenerated"); - - match &config.auth { - crate::sinks::azure_common::config::AzureAuthentication::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - assert_eq!(azure_tenant_id, ""); - assert_eq!(azure_client_id, ""); - let secret: String = azure_client_secret.inner().into(); - assert_eq!(secret, ""); - } - _ => panic!("Expected ClientSecretCredential variant"), - } + ); - let cx = SinkContext::default(); - let sink = config.build(cx).await; - match sink { - Ok(_) => panic!("Config build should have errored due to missing auth info"), + match config { + Ok(_) => panic!("Config parsing should have failed due to missing auth config"), Err(e) => { let err_str = e.to_string(); assert!( - err_str.contains("`auth.azure_tenant_id` is blank"), - "Config build did not complain about azure_tenant_id being blank: {}", + err_str.contains("missing field `auth`"), + "Config parsing did not complain about missing auth field: {}", err_str ); } @@ -87,6 +59,7 @@ fn basic_config_with_client_credentials() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -107,17 +80,17 @@ fn basic_config_with_client_credentials() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - AzureAuthentication::ClientSecretCredential { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, - } => { + }) => { assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); assert_eq!(azure_client_id, "mock-client-id"); let secret: String = azure_client_secret.inner().into(); assert_eq!(secret, "mock-client-secret"); } - _ => panic!("Expected ClientSecretCredential variant"), + _ => panic!("Expected Specific(ClientSecretCredential) variant"), } } @@ -180,6 +153,7 @@ async fn correct_request() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -290,6 +264,7 @@ async fn mock_healthcheck_with_400_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -359,6 +334,7 @@ async fn mock_healthcheck_with_403_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index e076472b91c66..9570d874bf456 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -38,7 +38,7 @@ generated: components: sinks: azure_blob: configuration: { } } auth: { - description: "Configuration of the authentication strategy for interacting with Azure services." + description: "Azure service principal authentication." required: false type: object: options: { azure_client_id: { @@ -47,7 +47,7 @@ generated: components: sinks: azure_blob: configuration: { [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" required: true type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] } @@ -57,7 +57,8 @@ generated: components: sinks: azure_blob: configuration: { [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: true + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] } azure_credential_kind: { @@ -66,6 +67,7 @@ generated: components: sinks: azure_blob: configuration: { type: string: enum: { azure_cli: "Use Azure CLI credentials" client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" managed_identity: "Use Managed Identity credentials" managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" workload_identity: "Use Workload Identity credentials" @@ -77,7 +79,7 @@ generated: components: sinks: azure_blob: configuration: { [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" required: true type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] } diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index 06c9a39630105..45741d5febc21 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -28,8 +28,8 @@ generated: components: sinks: azure_logs_ingestion: configuration: { } } auth: { - description: "Configuration of the authentication strategy for interacting with Azure services." - required: false + description: "Azure service principal authentication." + required: true type: object: options: { azure_client_id: { description: """ @@ -37,12 +37,9 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - relevant_when: "azure_credential_kind = \"client_certificate_credential\"" - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] } azure_client_secret: { description: """ @@ -50,11 +47,9 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] - } + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] } azure_credential_kind: { description: "The kind of Azure credential to use." @@ -62,6 +57,7 @@ generated: components: sinks: azure_logs_ingestion: configuration: { type: string: enum: { azure_cli: "Use Azure CLI credentials" client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" managed_identity: "Use Managed Identity credentials" managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" workload_identity: "Use Workload Identity credentials" @@ -73,12 +69,9 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - relevant_when: "azure_credential_kind = \"client_certificate_credential\"" - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] } certificate_password: { description: "The password for the client certificate, if applicable." From 69b8b3888e7ed4bee7992eb57bfc6313e1c04c88 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 10 Mar 2026 07:12:46 +0000 Subject: [PATCH 17/26] add azure_logs_ingestion to Azure integration tests Signed-off-by: Jed Laundry --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e77e0c15ce302..51e36d07856b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -968,7 +968,8 @@ aws-integration-tests = [ ] azure-integration-tests = [ - "azure-blob-integration-tests" + "azure-blob-integration-tests", + "azure-logs-ingestion-integration-tests", ] aws-cloudwatch-logs-integration-tests = ["sinks-aws_cloudwatch_logs"] @@ -982,6 +983,7 @@ aws-sqs-integration-tests = ["sinks-aws_sqs"] aws-sns-integration-tests = ["sinks-aws_sns"] axiom-integration-tests = ["sinks-axiom"] azure-blob-integration-tests = ["sinks-azure_blob"] +azure-logs-ingestion-integration-tests = ["sinks-azure_logs_ingestion"] chronicle-integration-tests = ["sinks-gcp"] clickhouse-integration-tests = ["sinks-clickhouse"] databend-integration-tests = ["sinks-databend"] From f2ec30f9419ab11d14dbb4a70a7db9f3ab436fcf Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 10 Mar 2026 08:22:46 +0000 Subject: [PATCH 18/26] Err out the other TlsConfig options Signed-off-by: Jed Laundry --- src/sinks/azure_blob/test.rs | 35 ++++++++++++++++++++++ src/sinks/azure_common/config.rs | 51 +++++++++++++++++++++++++++----- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index 024cbc15af1a0..5d5454a2a0ffa 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -437,6 +437,7 @@ async fn azure_blob_build_config_with_conflicting_connection_string_and_client_i } } } + #[tokio::test] async fn azure_blob_build_config_with_custom_ca_certificate() { let config: AzureBlobSinkConfig = toml::from_str::( @@ -459,3 +460,37 @@ async fn azure_blob_build_config_with_custom_ca_certificate() { .await .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); } + +#[tokio::test] +async fn azure_blob_build_config_with_crt_file() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [tls] + crt_file = "tests/data/ca/intermediate_client/certs/localhost.cert.pem" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => { + panic!("Config build should have errored due to `crt_file` being unsupported") + } + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str + .contains("TLS option `crt_file` is not supported"), + "Config build did not error about `crt_file` being unsupported: {}", + err_str + ); + } + } +} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 95d5d43408c15..4983086edb218 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -516,15 +516,52 @@ pub async fn build_client( } } - if let Some(tls_config) = &tls - && let Some(ca_file) = &tls_config.ca_file + if let Some(TlsConfig { + verify_certificate, + verify_hostname, + alpn_protocols, + ca_file, + crt_file, + key_file, + key_pass, + server_name, + }) = &tls { - let mut buf = Vec::new(); - File::open(ca_file)?.read_to_end(&mut buf)?; - let cert = reqwest_12::Certificate::from_pem(&buf)?; + if verify_certificate.is_some() { + return Err( + "TLS option `verify_certificate` is not supported for the azure_blob sink".into(), + ); + } + if verify_hostname.is_some() { + return Err( + "TLS option `verify_hostname` is not supported for the azure_blob sink".into(), + ); + } + if alpn_protocols.is_some() { + return Err( + "TLS option `alpn_protocols` is not supported for the azure_blob sink".into(), + ); + } + if let Some(ca_file) = ca_file { + let mut buf = Vec::new(); + File::open(ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_12::Certificate::from_pem(&buf)?; - warn!("Adding TLS root certificate from {}", ca_file.display()); - reqwest_builder = reqwest_builder.add_root_certificate(cert); + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); + } + if crt_file.is_some() { + return Err("TLS option `crt_file` is not supported for the azure_blob sink".into()); + } + if key_file.is_some() { + return Err("TLS option `key_file` is not supported for the azure_blob sink".into()); + } + if key_pass.is_some() { + return Err("TLS option `key_pass` is not supported for the azure_blob sink".into()); + } + if server_name.is_some() { + return Err("TLS option `server_name` is not supported for the azure_blob sink".into()); + } } options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( From 212a90a88d0e70ce45fb9c24aa26fb497126cc19 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 10 Mar 2026 08:33:21 +0000 Subject: [PATCH 19/26] Add warning message to connection_string Signed-off-by: Jed Laundry --- src/sinks/azure_blob/config.rs | 5 +++++ .../cue/reference/components/sinks/generated/azure_blob.cue | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 010b77fc2209b..21ddea56b7867 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -53,6 +53,11 @@ pub struct AzureBlobSinkConfig { /// are supported authentication methods. If using a non-account SAS, /// healthchecks will fail and will need to be disabled by setting /// `healthcheck.enabled` to `false` for this sink + /// + /// SECURITY WARNING: Access keys and SAS tokens can be used to gain unauthorized access to + /// Azure Blob Storage resources. Numerous security breaches have occurred due to leaked + /// connection strings. It is important to keep connection strings secure and not expose them + /// in logs, error messages, or version control systems. /// /// When generating an account SAS, the following are the minimum required option /// settings for Vector to access blob storage and pass a health check. diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 9570d874bf456..53574ca9d00f3 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -255,6 +255,11 @@ generated: components: sinks: azure_blob: configuration: { healthchecks will fail and will need to be disabled by setting `healthcheck.enabled` to `false` for this sink + SECURITY WARNING: Access keys and SAS tokens can be used to gain unauthorized access to + Azure Blob Storage resources. Numerous security breaches have occurred due to leaked + connection strings. It is important to keep connection strings secure and not expose them + in logs, error messages, or version control systems. + When generating an account SAS, the following are the minimum required option settings for Vector to access blob storage and pass a health check. | Option | Value | From 4ed432471c7523ddf35530fe4b63944be93a07ae Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 10 Mar 2026 09:14:53 +0000 Subject: [PATCH 20/26] Simplify TlsConfig Signed-off-by: Jed Laundry --- src/sinks/azure_blob/config.rs | 7 +- src/sinks/azure_blob/integration_tests.rs | 5 +- src/sinks/azure_blob/test.rs | 34 ------- src/sinks/azure_common/config.rs | 57 ++++------- .../components/sinks/generated/azure_blob.cue | 95 ++----------------- 5 files changed, 29 insertions(+), 169 deletions(-) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 21ddea56b7867..6752b9d8505fa 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -17,7 +17,7 @@ use crate::{ Healthcheck, VectorSink, azure_common::{ self, config::AzureAuthentication, config::AzureBlobRetryLogic, - service::AzureBlobService, sink::AzureBlobSink, + config::AzureBlobTlsConfig, service::AzureBlobService, sink::AzureBlobSink, }, util::{ BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt, @@ -25,7 +25,6 @@ use crate::{ }, }, template::Template, - tls::TlsConfig, }; #[derive(Clone, Copy, Debug)] @@ -53,7 +52,7 @@ pub struct AzureBlobSinkConfig { /// are supported authentication methods. If using a non-account SAS, /// healthchecks will fail and will need to be disabled by setting /// `healthcheck.enabled` to `false` for this sink - /// + /// /// SECURITY WARNING: Access keys and SAS tokens can be used to gain unauthorized access to /// Azure Blob Storage resources. Numerous security breaches have occurred due to leaked /// connection strings. It is important to keep connection strings secure and not expose them @@ -167,7 +166,7 @@ pub struct AzureBlobSinkConfig { #[configurable(derived)] #[serde(default)] - pub tls: Option, + pub tls: Option, } pub fn default_blob_prefix() -> Template { diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index d32cab88afaa3..dbe20a884ff86 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -26,7 +26,7 @@ use crate::{ components::{SINK_TAGS, assert_sink_compliance}, random_events_with_stream, random_lines, random_lines_with_stream, random_string, }, - tls::{self, TlsConfig}, + tls, }; #[tokio::test] @@ -284,9 +284,8 @@ impl AzureBlobSinkConfig { batch: Default::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), - tls: Some(TlsConfig { + tls: Some(azure_common::config::AzureBlobTlsConfig { ca_file: Some(tls::TEST_PEM_CA_PATH.into()), - ..Default::default() }), }; diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index 5d5454a2a0ffa..9b5a11e202dd5 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -460,37 +460,3 @@ async fn azure_blob_build_config_with_custom_ca_certificate() { .await .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); } - -#[tokio::test] -async fn azure_blob_build_config_with_crt_file() { - let config: AzureBlobSinkConfig = toml::from_str::( - r#" - account_name = "mylogstorage" - container_name = "my-logs" - - [encoding] - codec = "json" - - [tls] - crt_file = "tests/data/ca/intermediate_client/certs/localhost.cert.pem" - "#, - ) - .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); - - let cx = SinkContext::default(); - let sink = config.build(cx).await; - match sink { - Ok(_) => { - panic!("Config build should have errored due to `crt_file` being unsupported") - } - Err(e) => { - let err_str = e.to_string(); - assert!( - err_str - .contains("TLS option `crt_file` is not supported"), - "Config build did not error about `crt_file` being unsupported: {}", - err_str - ); - } - } -} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 4983086edb218..c53d7d423a9f6 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -36,9 +36,23 @@ use vector_lib::{ use crate::{ event::{EventFinalizers, EventStatus, Finalizable}, sinks::{Healthcheck, util::retries::RetryLogic}, - tls::TlsConfig, }; +/// TLS configuration. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] +pub struct AzureBlobTlsConfig { + /// Absolute path to an additional CA certificate file. + /// + /// The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + #[serde(alias = "ca_path")] + #[configurable(metadata(docs::examples = "/path/to/certificate_authority.crt"))] + #[configurable(metadata(docs::human_name = "CA File Path"))] + pub ca_file: Option, +} + /// Azure service principal authentication. #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] @@ -413,7 +427,7 @@ pub async fn build_client( connection_string: String, container_name: String, proxy: &crate::config::ProxyConfig, - tls: Option, + tls: Option, ) -> crate::Result> { // Parse connection string without legacy SDK let parsed = ParsedConnectionString::parse(&connection_string) @@ -516,32 +530,7 @@ pub async fn build_client( } } - if let Some(TlsConfig { - verify_certificate, - verify_hostname, - alpn_protocols, - ca_file, - crt_file, - key_file, - key_pass, - server_name, - }) = &tls - { - if verify_certificate.is_some() { - return Err( - "TLS option `verify_certificate` is not supported for the azure_blob sink".into(), - ); - } - if verify_hostname.is_some() { - return Err( - "TLS option `verify_hostname` is not supported for the azure_blob sink".into(), - ); - } - if alpn_protocols.is_some() { - return Err( - "TLS option `alpn_protocols` is not supported for the azure_blob sink".into(), - ); - } + if let Some(AzureBlobTlsConfig { ca_file }) = &tls { if let Some(ca_file) = ca_file { let mut buf = Vec::new(); File::open(ca_file)?.read_to_end(&mut buf)?; @@ -550,18 +539,6 @@ pub async fn build_client( warn!("Adding TLS root certificate from {}", ca_file.display()); reqwest_builder = reqwest_builder.add_root_certificate(cert); } - if crt_file.is_some() { - return Err("TLS option `crt_file` is not supported for the azure_blob sink".into()); - } - if key_file.is_some() { - return Err("TLS option `key_file` is not supported for the azure_blob sink".into()); - } - if key_pass.is_some() { - return Err("TLS option `key_pass` is not supported for the azure_blob sink".into()); - } - if server_name.is_some() { - return Err("TLS option `server_name` is not supported for the azure_blob sink".into()); - } } options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 53574ca9d00f3..c3170bed03204 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -968,95 +968,14 @@ generated: components: sinks: azure_blob: configuration: { tls: { description: "TLS configuration." required: false - type: object: options: { - alpn_protocols: { - description: """ - Sets the list of supported ALPN protocols. - - Declare the supported ALPN protocols, which are used during negotiation with a peer. They are prioritized in the order - that they are defined. - """ - required: false - type: array: items: type: string: examples: ["h2"] - } - ca_file: { - description: """ - Absolute path to an additional CA certificate file. - - The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. - """ - required: false - type: string: examples: ["/path/to/certificate_authority.crt"] - } - crt_file: { - description: """ - Absolute path to a certificate file used to identify this server. - - The certificate must be in DER, PEM (X.509), or PKCS#12 format. Additionally, the certificate can be provided as - an inline string in PEM format. - - If this is set _and_ is not a PKCS#12 archive, `key_file` must also be set. - """ - required: false - type: string: examples: ["/path/to/host_certificate.crt"] - } - key_file: { - description: """ - Absolute path to a private key file used to identify this server. - - The key must be in DER or PEM (PKCS#8) format. Additionally, the key can be provided as an inline string in PEM format. - """ - required: false - type: string: examples: ["/path/to/host_certificate.key"] - } - key_pass: { - description: """ - Passphrase used to unlock the encrypted key file. - - This has no effect unless `key_file` is set. - """ - required: false - type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"] - } - server_name: { - description: """ - Server name to use when using Server Name Indication (SNI). - - Only relevant for outgoing connections. - """ - required: false - type: string: examples: ["www.example.com"] - } - verify_certificate: { - description: """ - Enables certificate verification. For components that create a server, this requires that the - client connections have a valid client certificate. For components that initiate requests, - this validates that the upstream has a valid certificate. - - If enabled, certificates must not be expired and must be issued by a trusted - issuer. This verification operates in a hierarchical manner, checking that the leaf certificate (the - certificate presented by the client/server) is not only valid, but that the issuer of that certificate is also valid, and - so on, until the verification process reaches a root certificate. - - Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. - """ - required: false - type: bool: {} - } - verify_hostname: { - description: """ - Enables hostname verification. - - If enabled, the hostname used to connect to the remote host must be present in the TLS certificate presented by - the remote host, either as the Common Name or as an entry in the Subject Alternative Name extension. - - Only relevant for outgoing connections. + type: object: options: ca_file: { + description: """ + Absolute path to an additional CA certificate file. - Do NOT set this to `false` unless you understand the risks of not verifying the remote hostname. - """ - required: false - type: bool: {} - } + The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] } } } From e4b650fd6680f7e24fd979030725f11fa2666530 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 10 Mar 2026 21:28:24 +0000 Subject: [PATCH 21/26] add workload identity config options --- src/sinks/azure_common/config.rs | 38 ++++++++++++++++++- .../components/sinks/generated/azure_blob.cue | 26 +++++++++++++ .../sinks/generated/azure_logs_ingestion.cue | 26 +++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index c53d7d423a9f6..51f33ce9c1d13 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -18,6 +18,7 @@ use azure_identity::{ AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientCertificateCredential, ClientCertificateCredentialOptions, ClientSecretCredential, ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, WorkloadIdentityCredential, + WorkloadIdentityCredentialOptions, }; use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; @@ -166,7 +167,28 @@ pub enum SpecificAzureCredential { }, /// Use Workload Identity credentials - WorkloadIdentity {}, + WorkloadIdentity { + /// The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID}"))] + tenant_id: Option, + + /// The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID}"))] + client_id: Option, + + /// Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`. + #[configurable(metadata( + docs::examples = "/var/run/secrets/azure/tokens/azure-identity-token" + ))] + #[configurable(metadata(docs::examples = "${AZURE_FEDERATED_TOKEN_FILE}"))] + token_file_path: Option, + }, } #[derive(Debug)] @@ -312,7 +334,19 @@ impl SpecificAzureCredential { )? } - Self::WorkloadIdentity {} => WorkloadIdentityCredential::new(None)?, + Self::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + } => { + let mut options = WorkloadIdentityCredentialOptions::default(); + + options.tenant_id = tenant_id.clone(); + options.client_id = client_id.clone(); + options.token_file_path = token_file_path.clone(); + + WorkloadIdentityCredential::new(Some(options))? + } }; Ok(credential) } diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index c3170bed03204..46b914e5e57e1 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -107,6 +107,32 @@ generated: components: sinks: azure_blob: configuration: { required: true type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } user_assigned_managed_identity_id: { description: "The User Assigned Managed Identity (Client ID) to use." relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index 45741d5febc21..7eb21fb8e613e 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -97,6 +97,32 @@ generated: components: sinks: azure_logs_ingestion: configuration: { required: true type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } user_assigned_managed_identity_id: { description: "The User Assigned Managed Identity (Client ID) to use." relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" From 5af8d1a9b78a27d71142657b27eb1ab7fb53412a Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 12 Mar 2026 12:02:40 -0400 Subject: [PATCH 22/26] Move warning into docs::warnings --- src/sinks/azure_blob/config.rs | 10 +++++----- .../components/sinks/generated/azure_blob.cue | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 6752b9d8505fa..7db3a0ff576a8 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -53,11 +53,6 @@ pub struct AzureBlobSinkConfig { /// healthchecks will fail and will need to be disabled by setting /// `healthcheck.enabled` to `false` for this sink /// - /// SECURITY WARNING: Access keys and SAS tokens can be used to gain unauthorized access to - /// Azure Blob Storage resources. Numerous security breaches have occurred due to leaked - /// connection strings. It is important to keep connection strings secure and not expose them - /// in logs, error messages, or version control systems. - /// /// When generating an account SAS, the following are the minimum required option /// settings for Vector to access blob storage and pass a health check. /// | Option | Value | @@ -65,6 +60,11 @@ pub struct AzureBlobSinkConfig { /// | Allowed services | Blob | /// | Allowed resource types | Container & Object | /// | Allowed permissions | Read & Create | + #[configurable(metadata( + docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage\ + resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep\ + connection strings secure and not expose them in logs, error messages, or version control systems." + ))] #[configurable(metadata( docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net" ))] diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 46b914e5e57e1..9992cd1eb1d34 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -296,6 +296,7 @@ generated: components: sinks: azure_blob: configuration: { """ required: false type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken", "AccountName=mylogstorage"] + warnings: ["Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storageresources. Numerous security breaches have occurred due to leaked connection strings. It is important to keepconnection strings secure and not expose them in logs, error messages, or version control systems."] } container_name: { description: "The Azure Blob Storage Account container name." From 348cc24e5aebdf46088ed2484a71e2186f915477 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 12 Mar 2026 12:08:27 -0400 Subject: [PATCH 23/26] Properly regenerate cue file --- .../cue/reference/components/sinks/generated/azure_blob.cue | 5 ----- 1 file changed, 5 deletions(-) diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 9992cd1eb1d34..60b842d874193 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -281,11 +281,6 @@ generated: components: sinks: azure_blob: configuration: { healthchecks will fail and will need to be disabled by setting `healthcheck.enabled` to `false` for this sink - SECURITY WARNING: Access keys and SAS tokens can be used to gain unauthorized access to - Azure Blob Storage resources. Numerous security breaches have occurred due to leaked - connection strings. It is important to keep connection strings secure and not expose them - in logs, error messages, or version control systems. - When generating an account SAS, the following are the minimum required option settings for Vector to access blob storage and pass a health check. | Option | Value | From 9254aee11b1a0cdd39d0fe3f5e1ed927e3c9fd63 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 12 Mar 2026 12:10:38 -0400 Subject: [PATCH 24/26] Fix spaces --- src/sinks/azure_blob/config.rs | 4 ++-- .../cue/reference/components/sinks/generated/azure_blob.cue | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 7db3a0ff576a8..f6b107ebfec1b 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -61,8 +61,8 @@ pub struct AzureBlobSinkConfig { /// | Allowed resource types | Container & Object | /// | Allowed permissions | Read & Create | #[configurable(metadata( - docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage\ - resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep\ + docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage \ + resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep \ connection strings secure and not expose them in logs, error messages, or version control systems." ))] #[configurable(metadata( diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 60b842d874193..d6646c936a16e 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -291,7 +291,7 @@ generated: components: sinks: azure_blob: configuration: { """ required: false type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken", "AccountName=mylogstorage"] - warnings: ["Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storageresources. Numerous security breaches have occurred due to leaked connection strings. It is important to keepconnection strings secure and not expose them in logs, error messages, or version control systems."] + warnings: ["Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep connection strings secure and not expose them in logs, error messages, or version control systems."] } container_name: { description: "The Azure Blob Storage Account container name." From a18adfc069b522e5bcbb94c5c0824bddb5479c50 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 12 Mar 2026 17:28:53 +0000 Subject: [PATCH 25/26] tidy options creation Signed-off-by: Jed Laundry --- src/sinks/azure_common/config.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 51f33ce9c1d13..60b87e9e5f6ed 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -339,11 +339,12 @@ impl SpecificAzureCredential { client_id, token_file_path, } => { - let mut options = WorkloadIdentityCredentialOptions::default(); - - options.tenant_id = tenant_id.clone(); - options.client_id = client_id.clone(); - options.token_file_path = token_file_path.clone(); + let options = WorkloadIdentityCredentialOptions { + tenant_id: tenant_id.clone(), + client_id: client_id.clone(), + token_file_path: token_file_path.clone(), + ..Default::default() + }; WorkloadIdentityCredential::new(Some(options))? } @@ -564,15 +565,15 @@ pub async fn build_client( } } - if let Some(AzureBlobTlsConfig { ca_file }) = &tls { - if let Some(ca_file) = ca_file { - let mut buf = Vec::new(); - File::open(ca_file)?.read_to_end(&mut buf)?; - let cert = reqwest_12::Certificate::from_pem(&buf)?; + if let Some(AzureBlobTlsConfig { ca_file }) = &tls + && let Some(ca_file) = ca_file + { + let mut buf = Vec::new(); + File::open(ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_12::Certificate::from_pem(&buf)?; - warn!("Adding TLS root certificate from {}", ca_file.display()); - reqwest_builder = reqwest_builder.add_root_certificate(cert); - } + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); } options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( From 2d9f1a6d055efddba50b76679a5bd5327a6679bd Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 12 Mar 2026 17:59:32 +0000 Subject: [PATCH 26/26] add UAMI type option Signed-off-by: Jed Laundry --- src/sinks/azure_common/config.rs | 62 +++++++++++++++++-- .../components/sinks/generated/azure_blob.cue | 15 ++++- .../sinks/generated/azure_logs_ingestion.cue | 15 ++++- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 60b87e9e5f6ed..6f7c241ba084e 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -75,10 +75,26 @@ impl Default for AzureAuthentication { fn default() -> Self { Self::Specific(SpecificAzureCredential::ManagedIdentity { user_assigned_managed_identity_id: None, + user_assigned_managed_identity_id_type: None, }) } } +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +#[derive(Default)] +/// User Assigned Managed Identity Types. +pub enum UserAssignedManagedIdentityIdType { + #[default] + /// Client ID + ClientId, + /// Object ID + ObjectId, + /// Resource ID + ResourceId, +} + /// Specific Azure credential types. #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] @@ -144,19 +160,29 @@ pub enum SpecificAzureCredential { /// Use Managed Identity credentials ManagedIdentity { - /// The User Assigned Managed Identity (Client ID) to use. + /// The User Assigned Managed Identity to use. #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] #[serde(default, skip_serializing_if = "Option::is_none")] user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + /// or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, }, /// Use Managed Identity with Client Assertion credentials ManagedIdentityClientAssertion { - /// The User Assigned Managed Identity (Client ID) to use for the managed identity. + /// The User Assigned Managed Identity to use for the managed identity. #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata( + docs::examples = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-vector/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-vector-uami" + ))] #[serde(default, skip_serializing_if = "Option::is_none")] user_assigned_managed_identity_id: Option, + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + /// The target Tenant ID to use. #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] client_assertion_tenant_id: String, @@ -302,22 +328,50 @@ impl SpecificAzureCredential { Self::ManagedIdentity { user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, } => { let mut options = ManagedIdentityCredentialOptions::default(); if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; } ManagedIdentityCredential::new(Some(options))? } Self::ManagedIdentityClientAssertion { user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, client_assertion_tenant_id, client_assertion_client_id, } => { let mut options = ManagedIdentityCredentialOptions::default(); if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; } let msi: Arc = ManagedIdentityCredential::new(Some(options))?; let assertion = ManagedIdentityClientAssertion { diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index d6646c936a16e..f1e544cdaa008 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -134,11 +134,24 @@ generated: components: sinks: azure_blob: configuration: { type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] } user_assigned_managed_identity_id: { - description: "The User Assigned Managed Identity (Client ID) to use." + description: "The User Assigned Managed Identity to use." relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" required: false type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } } } batch: { diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index 7eb21fb8e613e..2aaf9b628e3c6 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -124,11 +124,24 @@ generated: components: sinks: azure_logs_ingestion: configuration: { type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] } user_assigned_managed_identity_id: { - description: "The User Assigned Managed Identity (Client ID) to use." + description: "The User Assigned Managed Identity to use." relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" required: false type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } } } batch: {