diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 60715b9c96c0e..ddd945006f1c4 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -379,6 +379,7 @@ mycie mycorp mydatabase mylabel +mylogstorage mypod myvalue Namazu diff --git a/Cargo.lock b/Cargo.lock index 75e89d42c3280..91b66e6de9e72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1768,6 +1768,7 @@ dependencies = [ "async-trait", "azure_core", "futures 0.3.31", + "openssl", "pin-project", "serde", "time", diff --git a/Cargo.toml b/Cargo.toml index ed8631a6d1e94..9323d0dc90c83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,7 +307,7 @@ aws-smithy-types = { version = "1.2.11", default-features = false, features = [" # Azure azure_core = { version = "0.30", features = ["reqwest", "hmac_openssl"], optional = true } -azure_identity = { version = "0.30", optional = true } +azure_identity = { version = "0.30", features = ["client_certificate"], optional = true } # Azure Storage azure_storage_blob = { version = "0.7", optional = true } @@ -879,7 +879,7 @@ sinks-aws_s3 = ["dep:base64", "dep:md-5", "aws-core", "dep:aws-sdk-s3"] sinks-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] sinks-aws_sns = ["aws-core", "dep:aws-sdk-sns"] sinks-axiom = ["sinks-http"] -sinks-azure_blob = ["dep:azure_core", "dep:azure_storage_blob"] +sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob"] sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity"] sinks-azure_monitor_logs = [] sinks-blackhole = [] @@ -991,7 +991,8 @@ aws-integration-tests = [ ] azure-integration-tests = [ - "azure-blob-integration-tests" + "azure-blob-integration-tests", + "azure-logs-ingestion-integration-tests", ] aws-cloudwatch-logs-integration-tests = ["sinks-aws_cloudwatch_logs"] @@ -1005,6 +1006,7 @@ aws-sqs-integration-tests = ["sinks-aws_sqs"] aws-sns-integration-tests = ["sinks-aws_sns"] axiom-integration-tests = ["sinks-axiom"] azure-blob-integration-tests = ["sinks-azure_blob"] +azure-logs-ingestion-integration-tests = ["sinks-azure_logs_ingestion"] chronicle-integration-tests = ["sinks-gcp"] clickhouse-integration-tests = ["sinks-clickhouse"] databend-integration-tests = ["sinks-databend"] diff --git a/changelog.d/24729_azure_blob_authentication.feature.md b/changelog.d/24729_azure_blob_authentication.feature.md new file mode 100644 index 0000000000000..7b224172563b7 --- /dev/null +++ b/changelog.d/24729_azure_blob_authentication.feature.md @@ -0,0 +1,3 @@ +Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. + +authors: jlaundry diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index db0521b4fa137..f6b107ebfec1b 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -16,7 +16,8 @@ use crate::{ sinks::{ Healthcheck, VectorSink, azure_common::{ - self, config::AzureBlobRetryLogic, service::AzureBlobService, sink::AzureBlobSink, + self, config::AzureAuthentication, config::AzureBlobRetryLogic, + config::AzureBlobTlsConfig, service::AzureBlobService, sink::AzureBlobSink, }, util::{ BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt, @@ -41,6 +42,10 @@ impl TowerRequestConfigDefaults for AzureBlobTowerRequestConfigDefaults { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct AzureBlobSinkConfig { + #[configurable(derived)] + #[serde(default)] + pub auth: Option, + /// The Azure Blob Storage Account connection string. /// /// Authentication with an access key or shared access signature (SAS) @@ -55,13 +60,33 @@ pub struct AzureBlobSinkConfig { /// | Allowed services | Blob | /// | Allowed resource types | Container & Object | /// | Allowed permissions | Read & Create | + #[configurable(metadata( + docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage \ + resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep \ + connection strings secure and not expose them in logs, error messages, or version control systems." + ))] #[configurable(metadata( docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net" ))] #[configurable(metadata( docs::examples = "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken" ))] - pub connection_string: SensitiveString, + #[configurable(metadata(docs::examples = "AccountName=mylogstorage"))] + pub connection_string: Option, + + /// The Azure Blob Storage Account name. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "mylogstorage"))] + pub(super) account_name: Option, + + /// The Azure Blob Storage endpoint. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "https://mylogstorage.blob.core.windows.net/"))] + pub(super) blob_endpoint: Option, /// The Azure Blob Storage Account container name. #[configurable(metadata(docs::examples = "my-logs"))] @@ -138,6 +163,10 @@ pub struct AzureBlobSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub tls: Option, } pub fn default_blob_prefix() -> Template { @@ -147,7 +176,10 @@ pub fn default_blob_prefix() -> Template { impl GenerateConfig for AzureBlobSinkConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - connection_string: String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into(), + auth: None, + connection_string: Some(String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into()), + account_name: None, + blob_endpoint: None, container_name: String::from("logs"), blob_prefix: default_blob_prefix(), blob_time_format: Some(String::from("%s")), @@ -157,6 +189,7 @@ impl GenerateConfig for AzureBlobSinkConfig { batch: BatchConfig::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), + tls: None, }) .unwrap() } @@ -166,11 +199,46 @@ impl GenerateConfig for AzureBlobSinkConfig { #[typetag::serde(name = "azure_blob")] impl SinkConfig for AzureBlobSinkConfig { async fn build(&self, cx: SinkContext) -> Result<(VectorSink, Healthcheck)> { + let connection_string: String = match ( + &self.connection_string, + &self.account_name, + &self.blob_endpoint, + ) { + (Some(connstr), None, None) => connstr.inner().into(), + (None, Some(account_name), None) => { + format!("AccountName={}", account_name) + } + (None, None, Some(blob_endpoint)) => { + // BlobEndpoint must always end in a trailing slash + let blob_endpoint = if blob_endpoint.ends_with('/') { + blob_endpoint.clone() + } else { + format!("{}/", blob_endpoint) + }; + format!("BlobEndpoint={}", blob_endpoint) + } + (None, None, None) => { + return Err("One of `connection_string`, `account_name`, or `blob_endpoint` must be provided".into()); + } + (Some(_), Some(_), _) => { + return Err("Cannot provide both `connection_string` and `account_name`".into()); + } + (Some(_), _, Some(_)) => { + return Err("Cannot provide both `connection_string` and `blob_endpoint`".into()); + } + (_, Some(_), Some(_)) => { + return Err("Cannot provide both `account_name` and `blob_endpoint`".into()); + } + }; + let client = azure_common::config::build_client( - self.connection_string.clone().into(), + self.auth.clone(), + connection_string.clone(), self.container_name.clone(), cx.proxy(), - )?; + self.tls.clone(), + ) + .await?; let healthcheck = azure_common::config::build_healthcheck( self.container_name.clone(), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 63943312fe66d..dbe20a884ff86 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -1,6 +1,8 @@ use std::io::{BufRead, BufReader}; +use std::sync::Arc; use azure_core::http::StatusCode; +use azure_storage_blob::BlobContainerClient; use bytes::{Buf, BytesMut}; use flate2::read::GzDecoder; @@ -24,17 +26,24 @@ use crate::{ components::{SINK_TAGS, assert_sink_compliance}, random_events_with_stream, random_lines, random_lines_with_stream, random_string, }, + tls, }; #[tokio::test] async fn azure_blob_healthcheck_passed() { let config = AzureBlobSinkConfig::new_emulator().await; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; + + azure_common::config::build_healthcheck(config.container_name, client) + .expect("Failed to build healthcheck") + .await + .expect("Failed to pass healthcheck"); +} + +#[tokio::test] +async fn azure_blob_healthcheck_passed_with_oauth() { + let config = AzureBlobSinkConfig::new_emulator_with_oauth().await; + let client = config.build_test_client().await; azure_common::config::build_healthcheck(config.container_name, client) .expect("Failed to build healthcheck") @@ -49,12 +58,7 @@ async fn azure_blob_healthcheck_unknown_container() { container_name: String::from("other-container-name"), ..config }; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; assert_eq!( azure_common::config::build_healthcheck(config.container_name, client) @@ -66,10 +70,8 @@ async fn azure_blob_healthcheck_unknown_container() { ); } -#[tokio::test] -async fn azure_blob_insert_lines_into_blob() { +async fn assert_insert_lines_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("lines/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), ..config @@ -88,9 +90,17 @@ async fn azure_blob_insert_lines_into_blob() { } #[tokio::test] -async fn azure_blob_insert_json_into_blob() { +async fn azure_blob_insert_lines_into_blob() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_lines_into_blob_with_oauth() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + +async fn assert_insert_json_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("json/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), encoding: ( @@ -117,6 +127,16 @@ async fn azure_blob_insert_json_into_blob() { assert_eq!(expected, blob_lines); } +#[tokio::test] +async fn azure_blob_insert_json_into_blob() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_json_into_blob_with_oauth() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + #[ignore] #[tokio::test] // This test fails to get the posted blob with "header not found content-length". @@ -177,14 +197,12 @@ async fn azure_blob_insert_json_into_blob_gzip() { assert_eq!(expected, blob_lines); } -#[tokio::test] -async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { +async fn assert_rotate_files_after_the_buffer_size_is_reached(mut config: AzureBlobSinkConfig) { let groups = 3; let (lines, size, input) = random_lines_with_stream_with_group_key(100, 30, groups); let size_per_group = (size / groups) + 10; let blob_prefix = format!("lines-rotate/into/blob/{}", random_string(10)); - let mut config = AzureBlobSinkConfig::new_emulator().await; config.batch.max_bytes = Some(size_per_group); let config = AzureBlobSinkConfig { @@ -211,52 +229,104 @@ async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { } } +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { + assert_rotate_files_after_the_buffer_size_is_reached(AzureBlobSinkConfig::new_emulator().await) + .await; +} + +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached_with_oauth() { + assert_rotate_files_after_the_buffer_size_is_reached( + AzureBlobSinkConfig::new_emulator_with_oauth().await, + ) + .await; +} + impl AzureBlobSinkConfig { pub async fn new_emulator() -> AzureBlobSinkConfig { - let address = std::env::var("AZURE_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { - connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into(), - container_name: "logs".to_string(), - blob_prefix: Default::default(), - blob_time_format: None, - blob_append_uuid: None, - encoding: (None::, TextSerializerConfig::default()).into(), - compression: Compression::None, - batch: Default::default(), - request: TowerRequestConfig::default(), - acknowledgements: Default::default(), - }; + auth: None, + connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: None, + }; config.ensure_container().await; config } - fn to_sink(&self) -> VectorSink { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), + pub async fn new_emulator_with_oauth() -> AzureBlobSinkConfig { + let address = std::env::var("AZURITE_OAUTH_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let config = AzureBlobSinkConfig { + auth: Some(azure_common::config::AzureAuthentication::MockCredential), + connection_string: Some(format!("DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: Some(azure_common::config::AzureBlobTlsConfig { + ca_file: Some(tls::TEST_PEM_CA_PATH.into()), + }), + }; + + config.ensure_container().await; + + config + } + + async fn build_test_client(&self) -> Arc { + azure_common::config::build_client( + self.auth.clone(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + self.tls.clone(), ) - .expect("Failed to create client"); + .await + .expect("Failed to create client") + } + async fn to_sink(&self) -> VectorSink { + let client = self.build_test_client().await; self.build_processor(client).expect("Failed to create sink") } async fn run_assert(&self, input: impl Stream + Send) { // `to_sink` needs to be inside the assertion check - assert_sink_compliance(&SINK_TAGS, async move { self.to_sink().run(input).await }) - .await - .expect("Running sink failed"); + assert_sink_compliance( + &SINK_TAGS, + async move { self.to_sink().await.run(input).await }, + ) + .await + .expect("Running sink failed"); } pub async fn list_blobs(&self, prefix: String) -> Vec { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; // Iterate pager results and collect blob names. Filter by prefix server-side. let mut pager = client @@ -276,12 +346,7 @@ impl AzureBlobSinkConfig { } pub async fn get_blob(&self, blob: String) -> (Option, Option, Vec) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; let blob_client = client.blob_client(&blob); @@ -337,12 +402,7 @@ impl AzureBlobSinkConfig { } async fn ensure_container(&self) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; let result = client.create_container(None).await; let response = match result { diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index b7f1a9689229f..9b5a11e202dd5 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -14,6 +14,8 @@ use super::{config::AzureBlobSinkConfig, request_builder::AzureBlobRequestOption use crate::{ codecs::{Encoder, EncodingConfigWithFraming}, event::{Event, LogEvent}, + sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}, + sinks::prelude::*, sinks::util::{ Compression, request_builder::{EncodeResult, RequestBuilder}, @@ -22,7 +24,10 @@ use crate::{ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { AzureBlobSinkConfig { + auth: Default::default(), connection_string: Default::default(), + account_name: Default::default(), + blob_endpoint: Default::default(), container_name: Default::default(), blob_prefix: Default::default(), blob_time_format: Default::default(), @@ -32,6 +37,7 @@ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { batch: Default::default(), request: Default::default(), acknowledgements: Default::default(), + tls: Default::default(), } } @@ -234,3 +240,223 @@ fn azure_blob_build_request_with_uuid() { assert_eq!(request.content_encoding, None); assert_eq!(request.content_type, "text/plain"); } + +#[tokio::test] +async fn azure_blob_build_config_with_null_auth() { + let config: Result = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + "#, + ); + + match config { + Ok(_) => panic!("Config parsing should have failed due to invalid auth config"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("data did not match any variant of untagged enum"), + "Config parsing did not complain about invalid auth config: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + }) => { + assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); + assert_eq!(azure_client_id, "mock-client-id"); + let secret: String = azure_client_secret.inner().into(); + assert_eq!(secret, "mock-client-secret"); + } + _ => panic!("Expected Specific(ClientSecretCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_certificate_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + certificate_path = "tests/data/ClientCertificateAuth.pfx" + certificate_password = "MockPassword123" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientCertificateCredential { + .. + }) => { + // Expected variant + } + _ => panic!("Expected Specific(ClientCertificateCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!( + "Config build should have errored due to conflicting connection_string and account_name" + ), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`connection_string` and `account_name`"), + "Config build did not complain about conflicting connection_string and account_name: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage;AccountKey=mockkey" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => { + panic!("Config build should have errored due to conflicting Shared Key and Client ID") + } + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str + .contains("Cannot use both Shared Key and another Azure Authentication method"), + "Config build did not complain about conflicting Shared Key and Client ID: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_custom_ca_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [tls] + ca_file = "tests/data/ca/certs/ca.cert.pem" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 139608d9847ab..6f7c241ba084e 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,19 +1,36 @@ +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; use std::sync::Arc; +use base64::prelude::*; + use azure_core::error::Error as AzureCoreError; use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; use crate::sinks::azure_common::shared_key_policy::SharedKeyAuthorizationPolicy; -use azure_core::http::Url; +use azure_core::http::{ClientMethodOptions, StatusCode, Url}; + +use azure_core::credentials::{TokenCredential, TokenRequestOptions}; +use azure_core::{Error, error::ErrorKind}; + +use azure_identity::{ + AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientCertificateCredential, + ClientCertificateCredentialOptions, ClientSecretCredential, ManagedIdentityCredential, + ManagedIdentityCredentialOptions, UserAssignedId, WorkloadIdentityCredential, + WorkloadIdentityCredentialOptions, +}; + use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; -use azure_core::http::StatusCode; use bytes::Bytes; use futures::FutureExt; use snafu::Snafu; use vector_lib::{ + configurable::configurable_component, json_size::JsonSize, request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, + sensitive_string::SensitiveString, stream::DriverResponse, }; @@ -22,6 +39,374 @@ use crate::{ sinks::{Healthcheck, util::retries::RetryLogic}, }; +/// TLS configuration. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] +pub struct AzureBlobTlsConfig { + /// Absolute path to an additional CA certificate file. + /// + /// The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + #[serde(alias = "ca_path")] + #[configurable(metadata(docs::examples = "/path/to/certificate_authority.crt"))] + #[configurable(metadata(docs::human_name = "CA File Path"))] + pub ca_file: Option, +} + +/// Azure service principal authentication. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, untagged)] +pub enum AzureAuthentication { + #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] + Specific(SpecificAzureCredential), + + /// Mock credential for testing — returns a static fake token + #[cfg(test)] + #[serde(skip)] + MockCredential, +} + +impl Default for AzureAuthentication { + // This should never be actually used. + // This is only needed when using Default::default() (such as unit tests), + // as serde requires `azure_credential_kind` to be specified. + fn default() -> Self { + Self::Specific(SpecificAzureCredential::ManagedIdentity { + user_assigned_managed_identity_id: None, + user_assigned_managed_identity_id_type: None, + }) + } +} + +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +#[derive(Default)] +/// User Assigned Managed Identity Types. +pub enum UserAssignedManagedIdentityIdType { + #[default] + /// Client ID + ClientId, + /// Object ID + ObjectId, + /// Resource ID + ResourceId, +} + +/// Specific Azure credential types. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde( + tag = "azure_credential_kind", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum SpecificAzureCredential { + /// Use Azure CLI credentials + #[cfg(not(target_arch = "wasm32"))] + AzureCli {}, + + /// Use certificate credentials + ClientCertificateCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// PKCS12 certificate with RSA private key. + #[configurable(metadata(docs::examples = "path/to/certificate.pfx"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"))] + certificate_path: PathBuf, + + /// The password for the client certificate, if applicable. + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PASSWORD}"))] + certificate_password: Option, + }, + + /// Use client ID/secret credentials + ClientSecretCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// The [Azure Client Secret][azure_client_secret]. + /// + /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_SECRET:?err}"))] + azure_client_secret: SensitiveString, + }, + + /// Use Managed Identity credentials + ManagedIdentity { + /// The User Assigned Managed Identity to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + /// or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + }, + + /// Use Managed Identity with Client Assertion credentials + ManagedIdentityClientAssertion { + /// The User Assigned Managed Identity to use for the managed identity. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata( + docs::examples = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-vector/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-vector-uami" + ))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + + /// The target Tenant ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_tenant_id: String, + + /// The target Client ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_client_id: String, + }, + + /// Use Workload Identity credentials + WorkloadIdentity { + /// The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID}"))] + tenant_id: Option, + + /// The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID}"))] + client_id: Option, + + /// Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`. + #[configurable(metadata( + docs::examples = "/var/run/secrets/azure/tokens/azure-identity-token" + ))] + #[configurable(metadata(docs::examples = "${AZURE_FEDERATED_TOKEN_FILE}"))] + token_file_path: Option, + }, +} + +#[derive(Debug)] +struct ManagedIdentityClientAssertion { + credential: Arc, + scope: String, +} + +#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] +impl ClientAssertion for ManagedIdentityClientAssertion { + async fn secret(&self, options: Option>) -> azure_core::Result { + Ok(self + .credential + .get_token( + &[&self.scope], + Some(TokenRequestOptions { + method_options: options.unwrap_or_default(), + }), + ) + .await? + .token + .secret() + .to_string()) + } +} + +impl AzureAuthentication { + /// Returns the provider for the credentials based on the authentication mechanism chosen. + pub async fn credential(&self) -> azure_core::Result> { + match self { + Self::Specific(specific) => specific.credential().await, + + #[cfg(test)] + Self::MockCredential => Ok(Arc::new(MockTokenCredential) as Arc), + } + } +} + +impl SpecificAzureCredential { + /// Returns the provider for the credentials based on the specific credential type. + pub async fn credential(&self) -> azure_core::Result> { + let credential: Arc = match self { + #[cfg(not(target_arch = "wasm32"))] + Self::AzureCli {} => AzureCliCredential::new(None)?, + + // requires azure_identity feature 'client_certificate' + Self::ClientCertificateCredential { + azure_tenant_id, + azure_client_id, + certificate_path, + certificate_password, + } => { + let certificate_bytes: Vec = std::fs::read(certificate_path).map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!( + "Failed to read certificate file {}: {e}", + certificate_path.display() + ), + ) + })?; + + // Note: in azure_identity 0.33.0+, this changes to SecretBytes, and the base64 encoding is no longer needed + let certificate_base64: azure_core::credentials::Secret = + BASE64_STANDARD.encode(&certificate_bytes).into(); + + let mut options: ClientCertificateCredentialOptions = + ClientCertificateCredentialOptions::default(); + if let Some(password) = certificate_password { + options.password = Some(password.inner().to_string().into()); + } + + ClientCertificateCredential::new( + azure_tenant_id.clone(), + azure_client_id.clone(), + certificate_base64, + Some(options), + )? + } + + Self::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + } => { + if azure_tenant_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_secret.inner().is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + + let secret: String = azure_client_secret.inner().into(); + ClientSecretCredential::new( + &azure_tenant_id.clone(), + azure_client_id.clone(), + secret.into(), + None, + )? + } + + Self::ManagedIdentity { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + ManagedIdentityCredential::new(Some(options))? + } + + Self::ManagedIdentityClientAssertion { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + client_assertion_tenant_id, + client_assertion_client_id, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + let msi: Arc = ManagedIdentityCredential::new(Some(options))?; + let assertion = ManagedIdentityClientAssertion { + credential: msi, + // Future: make this configurable for sovereign clouds? (no way to test...) + scope: "api://AzureADTokenExchange/.default".to_string(), + }; + + ClientAssertionCredential::new( + client_assertion_tenant_id.clone(), + client_assertion_client_id.clone(), + assertion, + None, + )? + } + + Self::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + } => { + let options = WorkloadIdentityCredentialOptions { + tenant_id: tenant_id.clone(), + client_id: client_id.clone(), + token_file_path: token_file_path.clone(), + ..Default::default() + }; + + WorkloadIdentityCredential::new(Some(options))? + } + }; + Ok(credential) + } +} + #[derive(Debug, Clone)] pub struct AzureBlobRequest { pub blob_data: Bytes, @@ -126,10 +511,12 @@ pub fn build_healthcheck( Ok(healthcheck.boxed()) } -pub fn build_client( +pub async fn build_client( + auth: Option, connection_string: String, container_name: String, proxy: &crate::config::ProxyConfig, + tls: Option, ) -> crate::Result> { // Parse connection string without legacy SDK let parsed = ParsedConnectionString::parse(&connection_string) @@ -140,16 +527,26 @@ pub fn build_client( .map_err(|e| format!("Failed to build container URL: {e}"))?; let url = Url::parse(&container_url).map_err(|e| format!("Invalid container URL: {e}"))?; + let mut credential: Option> = None; + // Prepare options; attach Shared Key policy if needed let mut options = BlobContainerClientOptions::default(); - match parsed.auth() { - Auth::Sas { .. } | Auth::None => { - // No extra policy; SAS is in the URL already (or anonymous) + match (parsed.auth(), &auth) { + (Auth::None, None) => { + warn!("No authentication method provided, requests will be anonymous"); + } + (Auth::Sas { .. }, None) => { + info!("Using SAS token authentication"); } - Auth::SharedKey { - account_name, - account_key, - } => { + ( + Auth::SharedKey { + account_name, + account_key, + }, + None, + ) => { + info!("Using Shared Key authentication"); + let policy = SharedKeyAuthorizationPolicy::new( account_name, account_key, @@ -162,6 +559,36 @@ pub fn build_client( .per_call_policies .push(Arc::new(policy)); } + (Auth::None, Some(AzureAuthentication::Specific(..))) => { + info!("Using Azure Authentication method"); + let credential_result: Arc = + auth.unwrap().credential().await.unwrap(); + credential = Some(credential_result); + } + (Auth::Sas { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both SAS token and another Azure Authentication method at the same time", + ))); + } + (Auth::SharedKey { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both Shared Key and another Azure Authentication method at the same time", + ))); + } + #[cfg(test)] + (Auth::None, Some(AzureAuthentication::MockCredential)) => { + warn!("Using mock token credential authentication"); + credential = Some(auth.unwrap().credential().await.unwrap()); + } + #[cfg(test)] + (_, Some(AzureAuthentication::MockCredential)) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both connection string auth and mock credential at the same time", + ))); + } } // Use reqwest v0.12 since Azure SDK only implements HttpClient for reqwest::Client v0.12 @@ -191,12 +618,90 @@ pub fn build_client( reqwest_builder = reqwest_builder.proxy(p); } } + + if let Some(AzureBlobTlsConfig { ca_file }) = &tls + && let Some(ca_file) = ca_file + { + let mut buf = Vec::new(); + File::open(ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_12::Certificate::from_pem(&buf)?; + + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); + } + options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( reqwest_builder .build() .map_err(|e| format!("Failed to build reqwest client: {e}"))?, ))); - let client = - BlobContainerClient::from_url(url, None, Some(options)).map_err(|e| format!("{e}"))?; + let client = BlobContainerClient::from_url(url, credential, Some(options)) + .map_err(|e| format!("{e}"))?; Ok(Arc::new(client)) } + +#[cfg(test)] +#[derive(Debug)] +struct MockTokenCredential; + +#[cfg(test)] +#[async_trait::async_trait] +impl TokenCredential for MockTokenCredential { + async fn get_token( + &self, + scopes: &[&str], + _options: Option>, + ) -> azure_core::Result { + let Some(scope) = scopes.first() else { + return Err(Error::with_message( + ErrorKind::Credential, + "no scopes were provided", + )); + }; + + // serde_json sometimes does and sometimes doesn't preserve order, be careful to sort + // the claims in alphabetical order to ensure a consistent base64 encoding for testing + let jwt = serde_json::json!({ + "aud": scope.strip_suffix("/.default").unwrap_or(*scope), + "exp": 2147483647, + "iat": 0, + "iss": "https://sts.windows.net/", + "nbf": 0, + }); + + // JWTs do not include standard base64 padding. + // this seemed cleaner than importing a new crates just for this function + let jwt_base64 = format!( + "e30.{}.", + BASE64_STANDARD + .encode(serde_json::to_string(&jwt).unwrap()) + .trim_end_matches("=") + ) + .to_string(); + + warn!( + "Using mock token credential, JWT: {}, base64: {}", + serde_json::to_string(&jwt).unwrap(), + jwt_base64 + ); + + Ok(azure_core::credentials::AccessToken::new( + jwt_base64, + azure_core::time::OffsetDateTime::now_utc() + std::time::Duration::from_secs(3600), + )) + } +} + +#[cfg(test)] +#[tokio::test] +async fn azure_mock_token_credential_test() { + let credential = MockTokenCredential; + let access_token = credential + .get_token(&["https://example.com/.default"], None) + .await + .expect("valid credential should return a token"); + assert_eq!( + access_token.token.secret(), + "e30.eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiZXhwIjoyMTQ3NDgzNjQ3LCJpYXQiOjAsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LyIsIm5iZiI6MH0." + ); +} diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 4e876a18a26bd..3b105d549d0c1 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -1,20 +1,14 @@ use std::sync::Arc; -use azure_core::credentials::{TokenCredential, TokenRequestOptions}; -use azure_core::http::ClientMethodOptions; -use azure_core::{Error, error::ErrorKind}; - -use azure_identity::{ - AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientSecretCredential, - ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, - WorkloadIdentityCredential, -}; -use vector_lib::{configurable::configurable_component, schema, sensitive_string::SensitiveString}; +use azure_core::credentials::TokenCredential; + +use vector_lib::{configurable::configurable_component, schema}; use vrl::value::Kind; use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ + azure_common::config::AzureAuthentication, prelude::*, util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, }, @@ -65,7 +59,6 @@ pub struct AzureLogsIngestionConfig { pub stream_name: String, #[configurable(derived)] - #[serde(default)] pub auth: AzureAuthentication, /// [Token scope][token_scope] for dedicated Azure regions. @@ -129,191 +122,6 @@ impl Default for AzureLogsIngestionConfig { } } -/// Configuration of the authentication strategy for interacting with Azure services. -#[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] -#[serde(deny_unknown_fields, untagged)] -pub enum AzureAuthentication { - /// Use client credentials - #[derivative(Default)] - ClientSecretCredential { - /// The [Azure Tenant ID][azure_tenant_id]. - /// - /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_tenant_id: String, - - /// The [Azure Client ID][azure_client_id]. - /// - /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_client_id: String, - - /// The [Azure Client Secret][azure_client_secret]. - /// - /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] - azure_client_secret: SensitiveString, - }, - - /// Use credentials from environment variables - #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] - Specific(SpecificAzureCredential), -} - -/// Specific Azure credential types. -#[configurable_component] -#[derive(Clone, Debug, Eq, PartialEq)] -#[serde( - tag = "azure_credential_kind", - rename_all = "snake_case", - deny_unknown_fields -)] -pub enum SpecificAzureCredential { - /// Use Azure CLI credentials - #[cfg(not(target_arch = "wasm32"))] - AzureCli {}, - - /// Use Managed Identity credentials - ManagedIdentity { - /// The User Assigned Managed Identity (Client ID) to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - }, - - /// Use Managed Identity with Client Assertion credentials - ManagedIdentityClientAssertion { - /// The User Assigned Managed Identity (Client ID) to use for the managed identity. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - - /// The target Tenant ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_tenant_id: String, - - /// The target Client ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_client_id: String, - }, - - /// Use Workload Identity credentials - WorkloadIdentity {}, -} - -#[derive(Debug)] -struct ManagedIdentityClientAssertion { - credential: Arc, - scope: String, -} - -#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] -impl ClientAssertion for ManagedIdentityClientAssertion { - async fn secret(&self, options: Option>) -> azure_core::Result { - Ok(self - .credential - .get_token( - &[&self.scope], - Some(TokenRequestOptions { - method_options: options.unwrap_or_default(), - }), - ) - .await? - .token - .secret() - .to_string()) - } -} - -impl AzureAuthentication { - /// Returns the provider for the credentials based on the authentication mechanism chosen. - pub async fn credential(&self) -> azure_core::Result> { - match self { - Self::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - if azure_tenant_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_secret.inner().is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - let secret: String = azure_client_secret.inner().into(); - let credential: Arc = ClientSecretCredential::new( - &azure_tenant_id.clone(), - azure_client_id.clone(), - secret.into(), - None, - )?; - Ok(credential) - } - - Self::Specific(specific) => specific.credential().await, - } - } -} - -impl SpecificAzureCredential { - /// Returns the provider for the credentials based on the specific credential type. - pub async fn credential(&self) -> azure_core::Result> { - let credential: Arc = match self { - #[cfg(not(target_arch = "wasm32"))] - Self::AzureCli {} => AzureCliCredential::new(None)?, - - Self::ManagedIdentity { - user_assigned_managed_identity_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - ManagedIdentityCredential::new(Some(options))? - } - - Self::ManagedIdentityClientAssertion { - user_assigned_managed_identity_id, - client_assertion_tenant_id, - client_assertion_client_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - let msi: Arc = ManagedIdentityCredential::new(Some(options))?; - let assertion = ManagedIdentityClientAssertion { - credential: msi, - // Future: make this configurable for sovereign clouds? (no way to test...) - scope: "api://AzureADTokenExchange/.default".to_string(), - }; - - ClientAssertionCredential::new( - client_assertion_tenant_id.clone(), - client_assertion_client_id.clone(), - assertion, - None, - )? - } - - Self::WorkloadIdentity {} => WorkloadIdentityCredential::new(None)?, - }; - Ok(credential) - } -} - impl AzureLogsIngestionConfig { #[allow(clippy::too_many_arguments)] pub(super) async fn build_inner( diff --git a/src/sinks/azure_logs_ingestion/tests.rs b/src/sinks/azure_logs_ingestion/tests.rs index 26eda91338836..46095071bd28c 100644 --- a/src/sinks/azure_logs_ingestion/tests.rs +++ b/src/sinks/azure_logs_ingestion/tests.rs @@ -8,6 +8,8 @@ use vector_lib::config::log_schema; use azure_core::credentials::{AccessToken, TokenCredential}; use azure_core::time::OffsetDateTime; +use crate::sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}; + use super::config::AzureLogsIngestionConfig; use crate::{ @@ -26,50 +28,22 @@ fn generate_config() { #[tokio::test] async fn basic_config_error_with_no_auth() { - let config: AzureLogsIngestionConfig = toml::from_str::( - r#" + let config: Result = + toml::from_str::( + r#" endpoint = "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" dcr_immutable_id = "dcr-00000000000000000000000000000000" stream_name = "Custom-UnitTest" "#, - ) - .expect("Config parsing failed"); - - assert_eq!( - config.endpoint, - "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" - ); - assert_eq!( - config.dcr_immutable_id, - "dcr-00000000000000000000000000000000" - ); - assert_eq!(config.stream_name, "Custom-UnitTest"); - assert_eq!(config.token_scope, "https://monitor.azure.com/.default"); - assert_eq!(config.timestamp_field, "TimeGenerated"); - - match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - assert_eq!(azure_tenant_id, ""); - assert_eq!(azure_client_id, ""); - let secret: String = azure_client_secret.inner().into(); - assert_eq!(secret, ""); - } - _ => panic!("Expected ClientSecretCredential variant"), - } + ); - let cx = SinkContext::default(); - let sink = config.build(cx).await; - match sink { - Ok(_) => panic!("Config build should have errored due to missing auth info"), + match config { + Ok(_) => panic!("Config parsing should have failed due to missing auth config"), Err(e) => { let err_str = e.to_string(); assert!( - err_str.contains("`auth.azure_tenant_id` is blank"), - "Config build did not complain about azure_tenant_id being blank: {}", + err_str.contains("missing field `auth`"), + "Config parsing did not complain about missing auth field: {}", err_str ); } @@ -85,6 +59,7 @@ fn basic_config_with_client_credentials() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -105,17 +80,17 @@ fn basic_config_with_client_credentials() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, - } => { + }) => { assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); assert_eq!(azure_client_id, "mock-client-id"); let secret: String = azure_client_secret.inner().into(); assert_eq!(secret, "mock-client-secret"); } - _ => panic!("Expected ClientSecretCredential variant"), + _ => panic!("Expected Specific(ClientSecretCredential) variant"), } } @@ -146,11 +121,7 @@ fn basic_config_with_managed_identity() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::Specific( - crate::sinks::azure_logs_ingestion::config::SpecificAzureCredential::ManagedIdentity { - .. - }, - ) => { + AzureAuthentication::Specific(SpecificAzureCredential::ManagedIdentity { .. }) => { // Expected variant } _ => panic!("Expected Specific(ManagedIdentity) variant"), @@ -182,6 +153,7 @@ async fn correct_request() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -292,6 +264,7 @@ async fn mock_healthcheck_with_400_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -361,6 +334,7 @@ async fn mock_healthcheck_with_403_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" diff --git a/tests/data/ClientCertificateAuth.pfx b/tests/data/ClientCertificateAuth.pfx new file mode 100644 index 0000000000000..e289b55375a4a Binary files /dev/null and b/tests/data/ClientCertificateAuth.pfx differ diff --git a/tests/data/Makefile b/tests/data/Makefile index f3550bf8bf100..6d4a17ab0d54c 100644 --- a/tests/data/Makefile +++ b/tests/data/Makefile @@ -61,6 +61,27 @@ ca/intermediate_server/certs/localhost.cert.pem: ca/intermediate_server/csr/loca ca/intermediate_server/certs/localhost-chain.cert.pem: ca/intermediate_server/certs/ca-chain.cert.pem ca/intermediate_server/certs/localhost.cert.pem cat ca/intermediate_server/certs/localhost.cert.pem ca/intermediate_server/certs/ca-chain.cert.pem > ca/intermediate_server/certs/localhost-chain.cert.pem +ca/intermediate_server/private/azurite.key.pem: + openssl genrsa -out ca/intermediate_server/private/azurite.key.pem 2048 + +ca/intermediate_server/csr/azurite.csr.pem: ca/intermediate_server/private/azurite.key.pem + openssl req -config ca/intermediate_server/openssl.cnf \ + -key ca/intermediate_server/private/azurite.key.pem \ + -subj '/CN=azurite/OU=Vector/O=Datadog/ST=New York/L=New York/C=US' \ + -addext 'subjectAltName=DNS:azurite' \ + -addext 'basicConstraints=critical,CA:FALSE' \ + -addext 'extendedKeyUsage=serverAuth' \ + -new -sha256 -out ca/intermediate_server/csr/azurite.csr.pem + +ca/intermediate_server/certs/azurite.cert.pem: ca/intermediate_server/csr/azurite.csr.pem + openssl ca -batch -config ca/intermediate_server/openssl.cnf \ + -extensions server_cert -days 3650 -notext -md sha256 \ + -in ca/intermediate_server/csr/azurite.csr.pem \ + -out ca/intermediate_server/certs/azurite.cert.pem + +ca/intermediate_server/certs/azurite-chain.cert.pem: ca/intermediate_server/certs/ca-chain.cert.pem ca/intermediate_server/certs/azurite.cert.pem + cat ca/intermediate_server/certs/azurite.cert.pem ca/intermediate_server/certs/ca-chain.cert.pem > ca/intermediate_server/certs/azurite-chain.cert.pem + ca/intermediate_server/private/elasticsearch-secure.key.pem: openssl genrsa -out ca/intermediate_server/private/elasticsearch-secure.key.pem 2048 diff --git a/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem new file mode 100644 index 0000000000000..f646a0fe3c20a --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem @@ -0,0 +1,98 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFtzCCA5+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwajESMBAGA1UEAwwJVmVj +dG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNV +BAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMwHhcN +MjIwNjA3MjIyNzUzWhcNMzIwNjA0MjIyNzUzWjBrMQswCQYDVQQGEwJVUzERMA8G +A1UECAwITmV3IFlvcmsxEDAOBgNVBAoMB0RhdGFkb2cxDzANBgNVBAsMBlZlY3Rv +cjEmMCQGA1UEAwwdVmVjdG9yIEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCy/mB0/ZwfgKrSZPQIFaGPtRA9xL2N +o2SsHndZ8k2XOCV225Hb2fzNH+o2WGNSjwmGjLP/uXb47KH0cHCAyFGzSjp++8/O +zoZaFiO0P5El02hQxmoabO3Cqu/N62EFsLfpSM828JM6YOn9p+WXUDn1+YPNoOOE +H142p4/RjFnXNHkzR3geXU4Pfi3KXDrMi8vK42lDqXPLPs6rhreBAfQ2dsYyqhz6 +tg6FzZuXxxzEYyYtNgGh+zTji99WCBMLbCmRcDurRjdTDO7m4O3PrwbGUy0xdLeb +HJiNGvUDCPH4bfwLiNqwVIZY38RBCAqbCnrqRhDaZIfAUev4mq3Kqh6KUeO/U7Vx +/5J5rL5ApREKOfWPATHMprBuEU2rs3N+MPBA04HoiFlu311urCxVEA1qsZCTkoCg +GHuDIVSU4E4hT4co95/J0to4zWgPlfPg1+cXyU8lAIMe7JdCGkG9cDe7Umw/GSbt +ZdoCMQZ6WyyiW2Hw+7sFD3V3VzYa5YA/rjKZRduPmGWKrs+mAa5J5pM2M22rrjbd +EpfTHWLS9s6cPN3/jxpCxn6Hv/KhIYRAcIterugag1+clvS1ajVjxBRavOxPBsf+ +hYdh7S5NTZnT98gjkc3yOuGQm7BPtXau+IYZRlWcB0dJ4/E2P69hmWQezSo9VVWh +5/K1RkbPvqTGZQIDAQABo2YwZDAdBgNVHQ4EFgQUPD06L8zVggN9mcRY8eHbNu+t +DUEwHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wEgYDVR0TAQH/BAgw +BgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAGqaGBuL +2J6Na8RHx/GmSeuZFiVcWhmd/I9bVpeMSYHSZujA2nay6OGaUYs0Lq/G5OKgsuT9 +AIHnsn7VUE1zqoDfXac/K8sXlOig8if7rTb+06jgymaP1YSELg3R+pBsdkZnXVil +izh/9FvzoyV+QQlIhojqCIybVFgxa1XFHq4QCPhDfwkg+tp9RctfwNmWgsJ63H19 +RmxN+H2xIrySvObwXnB4j6D4wvgu468QXQMEuSsnLcIQFg6Zteqe8fixbqTiOTBf +Dk1k+EpB9VMEkIPvMdfa48vseXdBEe6Ma9zGuJC76q4q1ZapVLTvOUP5Y24khlgd +cj5tfP7o7yc6HqymfXAcD1lzP2JQhqaRxA4I18Nrd+aHi+G1EM2c3cicuD3n6Iw9 +9oqdCwmMfS25fv5cyA5B6hRusIZ9wRopTi7at+JHl0GIt/FelaTYI7kRmAqgakQe +oEKLpXcH8lRJW802DmXm7ka4eQzwxa7Ngyf8O+JOFtGO0+EshuLJovxiPl6IyLyG +NJ/dHq3ad+46YVManbHdyjHxgT5PSvJFkq0Yluvf44NIyP5QRTCAvfH76bu7hXgS +QoQj5t5ILn6meQRTR79r2iwpQTanPLTEdoZvmrE4TeUBev9BA5KpiPPA3i3ZF/oV +0EYorXCNri7M/jylGW7AuWvNUyaVR6xgxAn6 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFujCCA6KgAwIBAgIJAKhPL9BkNaFGMA0GCSqGSIb3DQEBCwUAMGoxEjAQBgNV +BAMMCVZlY3RvciBDQTEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9n +MREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxCzAJBgNVBAYT +AlVTMB4XDTIyMDYwNzIyMjc1MloXDTQyMDYwMjIyMjc1MlowajESMBAGA1UEAwwJ +VmVjdG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9c1T+NXTNmqiiV36NSEJt +7mo0cyv8Byk2ZGdC85vHBm45QDY5USoh0vgonzPpWgSMggPn1WbR0f1y+LBwXdlM ++ZyZh2RVVeUrSjJ88lLHVn4DfywpdDkwQaFj1VmOsj2I9rMMrgc5x5n1Hj7lwZ+t +uPVSAGmgKp4iFfzLph9r/rjP1TUAnVUComfTUVS+Gd7zoGPOc14cMJXG6g2P2aAU +P6dg5uQlTxRmagnlx7bwm3lRwv6LMtnAdnjwBDBxr933nucAnk21GgE92GejiO3Z +OwlzIdzBI23lPcWi5pq+vCTgAArNq24W1Ha+7Jn5QewNTGKFyyYAJetZAwCUR8QS +Ip++2GE2pNhaGqcV5u1Tbwl02eD6p2qRqjfgLxmb+aC6xfl0n9kiFGPZppjCqDEW +sw+gX66nf+qxZVRWpJon2kWcFvhTnLqoa3T3+9+KIeamz2lW6wxMnki/Co2EA1Wa +mmedaUUcRPCgMx9aCktRkMyH6bEY8/vfJ07juxUsszOc46T00Scmn6Vkuo9Uc3Kf +2Q2N6Wo4jtyAiMO4gAwq5kzzpBAhNgRfLHOb83r2gAUj2Y4Vln/UUR/KR8ZbJi4i +r1BjX16Lz3yblJXXb1lp4uZynlbHNaAevXyGlRqHddM2ykKtAX/vgJcZRGSvms11 +uce/cqzrzx60AhpLRma5CwIDAQABo2MwYTAdBgNVHQ4EFgQURTWK6ARqnZkz8rkt +Uc5PrtasIh8wHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAEf5 +TR3hq/DtSAmsYotu1lAWz/OlTpG+7AdqSOHB878X4ETN3xaQ+KWvSwvf0K70ZDTV +tFOTh/r43cpzPifPKd1P+2ctnQEzrBtAacvyETLq1ABRK9VJOtfJ6Xk5KZXPhKdY +t353PQgBgW8YzQ2adq2B7FtgIlX7f1DIndjcMZBbolETR6xt9QwB/UnPI7Mwt01T ++bCBhr1fWAbZ4YAMlQ0xRam4qUOTjxgfmePrmSrv4HO7cXHMsRMLiXk+BLcx959/ +K/B6xzpzn6366Eqnqlo/uDiMpo5ud2I/Snz5PduB6oLztPMEf/8RmkG5tpHXYdWr +tM64WqNGO+ikluIrrtYvtyZS4DfsLAMfMYZcxX/Uw56gHo0i2c8I6+6JvGWdvOJ0 +FjrsKeIQoRlV77z025kI4V9jKi3XNMEsAIH+W7KNSut0X80yX7SugvQGoe0GDkXu +0fy8hMC3uTN2LEycYFRRfoIeKPLi6OZFK0PdS2E15d8PEU3n3W4eBCPgMtmiOKLY +d8QNBC8XLAuBoK9R8luCJpOJWUcFXjLpjcDab4V2hKTuAs+GQyDh/Xx4wF1yHX0r +zIkyN0EkOD/SvD8X4uFaM4mdsAh+ucn4ryUV7i5PgvDM9z4InHAMAee1ebBl0U+h ++NzMWF5c5OwxD5o6/Wh1HopmzJiVNT2v9u0kHT/f +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/certs/azurite.cert.pem b/tests/data/ca/intermediate_server/certs/azurite.cert.pem new file mode 100644 index 0000000000000..3e2f680ef37c0 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/csr/azurite.csr.pem b/tests/data/ca/intermediate_server/csr/azurite.csr.pem new file mode 100644 index 0000000000000..09cf2dd0bc3f7 --- /dev/null +++ b/tests/data/ca/intermediate_server/csr/azurite.csr.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC9TCCAd0CAQAwaDEQMA4GA1UEAwwHYXp1cml0ZTEPMA0GA1UECwwGVmVjdG9y +MRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwI +TmV3IFlvcmsxCzAJBgNVBAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVPbGW0LqD1WO42dSmUMQ2iRGsF +eHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bRRInmuyocDTvm4t7VU+ybnPD7 +MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3QZ4loS61BR5TxrCdnpE++gxh +7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ63PaXXvGpGiDwPqUqi0WhIYp +73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFnVr8lMVavcDcoTrHdJ8PXN4Ei +pgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABoEgwRgYJKoZIhvcNAQkOMTkwNzAS +BgNVHREECzAJggdhenVyaXRlMAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwDQYJKoZIhvcNAQELBQADggEBAIbILz+IbvX9126CRLqmtlp4s1q2L+A2 +jYz+qk1G1UhImumfEcLMBfj6X/ZQ5IRIrH8BZZtW9ungXUS/9eYKUCQP52GXPPZJ +WkS3sERjvktEP7orNrTLo7v36f9P0J9eMjN8XYhcqHmZB5OyoQ4S3ndqgE4+HV8j +OyP9lZC1Urjrre/S7pyxxnkVftxXtVHXzB0MgoQa77U/ukqeaMYIw3qWRqGwwl44 +JnZm05mukTpYnDVmo+J2Ra1uNOJf9/SIEnawyL1ROYCINxgIcdodrpuxz+dnYT0K +zvsbMGQ1MTacJK61Pw0EBbxERB9ZhWnApRdeuhrF5oq9uGEzD/76wTg= +-----END CERTIFICATE REQUEST----- diff --git a/tests/data/ca/intermediate_server/index.txt b/tests/data/ca/intermediate_server/index.txt index 0402ad0946130..b2a0481603b78 100644 --- a/tests/data/ca/intermediate_server/index.txt +++ b/tests/data/ca/intermediate_server/index.txt @@ -6,3 +6,4 @@ V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar +V 360301205253Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite diff --git a/tests/data/ca/intermediate_server/index.txt.old b/tests/data/ca/intermediate_server/index.txt.old index ab4efe2c2f6bd..0402ad0946130 100644 --- a/tests/data/ca/intermediate_server/index.txt.old +++ b/tests/data/ca/intermediate_server/index.txt.old @@ -5,3 +5,4 @@ V 320613195026Z 1004 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=kafka V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq +V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar diff --git a/tests/data/ca/intermediate_server/newcerts/1009.pem b/tests/data/ca/intermediate_server/newcerts/1009.pem new file mode 100644 index 0000000000000..3e2f680ef37c0 --- /dev/null +++ b/tests/data/ca/intermediate_server/newcerts/1009.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/private/azurite.key.pem b/tests/data/ca/intermediate_server/private/azurite.key.pem new file mode 100644 index 0000000000000..545d8904a3d73 --- /dev/null +++ b/tests/data/ca/intermediate_server/private/azurite.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC+cDIKkK8OivJI +I1qy1Vb3Hf7fVNrhJU9sZbQuoPVY7jZ1KZQxDaJEawV4e8ziC5euFeWgFV/fkvU7 +JZGn9uW0eWV4uuvnxtFEiea7KhwNO+bi3tVT7Juc8PsyF02xsyajYcGfXZwRjs+z +PFU7BxnyPRc4e0j697dBniWhLrUFHlPGsJ2ekT76DGHsqFQbXJOIqYQS3nzu9Ei5 +UFY2ssVWFhA3qZz7Yknrc9pde8akaIPA+pSqLRaEhinvddXIipseK8jbtOK6/+f2 +Bwc8C3ZTBVu/UCqoUWdWvyUxVq9wNyhOsd0nw9c3gSKmC+ywkXXiAUWDsvRTgA6P +jG7fV76JAgMBAAECggEAEXeC73sjw0a1QC6V9A8jQdkrdlp3FO1yInQVma1Ds5tt +vUNKB1HDz1itkMZyHU2I5Pu3Kv6q43u8KGeiu3Am023LA87JMmIG8a1gT0xmdERJ +QgfEM0VhZHyp3YdLpf/TjGq78p0IYofhvwPKoXZeR9yYk6KjJ/mugkM6GlWJXuWn +sE2ZYL4Wl6d//1/9D6xWfNVU1zCH7ScbFFvQKkBxURLpnQlgkfhr8HMrCQyV4Lps +6n166zcQKJO4mwGrWyu75HU8vcBYv1+6WbJ1Rgke6lL3dzjIhzE7C61i3dI636Aq +l0wZ3NVeysyrn5Xzu5vfq0D9/pt+Of1PE0FmmlRogQKBgQDhaY8OyUafMYhGnike +7K1ho60tFHE5vgxMpoUTinLyARgW8bA9VRtQ++PW90Z1sHzXAHjb30YZcYEuKaCW +SvIWg49/L4bORX2jY4UPZvESemJtnIXZkuuiGoYaXjIwNudorT1siCD/GYM24Uko +ax0QbFsNRtq89QE/ibKdFLRfUQKBgQDYR7JfauKmsY6OQjB8RvFe50AnnPfiJSZ+ +rD7H7gXWlt3q9zpsU3xBK+fwI9QsvTlUNsIkFhJpu6znCzK26nLIJCqGcm0W+9Ah +rJR+0TXyafQn1d1dTrtn7XVIL6QQEuUY9jggsAXWBejdPCeaVxplEiJa1i9x3YOp +OQWmMTvNuQKBgCiGF6fq231nJD691Fqw5gK1sD54fFqLJh7pmOcIbt2/AJuvW6XL +FRwcDLvqvIoP7oGgnhm5LBsK4tRvu2UJmDgf8r5ExxFyQMIM9DDuqsxNoEBgcVfK +J/5+kjlPUeqFFFknO/G1D2mNJp/JJKPVjeYT9NKQOGbcDRtlH+1JeZvhAoGAZkOg +Z9WWTdNu4H0Th+/TeVhG0XQ7EUcXqJWxKb+2Kv0y+ULk8QuYmQg1pyqJzI28acFq +kr2M/0mqO6Tj2fGJTHEtWl0Ij/GJPCLqI/ywUWsf8yYAgXoUytNQvU0peiA1C1SA +vZP9bnFk5hbncub0qA2nCOR1kpV3B7DapvZonKECgYEAjzpfOhaULZgwnkYOkoFw +VS+tDOVNVfjDOjuV82B/0zCgCW8dv2EeTCobEkIbSl8GYMzukNZQR8r3ZAxbs1By +Ieliad2FD/s0MtsPiGcxJN4twbDQO/SvLFtjQ+jCysgOVON6KiqpGVRcEL+6U4ZD +NiqT36xmXpmkZ05bMxOXPLM= +-----END PRIVATE KEY----- diff --git a/tests/data/ca/intermediate_server/serial b/tests/data/ca/intermediate_server/serial index 6cb3869343bf8..4e75b247d2247 100644 --- a/tests/data/ca/intermediate_server/serial +++ b/tests/data/ca/intermediate_server/serial @@ -1 +1 @@ -1009 +100A diff --git a/tests/data/ca/intermediate_server/serial.old b/tests/data/ca/intermediate_server/serial.old index 617ba1c154075..6cb3869343bf8 100644 --- a/tests/data/ca/intermediate_server/serial.old +++ b/tests/data/ca/intermediate_server/serial.old @@ -1 +1 @@ -1008 +1009 diff --git a/tests/integration/azure/config/compose.yaml b/tests/integration/azure/config/compose.yaml index 933a72d235a24..3db399d5f78ca 100644 --- a/tests/integration/azure/config/compose.yaml +++ b/tests/integration/azure/config/compose.yaml @@ -7,6 +7,14 @@ services: volumes: - /var/run:/var/run + azurite: + image: mcr.microsoft.com/azure-storage/azurite:${CONFIG_VERSION} + command: azurite --blobHost 0.0.0.0 --blobPort 14430 --queuePort 14431 --tablePort 14432 --loose --oauth basic --cert /certs/azurite-chain.cert.pem --key /private/azurite.key.pem + volumes: + - /var/run:/var/run + - ../../../data/ca/intermediate_server/certs:/certs + - ../../../data/ca/intermediate_server/private/azurite.key.pem:/private/azurite.key.pem + networks: default: name: ${VECTOR_NETWORK} diff --git a/tests/integration/azure/config/test.yaml b/tests/integration/azure/config/test.yaml index 191bea7256cea..88f46966896ee 100644 --- a/tests/integration/azure/config/test.yaml +++ b/tests/integration/azure/config/test.yaml @@ -4,7 +4,8 @@ features: test_filter: ::azure_ env: - AZURE_ADDRESS: local-azure-blob + AZURITE_ADDRESS: local-azure-blob + AZURITE_OAUTH_ADDRESS: azurite HEARTBEAT_ADDRESS: 0.0.0.0:8080 LOGSTASH_ADDRESS: 0.0.0.0:8081 diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 5ea0c1dd20221..f1e544cdaa008 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -1,6 +1,16 @@ package metadata generated: components: sinks: azure_blob: configuration: { + account_name: { + description: """ + The Azure Blob Storage Account name. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["mylogstorage"] + } acknowledgements: { description: """ Controls how acknowledgements are handled for this sink. @@ -27,6 +37,123 @@ generated: components: sinks: azure_blob: configuration: { type: bool: {} } } + auth: { + description: "Azure service principal authentication." + required: false + type: object: options: { + azure_client_id: { + description: """ + The [Azure Client ID][azure_client_id]. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] + } + azure_client_secret: { + description: """ + The [Azure Client Secret][azure_client_secret]. + + [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] + } + azure_credential_kind: { + description: "The kind of Azure credential to use." + required: true + type: string: enum: { + azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" + managed_identity: "Use Managed Identity credentials" + managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" + workload_identity: "Use Workload Identity credentials" + } + } + azure_tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] + } + client_assertion_client_id: { + description: "The target Client ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_assertion_tenant_id: { + description: "The target Tenant ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } + user_assigned_managed_identity_id: { + description: "The User Assigned Managed Identity to use." + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } + } + } batch: { description: "Event batching behavior." required: false @@ -74,6 +201,16 @@ generated: components: sinks: azure_blob: configuration: { required: false type: bool: {} } + blob_endpoint: { + description: """ + The Azure Blob Storage endpoint. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["https://mylogstorage.blob.core.windows.net/"] + } blob_prefix: { description: """ A prefix to apply to all blob keys. @@ -165,8 +302,9 @@ generated: components: sinks: azure_blob: configuration: { | Allowed resource types | Container & Object | | Allowed permissions | Read & Create | """ - required: true - type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken"] + required: false + type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken", "AccountName=mylogstorage"] + warnings: ["Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep connection strings secure and not expose them in logs, error messages, or version control systems."] } container_name: { description: "The Azure Blob Storage Account container name." @@ -862,4 +1000,17 @@ generated: components: sinks: azure_blob: configuration: { } } } + tls: { + description: "TLS configuration." + required: false + type: object: options: ca_file: { + description: """ + Absolute path to an additional CA certificate file. + + The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + } } diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index ec03373e5354e..2aaf9b628e3c6 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -28,8 +28,8 @@ generated: components: sinks: azure_logs_ingestion: configuration: { } } auth: { - description: "Configuration of the authentication strategy for interacting with Azure services." - required: false + description: "Azure service principal authentication." + required: true type: object: options: { azure_client_id: { description: """ @@ -37,11 +37,9 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] } azure_client_secret: { description: """ @@ -49,17 +47,17 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00-00~000000-0000000~0000000000000000000"] - } + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] } azure_credential_kind: { description: "The kind of Azure credential to use." required: true type: string: enum: { azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" managed_identity: "Use Managed Identity credentials" managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" workload_identity: "Use Workload Identity credentials" @@ -71,11 +69,21 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] } client_assertion_client_id: { description: "The target Client ID to use." @@ -89,12 +97,51 @@ generated: components: sinks: azure_logs_ingestion: configuration: { required: true type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } user_assigned_managed_identity_id: { - description: "The User Assigned Managed Identity (Client ID) to use." + description: "The User Assigned Managed Identity to use." relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" required: false type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } } } batch: {