diff --git a/scripts/integration/amqp/compose.yaml b/scripts/integration/amqp/compose.yaml index aa11f17a43af9..9c268b7ede3ff 100644 --- a/scripts/integration/amqp/compose.yaml +++ b/scripts/integration/amqp/compose.yaml @@ -5,7 +5,3 @@ services: image: docker.io/rabbitmq:${CONFIG_VERSION} ports: - 5672:5672 - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/aws/compose.yaml b/scripts/integration/aws/compose.yaml index 41f72695c4a01..07ed1fde4311d 100644 --- a/scripts/integration/aws/compose.yaml +++ b/scripts/integration/aws/compose.yaml @@ -14,7 +14,3 @@ services: volumes: - $DOCKER_SOCKET:/var/run/docker.sock - $HOME/.aws/:/home/.aws/ - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/azure/compose.yaml b/scripts/integration/azure/compose.yaml index 9fdf8a76a1cd4..6c7b00e37d6e1 100644 --- a/scripts/integration/azure/compose.yaml +++ b/scripts/integration/azure/compose.yaml @@ -6,7 +6,3 @@ services: command: azurite --blobHost 0.0.0.0 --loose volumes: - /var/run:/var/run - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/chronicle/compose.yaml b/scripts/integration/chronicle/compose.yaml index 75ae40bdbb54d..09097e24a97de 100644 --- a/scripts/integration/chronicle/compose.yaml +++ b/scripts/integration/chronicle/compose.yaml @@ -10,7 +10,3 @@ services: command: - -p - /public.pem - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/clickhouse/compose.yaml b/scripts/integration/clickhouse/compose.yaml index 4ac410aa1e6ac..fe11611e9265d 100644 --- a/scripts/integration/clickhouse/compose.yaml +++ b/scripts/integration/clickhouse/compose.yaml @@ -3,7 +3,3 @@ version: '3' services: clickhouse: image: docker.io/yandex/clickhouse-server:${CONFIG_VERSION} - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/databend/compose.yaml b/scripts/integration/databend/compose.yaml index 15bad6fd9b9cc..65ed7a9c809d2 100644 --- a/scripts/integration/databend/compose.yaml +++ b/scripts/integration/databend/compose.yaml @@ -19,7 +19,3 @@ services: - minio healthcheck: test: "curl -f localhost:8080/v1/health || exit 1" - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/datadog-agent/compose.yaml b/scripts/integration/datadog-agent/compose.yaml index de8d93037915b..5a36d822ac6ca 100644 --- a/scripts/integration/datadog-agent/compose.yaml +++ b/scripts/integration/datadog-agent/compose.yaml @@ -26,7 +26,3 @@ services: - DD_CMD_PORT=5002 - DD_USE_DOGSTATSD=false - DD_HOSTNAME=datadog-trace-agent - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/datadog-traces/compose.yaml b/scripts/integration/datadog-traces/compose.yaml index 1390ce472ab17..8ae4c0dd6c026 100644 --- a/scripts/integration/datadog-traces/compose.yaml +++ b/scripts/integration/datadog-traces/compose.yaml @@ -29,7 +29,3 @@ services: - DD_APM_MAX_MEMORY=0 - DD_APM_MAX_CPU_PERCENT=0 - DD_HOSTNAME=datadog-trace-agent-to-vector - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/dnstap/compose.yaml b/scripts/integration/dnstap/compose.yaml index 2ed599a55cf4a..ae5f973aff541 100644 --- a/scripts/integration/dnstap/compose.yaml +++ b/scripts/integration/dnstap/compose.yaml @@ -11,9 +11,5 @@ services: - dnstap-sockets:/bind2/etc/bind/socket - dnstap-sockets:/bind3/etc/bind/socket -networks: - default: - name: ${VECTOR_NETWORK} - volumes: dnstap-sockets: {} diff --git a/scripts/integration/elasticsearch/compose.yaml b/scripts/integration/elasticsearch/compose.yaml index 338cff73f2fde..cdaf5e2b1d64c 100644 --- a/scripts/integration/elasticsearch/compose.yaml +++ b/scripts/integration/elasticsearch/compose.yaml @@ -25,7 +25,3 @@ services: - ES_JAVA_OPTS=-Xms400m -Xmx400m volumes: - ../../../tests/data/ca:/usr/share/elasticsearch/config/certs:ro - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/eventstoredb/compose.yaml b/scripts/integration/eventstoredb/compose.yaml index c29906f7c033d..324bfeeb84c54 100644 --- a/scripts/integration/eventstoredb/compose.yaml +++ b/scripts/integration/eventstoredb/compose.yaml @@ -6,7 +6,3 @@ services: command: --insecure --stats-period-sec=1 volumes: - ../../../tests/data:/etc/vector:ro - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/gcp/compose.yaml b/scripts/integration/gcp/compose.yaml index b067a1c622a87..643362a2bc76e 100644 --- a/scripts/integration/gcp/compose.yaml +++ b/scripts/integration/gcp/compose.yaml @@ -6,7 +6,3 @@ services: environment: - PUBSUB_PROJECT1=testproject,topic1:subscription1 - PUBSUB_PROJECT2=sourceproject,topic2:subscription2 - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/http-client/compose.yaml b/scripts/integration/http-client/compose.yaml index c3b06ecb0d6d0..69ad1c6ef3d94 100644 --- a/scripts/integration/http-client/compose.yaml +++ b/scripts/integration/http-client/compose.yaml @@ -29,7 +29,3 @@ services: - ../../../tests/data/http-client/serve:/data - ../../../tests/data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem - ../../../tests/data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/humio/compose.yaml b/scripts/integration/humio/compose.yaml index bbef37c60c86c..fa95bec0826bd 100644 --- a/scripts/integration/humio/compose.yaml +++ b/scripts/integration/humio/compose.yaml @@ -3,7 +3,3 @@ version: '3' services: humio: image: docker.io/humio/humio:${CONFIG_VERSION} - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/influxdb/compose.yaml b/scripts/integration/influxdb/compose.yaml index d1972a3b5f6b2..9202280345c19 100644 --- a/scripts/integration/influxdb/compose.yaml +++ b/scripts/integration/influxdb/compose.yaml @@ -19,7 +19,3 @@ services: command: influxd --reporting-disabled environment: - INFLUXDB_REPORTING_DISABLED=true - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/kafka/compose.yaml b/scripts/integration/kafka/compose.yaml index 7287561df47ff..1f1ee04cd7baa 100644 --- a/scripts/integration/kafka/compose.yaml +++ b/scripts/integration/kafka/compose.yaml @@ -33,7 +33,3 @@ services: volumes: - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/logstash/compose.yaml b/scripts/integration/logstash/compose.yaml index ae8fcdbf602f9..9d1e16f2bee19 100644 --- a/scripts/integration/logstash/compose.yaml +++ b/scripts/integration/logstash/compose.yaml @@ -12,7 +12,3 @@ services: - /dev/null:/usr/share/logstash/pipeline/logstash.yml - ../../../tests/data/host.docker.internal.crt:/tmp/logstash.crt - ../../../tests/data/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/loki/compose.yaml b/scripts/integration/loki/compose.yaml index 102e3ea5e8837..bca8cd4a458e2 100644 --- a/scripts/integration/loki/compose.yaml +++ b/scripts/integration/loki/compose.yaml @@ -4,7 +4,3 @@ services: loki: image: docker.io/grafana/loki:${CONFIG_VERSION} command: -config.file=/etc/loki/local-config.yaml -auth.enabled=true - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/mongodb/compose.yaml b/scripts/integration/mongodb/compose.yaml index 45d01a6a1e3e5..8db5c198b8b36 100644 --- a/scripts/integration/mongodb/compose.yaml +++ b/scripts/integration/mongodb/compose.yaml @@ -30,7 +30,3 @@ services: - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor - MONGODB_REPLICA_SET_KEY=vector - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/nats/compose.yaml b/scripts/integration/nats/compose.yaml index f0a827111056f..fb36e78d819d4 100644 --- a/scripts/integration/nats/compose.yaml +++ b/scripts/integration/nats/compose.yaml @@ -43,7 +43,3 @@ services: - /usr/share/nats/config/nats-jwt.conf volumes: - ../../../tests/data/nats:/usr/share/nats/config - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/nginx/compose.yaml b/scripts/integration/nginx/compose.yaml index a702f7bb2c2e8..ddedd8eb2e05e 100644 --- a/scripts/integration/nginx/compose.yaml +++ b/scripts/integration/nginx/compose.yaml @@ -22,6 +22,4 @@ services: - proxy networks: - default: - name: ${VECTOR_NETWORK} proxy: {} diff --git a/scripts/integration/opentelemetry/compose.yaml b/scripts/integration/opentelemetry/compose.yaml index 1764eae606dce..e4b100750f085 100644 --- a/scripts/integration/opentelemetry/compose.yaml +++ b/scripts/integration/opentelemetry/compose.yaml @@ -5,7 +5,3 @@ services: image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION} volumes: - ../../../tests/data/opentelemetry/config.yaml:/etc/otelcol-contrib/config.yaml - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/postgres/compose.yaml b/scripts/integration/postgres/compose.yaml index 137715264442b..ab73d2bbc2303 100644 --- a/scripts/integration/postgres/compose.yaml +++ b/scripts/integration/postgres/compose.yaml @@ -12,9 +12,5 @@ services: - ../../../tests/data/postgres-init.sh:/postgres-init.sh:ro - ../../../tests/data/ca:/certs:ro -networks: - default: - name: ${VECTOR_NETWORK} - volumes: socket: {} diff --git a/scripts/integration/prometheus/compose.yaml b/scripts/integration/prometheus/compose.yaml index 1036ee17c163e..e41592c28eccf 100644 --- a/scripts/integration/prometheus/compose.yaml +++ b/scripts/integration/prometheus/compose.yaml @@ -21,7 +21,3 @@ services: command: --config.file=/etc/vector/prometheus.yaml volumes: - ../../../tests/data:/etc/vector:ro - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/pulsar/compose.yaml b/scripts/integration/pulsar/compose.yaml index a94a4021fe76d..b73d35909be9d 100644 --- a/scripts/integration/pulsar/compose.yaml +++ b/scripts/integration/pulsar/compose.yaml @@ -6,7 +6,3 @@ services: command: bin/pulsar standalone ports: - 6650:6650 - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/redis/compose.yaml b/scripts/integration/redis/compose.yaml index 1b08ef7c3ba2d..a5dd865e43579 100644 --- a/scripts/integration/redis/compose.yaml +++ b/scripts/integration/redis/compose.yaml @@ -3,7 +3,3 @@ version: '3' services: redis: image: docker.io/redis:${CONFIG_VERSION} - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/shutdown/compose.yaml b/scripts/integration/shutdown/compose.yaml index aa2ec34a0e4cb..a74d753e841bc 100644 --- a/scripts/integration/shutdown/compose.yaml +++ b/scripts/integration/shutdown/compose.yaml @@ -31,7 +31,3 @@ services: volumes: - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/splunk/compose.yaml b/scripts/integration/splunk/compose.yaml index 8805ad726dcc0..f42364adb2418 100644 --- a/scripts/integration/splunk/compose.yaml +++ b/scripts/integration/splunk/compose.yaml @@ -13,7 +13,3 @@ services: - 8000:8000 - 8088:8088 - 8089:8089 - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/scripts/integration/webhdfs/compose.yaml b/scripts/integration/webhdfs/compose.yaml index acf094c0bb882..9585927e84a10 100644 --- a/scripts/integration/webhdfs/compose.yaml +++ b/scripts/integration/webhdfs/compose.yaml @@ -30,7 +30,3 @@ services: interval: 5s timeout: 5s retries: 3 - -networks: - default: - name: ${VECTOR_NETWORK} diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index bf673fb60c993..30e63132cf5cc 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -35,5 +35,5 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.95" serde_yaml = "0.9.19" sha2 = "0.10.6" -tempfile = "3.4.0" +tempfile = "3.5.0" toml = { version = "0.7.2", default-features = false, features = ["parse"] } diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 8a2d7e4e263fa..e7bedd7093805 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -5,7 +5,7 @@ use std::{env, fs}; use anyhow::{bail, Context, Result}; use hashlink::LinkedHashMap; use itertools::{self, Itertools}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_yaml::Value; use crate::{app, util}; @@ -37,20 +37,49 @@ impl RustToolchainConfig { } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct ComposeConfig { pub services: BTreeMap, - #[serde(default)] + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub volumes: BTreeMap, + #[serde(default)] + pub networks: BTreeMap>, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct ComposeService { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hostname: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub container_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub build: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ports: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub env_file: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub volumes: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub environment: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub depends_on: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub healthcheck: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(untagged)] +pub enum Command { + Single(String), + Multiple(Vec), } impl ComposeConfig { - #[cfg(unix)] pub fn parse(path: &Path) -> Result { let contents = fs::read_to_string(path).with_context(|| format!("failed to read {path:?}"))?; diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index adf97e133362d..ecac804a98842 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -1,8 +1,8 @@ -use std::{path::Path, path::PathBuf, process::Command}; +use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command}; use anyhow::{bail, Context, Result}; +use tempfile::{Builder, NamedTempFile}; -#[cfg(unix)] use super::config::ComposeConfig; use super::config::{Environment, IntegrationTestConfig}; use super::runner::{ @@ -135,30 +135,52 @@ impl IntegrationTest { } struct Compose { - path: PathBuf, + original_path: PathBuf, test_dir: PathBuf, env: Environment, - #[cfg(unix)] + #[cfg_attr(target_family = "windows", allow(dead_code))] config: ComposeConfig, network: String, + temp_file: NamedTempFile, } impl Compose { fn new(test_dir: PathBuf, env: Environment, network: String) -> Result> { - let path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect(); - match path.try_exists() { - Err(error) => Err(error).with_context(|| format!("Could not lookup {path:?}")), + let original_path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect(); + + match original_path.try_exists() { + Err(error) => Err(error).with_context(|| format!("Could not lookup {original_path:?}")), Ok(false) => Ok(None), Ok(true) => { - #[cfg(unix)] - let config = ComposeConfig::parse(&path)?; + let mut config = ComposeConfig::parse(&original_path)?; + // Inject the networks block + config.networks.insert( + "default".to_string(), + BTreeMap::from_iter([("name".to_string(), network.clone())]), + ); + + // Create a named tempfile, there may be resource leakage here in case of SIGINT + // Tried tempfile::tempfile() but this returns a File object without a usable path + // https://docs.rs/tempfile/latest/tempfile/#resource-leaking + let temp_file = Builder::new() + .prefix("compose-temp-") + .suffix(".yaml") + .tempfile_in(&test_dir) + .with_context(|| "Failed to create temporary compose file")?; + + fs::write( + temp_file.path(), + serde_yaml::to_string(&config) + .with_context(|| "Failed to serialize modified compose.yaml")?, + )?; + Ok(Some(Self { - path, + original_path, test_dir, env, - #[cfg(unix)] config, network, + temp_file, })) } } @@ -178,8 +200,19 @@ impl Compose { let mut command = CONTAINER_TOOL.clone(); command.push("-compose"); let mut command = Command::new(command); + // When the integration test environment is already active, the tempfile path does not + // exist because `Compose::new()` has not been called. In this case, the `stop` command + // needs to use the calculated path from the integration name instead of the nonexistent + // tempfile path. This is because `stop` doesn't go through the same logic as `start` + // and doesn't create a new tempfile before calling docker compose. + // If stop command needs to use some of the injected bits then we need to rebuild it command.arg("--file"); - command.arg(&self.path); + if config.is_none() { + command.arg(&self.original_path); + } else { + command.arg(self.temp_file.path()); + } + command.args(args); command.current_dir(&self.test_dir);