enhancement(vdev): Load compose files and inject network block

scripts/integration/amqp/compose.yaml

@@ -5,7 +5,3 @@ services:
     image: docker.io/rabbitmq:${CONFIG_VERSION}
     ports:
     - 5672:5672
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/aws/compose.yaml

@@ -14,7 +14,3 @@ services:
     volumes:
     - $DOCKER_SOCKET:/var/run/docker.sock
     - $HOME/.aws/:/home/.aws/
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/azure/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: azurite --blobHost 0.0.0.0 --loose
     volumes:
     - /var/run:/var/run
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/chronicle/compose.yaml

@@ -10,7 +10,3 @@ services:
     command:
     - -p
     - /public.pem
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/clickhouse/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   clickhouse:
     image: docker.io/yandex/clickhouse-server:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/databend/compose.yaml

@@ -19,7 +19,3 @@ services:
       - minio
     healthcheck:
       test: "curl -f localhost:8080/v1/health || exit 1"
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/datadog-agent/compose.yaml

@@ -26,7 +26,3 @@ services:
     - DD_CMD_PORT=5002
     - DD_USE_DOGSTATSD=false
     - DD_HOSTNAME=datadog-trace-agent
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/datadog-traces/compose.yaml

@@ -29,7 +29,3 @@ services:
     - DD_APM_MAX_MEMORY=0
     - DD_APM_MAX_CPU_PERCENT=0
     - DD_HOSTNAME=datadog-trace-agent-to-vector
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/dnstap/compose.yaml

@@ -11,9 +11,5 @@ services:
       - dnstap-sockets:/bind2/etc/bind/socket
       - dnstap-sockets:/bind3/etc/bind/socket
 
-networks:
-  default:
-    name: ${VECTOR_NETWORK}
-
 volumes:
   dnstap-sockets: {}

scripts/integration/elasticsearch/compose.yaml

@@ -25,7 +25,3 @@ services:
     - ES_JAVA_OPTS=-Xms400m -Xmx400m
     volumes:
     - ../../../tests/data/ca:/usr/share/elasticsearch/config/certs:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/eventstoredb/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: --insecure --stats-period-sec=1
     volumes:
     - ../../../tests/data:/etc/vector:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/gcp/compose.yaml

@@ -6,7 +6,3 @@ services:
     environment:
     - PUBSUB_PROJECT1=testproject,topic1:subscription1
     - PUBSUB_PROJECT2=sourceproject,topic2:subscription2
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/http-client/compose.yaml

@@ -29,7 +29,3 @@ services:
     - ../../../tests/data/http-client/serve:/data
     - ../../../tests/data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem
     - ../../../tests/data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/humio/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   humio:
     image: docker.io/humio/humio:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/influxdb/compose.yaml

@@ -19,7 +19,3 @@ services:
     command: influxd --reporting-disabled
     environment:
     - INFLUXDB_REPORTING_DISABLED=true
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/kafka/compose.yaml

@@ -33,7 +33,3 @@ services:
     volumes:
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/logstash/compose.yaml

@@ -12,7 +12,3 @@ services:
     - /dev/null:/usr/share/logstash/pipeline/logstash.yml
     - ../../../tests/data/host.docker.internal.crt:/tmp/logstash.crt
     - ../../../tests/data/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/loki/compose.yaml

@@ -4,7 +4,3 @@ services:
   loki:
     image: docker.io/grafana/loki:${CONFIG_VERSION}
     command: -config.file=/etc/loki/local-config.yaml -auth.enabled=true
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/mongodb/compose.yaml

@@ -30,7 +30,3 @@ services:
     - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017
     - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor
     - MONGODB_REPLICA_SET_KEY=vector
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/nats/compose.yaml

@@ -43,7 +43,3 @@ services:
     - /usr/share/nats/config/nats-jwt.conf
     volumes:
     - ../../../tests/data/nats:/usr/share/nats/config
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/nginx/compose.yaml

@@ -20,8 +20,3 @@ services:
     - ../../../tests/data/nginx/:/etc/nginx:ro
     networks:
     - proxy
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}
-  proxy: {}

scripts/integration/opentelemetry/compose.yaml

@@ -5,7 +5,3 @@ services:
     image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION}
     volumes:
     - ../../../tests/data/opentelemetry/config.yaml:/etc/otelcol-contrib/config.yaml
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/postgres/compose.yaml

@@ -12,9 +12,5 @@ services:
     - ../../../tests/data/postgres-init.sh:/postgres-init.sh:ro
     - ../../../tests/data/ca:/certs:ro
 
-networks:
-  default:
-    name: ${VECTOR_NETWORK}
-
 volumes:
   socket: {}

scripts/integration/prometheus/compose.yaml

@@ -21,7 +21,3 @@ services:
     command: --config.file=/etc/vector/prometheus.yaml
     volumes:
     - ../../../tests/data:/etc/vector:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/pulsar/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: bin/pulsar standalone
     ports:
     - 6650:6650
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/redis/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   redis:
     image: docker.io/redis:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/shutdown/compose.yaml

@@ -31,7 +31,3 @@ services:
     volumes:
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/splunk/compose.yaml

@@ -13,7 +13,3 @@ services:
     - 8000:8000
     - 8088:8088
     - 8089:8089
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/webhdfs/compose.yaml

@@ -30,7 +30,3 @@ services:
       interval: 5s
       timeout: 5s
       retries: 3
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

vdev/src/testing/config.rs

@@ -5,7 +5,7 @@ use std::{env, fs};
 use anyhow::{bail, Context, Result};
 use hashlink::LinkedHashMap;
 use itertools::{self, Itertools};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use serde_yaml::Value;
 
 use crate::{app, util};
@@ -37,16 +37,68 @@ impl RustToolchainConfig {
     }
 }
 
-#[derive(Debug, Deserialize)]
+// Added the networks key, to match the structure of networks eg
+// networks: default: name: value: <some value here>
+#[derive(Debug, Deserialize, Serialize)]
 pub struct ComposeConfig {
     pub services: BTreeMap<String, ComposeService>,
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub volumes: BTreeMap<String, Value>,
+    #[serde(default)]
+    pub networks: BTreeMap<String, BTreeMap<String, String>>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct ComposeService {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub image: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hostname: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub container_name: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub build: Option<BuildConfig>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub command: Option<Command>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub ports: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub env_file: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub volumes: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub environment: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub depends_on: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub healthcheck: Option<HealthCheck>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct HealthCheck {
+    #[serde(rename = "test", skip_serializing_if = "Option::is_none")]
+    pub test: Option<Command>,
+    #[serde(rename = "interval", skip_serializing_if = "Option::is_none")]
+    pub interval: Option<String>,
+    #[serde(rename = "timeout", skip_serializing_if = "Option::is_none")]
+    pub timeout: Option<String>,
+    #[serde(rename = "retries", skip_serializing_if = "Option::is_none")]
+    pub retries: Option<u32>,
+    #[serde(rename = "start_period", skip_serializing_if = "Option::is_none")]
+    pub start_period: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct BuildConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub context: Option<String>
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(untagged)]
+pub enum Command {
+    Single(String),
+    Multiple(Vec<String>),
 }
 
 impl ComposeConfig {

vdev/src/testing/integration.rs

@@ -1,4 +1,5 @@
-use std::{path::Path, path::PathBuf, process::Command};
+use std::collections::BTreeMap;
+use std::{fs, path::Path, path::PathBuf, process::Command};
 
 use anyhow::{bail, Context, Result};
 
@@ -28,11 +29,28 @@ impl IntegrationTest {
         let integration = integration.into();
         let environment = environment.into();
         let (test_dir, config) = IntegrationTestConfig::load(&integration)?;
+        // Load thee compose_config
+        let mut compose_config = ComposeConfig::parse(&test_dir.join("compose.yaml"))?;
+        let network_name = format!("vector-integration-tests-{integration}");
+        // Inject the networks block
+        // TODO: Inject the labels block
+        compose_config.networks.insert("default".to_string(), {
+            let mut default_network = BTreeMap::new();
+            default_network.insert("name".to_string(), network_name.clone());
+            default_network
+        });
+
+        let temp_compose_path = test_dir.join("compose-temp.yaml");
+        fs::write(
+            temp_compose_path,
+            serde_yaml::to_string(&compose_config)
+                .with_context(|| "Failed to serialize modified compose.yml".to_string())?,
+        )?;
+
         let envs_dir = EnvsDir::new(&integration);
         let Some(env_config) = config.environments().get(&environment).map(Clone::clone) else {
             bail!("Could not find environment named {environment:?}");
         };
-        let network_name = format!("vector-integration-tests-{integration}");
         let compose = Compose::new(test_dir, env_config.clone(), Some(network_name.clone()))?;
         let runner = IntegrationTestRunner::new(
             integration.clone(),
@@ -128,12 +146,17 @@ impl IntegrationTest {
             self.runner.remove()?;
             compose.stop()?;
             self.envs_dir.remove()?;
+            std::fs::remove_file(&compose.path)?;
         }
 
         Ok(())
     }
 }
 
+// Question: Should I inject network here or should I keep it in the compose config?
+// I am leaning towards keeping it in the compose config. Since that seems to be the
+// Compose file direct mapping, but I would like guidance on this. For now I can
+// Add to network by reading the compose config.
 struct Compose {
     path: PathBuf,
     test_dir: PathBuf,
@@ -145,13 +168,17 @@ struct Compose {
 
 impl Compose {
     fn new(test_dir: PathBuf, env: Environment, network: Option<String>) -> Result<Option<Self>> {
-        let path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect();
+        let path: PathBuf = [&test_dir, Path::new("compose-temp.yaml")].iter().collect();
         match path.try_exists() {
             Err(error) => Err(error).with_context(|| format!("Could not lookup {path:?}")),
             Ok(false) => Ok(None),
             Ok(true) => {
                 #[cfg(unix)]
                 let config = ComposeConfig::parse(&path)?;
+                println!(
+                    "[jonathanpv:debugging]\nUsing compose file: {:?}, found in: {:?}",
+                    config, path
+                );
                 Ok(Some(Self {
                     path,
                     test_dir,