chore(vdev): Rework support for integration tests

scripts/integration/README.md

@@ -0,0 +1,31 @@
+This directory contains a set of integration test frameworks for vector which are executed by the
+`vdev` tool.
+
+Each directory contains two files:
+
+1. A `compose.yaml` file containing the instructions to `docker-compose` or `podman-compose` for how
+   to set up the containers in which to run the integrations, and
+2. A `test.yaml` file that describes how to run the integration tests, including a matrix of
+   software versions or other parameters over which the tests will be run.
+
+You can list these tests with `cargo vdev integration show`[1], which provides a list of all the
+integration test names followed by the extrapolated matrix of environments.
+
+Each test can be run using one of the following:
+
+1. Run a single test environment from the above list with `cargo vdev integration test NAME ENV`
+2. Run all the environments for one test with `cargo vdev integration test NAME`
+3. Run all the steps individually using the `start`, `test`, and then `stop` subcommands with the
+   same parameters as above (see below). This allows developers to start the environment once and
+   then repeat the testing step while working on a component.
+
+```shell
+cargo vdev integration start NAME ENVIRONMENT
+cargo vdev integration test NAME [ENVIRONMENT]
+cargo vdev integration stop NAME [ENVIRONMENT]
+```
+
+If no environment is named for the `test` and `stop` subcommands, all active environments are used.
+
+[1] Note that the `vdev` tool accepts abbreviated subcommand names, so this can also be run as
+`cargo vdev int show` for brevity.

scripts/integration/amqp/compose.yaml

@@ -0,0 +1,12 @@
+version: '3'
+
+services:
+  rabbitmq:
+    image: docker.io/rabbitmq:${AMQP_VERSION}
+    ports:
+    - 5672:5672
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/amqp/test.yaml

@@ -0,0 +1,8 @@
+args:
+- --features
+- amqp-integration-tests
+- --lib
+- '::amqp::'
+
+matrix:
+- version: ['3.8']

scripts/integration/apex/compose.yaml

@@ -0,0 +1,14 @@
+version: '3'
+
+services:
+  mock-apex:
+    image: mcasper/mock-apex:${APEX_VERSION}
+    environment:
+    - MOCK_API_TOKEN=token
+    ports:
+    - '4567'
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/apex/test.yaml

@@ -0,0 +1,11 @@
+args:
+- --features
+- apex-integration-tests
+- --lib
+- '::apex::'
+
+env:
+  MOCK_APEX_ADDRESS: http://mock-apex:4567
+
+matrix:
+- version: [latest]

scripts/integration/aws/compose.yaml

@@ -0,0 +1,21 @@
+version: '3'
+
+services:
+  mock-ec2-metadata:
+    image: public.ecr.aws/aws-ec2/amazon-ec2-metadata-mock:v1.11.2
+  mock-localstack:
+    image: docker.io/localstack/localstack-full:0.11.6
+    environment:
+    - SERVICES=kinesis,s3,cloudwatch,elasticsearch,es,firehose,sqs
+  mock-watchlogs:
+    image: docker.io/luciofranco/mockwatchlogs:latest
+  mock-ecs:
+    image: docker.io/amazon/amazon-ecs-local-container-endpoints:latest
+    volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+    - $HOME/.aws/:/home/.aws/
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/aws/test.yaml

@@ -0,0 +1,20 @@
+args:
+- --features
+- aws-integration-tests
+- --lib
+- ::aws_
+
+env:
+  AWS_ACCESS_KEY_ID: dummy
+  AWS_SECRET_ACCESS_KEY: dummy
+  CLOUDWATCH_ADDRESS: http://mock-localstack:4566
+  EC2_METADATA_ADDRESS: http://mock-ec2-metadata:8111
+  ECS_ADDRESS: http://mock-ecs
+  ELASTICSEARCH_ADDRESS: http://mock-localstack:4571
+  KINESIS_ADDRESS: http://mock-localstack:4566
+  S3_ADDRESS: http://mock-localstack:4566
+  SQS_ADDRESS: http://mock-localstack:4566
+  WATCHLOGS_ADDRESS: http://mock-watchlogs:6000
+
+matrix:
+- version: [latest]

scripts/integration/axiom/compose.yaml

@@ -0,0 +1,34 @@
+version: '3'
+
+services:
+  postgres:
+    image: postgres:${AXIOM_VERSION}
+    environment:
+      POSTGRES_USER: axiom
+      POSTGRES_PASSWORD: axiom
+    volumes:
+    - postgres_data:/var/lib/postgresql/data
+  axiom-db:
+    image: axiomhq/axiom-db:latest
+    restart: unless-stopped
+    environment:
+      AXIOM_STORAGE: file:///data
+      AXIOM_POSTGRES_URL: postgres://axiom:axiom@postgres?sslmode=disable&connect_timeout=5
+    depends_on:
+    - postgres
+  axiom-core:
+    image: axiomhq/axiom-core:latest
+    restart: unless-stopped
+    environment:
+      AXIOM_POSTGRES_URL: postgres://axiom:axiom@postgres?sslmode=disable&connect_timeout=5
+      AXIOM_DB_URL: http://axiom-db:8080
+      AXIOM_HTTP_PORT: '80'
+    depends_on:
+    - axiom-db
+    ports:
+    - 8081:80
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/axiom/test.yaml

@@ -0,0 +1,11 @@
+args:
+- --features
+- axiom-integration-tests
+- --lib
+- 'sinks::axiom::'
+
+env:
+  AXIOM_URL: http://axiom-core
+
+matrix:
+- version: [13-alpine]

scripts/integration/azure/compose.yaml

@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+  local-azure-blob:
+    image: mcr.microsoft.com/azure-storage/azurite:${AZURE_VERSION}
+    command: azurite --blobHost 0.0.0.0 --loose
+    volumes:
+    - /var/run:/var/run
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/azure/test.yaml

@@ -0,0 +1,13 @@
+args:
+- --features
+- azure-integration-tests
+- --lib
+- ::azure_
+
+env:
+  AZURE_ADDRESS: local-azure-blob
+  HEARTBEAT_ADDRESS: 0.0.0.0:8080
+  LOGSTASH_ADDRESS: 0.0.0.0:8081
+
+matrix:
+- version: [3.14.0]

scripts/integration/chronicle/compose.yaml

@@ -0,0 +1,17 @@
+version: '3'
+
+services:
+  chronicle-emulator:
+    image: docker.io/plork/chronicle-emulator:${CHRONICLE_VERSION}
+    ports:
+    - 3000:3000
+    volumes:
+    - ../chroniclepub.pem:/public.pem
+    command:
+    - -p
+    - /public.pem
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/chronicle/test.yaml

@@ -0,0 +1,11 @@
+args:
+- --features
+- chronicle-integration-tests
+- --lib
+- '::chronicle_unstructured::'
+
+env:
+  CHRONICLE_ADDRESS: http://chronicle-emulator:3000
+
+matrix:
+- version: [latest]

scripts/integration/clickhouse/compose.yaml

@@ -0,0 +1,10 @@
+version: '3'
+
+services:
+  clickhouse:
+    image: docker.io/yandex/clickhouse-server:${CLICKHOUSE_VERSION}
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/clickhouse/test.yaml

@@ -0,0 +1,11 @@
+args:
+- --features
+- clickhouse-integration-tests
+- --lib
+- '::clickhouse::'
+
+env:
+  CLICKHOUSE_ADDRESS: http://clickhouse:8123
+
+matrix:
+- version: ['19']

scripts/integration/datadog-agent/compose.yaml

@@ -0,0 +1,33 @@
+version: '3'
+
+services:
+  datadog-agent:
+    image: docker.io/datadog/agent:${DATADOG_AGENT_VERSION}
+    environment:
+    - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required}
+    - DD_APM_ENABLED=false
+    - DD_LOGS_ENABLED=true
+    - DD_LOGS_CONFIG_LOGS_DD_URL=runner:8080
+    - DD_LOGS_CONFIG_LOGS_NO_SSL=true
+    - DD_LOGS_CONFIG_USE_HTTP=true
+    - DD_HEALTH_PORT=8182
+    - DD_CMD_PORT=5001
+    - DD_USE_DOGSTATSD=false
+    - DD_HOSTNAME=datadog-agent
+    volumes:
+    - ../../../tests/data/datadog-agent/conf.yaml:/etc/datadog-agent/conf.d/test.d/conf.yaml
+  datadog-trace-agent:
+    image: docker.io/datadog/agent:7.31.0
+    environment:
+    - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required}
+    - DD_APM_ENABLED=true
+    - DD_APM_DD_URL=http://runner:8081
+    - DD_HEALTH_PORT=8183
+    - DD_CMD_PORT=5002
+    - DD_USE_DOGSTATSD=false
+    - DD_HOSTNAME=datadog-trace-agent
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/datadog-agent/test.yaml

@@ -0,0 +1,14 @@
+args:
+- --features
+- datadog-agent-integration-tests
+- --lib
+- 'sources::datadog_agent::integration_tests::'
+
+env:
+  AGENT_ADDRESS: datadog-agent:8181
+  AGENT_HEALTH_ADDRESS: http://datadog-agent:8182
+  TRACE_AGENT_HEALTH_ADDRESS: http://datadog-trace-agent:8183
+  TRACE_AGENT_URL: http://datadog-trace-agent:8126/v0.4/traces
+
+matrix:
+- version: ['7']

scripts/integration/datadog-traces/compose.yaml

@@ -0,0 +1,36 @@
+version: '3'
+
+services:
+  datadog-trace-agent:
+    image: docker.io/datadog/agent:${DATADOG_TRACES_VERSION}
+    environment:
+    - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required}
+    - DD_APM_ENABLED=true
+    - DD_APM_DD_URL=http://runner:8082
+    - DD_HEALTH_PORT=8183
+    - DD_CMD_PORT=5002
+    - DD_USE_DOGSTATSD=false
+    - DD_APM_MAX_TPS=999999
+    - DD_APM_ERROR_TPS=999999
+    - DD_APM_MAX_MEMORY=0
+    - DD_APM_MAX_CPU_PERCENT=0
+    - DD_HOSTNAME=datadog-trace-agent
+  datadog-trace-agent-to-vector:
+    image: docker.io/datadog/agent:${DATADOG_TRACES_VERSION}
+    environment:
+    - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required}
+    - DD_APM_ENABLED=true
+    - DD_APM_DD_URL=http://runner:8081
+    - DD_HEALTH_PORT=8183
+    - DD_CMD_PORT=5002
+    - DD_USE_DOGSTATSD=false
+    - DD_APM_MAX_TPS=999999
+    - DD_APM_ERROR_TPS=999999
+    - DD_APM_MAX_MEMORY=0
+    - DD_APM_MAX_CPU_PERCENT=0
+    - DD_HOSTNAME=datadog-trace-agent-to-vector
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/datadog-traces/test.yaml

@@ -0,0 +1,17 @@
+args:
+- --no-default-features
+- --features
+- datadog-traces-integration-tests
+- --lib
+- '::datadog::traces::'
+
+env:
+  AGENT_PORT: '8082'
+  TEST_DATADOG_API_KEY: ${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required}
+  TEST_LOG: ${TEST_LOG}
+  TRACE_AGENT_TO_VECTOR_URL: http://datadog-trace-agent-to-vector:8126/v0.3/traces
+  TRACE_AGENT_URL: http://datadog-trace-agent:8126/v0.3/traces
+  VECTOR_PORT: '8081'
+
+matrix:
+- version: [latest]