Introduce a new http -> pipelines -> blackhole soak

This commit introduces a new soak to investigate our pipelines in an ongoing
basis. The pipeline configuration for vector and the sample data was contributed
by work was contributed @vladimir-dd. I have expanded the soak to allow mounting
a TESTNAME/data to smuggle static data into the minikube, by which we feed
http_gen. Unfortunately this does not work as we cannot use the Virtual Box VM
for soak testing and kubernetes/minikube#12301 is
open. We will have to figure out some other way to smuggle data into the
minikube for this soak to function.

This commit depends on #10141 which
depends on DataDog/lading#119.

Signed-off-by: Brian L. Troutwine <[email protected]>

soaks/bin/run_experiment.sh

@@ -82,10 +82,21 @@ SOAK_CAPTURE_DIR="${CAPTURE_DIR}/${SOAK_NAME}"
 
 pushd "${__dir}"
 ./boot_minikube.sh --cpus "${SOAK_CPUS}" --memory "${SOAK_MEMORY}"
-mkdir -p "${SOAK_CAPTURE_DIR}"
+mkdir --parents "${SOAK_CAPTURE_DIR}"
+minikube image load "${IMAGE}"
+# Mount the capture directory. This is where the samples captured from inside
+# the minikube will be placed on the host.
 minikube mount "${SOAK_CAPTURE_DIR}:/captures" &
-minikube cache add "${IMAGE}"
-MOUNT_PID=$!
+CAPTURE_MOUNT_PID=$!
+popd
+
+pushd "${SOAK_ROOT}/tests/${SOAK_NAME}"
+mkdir --parents data
+# Mount the data directory. This is where the data that supports the test are
+# mounted into the minikube. The software running in the minikube will not write
+# to this directory.
+minikube mount "${SOAK_ROOT}/tests/${SOAK_NAME}/data:/data" &
+DATA_MOUNT_PID=$!
 popd
 
 pushd "${SOAK_ROOT}/tests/${SOAK_NAME}/terraform"
@@ -96,7 +107,8 @@ echo "[${VARIANT}] Sleeping for ${WARMUP_GRACE} seconds to allow warm-up"
 sleep "${WARMUP_GRACE}"
 echo "[${VARIANT}] Recording captures to ${SOAK_CAPTURE_DIR}"
 sleep "${TOTAL_SAMPLES}"
-kill "${MOUNT_PID}"
+kill "${CAPTURE_MOUNT_PID}"
+kill "${DATA_MOUNT_PID}"
 popd
 
 pushd "${__dir}"

soaks/common/terraform/modules/lading_http_gen/main.tf

@@ -70,6 +70,11 @@ resource "kubernetes_deployment" "http-gen" {
           name              = "http-gen"
           command           = ["/http_gen"]
 
+          volume_mount {
+            mount_path = "/data"
+            name       = "data"
+            read_only  = true
+          }
           volume_mount {
             mount_path = "/etc/lading"
             name       = "etc-lading"
@@ -106,6 +111,12 @@ resource "kubernetes_deployment" "http-gen" {
             name = kubernetes_config_map.lading.metadata[0].name
           }
         }
+        volume {
+          name = "data"
+          host_path {
+            path = "/data"
+          }
+        }
       }
     }
   }

soaks/tests/http_pipelines_blackhole/README.md

@@ -0,0 +1,9 @@
+# HTTP -> Pipelines -> Datadog Logs
+
+This soak tests the http sink feeding through multiple pipeline transforms down
+to blackhole sink. It is a complicated topology.
+
+## Method
+
+Lading `http_gen` is used to generate log load into vector. The vector internal
+blackhole is sink for this configuration.

soaks/tests/http_pipelines_blackhole/terraform/http_gen.yaml

@@ -0,0 +1,15 @@
+worker_threads: 1
+prometheus_addr: "0.0.0.0:9090"
+
+targets:
+  vector:
+    headers: {}
+    target_uri: "http://vector:8282/v1/input"
+    bytes_per_second: "500 Mb"
+    parallel_connections: 10
+    method:
+      post:
+        maximum_prebuild_cache_size_bytes: "256 Mb"
+        variant:
+          static:
+            static_path: "/tmp/input.log"

soaks/tests/http_pipelines_blackhole/terraform/main.tf

@@ -0,0 +1,53 @@
+# Set up the providers needed to run this soak and other terraform related
+# business. Here we only require 'kubernetes' to interact with the soak
+# minikube.
+terraform {
+  required_providers {
+    kubernetes = {
+      version = "~> 2.5.0"
+      source  = "hashicorp/kubernetes"
+    }
+  }
+}
+
+# Rig the kubernetes provider to communicate with minikube. The details of
+# adjusting `~/.kube/config` are addressed by the soak control scripts.
+provider "kubernetes" {
+  config_path = "~/.kube/config"
+}
+
+# Setup background monitoring details. These are needed by the soak control to
+# understand what vector et al's running behavior is.
+module "monitoring" {
+  source       = "../../../common/terraform/modules/monitoring"
+  type         = var.type
+  vector_image = var.vector_image
+}
+
+# Setup the soak pieces
+#
+# This soak config sets up a vector soak with lading/http-gen feeding into vector,
+# lading/http-blackhole receiving.
+resource "kubernetes_namespace" "soak" {
+  metadata {
+    name = "soak"
+  }
+}
+
+module "vector" {
+  source       = "../../../common/terraform/modules/vector"
+  type         = var.type
+  vector_image = var.vector_image
+  vector-toml  = file("${path.module}/vector.toml")
+  namespace    = kubernetes_namespace.soak.metadata[0].name
+  vector_cpus  = var.vector_cpus
+  depends_on   = [module.monitoring]
+}
+module "http-gen" {
+  source        = "../../../common/terraform/modules/lading_http_gen"
+  type          = var.type
+  http-gen-yaml = file("${path.module}/http_gen.yaml")
+  namespace     = kubernetes_namespace.soak.metadata[0].name
+  lading_image  = var.lading_image
+  depends_on    = [module.monitoring, module.vector]
+}

soaks/tests/http_pipelines_blackhole/terraform/variables.tf

@@ -0,0 +1,19 @@
+variable "type" {
+  description = "The type of the vector install, whether 'baseline' or 'comparison'"
+  type        = string
+}
+
+variable "vector_image" {
+  description = "The image of vector to use in this investigation"
+  type        = string
+}
+
+variable "vector_cpus" {
+  description = "The total number of CPUs to give to vector"
+  type        = number
+}
+
+variable "lading_image" {
+  description = "The lading image to run"
+  type        = string
+}