Dockerized benchmarking framework

.circleci/config.yml

@@ -41,6 +41,16 @@ jobs:
       - store_artifacts:
           path: /root/project/generated
           destination: /
+  benchmark:
+    docker:
+      - image: *envoy-build-image
+    resource_class: xlarge
+    steps:
+      - checkout
+      - run: ci/do_ci.sh benchmark_with_own_binaries
+      - store_artifacts:
+          path: /root/project/generated
+          destination: /
   asan:
     docker:
       - image: *envoy-build-image
@@ -83,5 +93,6 @@ workflows:
       - coverage
       - asan
       - tsan
+      - benchmark
       - format
       - docker

WORKSPACE

@@ -25,13 +25,13 @@ load("@envoy//bazel:dependency_imports.bzl", "envoy_dependency_imports")
 envoy_dependency_imports()
 
 # For PIP support:
-load("@rules_python//python:pip.bzl", "pip_import", "pip_repositories")
+load("@rules_python//python:pip.bzl", "pip3_import", "pip_repositories")
 
 pip_repositories()
 
 # This rule translates the specified requirements.txt into
 # @my_deps//:requirements.bzl, which itself exposes a pip_install method.
-pip_import(
+pip3_import(
     name = "python_pip_deps",
     requirements = "//:requirements.txt",
 )

benchmarks/.gitignore

@@ -0,0 +1 @@
+tmp/*

benchmarks/BUILD

@@ -1,21 +1,38 @@
-load("@rules_python//python:defs.bzl", "py_test")
-load(
-    "@envoy//bazel:envoy_build_system.bzl",
-    "envoy_package",
-)
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 
 licenses(["notice"])  # Apache 2
 
-envoy_package()
+py_binary(
+    name = "benchmarks",
+    srcs = [
+        "benchmarks.py",
+    ],
+    deps = [
+        ":benchmarks_envoy_proxy_lib",
+        "//test/integration:integration_test_base_lean",
+    ],
+)
 
 py_test(
-    name = "benchmarks",
+    name = "test_benchmarks",
     srcs = [
         "benchmarks.py",
-        "test_benchmark.py",
+        "test/test_discovery.py",
     ],
+    main = "benchmarks.py",
     deps = [
+        ":benchmarks_envoy_proxy_lib",
         "//test/integration:integration_test_base",
-        "//test/integration:test_integration_basics_lib",
+    ],
+)
+
+py_library(
+    name = "benchmarks_envoy_proxy_lib",
+    srcs = [
+        "envoy_proxy.py",
+    ],
+    data = [
+        "configurations/envoy_proxy.yaml",
+        "test/templates/simple_plot.html",
     ],
 )

benchmarks/README.md

@@ -0,0 +1,130 @@
+# Benchmarking testsuite
+
+The NH benchmark test suite builds on top Nighthawk's integration test framework, and
+can be used to scaffold tests and obtain latency reports as well as flamegraphs.
+
+## Immediate goals
+
+The goal is to be able to:
+
+- facilitate integration into CI flows (via fully dockerized flow)
+- facilitate A/B testing by developers
+- run the suite against arbitrary Envoy revisions
+- persist profile dumps, flamegraphs, and latency numbers per test
+- offer stock tests, but also allow scaffolding consumer-specific tests
+
+## Example output visualization
+
+The benchmark will drop a visual in each test directory.
+
+![Example visualization](benchmark-visual-example.png)
+
+## Example: Docker based execution, scavaging benchmark/
+
+This scripts shows how to use the benchmarking suite.
+It will run a selection of an example [benchmarks](test/test_discovery.py)
+extracted from `/benchmarks`, which injects Envoy between the benchmark client and test server.
+
+```bash
+git clone https://github.com/oschaaf/nighthawk.git benchmark-test
+cd benchmark-test
+bazel build //benchmarks:benchmarks
+
+# Specify the ip address family we'll be using. [v4only|v6only|all]
+export ENVOY_IP_TEST_VERSIONS=v4only
+# Explicit tmpdir for OSX Docker, to make sure we'll use a volume that works when
+export TMPDIR="$(pwd)/benchmarks/tmp"
+# Nighthawk tools will be sourced from this Docker image
+export NH_DOCKER_IMAGE="envoyproxy/nighthawk-dev:latest"
+# Envoy Docker image that we'll use to inject the Envoy proxy
+export ENVOY_DOCKER_IMAGE_TO_TEST="envoyproxy/envoy-dev:74290ef76a76fbbf50f072dc33438791f93f68c7"
+# Envoy is called 'Envoy' in the Envoy Docker image.
+export ENVOY_PATH="envoy"
+
+# run all tests starting with test_http_h1_small in benchmarks/
+bazel-bin/benchmarks/benchmarks --log-cli-level=info -vvvv -k test_http_h1_small benchmarks/
+```
+
+## Example: running with binaries
+
+This will build the Nighthawk binaries from the C++ code, and use those to
+execute the benchmarks. Environment variable `ENVOY_PATH` can be used to
+specify a custom Envoy binary to use to inject as a proxy between the test
+client and server. If not set, the benchmark suite will fall back to configuring
+Nighthawk's test server for that. Note that the build can be a lengthy process.
+
+```bash
+git clone https://github.com/oschaaf/nighthawk.git benchmark-test
+cd benchmark-test
+bazel test \
+  --test_summary=detailed \
+  --test_output=all \
+  --test_arg=--log-cli-level=info \
+  --test_env=ENVOY_IP_TEST_VERSIONS=v4only \
+  --test_env=HEAPPROFILE= \
+  --test_env=HEAPCHECK= \
+  --cache_test_results=no \
+  --compilation_mode=opt \
+  --cxxopt=-g \
+  --cxxopt=-ggdb3 \
+  //benchmarks:*
+```
+
+## Example: fully dockerized flow
+
+The framework can be run via Docker and used that way to execute
+Python benchmarks scripts not sourced from this repository, but
+elsewhere. An example:
+
+```bash
+# This script runs the dockerized benchmarking framework, which in
+# turn will pull Nighthawk and Envoy in via Docker.
+
+set -eo pipefail
+set +x
+set -u
+
+# The benchmark logs and artifacts will be dropped here
+OUTDIR="/my-artifacts-dir/"
+# Used to map the test that we want to see executed into the Docker container
+# Note: the contents could be fetched via http, for example.
+TEST_DIR="/dir-that-has-my-tests/"
+
+# Rebuild the Docker in case something changed.
+./docker_build.sh &&
+docker run -it --rm \
+  -v "/var/run/docker.sock:/var/run/docker.sock:rw" \
+  -v "${OUTDIR}:${OUTDIR}:rw" \
+  -v "${TEST_DIR}:/usr/local/bin/benchmarks/benchmarks.runfiles/nighthawk/benchmarks/external_tests/" \
+  --network=host \
+  --env NH_DOCKER_IMAGE="envoyproxy/nighthawk-dev:latest" \
+  --env ENVOY_DOCKER_IMAGE_TO_TEST="envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" \
+  --env TMPDIR="${OUTDIR}" \
+  oschaaf/benchmark-dev:latest ./benchmarks --log-cli-level=info -vvvv
+```
+
+# TODOs
+
+- Copy out the artifacts and push those to a gcp bucket. Current status:
+  - cpu profiles are dumped to tmp per test (named according to the test). ideally we'd
+    also dump flamegraph svg's
+- Enhance the UI. Currently we rely on CircleCI to directory-list our artifacts. This is
+  is pretty bare-bones, ideally we'd have something more fancy and we would be able to
+  visualize test data historically.
+- Use taskset/cpuset when starting processes.
+- Offer a docker image with binaries pre-built for profiling + the python test framework.
+- Multi-origin support. The integration tests support it, but this isn't fitted into
+  the benchmark framework yet. This _might_ just be a doc issue.
+
+# FUTURE
+
+- The current status can collect CPU profiles, but these are useless
+  without access to the binary & libs that have been involved in producing them.
+- profiling / flamegraphing via perf/bcc tools
+- Allow injection of other proxies: nginx, haproxy
+- Allow using alt clients, like Fortio & wrk2
+- An app that integrates fortios UI, pprof's web UI
+- Have a mode where nighthawk_test_server provides high-res control timings in its
+  access logs
+- The ability to repeat the runs multiple times and obtain stats, e.g. how much variance there is, mean, etc.
+- The ability to do A/B testing, similar to https://github.com/envoyproxy/envoy-perf/blob/master/siege/siege.py#L3.

benchmarks/benchmarks.py

@@ -1,24 +1,13 @@
 #!/usr/bin/env python3
-"""@package integration_test.py
-Entry point for our integration testing
+"""@package benchmarks.
+
+Entry point for benchmark execution.
 """
-import logging
 import os
 import sys
 import pytest
 
 if __name__ == '__main__':
   path = os.path.dirname(os.path.realpath(__file__))
-  test_selection_arg = sys.argv[1] if len(sys.argv) > 1 else ""
-  r = pytest.main([
-      "--rootdir=" + path,
-      "-vvvv",
-      "--showlocals",  # Don't abbreviate/truncate long values in asserts.
-      "-p",
-      "no:cacheprovider",  # Avoid a bunch of warnings on readonly filesystems 
-      "-x",
-      path,
-      "--log-cli-level",
-      "info"
-  ])
+  r = pytest.main(["--rootdir=" + path, "-x", path, "-p", "no:cacheprovider", *sys.argv])
   exit(r)

benchmarks/configurations/envoy_proxy.yaml

@@ -0,0 +1,42 @@
+admin:
+  access_log_path: $tmpdir/envoyproxy-admin-access.log
+  profile_path: $tmpdir/envoyproxy.prof
+  address:
+    socket_address: { address: $proxy_ip, port_value: 0 }
+static_resources:
+  listeners:
+    - address:
+        socket_address:
+          address: $proxy_ip
+          port_value: 0
+      filter_chains:
+        - filters:
+            - name: envoy.http_connection_manager
+              config:
+                generate_request_id: false
+                codec_type: auto
+                stat_prefix: ingress_http
+                route_config:
+                  name: local_route
+                  virtual_hosts:
+                    - name: service
+                      domains:
+                        - "*"
+                      routes:
+                        - match:
+                            prefix: /
+                          route:
+                            cluster: local_service
+                http_filters:
+                  - name: envoy.router
+                    config:
+                      dynamic_stats: false
+  clusters:
+    - name: local_service
+      connect_timeout: 0.25s
+      type: strict_dns
+      lb_policy: round_robin
+      hosts:
+        - socket_address:
+            address: $server_ip
+            port_value: $server_port

benchmarks/docker/Dockerfile-benchmark

@@ -0,0 +1,14 @@
+FROM frolvlad/alpine-python3
+
+RUN apk add docker openrc
+RUN rc-update add docker boot
+
+ADD benchmarks /usr/local/bin/benchmarks
+
+WORKDIR /usr/local/bin/benchmarks
+
+ENV ENVOY_PATH="envoy" \
+    RUNFILES_DIR="/usr/local/bin/benchmarks/benchmarks.runfiles/" \
+    ENVOY_IP_TEST_VERSIONS="v4only"
+
+CMD ["./benchmarks", "--help"]

benchmarks/docker/docker_build.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Builds a docker image nighthawk-dev:latest containing the stripped binaries
+# based on a pre-build bazel-bin directory (with "-c opt" set).
+
+set -eo pipefail
+set +x
+set -u
+
+# NOTE: explicit no -x for verbose commands. Because this is run in CI, doing so may result in
+# publishing sensitive information into public CI logs if someone makes a change in a
+# consuming script that is off guard.
+
+DOCKER_NAME="benchmark"
+DOCKER_IMAGE_PREFIX="${USER}/${DOCKER_NAME}"
+BAZEL_BIN="$(bazel info bazel-bin)"
+WORKSPACE="$(bazel info workspace)"
+bazel build //benchmarks:benchmarks
+TMP_DIR="$(mktemp -d)"
+PUSH=${PUSH:-0}  
+
+echo "Preparing docker build context in ${TMP_DIR}"
+# We flatten any symlinks to make this work on Linux (OSX doesn't need this)
+cp -Lr "${WORKSPACE}/benchmarks/docker/" "${TMP_DIR}/"
+cp -Lr "${BAZEL_BIN}/benchmarks" "${TMP_DIR}/"
+
+
+cd "${TMP_DIR}"
+echo "running docker build ... "
+docker build -f "${TMP_DIR}/docker/Dockerfile-${DOCKER_NAME}" -t "${DOCKER_IMAGE_PREFIX}-dev:latest" .
+rm -rf "${TMP_DIR}"
+echo "docker build finished"
+
+if [[ $PUSH == "1" ]]; then
+    echo "pushing ${DOCKER_IMAGE_PREFIX}-dev:latest .."
+    docker tag "${DOCKER_IMAGE_PREFIX}-dev:latest" "${DOCKER_IMAGE_PREFIX}-dev:latest"
+    docker push "${DOCKER_IMAGE_PREFIX}-dev:latest"
+    echo "docker image pushed"
+fi