sync to main

.bazelrc

@@ -22,8 +22,6 @@ startup --host_jvm_args=-Xmx2g
 build --workspace_status_command="bash bazel/get_workspace_status"
 build --experimental_strict_action_env=true
 build --host_force_python=PY3
-build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a
-build --action_env=BAZEL_LINKOPTS=-lm
 build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11
 build --javabase=@bazel_tools//tools/jdk:remote_jdk11
 build --enable_platform_specific_config
@@ -35,6 +33,8 @@ build:linux --cxxopt=-std=c++17
 build:linux --conlyopt=-fexceptions
 build:linux --fission=dbg,opt
 build:linux --features=per_object_debug_info
+build:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a
+build:linux --action_env=BAZEL_LINKOPTS=-lm
 
 # We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace.
 build --define absl=1
@@ -83,6 +83,7 @@ build:clang-asan --linkopt -fuse-ld=lld
 
 # macOS ASAN/UBSAN
 build:macos --cxxopt=-std=c++17
+
 build:macos-asan --config=asan
 # Workaround, see https://github.com/bazelbuild/bazel/issues/6932
 build:macos-asan --copt -Wno-macro-redefined
@@ -154,7 +155,7 @@ build:coverage --strategy=CoverageReport=sandboxed,local
 build:coverage --experimental_use_llvm_covmap
 build:coverage --collect_code_coverage
 build:coverage --test_tag_filters=-nocoverage
-build:coverage --instrumentation_filter="//source(?!/common/chromium_url|/extensions/quic_listeners/quiche/platform)[/:],//include[/:]"
+build:coverage --instrumentation_filter="//source(?!/common/chromium_url|/common/quic/platform)[/:],//include[/:]"
 build:test-coverage --test_arg="-l trace"
 build:fuzz-coverage --config=plain-fuzzer
 build:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh
@@ -257,7 +258,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl
 
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/main/toolchains/rbe_toolchains_config.bzl#L8
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:e33c93e6d79804bf95ff80426d10bdcc9096c785
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:3d0491e2034287959a292806e3891fd0b7dd2703
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker
@@ -363,7 +364,6 @@ build:clang-cl --copt="-Wno-builtin-macro-redefined"
 #     overrides a member function but is not marked 'override'
 #   MOCK_METHOD(void, addCallbacks, (StreamCallbacks & callbacks));
 build:clang-cl --copt="-Wno-inconsistent-missing-override"
-build:clang-cl --action_env=USE_CLANG_CL=1
 
 # Defaults to 'auto' - Off for windows, so override to linux behavior
 build:windows --enable_runfiles=yes

.circleci/config.yml

@@ -1,6 +1,6 @@
 references:
   envoy-build-image: &envoy-build-image # March 8th, 2021
-    envoyproxy/envoy-build-ubuntu:e33c93e6d79804bf95ff80426d10bdcc9096c785
+    envoyproxy/envoy-build-ubuntu:3d0491e2034287959a292806e3891fd0b7dd2703
 version: 2
 jobs:
   build:

.vscode/tasks.json

@@ -223,6 +223,7 @@
       "options": [
           "//test/sink:sink_test",
           "//test/sink:nighthawk_sink_client_test",
+          "//test/sink:sink_service_test",
           "//test/server:http_time_tracking_filter_integration_test",
           "//test/server:http_test_server_filter_integration_test",
           "//test/server:http_filter_base_test",

api/client/options.proto

@@ -106,7 +106,7 @@ message H1ConnectionReuseStrategy {
 
 // TODO(oschaaf): Ultimately this will be a load test specification. The fact that it
 // can arrive via CLI is just a concrete detail. Change this to reflect that.
-// highest unused number is 39
+// highest unused number is 107
 message CommandLineOptions {
   // The target requests-per-second rate. Default: 5.
   google.protobuf.UInt32Value requests_per_second = 1
@@ -225,4 +225,10 @@ message CommandLineOptions {
   google.protobuf.Timestamp scheduled_start = 105;
 
   reserved 38; // deprecated
+  // Provide a unique execution id. Will be reflected in the output when set.
+  // This is intended for future use in horizontally scaled scenarios in mind.
+  // When populating this field, it is recommended to use unique identifiers for forward
+  // compatibility purposes. In the future, this field may be auto-populated when left unset and
+  // circumstances mandate so (distributed load test execution).
+  google.protobuf.StringValue execution_id = 106;
 }

api/distributor/distributor.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package nighthawk;
 
+import "google/rpc/status.proto";
+
 import "envoy/config/core/v3/address.proto";
 
 import "api/client/service.proto";
@@ -16,8 +18,18 @@ message DistributedRequest {
   repeated envoy.config.core.v3.Address services = 3 [(validate.rules).repeated .min_items = 1];
 }
 
+message DistributedServiceResponse {
+  oneof distributed_response_type {
+    google.rpc.Status error = 1;
+    nighthawk.client.ExecutionResponse execution_response = 2;
+  }
+  // The service that is associated to this fragment.
+  envoy.config.core.v3.Address service = 3;
+}
+
 // Carries responses associated with a DistributedRequest.
 message DistributedResponse {
+  repeated DistributedServiceResponse service_response = 1;
 }
 
 // Service which distributes messages to one or more other services for handling, and streams back

bazel/repositories.bzl

@@ -1,7 +1,7 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-ENVOY_COMMIT = "031f75dd113e2f7be41b94a0365145e7bf1e6c12"  # March 14th, 2021
-ENVOY_SHA = "20f5f98cc9a5c1ddd2c3de434b019273e3dbe02afe52396b90d74fab38dba4ba"
+ENVOY_COMMIT = "751be72c2eb967ff03759b6e50dde106c8cf5c84"  # April 19, 2021
+ENVOY_SHA = "43a538a299f0169cd6b15d3a8d757b51870487bb7e50766108f0917e5974f412"
 
 HDR_HISTOGRAM_C_VERSION = "0.11.2"  # October 12th, 2020
 HDR_HISTOGRAM_C_SHA = "637f28b5f64de2e268131e4e34e6eef0b91cf5ff99167db447d9b2825eae6bad"

ci/do_ci.sh

@@ -17,6 +17,7 @@ export BAZEL_OPTIONS=${BAZEL_OPTIONS:=""}
 export BAZEL_BUILD_EXTRA_OPTIONS=${BAZEL_BUILD_EXTRA_OPTIONS:=""}
 export SRCDIR=${SRCDIR:="${PWD}"}
 export CLANG_FORMAT=clang-format
+export NIGHTHAWK_BUILD_ARCH=$(uname -m)
 
 function do_build () {
     bazel build $BAZEL_BUILD_OPTIONS //:nighthawk
@@ -41,7 +42,7 @@ function do_clang_tidy() {
 
 function do_unit_test_coverage() {
     export TEST_TARGETS="//test/... -//test:python_test"
-    export COVERAGE_THRESHOLD=94.2
+    export COVERAGE_THRESHOLD=94.1
     echo "bazel coverage build with tests ${TEST_TARGETS}"
     test/run_nighthawk_bazel_coverage.sh ${TEST_TARGETS}
     exit 0
@@ -60,6 +61,8 @@ function setup_gcc_toolchain() {
     export CC=gcc
     export CXX=g++
     export BAZEL_COMPILER=gcc
+    [[ "${NIGHTHAWK_BUILD_ARCH}" == "aarch64" ]] && BAZEL_BUILD_OPTIONS="$BAZEL_BUILD_OPTIONS --copt -march=armv8-a+crypto"
+    [[ "${NIGHTHAWK_BUILD_ARCH}" == "aarch64" ]] && BAZEL_TEST_OPTIONS="$BAZEL_TEST_OPTIONS --copt -march=armv8-a+crypto"
     echo "$CC/$CXX toolchain configured"
 }
 

ci/docker/Dockerfile-nighthawk-benchmark

@@ -8,7 +8,7 @@ WORKDIR /usr/local/bin/benchmarks
 
 COPY benchmarks /usr/local/bin/benchmarks/
 
-RUN apk add --no-cache docker=20.10.3-r0 openrc=0.42.1-r19 python3>=3.8.7-r0
+RUN apk add --no-cache docker>=20.10.3-r0 openrc>=0.42.1-r19 python3>=3.8.7-r0
 RUN rc-update add docker boot
 
 RUN if [ ! -e /usr/bin/python ]; then ln -sf python3 /usr/bin/python; fi && \

ci/run_envoy_docker.sh

@@ -22,10 +22,10 @@ if is_windows; then
   # CI sets it to a Linux-specific value. Undo this once https://github.com/envoyproxy/envoy/issues/13272
   # is resolved.
   ENVOY_DOCKER_OPTIONS=()
-  DEFAULT_ENVOY_DOCKER_BUILD_DIR=C:/Windows/Temp/envoy-docker-build
+  # Replace MSYS style drive letter (/c/) with Windows drive letter designation (C:/)
+  DEFAULT_ENVOY_DOCKER_BUILD_DIR=$(echo "${TEMP}" | sed -E "s#^/([a-zA-Z])/#\1:/#")/envoy-docker-build
   BUILD_DIR_MOUNT_DEST=C:/build
-  # Replace MSYS style drive letter (/c/) with driver letter designation (C:/)
-  SOURCE_DIR=$(echo "${PWD}" | sed -E "s#/([a-zA-Z])/#\1:/#")
+  SOURCE_DIR=$(echo "${PWD}" | sed -E "s#^/([a-zA-Z])/#\1:/#")
   SOURCE_DIR_MOUNT_DEST=C:/source
   START_COMMAND=("bash" "-c" "cd source && $*")
 else

docs/root/examples/HEADER_BASED_LATENCY.md

@@ -0,0 +1,49 @@
+# Header based latency tracking
+
+##  Tracking latencies contained in response headers.
+
+The api allows one to [point out a response header](https://github.com/envoyproxy/nighthawk/blob/211b3b53f60d5ed3855d15eb8a4c2d7a3edc0724/api/client/options.proto#L222) whose values will be interpreted as latencies, to be tracked in a histogram by the client.
+
+A sample invokation of the CLI to track latencies communicated via a response header called `x-origin-request-receipt-delta` looks like:
+
+```bash
+./nighthawk_client --latency-response-header-name x-origin-request-receipt-delta http://foo/
+```
+
+## Tracking upstream processing time
+
+One use case for tracking latencies via response headers, is measuring upstream processing time. For example, Envoy proxy can be set up to emit `x-envoy-upstream-service-time`, which allows tracking that as a component of the overal request/reply time. Also, when using synhetic delays on the test server, the theoretical expected distribution of the synthetic delays should ideally closely match the actual observed response latencies, and any unexpected divergence might be an interesting data point.
+
+## Tracking request-arrival timing deltas using the test server
+
+Another use-case is tracking request-arrival time deltas using a feature of the test server.
+
+```yaml
+# The time-tracking extension will emit request-arrival timing deltas in a response header.
+    http_filters:
+    - name: time-tracking
+    typed_config:
+        "@type": type.googleapis.com/nighthawk.server.ResponseOptions
+        emit_previous_request_delta_in_response_header: x-origin-request-receipt-delta
+    - name: test-server
+    typed_config:
+        "@type": type.googleapis.com/nighthawk.server.ResponseOptions
+        response_body_size: 10
+```
+
+The resulting histogram of arrival timing-deltas can be compared to the configured distribution of request-release timings. In an ideal world, 1000qps would yield a flat 1ms delta, and adding uniform jitter would yield perfect bell curve / normal distribution.
+
+A real world example of the output for this using straight 1000 qps linear pacing in a baseline test against the test server looks like:
+
+```
+  0.5         2519        0s 000ms 999us 
+  0.75        3767        0s 001ms 001us 
+  0.8         4002        0s 001ms 002us 
+  0.9         4508        0s 001ms 003us 
+  0.95        4756        0s 001ms 004us 
+  0.990625    4953        0s 001ms 005us 
+  0.99902344  4995        0s 001ms 015us
+```
+
+The way this changes when introducing a proxy can be interesting when comparing to baseline, as this divergence may say something about distortion introduced on the path from the client to the test server, and perhaps can even be used to quantify that.
+

include/nighthawk/client/BUILD

@@ -8,17 +8,33 @@ licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_basic_cc_library(
+    name = "options_lib",
+    hdrs = [
+        "options.h",
+    ],
+    include_prefix = "nighthawk/client",
+    deps = [
+        "//api/client:base_cc_proto",
+        "//include/nighthawk/common:base_includes",
+        "@envoy//include/envoy/common:time_interface",
+        "@envoy//source/common/common:minimal_logger_lib_with_external_headers",
+        "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
+        "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto",
+    ],
+)
+
 envoy_basic_cc_library(
     name = "client_includes",
     hdrs = [
         "benchmark_client.h",
         "client_worker.h",
         "factories.h",
-        "options.h",
         "process.h",
     ],
     include_prefix = "nighthawk/client",
     deps = [
+        ":options_lib",
         ":output_collector_lib",
         ":output_formatter_lib",
         "//api/client:base_cc_proto",

include/nighthawk/client/options.h

@@ -77,6 +77,8 @@ class Options {
   virtual std::string responseHeaderWithLatencyInput() const PURE;
 
   virtual absl::optional<Envoy::SystemTime> scheduled_start() const PURE;
+  virtual absl::optional<std::string> executionId() const PURE;
+
   /**
    * Converts an Options instance to an equivalent CommandLineOptions instance in terms of option
    * values.

include/nighthawk/client/output_formatter.h

@@ -8,6 +8,9 @@
 
 #include "api/client/output.pb.validate.h"
 
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+
 namespace Nighthawk {
 namespace Client {
 
@@ -19,10 +22,11 @@ class OutputFormatter {
   virtual ~OutputFormatter() = default;
 
   /**
-   * @return std::string serialized representation of output. The specific format depends
-   * on the derived class, for example human-readable or json.
+   * @return absl::StatusOr<std::string> serialized representation of output, or an error.
+   * The specific format depends on the derived class, for example human-readable or json.
    */
-  virtual std::string formatProto(const nighthawk::client::Output& output) const PURE;
+  virtual absl::StatusOr<std::string>
+  formatProto(const nighthawk::client::Output& output) const PURE;
 };
 
 using OutputFormatterPtr = std::unique_ptr<OutputFormatter>;

include/nighthawk/sink/sink.h

@@ -30,7 +30,7 @@ class Sink {
    * @return absl::Status Indicates if the operation succeeded or not.
    */
   virtual absl::Status
-  StoreExecutionResultPiece(const nighthawk::client::ExecutionResponse& response) const PURE;
+  StoreExecutionResultPiece(const nighthawk::client::ExecutionResponse& response) PURE;
 
   /**
    * Attempt to load a vector of ExecutionResponse instances associated to an execution id.

internal_proto/README.md

@@ -0,0 +1,3 @@
+# Internal proto definitions
+
+Warning: this directory contains proto defininitions which are used internally by Nighthawk, and are subject to change.

internal_proto/statistic/BUILD

@@ -0,0 +1,11 @@
+load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library")
+
+licenses(["notice"])  # Apache 2
+
+api_cc_py_proto_library(
+    name = "statistic",
+    srcs = [
+        "statistic.proto",
+    ],
+    visibility = ["//visibility:public"],
+)

internal_proto/statistic/statistic.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package nighthawk.internal;
+
+// This package contains proto definitions which allow serialization / deserialization
+// of statistics implementations. Naming maps 1:1 with the statistics implementations
+// over at source/common/statistics_impl.h. See the code & doc comments there for further
+// information about the corresponding statistics implementations.
+
+message SimpleStatistic {
+  uint64 count = 1;
+  string id = 2;
+  uint64 min = 5;
+  uint64 max = 6;
+  double sum_x = 7;
+  double sum_x_2 = 8;
+}
+
+message StreamingStatistic {
+  uint64 count = 1;
+  string id = 2;
+  uint64 min = 5;
+  uint64 max = 6;
+  double mean = 7;
+  double accumulated_variance = 8;
+}

requirements.txt

@@ -2,6 +2,6 @@ requests==2.24.0
 pytest==6.0.1
 pytest-dependency==0.5.1
 pytest-xdist==1.34.0
-pyyaml==5.3.1
+pyyaml==5.4
 zipp==3.1.0
 importlib_metadata==1.7.0

source/adaptive_load/adaptive_load_client_main.cc

@@ -39,7 +39,9 @@ namespace {
  */
 void WriteFileOrThrow(Envoy::Filesystem::Instance& filesystem, absl::string_view path,
                       absl::string_view contents) {
-  Envoy::Filesystem::FilePtr file = filesystem.createFile(std::string(path));
+  Envoy::Filesystem::FilePathAndType file_path_and_type(Envoy::Filesystem::DestinationType::File,
+                                                        path);
+  Envoy::Filesystem::FilePtr file = filesystem.createFile(file_path_and_type);
   const Envoy::Api::IoCallBoolResult open_result =
       file->open(((1 << Envoy::Filesystem::File::Operation::Write)) |
                  (1 << (Envoy::Filesystem::File::Operation::Create)));