Adding config factory

api/client/options.proto

@@ -18,6 +18,12 @@ message RequestOptions {
   google.protobuf.UInt32Value request_body_size = 3 [(validate.rules).uint32 = {lte: 4194304}];
 }
 
+// Used for providing multiple request options, especially for RequestSourcePlugins.
+message RequestOptionsList {
+  // Each option is used for a separate request to be sent by the requestSource.
+  repeated RequestOptions options = 1;
+}
+
 // Configures a remote gRPC source that will deliver to-be-replayed request data to Nighthawks
 // workers.
 message RequestSource {

api/request_source/BUILD

@@ -16,6 +16,18 @@ api_cc_py_proto_library(
     ],
 )
 
+api_cc_py_proto_library(
+    name = "request_source_plugin",
+    srcs = [
+        "request_source_plugin.proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@envoy_api//envoy/config/core/v3:pkg",
+        "@nighthawk//api/client:base",
+    ],
+)
+
 cc_grpc_library(
     name = "grpc_request_source_service_lib",
     srcs = [

api/request_source/request_source_plugin.proto

@@ -0,0 +1,36 @@
+// Config protos for the Request Source Plugin Config Factories.
+syntax = "proto3";
+
+package nighthawk.request_source;
+
+import "google/protobuf/wrappers.proto";
+import "validate/validate.proto";
+import "api/client/options.proto";
+
+// Configuration for FileBasedPluginRequestSource (plugin name:
+// "nighthawk.file-based-request-source-plugin")
+// The factory will load the RequestOptionsList from the file, and then passes it to the
+// requestSource it generates. The resulting request source will loop over the RequestOptionsList it
+// is passed.
+message FileBasedPluginConfig {
+  // The file_path is the path to a file that contains a RequestOptionList in json or yaml format.
+  string file_path = 1;
+  // The pluginfactory makes requestSources that will generate requests from the RequestOptionList
+  // up to num_requests number of times. If num_requests exceeds the number of RequestOptions in the
+  // RequestOptionList located in the file at file_path, it will loop. num_requests = 0 means no
+  // limit on the number of requests to be produced.
+  google.protobuf.UInt32Value num_requests = 2 [(validate.rules).uint32 = {gte: 0, lte: 1000000}];
+  // The pluginfactory will load the file located in file_path as long as it is below max_file_size,
+  // if it's too large it will throw an error.
+  google.protobuf.UInt32Value max_file_size = 3 [(validate.rules).uint32 = {lte: 1000000}];
+}
+
+// Configuration for StubPluginRequestSource (plugin name: "nighthawk.stub-request-source-plugin")
+// The plugin does nothing. This is for testing and comparison of the Request Source Plugin Factory
+// mechanism using a minimal version of plugin that does not require a more complicated proto or
+// file reading.
+message StubPluginConfig {
+  // test input value which is the only output value in the headers produced from the
+  // requestGenerator for the StubRequestSource.
+  google.protobuf.DoubleValue test_value = 1;
+}

include/nighthawk/request_source/BUILD

@@ -0,0 +1,24 @@
+load(
+    "@envoy//bazel:envoy_build_system.bzl",
+    "envoy_basic_cc_library",
+    "envoy_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_basic_cc_library(
+    name = "request_source_plugin_config_factory_lib",
+    hdrs = [
+        "request_source_plugin_config_factory.h",
+    ],
+    include_prefix = "nighthawk/request_source",
+    deps = [
+        "//api/request_source:request_source_plugin_cc_proto",
+        "//include/nighthawk/common:request_source_lib",
+        "@envoy//include/envoy/common:base_includes",
+        "@envoy//include/envoy/config:typed_config_interface",
+        "@envoy//source/common/api:api_lib_with_external_headers",
+    ],
+)

include/nighthawk/request_source/request_source_plugin_config_factory.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "envoy/api/api.h"
+#include "envoy/common/pure.h"
+#include "envoy/config/typed_config.h"
+
+#include "nighthawk/common/request_source.h"
+
+namespace Nighthawk {
+
+// A factory that must be implemented for each RequestSourcePlugin. It instantiates the specific
+// RequestSourcePlugin class after unpacking the plugin-specific config proto.
+class RequestSourcePluginConfigFactory : public Envoy::Config::TypedFactory {
+public:
+  ~RequestSourcePluginConfigFactory() override = default;
+  // All request source plugins will be in this category.
+  std::string category() const override { return "nighthawk.request_source_plugin"; }
+
+  // Instantiates the specific RequestSourcePlugin class. Casts |message| to Any, unpacks it to the
+  // plugin-specific proto, and passes the strongly typed proto to the plugin constructor.
+  //
+  // @param typed_config Any typed_config proto taken from the TypedExtensionConfig. This should be
+  // a type listed in request_source_plugin_config.proto
+  //
+  // @param api Api parameter that contains timesystem, filesystem, and threadfactory.
+  //
+  // @param header RequestHeaderMapPtr parameter that acts as a template header for the
+  // requestSource to modify when generating requests.
+  //
+  // @return RequestSourcePtr Pointer to the new instance of RequestSource.
+  //
+  // @throw Envoy::EnvoyException If the Any proto cannot be unpacked as the type expected by the
+  // plugin.
+  virtual RequestSourcePtr createRequestSourcePlugin(const Envoy::Protobuf::Message& typed_config,
+                                                     Envoy::Api::Api& api,
+                                                     Envoy::Http::RequestHeaderMapPtr header) PURE;
+};
+
+} // namespace Nighthawk

source/request_source/BUILD

@@ -0,0 +1,33 @@
+load(
+    "@envoy//bazel:envoy_build_system.bzl",
+    "envoy_cc_library",
+    "envoy_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_cc_library(
+    name = "request_options_list_plugin_impl",
+    srcs = [
+        "request_options_list_plugin_impl.cc",
+    ],
+    hdrs = [
+        "request_options_list_plugin_impl.h",
+    ],
+    repository = "@envoy",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//include/nighthawk/request_source:request_source_plugin_config_factory_lib",
+        "//source/common:nighthawk_common_lib",
+        "//source/common:request_impl_lib",
+        "//source/common:request_source_impl_lib",
+        "@envoy//source/common/common:thread_lib_with_external_headers",
+        "@envoy//source/common/protobuf:message_validator_lib_with_external_headers",
+        "@envoy//source/common/protobuf:protobuf_with_external_headers",
+        "@envoy//source/common/protobuf:utility_lib_with_external_headers",
+        "@envoy//source/exe:platform_header_lib_with_external_headers",
+        "@envoy//source/exe:platform_impl_lib",
+    ],
+)

source/request_source/request_options_list_plugin_impl.cc

@@ -0,0 +1,89 @@
+#include "request_source/request_options_list_plugin_impl.h"
+
+#include "external/envoy/source/common/protobuf/message_validator_impl.h"
+#include "external/envoy/source/common/protobuf/utility.h"
+#include "external/envoy/source/exe/platform_impl.h"
+
+#include "api/client/options.pb.h"
+
+#include "common/request_impl.h"
+#include "common/request_source_impl.h"
+
+namespace Nighthawk {
+std::string OptionsListFromFileRequestSourceFactory::name() const {
+  return "nighthawk.file-based-request-source-plugin";
+}
+
+Envoy::ProtobufTypes::MessagePtr OptionsListFromFileRequestSourceFactory::createEmptyConfigProto() {
+  return std::make_unique<nighthawk::request_source::FileBasedPluginConfig>();
+}
+
+RequestSourcePtr OptionsListFromFileRequestSourceFactory::createRequestSourcePlugin(
+    const Envoy::Protobuf::Message& message, Envoy::Api::Api& api,
+    Envoy::Http::RequestHeaderMapPtr header) {
+  const auto& any = dynamic_cast<const Envoy::ProtobufWkt::Any&>(message);
+  nighthawk::request_source::FileBasedPluginConfig config;
+  Envoy::MessageUtil util;
+
+  util.unpackTo(any, config);
+  if (api.fileSystem().fileSize(config.file_path()) > config.max_file_size().value()) {
+    throw NighthawkException("file size must be less than max_file_size");
+  }
+
+  // Locking to avoid issues with multiple threads reading the same file.
+  {
+    Envoy::Thread::LockGuard lock_guard(file_lock_);
+    // Reading the file only the first time.
+    if (options_list_.options_size() == 0) {
+      util.loadFromFile(config.file_path(), options_list_,
+                        Envoy::ProtobufMessage::getStrictValidationVisitor(), api, true);
+    }
+  }
+  return std::make_unique<RequestOptionsListRequestSource>(config.num_requests().value(),
+                                                           std::move(header), options_list_);
+}
+
+REGISTER_FACTORY(OptionsListFromFileRequestSourceFactory, RequestSourcePluginConfigFactory);
+
+RequestOptionsListRequestSource::RequestOptionsListRequestSource(
+    const uint32_t total_requests, Envoy::Http::RequestHeaderMapPtr header,
+    const nighthawk::client::RequestOptionsList& options_list)
+    : header_(std::move(header)), options_list_(options_list), total_requests_(total_requests) {}
+
+RequestGenerator RequestOptionsListRequestSource::get() {
+  request_count_.push_back(0);
+  uint32_t& lambda_counter = request_count_.back();
+  RequestGenerator request_generator = [this, lambda_counter]() mutable -> RequestPtr {
+    // if request_max is 0, then we never stop generating requests.
+    if (lambda_counter >= total_requests_ && total_requests_ != 0) {
+      return nullptr;
+    }
+
+    // Increment the counter and get the request_option from the list for the current iteration.
+    const uint32_t index = lambda_counter % options_list_.options_size();
+    nighthawk::client::RequestOptions request_option = options_list_.options().at(index);
+    ++lambda_counter;
+
+    // Initialize the header with the values from the default header.
+    Envoy::Http::RequestHeaderMapPtr header = Envoy::Http::RequestHeaderMapImpl::create();
+    Envoy::Http::HeaderMapImpl::copyFrom(*header, *header_);
+
+    // Override the default values with the values from the request_option
+    header->setMethod(envoy::config::core::v3::RequestMethod_Name(request_option.request_method()));
+    const uint32_t content_length = request_option.request_body_size().value();
+    if (content_length > 0) {
+      header->setContentLength(content_length);
+    }
+    for (const envoy::config::core::v3::HeaderValueOption& option_header :
+         request_option.request_headers()) {
+      auto lower_case_key = Envoy::Http::LowerCaseString(std::string(option_header.header().key()));
+      header->setCopy(lower_case_key, std::string(option_header.header().value()));
+    }
+    return std::make_unique<RequestImpl>(std::move(header));
+  };
+  return request_generator;
+}
+
+void RequestOptionsListRequestSource::initOnThread() {}
+
+} // namespace Nighthawk

source/request_source/request_options_list_plugin_impl.h

@@ -0,0 +1,84 @@
+// Implementations of RequestSourceConfigFactories that make a RequestOptionsListRequestSource.
+#pragma once
+
+#include "envoy/registry/registry.h"
+
+#include "nighthawk/request_source/request_source_plugin_config_factory.h"
+
+#include "external/envoy/source/common/common/lock_guard.h"
+#include "external/envoy/source/common/common/thread.h"
+
+#include "api/client/options.pb.h"
+#include "api/request_source/request_source_plugin.pb.h"
+
+#include "common/uri_impl.h"
+
+namespace Nighthawk {
+
+// Sample Request Source for small RequestOptionsLists. Loads a copy of the RequestOptionsList in
+// memory and replays them.
+// @param total_requests The number of requests the requestGenerator produced by get() will
+// generate. 0 means it is unlimited.
+// @param header the default header that will be overridden by values taken from the options_list,
+// any values not overridden will be used.
+// @param options_list This is const because it is intended to be shared by multiple threads. The
+// RequestGenerator produced by get() will use options from the options_list to overwrite values in
+// the default header, and create new requests. if total_requests is greater than the length of
+// options_list, it will loop. This is not thread safe.
+class RequestOptionsListRequestSource : public RequestSource {
+public:
+  RequestOptionsListRequestSource(const uint32_t total_requests,
+                                  Envoy::Http::RequestHeaderMapPtr header,
+                                  const nighthawk::client::RequestOptionsList& options_list);
+
+  // This get function is not thread safe, because multiple threads calling get simultaneously will
+  // result in a collision as it attempts to update its request_count_.
+  RequestGenerator get() override;
+
+  // default implementation
+  void initOnThread() override;
+
+private:
+  Envoy::Http::RequestHeaderMapPtr header_;
+  const nighthawk::client::RequestOptionsList& options_list_;
+  std::vector<uint32_t> request_count_;
+  const uint32_t total_requests_;
+};
+
+// Factory that creates a RequestOptionsListRequestSource from a FileBasedPluginConfig proto.
+// Registered as an Envoy plugin.
+// Implementation of RequestSourceConfigFactory which produces a RequestSource that keeps an
+// RequestOptionsList in memory, and loads it with the RequestOptions taken from a file. All plugins
+// configuration are specified in the request_source_plugin.proto. This class is not thread-safe,
+// because it loads its RequestOptionlist in memory from a file when first called.
+// Usage: assume you are passed an appropriate Any type object called config, an Api object called
+// api, and a default header called header. auto& config_factory =
+//     Envoy::Config::Utility::getAndCheckFactoryByName<RequestSourcePluginConfigFactory>(
+//         "nighthawk.file-based-request-source-plugin");
+// RequestSourcePtr plugin =
+//     config_factory.createRequestSourcePlugin(config, std::move(api), std::move(header));
+class OptionsListFromFileRequestSourceFactory : public virtual RequestSourcePluginConfigFactory {
+public:
+  std::string name() const override;
+
+  Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override;
+
+  // This implementation is not thread safe. Only the first call to createRequestSourcePlugin will
+  // load the file from memory and subsequent calls just make a copy of the options_list that was
+  // already loaded. The OptionsListFromFileRequestSourceFactory will not work with multiple
+  // different files for this reason.
+  // This method will also error if the file can not be loaded correctly, e.g. the file is too large
+  // or could not be found.
+  RequestSourcePtr createRequestSourcePlugin(const Envoy::Protobuf::Message& message,
+                                             Envoy::Api::Api& api,
+                                             Envoy::Http::RequestHeaderMapPtr header) override;
+
+private:
+  Envoy::Thread::MutexBasicLockable file_lock_;
+  nighthawk::client::RequestOptionsList options_list_;
+};
+
+// This factory will be activated through RequestSourceFactory in factories.h
+DECLARE_FACTORY(OptionsListFromFileRequestSourceFactory);
+
+} // namespace Nighthawk

test/request_source/BUILD

@@ -0,0 +1,48 @@
+load(
+    "@envoy//bazel:envoy_build_system.bzl",
+    "envoy_cc_test",
+    "envoy_cc_test_library",
+    "envoy_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_cc_test_library(
+    name = "stub_plugin_impl",
+    srcs = [
+        "stub_plugin_impl.cc",
+    ],
+    hdrs = [
+        "stub_plugin_impl.h",
+    ],
+    repository = "@envoy",
+    deps = [
+        "//include/nighthawk/request_source:request_source_plugin_config_factory_lib",
+        "//source/common:nighthawk_common_lib",
+        "//source/common:request_impl_lib",
+        "//source/common:request_source_impl_lib",
+        "@envoy//source/common/protobuf:message_validator_lib_with_external_headers",
+        "@envoy//source/common/protobuf:protobuf_with_external_headers",
+        "@envoy//source/common/protobuf:utility_lib_with_external_headers",
+        "@envoy//source/exe:platform_header_lib_with_external_headers",
+        "@envoy//source/exe:platform_impl_lib",
+    ],
+)
+
+envoy_cc_test(
+    name = "request_source_plugin_test",
+    srcs = ["request_source_plugin_test.cc"],
+    data = [
+        "test_data/test-config.yaml",
+    ],
+    repository = "@envoy",
+    deps = [
+        "//source/request_source:request_options_list_plugin_impl",
+        "//test/request_source:stub_plugin_impl",
+        "//test/test_common:environment_lib",
+        "@envoy//source/common/config:utility_lib_with_external_headers",
+        "@envoy//test/mocks/api:api_mocks",
+    ],
+)