[fuzz] Added random load balancer fuzz

test/common/upstream/BUILD

@@ -242,6 +242,54 @@ envoy_cc_test(
     ],
 )
 
+envoy_cc_test_library(
+    name = "load_balancer_fuzz_lib",
+    srcs = ["load_balancer_fuzz_base.cc"],
+    hdrs = ["load_balancer_fuzz_base.h"],
+    deps = [
+        ":load_balancer_fuzz_proto_cc_proto",
+        ":utility_lib",
+        "//source/common/upstream:load_balancer_lib",
+        "//test/fuzz:random_lib",
+        "//test/mocks:common_lib",
+        "//test/mocks/runtime:runtime_mocks",
+        "//test/mocks/upstream:cluster_info_mocks",
+        "//test/mocks/upstream:host_set_mocks",
+        "//test/mocks/upstream:load_balancer_context_mock",
+        "//test/mocks/upstream:priority_set_mocks",
+        "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_proto_library(
+    name = "load_balancer_fuzz_proto",
+    srcs = ["load_balancer_fuzz.proto"],
+    deps = [
+        "//test/fuzz:common_proto",
+        "@envoy_api//envoy/config/cluster/v3:pkg",
+    ],
+)
+
+envoy_proto_library(
+    name = "random_load_balancer_fuzz_proto",
+    srcs = ["random_load_balancer_fuzz.proto"],
+    deps = [
+        "//test/common/upstream:load_balancer_fuzz_proto",
+    ],
+)
+
+envoy_cc_fuzz_test(
+    name = "random_load_balancer_fuzz_test",
+    srcs = ["random_load_balancer_fuzz_test.cc"],
+    corpus = "//test/common/upstream:random_load_balancer_corpus",
+    deps = [
+        ":load_balancer_fuzz_lib",
+        ":load_balancer_fuzz_proto_cc_proto",
+        ":random_load_balancer_fuzz_proto_cc_proto",
+        ":utility_lib",
+    ],
+)
+
 envoy_cc_test(
     name = "load_balancer_simulation_test",
     srcs = ["load_balancer_simulation_test.cc"],

test/common/upstream/load_balancer_fuzz.proto

@@ -0,0 +1,61 @@
+syntax = "proto3";
+
+package test.common.upstream;
+
+import "validate/validate.proto";
+import "envoy/config/cluster/v3/cluster.proto";
+import "google/protobuf/empty.proto";
+
+message UpdateHealthFlags {
+  // The host priority determines what host set within the priority set which will get updated.
+  uint64 host_priority = 1;
+  // These will determine how many hosts will get placed into health hosts, degraded hosts, and
+  // excluded hosts from the full host list.
+  uint32 num_healthy_hosts = 2;
+  uint32 num_degraded_hosts = 3;
+  uint32 num_excluded_hosts = 4;
+  // This is used to determine which hosts get marked as healthy, degraded, and excluded.
+  bytes random_bytestring = 5 [(validate.rules).bytes = {min_len: 1, max_len: 256}];
+}
+
+message LbAction {
+  oneof action_selector {
+    option (validate.required) = true;
+    // This updates the health flags of hosts at a certain priority level. The number of hosts in each priority level/in localities is static,
+    // as untrusted upstreams cannot change that, and can only change their health flags.
+    UpdateHealthFlags update_health_flags = 1;
+    // Prefetches a host using the encapsulated specific load balancer.
+    google.protobuf.Empty prefetch = 2;
+    // Chooses a host using the encapsulated specific load balancer.
+    google.protobuf.Empty choose_host = 3;
+  }
+}
+
+message SetupPriorityLevel {
+  uint32 num_hosts_in_priority_level = 1 [(validate.rules).uint32.lte = 500];
+  // Doesn't need a cap on sum, as subset utility class handles "overflowing" subsets
+  uint32 num_hosts_locality_one = 2 [(validate.rules).uint32.lte = 500];
+  uint32 num_hosts_locality_two = 3 [(validate.rules).uint32.lte = 500];
+  // Hard cap at 3 localities for simplicity
+  uint32 num_hosts_locality_three = 4 [(validate.rules).uint32.lte = 500];
+  // For choosing which hosts go in which locality
+  bytes random_bytestring = 5 [(validate.rules).bytes = {min_len: 1, max_len: 256}];
+}
+
+// This message represents what LoadBalancerFuzzBase will interact with, performing setup of host sets and calling into load balancers.
+// The logic that this message represents and the base class for load balancing fuzzing will be logic that maps to all types of load balancing
+// and can be used in a modular way at the highest level for each load balancer.
+message LoadBalancerTestCase {
+  envoy.config.cluster.v3.Cluster.CommonLbConfig common_lb_config = 1
+      [(validate.rules).message.required = true];
+  repeated LbAction actions = 2;
+
+  // Each generated integer will cause the fuzzer to initialize hosts at a certain priority level, each integer generated adding a priority
+  // level with integer generated hosts in that new priority level. Capped at 20 for simplicity.
+  repeated SetupPriorityLevel setup_priority_levels = 3
+      [(validate.rules).repeated = {min_items: 1, max_items: 20}];
+
+  // This number is used to instantiate the prng. The prng takes the place of random() calls, allowing a representative random distribution
+  // which is also deterministic.
+  uint64 seed_for_prng = 4 [(validate.rules).uint64.gt = 0];
+}

test/common/upstream/load_balancer_fuzz_base.cc

@@ -0,0 +1,193 @@
+#include "test/common/upstream/load_balancer_fuzz_base.h"
+
+#include "test/common/upstream/utility.h"
+
+namespace Envoy {
+namespace Upstream {
+
+namespace {
+
+constexpr uint32_t MaxNumHostsPerPriorityLevel = 256;
+
+} // namespace
+
+void LoadBalancerFuzzBase::initializeASingleHostSet(
+    const test::common::upstream::SetupPriorityLevel& setup_priority_level,
+    const uint8_t priority_level) {
+  const uint32_t num_hosts_in_priority_level = setup_priority_level.num_hosts_in_priority_level();
+  ENVOY_LOG_MISC(trace, "Will attempt to initialize host set {} with {} hosts.", priority_level,
+                 num_hosts_in_priority_level);
+  MockHostSet& host_set = *priority_set_.getMockHostSet(priority_level);
+  uint32_t hosts_made = 0;
+  // Cap each host set at 256 hosts for efficiency - Leave port clause in for future changes
+  while (hosts_made < std::min(num_hosts_in_priority_level, MaxNumHostsPerPriorityLevel) &&
+         port_ < 65535) {
+    host_set.hosts_.push_back(makeTestHost(info_, "tcp://127.0.0.1:" + std::to_string(port_)));
+    ++port_;
+    ++hosts_made;
+  }
+
+  Fuzz::ProperSubsetSelector subset_selector(setup_priority_level.random_bytestring());
+
+  const std::vector<std::vector<uint8_t>> localities = subset_selector.constructSubsets(
+      {setup_priority_level.num_hosts_locality_one(), setup_priority_level.num_hosts_locality_two(),
+       setup_priority_level.num_hosts_locality_three()},
+      std::min(num_hosts_in_priority_level, MaxNumHostsPerPriorityLevel));
+
+  HostVector locality_one = {};
+  HostVector locality_two = {};
+  HostVector locality_three = {};
+  // Used to index into correct locality in iteration through subsets
+  std::array<HostVector, 3> locality_indexes = {locality_one, locality_two, locality_three};
+
+  for (uint8_t locality = 0; locality < locality_indexes.size(); locality++) {
+    for (uint8_t index : localities[locality]) {
+      locality_indexes[locality].push_back(host_set.hosts_[index]);
+      locality_indexes_[index] = locality;
+    }
+  }
+
+  ENVOY_LOG_MISC(trace, "Added these hosts to locality 1: ", absl::StrJoin(localities[0], " "));
+  ENVOY_LOG_MISC(trace, "Added these hosts to locality 2: ", absl::StrJoin(localities[1], " "));
+  ENVOY_LOG_MISC(trace, "Added these hosts to locality 3: ", absl::StrJoin(localities[2], " "));
+
+  host_set.hosts_per_locality_ = makeHostsPerLocality({locality_one, locality_two, locality_three});
+}
+
+// Initializes random and fixed host sets
+void LoadBalancerFuzzBase::initializeLbComponents(
+    const test::common::upstream::LoadBalancerTestCase& input) {
+  random_.initializeSeed(input.seed_for_prng());
+  for (uint8_t priority_of_host_set = 0;
+       priority_of_host_set < input.setup_priority_levels().size(); ++priority_of_host_set) {
+    initializeASingleHostSet(input.setup_priority_levels().at(priority_of_host_set),
+                             priority_of_host_set);
+  }
+  num_priority_levels_ = input.setup_priority_levels().size();
+}
+
+// Updating host sets is shared amongst all the load balancer tests. Since logically, we're just
+// setting the mock priority set to have certain values, and all load balancers interface with host
+// sets and their health statuses, this action maps to all load balancers.
+void LoadBalancerFuzzBase::updateHealthFlagsForAHostSet(const uint64_t host_priority,
+                                                        const uint32_t num_healthy_hosts,
+                                                        const uint32_t num_degraded_hosts,
+                                                        const uint32_t num_excluded_hosts,
+                                                        const std::string random_bytestring) {
+  const uint8_t priority_of_host_set = host_priority % num_priority_levels_;
+  ENVOY_LOG_MISC(trace, "Updating health flags for host set at priority: {}", priority_of_host_set);
+  MockHostSet& host_set = *priority_set_.getMockHostSet(priority_of_host_set);
+  // This downcast will not overflow because size is capped by port numbers
+  const uint32_t host_set_size = host_set.hosts_.size();
+  host_set.healthy_hosts_.clear();
+  host_set.degraded_hosts_.clear();
+  host_set.excluded_hosts_.clear();
+
+  Fuzz::ProperSubsetSelector subset_selector(random_bytestring);
+
+  const std::vector<std::vector<uint8_t>> subsets = subset_selector.constructSubsets(
+      {num_healthy_hosts, num_degraded_hosts, num_excluded_hosts}, host_set_size);
+
+  // Healthy hosts are first subset
+  for (uint8_t index : subsets.at(0)) {
+    host_set.healthy_hosts_.push_back(host_set.hosts_[index]);
+  }
+  ENVOY_LOG_MISC(trace, "Hosts made healthy at priority level {}: {}", priority_of_host_set,
+                 absl::StrJoin(subsets.at(0), " "));
+
+  // Degraded hosts are second subset
+  for (uint8_t index : subsets.at(1)) {
+    host_set.degraded_hosts_.push_back(host_set.hosts_[index]);
+  }
+  ENVOY_LOG_MISC(trace, "Hosts made degraded at priority level {}: {}", priority_of_host_set,
+                 absl::StrJoin(subsets.at(1), " "));
+
+  // Excluded hosts are third subset
+  for (uint8_t index : subsets.at(2)) {
+    host_set.excluded_hosts_.push_back(host_set.hosts_[index]);
+  }
+  ENVOY_LOG_MISC(trace, "Hosts made excluded at priority level {}: {}", priority_of_host_set,
+                 absl::StrJoin(subsets.at(2), " "));
+
+  // Handle updating health flags for hosts_per_locality_
+
+  // The index within the array of the vector represents the locality
+  std::array<HostVector, 3> healthy_hosts_per_locality;
+  std::array<HostVector, 3> degraded_hosts_per_locality;
+  std::array<HostVector, 3> excluded_hosts_per_locality;
+
+  // Wrap those three in an array here, where the index represents health flag of
+  // healthy/degraded/excluded, used for indexing during iteration through subsets
+  std::array<std::array<HostVector, 3>, 3> locality_health_flags = {
+      healthy_hosts_per_locality, degraded_hosts_per_locality, excluded_hosts_per_locality};
+
+  // Iterate through subsets
+  for (uint8_t health_flag = 0; health_flag < locality_health_flags.size(); health_flag++) {
+    for (uint8_t index : subsets.at(health_flag)) { // Each subset logically represents a health
+                                                    // flag
+      // If the host is in a locality, we have to update the corresponding health flag host vector
+      if (!(locality_indexes_.find(index) == locality_indexes_.end())) {
+        // First dimension of array represents health_flag, second represents locality, which is
+        // pulled from map
+        locality_health_flags[health_flag][locality_indexes_[index]].push_back(
+            host_set.hosts_[index]);
+        ENVOY_LOG_MISC(trace, "Added host at index {} in locality {} to health flag set {}", index,
+                       locality_indexes_[index], health_flag + 1);
+      }
+    }
+  }
+
+  // This overrides what is currently present in the host set, thus not having to explicitly call
+  // vector.clear()
+  host_set.healthy_hosts_per_locality_ =
+      makeHostsPerLocality({healthy_hosts_per_locality[0], healthy_hosts_per_locality[1],
+                            healthy_hosts_per_locality[2]});
+  host_set.degraded_hosts_per_locality_ =
+      makeHostsPerLocality({degraded_hosts_per_locality[0], degraded_hosts_per_locality[1],
+                            degraded_hosts_per_locality[2]});
+  host_set.excluded_hosts_per_locality_ =
+      makeHostsPerLocality({excluded_hosts_per_locality[0], excluded_hosts_per_locality[1],
+                            excluded_hosts_per_locality[2]});
+
+  host_set.runCallbacks({}, {});
+}
+
+void LoadBalancerFuzzBase::prefetch() {
+  // TODO: context, could generate it in proto action
+  lb_->peekAnotherHost(nullptr);
+}
+
+void LoadBalancerFuzzBase::chooseHost() {
+  // TODO: context, could generate it in proto action
+  lb_->chooseHost(nullptr);
+}
+
+void LoadBalancerFuzzBase::replay(
+    const Protobuf::RepeatedPtrField<test::common::upstream::LbAction>& actions) {
+  constexpr auto max_actions = 64;
+  for (int i = 0; i < std::min(max_actions, actions.size()); ++i) {
+    const auto& event = actions.at(i);
+    ENVOY_LOG_MISC(trace, "Action: {}", event.DebugString());
+    switch (event.action_selector_case()) {
+    case test::common::upstream::LbAction::kUpdateHealthFlags: {
+      updateHealthFlagsForAHostSet(event.update_health_flags().host_priority(),
+                                   event.update_health_flags().num_healthy_hosts(),
+                                   event.update_health_flags().num_degraded_hosts(),
+                                   event.update_health_flags().num_excluded_hosts(),
+                                   event.update_health_flags().random_bytestring());
+      break;
+    }
+    case test::common::upstream::LbAction::kPrefetch:
+      prefetch();
+      break;
+    case test::common::upstream::LbAction::kChooseHost:
+      chooseHost();
+      break;
+    default:
+      break;
+    }
+  }
+}
+
+} // namespace Upstream
+} // namespace Envoy

test/common/upstream/load_balancer_fuzz_base.h

@@ -0,0 +1,67 @@
+#include "envoy/config/cluster/v3/cluster.pb.h"
+
+#include "common/upstream/load_balancer_impl.h"
+
+#include "test/common/upstream/load_balancer_fuzz.pb.validate.h"
+#include "test/fuzz/random.h"
+#include "test/mocks/common.h"
+#include "test/mocks/runtime/mocks.h"
+#include "test/mocks/upstream/cluster_info.h"
+#include "test/mocks/upstream/host_set.h"
+#include "test/mocks/upstream/load_balancer_context.h"
+#include "test/mocks/upstream/priority_set.h"
+
+namespace Envoy {
+namespace Upstream {
+
+// This class implements replay logic, and also handles the initial setup of static host sets and
+// the subsequent updates to those sets.
+class LoadBalancerFuzzBase {
+public:
+  LoadBalancerFuzzBase() : stats_(ClusterInfoImpl::generateStats(stats_store_)){};
+
+  // Initializes load balancer components shared amongst every load balancer, random_, and
+  // priority_set_
+  void initializeLbComponents(const test::common::upstream::LoadBalancerTestCase& input);
+  void updateHealthFlagsForAHostSet(const uint64_t host_priority, const uint32_t num_healthy_hosts,
+                                    const uint32_t num_degraded_hosts,
+                                    const uint32_t num_excluded_hosts,
+                                    const std::string random_bytestring);
+  // These two actions have a lot of logic attached to them. However, all the logic that the load
+  // balancer needs to run its algorithm is already encapsulated within the load balancer. Thus,
+  // once the load balancer is constructed, all this class has to do is call lb_->peekAnotherHost()
+  // and lb_->chooseHost().
+  void prefetch();
+  void chooseHost();
+  ~LoadBalancerFuzzBase() = default;
+  void replay(const Protobuf::RepeatedPtrField<test::common::upstream::LbAction>& actions);
+
+  // These public objects shared amongst all types of load balancers will be used to construct load
+  // balancers in specific load balancer fuzz classes
+  Stats::IsolatedStoreImpl stats_store_;
+  ClusterStats stats_;
+  NiceMock<Runtime::MockLoader> runtime_;
+  Random::PsuedoRandomGenerator64 random_;
+  NiceMock<MockPrioritySet> priority_set_;
+  std::shared_ptr<MockClusterInfo> info_{new NiceMock<MockClusterInfo>()};
+  std::unique_ptr<LoadBalancerBase> lb_;
+
+private:
+  // Untrusted upstreams don't have the ability to change the host set size, so keep it constant
+  // over the fuzz iteration.
+  void
+  initializeASingleHostSet(const test::common::upstream::SetupPriorityLevel& setup_priority_level,
+                           const uint8_t priority_level);
+
+  // There are used to construct the priority set at the beginning of the fuzz iteration
+  uint16_t port_ = 80;
+  uint8_t num_priority_levels_ = 0;
+
+  // This map used when updating health flags - making sure the health flags are updated hosts in
+  // localities Key - index of host within full host list, value - locality level host at index is
+  // in
+  absl::node_hash_map<uint8_t, uint8_t> locality_indexes_;
+};
+
+} // namespace Upstream
+} // namespace Envoy