Redis fault injection

api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto

@@ -22,7 +22,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // Redis Proxy :ref:`configuration overview <config_network_filters_redis_proxy>`.
 // [#extension: envoy.filters.network.redis_proxy]
 
-// [#next-free-field: 7]
+// [#next-free-field: 8]
 message RedisProxy {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.network.redis_proxy.v2.RedisProxy";
@@ -182,6 +182,32 @@ message RedisProxy {
     Route catch_all_route = 4;
   }
 
+  // RedisFault defines faults used for fault injection.
+  message RedisFault {
+    option (udpa.annotations.versioning).previous_message_type =
+        "envoy.config.filter.network.redis_proxy.v2.RedisProxy.RedisFault";
+
+    enum RedisFaultType {
+      // Delays requests. This is the base fault; other faults can have delays added.
+      DELAY = 0;
+
+      // Returns errors on requests.
+      ERROR = 1;
+    }
+
+    // Fault type.
+    RedisFaultType fault_type = 1 [(validate.rules).enum = {defined_only: true}];
+
+    // Percentage of requests fault applies to.
+    config.core.v3.RuntimeFractionalPercent fault_enabled = 2;
+
+    // Delay for all faults. If not set, defaults to zero
+    google.protobuf.Duration delay = 3;
+
+    // Commands fault is restricted to, if any.
+    repeated string commands = 4;
+  }
+
   reserved 2;
 
   reserved "cluster";
@@ -234,6 +260,35 @@ message RedisProxy {
   // client. If an AUTH command is received when the password is not set, then an "ERR Client sent
   // AUTH, but no password is set" error will be returned.
   config.core.v3.DataSource downstream_auth_password = 6 [(udpa.annotations.sensitive) = true];
+
+  // List of faults to inject. Faults currently come in two flavors:
+  // - Delay, which delays a request.
+  // - Error, which responds to a request with an error. Errors can also have delays attached.
+  //
+  // Example:
+  //
+  // .. code-block:: yaml
+  //
+  //    faults:
+  //    - fault_type: ERROR
+  //      fault_enabled:
+  //        default_value:
+  //          numerator: 10
+  //          denominator: HUNDRED
+  //        runtime_key: "bogus_key"
+  //        commands:
+  //        - GET
+  //      - fault_type: DELAY
+  //        fault_enabled:
+  //          default_value:
+  //            numerator: 10
+  //            denominator: HUNDRED
+  //          runtime_key: "bogus_key"
+  //        delay: 2s
+  //
+  // See the :ref:`fault injection section
+  // <config_network_filters_redis_proxy_fault_injection>` for more information on how to configure this.
+  repeated RedisFault faults = 7;
 }
 
 // RedisProtocolOptions specifies Redis upstream protocol options. This object is used in

docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst

@@ -58,7 +58,9 @@ changed to microseconds by setting the configuration parameter :ref:`latency_in_
   total, Counter, Number of commands
   success, Counter, Number of commands that were successful
   error, Counter, Number of commands that returned a partial or complete error response
-  latency, Histogram, Command execution time in milliseconds
+  latency, Histogram, Command execution time in milliseconds (including delay faults)
+  error_fault, Counter, Number of commands that had an error fault injected
+  delay_fault, Counter, Number of commands that had a delay fault injected
 
 .. _config_network_filters_redis_proxy_per_command_stats:
 
@@ -70,3 +72,16 @@ The Redis proxy filter supports the following runtime settings:
 redis.drain_close_enabled
   % of connections that will be drain closed if the server is draining and would otherwise
   attempt a drain close. Defaults to 100.
+
+.. _config_network_filters_redis_proxy_fault_injection:
+
+Fault Injection
+---------------
+
+The Redis filter can perform fault injection. Currently, Delay and Error faults are supported.
+Delay faults delay a request, and Error faults respond with an error. Moreover, errors can be delayed.
+
+Note that if a delay is injected, the delay is additive- if the request took 400ms and a delay of 100ms
+is injected, then the total request latency is 500ms. Also, due to implementation of the redis proxy,
+a delayed request will delay everything that comes in after it, due to the proxy's need to respect the 
+order of commands it receives.

docs/root/version_history/current.rst

@@ -19,6 +19,7 @@ Changes
   `envoy.reloadable_features.new_http2_connection_pool_behavior`.
 * logger: added :ref:`--log-format-prefix-with-location <operations_cli>` command line option to prefix '%v' with file path and line number.
 * network filters: added a :ref:`postgres proxy filter <config_network_filters_postgres_proxy>`.
+* redis fault injection: added :ref:`fault injection for redis proxy <envoy_v3_api_field_extensions.filters.network.redis_proxy.v3.RedisProxy.faults>`, described further in :ref:`configuration documentation <config_network_filters_redis_proxy>`.
 * request_id: added to :ref:`always_set_request_id_in_response setting <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.always_set_request_id_in_response>`
   to set :ref:`x-request-id <config_http_conn_man_headers_x-request-id>` header in response even if
   tracing is not forced.

source/extensions/filters/network/common/redis/BUILD

@@ -99,3 +99,22 @@ envoy_cc_library(
         "//source/common/stats:timespan_lib",
     ],
 )
+
+envoy_cc_library(
+    name = "fault_interface",
+    hdrs = ["fault.h"],
+    deps = [],
+)
+
+envoy_cc_library(
+    name = "fault_lib",
+    srcs = ["fault_impl.cc"],
+    hdrs = ["fault_impl.h"],
+    deps = [
+        ":codec_lib",
+        ":fault_interface",
+        "//include/envoy/upstream:upstream_interface",
+        "//source/common/protobuf:utility_lib",
+        "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto",
+    ],
+)

source/extensions/filters/network/common/redis/fault.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/api/api.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Common {
+namespace Redis {
+
+/**
+ * Fault Type.
+ */
+enum class FaultType { Delay, Error };
+
+class FaultManager {
+public:
+  virtual ~FaultManager() = default;
+
+  /**
+   * Get fault type and delay given a Redis command.
+   * @param command supplies the Redis command string.
+   */
+  virtual absl::optional<std::pair<FaultType, std::chrono::milliseconds>>
+  getFaultForCommand(std::string command) PURE;
+};
+
+using FaultManagerPtr = std::shared_ptr<FaultManager>;
+
+} // namespace Redis
+} // namespace Common
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy

source/extensions/filters/network/common/redis/fault_impl.cc

@@ -0,0 +1,141 @@
+#include "extensions/filters/network/common/redis/fault_impl.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Common {
+namespace Redis {
+
+Fault::Fault(envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault base_fault)
+    : commands_(getCommands(base_fault)) {
+  // // Get delay
+  delay_ms_ = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(base_fault, delay, 0));
+
+  // Get fault type
+  switch (base_fault.fault_type()) {
+  case envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::RedisFault::DELAY:
+    fault_type_ = FaultType::Delay;
+    break;
+  case envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::RedisFault::ERROR:
+    fault_type_ = FaultType::Error;
+    break;
+  default:
+    NOT_REACHED_GCOVR_EXCL_LINE;
+    break;
+  }
+
+  // Get the default value/runtime key
+  if (base_fault.has_fault_enabled()) {
+    has_fault_enabled_ = true;
+    if (base_fault.fault_enabled().has_default_value()) {
+      if (base_fault.fault_enabled().default_value().denominator() ==
+          envoy::type::v3::FractionalPercent::HUNDRED) {
+        default_value_ = base_fault.fault_enabled().default_value().numerator();
+      } else {
+        auto denominator = ProtobufPercentHelper::fractionalPercentDenominatorToInt(
+            base_fault.fault_enabled().default_value().denominator());
+        default_value_ =
+            (base_fault.fault_enabled().default_value().numerator() * 100) / denominator;
+      }
+    }
+    runtime_key_ = base_fault.fault_enabled().runtime_key();
+  }
+};
+
+std::vector<std::string> Fault::getCommands(
+    envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault base_fault) {
+  std::vector<std::string> commands;
+  for (auto command : base_fault.commands()) {
+    commands.emplace_back(absl::AsciiStrToLower(command));
+  }
+  return commands;
+}
+
+FaultManagerImpl::FaultManagerImpl(
+    Runtime::RandomGenerator& random, Runtime::Loader& runtime,
+    const Protobuf::RepeatedPtrField<
+        ::envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault>
+        faults)
+    : random_(random), runtime_(runtime) {
+
+  // Next, create the fault map that maps commands to pointers to Fault objects.
+  // Group faults by command
+  for (auto base_fault : faults) {
+    auto fault_ptr = std::make_shared<Fault>(base_fault);
+    if (fault_ptr->commands_.size() > 0) {
+      for (std::string command : fault_ptr->commands_) {
+        fault_map_[command].push_back(fault_ptr);
+      }
+    } else {
+      // Generic "ALL" entry in map for faults that map to all keys; also add to each command
+      fault_map_[ALL_KEY].push_back(fault_ptr);
+    }
+  }
+
+  // Add the ALL keys faults to each command too so that we can just query faults by command.
+  // Get all ALL_KEY faults.
+  FaultMap::iterator it_outer = fault_map_.find(ALL_KEY);
+  if (it_outer != fault_map_.end()) {
+    // For each ALL_KEY fault...
+    for (auto fault_ptr : it_outer->second) {
+      // Loop through all unique commands other than ALL_KEY and add the fault.
+      FaultMap::iterator it_inner;
+      for (it_inner = fault_map_.begin(); it_inner != fault_map_.end(); it_inner++) {
+        std::string command = it_inner->first;
+        if (command != ALL_KEY) {
+          fault_map_[command].push_back(fault_ptr);
+        }
+      }
+    }
+  }
+}
+
+// Fault checking algorithm:
+//
+// For example, if we have an ERROR fault at 5% for all commands, and a DELAY fault at 10% for GET,
+// if we receive a GET, we want 5% of GETs to get DELAY, and 5% to get ERROR. Thus, we need to
+// amortize the percentages.
+//
+// 0. Get random number.
+// 1. Get faults for given command.s
+// 2. For each fault, calculate the amortized fault injection percentage.
+absl::optional<std::pair<FaultType, std::chrono::milliseconds>>
+FaultManagerImpl::getFaultForCommandInternal(std::string command) {
+  auto random_number = random_.random() % 100;
+  int amortized_fault = 0;
+
+  FaultMap::iterator it_outer = fault_map_.find(command);
+  if (it_outer != fault_map_.end()) {
+    for (auto fault_ptr : it_outer->second) {
+      auto fault_injection_percentage = runtime_.snapshot().getInteger(
+          fault_ptr->runtime_key_.value(), fault_ptr->default_value_.value());
+      if (random_number < (fault_injection_percentage + amortized_fault)) {
+        return std::make_pair(fault_ptr->fault_type_, fault_ptr->delay_ms_);
+      } else {
+        amortized_fault += fault_injection_percentage;
+      }
+    }
+  }
+
+  return absl::nullopt;
+}
+
+absl::optional<std::pair<FaultType, std::chrono::milliseconds>>
+FaultManagerImpl::getFaultForCommand(std::string command) {
+  // Check if faults exist for given command; else use ALL_KEY and search for general faults.
+  if (!fault_map_.empty()) {
+    if (fault_map_.count(command) > 0) {
+      return getFaultForCommandInternal(command);
+    } else {
+      return getFaultForCommandInternal(ALL_KEY);
+    }
+  }
+
+  return absl::nullopt;
+}
+
+} // namespace Redis
+} // namespace Common
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy