Redis fault injection

api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto

@@ -22,7 +22,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // Redis Proxy :ref:`configuration overview <config_network_filters_redis_proxy>`.
 // [#extension: envoy.filters.network.redis_proxy]
 
-// [#next-free-field: 7]
+// [#next-free-field: 8]
 message RedisProxy {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.network.redis_proxy.v2.RedisProxy";
@@ -182,6 +182,33 @@ message RedisProxy {
     Route catch_all_route = 4;
   }
 
+  // RedisFault defines faults used for fault injection.
+  message RedisFault {
+    option (udpa.annotations.versioning).previous_message_type =
+        "envoy.config.filter.network.redis_proxy.v2.RedisProxy.RedisFault";
+
+    enum RedisFaultType {
+      // Delays requests. This is the base fault; other faults can have delays added.
+      DELAY = 0;
+
+      // Returns errors on requests.
+      ERROR = 1;
+    }
+
+    // Fault type.
+    RedisFaultType fault_type = 1 [(validate.rules).enum = {defined_only: true}];
+
+    // Percentage of requests fault applies to.
+    config.core.v3.RuntimeFractionalPercent fault_enabled = 2
+        [(validate.rules).message = {required: true}];
+
+    // Delay for all faults. If not set, defaults to zero
+    google.protobuf.Duration delay = 3;
+
+    // Commands fault is restricted to, if any.
+    repeated string commands = 4;
+  }
+
   reserved 2;
 
   reserved "cluster";
@@ -234,6 +261,35 @@ message RedisProxy {
   // client. If an AUTH command is received when the password is not set, then an "ERR Client sent
   // AUTH, but no password is set" error will be returned.
   config.core.v3.DataSource downstream_auth_password = 6 [(udpa.annotations.sensitive) = true];
+
+  // List of faults to inject. Faults currently come in two flavors:
+  // - Delay, which delays a request.
+  // - Error, which responds to a request with an error. Errors can also have delays attached.
+  //
+  // Example:
+  //
+  // .. code-block:: yaml
+  //
+  //    faults:
+  //    - fault_type: ERROR
+  //      fault_enabled:
+  //        default_value:
+  //          numerator: 10
+  //          denominator: HUNDRED
+  //        runtime_key: "bogus_key"
+  //        commands:
+  //        - GET
+  //      - fault_type: DELAY
+  //        fault_enabled:
+  //          default_value:
+  //            numerator: 10
+  //            denominator: HUNDRED
+  //          runtime_key: "bogus_key"
+  //        delay: 2s
+  //
+  // See the :ref:`fault injection section
+  // <config_network_filters_redis_proxy_fault_injection>` for more information on how to configure this.
+  repeated RedisFault faults = 7;
 }
 
 // RedisProtocolOptions specifies Redis upstream protocol options. This object is used in

docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst

@@ -58,7 +58,9 @@ changed to microseconds by setting the configuration parameter :ref:`latency_in_
   total, Counter, Number of commands
   success, Counter, Number of commands that were successful
   error, Counter, Number of commands that returned a partial or complete error response
-  latency, Histogram, Command execution time in milliseconds
+  latency, Histogram, Command execution time in milliseconds (including delay faults)
+  error_fault, Counter, Number of commands that had an error fault injected
+  delay_fault, Counter, Number of commands that had a delay fault injected
 
 .. _config_network_filters_redis_proxy_per_command_stats:
 
@@ -70,3 +72,49 @@ The Redis proxy filter supports the following runtime settings:
 redis.drain_close_enabled
   % of connections that will be drain closed if the server is draining and would otherwise
   attempt a drain close. Defaults to 100.
+
+.. _config_network_filters_redis_proxy_fault_injection:
+
+Fault Injection
+---------------
+
+The Redis filter can perform fault injection. Currently, Delay and Error faults are supported.
+Delay faults delay a request, and Error faults respond with an error. Moreover, errors can be delayed.
+
+If multiple faults are specified, the fault injection percentage should not exceed 100% for a given 
+fault and Redis command combination. For example, if you have two faults; one applying to GET at 60
+%, and one applying to all commands at 50%, that is a bad configuration as GET now has 110% chance of
+applying a fault. The Redis filter does not check for correctness in your configuration- it is user's
+responsibility to make sure both the default and runtime percentages are correct!
+
+If a delay is injected, the delay is additive- if the request took 400ms and a delay of 100ms
+is injected, then the total request latency is 500ms. Also, due to implementation of the redis protocol,
+a delayed request will delay everything that comes in after it, due to the proxy's need to respect the 
+order of commands it receives.
+
+Note that faults must have a `fault_enabled` field, and are not enabled by default (if no default value
+or runtime key are set).
+
+Example configuration:
+
+.. code-block:: yaml
+
+  faults:
+  - fault_type: ERROR
+    fault_enabled:
+      default_value:
+        numerator: 10
+        denominator: HUNDRED
+      runtime_key: "bogus_key"
+      commands:
+      - GET
+    - fault_type: DELAY
+      fault_enabled:
+        default_value:
+          numerator: 10
+          denominator: HUNDRED
+        runtime_key: "bogus_key"
+      delay: 2s
+
+This creates two faults- an error, applying only to GET commands at 10%, and a delay, applying to all
+commands at 10%. This means that 20% of GET commands will have a fault applied, as discussed earlier.

docs/root/version_history/current.rst

@@ -42,6 +42,7 @@ Changes
 * network filters: added a :ref:`postgres proxy filter <config_network_filters_postgres_proxy>`.
 * network filters: added a :ref:`rocketmq proxy filter <config_network_filters_rocketmq_proxy>`.
 * prometheus stats: fix the sort order of output lines to comply with the standard.
+* redis fault injection: added :ref:`fault injection for redis proxy <envoy_v3_api_field_extensions.filters.network.redis_proxy.v3.RedisProxy.faults>`, described further in :ref:`configuration documentation <config_network_filters_redis_proxy>`.
 * request_id: added to :ref:`always_set_request_id_in_response setting <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.always_set_request_id_in_response>`
   to set :ref:`x-request-id <config_http_conn_man_headers_x-request-id>` header in response even if
   tracing is not forced.

source/extensions/filters/network/common/redis/BUILD

@@ -100,3 +100,22 @@ envoy_cc_library(
         "//source/common/stats:utility_lib",
     ],
 )
+
+envoy_cc_library(
+    name = "fault_interface",
+    hdrs = ["fault.h"],
+    deps = [],
+)
+
+envoy_cc_library(
+    name = "fault_lib",
+    srcs = ["fault_impl.cc"],
+    hdrs = ["fault_impl.h"],
+    deps = [
+        ":codec_lib",
+        ":fault_interface",
+        "//include/envoy/upstream:upstream_interface",
+        "//source/common/protobuf:utility_lib",
+        "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto",
+    ],
+)

source/extensions/filters/network/common/redis/fault.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <chrono>
+#include <string>
+
+#include "envoy/common/pure.h"
+
+#include "absl/types/optional.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Common {
+namespace Redis {
+
+/**
+ * Fault Type.
+ */
+enum class FaultType { Delay, Error };
+
+class FaultManager {
+public:
+  virtual ~FaultManager() = default;
+
+  /**
+   * Get fault type and delay given a Redis command.
+   * @param command supplies the Redis command string.
+   */
+  virtual absl::optional<std::pair<FaultType, std::chrono::milliseconds>>
+  getFaultForCommand(std::string command) const PURE;
+};
+
+using FaultManagerPtr = std::shared_ptr<FaultManager>;
+
+} // namespace Redis
+} // namespace Common
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy