Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/envoy/config/filter/fault/v2/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ api_proto_library_internal(
"//envoy/config/filter/http/fault/v2:__pkg__",
"//envoy/config/filter/network/mongo_proxy/v2:__pkg__",
],
deps = ["//envoy/type:percent"],
)
11 changes: 10 additions & 1 deletion api/envoy/config/filter/fault/v2/fault.proto
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ syntax = "proto3";
package envoy.config.filter.fault.v2;
option go_package = "v2";

import "envoy/type/percent.proto";

import "google/protobuf/duration.proto";

import "validate/validate.proto";
Expand All @@ -24,7 +26,11 @@ message FaultDelay {

// An integer between 0-100 indicating the percentage of operations/connection requests
// on which the delay will be injected.
uint32 percent = 2 [(validate.rules).uint32.lte = 100];
//
// .. attention::
//
// This field is deprecated and `percentage` should be used instead.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use of integer percent value is deprecated. Use fractional percentage field instead

uint32 percent = 2 [(validate.rules).uint32.lte = 100, deprecated = true];

oneof fault_delay_secifier {
option (validate.required) = true;
Expand All @@ -37,4 +43,7 @@ message FaultDelay {
google.protobuf.Duration fixed_delay = 3
[(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true];
}

// The percentage of operations/connection requests on which the delay will be injected.
envoy.type.FractionalPercent percentage = 4;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you do the same for aborts as well ?

}
21 changes: 17 additions & 4 deletions include/envoy/runtime/runtime.h
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,19 @@ class Snapshot {
virtual bool featureEnabled(const std::string& key, uint64_t default_value,
uint64_t random_value) const PURE;

/**
* Test if a feature is enabled using the built in random generator and total number of buckets
* for sampling.
* @param key supplies the feature key to lookup.
* @param default_value supplies the default value that will be used if either the feature key
* does not exist or it is not an integer.
* @param num_buckets control max number of buckets for sampling. Sampled value will be in a range
* of [0, num_buckets).
* @return true if the feature is enabled.
*/
virtual bool sampleFeatureEnabled(const std::string& key, uint64_t default_value,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I see why you had to change the name here, but IMO "sample" is a bit strange in the sense that the other featureEnabled() functions also do sampling. A few options here:

  1. Get rid of the versions of featureEnabled() that don't require specifying number of buckets. Larger change.
  2. Some other name. featureEnabledEx ? (Bad, but not sure what would be better).

@venilnoronha venilnoronha Aug 8, 2018

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mattklein123 thanks for reviewing! I've addressed your other comments in f197fe2, and I'm working on this one now.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Implemented option 1 in f98bcc0.

uint64_t num_buckets) const PURE;

/**
* Test if a feature is enabled using a supplied stable random value and total number of buckets
* for sampling.
Expand All @@ -112,12 +125,12 @@ class Snapshot {
* does not exist or it is not an integer.
* @param random_value supplies the stable random value to use for determining whether the feature
* is enabled.
* @param control max number of buckets for sampling. Sampled value will be in a range of
* [0, num_buckets).
* @param num_buckets control max number of buckets for sampling. Sampled value will be in a range
* of [0, num_buckets).
* @return true if the feature is enabled.
*/
virtual bool featureEnabled(const std::string& key, uint64_t default_value, uint64_t random_value,
uint64_t num_buckets) const PURE;
virtual bool sampleFeatureEnabled(const std::string& key, uint64_t default_value,
uint64_t random_value, uint64_t num_buckets) const PURE;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mattklein123 how are you guys doing fractional percent with runtimes? IOW, do you need runtime support for fractional percent? If not, all changes to this file can be eliminated

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think access_log_impl uses runtime for sampling the feature status as seen here. Should we just stick to a similar approach for faults?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking again, why is this being renamed ?

@venilnoronha venilnoronha Aug 1, 2018

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because of a signature conflict between the routines highlighted here.


/**
* Fetch raw runtime data based on key.
Expand Down
2 changes: 1 addition & 1 deletion source/common/access_log/access_log_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ bool RuntimeFilter::evaluate(const RequestInfo::RequestInfo&,
random_value = random_.random();
}

return runtime_.snapshot().featureEnabled(
return runtime_.snapshot().sampleFeatureEnabled(
runtime_key_, percent_.numerator(), random_value,
ProtobufPercentHelper::fractionalPercentDenominatorToInt(percent_));
}
Expand Down
10 changes: 10 additions & 0 deletions source/common/config/filter_json.cc
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,11 @@ void FilterJson::translateMongoProxy(
delay->set_type(envoy::config::filter::fault::v2::FaultDelay::FIXED);
delay->set_percent(static_cast<uint32_t>(json_fault->getInteger("percent")));
JSON_UTIL_SET_DURATION_FROM_FIELD(*json_fault, *delay, fixed_delay, duration);

if (json_fault->hasObject("percentage")) {
const auto json_percentage = json_fault->getObject("percentage");
JSON_UTIL_SET_FRACTIONALPERCENT(*json_percentage, *delay, percentage);
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here

}
}

Expand All @@ -280,6 +285,11 @@ void FilterJson::translateFaultFilter(
delay->set_type(envoy::config::filter::fault::v2::FaultDelay::FIXED);
delay->set_percent(static_cast<uint32_t>(json_config_delay->getInteger("fixed_delay_percent")));
JSON_UTIL_SET_DURATION_FROM_FIELD(*json_config_delay, *delay, fixed_delay, fixed_duration);

if (json_config_delay->hasObject("fixed_delay_percentage")) {
const auto json_percentage = json_config_delay->getObject("fixed_delay_percentage");
JSON_UTIL_SET_FRACTIONALPERCENT(*json_percentage, *delay, percentage);
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You don’t need this. This is for translation of legacy config only.

}

for (const auto json_header_matcher : json_config.getObjectArray("headers", true)) {
Expand Down
15 changes: 15 additions & 0 deletions source/common/config/json_utility.h
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,18 @@
Protobuf::util::TimeUtil::MillisecondsToDuration((json).getInteger(#src_field "_ms"))); \
} \
} while (0)

// Set a envoy.type.FractionalPercent compatible field in a protobuf message with the corresponding
// numerator and denominator values from a JSON object if the fields are set in the JSON object.
#define JSON_UTIL_SET_FRACTIONALPERCENT(json, message, field_name) \
do { \
if ((json).hasObject("numerator")) { \
(message).mutable_##field_name()->set_numerator( \
static_cast<uint32_t>((json).getInteger("numerator"))); \
} \
if ((json).hasObject("denominator")) { \
(message).mutable_##field_name()->set_denominator( \
ProtobufPercentHelper::stringToFractionalPercentDenominatorType( \
(json).getString("denominator"))); \
} \
} while (0)
6 changes: 3 additions & 3 deletions source/common/http/conn_manager_utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,9 @@ void ConnectionManagerUtility::mutateTracingRequestHeader(Http::HeaderMap& reque
UuidUtils::setTraceableUuid(x_request_id, UuidTraceStatus::Client);
} else if (request_headers.EnvoyForceTrace()) {
UuidUtils::setTraceableUuid(x_request_id, UuidTraceStatus::Forced);
} else if (runtime.snapshot().featureEnabled("tracing.random_sampling",
config.tracingConfig()->random_sampling_, result,
10000)) {
} else if (runtime.snapshot().sampleFeatureEnabled("tracing.random_sampling",
config.tracingConfig()->random_sampling_,
result, 10000)) {
UuidUtils::setTraceableUuid(x_request_id, UuidTraceStatus::Sampled);
}
}
Expand Down
14 changes: 14 additions & 0 deletions source/common/protobuf/utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,20 @@ uint64_t convertPercent(double percent, uint64_t max_value) {
return max_value * (percent / 100.0);
}

envoy::type::FractionalPercent::DenominatorType
stringToFractionalPercentDenominatorType(const std::string denominator) {
if (denominator == "HUNDRED") {
return envoy::type::FractionalPercent::HUNDRED;
} else if (denominator == "TEN_THOUSAND") {
return envoy::type::FractionalPercent::TEN_THOUSAND;
} else if (denominator == "MILLION") {
return envoy::type::FractionalPercent::MILLION;
} else {
// Checked by schema.
NOT_REACHED_GCOVR_EXCL_LINE;
}
}

uint64_t fractionalPercentDenominatorToInt(const envoy::type::FractionalPercent& percent) {
switch (percent.denominator()) {
case envoy::type::FractionalPercent::HUNDRED:
Expand Down
8 changes: 8 additions & 0 deletions source/common/protobuf/utility.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ namespace ProtobufPercentHelper {
uint64_t checkAndReturnDefault(uint64_t default_value, uint64_t max_value);
uint64_t convertPercent(double percent, uint64_t max_value);

/**
* Convert a string to a fractional percent DenominatorType enum value.
* @param denominator supplies the denominator string to convert.
* @return the converted DenominatorType.
*/
envoy::type::FractionalPercent::DenominatorType
stringToFractionalPercentDenominatorType(const std::string denominator);

/**
* Convert a fractional percent denominator enum into an integer.
* @param percent supplies percent to convert.
Expand Down
2 changes: 1 addition & 1 deletion source/common/router/router.cc
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ bool FilterUtility::shouldShadow(const ShadowPolicy& policy, Runtime::Loader& ru
}

if (!policy.runtimeKey().empty() &&
!runtime.snapshot().featureEnabled(policy.runtimeKey(), 0, stable_random, 10000UL)) {
!runtime.snapshot().sampleFeatureEnabled(policy.runtimeKey(), 0, stable_random, 10000UL)) {
return false;
}

Expand Down
11 changes: 8 additions & 3 deletions source/common/runtime/runtime_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,13 @@ std::string RandomGeneratorImpl::uuid() {
return std::string(uuid, UUID_LENGTH);
}

bool SnapshotImpl::featureEnabled(const std::string& key, uint64_t default_value,
uint64_t random_value, uint64_t num_buckets) const {
bool SnapshotImpl::sampleFeatureEnabled(const std::string& key, uint64_t default_value,
uint64_t num_buckets) const {
return sampleFeatureEnabled(key, default_value, generator_.random(), num_buckets);
}

bool SnapshotImpl::sampleFeatureEnabled(const std::string& key, uint64_t default_value,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Skip renaming and overload old function name ?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ignore

uint64_t random_value, uint64_t num_buckets) const {
return random_value % num_buckets < std::min(getInteger(key, default_value), num_buckets);
}

Expand All @@ -165,7 +170,7 @@ bool SnapshotImpl::featureEnabled(const std::string& key, uint64_t default_value

bool SnapshotImpl::featureEnabled(const std::string& key, uint64_t default_value,
uint64_t random_value) const {
return featureEnabled(key, default_value, random_value, 100);
return sampleFeatureEnabled(key, default_value, random_value, 100);
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These two functions are not equivalent.. The stable random value is not same as number of buckets above. Am I missing something here>?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

random_value and num_buckets aren't being used interchangeably. Either we pass both random_value and num_buckets to featureEnabled, or we only pass num_buckets. There's no way to pass random_value without num_buckets.

const std::string& SnapshotImpl::get(const std::string& key) const {
Expand Down
6 changes: 4 additions & 2 deletions source/common/runtime/runtime_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,10 @@ class SnapshotImpl : public Snapshot, public ThreadLocal::ThreadLocalObject {
std::vector<OverrideLayerConstPtr>&& layers);

// Runtime::Snapshot
bool featureEnabled(const std::string& key, uint64_t default_value, uint64_t random_value,
uint64_t num_buckets) const override;
bool sampleFeatureEnabled(const std::string& key, uint64_t default_value,
uint64_t num_buckets) const override;
bool sampleFeatureEnabled(const std::string& key, uint64_t default_value, uint64_t random_value,
uint64_t num_buckets) const override;
bool featureEnabled(const std::string& key, uint64_t default_value) const override;
bool featureEnabled(const std::string& key, uint64_t default_value,
uint64_t random_value) const override;
Expand Down
25 changes: 19 additions & 6 deletions source/extensions/filters/http/fault/fault_filter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ namespace HttpFilters {
namespace Fault {

const std::string FaultFilter::DELAY_PERCENT_KEY = "fault.http.delay.fixed_delay_percent";
const std::string FaultFilter::DELAY_PERCENTAGE_KEY = "fault.http.delay.fixed_delay_percentage";
const std::string FaultFilter::ABORT_PERCENT_KEY = "fault.http.abort.abort_percent";
const std::string FaultFilter::DELAY_DURATION_KEY = "fault.http.delay.fixed_duration_ms";
const std::string FaultFilter::ABORT_HTTP_STATUS_KEY = "fault.http.abort.http_status";
Expand Down Expand Up @@ -129,14 +130,26 @@ Http::FilterHeadersStatus FaultFilter::decodeHeaders(Http::HeaderMap& headers, b
}

bool FaultFilter::isDelayEnabled() {
bool enabled = config_->runtime().snapshot().featureEnabled(DELAY_PERCENT_KEY,
fault_settings_->delayPercent());

if (!downstream_cluster_delay_percent_key_.empty()) {
enabled |= config_->runtime().snapshot().featureEnabled(downstream_cluster_delay_percent_key_,
bool enabled = false;
if (fault_settings_->delayPercent() != 0) {
enabled |= config_->runtime().snapshot().featureEnabled(DELAY_PERCENT_KEY,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not right. Runtime overrides config. If there is no runtime the config value becomes the default.

fault_settings_->delayPercent());
if (!downstream_cluster_delay_percent_key_.empty()) {
enabled |= config_->runtime().snapshot().featureEnabled(downstream_cluster_delay_percent_key_,
fault_settings_->delayPercent());
}
} else if (fault_settings_->delayPercentage().numerator() != 0) {
enabled |= config_->runtime().snapshot().sampleFeatureEnabled(
DELAY_PERCENTAGE_KEY, fault_settings_->delayPercentage().numerator(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_settings_->delayPercentage()));
if (!downstream_cluster_delay_percent_key_.empty()) {
enabled |= config_->runtime().snapshot().sampleFeatureEnabled(
downstream_cluster_delay_percent_key_, fault_settings_->delayPercentage().numerator(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_settings_->delayPercentage()));
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Flip the logic. Read new runtime or percentage. If not present read old runtime.

}

return enabled;
}

Expand Down
13 changes: 8 additions & 5 deletions source/extensions/filters/http/fault/fault_filter.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,19 @@ class FaultSettings : public Router::RouteSpecificFilterConfig {
}
uint64_t abortPercent() const { return abort_percent_; }
uint64_t delayPercent() const { return fixed_delay_percent_; }
envoy::type::FractionalPercent delayPercentage() const { return fixed_delay_percentage_; }
uint64_t delayDuration() const { return fixed_duration_ms_; }
uint64_t abortCode() const { return http_status_; }
const std::string& upstreamCluster() const { return upstream_cluster_; }
const std::unordered_set<std::string>& downstreamNodes() const { return downstream_nodes_; }

private:
uint64_t abort_percent_{}; // 0-100
uint64_t http_status_{}; // HTTP or gRPC return codes
uint64_t fixed_delay_percent_{}; // 0-100
uint64_t fixed_duration_ms_{}; // in milliseconds
std::string upstream_cluster_; // restrict faults to specific upstream cluster
uint64_t abort_percent_{}; // 0-100

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since you are doing this for delays, why don't you change the aborts as well to fractional ? It makes the implementation streamlined.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this PR is big in itself, should we do it in another PR?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The abort stuff is small. Confined to http only. If done right, it should be touching same files and not much.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I'm on it right now. Figured out it wasn't as big of a change as I thought it would be.

uint64_t http_status_{}; // HTTP or gRPC return codes
uint64_t fixed_delay_percent_{}; // 0-100
envoy::type::FractionalPercent fixed_delay_percentage_{}; // 0-100

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is a simpler way. Convert both percents to fractional (delay and abort).
Then when reading/translating config, if there is no fractional but old percent field is set, read it and convert it to fractions (out of 100).

Then in fault filter you simply have to check for new runtime fields for fractional percentage. If absent check old field. If that’s also absent use the value provided in the config.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer to implement the change to abort in a separate PR.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fix comment to indicate that this is fractional

uint64_t fixed_duration_ms_{}; // in milliseconds
std::string upstream_cluster_; // restrict faults to specific upstream cluster
std::vector<Http::HeaderUtility::HeaderData> fault_filter_headers_;
std::unordered_set<std::string> downstream_nodes_{}; // Inject failures for specific downstream
};
Expand Down Expand Up @@ -131,6 +133,7 @@ class FaultFilter : public Http::StreamDecoderFilter {
std::string downstream_cluster_abort_http_status_key_{};

const static std::string DELAY_PERCENT_KEY;
const static std::string DELAY_PERCENTAGE_KEY;
const static std::string ABORT_PERCENT_KEY;
const static std::string DELAY_DURATION_KEY;
const static std::string ABORT_HTTP_STATUS_KEY;
Expand Down
11 changes: 9 additions & 2 deletions source/extensions/filters/network/mongo_proxy/proxy.cc
Original file line number Diff line number Diff line change
Expand Up @@ -319,8 +319,15 @@ absl::optional<uint64_t> ProxyFilter::delayDuration() {
return result;
}

if (!runtime_.snapshot().featureEnabled(MongoRuntimeConfig::get().FixedDelayPercent,
fault_config_->delayPercent())) {
if ((fault_config_->delayPercent() != 0 &&
!runtime_.snapshot().featureEnabled(MongoRuntimeConfig::get().FixedDelayPercent,
fault_config_->delayPercent())) ||
(fault_config_->delayPercentage().numerator() != 0 &&
!runtime_.snapshot().sampleFeatureEnabled(
MongoRuntimeConfig::get().FixedDelayPercentage,
fault_config_->delayPercentage().numerator(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_config_->delayPercentage())))) {
return result;
}

Expand Down
5 changes: 4 additions & 1 deletion source/extensions/filters/network/mongo_proxy/proxy.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ namespace MongoProxy {
class MongoRuntimeConfigKeys {
public:
const std::string FixedDelayPercent{"mongo.fault.fixed_delay.percent"};
const std::string FixedDelayPercentage{"mongo.fault.fixed_delay.percentage"};
const std::string FixedDelayDurationMs{"mongo.fault.fixed_delay.duration_ms"};
const std::string LoggingEnabled{"mongo.logging_enabled"};
const std::string ProxyEnabled{"mongo.proxy_enabled"};
Expand Down Expand Up @@ -102,13 +103,15 @@ typedef std::shared_ptr<AccessLog> AccessLogSharedPtr;
class FaultConfig {
public:
FaultConfig(const envoy::config::filter::fault::v2::FaultDelay& fault_config)
: delay_percent_(fault_config.percent()),
: delay_percent_(fault_config.percent()), delay_percentage_(fault_config.percentage()),
duration_ms_(PROTOBUF_GET_MS_REQUIRED(fault_config, fixed_delay)) {}
uint32_t delayPercent() const { return delay_percent_; }
envoy::type::FractionalPercent delayPercentage() const { return delay_percentage_; }
uint64_t delayDuration() const { return duration_ms_; }

private:
const uint32_t delay_percent_;
const envoy::type::FractionalPercent delay_percentage_;
const uint64_t duration_ms_;
};

Expand Down
Loading