From 5b7581672e6197aee7aa3abf922c626a3da726b4 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Thu, 13 Oct 2022 15:25:11 +0000 Subject: [PATCH 01/21] load balancer: update related methodsfor load balancer extensions Signed-off-by: wbpcode --- envoy/upstream/load_balancer.h | 33 +++++++++++++++---- envoy/upstream/upstream.h | 7 ++-- .../common/upstream/cluster_manager_impl.cc | 5 ++- source/common/upstream/original_dst_cluster.h | 1 + source/common/upstream/thread_aware_lb_impl.h | 2 ++ source/common/upstream/upstream_impl.cc | 14 +++++--- source/common/upstream/upstream_impl.h | 8 ++--- .../extensions/clusters/aggregate/cluster.h | 1 + .../clusters/dynamic_forward_proxy/cluster.h | 1 + .../upstream/cluster_manager_impl_test.cc | 3 +- .../load_balancers/custom_lb_policy.h | 8 ++--- test/mocks/upstream/cluster_info.h | 3 +- .../upstream/typed_load_balancer_factory.h | 5 ++- 13 files changed, 59 insertions(+), 32 deletions(-) diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h index 07d72fc5e3f5c..62d3624e472f5 100644 --- a/envoy/upstream/load_balancer.h +++ b/envoy/upstream/load_balancer.h @@ -150,6 +150,16 @@ class LoadBalancer { using LoadBalancerPtr = std::unique_ptr; +/** + * Necessary parameters for creating a worker local load balancer. + */ +struct LoadBalancerParams { + // The worker local priority set of the target cluster. + const PrioritySet& priority_set; + // The worker local priority set of the local cluster. + const PrioritySet* local_priority_set{}; +}; + /** * Factory for load balancers. */ @@ -158,9 +168,15 @@ class LoadBalancerFactory { virtual ~LoadBalancerFactory() = default; /** - * @return LoadBalancerPtr a new load balancer. + * @return LoadBalancerPtr a new worker local load balancer. + * TODO(wbpcode): remove this method in the future and used the new method below. */ virtual LoadBalancerPtr create() PURE; + + /** + * @return LoadBalancerPtr a new worker local load balancer. + */ + virtual LoadBalancerPtr create(LoadBalancerParams params) PURE; }; using LoadBalancerFactorySharedPtr = std::shared_ptr; @@ -213,7 +229,7 @@ class ThreadAwareLoadBalancer { using ThreadAwareLoadBalancerPtr = std::unique_ptr; /** - * Factory for (thread-aware) load balancers. To support a load balancing policy of + * Factory config for load balancers. To support a load balancing policy of * LOAD_BALANCING_POLICY_CONFIG, at least one load balancer factory corresponding to a policy in * load_balancing_policy must be registered with Envoy. Envoy will use the first policy for which * it has a registered factory. @@ -224,13 +240,18 @@ class TypedLoadBalancerFactory : public Config::TypedFactory { /** * @return ThreadAwareLoadBalancerPtr a new thread-aware load balancer. + * + * @param cluster_info supplies the cluster info. + * @param priority_set supplies the priority set. + * @param runtime supplies the runtime loader. + * @param random supplies the random generator. + * @param time_source supplies the time source. */ virtual ThreadAwareLoadBalancerPtr - create(const PrioritySet& priority_set, ClusterStats& stats, Stats::Scope& stats_scope, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const ::envoy::config::cluster::v3::LoadBalancingPolicy_Policy& lb_policy) PURE; + create(const ClusterInfo& cluster_info, const PrioritySet& priority_set, Runtime::Loader& runtime, + Random::RandomGenerator& random, TimeSource& time_source) PURE; - std::string category() const override { return "envoy.load_balancers"; } + std::string category() const override { return "envoy.load_balancing_policies"; } }; } // namespace Upstream diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h index 69ff8cb0895cd..5fb83e9be9bf5 100644 --- a/envoy/upstream/upstream.h +++ b/envoy/upstream/upstream.h @@ -836,11 +836,10 @@ class ClusterInfo : public Http::FilterChainFactory { } /** - * @return const envoy::config::cluster::v3::LoadBalancingPolicy_Policy& the load balancing policy - * to use for this cluster. + * @return const ProtobufWkt::Message& the validated load balancing policy configuration to use + * for this cluster. */ - virtual const envoy::config::cluster::v3::LoadBalancingPolicy_Policy& - loadBalancingPolicy() const PURE; + virtual const ProtobufTypes::MessagePtr& loadBalancingPolicy() const PURE; /** * @return the load balancer factory for this cluster if the load balancing type is diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index a03026aa72839..eb558edfdde63 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -898,12 +898,11 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) { cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second); } else if (cluster_reference.info()->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) { - const auto& policy = cluster_reference.info()->loadBalancingPolicy(); TypedLoadBalancerFactory* typed_lb_factory = cluster_reference.info()->loadBalancerFactory(); RELEASE_ASSERT(typed_lb_factory != nullptr, "ClusterInfo should contain a valid factory"); cluster_entry_it->second->thread_aware_lb_ = - typed_lb_factory->create(cluster_reference.prioritySet(), cluster_reference.info()->stats(), - cluster_reference.info()->statsScope(), runtime_, random_, policy); + typed_lb_factory->create(*cluster_reference.info(), cluster_reference.prioritySet(), + runtime_, random_, time_source_); } updateClusterCounts(); diff --git a/source/common/upstream/original_dst_cluster.h b/source/common/upstream/original_dst_cluster.h index a6f6c6d45fd75..33193bc012558 100644 --- a/source/common/upstream/original_dst_cluster.h +++ b/source/common/upstream/original_dst_cluster.h @@ -107,6 +107,7 @@ class OriginalDstCluster : public ClusterImplBase { // Upstream::LoadBalancerFactory Upstream::LoadBalancerPtr create() override { return std::make_unique(cluster_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } const std::shared_ptr cluster_; }; diff --git a/source/common/upstream/thread_aware_lb_impl.h b/source/common/upstream/thread_aware_lb_impl.h index 25ce4ace6c277..985e2921e3e83 100644 --- a/source/common/upstream/thread_aware_lb_impl.h +++ b/source/common/upstream/thread_aware_lb_impl.h @@ -156,6 +156,8 @@ class ThreadAwareLoadBalancerBase : public LoadBalancerBase, public ThreadAwareL // Upstream::LoadBalancerFactory LoadBalancerPtr create() override; + // Ignore the params for the thread-aware LB. + LoadBalancerPtr create(LoadBalancerParams) override { return create(); } ClusterStats& stats_; Random::RandomGenerator& random_; diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 84c7f1c6c1846..340ae4f2b340c 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -968,7 +968,7 @@ ClusterInfoImpl::ClusterInfoImpl( // If load_balancing_policy is set we will use it directly, ignoring lb_policy. if (config.has_load_balancing_policy()) { - configureLbPolicies(config); + configureLbPolicies(config, server_context); } else { switch (config.lb_policy()) { PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; @@ -997,7 +997,7 @@ ClusterInfoImpl::ClusterInfoImpl( lb_type_ = LoadBalancerType::ClusterProvided; break; case envoy::config::cluster::v3::Cluster::LOAD_BALANCING_POLICY_CONFIG: { - configureLbPolicies(config); + configureLbPolicies(config, server_context); break; } } @@ -1086,7 +1086,8 @@ ClusterInfoImpl::ClusterInfoImpl( } // Configures the load balancer based on config.load_balancing_policy -void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Cluster& config) { +void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Cluster& config, + Server::Configuration::ServerFactoryContext& context) { if (config.has_lb_subset_config()) { throw EnvoyException( fmt::format("cluster: LB policy {} cannot be combined with lb_subset_config", @@ -1110,7 +1111,12 @@ void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Clus Config::Utility::getAndCheckFactory( policy.typed_extension_config(), /*is_optional=*/true); if (factory != nullptr) { - load_balancing_policy_ = policy; + // Load and validate the configuration. + load_balancing_policy_ = factory->createEmptyConfigProto(); + Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(), + context.messageValidationVisitor(), + *load_balancing_policy_); + load_balancer_factory_ = factory; break; } diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 2dec305e6339d..32e394f275475 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -660,8 +660,7 @@ class ClusterInfoImpl : public ClusterInfo, // Upstream::ClusterInfo bool addedViaApi() const override { return added_via_api_; } - const envoy::config::cluster::v3::LoadBalancingPolicy_Policy& - loadBalancingPolicy() const override { + const ProtobufTypes::MessagePtr& loadBalancingPolicy() const override { return load_balancing_policy_; } TypedLoadBalancerFactory* loadBalancerFactory() const override { return load_balancer_factory_; } @@ -693,7 +692,8 @@ class ClusterInfoImpl : public ClusterInfo, const envoy::config::core::v3::HttpProtocolOptions& commonHttpProtocolOptions() const override { return http_protocol_options_->common_http_protocol_options_; } - void configureLbPolicies(const envoy::config::cluster::v3::Cluster& config); + void configureLbPolicies(const envoy::config::cluster::v3::Cluster& config, + Server::Configuration::ServerFactoryContext& context); ProtocolOptionsConfigConstSharedPtr extensionProtocolOptions(const std::string& name) const override; LoadBalancerType lbType() const override { return lb_type_; } @@ -869,7 +869,7 @@ class ClusterInfoImpl : public ClusterInfo, LoadBalancerSubsetInfoImpl lb_subset_; const envoy::config::core::v3::Metadata metadata_; Envoy::Config::TypedMetadataImpl typed_metadata_; - envoy::config::cluster::v3::LoadBalancingPolicy_Policy load_balancing_policy_; + ProtobufTypes::MessagePtr load_balancing_policy_; TypedLoadBalancerFactory* load_balancer_factory_ = nullptr; const envoy::config::cluster::v3::Cluster::CommonLbConfig common_lb_config_; const Network::ConnectionSocket::OptionsSharedPtr cluster_socket_options_; diff --git a/source/extensions/clusters/aggregate/cluster.h b/source/extensions/clusters/aggregate/cluster.h index ba2dbe908e332..90a267a606ab3 100644 --- a/source/extensions/clusters/aggregate/cluster.h +++ b/source/extensions/clusters/aggregate/cluster.h @@ -147,6 +147,7 @@ struct AggregateLoadBalancerFactory : public Upstream::LoadBalancerFactory { cluster_.info(), cluster_.cluster_manager_, cluster_.runtime_, cluster_.random_, cluster_.clusters_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } const Cluster& cluster_; }; diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.h b/source/extensions/clusters/dynamic_forward_proxy/cluster.h index 7bb6046fbfdf4..1cd876e6b916f 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.h +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.h @@ -112,6 +112,7 @@ class Cluster : public Upstream::BaseDynamicClusterImpl, // Upstream::LoadBalancerFactory Upstream::LoadBalancerPtr create() override { return std::make_unique(cluster_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } private: Cluster& cluster_; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 3f4205e9a4ed4..7c0fb3f7d2967 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1132,8 +1132,7 @@ TEST_F(ClusterManagerImplTest, LbPolicyConfig) { create(parseBootstrapFromV3Yaml(yaml)); const auto& cluster = cluster_manager_->clusters().getCluster("cluster_1"); EXPECT_NE(cluster, absl::nullopt); - EXPECT_EQ(cluster->get().info()->loadBalancingPolicy().typed_extension_config().name(), - "envoy.load_balancers.custom_lb"); + EXPECT_NE(cluster->get().info()->loadBalancingPolicy(), nullptr); } // Verify that if Envoy does not have a factory for any of the load balancing policies specified in diff --git a/test/integration/load_balancers/custom_lb_policy.h b/test/integration/load_balancers/custom_lb_policy.h index cbdcd49ff8a8e..e166472248bd3 100644 --- a/test/integration/load_balancers/custom_lb_policy.h +++ b/test/integration/load_balancers/custom_lb_policy.h @@ -47,6 +47,7 @@ class ThreadAwareLbImpl : public Upstream::ThreadAwareLoadBalancer { LbFactory(const Upstream::HostSharedPtr& host) : host_(host) {} Upstream::LoadBalancerPtr create() override { return std::make_unique(host_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } const Upstream::HostSharedPtr host_; }; @@ -62,10 +63,9 @@ class CustomLbFactory : public Upstream::TypedLoadBalancerFactoryBase { return Envoy::ProtobufTypes::MessagePtr{new ::test::integration::custom_lb::CustomLbConfig()}; } - Upstream::ThreadAwareLoadBalancerPtr - create(const Upstream::PrioritySet&, Upstream::ClusterStats&, Stats::Scope&, Runtime::Loader&, - Random::RandomGenerator&, - const ::envoy::config::cluster::v3::LoadBalancingPolicy_Policy&) override { + Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo&, + const Upstream::PrioritySet&, Runtime::Loader&, + Random::RandomGenerator&, TimeSource&) override { return std::make_unique(); } }; diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 337836d77baeb..b148b0a068d6d 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -113,8 +113,7 @@ class MockClusterInfo : public ClusterInfo { (const)); MOCK_METHOD(ProtocolOptionsConfigConstSharedPtr, extensionProtocolOptions, (const std::string&), (const)); - MOCK_METHOD(const envoy::config::cluster::v3::LoadBalancingPolicy_Policy&, loadBalancingPolicy, - (), (const)); + MOCK_METHOD(const ProtobufTypes::MessagePtr&, loadBalancingPolicy, (), (const)); MOCK_METHOD(TypedLoadBalancerFactory*, loadBalancerFactory, (), (const)); MOCK_METHOD(const envoy::config::cluster::v3::Cluster::CommonLbConfig&, lbConfig, (), (const)); MOCK_METHOD(LoadBalancerType, lbType, (), (const)); diff --git a/test/mocks/upstream/typed_load_balancer_factory.h b/test/mocks/upstream/typed_load_balancer_factory.h index 98a3e2fa6a287..eb8b0cb8d4f30 100644 --- a/test/mocks/upstream/typed_load_balancer_factory.h +++ b/test/mocks/upstream/typed_load_balancer_factory.h @@ -15,9 +15,8 @@ class MockTypedLoadBalancerFactory : public TypedLoadBalancerFactory { // Upstream::TypedLoadBalancerFactory MOCK_METHOD(std::string, name, (), (const)); MOCK_METHOD(ThreadAwareLoadBalancerPtr, create, - (const PrioritySet& priority_set, ClusterStats& stats, Stats::Scope& stats_scope, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const ::envoy::config::cluster::v3::LoadBalancingPolicy_Policy& lb_policy)); + (const ClusterInfo& cluster_info, const PrioritySet& priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source)); ProtobufTypes::MessagePtr createEmptyConfigProto() override { // Using Struct instead of a custom per-filter empty config proto From d5947bbaf623d7a4506bcddd587b437fa6e5f8be Mon Sep 17 00:00:00 2001 From: wbpcode Date: Thu, 13 Oct 2022 15:25:50 +0000 Subject: [PATCH 02/21] load balancer: added least request load balancer extension Signed-off-by: wbpcode --- .../least_request/v3/least_request.proto | 2 +- ci/do_ci.sh | 6 +- source/extensions/extensions_build_config.bzl | 5 ++ source/extensions/extensions_metadata.yaml | 7 ++ .../load_balancing_policies/common/BUILD | 17 +++++ .../common/factory_base.h | 70 +++++++++++++++++++ .../least_request/BUILD | 21 ++++++ .../least_request/config.cc | 38 ++++++++++ .../least_request/config.h | 34 +++++++++ 9 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 source/extensions/load_balancing_policies/common/BUILD create mode 100644 source/extensions/load_balancing_policies/common/factory_base.h create mode 100644 source/extensions/load_balancing_policies/least_request/BUILD create mode 100644 source/extensions/load_balancing_policies/least_request/config.cc create mode 100644 source/extensions/load_balancing_policies/least_request/config.h diff --git a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto index 97efd9183250a..6cb5ca135b139 100644 --- a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto +++ b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto @@ -21,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This configuration allows the built-in LEAST_REQUEST LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` for more information. -// [#extension: envoy.clusters.lb_policy] +// [#extension: envoy.load_balancing_policies] message LeastRequest { // The number of random healthy hosts from which the host with the fewest active requests will // be chosen. Defaults to 2 so that we perform two-choice selection if the field is not set. diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 70a2f4726d79a..2cfdea0b80c93 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -12,7 +12,7 @@ if [[ "$1" == "format" || "$1" == "fix_proto_format" || "$1" == "check_proto_for || "$1" == "verify_distro" ]]; then build_setup_args="-nofetch" fi - +build_setup_args="-nofetch" # TODO(phlax): Clarify and/or integrate SRCDIR and ENVOY_SRCDIR export SRCDIR="${SRCDIR:-$PWD}" export ENVOY_SRCDIR="${ENVOY_SRCDIR:-$PWD}" @@ -225,10 +225,10 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c opt "${TEST_TARGETS[@]}" echo "bazel release build..." - bazel_envoy_binary_build release + # bazel_envoy_binary_build release echo "bazel contrib release build..." - bazel_contrib_binary_build release + # bazel_contrib_binary_build release exit 0 elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index df452a4beb737..e735057fd613f 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -367,6 +367,11 @@ EXTENSIONS = { # "envoy.route.early_data_policy.default": "//source/extensions/early_data:default_early_data_policy_lib", + + # + # Load balancing policies for upstream + # + "envoy.load_balancing_policies.least_request": "//source/extensions/load_balancing_policies/least_request:config", } # These can be changed to ["//visibility:public"], for downstream builds which diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 1c7b160eb094c..fe69a3034de66 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -1280,3 +1280,10 @@ envoy.matching.custom_matchers.trie_matcher: status: alpha type_urls: - xds.type.matcher.v3.IPMatcher +envoy.load_balancing_policies.least_request: + categories: + - envoy.load_balancing_policies + security_posture: unknown + status: alpha + type_urls: + - envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest diff --git a/source/extensions/load_balancing_policies/common/BUILD b/source/extensions/load_balancing_policies/common/BUILD new file mode 100644 index 0000000000000..4c045e2ec9de5 --- /dev/null +++ b/source/extensions/load_balancing_policies/common/BUILD @@ -0,0 +1,17 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "factory_base", + hdrs = ["factory_base.h"], + deps = [ + "//envoy/upstream:load_balancer_interface", + ], +) diff --git a/source/extensions/load_balancing_policies/common/factory_base.h b/source/extensions/load_balancing_policies/common/factory_base.h new file mode 100644 index 0000000000000..a957c9dfec2c1 --- /dev/null +++ b/source/extensions/load_balancing_policies/common/factory_base.h @@ -0,0 +1,70 @@ +#pragma once + +#include + +#include "envoy/upstream/load_balancer.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Common { + +template +class FactoryBase : public Upstream::TypedLoadBalancerFactory { +public: + FactoryBase(absl::string_view name) : name_(name) {} + + Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, + Random::RandomGenerator& random, + TimeSource& time_source) override { + return std::make_unique( + std::make_shared(cluster_info, priority_set, runtime, random, time_source)); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return name_; } + +private: + class LbFactory : public Upstream::LoadBalancerFactory { + public: + LbFactory(const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) + : cluster_info_(cluster_info), priority_set_(priority_set), runtime_(runtime), + random_(random), time_source_(time_source) {} + + Upstream::LoadBalancerPtr create() override { PANIC("not implemented"); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams params) override { + return Impl()(params, cluster_info_, priority_set_, runtime_, random_, time_source_); + } + + public: + const Upstream::ClusterInfo& cluster_info_; + const Upstream::PrioritySet& priority_set_; + Runtime::Loader& runtime_; + Random::RandomGenerator& random_; + TimeSource& time_source_; + }; + + class ThreadAwareLb : public Upstream::ThreadAwareLoadBalancer { + public: + ThreadAwareLb(Upstream::LoadBalancerFactorySharedPtr factory) : factory_(std::move(factory)) {} + + Upstream::LoadBalancerFactorySharedPtr factory() override { return factory_; } + void initialize() override {} + + private: + Upstream::LoadBalancerFactorySharedPtr factory_; + }; + + const std::string name_; +}; + +} // namespace Common +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/least_request/BUILD b/source/extensions/load_balancing_policies/least_request/BUILD new file mode 100644 index 0000000000000..5f1d06067544c --- /dev/null +++ b/source/extensions/load_balancing_policies/least_request/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//source/common/common:minimal_logger_lib", + "//source/common/upstream:load_balancer_lib", + "//source/extensions/load_balancing_policies/common:factory_base", + "@envoy_api//envoy/extensions/load_balancing_policies/least_request/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc new file mode 100644 index 0000000000000..e41cb43bfc54f --- /dev/null +++ b/source/extensions/load_balancing_policies/least_request/config.cc @@ -0,0 +1,38 @@ +#include "source/extensions/load_balancing_policies/least_request/config.h" + +#include "source/common/upstream/load_balancer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace LeastRequest { + +Upstream::LoadBalancerPtr LeastRequestCreator::operator()(Upstream::LoadBalancerParams params, + const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet&, + Runtime::Loader& runtime, + Random::RandomGenerator& random, + TimeSource& time_source) { + // Use the old proto type for now. + envoy::config::cluster::v3::Cluster::LeastRequestLbConfig old_type_config; + try { + MessageUtil::wireCast(*cluster_info.loadBalancingPolicy(), old_type_config); + } catch (EnvoyException& e) { + // Cast failed and exit in debug mode. Continue in release mode with a error message. + ASSERT(false, + fmt::format("Failed to cast load balancing policy to least request: {}", e.what())); + ENVOY_LOG(error, + "Cannot cast least request load balancing policy configuration to " + "envoy::config::cluster::v3::Cluster::LeastRequestLbConfig: {}", + e.what()); + old_type_config.Clear(); + } + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.stats(), runtime, random, + cluster_info.lbConfig(), old_type_config, time_source); +} + +} // namespace LeastRequest +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/least_request/config.h b/source/extensions/load_balancing_policies/least_request/config.h new file mode 100644 index 0000000000000..44fd31c8b13da --- /dev/null +++ b/source/extensions/load_balancing_policies/least_request/config.h @@ -0,0 +1,34 @@ +#pragma once + +#include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.h" +#include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.validate.h" +#include "envoy/upstream/load_balancer.h" + +#include "source/extensions/load_balancing_policies/common/factory_base.h" + +#include "/source/common/common/logger.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace LeastRequest { + +struct LeastRequestCreator : public Logger::Loggable { + Upstream::LoadBalancerPtr operator()(Upstream::LoadBalancerParams params, + const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, + TimeSource& time_source); +}; + +class Factory : Common::FactoryBase< + envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest, + LeastRequestCreator> { +public: + Factory() : FactoryBase("envoy.load_balancing_policies.least_request") {} +}; + +} // namespace LeastRequest +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy From 61520ae73d5631aa4afdb90d5299c7616d477961 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Thu, 13 Oct 2022 17:04:52 +0000 Subject: [PATCH 03/21] fix build and make it just run Signed-off-by: wbpcode --- source/common/upstream/cluster_manager_impl.cc | 4 ++-- source/extensions/clusters/redis/redis_cluster_lb.h | 1 + .../load_balancing_policies/least_request/config.cc | 5 +++++ .../load_balancing_policies/least_request/config.h | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index eb558edfdde63..c676fac636f3b 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -1182,7 +1182,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::updateHost // If an LB is thread aware, create a new worker local LB on membership changes. if (lb_factory_ != nullptr) { ENVOY_LOG(debug, "re-creating local LB for TLS cluster {}", name); - lb_ = lb_factory_->create(); + lb_ = lb_factory_->create({priority_set_, parent_.local_priority_set_}); } } @@ -1550,7 +1550,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry( case LoadBalancerType::Maglev: case LoadBalancerType::OriginalDst: { ASSERT(lb_factory_ != nullptr); - lb_ = lb_factory_->create(); + lb_ = lb_factory_->create({priority_set_, parent_.local_priority_set_}); break; } } diff --git a/source/extensions/clusters/redis/redis_cluster_lb.h b/source/extensions/clusters/redis/redis_cluster_lb.h index 1a311db0dea1e..fe2870adb912f 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.h +++ b/source/extensions/clusters/redis/redis_cluster_lb.h @@ -150,6 +150,7 @@ class RedisClusterLoadBalancerFactory : public ClusterSlotUpdateCallBack, // Upstream::LoadBalancerFactory Upstream::LoadBalancerPtr create() override; + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } private: class RedisShard { diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc index e41cb43bfc54f..047daa47f32ec 100644 --- a/source/extensions/load_balancing_policies/least_request/config.cc +++ b/source/extensions/load_balancing_policies/least_request/config.cc @@ -32,6 +32,11 @@ Upstream::LoadBalancerPtr LeastRequestCreator::operator()(Upstream::LoadBalancer cluster_info.lbConfig(), old_type_config, time_source); } +/** + * Static registration for the Factory. @see RegisterFactory. + */ +REGISTER_FACTORY(Factory, Upstream::TypedLoadBalancerFactory); + } // namespace LeastRequest } // namespace LoadBalancingPolices } // namespace Extensions diff --git a/source/extensions/load_balancing_policies/least_request/config.h b/source/extensions/load_balancing_policies/least_request/config.h index 44fd31c8b13da..702644a93e6f6 100644 --- a/source/extensions/load_balancing_policies/least_request/config.h +++ b/source/extensions/load_balancing_policies/least_request/config.h @@ -6,7 +6,7 @@ #include "source/extensions/load_balancing_policies/common/factory_base.h" -#include "/source/common/common/logger.h" +#include "source/common/common/logger.h" namespace Envoy { namespace Extensions { @@ -21,7 +21,7 @@ struct LeastRequestCreator : public Logger::Loggable { TimeSource& time_source); }; -class Factory : Common::FactoryBase< +class Factory : public Common::FactoryBase< envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest, LeastRequestCreator> { public: From fd728cecfb0434a50dde75c03aea1b853f1c30ce Mon Sep 17 00:00:00 2001 From: wbpcode Date: Sat, 29 Oct 2022 09:06:48 +0000 Subject: [PATCH 04/21] reverse unnecessary update Signed-off-by: wbpcode --- ci/do_ci.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 2cfdea0b80c93..70a2f4726d79a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -12,7 +12,7 @@ if [[ "$1" == "format" || "$1" == "fix_proto_format" || "$1" == "check_proto_for || "$1" == "verify_distro" ]]; then build_setup_args="-nofetch" fi -build_setup_args="-nofetch" + # TODO(phlax): Clarify and/or integrate SRCDIR and ENVOY_SRCDIR export SRCDIR="${SRCDIR:-$PWD}" export ENVOY_SRCDIR="${ENVOY_SRCDIR:-$PWD}" @@ -225,10 +225,10 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c opt "${TEST_TARGETS[@]}" echo "bazel release build..." - # bazel_envoy_binary_build release + bazel_envoy_binary_build release echo "bazel contrib release build..." - # bazel_contrib_binary_build release + bazel_contrib_binary_build release exit 0 elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then From 89c15829092f1e8f4e7ed0f073d2e3089c29c295 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Sat, 29 Oct 2022 10:32:21 +0000 Subject: [PATCH 05/21] add test Signed-off-by: wbpcode --- source/common/upstream/BUILD | 1 + source/common/upstream/load_balancer_impl.h | 18 +++---- .../least_request/config.cc | 25 ++++----- .../least_request/BUILD | 24 +++++++++ .../least_request/config_test.cc | 51 +++++++++++++++++++ 5 files changed, 94 insertions(+), 25 deletions(-) create mode 100644 test/extensions/load_balancing_policies/least_request/BUILD create mode 100644 test/extensions/load_balancing_policies/least_request/config_test.cc diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 50206adf02562..531246cf0bf30 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -260,6 +260,7 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_protos_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/least_request/v3:pkg_cc_proto", ], ) diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 6e677afa2d1d8..bc7f4bc4537e7 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -11,6 +11,8 @@ #include "envoy/common/callback.h" #include "envoy/common/random_generator.h" #include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.h" +#include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.validate.h" #include "envoy/runtime/runtime.h" #include "envoy/upstream/load_balancer.h" #include "envoy/upstream/upstream.h" @@ -575,25 +577,21 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase { const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterStats& stats, Runtime::Loader& runtime, Random::RandomGenerator& random, const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config, - const absl::optional + const envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest& least_request_config, TimeSource& time_source) : EdfLoadBalancerBase( priority_set, local_priority_set, stats, runtime, random, common_config, - (least_request_config.has_value() && - least_request_config.value().has_slow_start_config()) + (least_request_config.has_slow_start_config()) ? absl::optional( - least_request_config.value().slow_start_config()) + least_request_config.slow_start_config()) : absl::nullopt, time_source), - choice_count_( - least_request_config.has_value() - ? PROTOBUF_GET_WRAPPED_OR_DEFAULT(least_request_config.value(), choice_count, 2) - : 2), + choice_count_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(least_request_config, choice_count, 2)), active_request_bias_runtime_( - least_request_config.has_value() && least_request_config->has_active_request_bias() + least_request_config.has_active_request_bias() ? absl::optional( - {least_request_config->active_request_bias(), runtime}) + {least_request_config.active_request_bias(), runtime}) : absl::nullopt) { initialize(); } diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc index 047daa47f32ec..34aa562d567ed 100644 --- a/source/extensions/load_balancing_policies/least_request/config.cc +++ b/source/extensions/load_balancing_policies/least_request/config.cc @@ -1,5 +1,6 @@ #include "source/extensions/load_balancing_policies/least_request/config.h" +#include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.h" #include "source/common/upstream/load_balancer_impl.h" namespace Envoy { @@ -13,23 +14,17 @@ Upstream::LoadBalancerPtr LeastRequestCreator::operator()(Upstream::LoadBalancer Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) { - // Use the old proto type for now. - envoy::config::cluster::v3::Cluster::LeastRequestLbConfig old_type_config; - try { - MessageUtil::wireCast(*cluster_info.loadBalancingPolicy(), old_type_config); - } catch (EnvoyException& e) { - // Cast failed and exit in debug mode. Continue in release mode with a error message. - ASSERT(false, - fmt::format("Failed to cast load balancing policy to least request: {}", e.what())); - ENVOY_LOG(error, - "Cannot cast least request load balancing policy configuration to " - "envoy::config::cluster::v3::Cluster::LeastRequestLbConfig: {}", - e.what()); - old_type_config.Clear(); - } + + const auto* typed_config = dynamic_cast< + const envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest*>( + cluster_info.loadBalancingPolicy().get()); + + RELEASE_ASSERT(typed_config != nullptr, + "Invalid load balancing policy configuration for least request load balancer"); + return std::make_unique( params.priority_set, params.local_priority_set, cluster_info.stats(), runtime, random, - cluster_info.lbConfig(), old_type_config, time_source); + cluster_info.lbConfig(), *typed_config, time_source); } /** diff --git a/test/extensions/load_balancing_policies/least_request/BUILD b/test/extensions/load_balancing_policies/least_request/BUILD new file mode 100644 index 0000000000000..48531783d24d6 --- /dev/null +++ b/test/extensions/load_balancing_policies/least_request/BUILD @@ -0,0 +1,24 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.load_balancing_policies.least_request"], + deps = [ + "//source/extensions/load_balancing_policies/least_request:config", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:priority_set_mocks", + ], +) diff --git a/test/extensions/load_balancing_policies/least_request/config_test.cc b/test/extensions/load_balancing_policies/least_request/config_test.cc new file mode 100644 index 0000000000000..aca42afb16a67 --- /dev/null +++ b/test/extensions/load_balancing_policies/least_request/config_test.cc @@ -0,0 +1,51 @@ +#include "envoy/config/core/v3/extension.pb.h" +#include "source/extensions/load_balancing_policies/least_request/config.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/mocks/upstream/priority_set.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace LeastRequest { +namespace { + +TEST(LeastRequestConfigTest, ValidateFail) { + NiceMock context; + NiceMock cluster_info; + NiceMock main_thread_priority_set; + NiceMock thread_local_priority_set; + + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("envoy.load_balancing_policies.least_request"); + envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest config_msg; + config.mutable_typed_config()->PackFrom(config_msg); + + auto& factory = Config::Utility::getAndCheckFactory(config); + EXPECT_EQ("envoy.load_balancing_policies.least_request", factory.name()); + + auto messsage_ptr = factory.createEmptyConfigProto(); + EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(messsage_ptr)); + + auto thread_aware_lb = + factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_, + context.api_.random_, context.time_system_); + EXPECT_NE(nullptr, thread_aware_lb); + + thread_aware_lb->initialize(); + + auto thread_local_lb_factory = thread_aware_lb->factory(); + EXPECT_NE(nullptr, thread_local_lb_factory); + + auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr}); + EXPECT_NE(nullptr, thread_local_lb); +} + +} // namespace +} // namespace LeastRequest +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy From 8f6902204864318f7b1801f3a56758869721ac20 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Sat, 29 Oct 2022 13:24:06 +0000 Subject: [PATCH 06/21] fix docs, release note, format and everything Signed-off-by: wbpcode --- CODEOWNERS | 3 +++ api/BUILD | 1 + api/envoy/config/cluster/v3/cluster.proto | 1 + .../least_request/v3/least_request.proto | 3 +-- changelogs/current.yaml | 3 +++ docs/root/api-v3/config/config.rst | 1 + .../load_balancing_policies/load_balancing_policies.rst | 8 ++++++++ .../load_balancing_policies/least_request/config.cc | 1 + .../load_balancing_policies/least_request/config.h | 3 +-- .../load_balancing_policies/least_request/BUILD | 1 + .../load_balancing_policies/least_request/config_test.cc | 1 + tools/extensions/extensions_schema.yaml | 1 + 12 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst diff --git a/CODEOWNERS b/CODEOWNERS index b30e99bf10e24..82634d866fb55 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -273,6 +273,9 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/path/rewrite/uri_template @alyssawilk @yanjunxiang-google # Dubbo codec /*/extensions/common/dubbo @wbpcode @lizan +# upstream load balancing policies +/*/extensions/load_balancing_policies/common @wbpcode @UNOWNED +/*/extensions/load_balancing_policies/least_request @wbpcode @UNOWNED # Intentionally exempt (treated as core code) /*/extensions/filters/common @UNOWNED @UNOWNED diff --git a/api/BUILD b/api/BUILD index 29414b11265be..eae3bc6ed65c3 100644 --- a/api/BUILD +++ b/api/BUILD @@ -239,6 +239,7 @@ proto_library( "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", "//envoy/extensions/key_value/file_based/v3:pkg", + "//envoy/extensions/load_balancing_policies/least_request/v3:pkg", "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg", "//envoy/extensions/matching/common_inputs/network/v3:pkg", "//envoy/extensions/matching/common_inputs/ssl/v3:pkg", diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index 79c9909b40785..953ca4eb41edf 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -1215,6 +1215,7 @@ message LoadBalancingPolicy { reserved "config", "name", "typed_config"; + // [#extension-category: envoy.load_balancing_policies] core.v3.TypedExtensionConfig typed_extension_config = 4; } diff --git a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto index fd34260d05ede..7de2d8bea0ae6 100644 --- a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto +++ b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto @@ -17,12 +17,11 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/loa option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Least Request Load Balancing Policy] -// [#not-implemented-hide:] +// [#extension: envoy.load_balancing_policies.least_request] // This configuration allows the built-in LEAST_REQUEST LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` for more information. -// [#extension: envoy.load_balancing_policies] message LeastRequest { // The number of random healthy hosts from which the host with the fewest active requests will // be chosen. Defaults to 2 so that we perform two-choice selection if the field is not set. diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 23ee1c0ecb6cb..de231f129daaf 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -62,5 +62,8 @@ new_features: - area: http change: | allowing the dynamic forward proxy cluster to :ref:`allow_coalesced_connections ` for HTTP/2 and HTTP/3 connections. +- area: upstream + change: | + added :ref:`least request extension ` to suppport the :ref:`load balancer policy `. deprecated: diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst index cfb4b26e6ae5c..c70d9dadc7478 100644 --- a/docs/root/api-v3/config/config.rst +++ b/docs/root/api-v3/config/config.rst @@ -40,3 +40,4 @@ Extensions upstream/upstream wasm/wasm watchdog/watchdog + load_balancing_policies/load_balancing_policies diff --git a/docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst b/docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst new file mode 100644 index 0000000000000..c3b8c2f6d1f4d --- /dev/null +++ b/docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst @@ -0,0 +1,8 @@ +Load balancing policies +======================= + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../extensions/load_balancing_policies/*/v3/* diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc index 34aa562d567ed..8cbee7e303ebf 100644 --- a/source/extensions/load_balancing_policies/least_request/config.cc +++ b/source/extensions/load_balancing_policies/least_request/config.cc @@ -1,6 +1,7 @@ #include "source/extensions/load_balancing_policies/least_request/config.h" #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.h" + #include "source/common/upstream/load_balancer_impl.h" namespace Envoy { diff --git a/source/extensions/load_balancing_policies/least_request/config.h b/source/extensions/load_balancing_policies/least_request/config.h index 702644a93e6f6..e2e787936c283 100644 --- a/source/extensions/load_balancing_policies/least_request/config.h +++ b/source/extensions/load_balancing_policies/least_request/config.h @@ -4,9 +4,8 @@ #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.validate.h" #include "envoy/upstream/load_balancer.h" -#include "source/extensions/load_balancing_policies/common/factory_base.h" - #include "source/common/common/logger.h" +#include "source/extensions/load_balancing_policies/common/factory_base.h" namespace Envoy { namespace Extensions { diff --git a/test/extensions/load_balancing_policies/least_request/BUILD b/test/extensions/load_balancing_policies/least_request/BUILD index 48531783d24d6..e0d46ad819d45 100644 --- a/test/extensions/load_balancing_policies/least_request/BUILD +++ b/test/extensions/load_balancing_policies/least_request/BUILD @@ -20,5 +20,6 @@ envoy_extension_cc_test( "//test/mocks/server:factory_context_mocks", "//test/mocks/upstream:cluster_info_mocks", "//test/mocks/upstream:priority_set_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/load_balancing_policies/least_request/config_test.cc b/test/extensions/load_balancing_policies/least_request/config_test.cc index aca42afb16a67..11d1ddeed489f 100644 --- a/test/extensions/load_balancing_policies/least_request/config_test.cc +++ b/test/extensions/load_balancing_policies/least_request/config_test.cc @@ -1,4 +1,5 @@ #include "envoy/config/core/v3/extension.pb.h" + #include "source/extensions/load_balancing_policies/least_request/config.h" #include "test/mocks/server/factory_context.h" diff --git a/tools/extensions/extensions_schema.yaml b/tools/extensions/extensions_schema.yaml index d51644e2a6965..4932926560200 100644 --- a/tools/extensions/extensions_schema.yaml +++ b/tools/extensions/extensions_schema.yaml @@ -117,6 +117,7 @@ categories: - envoy.tls_handshakers - envoy.generic_proxy.filters - envoy.generic_proxy.codecs +- envoy.load_balancing_policies status_values: - name: stable From 2c9b3212ae1e20675f62b8e8fab9cb65387d649a Mon Sep 17 00:00:00 2001 From: wbpcode Date: Sat, 29 Oct 2022 13:47:46 +0000 Subject: [PATCH 07/21] revert unexpected deltetion Signed-off-by: wbpcode --- source/common/upstream/load_balancer_impl.h | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index bc7f4bc4537e7..3f497d6d506e6 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -573,6 +573,33 @@ class RoundRobinLoadBalancer : public EdfLoadBalancerBase { */ class LeastRequestLoadBalancer : public EdfLoadBalancerBase { public: + LeastRequestLoadBalancer( + const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterStats& stats, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config, + const absl::optional + least_request_config, + TimeSource& time_source) + : EdfLoadBalancerBase( + priority_set, local_priority_set, stats, runtime, random, common_config, + (least_request_config.has_value() && + least_request_config.value().has_slow_start_config()) + ? absl::optional( + least_request_config.value().slow_start_config()) + : absl::nullopt, + time_source), + choice_count_( + least_request_config.has_value() + ? PROTOBUF_GET_WRAPPED_OR_DEFAULT(least_request_config.value(), choice_count, 2) + : 2), + active_request_bias_runtime_( + least_request_config.has_value() && least_request_config->has_active_request_bias() + ? absl::optional( + {least_request_config->active_request_bias(), runtime}) + : absl::nullopt) { + initialize(); + } + LeastRequestLoadBalancer( const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterStats& stats, Runtime::Loader& runtime, Random::RandomGenerator& random, From 2fc6360cfb91db1f286fdd7004ada536310d036a Mon Sep 17 00:00:00 2001 From: wbpcode Date: Sun, 30 Oct 2022 03:23:42 +0000 Subject: [PATCH 08/21] try for coverage Signed-off-by: wbpcode --- .../load_balancing_policies/least_request/config_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/extensions/load_balancing_policies/least_request/config_test.cc b/test/extensions/load_balancing_policies/least_request/config_test.cc index 11d1ddeed489f..ef4fd506d0006 100644 --- a/test/extensions/load_balancing_policies/least_request/config_test.cc +++ b/test/extensions/load_balancing_policies/least_request/config_test.cc @@ -43,6 +43,8 @@ TEST(LeastRequestConfigTest, ValidateFail) { auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr}); EXPECT_NE(nullptr, thread_local_lb); + + EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented"); } } // namespace From 0654452fa33cec4f8e2e2c581beebde902596667 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Sun, 30 Oct 2022 09:30:21 +0000 Subject: [PATCH 09/21] update per file coverage Signed-off-by: wbpcode --- test/per_file_coverage.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 81bf2ea3aabaf..46ae79284a824 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -96,6 +96,8 @@ declare -a KNOWN_LOW_COVERAGE=( "source/server/admin:97.5" "source/server/admin:profiler-lib:83" "source/server/config_validation:74.8" +"source/extensions/load_balancing_policies:96" +"source/extensions/load_balancing_policies/common:94" # Death tests don't report LCOV ) [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" From 5fc44ed7bfc1e69f9a3613f2c20d8eccf9e7d255 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Tue, 20 Dec 2022 13:00:25 +0000 Subject: [PATCH 10/21] upate all Signed-off-by: wbpcode --- CODEOWNERS | 2 + .../common/v3/common.proto | 1 - .../random/v3/random.proto | 2 +- .../round_robin/v3/round_robin.proto | 2 +- source/common/upstream/BUILD | 2 + source/common/upstream/load_balancer_impl.cc | 75 ++++++--- source/common/upstream/load_balancer_impl.h | 144 ++++++++++++++---- source/extensions/extensions_build_config.bzl | 3 + source/extensions/extensions_metadata.yaml | 14 ++ .../least_request/config.cc | 6 +- .../load_balancing_policies/random/BUILD | 21 +++ .../load_balancing_policies/random/config.cc | 41 +++++ .../load_balancing_policies/random/config.h | 32 ++++ .../load_balancing_policies/round_robin/BUILD | 21 +++ .../round_robin/config.cc | 41 +++++ .../round_robin/config.h | 33 ++++ .../load_balancing_policies/random/BUILD | 25 +++ .../random/config_test.cc | 54 +++++++ .../load_balancing_policies/round_robin/BUILD | 25 +++ .../round_robin/config_test.cc | 54 +++++++ 20 files changed, 540 insertions(+), 58 deletions(-) create mode 100644 source/extensions/load_balancing_policies/random/BUILD create mode 100644 source/extensions/load_balancing_policies/random/config.cc create mode 100644 source/extensions/load_balancing_policies/random/config.h create mode 100644 source/extensions/load_balancing_policies/round_robin/BUILD create mode 100644 source/extensions/load_balancing_policies/round_robin/config.cc create mode 100644 source/extensions/load_balancing_policies/round_robin/config.h create mode 100644 test/extensions/load_balancing_policies/random/BUILD create mode 100644 test/extensions/load_balancing_policies/random/config_test.cc create mode 100644 test/extensions/load_balancing_policies/round_robin/BUILD create mode 100644 test/extensions/load_balancing_policies/round_robin/config_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 488be7c63e519..cf716e0eb61d1 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -287,6 +287,8 @@ extensions/filters/http/oauth2 @derekargueta @snowp # upstream load balancing policies /*/extensions/load_balancing_policies/common @wbpcode @UNOWNED /*/extensions/load_balancing_policies/least_request @wbpcode @UNOWNED +/*/extensions/load_balancing_policies/random @wbpcode @UNOWNED +/*/extensions/load_balancing_policies/round_robin @wbpcode @UNOWNED # Early header mutation /*/extensions/http/early_header_mutation/header_mutation @wbpcode @UNOWNED diff --git a/api/envoy/extensions/load_balancing_policies/common/v3/common.proto b/api/envoy/extensions/load_balancing_policies/common/v3/common.proto index a19765f77b4b7..51520690a29ac 100644 --- a/api/envoy/extensions/load_balancing_policies/common/v3/common.proto +++ b/api/envoy/extensions/load_balancing_policies/common/v3/common.proto @@ -18,7 +18,6 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/loa option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Common configuration for two or more load balancing policy extensions] -// [#not-implemented-hide:] message LocalityLbConfig { // Configuration for :ref:`zone aware routing diff --git a/api/envoy/extensions/load_balancing_policies/random/v3/random.proto b/api/envoy/extensions/load_balancing_policies/random/v3/random.proto index 37efa7c13305c..1fc0a49049be9 100644 --- a/api/envoy/extensions/load_balancing_policies/random/v3/random.proto +++ b/api/envoy/extensions/load_balancing_policies/random/v3/random.proto @@ -13,7 +13,7 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/loa option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Random Load Balancing Policy] -// [#not-implemented-hide:] +// [#extension: envoy.load_balancing_policies.random] // This configuration allows the built-in Random LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview diff --git a/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto b/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto index 01b8a62e002a4..63fe49e9b0bca 100644 --- a/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto +++ b/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto @@ -13,7 +13,7 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/loa option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Round Robin Load Balancing Policy] -// [#not-implemented-hide:] +// [#extension: envoy.load_balancing_policies.round_robin] // This configuration allows the built-in ROUND_ROBIN LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 7d344b7081691..0c2a90d3f6336 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -266,6 +266,8 @@ envoy_cc_library( "//source/common/runtime:runtime_protos_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/least_request/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/random/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/round_robin/v3:pkg_cc_proto", ], ) diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index eb719f7244d6c..8c2fb90271dd1 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -77,6 +77,36 @@ bool hostWeightsAreEqual(const HostVector& hosts) { } // namespace +absl::optional +LoadBalancerConfigHelper::localityLbConfigFromCommonLbConfig( + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) { + + if (common_config.has_locality_weighted_lb_config()) { + envoy::extensions::load_balancing_policies::common::v3::LocalityLbConfig locality_lb_config; + locality_lb_config.mutable_locality_weighted_lb_config(); + return locality_lb_config; + } else if (common_config.has_zone_aware_lb_config()) { + envoy::extensions::load_balancing_policies::common::v3::LocalityLbConfig locality_lb_config; + auto& zone_aware_lb_config = *locality_lb_config.mutable_zone_aware_lb_config(); + + const auto& legacy_zone_aware_lb_config = common_config.zone_aware_lb_config(); + if (legacy_zone_aware_lb_config.has_routing_enabled()) { + *zone_aware_lb_config.mutable_routing_enabled() = + legacy_zone_aware_lb_config.routing_enabled(); + } + if (legacy_zone_aware_lb_config.has_min_cluster_size()) { + *zone_aware_lb_config.mutable_min_cluster_size() = + legacy_zone_aware_lb_config.min_cluster_size(); + } + zone_aware_lb_config.set_fail_traffic_on_panic( + legacy_zone_aware_lb_config.fail_traffic_on_panic()); + + return locality_lb_config; + } + + return {}; +} + std::pair LoadBalancerBase::choosePriority(uint64_t hash, const HealthyLoad& healthy_per_priority_load, const DegradedLoad& degraded_per_priority_load) { @@ -108,14 +138,11 @@ LoadBalancerBase::choosePriority(uint64_t hash, const HealthyLoad& healthy_per_p return {0, HostAvailability::Healthy}; } -LoadBalancerBase::LoadBalancerBase( - const PrioritySet& priority_set, ClusterLbStats& stats, Runtime::Loader& runtime, - Random::RandomGenerator& random, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) +LoadBalancerBase::LoadBalancerBase(const PrioritySet& priority_set, ClusterLbStats& stats, + Runtime::Loader& runtime, Random::RandomGenerator& random, + uint32_t healthy_panic_threshold) : stats_(stats), runtime_(runtime), random_(random), - default_healthy_panic_percent_(PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( - common_config, healthy_panic_threshold, 100, 50)), - priority_set_(priority_set) { + default_healthy_panic_percent_(healthy_panic_threshold), priority_set_(priority_set) { for (auto& host_set : priority_set_.hostSetsPerPriority()) { recalculatePerPriorityState(host_set->priority(), priority_set_, per_priority_load_, per_priority_health_, per_priority_degraded_, total_healthy_hosts_); @@ -352,16 +379,23 @@ LoadBalancerBase::chooseHostSet(LoadBalancerContext* context, uint64_t hash) con ZoneAwareLoadBalancerBase::ZoneAwareLoadBalancerBase( const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) - : LoadBalancerBase(priority_set, stats, runtime, random, common_config), + Runtime::Loader& runtime, Random::RandomGenerator& random, uint32_t healthy_panic_threshold, + const absl::optional locality_config) + : LoadBalancerBase(priority_set, stats, runtime, random, healthy_panic_threshold), local_priority_set_(local_priority_set), - locality_weighted_balancing_(common_config.has_locality_weighted_lb_config()), - routing_enabled_(PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( - common_config.zone_aware_lb_config(), routing_enabled, 100, 100)), - min_cluster_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(common_config.zone_aware_lb_config(), - min_cluster_size, 6U)), - fail_traffic_on_panic_(common_config.zone_aware_lb_config().fail_traffic_on_panic()) { + locality_weighted_balancing_(locality_config.has_value() && + locality_config->has_locality_weighted_lb_config()), + routing_enabled_(locality_config.has_value() + ? PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( + locality_config->zone_aware_lb_config(), routing_enabled, 100, 100) + : 100), + min_cluster_size_(locality_config.has_value() + ? PROTOBUF_GET_WRAPPED_OR_DEFAULT( + locality_config->zone_aware_lb_config(), min_cluster_size, 6U) + : 6U), + fail_traffic_on_panic_(locality_config.has_value() + ? locality_config->zone_aware_lb_config().fail_traffic_on_panic() + : false) { ASSERT(!priority_set.hostSetsPerPriority().empty()); resizePerPriorityState(); priority_update_cb_ = priority_set_.addPriorityUpdateCb( @@ -727,12 +761,11 @@ const HostVector& ZoneAwareLoadBalancerBase::hostSourceToHosts(HostsSource hosts EdfLoadBalancerBase::EdfLoadBalancerBase( const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config, - const absl::optional slow_start_config, - TimeSource& time_source) + Runtime::Loader& runtime, Random::RandomGenerator& random, uint32_t healthy_panic_threshold, + const absl::optional locality_config, + const absl::optional slow_start_config, TimeSource& time_source) : ZoneAwareLoadBalancerBase(priority_set, local_priority_set, stats, runtime, random, - common_config), + healthy_panic_threshold, locality_config), seed_(random_.random()), slow_start_window_(slow_start_config.has_value() ? std::chrono::milliseconds(DurationUtil::durationToMilliseconds( diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 5b9a7253c3eff..bf3db5242e3c1 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -11,8 +11,13 @@ #include "envoy/common/callback.h" #include "envoy/common/random_generator.h" #include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/extensions/load_balancing_policies/common/v3/common.pb.h" #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.h" #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.validate.h" +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h" +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.validate.h" +#include "envoy/extensions/load_balancing_policies/random/v3/random.pb.h" +#include "envoy/extensions/load_balancing_policies/random/v3/random.pb.validate.h" #include "envoy/runtime/runtime.h" #include "envoy/upstream/load_balancer.h" #include "envoy/upstream/upstream.h" @@ -27,6 +32,53 @@ namespace Upstream { // Priority levels and localities are considered overprovisioned with this factor. static constexpr uint32_t kDefaultOverProvisioningFactor = 140; +class LoadBalancerConfigHelper { +public: + template + static absl::optional + slowStartConfigFromLegacyProto(const LegacyProto& proto_config) { + if (!proto_config.has_slow_start_config()) { + return {}; + } + + envoy::extensions::load_balancing_policies::common::v3::SlowStartConfig slow_start_config; + const auto& legacy_slow_start_config = proto_config.slow_start_config(); + if (legacy_slow_start_config.has_slow_start_window()) { + *slow_start_config.mutable_slow_start_window() = legacy_slow_start_config.slow_start_window(); + } + if (legacy_slow_start_config.has_aggression()) { + *slow_start_config.mutable_aggression() = legacy_slow_start_config.aggression(); + } + if (legacy_slow_start_config.has_min_weight_percent()) { + *slow_start_config.mutable_min_weight_percent() = + legacy_slow_start_config.min_weight_percent(); + } + return slow_start_config; + } + + template + static absl::optional + slowStartConfigFromProto(const Proto& proto_config) { + if (!proto_config.has_slow_start_config()) { + return {}; + } + return proto_config.slow_start_config(); + } + + static absl::optional + localityLbConfigFromCommonLbConfig( + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config); + + template + static absl::optional + localityLbConfigFromProto(const Proto& proto_config) { + if (!proto_config.has_locality_lb_config()) { + return {}; + } + return proto_config.locality_lb_config(); + } +}; + /** * Base class for all LB implementations. */ @@ -71,8 +123,7 @@ class LoadBalancerBase : public LoadBalancer { void recalculateLoadInTotalPanic(); LoadBalancerBase(const PrioritySet& priority_set, ClusterLbStats& stats, Runtime::Loader& runtime, - Random::RandomGenerator& random, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config); + Random::RandomGenerator& random, uint32_t healthy_panic_threshold); // Choose host set randomly, based on the healthy_per_priority_load_ and // degraded_per_priority_load_. per_priority_load_ is consulted first, spilling over to @@ -196,14 +247,16 @@ class LoadBalancerContextBase : public LoadBalancerContext { */ class ZoneAwareLoadBalancerBase : public LoadBalancerBase { public: + using LocalityLbConfig = envoy::extensions::load_balancing_policies::common::v3::LocalityLbConfig; + HostConstSharedPtr chooseHost(LoadBalancerContext* context) override; protected: // Both priority_set and local_priority_set if non-null must have at least one host set. - ZoneAwareLoadBalancerBase( - const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config); + ZoneAwareLoadBalancerBase(const PrioritySet& priority_set, const PrioritySet* local_priority_set, + ClusterLbStats& stats, Runtime::Loader& runtime, + Random::RandomGenerator& random, uint32_t healthy_panic_threshold, + const absl::optional locality_config); // When deciding which hosts to use on an LB decision, we need to know how to index into the // priority_set. This priority_set cursor is used by ZoneAwareLoadBalancerBase subclasses, e.g. @@ -406,12 +459,14 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase { class EdfLoadBalancerBase : public ZoneAwareLoadBalancerBase, Logger::Loggable { public: - EdfLoadBalancerBase( - const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config, - const absl::optional slow_start_cofig, - TimeSource& time_source); + using SlowStartConfig = envoy::extensions::load_balancing_policies::common::v3::SlowStartConfig; + + EdfLoadBalancerBase(const PrioritySet& priority_set, const PrioritySet* local_priority_set, + ClusterLbStats& stats, Runtime::Loader& runtime, + Random::RandomGenerator& random, uint32_t healthy_panic_threshold, + const absl::optional locality_config, + const absl::optional slow_start_config, + TimeSource& time_source); // Upstream::ZoneAwareLoadBalancerBase HostConstSharedPtr peekAnotherHost(LoadBalancerContext* context) override; @@ -482,15 +537,31 @@ class RoundRobinLoadBalancer : public EdfLoadBalancerBase { round_robin_config, TimeSource& time_source) : EdfLoadBalancerBase( - priority_set, local_priority_set, stats, runtime, random, common_config, - (round_robin_config.has_value() && round_robin_config.value().has_slow_start_config()) - ? absl::optional( - round_robin_config.value().slow_start_config()) + priority_set, local_priority_set, stats, runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(common_config, healthy_panic_threshold, + 100, 50), + LoadBalancerConfigHelper::localityLbConfigFromCommonLbConfig(common_config), + round_robin_config.has_value() + ? LoadBalancerConfigHelper::slowStartConfigFromLegacyProto( + round_robin_config.value()) : absl::nullopt, time_source) { initialize(); } + RoundRobinLoadBalancer( + const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, + Runtime::Loader& runtime, Random::RandomGenerator& random, uint32_t healthy_panic_threshold, + const envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin& + round_robin_config, + TimeSource& time_source) + : EdfLoadBalancerBase( + priority_set, local_priority_set, stats, runtime, random, healthy_panic_threshold, + LoadBalancerConfigHelper::localityLbConfigFromProto(round_robin_config), + LoadBalancerConfigHelper::slowStartConfigFromProto(round_robin_config), time_source) { + initialize(); + } + private: void refreshHostSource(const HostsSource& source) override { // insert() is used here on purpose so that we don't overwrite the index if the host source @@ -560,11 +631,13 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase { least_request_config, TimeSource& time_source) : EdfLoadBalancerBase( - priority_set, local_priority_set, stats, runtime, random, common_config, - (least_request_config.has_value() && - least_request_config.value().has_slow_start_config()) - ? absl::optional( - least_request_config.value().slow_start_config()) + priority_set, local_priority_set, stats, runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(common_config, healthy_panic_threshold, + 100, 50), + LoadBalancerConfigHelper::localityLbConfigFromCommonLbConfig(common_config), + least_request_config.has_value() + ? LoadBalancerConfigHelper::slowStartConfigFromLegacyProto( + least_request_config.value()) : absl::nullopt, time_source), choice_count_( @@ -580,19 +653,15 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase { } LeastRequestLoadBalancer( - const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterStats& stats, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config, + const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, + Runtime::Loader& runtime, Random::RandomGenerator& random, uint32_t healthy_panic_threshold, const envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest& least_request_config, TimeSource& time_source) : EdfLoadBalancerBase( - priority_set, local_priority_set, stats, runtime, random, common_config, - (least_request_config.has_slow_start_config()) - ? absl::optional( - least_request_config.slow_start_config()) - : absl::nullopt, - time_source), + priority_set, local_priority_set, stats, runtime, random, healthy_panic_threshold, + LoadBalancerConfigHelper::localityLbConfigFromProto(least_request_config), + LoadBalancerConfigHelper::slowStartConfigFromProto(least_request_config), time_source), choice_count_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(least_request_config, choice_count, 2)), active_request_bias_runtime_( least_request_config.has_active_request_bias() @@ -679,8 +748,19 @@ class RandomLoadBalancer : public ZoneAwareLoadBalancerBase, ClusterLbStats& stats, Runtime::Loader& runtime, Random::RandomGenerator& random, const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) - : ZoneAwareLoadBalancerBase(priority_set, local_priority_set, stats, runtime, random, - common_config) {} + : ZoneAwareLoadBalancerBase( + priority_set, local_priority_set, stats, runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(common_config, healthy_panic_threshold, + 100, 50), + LoadBalancerConfigHelper::localityLbConfigFromCommonLbConfig(common_config)) {} + + RandomLoadBalancer( + const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, + Runtime::Loader& runtime, Random::RandomGenerator& random, uint32_t healthy_panic_threshold, + const envoy::extensions::load_balancing_policies::random::v3::Random& random_config) + : ZoneAwareLoadBalancerBase( + priority_set, local_priority_set, stats, runtime, random, healthy_panic_threshold, + LoadBalancerConfigHelper::localityLbConfigFromProto(random_config)) {} // Upstream::ZoneAwareLoadBalancerBase HostConstSharedPtr chooseHostOnce(LoadBalancerContext* context) override; diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index f215462844930..eccab9f9b9715 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -396,6 +396,9 @@ EXTENSIONS = { # Load balancing policies for upstream # "envoy.load_balancing_policies.least_request": "//source/extensions/load_balancing_policies/least_request:config", + "envoy.load_balancing_policies.random": "//source/extensions/load_balancing_policies/random:config", + "envoy.load_balancing_policies.round_robin": "//source/extensions/load_balancing_policies/round_robin:config", + # HTTP Early Header Mutation # diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 4e3a22099f9c1..c0c57fb413626 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -1350,6 +1350,20 @@ envoy.load_balancing_policies.least_request: status: alpha type_urls: - envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest +envoy.load_balancing_policies.random: + categories: + - envoy.load_balancing_policies + security_posture: unknown + status: alpha + type_urls: + - envoy.extensions.load_balancing_policies.random.v3.Random +envoy.load_balancing_policies.round_robin: + categories: + - envoy.load_balancing_policies + security_posture: unknown + status: alpha + type_urls: + - envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin envoy.http.early_header_mutation.header_mutation: categories: - envoy.http.early_header_mutation diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc index 8cbee7e303ebf..ec8a6081644f3 100644 --- a/source/extensions/load_balancing_policies/least_request/config.cc +++ b/source/extensions/load_balancing_policies/least_request/config.cc @@ -24,8 +24,10 @@ Upstream::LoadBalancerPtr LeastRequestCreator::operator()(Upstream::LoadBalancer "Invalid load balancing policy configuration for least request load balancer"); return std::make_unique( - params.priority_set, params.local_priority_set, cluster_info.stats(), runtime, random, - cluster_info.lbConfig(), *typed_config, time_source); + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), + healthy_panic_threshold, 100, 50), + *typed_config, time_source); } /** diff --git a/source/extensions/load_balancing_policies/random/BUILD b/source/extensions/load_balancing_policies/random/BUILD new file mode 100644 index 0000000000000..a803dbb22f403 --- /dev/null +++ b/source/extensions/load_balancing_policies/random/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//source/common/common:minimal_logger_lib", + "//source/common/upstream:load_balancer_lib", + "//source/extensions/load_balancing_policies/common:factory_base", + "@envoy_api//envoy/extensions/load_balancing_policies/random/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/load_balancing_policies/random/config.cc b/source/extensions/load_balancing_policies/random/config.cc new file mode 100644 index 0000000000000..b36dda5d65b25 --- /dev/null +++ b/source/extensions/load_balancing_policies/random/config.cc @@ -0,0 +1,41 @@ +#include "source/extensions/load_balancing_policies/random/config.h" + +#include "envoy/extensions/load_balancing_policies/random/v3/random.pb.h" + +#include "source/common/upstream/load_balancer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Random { + +Upstream::LoadBalancerPtr RandomCreator::operator()(Upstream::LoadBalancerParams params, + const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet&, + Runtime::Loader& runtime, + Envoy::Random::RandomGenerator& random, + TimeSource&) { + + const auto* typed_config = + dynamic_cast( + cluster_info.loadBalancingPolicy().get()); + + RELEASE_ASSERT(typed_config != nullptr, + "Invalid load balancing policy configuration for random load balancer"); + + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), + healthy_panic_threshold, 100, 50), + *typed_config); +} + +/** + * Static registration for the Factory. @see RegisterFactory. + */ +REGISTER_FACTORY(Factory, Upstream::TypedLoadBalancerFactory); + +} // namespace Random +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/random/config.h b/source/extensions/load_balancing_policies/random/config.h new file mode 100644 index 0000000000000..b40754536a670 --- /dev/null +++ b/source/extensions/load_balancing_policies/random/config.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/extensions/load_balancing_policies/random/v3/random.pb.h" +#include "envoy/extensions/load_balancing_policies/random/v3/random.pb.validate.h" +#include "envoy/upstream/load_balancer.h" + +#include "source/common/common/logger.h" +#include "source/extensions/load_balancing_policies/common/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Random { + +struct RandomCreator : public Logger::Loggable { + Upstream::LoadBalancerPtr + operator()(Upstream::LoadBalancerParams params, const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime, + Envoy::Random::RandomGenerator& random, TimeSource& time_source); +}; + +class Factory + : public Common::FactoryBase { +public: + Factory() : FactoryBase("envoy.load_balancing_policies.random") {} +}; + +} // namespace Random +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/round_robin/BUILD b/source/extensions/load_balancing_policies/round_robin/BUILD new file mode 100644 index 0000000000000..a047662218360 --- /dev/null +++ b/source/extensions/load_balancing_policies/round_robin/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//source/common/common:minimal_logger_lib", + "//source/common/upstream:load_balancer_lib", + "//source/extensions/load_balancing_policies/common:factory_base", + "@envoy_api//envoy/extensions/load_balancing_policies/round_robin/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/load_balancing_policies/round_robin/config.cc b/source/extensions/load_balancing_policies/round_robin/config.cc new file mode 100644 index 0000000000000..e78661a72df44 --- /dev/null +++ b/source/extensions/load_balancing_policies/round_robin/config.cc @@ -0,0 +1,41 @@ +#include "source/extensions/load_balancing_policies/round_robin/config.h" + +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h" + +#include "source/common/upstream/load_balancer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace RoundRobin { + +Upstream::LoadBalancerPtr RoundRobinCreator::operator()(Upstream::LoadBalancerParams params, + const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet&, + Runtime::Loader& runtime, + Random::RandomGenerator& random, + TimeSource& time_source) { + + const auto* typed_config = + dynamic_cast( + cluster_info.loadBalancingPolicy().get()); + + RELEASE_ASSERT(typed_config != nullptr, + "Invalid load balancing policy configuration for least request load balancer"); + + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), + healthy_panic_threshold, 100, 50), + *typed_config, time_source); +} + +/** + * Static registration for the Factory. @see RegisterFactory. + */ +REGISTER_FACTORY(Factory, Upstream::TypedLoadBalancerFactory); + +} // namespace RoundRobin +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/round_robin/config.h b/source/extensions/load_balancing_policies/round_robin/config.h new file mode 100644 index 0000000000000..b0138b86dc015 --- /dev/null +++ b/source/extensions/load_balancing_policies/round_robin/config.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h" +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.validate.h" +#include "envoy/upstream/load_balancer.h" + +#include "source/common/common/logger.h" +#include "source/extensions/load_balancing_policies/common/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace RoundRobin { + +struct RoundRobinCreator : public Logger::Loggable { + Upstream::LoadBalancerPtr operator()(Upstream::LoadBalancerParams params, + const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, + TimeSource& time_source); +}; + +class Factory : public Common::FactoryBase< + envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin, + RoundRobinCreator> { +public: + Factory() : FactoryBase("envoy.load_balancing_policies.round_robin") {} +}; + +} // namespace RoundRobin +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/random/BUILD b/test/extensions/load_balancing_policies/random/BUILD new file mode 100644 index 0000000000000..333d879eadbee --- /dev/null +++ b/test/extensions/load_balancing_policies/random/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.load_balancing_policies.random"], + deps = [ + "//source/extensions/load_balancing_policies/random:config", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:priority_set_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/load_balancing_policies/random/config_test.cc b/test/extensions/load_balancing_policies/random/config_test.cc new file mode 100644 index 0000000000000..505855dde3d06 --- /dev/null +++ b/test/extensions/load_balancing_policies/random/config_test.cc @@ -0,0 +1,54 @@ +#include "envoy/config/core/v3/extension.pb.h" + +#include "source/extensions/load_balancing_policies/random/config.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/mocks/upstream/priority_set.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Random { +namespace { + +TEST(RandomConfigTest, ValidateFail) { + NiceMock context; + NiceMock cluster_info; + NiceMock main_thread_priority_set; + NiceMock thread_local_priority_set; + + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("envoy.load_balancing_policies.random"); + envoy::extensions::load_balancing_policies::random::v3::Random config_msg; + config.mutable_typed_config()->PackFrom(config_msg); + + auto& factory = Config::Utility::getAndCheckFactory(config); + EXPECT_EQ("envoy.load_balancing_policies.random", factory.name()); + + auto messsage_ptr = factory.createEmptyConfigProto(); + EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(messsage_ptr)); + + auto thread_aware_lb = + factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_, + context.api_.random_, context.time_system_); + EXPECT_NE(nullptr, thread_aware_lb); + + thread_aware_lb->initialize(); + + auto thread_local_lb_factory = thread_aware_lb->factory(); + EXPECT_NE(nullptr, thread_local_lb_factory); + + auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr}); + EXPECT_NE(nullptr, thread_local_lb); + + EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented"); +} + +} // namespace +} // namespace Random +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/round_robin/BUILD b/test/extensions/load_balancing_policies/round_robin/BUILD new file mode 100644 index 0000000000000..af781df988bed --- /dev/null +++ b/test/extensions/load_balancing_policies/round_robin/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.load_balancing_policies.round_robin"], + deps = [ + "//source/extensions/load_balancing_policies/round_robin:config", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:priority_set_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/load_balancing_policies/round_robin/config_test.cc b/test/extensions/load_balancing_policies/round_robin/config_test.cc new file mode 100644 index 0000000000000..533218cb71df9 --- /dev/null +++ b/test/extensions/load_balancing_policies/round_robin/config_test.cc @@ -0,0 +1,54 @@ +#include "envoy/config/core/v3/extension.pb.h" + +#include "source/extensions/load_balancing_policies/round_robin/config.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/mocks/upstream/priority_set.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace RoundRobin { +namespace { + +TEST(RoundRobinConfigTest, ValidateFail) { + NiceMock context; + NiceMock cluster_info; + NiceMock main_thread_priority_set; + NiceMock thread_local_priority_set; + + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("envoy.load_balancing_policies.round_robin"); + envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin config_msg; + config.mutable_typed_config()->PackFrom(config_msg); + + auto& factory = Config::Utility::getAndCheckFactory(config); + EXPECT_EQ("envoy.load_balancing_policies.round_robin", factory.name()); + + auto messsage_ptr = factory.createEmptyConfigProto(); + EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(messsage_ptr)); + + auto thread_aware_lb = + factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_, + context.api_.random_, context.time_system_); + EXPECT_NE(nullptr, thread_aware_lb); + + thread_aware_lb->initialize(); + + auto thread_local_lb_factory = thread_aware_lb->factory(); + EXPECT_NE(nullptr, thread_local_lb_factory); + + auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr}); + EXPECT_NE(nullptr, thread_local_lb); + + EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented"); +} + +} // namespace +} // namespace RoundRobin +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy From b8e0978ca1a1674a6c1d4178981622b3c5cd0a58 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Wed, 21 Dec 2022 10:00:31 +0000 Subject: [PATCH 11/21] fix format --- source/common/upstream/BUILD | 1 + source/common/upstream/load_balancer_impl.h | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 0c2a90d3f6336..cf3b782ebe5e4 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -265,6 +265,7 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_protos_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/common/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/least_request/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/random/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/round_robin/v3:pkg_cc_proto", diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index bf3db5242e3c1..97685a7102990 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -14,10 +14,10 @@ #include "envoy/extensions/load_balancing_policies/common/v3/common.pb.h" #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.h" #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.validate.h" -#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h" -#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.validate.h" #include "envoy/extensions/load_balancing_policies/random/v3/random.pb.h" #include "envoy/extensions/load_balancing_policies/random/v3/random.pb.validate.h" +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h" +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.validate.h" #include "envoy/runtime/runtime.h" #include "envoy/upstream/load_balancer.h" #include "envoy/upstream/upstream.h" From 560c051cb042c884fb57221e6d9e6a5e5f9c13ca Mon Sep 17 00:00:00 2001 From: wbpcode Date: Thu, 22 Dec 2022 03:56:13 +0000 Subject: [PATCH 12/21] fix test Signed-off-by: wbpcode --- source/common/upstream/thread_aware_lb_impl.h | 4 +++- .../common/upstream/load_balancer_impl_test.cc | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/source/common/upstream/thread_aware_lb_impl.h b/source/common/upstream/thread_aware_lb_impl.h index 2cd2b78a17cb6..12dd97139c814 100644 --- a/source/common/upstream/thread_aware_lb_impl.h +++ b/source/common/upstream/thread_aware_lb_impl.h @@ -110,7 +110,9 @@ class ThreadAwareLoadBalancerBase : public LoadBalancerBase, public ThreadAwareL const PrioritySet& priority_set, ClusterLbStats& stats, Runtime::Loader& runtime, Random::RandomGenerator& random, const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) - : LoadBalancerBase(priority_set, stats, runtime, random, common_config), + : LoadBalancerBase(priority_set, stats, runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( + common_config, healthy_panic_threshold, 100, 50)), factory_(new LoadBalancerFactoryImpl(stats, random)) {} private: diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index ac565799e85e2..77ca220247b36 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -57,8 +57,11 @@ class TestZoneAwareLoadBalancer : public ZoneAwareLoadBalancerBase { const PrioritySet& priority_set, ClusterLbStats& lb_stats, Runtime::Loader& runtime, Random::RandomGenerator& random, const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) - : ZoneAwareLoadBalancerBase(priority_set, nullptr, lb_stats, runtime, random, common_config) { - } + : ZoneAwareLoadBalancerBase( + priority_set, nullptr, lb_stats, runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(common_config, healthy_panic_threshold, + 100, 50), + LoadBalancerConfigHelper::localityLbConfigFromCommonLbConfig(common_config)) {} void runInvalidLocalitySourceType() { localitySourceType(static_cast(123)); } @@ -100,7 +103,9 @@ class TestLb : public LoadBalancerBase { TestLb(const PrioritySet& priority_set, ClusterLbStats& lb_stats, Runtime::Loader& runtime, Random::RandomGenerator& random, const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) - : LoadBalancerBase(priority_set, lb_stats, runtime, random, common_config) {} + : LoadBalancerBase(priority_set, lb_stats, runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( + common_config, healthy_panic_threshold, 100, 50)) {} using LoadBalancerBase::chooseHostSet; using LoadBalancerBase::isInPanic; using LoadBalancerBase::percentageDegradedLoad; @@ -584,8 +589,11 @@ class TestZoneAwareLb : public ZoneAwareLoadBalancerBase { TestZoneAwareLb(const PrioritySet& priority_set, ClusterLbStats& lb_stats, Runtime::Loader& runtime, Random::RandomGenerator& random, const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) - : ZoneAwareLoadBalancerBase(priority_set, nullptr, lb_stats, runtime, random, common_config) { - } + : ZoneAwareLoadBalancerBase( + priority_set, nullptr, lb_stats, runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(common_config, healthy_panic_threshold, + 100, 50), + LoadBalancerConfigHelper::localityLbConfigFromCommonLbConfig(common_config)) {} HostConstSharedPtr chooseHostOnce(LoadBalancerContext*) override { return choose_host_once_host_; From 40eaf43ebd15152e5dc4780bc876ffa0453d8ebd Mon Sep 17 00:00:00 2001 From: wbpcode Date: Fri, 23 Dec 2022 02:11:46 +0000 Subject: [PATCH 13/21] fix test Signed-off-by: wbpcode --- source/extensions/clusters/aggregate/cluster.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/extensions/clusters/aggregate/cluster.h b/source/extensions/clusters/aggregate/cluster.h index 7d0a215d6e9ec..d71d5a558137e 100644 --- a/source/extensions/clusters/aggregate/cluster.h +++ b/source/extensions/clusters/aggregate/cluster.h @@ -95,7 +95,8 @@ class AggregateClusterLoadBalancer : public Upstream::LoadBalancer, Runtime::Loader& runtime, Random::RandomGenerator& random, const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) : Upstream::LoadBalancerBase(priority_context.priority_set_, lb_stats, runtime, random, - common_config), + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( + common_config, healthy_panic_threshold, 100, 50)), priority_context_(priority_context) {} // Upstream::LoadBalancer From f937219ee5540b130893ad1121453b3323b6c78e Mon Sep 17 00:00:00 2001 From: wbpcode Date: Fri, 23 Dec 2022 03:17:18 +0000 Subject: [PATCH 14/21] full namespace for random generator Signed-off-by: wbpcode --- .../load_balancing_policies/common/factory_base.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/extensions/load_balancing_policies/common/factory_base.h b/source/extensions/load_balancing_policies/common/factory_base.h index a957c9dfec2c1..d532ec3d439f6 100644 --- a/source/extensions/load_balancing_policies/common/factory_base.h +++ b/source/extensions/load_balancing_policies/common/factory_base.h @@ -17,7 +17,7 @@ class FactoryBase : public Upstream::TypedLoadBalancerFactory { Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime, - Random::RandomGenerator& random, + Envoy::Random::RandomGenerator& random, TimeSource& time_source) override { return std::make_unique( std::make_shared(cluster_info, priority_set, runtime, random, time_source)); @@ -33,7 +33,8 @@ class FactoryBase : public Upstream::TypedLoadBalancerFactory { class LbFactory : public Upstream::LoadBalancerFactory { public: LbFactory(const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, - Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) + Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random, + TimeSource& time_source) : cluster_info_(cluster_info), priority_set_(priority_set), runtime_(runtime), random_(random), time_source_(time_source) {} @@ -46,7 +47,7 @@ class FactoryBase : public Upstream::TypedLoadBalancerFactory { const Upstream::ClusterInfo& cluster_info_; const Upstream::PrioritySet& priority_set_; Runtime::Loader& runtime_; - Random::RandomGenerator& random_; + Envoy::Random::RandomGenerator& random_; TimeSource& time_source_; }; From 6c27a1ac25f5f262f62864f382848e8f7ce0909d Mon Sep 17 00:00:00 2001 From: wbpcode Date: Wed, 4 Jan 2023 08:48:31 +0000 Subject: [PATCH 15/21] update change log Signed-off-by: wbpcode --- changelogs/current.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 1cb6acccb1a4b..855af3701416c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -209,6 +209,12 @@ new_features: - area: upstream change: | added :ref:`least request extension ` to suppport the :ref:`load balancer policy `. +- area: upstream + change: | + added :ref:`random extension ` to suppport the :ref:`load balancer policy `. +- area: upstream + change: | + added :ref:`round robin extension ` to suppport the :ref:`load balancer policy `. - area: generic_proxy change: | added :ref:`generic rds support `. From 3db7808d91a331686cf64b7b05fc83bbb33cdd22 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Wed, 4 Jan 2023 12:41:00 +0000 Subject: [PATCH 16/21] more docs Signed-off-by: wbpcode --- .../load_balancing_policies.rst | 2 + .../upstream/load_balancing_policies.rst | 39 +++++++++++++++++++ docs/root/configuration/upstream/upstream.rst | 1 + source/extensions/extensions_metadata.yaml | 12 +++--- 4 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 docs/root/configuration/upstream/load_balancing_policies.rst diff --git a/docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst b/docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst index c3b8c2f6d1f4d..39133e7a38da9 100644 --- a/docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst +++ b/docs/root/api-v3/config/load_balancing_policies/load_balancing_policies.rst @@ -1,3 +1,5 @@ +.. _envoy_v3_api_config_load_balancer_policies: + Load balancing policies ======================= diff --git a/docs/root/configuration/upstream/load_balancing_policies.rst b/docs/root/configuration/upstream/load_balancing_policies.rst new file mode 100644 index 0000000000000..680860ec33b2b --- /dev/null +++ b/docs/root/configuration/upstream/load_balancing_policies.rst @@ -0,0 +1,39 @@ +Load balancing policies +======================= + +Extendable load balancing policies could be configured for each cluster by +:ref:`load balancer policy `. +And the developer can implement a custom load balancing policy and configured it. + + +See built-in load balancing policies by :ref:`APIs `. + + +:ref:`Enum based load balancing policies ` +will be suupported for backward compatibility and marked as deprecated. The new extendable load +balancing policies should be used as a priority if the related policies are implemented. + + +Use :ref:`random load balancing policy ` +as an example: + +.. code-block:: yaml + + name: example_cluster + type: STRICT_DNS + connect_timeout: 0.25s + load_assignment: + cluster_name: example_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: example.com + port_value: 80 + load_balancing_policy: + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.random + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random diff --git a/docs/root/configuration/upstream/upstream.rst b/docs/root/configuration/upstream/upstream.rst index f50ae1a2aea7b..903137de51574 100644 --- a/docs/root/configuration/upstream/upstream.rst +++ b/docs/root/configuration/upstream/upstream.rst @@ -7,3 +7,4 @@ Upstream clusters clusters cluster_manager/cluster_manager health_checkers/health_checkers + load_balancing_policies diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index f1adc2a51f3ed..31f424b44da77 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -1352,27 +1352,27 @@ envoy.load_balancing_policies.least_request: categories: - envoy.load_balancing_policies security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest envoy.load_balancing_policies.random: categories: - envoy.load_balancing_policies - security_posture: unknown - status: alpha + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable type_urls: - envoy.extensions.load_balancing_policies.random.v3.Random envoy.load_balancing_policies.round_robin: categories: - envoy.load_balancing_policies - security_posture: unknown - status: alpha + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable type_urls: - envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin envoy.http.early_header_mutation.header_mutation: categories: - envoy.http.early_header_mutation - security_posture: unknown + security_posture: robust_to_untrusted_downstream_and_upstream status: alpha type_urls: - envoy.extensions.http.early_header_mutation.header_mutation.v3.HeaderMutation From bd88fdd7e0b77f92844a014dc6731d35e1983622 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Wed, 4 Jan 2023 16:23:19 +0000 Subject: [PATCH 17/21] add some integration test Signed-off-by: wbpcode --- docs/root/configuration/upstream/upstream.rst | 1 - .../upstream/load_balancing_policies.rst | 0 .../intro/arch_overview/upstream/upstream.rst | 1 + .../least_request/BUILD | 13 +++ .../least_request/integration_test.cc | 103 +++++++++++++++++ .../load_balancing_policies/random/BUILD | 13 +++ .../random/integration_test.cc | 103 +++++++++++++++++ .../load_balancing_policies/round_robin/BUILD | 13 +++ .../round_robin/integration_test.cc | 108 ++++++++++++++++++ 9 files changed, 354 insertions(+), 1 deletion(-) rename docs/root/{configuration => intro/arch_overview}/upstream/load_balancing_policies.rst (100%) create mode 100644 test/extensions/load_balancing_policies/least_request/integration_test.cc create mode 100644 test/extensions/load_balancing_policies/random/integration_test.cc create mode 100644 test/extensions/load_balancing_policies/round_robin/integration_test.cc diff --git a/docs/root/configuration/upstream/upstream.rst b/docs/root/configuration/upstream/upstream.rst index 903137de51574..f50ae1a2aea7b 100644 --- a/docs/root/configuration/upstream/upstream.rst +++ b/docs/root/configuration/upstream/upstream.rst @@ -7,4 +7,3 @@ Upstream clusters clusters cluster_manager/cluster_manager health_checkers/health_checkers - load_balancing_policies diff --git a/docs/root/configuration/upstream/load_balancing_policies.rst b/docs/root/intro/arch_overview/upstream/load_balancing_policies.rst similarity index 100% rename from docs/root/configuration/upstream/load_balancing_policies.rst rename to docs/root/intro/arch_overview/upstream/load_balancing_policies.rst diff --git a/docs/root/intro/arch_overview/upstream/upstream.rst b/docs/root/intro/arch_overview/upstream/upstream.rst index 2e7350917ea64..aaa72f3b6cde3 100644 --- a/docs/root/intro/arch_overview/upstream/upstream.rst +++ b/docs/root/intro/arch_overview/upstream/upstream.rst @@ -17,3 +17,4 @@ Upstream clusters circuit_breaking upstream_filters load_reporting_service + load_balancing_policies diff --git a/test/extensions/load_balancing_policies/least_request/BUILD b/test/extensions/load_balancing_policies/least_request/BUILD index e0d46ad819d45..74d75e12ea48d 100644 --- a/test/extensions/load_balancing_policies/least_request/BUILD +++ b/test/extensions/load_balancing_policies/least_request/BUILD @@ -23,3 +23,16 @@ envoy_extension_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_extension_cc_test( + name = "integration_test", + srcs = ["integration_test.cc"], + extension_names = ["envoy.load_balancing_policies.least_request"], + deps = [ + "//source/common/protobuf", + "//source/extensions/load_balancing_policies/least_request:config", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/load_balancing_policies/least_request/integration_test.cc b/test/extensions/load_balancing_policies/least_request/integration_test.cc new file mode 100644 index 0000000000000..d2c0365f2d29a --- /dev/null +++ b/test/extensions/load_balancing_policies/least_request/integration_test.cc @@ -0,0 +1,103 @@ +#include +#include + +#include "envoy/config/endpoint/v3/endpoint_components.pb.h" + +#include "source/common/common/base64.h" +#include "source/common/http/utility.h" +#include "source/common/protobuf/protobuf.h" +#include "source/extensions/load_balancing_policies/least_request/config.h" + +#include "test/integration/http_integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace LeastRequest { +namespace { + +class LeastRequestIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + LeastRequestIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) { + // Create 3 different upstream server for stateful session test. + setUpstreamCount(3); + + // Update endpoints of default cluster `cluster_0` to 3 different fake upstreams. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + const std::string endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + )EOF"; + + TestUtility::loadFromYaml(endpoints_yaml, *endpoint); + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.least_request + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, LeastRequestIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(LeastRequestIntegrationTest, NormalLoadBalancing) { + initialize(); + + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); + + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + cleanupUpstreamAndDownstream(); + } +} + +} // namespace +} // namespace LeastRequest +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/random/BUILD b/test/extensions/load_balancing_policies/random/BUILD index 333d879eadbee..226c5fccb6b95 100644 --- a/test/extensions/load_balancing_policies/random/BUILD +++ b/test/extensions/load_balancing_policies/random/BUILD @@ -23,3 +23,16 @@ envoy_extension_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_extension_cc_test( + name = "integration_test", + srcs = ["integration_test.cc"], + extension_names = ["envoy.load_balancing_policies.random"], + deps = [ + "//source/common/protobuf", + "//source/extensions/load_balancing_policies/random:config", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/load_balancing_policies/random/integration_test.cc b/test/extensions/load_balancing_policies/random/integration_test.cc new file mode 100644 index 0000000000000..2734412a21801 --- /dev/null +++ b/test/extensions/load_balancing_policies/random/integration_test.cc @@ -0,0 +1,103 @@ +#include +#include + +#include "envoy/config/endpoint/v3/endpoint_components.pb.h" + +#include "source/common/common/base64.h" +#include "source/common/http/utility.h" +#include "source/common/protobuf/protobuf.h" +#include "source/extensions/load_balancing_policies/random/config.h" + +#include "test/integration/http_integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Random { +namespace { + +class RandomIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + RandomIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) { + // Create 3 different upstream server for stateful session test. + setUpstreamCount(3); + + // Update endpoints of default cluster `cluster_0` to 3 different fake upstreams. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + const std::string endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + )EOF"; + + TestUtility::loadFromYaml(endpoints_yaml, *endpoint); + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.random + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, RandomIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(RandomIntegrationTest, NormalLoadBalancing) { + initialize(); + + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); + + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + cleanupUpstreamAndDownstream(); + } +} + +} // namespace +} // namespace Random +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/round_robin/BUILD b/test/extensions/load_balancing_policies/round_robin/BUILD index af781df988bed..b52558c9667c5 100644 --- a/test/extensions/load_balancing_policies/round_robin/BUILD +++ b/test/extensions/load_balancing_policies/round_robin/BUILD @@ -23,3 +23,16 @@ envoy_extension_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_extension_cc_test( + name = "integration_test", + srcs = ["integration_test.cc"], + extension_names = ["envoy.load_balancing_policies.round_robin"], + deps = [ + "//source/common/protobuf", + "//source/extensions/load_balancing_policies/round_robin:config", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/load_balancing_policies/round_robin/integration_test.cc b/test/extensions/load_balancing_policies/round_robin/integration_test.cc new file mode 100644 index 0000000000000..31ab07ea68ea8 --- /dev/null +++ b/test/extensions/load_balancing_policies/round_robin/integration_test.cc @@ -0,0 +1,108 @@ +#include +#include + +#include "envoy/config/endpoint/v3/endpoint_components.pb.h" + +#include "source/common/common/base64.h" +#include "source/common/http/utility.h" +#include "source/common/protobuf/protobuf.h" +#include "source/extensions/load_balancing_policies/round_robin/config.h" + +#include "test/integration/http_integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace RoundRobin { +namespace { + +class RoundRobinIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + RoundRobinIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) { + // Create 3 different upstream server for stateful session test. + setUpstreamCount(3); + + // Update endpoints of default cluster `cluster_0` to 3 different fake upstreams. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + const std::string endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + )EOF"; + + TestUtility::loadFromYaml(endpoints_yaml, *endpoint); + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.round_robin + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, RoundRobinIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(RoundRobinIntegrationTest, NormalLoadBalancing) { + initialize(); + + std::vector indexs; + + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); + + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); + indexs.push_back(upstream_index.value()); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + cleanupUpstreamAndDownstream(); + } + + EXPECT_EQ(std::vector({0, 1, 2, 0, 1, 2, 0, 1}), indexs); +} + +} // namespace +} // namespace RoundRobin +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy From 82b05d0ac09a63c45b5b0ea170925b9b87220cb5 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Thu, 5 Jan 2023 02:49:40 +0000 Subject: [PATCH 18/21] fix test and update assert Signed-off-by: wbpcode --- .../load_balancing_policies/least_request/config.cc | 6 ++++-- source/extensions/load_balancing_policies/random/config.cc | 6 ++++-- .../load_balancing_policies/round_robin/config.cc | 6 ++++-- .../load_balancing_policies/round_robin/integration_test.cc | 5 ++++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc index ec8a6081644f3..d2dab339ec1fa 100644 --- a/source/extensions/load_balancing_policies/least_request/config.cc +++ b/source/extensions/load_balancing_policies/least_request/config.cc @@ -20,8 +20,10 @@ Upstream::LoadBalancerPtr LeastRequestCreator::operator()(Upstream::LoadBalancer const envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest*>( cluster_info.loadBalancingPolicy().get()); - RELEASE_ASSERT(typed_config != nullptr, - "Invalid load balancing policy configuration for least request load balancer"); + // The load balancing policy configuration will be loaded and validated in the main thread when we + // load the cluster configuration. So we can assume the configuration is valid here. + ASSERT(typed_config != nullptr, + "Invalid load balancing policy configuration for least request load balancer"); return std::make_unique( params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, diff --git a/source/extensions/load_balancing_policies/random/config.cc b/source/extensions/load_balancing_policies/random/config.cc index b36dda5d65b25..86d8394f32f7d 100644 --- a/source/extensions/load_balancing_policies/random/config.cc +++ b/source/extensions/load_balancing_policies/random/config.cc @@ -20,8 +20,10 @@ Upstream::LoadBalancerPtr RandomCreator::operator()(Upstream::LoadBalancerParams dynamic_cast( cluster_info.loadBalancingPolicy().get()); - RELEASE_ASSERT(typed_config != nullptr, - "Invalid load balancing policy configuration for random load balancer"); + // The load balancing policy configuration will be loaded and validated in the main thread when we + // load the cluster configuration. So we can assume the configuration is valid here. + ASSERT(typed_config != nullptr, + "Invalid load balancing policy configuration for random load balancer"); return std::make_unique( params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, diff --git a/source/extensions/load_balancing_policies/round_robin/config.cc b/source/extensions/load_balancing_policies/round_robin/config.cc index e78661a72df44..a55d47fa9c680 100644 --- a/source/extensions/load_balancing_policies/round_robin/config.cc +++ b/source/extensions/load_balancing_policies/round_robin/config.cc @@ -20,8 +20,10 @@ Upstream::LoadBalancerPtr RoundRobinCreator::operator()(Upstream::LoadBalancerPa dynamic_cast( cluster_info.loadBalancingPolicy().get()); - RELEASE_ASSERT(typed_config != nullptr, - "Invalid load balancing policy configuration for least request load balancer"); + // The load balancing policy configuration will be loaded and validated in the main thread when we + // load the cluster configuration. So we can assume the configuration is valid here. + ASSERT(typed_config != nullptr, + "Invalid load balancing policy configuration for least request load balancer"); return std::make_unique( params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, diff --git a/test/extensions/load_balancing_policies/round_robin/integration_test.cc b/test/extensions/load_balancing_policies/round_robin/integration_test.cc index 31ab07ea68ea8..edc5f944b0340 100644 --- a/test/extensions/load_balancing_policies/round_robin/integration_test.cc +++ b/test/extensions/load_balancing_policies/round_robin/integration_test.cc @@ -98,7 +98,10 @@ TEST_P(RoundRobinIntegrationTest, NormalLoadBalancing) { cleanupUpstreamAndDownstream(); } - EXPECT_EQ(std::vector({0, 1, 2, 0, 1, 2, 0, 1}), indexs); + for (uint64_t i = 2; i < 8; i++) { + EXPECT_NE(indexs[i], indexs[i - 1]); + EXPECT_NE(indexs[i - 1], indexs[i - 2]); + } } } // namespace From 1e6e1a60b4d908b9d98fe118865b1ad57421a146 Mon Sep 17 00:00:00 2001 From: wbpcode Date: Thu, 5 Jan 2023 09:15:32 +0000 Subject: [PATCH 19/21] fix test for ipv6 Signed-off-by: wbpcode --- .../least_request/integration_test.cc | 10 ++++++---- .../load_balancing_policies/random/integration_test.cc | 10 ++++++---- .../round_robin/integration_test.cc | 10 ++++++---- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/test/extensions/load_balancing_policies/least_request/integration_test.cc b/test/extensions/load_balancing_policies/least_request/integration_test.cc index d2c0365f2d29a..4feb0585c182b 100644 --- a/test/extensions/load_balancing_policies/least_request/integration_test.cc +++ b/test/extensions/load_balancing_policies/least_request/integration_test.cc @@ -36,21 +36,23 @@ class LeastRequestIntegrationTest : public testing::TestWithParammutable_load_balancing_policy(); diff --git a/test/extensions/load_balancing_policies/random/integration_test.cc b/test/extensions/load_balancing_policies/random/integration_test.cc index 2734412a21801..aae4a415c5460 100644 --- a/test/extensions/load_balancing_policies/random/integration_test.cc +++ b/test/extensions/load_balancing_policies/random/integration_test.cc @@ -36,21 +36,23 @@ class RandomIntegrationTest : public testing::TestWithParammutable_load_balancing_policy(); diff --git a/test/extensions/load_balancing_policies/round_robin/integration_test.cc b/test/extensions/load_balancing_policies/round_robin/integration_test.cc index edc5f944b0340..ee9939fccded6 100644 --- a/test/extensions/load_balancing_policies/round_robin/integration_test.cc +++ b/test/extensions/load_balancing_policies/round_robin/integration_test.cc @@ -36,21 +36,23 @@ class RoundRobinIntegrationTest : public testing::TestWithParammutable_load_balancing_policy(); From f71432f2911a61d6cfe757ee450557b702931c70 Mon Sep 17 00:00:00 2001 From: Rafal Augustyniak Date: Fri, 6 Jan 2023 15:29:21 -0500 Subject: [PATCH 20/21] test Signed-off-by: Rafal Augustyniak --- mobile/examples/objective-c/hello_world/ViewController.m | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/examples/objective-c/hello_world/ViewController.m b/mobile/examples/objective-c/hello_world/ViewController.m index 4d1e6619bc769..2d88c048a30a9 100644 --- a/mobile/examples/objective-c/hello_world/ViewController.m +++ b/mobile/examples/objective-c/hello_world/ViewController.m @@ -39,6 +39,7 @@ - (void)startEnvoy { NSLog(@"starting Envoy..."); EngineBuilder *builder = [[EngineBuilder alloc] init]; [builder addLogLevel:LogLevelDebug]; + [builder addGrpcStatsDomain:@"example.com"]; [builder enableDNSCache:YES]; [builder addKeyValueStoreWithName:@"reserved.platform_store" keyValueStore:NSUserDefaults.standardUserDefaults]; From 384b0d59a058153b1acfca5caf09aed686e76190 Mon Sep 17 00:00:00 2001 From: Rafal Augustyniak Date: Fri, 6 Jan 2023 15:52:30 -0500 Subject: [PATCH 21/21] change Signed-off-by: Rafal Augustyniak --- mobile/examples/objective-c/hello_world/ViewController.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/examples/objective-c/hello_world/ViewController.m b/mobile/examples/objective-c/hello_world/ViewController.m index 2d88c048a30a9..6002692a7d9f1 100644 --- a/mobile/examples/objective-c/hello_world/ViewController.m +++ b/mobile/examples/objective-c/hello_world/ViewController.m @@ -48,7 +48,7 @@ - (void)startEnvoy { }]; id engine = [builder build]; - NSLog(@"started Envoy, beginning requests..."); + NSLog(@"started Envoy, beginning requeststest.."); self.streamClient = [engine streamClient]; self.pulseClient = [engine pulseClient]; [self startRequests];