diff --git a/api/envoy/admin/v2alpha/server_info.proto b/api/envoy/admin/v2alpha/server_info.proto index 18dcc70b805cc..13dea7ae8eb76 100644 --- a/api/envoy/admin/v2alpha/server_info.proto +++ b/api/envoy/admin/v2alpha/server_info.proto @@ -128,4 +128,7 @@ message CommandLineOptions { // See :option:`--restart-epoch` for details. uint32 restart_epoch = 24; + + // See :option:`--cpuset-threads` for details. + bool cpuset_threads = 25; } diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index a1e3eedb15e72..9ee0a7a40adee 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -14,6 +14,7 @@ Version history * config: removed deprecated --v2-config-only from command line config. * config: removed deprecated_v1 sds_config from :ref:`Bootstrap config `. * config: removed REST_LEGACY as a valid :ref:`ApiType `. +* config: use Envoy cpuset size to set the default number or worker threads if :option:`--cpuset-threads` is enabled. * cors: added :ref:`filter_enabled & shadow_enabled RuntimeFractionalPercent flags ` to filter. * ext_authz: added an configurable option to make the gRPC service cross-compatible with V2Alpha. Note that this feature is already deprecated. It should be used for a short time, and only when transitioning from alpha to V2 release version. * ext_authz: migrated from V2alpha to V2 and improved the documentation. diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index 3b0796fcfd929..e88da1ac382ab 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -213,7 +213,8 @@ modify different aspects of the server: "restart_epoch": 0, "file_flush_interval": "10s", "drain_time": "600s", - "parent_shutdown_time": "900s" + "parent_shutdown_time": "900s", + "cpuset_threads": false }, "uptime_current_epoch": "6s", "uptime_all_epochs": "6s" diff --git a/docs/root/operations/cli.rst b/docs/root/operations/cli.rst index 6e2b45d10ba22..c863a47e693c6 100644 --- a/docs/root/operations/cli.rst +++ b/docs/root/operations/cli.rst @@ -76,6 +76,14 @@ following are the command line options that Envoy supports. `connection` component to run at `trace` level, you should pass ``upstream:debug,connection:trace`` to this flag. See ``ALL_LOGGER_IDS`` in :repo:`/source/common/common/logger.h` for a list of components. +.. option:: --cpuset-threads + + *(optional)* This flag is used to control the number of worker threads if :option:`--concurrency` is + not set. If enabled, the assigned cpuset size is used to determine the number of worker threads on + Linux-based systems. Otherwise the number of worker threads is set to the number of hardware threads + on the machine. You can read more about cpusets in the + `kernel documentation `_. + .. option:: --log-path *(optional)* The output file path where logs should be written. This file will be re-opened diff --git a/include/envoy/api/BUILD b/include/envoy/api/BUILD index 265e047ec9789..0b2b6df8cc2d4 100644 --- a/include/envoy/api/BUILD +++ b/include/envoy/api/BUILD @@ -25,5 +25,9 @@ envoy_cc_library( envoy_cc_library( name = "os_sys_calls_interface", - hdrs = ["os_sys_calls.h"], + hdrs = [ + "os_sys_calls.h", + "os_sys_calls_common.h", + "os_sys_calls_linux.h", + ], ) diff --git a/include/envoy/api/os_sys_calls.h b/include/envoy/api/os_sys_calls.h index d3edee58fa6df..6f5c200ee7d5d 100644 --- a/include/envoy/api/os_sys_calls.h +++ b/include/envoy/api/os_sys_calls.h @@ -9,33 +9,12 @@ #include #include +#include "envoy/api/os_sys_calls_common.h" #include "envoy/common/pure.h" namespace Envoy { namespace Api { -/** - * SysCallResult holds the rc and errno values resulting from a system call. - */ -template struct SysCallResult { - - /** - * The return code from the system call. - */ - T rc_; - - /** - * The errno value as captured after the system call. - */ - int errno_; -}; - -typedef SysCallResult SysCallIntResult; -typedef SysCallResult SysCallSizeResult; -typedef SysCallResult SysCallPtrResult; -typedef SysCallResult SysCallStringResult; -typedef SysCallResult SysCallBoolResult; - class OsSysCalls { public: virtual ~OsSysCalls() {} diff --git a/include/envoy/api/os_sys_calls_common.h b/include/envoy/api/os_sys_calls_common.h new file mode 100644 index 0000000000000..3c283e064bbfd --- /dev/null +++ b/include/envoy/api/os_sys_calls_common.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +namespace Envoy { +namespace Api { +/** + * SysCallResult holds the rc and errno values resulting from a system call. + */ +template struct SysCallResult { + + /** + * The return code from the system call. + */ + T rc_; + + /** + * The errno value as captured after the system call. + */ + int errno_; +}; + +typedef SysCallResult SysCallIntResult; +typedef SysCallResult SysCallSizeResult; +typedef SysCallResult SysCallPtrResult; +typedef SysCallResult SysCallStringResult; +typedef SysCallResult SysCallBoolResult; + +} // namespace Api +} // namespace Envoy diff --git a/include/envoy/api/os_sys_calls_linux.h b/include/envoy/api/os_sys_calls_linux.h new file mode 100644 index 0000000000000..cd90daea538df --- /dev/null +++ b/include/envoy/api/os_sys_calls_linux.h @@ -0,0 +1,28 @@ +#pragma once + +#if !defined(__linux__) +#error "Linux platform file is part of non-Linux build." +#endif + +#include + +#include "envoy/api/os_sys_calls_common.h" +#include "envoy/common/pure.h" + +namespace Envoy { +namespace Api { + +class LinuxOsSysCalls { +public: + virtual ~LinuxOsSysCalls() {} + + /** + * @see sched_getaffinity (man 2 sched_getaffinity) + */ + virtual SysCallIntResult sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t* mask) PURE; +}; + +typedef std::unique_ptr LinuxOsSysCallsPtr; + +} // namespace Api +} // namespace Envoy diff --git a/include/envoy/server/options.h b/include/envoy/server/options.h index 89ee2d132690d..905a218f9cc3f 100644 --- a/include/envoy/server/options.h +++ b/include/envoy/server/options.h @@ -174,6 +174,11 @@ class Options { */ virtual bool mutexTracingEnabled() const PURE; + /** + * @return bool indicating whether cpuset size should determine the number of worker threads. + */ + virtual bool cpusetThreadsEnabled() const PURE; + /** * Converts the Options in to CommandLineOptions proto message defined in server_info.proto. * @return CommandLineOptionsPtr the protobuf representation of the options. diff --git a/source/common/api/BUILD b/source/common/api/BUILD index 92623ea51279f..d4dd9c3950223 100644 --- a/source/common/api/BUILD +++ b/source/common/api/BUILD @@ -22,8 +22,16 @@ envoy_cc_library( envoy_cc_library( name = "os_sys_calls_lib", - srcs = ["os_sys_calls_impl.cc"], - hdrs = ["os_sys_calls_impl.h"], + srcs = ["os_sys_calls_impl.cc"] + select({ + "@bazel_tools//src/conditions:linux_x86_64": ["os_sys_calls_impl_linux.cc"], + "@bazel_tools//src/conditions:linux_aarch64": ["os_sys_calls_impl_linux.cc"], + "//conditions:default": [], + }), + hdrs = ["os_sys_calls_impl.h"] + select({ + "@bazel_tools//src/conditions:linux_x86_64": ["os_sys_calls_impl_linux.h"], + "@bazel_tools//src/conditions:linux_aarch64": ["os_sys_calls_impl_linux.h"], + "//conditions:default": [], + }), deps = [ "//include/envoy/api:os_sys_calls_interface", "//source/common/singleton:threadsafe_singleton", diff --git a/source/common/api/os_sys_calls_impl_linux.cc b/source/common/api/os_sys_calls_impl_linux.cc new file mode 100644 index 0000000000000..fcf2fafdc7d0d --- /dev/null +++ b/source/common/api/os_sys_calls_impl_linux.cc @@ -0,0 +1,20 @@ +#if !defined(__linux__) +#error "Linux platform file is part of non-Linux build." +#endif + +#include "common/api/os_sys_calls_impl_linux.h" + +#include +#include + +namespace Envoy { +namespace Api { + +SysCallIntResult LinuxOsSysCallsImpl::sched_getaffinity(pid_t pid, size_t cpusetsize, + cpu_set_t* mask) { + const int rc = ::sched_getaffinity(pid, cpusetsize, mask); + return {rc, errno}; +} + +} // namespace Api +} // namespace Envoy diff --git a/source/common/api/os_sys_calls_impl_linux.h b/source/common/api/os_sys_calls_impl_linux.h new file mode 100644 index 0000000000000..d3b08fe427d9f --- /dev/null +++ b/source/common/api/os_sys_calls_impl_linux.h @@ -0,0 +1,23 @@ +#pragma once + +#if !defined(__linux__) +#error "Linux platform file is part of non-Linux build." +#endif + +#include "envoy/api/os_sys_calls_linux.h" + +#include "common/singleton/threadsafe_singleton.h" + +namespace Envoy { +namespace Api { + +class LinuxOsSysCallsImpl : public LinuxOsSysCalls { +public: + // Api::LinuxOsSysCalls + SysCallIntResult sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t* mask) override; +}; + +typedef ThreadSafeSingleton LinuxOsSysCallsSingleton; + +} // namespace Api +} // namespace Envoy diff --git a/source/server/BUILD b/source/server/BUILD index 70d5d13b0a6bf..3943a76618b28 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -151,13 +151,26 @@ envoy_cc_library( envoy_cc_library( name = "options_lib", - srcs = ["options_impl.cc"], - hdrs = ["options_impl.h"], + srcs = ["options_impl.cc"] + select({ + "@bazel_tools//src/conditions:linux_x86_64": ["options_impl_platform_linux.cc"], + "@bazel_tools//src/conditions:linux_aarch64": ["options_impl_platform_linux.cc"], + "//conditions:default": ["options_impl_platform_default.cc"], + }), + hdrs = [ + "options_impl.h", + "options_impl_platform.h", + ] + select({ + "@bazel_tools//src/conditions:linux_x86_64": ["options_impl_platform_linux.h"], + "@bazel_tools//src/conditions:linux_aarch64": ["options_impl_platform_linux.h"], + "//conditions:default": [], + }), external_deps = ["tclap"], deps = [ "//include/envoy/network:address_interface", "//include/envoy/server:options_interface", "//include/envoy/stats:stats_interface", + "//source/common/api:os_sys_calls_lib", + "//source/common/common:logger_lib", "//source/common/common:macros", "//source/common/common:version_lib", "//source/common/protobuf:utility_lib", diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index 62971fa3e6a35..92e05e3fa3190 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -11,6 +11,8 @@ #include "common/common/version.h" #include "common/protobuf/utility.h" +#include "server/options_impl_platform.h" + #include "absl/strings/str_split.h" #include "spdlog/spdlog.h" #include "tclap/CmdLine.h" @@ -117,6 +119,8 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, "Disable hot restart functionality", cmd, false); TCLAP::SwitchArg enable_mutex_tracing( "", "enable-mutex-tracing", "Enable mutex contention tracing functionality", cmd, false); + TCLAP::SwitchArg cpuset_threads( + "", "cpuset-threads", "Get the default # of worker threads from cpuset size", cmd, false); cmd.setExceptionHandling(false); try { @@ -154,6 +158,8 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, mutex_tracing_enabled_ = enable_mutex_tracing.getValue(); + cpuset_threads_ = cpuset_threads.getValue(); + log_level_ = default_log_level; for (size_t i = 0; i < ARRAY_SIZE(spdlog::level::level_string_views); i++) { if (log_level.getValue() == spdlog::level::level_string_views[i]) { @@ -188,7 +194,20 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, // For base ID, scale what the user inputs by 10 so that we have spread for domain sockets. base_id_ = base_id.getValue() * 10; - concurrency_ = std::max(1U, concurrency.getValue()); + + if (!concurrency.isSet() && cpuset_threads_) { + // The 'concurrency' command line option wasn't set but the 'cpuset-threads' + // option was set. Use the number of CPUs assigned to the process cpuset, if + // that can be known. + concurrency_ = OptionsImplPlatform::getCpuCount(); + } else { + if (concurrency.isSet() && cpuset_threads_ && cpuset_threads.isSet()) { + ENVOY_LOG(warn, "Both --concurrency and --cpuset-threads options are set; not applying " + "--cpuset-threads."); + } + concurrency_ = std::max(1U, concurrency.getValue()); + } + config_path_ = config_path.getValue(); config_yaml_ = config_yaml.getValue(); allow_unknown_fields_ = allow_unknown_fields.getValue(); @@ -291,6 +310,7 @@ Server::CommandLineOptionsPtr OptionsImpl::toCommandLineOptions() const { command_line_options->set_max_obj_name_len(statsOptions().maxObjNameLength()); command_line_options->set_disable_hot_restart(hotRestartDisabled()); command_line_options->set_enable_mutex_tracing(mutexTracingEnabled()); + command_line_options->set_cpuset_threads(cpusetThreadsEnabled()); command_line_options->set_restart_epoch(restartEpoch()); return command_line_options; } @@ -303,6 +323,6 @@ OptionsImpl::OptionsImpl(const std::string& service_cluster, const std::string& service_cluster_(service_cluster), service_node_(service_node), service_zone_(service_zone), file_flush_interval_msec_(10000), drain_time_(600), parent_shutdown_time_(900), mode_(Server::Mode::Serve), max_stats_(ENVOY_DEFAULT_MAX_STATS), hot_restart_disabled_(false), - signal_handling_enabled_(true), mutex_tracing_enabled_(false) {} + signal_handling_enabled_(true), mutex_tracing_enabled_(false), cpuset_threads_(false) {} } // namespace Envoy diff --git a/source/server/options_impl.h b/source/server/options_impl.h index cd0ec6a9d180f..b23faf2c34bbb 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -8,6 +8,7 @@ #include "envoy/server/options.h" #include "envoy/stats/stats_options.h" +#include "common/common/logger.h" #include "common/stats/stats_options_impl.h" #include "spdlog/spdlog.h" @@ -16,7 +17,7 @@ namespace Envoy { /** * Implementation of Server::Options. */ -class OptionsImpl : public Server::Options { +class OptionsImpl : public Server::Options, protected Logger::Loggable { public: /** * Parameters are max_num_stats, max_stat_name_len, hot_restart_enabled @@ -73,6 +74,7 @@ class OptionsImpl : public Server::Options { void setSignalHandling(bool signal_handling_enabled) { signal_handling_enabled_ = signal_handling_enabled; } + void setCpusetThreads(bool cpuset_threads_enabled) { cpuset_threads_ = cpuset_threads_enabled; } // Server::Options uint64_t baseId() const override { return base_id_; } @@ -107,6 +109,7 @@ class OptionsImpl : public Server::Options { bool mutexTracingEnabled() const override { return mutex_tracing_enabled_; } virtual Server::CommandLineOptionsPtr toCommandLineOptions() const override; void parseComponentLogLevels(const std::string& component_log_levels); + bool cpusetThreadsEnabled() const override { return cpuset_threads_; } uint32_t count() const; private: @@ -137,6 +140,7 @@ class OptionsImpl : public Server::Options { bool hot_restart_disabled_; bool signal_handling_enabled_; bool mutex_tracing_enabled_; + bool cpuset_threads_; uint32_t count_; }; diff --git a/source/server/options_impl_platform.h b/source/server/options_impl_platform.h new file mode 100644 index 0000000000000..7d628d3699025 --- /dev/null +++ b/source/server/options_impl_platform.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +#include "common/common/logger.h" + +namespace Envoy { +class OptionsImplPlatform : protected Logger::Loggable { +public: + static uint32_t getCpuCount(); +}; +} // namespace Envoy diff --git a/source/server/options_impl_platform_default.cc b/source/server/options_impl_platform_default.cc new file mode 100644 index 0000000000000..3b4cbbe3118e9 --- /dev/null +++ b/source/server/options_impl_platform_default.cc @@ -0,0 +1,14 @@ +#include + +#include "common/common/logger.h" + +#include "server/options_impl_platform.h" + +namespace Envoy { + +uint32_t OptionsImplPlatform::getCpuCount() { + ENVOY_LOG(warn, "CPU number provided by HW thread count (instead of cpuset)."); + return std::thread::hardware_concurrency(); +} + +} // namespace Envoy diff --git a/source/server/options_impl_platform_linux.cc b/source/server/options_impl_platform_linux.cc new file mode 100644 index 0000000000000..069c68ab83da1 --- /dev/null +++ b/source/server/options_impl_platform_linux.cc @@ -0,0 +1,46 @@ +#if !defined(__linux__) +#error "Linux platform file is part of non-Linux build." +#endif + +#include "server/options_impl_platform_linux.h" + +#include + +#include + +#include "common/api/os_sys_calls_impl_linux.h" + +#include "server/options_impl_platform.h" + +namespace Envoy { + +uint32_t OptionsImplPlatformLinux::getCpuAffinityCount(unsigned int hw_threads) { + unsigned int threads = 0; + pid_t pid = getpid(); + cpu_set_t mask; + auto& linux_os_syscalls = Api::LinuxOsSysCallsSingleton::get(); + + CPU_ZERO(&mask); + const Api::SysCallIntResult result = + linux_os_syscalls.sched_getaffinity(pid, sizeof(cpu_set_t), &mask); + if (result.rc_ == -1) { + // Fall back to number of hardware threads. + return hw_threads; + } + + threads = CPU_COUNT(&mask); + + // Sanity check. + if (threads > 0 && threads <= hw_threads) { + return threads; + } + + return hw_threads; +} + +uint32_t OptionsImplPlatform::getCpuCount() { + unsigned int hw_threads = std::max(1U, std::thread::hardware_concurrency()); + return OptionsImplPlatformLinux::getCpuAffinityCount(hw_threads); +} + +} // namespace Envoy diff --git a/source/server/options_impl_platform_linux.h b/source/server/options_impl_platform_linux.h new file mode 100644 index 0000000000000..dfdb0c7efae0b --- /dev/null +++ b/source/server/options_impl_platform_linux.h @@ -0,0 +1,15 @@ +#pragma once + +#if !defined(__linux__) +#error "Linux platform file is part of non-Linux build." +#endif + +#include +#include + +namespace Envoy { +class OptionsImplPlatformLinux { +public: + static uint32_t getCpuAffinityCount(unsigned int hw_threads); +}; +} // namespace Envoy diff --git a/test/mocks/api/mocks.h b/test/mocks/api/mocks.h index 4a1be0b0b0df5..711738750ca69 100644 --- a/test/mocks/api/mocks.h +++ b/test/mocks/api/mocks.h @@ -10,6 +10,10 @@ #include "common/api/os_sys_calls_impl.h" +#if defined(__linux__) +#include "common/api/os_sys_calls_impl_linux.h" +#endif + #include "test/mocks/filesystem/mocks.h" #include "test/test_common/test_time.h" @@ -74,5 +78,13 @@ class MockOsSysCalls : public OsSysCallsImpl { std::map boolsockopts_; }; +#if defined(__linux__) +class MockLinuxOsSysCalls : public LinuxOsSysCallsImpl { +public: + // Api::LinuxOsSysCalls + MOCK_METHOD3(sched_getaffinity, SysCallIntResult(pid_t pid, size_t cpusetsize, cpu_set_t* mask)); +}; +#endif + } // namespace Api } // namespace Envoy diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index af2b5f299799e..5f58a2c6dada5 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -34,6 +34,7 @@ MockOptions::MockOptions(const std::string& config_path) : config_path_(config_p ON_CALL(*this, hotRestartDisabled()).WillByDefault(ReturnPointee(&hot_restart_disabled_)); ON_CALL(*this, signalHandlingEnabled()).WillByDefault(ReturnPointee(&signal_handling_enabled_)); ON_CALL(*this, mutexTracingEnabled()).WillByDefault(ReturnPointee(&mutex_tracing_enabled_)); + ON_CALL(*this, cpusetThreadsEnabled()).WillByDefault(ReturnPointee(&cpuset_threads_enabled_)); ON_CALL(*this, toCommandLineOptions()).WillByDefault(Invoke([] { return std::make_unique(); })); diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 7c57b6e0da8e2..ec26b4bcbc2f6 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -79,6 +79,7 @@ class MockOptions : public Options { MOCK_CONST_METHOD0(hotRestartDisabled, bool()); MOCK_CONST_METHOD0(signalHandlingEnabled, bool()); MOCK_CONST_METHOD0(mutexTracingEnabled, bool()); + MOCK_CONST_METHOD0(cpusetThreadsEnabled, bool()); MOCK_CONST_METHOD0(toCommandLineOptions, Server::CommandLineOptionsPtr()); std::string config_path_; @@ -95,6 +96,7 @@ class MockOptions : public Options { bool hot_restart_disabled_{}; bool signal_handling_enabled_{true}; bool mutex_tracing_enabled_{}; + bool cpuset_threads_enabled_{}; }; class MockConfigTracker : public ConfigTracker { diff --git a/test/server/BUILD b/test/server/BUILD index 813ac24ee5267..1c64006680419 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -114,6 +114,10 @@ envoy_cc_test( "//source/common/common:utility_lib", "//source/common/stats:stats_lib", "//source/server:options_lib", + "//test/mocks/api:api_mocks", + "//test/test_common:environment_lib", + "//test/test_common:logging_lib", + "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 52ac0dc503423..f7a72820b28ea 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -8,7 +9,12 @@ #include "common/common/utility.h" #include "server/options_impl.h" +#include "server/options_impl_platform_linux.h" +#include "test/mocks/api/mocks.h" +#include "test/test_common/environment.h" +#include "test/test_common/logging.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -67,7 +73,7 @@ TEST_F(OptionsImplTest, All) { "--service-cluster cluster --service-node node --service-zone zone " "--file-flush-interval-msec 9000 " "--drain-time-s 60 --log-format [%v] --parent-shutdown-time-s 90 --log-path /foo/bar " - "--disable-hot-restart"); + "--disable-hot-restart --cpuset-threads"); EXPECT_EQ(Server::Mode::Validate, options->mode()); EXPECT_EQ(2U, options->concurrency()); EXPECT_EQ("hello", options->configPath()); @@ -85,6 +91,7 @@ TEST_F(OptionsImplTest, All) { EXPECT_EQ(std::chrono::seconds(60), options->drainTime()); EXPECT_EQ(std::chrono::seconds(90), options->parentShutdownTime()); EXPECT_EQ(true, options->hotRestartDisabled()); + EXPECT_EQ(true, options->cpusetThreadsEnabled()); options = createOptionsImpl("envoy --mode init_only"); EXPECT_EQ(Server::Mode::InitOnly, options->mode()); @@ -94,6 +101,7 @@ TEST_F(OptionsImplTest, SetAll) { std::unique_ptr options = createOptionsImpl("envoy -c hello"); bool hot_restart_disabled = options->hotRestartDisabled(); bool signal_handling_enabled = options->signalHandlingEnabled(); + bool cpuset_threads_enabled = options->cpusetThreadsEnabled(); Stats::StatsOptionsImpl stats_options; stats_options.max_obj_name_length_ = 54321; stats_options.max_stat_suffix_length_ = 1234; @@ -119,6 +127,7 @@ TEST_F(OptionsImplTest, SetAll) { options->setStatsOptions(stats_options); options->setHotRestartDisabled(!options->hotRestartDisabled()); options->setSignalHandling(!options->signalHandlingEnabled()); + options->setCpusetThreads(!options->cpusetThreadsEnabled()); EXPECT_EQ(109876, options->baseId()); EXPECT_EQ(42U, options->concurrency()); @@ -142,6 +151,7 @@ TEST_F(OptionsImplTest, SetAll) { EXPECT_EQ(stats_options.max_stat_suffix_length_, options->statsOptions().maxStatSuffixLength()); EXPECT_EQ(!hot_restart_disabled, options->hotRestartDisabled()); EXPECT_EQ(!signal_handling_enabled, options->signalHandlingEnabled()); + EXPECT_EQ(!cpuset_threads_enabled, options->cpusetThreadsEnabled()); // Validate that CommandLineOptions is constructed correctly. Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); @@ -172,6 +182,7 @@ TEST_F(OptionsImplTest, SetAll) { EXPECT_EQ(options->statsOptions().maxObjNameLength(), command_line_options->max_obj_name_len()); EXPECT_EQ(options->hotRestartDisabled(), command_line_options->disable_hot_restart()); EXPECT_EQ(options->mutexTracingEnabled(), command_line_options->enable_mutex_tracing()); + EXPECT_EQ(options->cpusetThreadsEnabled(), command_line_options->cpuset_threads()); } TEST_F(OptionsImplTest, DefaultParams) { @@ -182,6 +193,7 @@ TEST_F(OptionsImplTest, DefaultParams) { EXPECT_EQ(Network::Address::IpVersion::v4, options->localAddressIpVersion()); EXPECT_EQ(Server::Mode::Serve, options->mode()); EXPECT_EQ(false, options->hotRestartDisabled()); + EXPECT_EQ(false, options->cpusetThreadsEnabled()); // Validate that CommandLineOptions is constructed correctly with default params. Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); @@ -193,6 +205,7 @@ TEST_F(OptionsImplTest, DefaultParams) { command_line_options->local_address_ip_version()); EXPECT_EQ(envoy::admin::v2alpha::CommandLineOptions::Serve, command_line_options->mode()); EXPECT_EQ(false, command_line_options->disable_hot_restart()); + EXPECT_EQ(false, command_line_options->cpuset_threads()); } // Validates that the server_info proto is in sync with the options. @@ -304,7 +317,62 @@ TEST_F(OptionsImplTest, SaneTestConstructor) { EXPECT_EQ(regular_options_impl->statsOptions().maxStatSuffixLength(), test_options_impl.statsOptions().maxStatSuffixLength()); EXPECT_EQ(regular_options_impl->hotRestartDisabled(), test_options_impl.hotRestartDisabled()); + EXPECT_EQ(regular_options_impl->cpusetThreadsEnabled(), test_options_impl.cpusetThreadsEnabled()); } +TEST_F(OptionsImplTest, SetBothConcurrencyAndCpuset) { + EXPECT_LOG_CONTAINS( + "warning", + "Both --concurrency and --cpuset-threads options are set; not applying --cpuset-threads.", + std::unique_ptr options = + createOptionsImpl("envoy -c hello --concurrency 42 --cpuset-threads")); +} + +#if defined(__linux__) + +using testing::Return; + +class OptionsImplPlatformLinuxTest : public testing::Test { +public: +}; + +TEST_F(OptionsImplPlatformLinuxTest, AffinityTest1) { + // Success case: cpuset size and hardware thread count are the same. + unsigned int fake_cpuset_size = std::thread::hardware_concurrency(); + unsigned int fake_hw_threads = fake_cpuset_size; + + EXPECT_EQ(OptionsImplPlatformLinux::getCpuAffinityCount(fake_hw_threads), fake_cpuset_size); +} + +TEST_F(OptionsImplPlatformLinuxTest, AffinityTest2) { + // Success case: cpuset size is half of the hardware thread count. + unsigned int fake_cpuset_size = std::thread::hardware_concurrency(); + unsigned int fake_hw_threads = 2 * fake_cpuset_size; + + EXPECT_EQ(OptionsImplPlatformLinux::getCpuAffinityCount(fake_hw_threads), fake_cpuset_size); +} + +TEST_F(OptionsImplPlatformLinuxTest, AffinityTest3) { + // Failure case: cpuset size is bigger than the hardware thread count. + unsigned int fake_cpuset_size = std::thread::hardware_concurrency(); + unsigned int fake_hw_threads = fake_cpuset_size - 1; + + EXPECT_EQ(OptionsImplPlatformLinux::getCpuAffinityCount(fake_hw_threads), fake_hw_threads); +} + +TEST_F(OptionsImplPlatformLinuxTest, AffinityTest4) { + // When sched_getaffinity() fails, expect to get the hardware thread count. + unsigned int fake_cpuset_size = std::thread::hardware_concurrency(); + unsigned int fake_hw_threads = 2 * fake_cpuset_size; + Api::MockLinuxOsSysCalls linux_os_sys_calls; + TestThreadsafeSingletonInjector linux_os_calls(&linux_os_sys_calls); + + EXPECT_CALL(linux_os_sys_calls, sched_getaffinity(_, _, _)) + .WillOnce(Return(Api::SysCallIntResult{-1, 0})); + EXPECT_EQ(OptionsImplPlatformLinux::getCpuAffinityCount(fake_hw_threads), fake_hw_threads); +} + +#endif + } // namespace } // namespace Envoy diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 9b59f844f3ce3..be26db203fb67 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -633,6 +633,7 @@ sanitization sanitizer scala scalability +sched schemas serializable serializer