diff --git a/mobile/library/common/BUILD b/mobile/library/common/BUILD index 0466044dc7f72..55ec285fd010c 100644 --- a/mobile/library/common/BUILD +++ b/mobile/library/common/BUILD @@ -53,7 +53,7 @@ envoy_cc_library( "@envoy//source/common/common:minimal_logger_lib", "@envoy//source/common/common:random_generator_lib", "@envoy//source/common/runtime:runtime_lib", - "@envoy//source/exe:main_common_lib", + "@envoy//source/exe:stripped_main_base_lib", ] + select({ "@envoy//bazel:disable_signal_trace": [], "//conditions:default": [ @@ -67,6 +67,6 @@ envoy_cc_library( repository = "@envoy", deps = [ ":engine_common_lib", - "@envoy//source/exe:envoy_main_common_lib", + "@envoy//source/exe:envoy_stripped_main_base_lib", ], ) diff --git a/mobile/library/common/engine_common.cc b/mobile/library/common/engine_common.cc index b20b31f147b20..c11b166b57851 100644 --- a/mobile/library/common/engine_common.cc +++ b/mobile/library/common/engine_common.cc @@ -5,8 +5,10 @@ namespace Envoy { +std::string hotRestartVersion(bool) { return "disabled"; } + EngineCommon::EngineCommon(int argc, const char* const* argv) - : options_(argc, argv, &MainCommon::hotRestartVersion, spdlog::level::info), + : options_(argc, argv, &hotRestartVersion, spdlog::level::info), base_(options_, real_time_system_, default_listener_hooks_, prod_component_factory_, std::make_unique(), std::make_unique(), nullptr) { diff --git a/mobile/library/common/engine_common.h b/mobile/library/common/engine_common.h index c979737c7a9a0..bae7acd5fb8cc 100644 --- a/mobile/library/common/engine_common.h +++ b/mobile/library/common/engine_common.h @@ -4,8 +4,8 @@ #include "envoy/server/instance.h" #include "source/common/event/real_time_system.h" -#include "source/exe/main_common.h" #include "source/exe/platform_impl.h" +#include "source/exe/stripped_main_base.h" #include "source/server/listener_hooks.h" #include "source/server/options_impl.h" @@ -18,12 +18,15 @@ namespace Envoy { /** * This class is used instead of Envoy::MainCommon to customize logic for the Envoy Mobile setting. - * It largely leverages Envoy::MainCommonBase. + * It largely leverages Envoy::StrippedMainBase. */ class EngineCommon { public: EngineCommon(int argc, const char* const* argv); - bool run() { return base_.run(); } + bool run() { + base_.runServer(); + return true; + } /** * @return a pointer to the server instance, or nullptr if initialized into @@ -38,13 +41,12 @@ class EngineCommon { Envoy::SignalAction handle_sigs_; Envoy::TerminateHandler log_on_terminate_; #endif - Thread::MainThread register_main_thread_; Envoy::OptionsImpl options_; Event::RealTimeSystem real_time_system_; // NO_CHECK_FORMAT(real_time) DefaultListenerHooks default_listener_hooks_; ProdComponentFactory prod_component_factory_; - MainCommonBase base_; + StrippedMainBase base_; }; } // namespace Envoy diff --git a/source/exe/BUILD b/source/exe/BUILD index 390bd8a4d84d0..a34d06a04d2e4 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -61,10 +61,40 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "stripped_main_base_lib", + srcs = ["stripped_main_base.cc"], + hdrs = ["stripped_main_base.h"], + deps = [ + ":envoy_common_lib", + ":platform_impl_lib", + ":process_wide_lib", + "//source/common/thread_local:thread_local_lib", + "//source/common/api:os_sys_calls_lib", + "//source/common/common:compiler_requirements_lib", + "//source/common/common:perf_annotation_lib", + "//source/common/grpc:google_grpc_context_lib", + "//source/server:hot_restart_lib", + "//source/server:hot_restart_nop_lib", + ] + select({ + "//bazel:disable_signal_trace": [], + "//conditions:default": [ + "//source/common/signal:sigaction_lib", + ":terminate_handler_lib", + ], + }), +) + envoy_cc_library( name = "main_common_lib", - srcs = ["main_common.cc"], - hdrs = ["main_common.h"], + srcs = [ + "main_common.cc", + "stripped_main_base.cc", + ], + hdrs = [ + "main_common.h", + "stripped_main_base.h", + ], deps = [ ":envoy_common_lib", ":platform_impl_lib", @@ -85,6 +115,7 @@ envoy_cc_library( }), ) +# provides a library target for Envoy server builds with the versioning information set up correctly. envoy_cc_library( name = "envoy_main_common_lib", deps = [ @@ -96,6 +127,15 @@ envoy_cc_library( ], ) +# provides a library target for Envoy Mobile builds with the versioning information set up correctly. +envoy_cc_library( + name = "envoy_stripped_main_base_lib", + deps = [ + ":stripped_main_base_lib", + "//source/common/version:version_linkstamp", + ], +) + envoy_cc_library( name = "envoy_common_with_core_extensions_lib", deps = [ @@ -117,8 +157,14 @@ envoy_cc_library( envoy_cc_library( name = "envoy_main_common_with_core_extensions_lib", - srcs = ["main_common.cc"], - hdrs = ["main_common.h"], + srcs = [ + "main_common.cc", + "stripped_main_base.cc", + ], + hdrs = [ + "main_common.h", + "stripped_main_base.h", + ], deps = [ ":envoy_common_with_core_extensions_lib", ":platform_impl_lib", diff --git a/source/exe/main_common.cc b/source/exe/main_common.cc index ea84f4037ba6e..ba0edd482cacb 100644 --- a/source/exe/main_common.cc +++ b/source/exe/main_common.cc @@ -29,154 +29,15 @@ namespace Envoy { -Server::DrainManagerPtr ProdComponentFactory::createDrainManager(Server::Instance& server) { - // The global drain manager only triggers on listener modification, which effectively is - // hot restart at the global level. The per-listener drain managers decide whether to - // to include /healthcheck/fail status. - return std::make_unique( - server, envoy::config::listener::v3::Listener::MODIFY_ONLY, server.dispatcher()); -} - -Runtime::LoaderPtr ProdComponentFactory::createRuntime(Server::Instance& server, - Server::Configuration::Initial& config) { - return Server::InstanceUtil::createRuntime(server, config); -} - -MainCommonBase::MainCommonBase(const Server::Options& options, Event::TimeSystem& time_system, - ListenerHooks& listener_hooks, - Server::ComponentFactory& component_factory, - std::unique_ptr platform_impl, - std::unique_ptr&& random_generator, - std::unique_ptr process_context) - : platform_impl_(std::move(platform_impl)), options_(options), - component_factory_(component_factory), stats_allocator_(symbol_table_) { - // Process the option to disable extensions as early as possible, - // before we do any configuration loading. - OptionsImpl::disableExtensions(options.disabledExtensions()); - - // Enable core dumps as early as possible. - if (options_.coreDumpEnabled()) { - const auto ret = platform_impl_->enableCoreDump(); - if (ret) { - ENVOY_LOG_MISC(info, "core dump enabled"); - } else { - ENVOY_LOG_MISC(warn, "failed to enable core dump"); - } - } - - switch (options_.mode()) { - case Server::Mode::InitOnly: - case Server::Mode::Serve: { - configureHotRestarter(*random_generator); - - tls_ = std::make_unique(); - Thread::BasicLockable& log_lock = restarter_->logLock(); - Thread::BasicLockable& access_log_lock = restarter_->accessLogLock(); - auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion()); - logging_context_ = std::make_unique(options_.logLevel(), options_.logFormat(), - log_lock, options_.logFormatEscaped(), - options_.enableFineGrainLogging()); - - configureComponentLogLevels(); - - // Provide consistent behavior for out-of-memory, regardless of whether it occurs in a try/catch - // block or not. - std::set_new_handler([]() { PANIC("out of memory"); }); - - stats_store_ = std::make_unique(stats_allocator_); - - server_ = std::make_unique( - *init_manager_, options_, time_system, local_address, listener_hooks, *restarter_, - *stats_store_, access_log_lock, component_factory, std::move(random_generator), *tls_, - platform_impl_->threadFactory(), platform_impl_->fileSystem(), std::move(process_context)); - - break; - } - case Server::Mode::Validate: - restarter_ = std::make_unique(); - logging_context_ = - std::make_unique(options_.logLevel(), options_.logFormat(), - restarter_->logLock(), options_.logFormatEscaped()); - break; - } -} - -void MainCommonBase::configureComponentLogLevels() { - for (auto& component_log_level : options_.componentLogLevels()) { - Logger::Logger* logger_to_change = Logger::Registry::logger(component_log_level.first); - ASSERT(logger_to_change); - logger_to_change->setLevel(component_log_level.second); - } -} - -void MainCommonBase::configureHotRestarter(Random::RandomGenerator& random_generator) { -#ifdef ENVOY_HOT_RESTART - if (!options_.hotRestartDisabled()) { - uint32_t base_id = options_.baseId(); - - if (options_.useDynamicBaseId()) { - ASSERT(options_.restartEpoch() == 0, "cannot use dynamic base id during hot restart"); - - std::unique_ptr restarter; - - // Try 100 times to get an unused base ID and then give up under the assumption - // that some other problem has occurred to prevent binding the domain socket. - for (int i = 0; i < 100 && restarter == nullptr; i++) { - // HotRestartImpl is going to multiply this value by 10, so leave head room. - base_id = static_cast(random_generator.random()) & 0x0FFFFFFF; - - TRY_ASSERT_MAIN_THREAD { - restarter = std::make_unique(base_id, 0, options_.socketPath(), - options_.socketMode()); - } - END_TRY - catch (Server::HotRestartDomainSocketInUseException& ex) { - // No luck, try again. - ENVOY_LOG_MISC(debug, "dynamic base id: {}", ex.what()); - } - } - - if (restarter == nullptr) { - throw EnvoyException("unable to select a dynamic base id"); - } - - restarter_.swap(restarter); - } else { - restarter_ = std::make_unique( - base_id, options_.restartEpoch(), options_.socketPath(), options_.socketMode()); - } - - // Write the base-id to the requested path whether we selected it - // dynamically or not. - if (!options_.baseIdPath().empty()) { - std::ofstream base_id_out_file(options_.baseIdPath()); - if (!base_id_out_file) { - ENVOY_LOG_MISC(critical, "cannot open base id output file {} for writing.", - options_.baseIdPath()); - } else { - base_id_out_file << base_id; - } - } - } -#else - UNREFERENCED_PARAMETER(random_generator); -#endif - - if (restarter_ == nullptr) { - restarter_ = std::make_unique(); - } -} - bool MainCommonBase::run() { switch (options_.mode()) { case Server::Mode::Serve: - server_->run(); + runServer(); return true; - case Server::Mode::Validate: { - auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion()); - return Server::validateConfig(options_, local_address, component_factory_, - platform_impl_->threadFactory(), platform_impl_->fileSystem()); - } + case Server::Mode::Validate: + return Server::validateConfig( + options_, Network::Utility::getLocalAddress(options_.localAddressIpVersion()), + component_factory_, platform_impl_->threadFactory(), platform_impl_->fileSystem()); case Server::Mode::InitOnly: PERF_DUMP(); return true; diff --git a/source/exe/main_common.h b/source/exe/main_common.h index f8dff2b702123..f101e578986aa 100644 --- a/source/exe/main_common.h +++ b/source/exe/main_common.h @@ -11,6 +11,7 @@ #include "source/common/stats/thread_local_store.h" #include "source/common/thread_local/thread_local_impl.h" #include "source/exe/process_wide.h" +#include "source/exe/stripped_main_base.h" #include "source/server/listener_hooks.h" #include "source/server/options_impl.h" #include "source/server/server.h" @@ -22,29 +23,12 @@ namespace Envoy { -class ProdComponentFactory : public Server::ComponentFactory { +class MainCommonBase : public StrippedMainBase { public: - // Server::DrainManagerFactory - Server::DrainManagerPtr createDrainManager(Server::Instance& server) override; - Runtime::LoaderPtr createRuntime(Server::Instance& server, - Server::Configuration::Initial& config) override; -}; - -class MainCommonBase { -public: - // Consumer must guarantee that all passed references are alive until this object is - // destructed. - MainCommonBase(const Server::Options& options, Event::TimeSystem& time_system, - ListenerHooks& listener_hooks, Server::ComponentFactory& component_factory, - std::unique_ptr platform_impl, - std::unique_ptr&& random_generator, - std::unique_ptr process_context); + using StrippedMainBase::StrippedMainBase; bool run(); - // Will be null if options.mode() == Server::Mode::Validate - Server::Instance* server() { return server_.get(); } - #ifdef ENVOY_ADMIN_FUNCTIONALITY using AdminRequestFn = std::function; @@ -64,38 +48,10 @@ class MainCommonBase { void adminRequest(absl::string_view path_and_query, absl::string_view method, const AdminRequestFn& handler); #endif - -protected: - std::unique_ptr platform_impl_; - ProcessWide process_wide_; // Process-wide state setup/teardown (excluding grpc). - // We instantiate this class regardless of ENVOY_GOOGLE_GRPC, to avoid having - // an ifdef in a header file exposed in a C++ library. It is too easy to have - // the ifdef be inconsistent across build-system boundaries. - Grpc::GoogleGrpcContext google_grpc_context_; - const Envoy::Server::Options& options_; - Server::ComponentFactory& component_factory_; - Stats::SymbolTableImpl symbol_table_; - Stats::AllocatorImpl stats_allocator_; - - ThreadLocal::InstanceImplPtr tls_; - std::unique_ptr restarter_; - Stats::ThreadLocalStoreImplPtr stats_store_; - std::unique_ptr logging_context_; - std::unique_ptr init_manager_{std::make_unique("Server")}; - std::unique_ptr server_; - -private: - void configureComponentLogLevels(); - void configureHotRestarter(Random::RandomGenerator& random_generator); - - // Declaring main thread here allows custom integrations to instantiate - // MainCommonBase directly, with environment-specific dependency injection. - // Note that MainThread must also be declared in MainCommon. - Thread::MainThread main_thread_; }; -// TODO(jmarantz): consider removing this class; I think it'd be more useful to -// go through MainCommonBase directly. +// This is separate from MainCommonBase for legacy reasons: sufficient +// downstream tests use one or the other that resolving is deemed problematic. class MainCommon { public: // Hook to run after a server is created. diff --git a/source/exe/stripped_main_base.cc b/source/exe/stripped_main_base.cc new file mode 100644 index 0000000000000..9d532a993b90c --- /dev/null +++ b/source/exe/stripped_main_base.cc @@ -0,0 +1,169 @@ +#include "source/exe/stripped_main_base.h" + +#include +#include +#include +#include + +#include "envoy/config/listener/v3/listener.pb.h" + +#include "source/common/common/compiler_requirements.h" +#include "source/common/common/logger.h" +#include "source/common/common/perf_annotation.h" +#include "source/common/network/utility.h" +#include "source/common/stats/thread_local_store.h" +#include "source/exe/platform_impl.h" +#include "source/server/drain_manager_impl.h" +#include "source/server/hot_restart_nop_impl.h" +#include "source/server/listener_hooks.h" +#include "source/server/options_impl.h" +#include "source/server/server.h" + +#include "absl/debugging/symbolize.h" +#include "absl/strings/str_split.h" + +#ifdef ENVOY_HOT_RESTART +#include "source/server/hot_restart_impl.h" +#endif + +namespace Envoy { + +Server::DrainManagerPtr ProdComponentFactory::createDrainManager(Server::Instance& server) { + // The global drain manager only triggers on listener modification, which effectively is + // hot restart at the global level. The per-listener drain managers decide whether to + // to include /healthcheck/fail status. + return std::make_unique( + server, envoy::config::listener::v3::Listener::MODIFY_ONLY, server.dispatcher()); +} + +Runtime::LoaderPtr ProdComponentFactory::createRuntime(Server::Instance& server, + Server::Configuration::Initial& config) { + return Server::InstanceUtil::createRuntime(server, config); +} + +StrippedMainBase::StrippedMainBase(const Server::Options& options, Event::TimeSystem& time_system, + ListenerHooks& listener_hooks, + Server::ComponentFactory& component_factory, + std::unique_ptr platform_impl, + std::unique_ptr&& random_generator, + std::unique_ptr process_context) + : platform_impl_(std::move(platform_impl)), options_(options), + component_factory_(component_factory), stats_allocator_(symbol_table_) { + // Process the option to disable extensions as early as possible, + // before we do any configuration loading. + OptionsImpl::disableExtensions(options.disabledExtensions()); + + // Enable core dumps as early as possible. + if (options_.coreDumpEnabled()) { + const auto ret = platform_impl_->enableCoreDump(); + if (ret) { + ENVOY_LOG_MISC(info, "core dump enabled"); + } else { + ENVOY_LOG_MISC(warn, "failed to enable core dump"); + } + } + + switch (options_.mode()) { + case Server::Mode::InitOnly: + case Server::Mode::Serve: { + configureHotRestarter(*random_generator); + + tls_ = std::make_unique(); + Thread::BasicLockable& log_lock = restarter_->logLock(); + Thread::BasicLockable& access_log_lock = restarter_->accessLogLock(); + auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion()); + logging_context_ = std::make_unique(options_.logLevel(), options_.logFormat(), + log_lock, options_.logFormatEscaped(), + options_.enableFineGrainLogging()); + + configureComponentLogLevels(); + + // Provide consistent behavior for out-of-memory, regardless of whether it occurs in a try/catch + // block or not. + std::set_new_handler([]() { PANIC("out of memory"); }); + + stats_store_ = std::make_unique(stats_allocator_); + + server_ = std::make_unique( + *init_manager_, options_, time_system, local_address, listener_hooks, *restarter_, + *stats_store_, access_log_lock, component_factory, std::move(random_generator), *tls_, + platform_impl_->threadFactory(), platform_impl_->fileSystem(), std::move(process_context)); + + break; + } + case Server::Mode::Validate: + restarter_ = std::make_unique(); + logging_context_ = + std::make_unique(options_.logLevel(), options_.logFormat(), + restarter_->logLock(), options_.logFormatEscaped()); + break; + } +} + +void StrippedMainBase::configureComponentLogLevels() { + for (auto& component_log_level : options_.componentLogLevels()) { + Logger::Logger* logger_to_change = Logger::Registry::logger(component_log_level.first); + ASSERT(logger_to_change); + logger_to_change->setLevel(component_log_level.second); + } +} + +void StrippedMainBase::configureHotRestarter(Random::RandomGenerator& random_generator) { +#ifdef ENVOY_HOT_RESTART + if (!options_.hotRestartDisabled()) { + uint32_t base_id = options_.baseId(); + + if (options_.useDynamicBaseId()) { + ASSERT(options_.restartEpoch() == 0, "cannot use dynamic base id during hot restart"); + + std::unique_ptr restarter; + + // Try 100 times to get an unused base ID and then give up under the assumption + // that some other problem has occurred to prevent binding the domain socket. + for (int i = 0; i < 100 && restarter == nullptr; i++) { + // HotRestartImpl is going to multiply this value by 10, so leave head room. + base_id = static_cast(random_generator.random()) & 0x0FFFFFFF; + + TRY_ASSERT_MAIN_THREAD { + restarter = std::make_unique(base_id, 0, options_.socketPath(), + options_.socketMode()); + } + END_TRY + catch (Server::HotRestartDomainSocketInUseException& ex) { + // No luck, try again. + ENVOY_LOG_MISC(debug, "dynamic base id: {}", ex.what()); + } + } + + if (restarter == nullptr) { + throw EnvoyException("unable to select a dynamic base id"); + } + + restarter_.swap(restarter); + } else { + restarter_ = std::make_unique( + base_id, options_.restartEpoch(), options_.socketPath(), options_.socketMode()); + } + + // Write the base-id to the requested path whether we selected it + // dynamically or not. + if (!options_.baseIdPath().empty()) { + std::ofstream base_id_out_file(options_.baseIdPath()); + if (!base_id_out_file) { + ENVOY_LOG_MISC(critical, "cannot open base id output file {} for writing.", + options_.baseIdPath()); + } else { + base_id_out_file << base_id; + } + } + } +#else + UNREFERENCED_PARAMETER(random_generator); +#endif + + if (restarter_ == nullptr) { + restarter_ = std::make_unique(); + } +} + +} // namespace Envoy diff --git a/source/exe/stripped_main_base.h b/source/exe/stripped_main_base.h new file mode 100644 index 0000000000000..10a2bcf0f73bc --- /dev/null +++ b/source/exe/stripped_main_base.h @@ -0,0 +1,86 @@ +#pragma once + +#include "envoy/event/timer.h" +#include "envoy/runtime/runtime.h" +#include "envoy/server/platform.h" + +#include "source/common/common/thread.h" +#include "source/common/event/real_time_system.h" +#include "source/common/grpc/google_grpc_context.h" +#include "source/common/stats/symbol_table.h" +#include "source/common/stats/thread_local_store.h" +#include "source/common/thread_local/thread_local_impl.h" +#include "source/exe/process_wide.h" +#include "source/server/listener_hooks.h" +#include "source/server/options_impl.h" +#include "source/server/server.h" + +#ifdef ENVOY_HANDLE_SIGNALS +#include "source/common/signal/signal_action.h" +#include "source/exe/terminate_handler.h" +#endif + +namespace Envoy { + +class ProdComponentFactory : public Server::ComponentFactory { +public: + // Server::DrainManagerFactory + Server::DrainManagerPtr createDrainManager(Server::Instance& server) override; + Runtime::LoaderPtr createRuntime(Server::Instance& server, + Server::Configuration::Initial& config) override; +}; + +// This is the common main between Envoy and Envoy mobile. +// It is stripped down to functionality required by Envoy Mobile: anything +// server-specific should live in MainCommonBase or MainCommon which remain +// separate for legacy reasons. +class StrippedMainBase { +public: + static std::string hotRestartVersion(bool hot_restart_enabled); + + // Consumer must guarantee that all passed references are alive until this object is + // destructed. + StrippedMainBase(const Server::Options& options, Event::TimeSystem& time_system, + ListenerHooks& listener_hooks, Server::ComponentFactory& component_factory, + std::unique_ptr platform_impl, + std::unique_ptr&& random_generator, + std::unique_ptr process_context); + + void runServer() { + ASSERT(options_.mode() == Server::Mode::Serve); + server_->run(); + } + + // Will be null if options.mode() == Server::Mode::Validate + Server::Instance* server() { return server_.get(); } + +protected: + std::unique_ptr platform_impl_; + ProcessWide process_wide_; // Process-wide state setup/teardown (excluding grpc). + // We instantiate this class regardless of ENVOY_GOOGLE_GRPC, to avoid having + // an ifdef in a header file exposed in a C++ library. It is too easy to have + // the ifdef be inconsistent across build-system boundaries. + Grpc::GoogleGrpcContext google_grpc_context_; + const Envoy::Server::Options& options_; + Server::ComponentFactory& component_factory_; + Stats::SymbolTableImpl symbol_table_; + Stats::AllocatorImpl stats_allocator_; + + ThreadLocal::InstanceImplPtr tls_; + std::unique_ptr restarter_; + Stats::ThreadLocalStoreImplPtr stats_store_; + std::unique_ptr logging_context_; + std::unique_ptr init_manager_{std::make_unique("Server")}; + std::unique_ptr server_; + +private: + void configureComponentLogLevels(); + void configureHotRestarter(Random::RandomGenerator& random_generator); + + // Declaring main thread here allows custom integrations to instantiate + // StrippedMainBase directly, with environment-specific dependency injection. + // Note that MainThread must also be declared in MainCommon. + Thread::MainThread main_thread_; +}; + +} // namespace Envoy diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh old mode 100755 new mode 100644