[Windows] Onboard Envoy to SCM

bazel/envoy_binary.bzl

@@ -69,6 +69,7 @@ def _envoy_linkopts():
         "@envoy//bazel:windows_x86_64": [
             "-DEFAULTLIB:ws2_32.lib",
             "-DEFAULTLIB:iphlpapi.lib",
+            "-DEFAULTLIB:shell32.lib",
             "-WX",
         ],
         "//conditions:default": [

examples/mysql/envoy.yaml

@@ -32,7 +32,7 @@ static_resources:
                 port_value: 3306
 
 admin:
-  access_log_path: "/dev/null"
+  access_log_path: "NUL"
   address:
     socket_address:
       address: 0.0.0.0

source/exe/BUILD

@@ -52,6 +52,7 @@ envoy_cc_library(
     deps = [
         ":envoy_main_common_lib",
         ":platform_impl_lib",
+        ":scm_impl_lib",
     ],
 )
 
@@ -176,6 +177,34 @@ envoy_cc_win32_library(
     ],
 )
 
+envoy_cc_library(
+    name = "scm_impl_lib",
+    deps = envoy_cc_platform_dep("scm_impl_lib"),
+    hdrs = select({
+        "//bazel:windows_x86_64": [
+            "win32/service_base.h",
+        ]}),
+    strip_include_prefix = select({
+        "//bazel:windows_x86_64": "win32",
+    }),
+)
+
+envoy_cc_win32_library(
+    name = "scm_impl_lib",
+    srcs = ["win32/service_base.cc"],
+    hdrs = [
+        "win32/service_base.h",
+        "win32/service_status.h",],
+    strip_include_prefix = "win32",
+    deps = [
+        ":main_common_lib",
+        "//source/common/buffer:buffer_lib",
+        "//source/common/common:assert_lib",
+        "//source/common/common:thread_lib",
+        "//source/common/event:signal_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "terminate_handler_lib",
     srcs = ["terminate_handler.cc"],

source/exe/main.cc

@@ -1,5 +1,7 @@
 #include "exe/main_common.h"
 
+#include "exe/service_base.h"
+
 // NOLINT(namespace-envoy)
 
 /**
@@ -9,4 +11,13 @@
  * deployment such as initializing signal handling. It calls main_common
  * after setting up command line options.
  */
-int main(int argc, char** argv) { return Envoy::MainCommon::main(argc, argv); }
+int main(int argc, char** argv) {
+#ifdef WIN32
+  Envoy::ServiceBase service;
+  if (!Envoy::ServiceBase::TryRunAsService(service)) {
+    return Envoy::MainCommon::main(argc, argv);
+  }
+  return EXIT_SUCCESS;
+#endif
+  return Envoy::MainCommon::main(argc, argv);
+}

source/exe/main_common.cc

@@ -185,6 +185,12 @@ void MainCommonBase::adminRequest(absl::string_view path_and_query, absl::string
   });
 }
 
+MainCommon::MainCommon(std::vector<std::string> args)
+    : options_(args, &MainCommon::hotRestartVersion, spdlog::level::info),
+      base_(options_, real_time_system_, default_listener_hooks_, prod_component_factory_,
+            std::make_unique<Random::RandomGeneratorImpl>(), platform_impl_.threadFactory(),
+            platform_impl_.fileSystem(), nullptr) {}
+
 MainCommon::MainCommon(int argc, const char* const* argv)
     : options_(argc, argv, &MainCommon::hotRestartVersion, spdlog::level::info),
       base_(options_, real_time_system_, default_listener_hooks_, prod_component_factory_,
@@ -208,13 +214,13 @@ int MainCommon::main(int argc, char** argv, PostServerHook hook) {
   // handling, such as running in a chroot jail.
   absl::InitializeSymbolizer(argv[0]);
 #endif
-  std::unique_ptr<Envoy::MainCommon> main_common;
+  std::shared_ptr<Envoy::MainCommon> main_common;
 
   // Initialize the server's main context under a try/catch loop and simply return EXIT_FAILURE
   // as needed. Whatever code in the initialization path that fails is expected to log an error
   // message so the user can diagnose.
   try {
-    main_common = std::make_unique<Envoy::MainCommon>(argc, argv);
+    main_common = std::make_shared<Envoy::MainCommon>(argc, argv);
     Envoy::Server::Instance* server = main_common->server();
     if (server != nullptr && hook != nullptr) {
       hook(*server);
@@ -228,9 +234,6 @@ int MainCommon::main(int argc, char** argv, PostServerHook hook) {
     std::cerr << e.what() << std::endl;
     return EXIT_FAILURE;
   }
-
-  // Run the server listener loop outside try/catch blocks, so that unexpected exceptions
-  // show up as a core-dumps for easier diagnostics.
   return main_common->run() ? EXIT_SUCCESS : EXIT_FAILURE;
 }
 

source/exe/main_common.h

@@ -98,6 +98,8 @@ class MainCommon {
   using PostServerHook = std::function<void(Server::Instance& server)>;
 
   MainCommon(int argc, const char* const* argv);
+  MainCommon(std::vector<std::string> args);
+
   bool run() { return base_.run(); }
   // Only tests have a legitimate need for this today.
   Event::Dispatcher& dispatcherForTest() { return base_.server()->dispatcher(); }

source/exe/win32/service_base.cc

@@ -0,0 +1,179 @@
+#include "exe/service_base.h"
+
+#include <codecvt>
+#include <locale>
+#include <processenv.h>
+#include <shellapi.h>
+
+#include "common/buffer/buffer_impl.h"
+#include "common/common/assert.h"
+#include "common/event/signal_impl.h"
+#include "exe/main_common.h"
+
+#include "absl/debugging/symbolize.h"
+
+#define SVCNAME TEXT("ENVOY")
+
+namespace Envoy {
+
+namespace {
+DWORD Win32FromHResult(HRESULT value) { return value & ~0x80070000; }
+
+} // namespace
+
+ServiceBase* ServiceBase::service_static = nullptr;
+
+ServiceBase::ServiceBase(DWORD controlsAccepted) : handle_(0) {
+  status_.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
+  status_.dwCurrentState = SERVICE_START_PENDING;
+  status_.dwControlsAccepted = controlsAccepted;
+}
+
+bool ServiceBase::TryRunAsService(ServiceBase& service) {
+  // We need to be extremely defensive when programming between the start of the program until
+  // `main_common` starts because the loggers have not been initialized
+  //  so we dont have a good way to know what is happening if the program fails.
+  service_static = &service;
+  RELEASE_ASSERT(service_static != nullptr, "Global pointer to service should not be null");
+
+  SERVICE_TABLE_ENTRYA service_table[] = {// Even though the service name is ignored for own process
+                                          // services, it must be a valid string and cannot be 0.
+                                          {SVCNAME, (LPSERVICE_MAIN_FUNCTIONA)ServiceMain},
+                                          // Designates the end of table.
+                                          {0, 0}};
+
+  if (!::StartServiceCtrlDispatcherA(service_table)) {
+    auto last_error = ::GetLastError();
+    if (last_error == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
+      return false;
+    } else {
+      PANIC(
+          fmt::format("Could not dispatch Envoy to start as a service with error {}", last_error));
+    }
+  }
+  return true;
+}
+
+DWORD ServiceBase::Start(std::vector<std::string> args, DWORD control) {
+  // Run the server listener loop outside try/catch blocks, so that unexpected exceptions
+  // show up as a core-dumps for easier diagnostics.
+  absl::InitializeSymbolizer(args[0].c_str());
+  std::shared_ptr<Envoy::MainCommon> main_common;
+
+  // Initialize the server's main context under a try/catch loop and simply return `EXIT_FAILURE`
+  // as needed. Whatever code in the initialization path that fails is expected to log an error
+  // message so the user can diagnose.
+  try {
+    main_common = std::make_shared<Envoy::MainCommon>(args);
+    Envoy::Server::Instance* server = main_common->server();
+    if (!server->options().signalHandlingEnabled()) {
+      // This means that the Envoy has not registered `ENVOY_SIGTERM`.
+      // we need to manually enable it as we are going to use it to
+      // handle close requests from `SCM`.
+      sigterm_ = server->dispatcher().listenForSignal(ENVOY_SIGTERM, [server]() {
+        ENVOY_LOG_MISC(warn, "caught ENVOY_SIGTERM");
+        server->shutdown();
+      });
+    }
+  } catch (const Envoy::NoServingException& e) {
+    return S_OK;
+  } catch (const Envoy::MalformedArgvException& e) {
+    return E_INVALIDARG;
+  } catch (const Envoy::EnvoyException& e) {
+    ENVOY_LOG_MISC(warn, "Envoy failed to start with {}", e.what());
+    return E_FAIL;
+  }
+
+  main_common->run();
+  return main_common->run() ? S_OK : E_FAIL;
+}
+
+void ServiceBase::Stop(DWORD control) {
+  auto handler = Event::eventBridgeHandlersSingleton::get()[ENVOY_SIGTERM];
+  if (!handler) {
+    PANIC("No handler is registered to stop gracefully, aborting the program.");
+  }
+
+  char data[] = {'a'};
+  Buffer::RawSlice buffer{data, 1};
+  auto result = handler->writev(&buffer, 1);
+  RELEASE_ASSERT(result.rc_ == 1,
+                 fmt::format("failed to write 1 byte: {}", result.err_->getErrorDetails()));
+}
+
+void ServiceBase::UpdateState(DWORD state, HRESULT errorCode, bool serviceError) {
+  status_.dwCurrentState = state;
+  if (FAILED(errorCode)) {
+    if (!serviceError) {
+      status_.dwWin32ExitCode = Win32FromHResult(errorCode);
+    } else {
+      status_.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR;
+      status_.dwServiceSpecificExitCode = errorCode;
+    }
+  }
+  SetServiceStatus();
+}
+
+void ServiceBase::SetServiceStatus() {
+  RELEASE_ASSERT(service_static != nullptr, "Global pointer to service should not be null");
+  if (!::SetServiceStatus(handle_, &status_)) {
+    PANIC(
+        fmt::format("Could not start StartServiceCtrlDispatcher with error {}", ::GetLastError()));
+  }
+}
+
+void WINAPI ServiceBase::ServiceMain(DWORD argc, LPSTR* argv) {
+  RELEASE_ASSERT(service_static != nullptr, "Global pointer to service should not be null");
+  if (argc != 1 || argv == 0 || argv[0] == 0) {
+    service_static->UpdateState(SERVICE_STOPPED, E_INVALIDARG, true);
+  }
+
+  service_static->handle_ = ::RegisterServiceCtrlHandler(SVCNAME, Handler);
+  if (service_static->handle_ == 0) {
+    service_static->UpdateState(SERVICE_STOPPED, ::GetLastError(), false);
+  }
+
+  // Windows Services can get their arguments in two different ways
+  // 1. With arguments coming from StartServiceA.
+  // 2. With command line arguments that have been registered when the service gets created.
+  // We merge the two cases into one vector of arguments that we provide to main common.
+  auto cli = std::wstring(::GetCommandLineW());
+  int envoyArgCount = 0;
+  LPWSTR* argvEnvoy = CommandLineToArgvW(cli.c_str(), &envoyArgCount);
+  std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
+  std::vector<std::string> args;
+  args.reserve(envoyArgCount + argc - 1);
+  for (int i = 0; i < envoyArgCount; ++i) {
+    args.emplace_back(converter.to_bytes(std::wstring(argvEnvoy[i])));
+  }
+
+  for (int i = 1; i < argc; i++) {
+    args.emplace_back(std::string(argv[i]));
+  }
+
+  service_static->SetServiceStatus();
+  service_static->UpdateState(SERVICE_RUNNING);
+  DWORD rc = service_static->Start(args, 0);
+  service_static->UpdateState(SERVICE_STOPPED, rc, true);
+}
+
+void WINAPI ServiceBase::Handler(DWORD control) {
+  // When the service control manager sends a control code to a service, it waits for the handler
+  // function to return before sending additional control codes to other services.
+  // The control handler should return as quickly as possible; if it does not return within 30
+  // seconds, the SCM returns an error. If a service must do lengthy processing when the service is
+  // executing the control handler, it should create a secondary thread to perform the lengthy
+  // processing, and then return from the control handler.
+  switch (control) {
+  case SERVICE_CONTROL_SHUTDOWN:
+  case SERVICE_CONTROL_PRESHUTDOWN:
+  case SERVICE_CONTROL_STOP: {
+    ENVOY_BUG(service_static->status_.dwCurrentState == SERVICE_RUNNING,
+              "Attempting to stop Envoy service when it is not running");
+    service_static->UpdateState(SERVICE_STOP_PENDING);
+    service_static->Stop(control);
+    break;
+  }
+  }
+}
+} // namespace Envoy

source/exe/win32/service_base.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <functional>
+#include <string>
+
+#include "exe/service_status.h"
+#include "envoy/event/signal.h"
+
+namespace Envoy {
+class ServiceBase {
+public:
+  virtual ~ServiceBase() = default;
+
+  static bool TryRunAsService(ServiceBase& service);
+
+  ServiceBase(DWORD controlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN |
+                                       SERVICE_CONTROL_PRESHUTDOWN);
+
+  DWORD Start(std::vector<std::string> args, DWORD control);
+  void Stop(DWORD control);
+
+  void UpdateState(DWORD state, HRESULT errorCode = S_OK, bool serviceError = true);
+
+private:
+  void SetServiceStatus();
+
+  static void WINAPI ServiceMain(DWORD argc, LPSTR* argv);
+
+  static void WINAPI Handler(DWORD control);
+
+  static ServiceBase* service_static;
+  SERVICE_STATUS_HANDLE handle_;
+  ServiceStatus status_;
+  Event::SignalEventPtr sigterm_;
+};
+} // namespace Envoy

source/exe/win32/service_status.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "envoy/common/platform.h"
+#include <winsvc.h>
+
+namespace Envoy {
+class ServiceStatus : public SERVICE_STATUS {
+public:
+  ServiceStatus() {
+    static_assert(sizeof(*this) == sizeof(SERVICE_STATUS));
+    ::ZeroMemory(this, sizeof(SERVICE_STATUS));
+  }
+};
+} // namespace Envoy