[Windows] Onboard Envoy to SCM

bazel/envoy_binary.bzl

@@ -63,12 +63,14 @@ def _envoy_linkopts():
         "@envoy//bazel:windows_opt_build": [
             "-DEFAULTLIB:ws2_32.lib",
             "-DEFAULTLIB:iphlpapi.lib",
+            "-DEFAULTLIB:shell32.lib",
             "-DEBUG:FULL",
             "-WX",
         ],
         "@envoy//bazel:windows_x86_64": [
             "-DEFAULTLIB:ws2_32.lib",
             "-DEFAULTLIB:iphlpapi.lib",
+            "-DEFAULTLIB:shell32.lib",
             "-WX",
         ],
         "//conditions:default": [

docs/root/start/quick-start/run-envoy.rst

@@ -104,6 +104,51 @@ The ``-c`` or ``--config-path`` flag tells Envoy the path to its initial configu
                    -c /envoy-custom.yaml
          ...
 
+   .. tab:: Windows Service
+
+      .. note::
+
+         This feature is still in Experimental state.
+
+      You can start Envoy as Windows Service that is managed under `Windows Service Control Manager <https://docs.microsoft.com/en-us/windows/win32/services/using-services/>`_. 
+
+      First, you need to create the service. Assuming you have a custom configuration in the current directory named ``envoy-custom.yaml``. After you create the service you
+      can start it.
+
+      From an **administrator** prompt run the following commands (note that you need replace C:\EnvoyProxy\ with the path to the envoy.exe binary and the config file):
+
+      .. substitution-code-block:: console
+
+         > sc create EnvoyProxy binpath="C:\EnvoyProxy\envoy.exe --config-path C:\EnvoyProxy\envoy-demo.yaml" start=auto depend=Tcpip/Afd
+            [SC] CreateService SUCCESS
+         > sc start EnvoyProxy
+            SERVICE_NAME: envoyproxy
+               TYPE               : 10  WIN32_OWN_PROCESS
+               STATE              : 2  START_PENDING
+                                       (NOT_STOPPABLE, NOT_PAUSABLE, IGNORES_SHUTDOWN)
+               WIN32_EXIT_CODE    : 0  (0x0)
+               SERVICE_EXIT_CODE  : 0  (0x0)
+               CHECKPOINT         : 0x0
+               WAIT_HINT          : 0x7d0
+               PID                : 3924
+               FLAGS              :
+         > sc query EnvoyProxy
+            SERVICE_NAME: envoyproxy
+               TYPE               : 10  WIN32_OWN_PROCESS
+               STATE              : 4  RUNNING
+                                       (STOPPABLE, NOT_PAUSABLE, ACCEPTS_SHUTDOWN)
+               WIN32_EXIT_CODE    : 0  (0x0)
+               SERVICE_EXIT_CODE  : 0  (0x0)
+               CHECKPOINT         : 0x0
+               WAIT_HINT          : 0x0
+         ...
+
+      Use `sc.exe <https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/sc-create/>`_ to configure the service startup and error handling.
+
+      .. tip::
+
+         The output of ``sc query envoyproxy`` contains the exit code of Envoy Proxy. In case the arguments are invalid we set it to ``E_INVALIDARG``.
+
 Check Envoy is proxying on http://localhost:10000.
 
 .. code-block:: console

source/exe/BUILD

@@ -52,6 +52,7 @@ envoy_cc_library(
     deps = [
         ":envoy_main_common_lib",
         ":platform_impl_lib",
+        ":scm_impl_lib",
     ],
 )
 
@@ -176,6 +177,37 @@ envoy_cc_win32_library(
     ],
 )
 
+envoy_cc_library(
+    name = "scm_impl_lib",
+    srcs = select({
+        "//bazel:windows_x86_64": [
+            "win32/service_base.cc",
+        ],
+        "//conditions:default": [],
+    }),
+    hdrs = select({
+        "//bazel:windows_x86_64": [
+            "win32/service_base.h",
+            "win32/service_status.h",
+        ],
+        "//conditions:default": [],
+    }),
+    strip_include_prefix = select({
+        "//bazel:windows_x86_64": "win32",
+        "//conditions:default": "",
+    }),
+    deps = select({
+        "//bazel:windows_x86_64": [
+            ":main_common_lib",
+            "//source/common/buffer:buffer_lib",
+            "//source/common/common:assert_lib",
+            "//source/common/common:thread_lib",
+            "//source/common/event:signal_lib",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
 envoy_cc_library(
     name = "terminate_handler_lib",
     srcs = ["terminate_handler.cc"],

source/exe/main.cc

@@ -1,5 +1,9 @@
 #include "exe/main_common.h"
 
+#ifdef WIN32
+#include "exe/service_base.h"
+#endif
+
 // NOLINT(namespace-envoy)
 
 /**
@@ -9,4 +13,13 @@
  * deployment such as initializing signal handling. It calls main_common
  * after setting up command line options.
  */
-int main(int argc, char** argv) { return Envoy::MainCommon::main(argc, argv); }
+int main(int argc, char** argv) {
+#ifdef WIN32
+  Envoy::ServiceBase service;
+  if (!Envoy::ServiceBase::TryRunAsService(service)) {
+    return Envoy::MainCommon::main(argc, argv);
+  }
+  return EXIT_SUCCESS;
+#endif
+  return Envoy::MainCommon::main(argc, argv);
+}

source/exe/main_common.cc

@@ -185,6 +185,12 @@ void MainCommonBase::adminRequest(absl::string_view path_and_query, absl::string
   });
 }
 
+MainCommon::MainCommon(const std::vector<std::string>& args)
+    : options_(args, &MainCommon::hotRestartVersion, spdlog::level::info),
+      base_(options_, real_time_system_, default_listener_hooks_, prod_component_factory_,
+            std::make_unique<Random::RandomGeneratorImpl>(), platform_impl_.threadFactory(),
+            platform_impl_.fileSystem(), nullptr) {}
+
 MainCommon::MainCommon(int argc, const char* const* argv)
     : options_(argc, argv, &MainCommon::hotRestartVersion, spdlog::level::info),
       base_(options_, real_time_system_, default_listener_hooks_, prod_component_factory_,

source/exe/main_common.h

@@ -98,6 +98,8 @@ class MainCommon {
   using PostServerHook = std::function<void(Server::Instance& server)>;
 
   MainCommon(int argc, const char* const* argv);
+  MainCommon(const std::vector<std::string>& args);
+
   bool run() { return base_.run(); }
   // Only tests have a legitimate need for this today.
   Event::Dispatcher& dispatcherForTest() { return base_.server()->dispatcher(); }

source/exe/win32/service_base.cc

@@ -0,0 +1,180 @@
+#include <codecvt>
+#include <locale>
+
+#include "common/buffer/buffer_impl.h"
+#include "common/common/assert.h"
+#include "common/event/signal_impl.h"
+
+#include "exe/main_common.h"
+#include "exe/service_base.h"
+
+#include "absl/debugging/symbolize.h"
+
+namespace Envoy {
+
+namespace {
+DWORD Win32FromHResult(HRESULT value) { return value & ~0x80070000; }
+
+} // namespace
+
+ServiceBase* ServiceBase::service_static = nullptr;
+
+ServiceBase::ServiceBase(DWORD controlsAccepted) : handle_(0) {
+  status_.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
+  status_.dwCurrentState = SERVICE_START_PENDING;
+  status_.dwControlsAccepted = controlsAccepted;
+}
+
+bool ServiceBase::TryRunAsService(ServiceBase& service) {
+  // We need to be extremely defensive when programming between the start of the program until
+  // `main_common` starts because the loggers have not been initialized
+  //  so we do not have a good way to know what is happening if the program fails.
+  service_static = &service;
+
+  // The `SERVICE_TABLE_ENTRY` struct requires a volatile `LPSTR`
+  char nullstr[1] = "";
+  SERVICE_TABLE_ENTRYA service_table[] = {// Even though the service name is ignored for own process
+                                          // services, it must be a valid string and cannot be 0.
+                                          {nullstr, (LPSERVICE_MAIN_FUNCTIONA)ServiceMain},
+                                          // Designates the end of table.
+                                          {0, 0}};
+
+  if (!::StartServiceCtrlDispatcherA(service_table)) {
+    auto last_error = ::GetLastError();
+    // There are two cases:
+    // 1. The user is trying to run Envoy as a console app. In that the last error is
+    // `ERROR_FAILED_SERVICE_CONTROLLER_CONNECT`. In that case we return false and let Envoy start
+    // as normal.
+    // 2. Other programmatic errors.
+    if (last_error == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
+      return false;
+    } else {
+      PANIC(
+          fmt::format("Could not dispatch Envoy to start as a service with error {}", last_error));
+    }
+  }
+  return true;
+}
+
+DWORD ServiceBase::Start(std::vector<std::string> args) {
+  // Run the server listener loop outside try/catch blocks, so that unexpected exceptions
+  // show up as a core-dumps for easier diagnostics.
+  absl::InitializeSymbolizer(args[0].c_str());
+  std::shared_ptr<Envoy::MainCommon> main_common;
+
+  // Initialize the server's main context under a try/catch loop and simply return `EXIT_FAILURE`
+  // as needed. Whatever code in the initialization path that fails is expected to log an error
+  // message so the user can diagnose.
+  try {
+    main_common = std::make_shared<Envoy::MainCommon>(args);
+    Envoy::Server::Instance* server = main_common->server();
+    if (!server->options().signalHandlingEnabled()) {
+      // This means that the Envoy has not registered `ENVOY_SIGTERM`.
+      // we need to manually enable it as we are going to use it to
+      // handle close requests from `SCM`.
+      sigterm_ = server->dispatcher().listenForSignal(ENVOY_SIGTERM, [server]() {
+        ENVOY_LOG_MISC(warn, "caught ENVOY_SIGTERM");
+        server->shutdown();
+      });
+    }
+  } catch (const Envoy::NoServingException& e) {
+    return S_OK;
+  } catch (const Envoy::MalformedArgvException& e) {
+    return E_INVALIDARG;
+  } catch (const Envoy::EnvoyException& e) {
+    ENVOY_LOG_MISC(warn, "Envoy failed to start with {}", e.what());
+    return E_FAIL;
+  }
+
+  return main_common->run() ? S_OK : E_FAIL;
+}
+
+void ServiceBase::Stop(DWORD control) {
+  auto handler = Event::eventBridgeHandlersSingleton::get()[ENVOY_SIGTERM];
+  if (!handler) {
+    PANIC("No handler is registered to stop gracefully, aborting the program.");
+  }
+
+  char data[] = {'a'};
+  Buffer::RawSlice buffer{data, 1};
+  auto result = handler->writev(&buffer, 1);
+  RELEASE_ASSERT(result.rc_ == 1,
+                 fmt::format("failed to write 1 byte: {}", result.err_->getErrorDetails()));
+}
+
+void ServiceBase::UpdateState(DWORD state, HRESULT errorCode, bool serviceError) {
+  status_.dwCurrentState = state;
+  if (FAILED(errorCode)) {
+    if (!serviceError) {
+      status_.dwWin32ExitCode = Win32FromHResult(errorCode);
+    } else {
+      status_.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR;
+      status_.dwServiceSpecificExitCode = errorCode;
+    }
+  }
+  SetServiceStatus();
+}
+
+void ServiceBase::SetServiceStatus() {
+  RELEASE_ASSERT(service_static != nullptr, "Global pointer to service should not be null");
+  if (!::SetServiceStatus(handle_, &status_)) {
+    PANIC(
+        fmt::format("Could not start StartServiceCtrlDispatcher with error {}", ::GetLastError()));
+  }
+}
+
+void WINAPI ServiceBase::ServiceMain(DWORD argc, LPSTR* argv) {
+  RELEASE_ASSERT(service_static != nullptr, "Global pointer to service should not be null");
+  if (argc < 1 || argv == 0 || argv[0] == 0) {
+    service_static->UpdateState(SERVICE_STOPPED, E_INVALIDARG, true);
+  }
+
+  service_static->handle_ = ::RegisterServiceCtrlHandlerA("ENVOY\0", Handler);
+  if (service_static->handle_ == 0) {
+    service_static->UpdateState(SERVICE_STOPPED, ::GetLastError(), false);
+  }
+
+  // Windows Services can get their arguments in two different ways
+  // 1. With command line arguments that have been registered when the service gets created.
+  // 2. With arguments coming from StartServiceA.
+  // We merge the two cases into one vector of arguments that we provide to main common.
+  auto cli = std::wstring(::GetCommandLineW());
+  int envoyArgCount = 0;
+  LPWSTR* argvEnvoy = CommandLineToArgvW(cli.c_str(), &envoyArgCount);
+  std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
+  std::vector<std::string> args;
+  args.reserve(envoyArgCount + argc - 1);
+  for (int i = 0; i < envoyArgCount; ++i) {
+    args.emplace_back(converter.to_bytes(std::wstring(argvEnvoy[i])));
+  }
+
+  for (int i = 1; i < argc; i++) {
+    args.emplace_back(std::string(argv[i]));
+  }
+
+  service_static->SetServiceStatus();
+  service_static->UpdateState(SERVICE_RUNNING);
+  DWORD rc = service_static->Start(args);
+  service_static->UpdateState(SERVICE_STOPPED, rc, true);
+}
+
+void WINAPI ServiceBase::Handler(DWORD control) {
+  // When the service control manager sends a control code to a service, it waits for the handler
+  // function to return before sending additional control codes to other services.
+  // The control handler should return as quickly as possible; if it does not return within 30
+  // seconds, the SCM returns an error. If a service must do lengthy processing when the service is
+  // executing the control handler, it should create a secondary thread to perform the lengthy
+  // processing, and then return from the control handler.
+  switch (control) {
+  case SERVICE_CONTROL_SHUTDOWN:
+  case SERVICE_CONTROL_PRESHUTDOWN:
+  case SERVICE_CONTROL_STOP: {
+    ENVOY_BUG(service_static->status_.dwCurrentState == SERVICE_RUNNING,
+              "Attempting to stop Envoy service when it is not running");
+    service_static->UpdateState(SERVICE_STOP_PENDING);
+    service_static->Stop(control);
+    break;
+  }
+  }
+}
+} // namespace Envoy

source/exe/win32/service_base.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <processenv.h>
+#include <shellapi.h>
+#include <winsvc.h>
+
+#include <functional>
+#include <string>
+
+#include "envoy/event/signal.h"
+
+#include "exe/service_status.h"
+
+namespace Envoy {
+class ServiceBase {
+public:
+  virtual ~ServiceBase() = default;
+  /**
+   * Try to register Envoy as a service. For this we register `ServiceMain` and `Handler`
+   * @param ServiceBase instance of the service.
+   * @return false if Envoy can not be registered to run as service.
+   */
+  static bool TryRunAsService(ServiceBase& service);
+
+  ServiceBase(DWORD controlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_PRESHUTDOWN);
+  /**
+   * Start the service.
+   * @return exit code of Envoy.
+   */
+  DWORD Start(std::vector<std::string> args);
+
+  /**
+   * Stop the service.
+   * @param control the control code Service Control Manager requested.
+   */
+  void Stop(DWORD control);
+
+  /**
+   * Update the state of the service.
+   * @param state New service state.
+   * @param errorCode Last error code encountered by Envoy.
+   * @param serviceError If the error is coming from the service or a Win32 interaction.
+   */
+  void UpdateState(DWORD state, HRESULT errorCode = S_OK, bool serviceError = true);
+
+private:
+  void SetServiceStatus();
+
+  static void WINAPI ServiceMain(DWORD argc, LPSTR* argv);
+
+  static void WINAPI Handler(DWORD control);
+
+  static ServiceBase* service_static;
+  SERVICE_STATUS_HANDLE handle_;
+  ServiceStatus status_;
+  Event::SignalEventPtr sigterm_;
+};
+} // namespace Envoy