[Store] HA: drain the master on SIGTERM/SIGINT before exit (part 1 of #4710)

mooncake-store/include/ha/leadership/master_service_supervisor.h

@@ -1,10 +1,13 @@
 #pragma once
 
 #include <atomic>
+#include <chrono>
+#include <functional>
 #include <mutex>
 #include <utility>
 
 #include "master_config.h"
+#include "shutdown_request.h"
 
 namespace mooncake {
 namespace ha {
@@ -46,15 +49,25 @@ class ServingStateGate {
 
 }  // namespace detail
 
+// Bound on an HA shutdown, from the first observation of the request to
+// process exit. main() arms the shutdown watchdog with it.
+inline constexpr std::chrono::seconds kHaShutdownDeadline{15};
+
 class MasterServiceSupervisor {
    public:
+    // shutdown_requested defaults to the process signal flag; tests inject
+    // their own predicate. The predicate can run concurrently on the
+    // supervisor and trigger threads, so it must be thread-safe and must not
+    // throw.
     explicit MasterServiceSupervisor(
-        const MasterServiceSupervisorConfig& config);
+        const MasterServiceSupervisorConfig& config,
+        std::function<bool()> shutdown_requested = ShutdownRequested);
 
     int Start();
 
    private:
     MasterServiceSupervisorConfig config_;
+    std::function<bool()> shutdown_requested_;
 };
 
 }  // namespace ha

mooncake-store/include/shutdown_request.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <signal.h>
+
+#include <chrono>
+
+namespace mooncake {
+
+// Process-wide SIGINT/SIGTERM intake for a process that drains before it
+// exits. The first signal sets a sticky flag. A second signal ends the
+// process at once with _exit(128 + signo).
+
+// Installs the SIGINT and SIGTERM handlers with sigaction and SA_ONSTACK,
+// which the Go c-shared HA backends in the master process require. Returns
+// false with errno set if a sigaction call fails. A handler installed before
+// the failure stays installed.
+bool InstallShutdownSignalHandlers();
+
+// True once SIGINT or SIGTERM was received. Safe from any thread.
+bool ShutdownRequested();
+
+// Records a shutdown request that code other than the signal handler
+// observed, such as an injected predicate. Sticky and idempotent. Call it
+// before any shutdown work: it starts the watchdog deadline.
+void LatchShutdownRequest();
+
+// True once LatchShutdownRequest or ExitAfterShutdown ran.
+bool ShutdownLatched();
+
+// Starts a detached watchdog thread. Once a signal was received or a request
+// was latched, the watchdog ends the process with _exit(EXIT_FAILURE) if it
+// is still alive `deadline` later. The deadline starts once and is never
+// reset. The thread takes no locks and does no I/O. Call it once, serially,
+// before the work it bounds; a repeated serial call returns true without a
+// second thread. Concurrent calls are not supported. Returns false if the
+// thread cannot be created.
+bool StartShutdownWatchdog(std::chrono::milliseconds deadline);
+
+// Ends the process after a completed shutdown. Latches the request, flushes
+// glog only when the watchdog thread exists, so a blocked flush still ends
+// at the deadline, then calls _exit(status); no destructors run.
+[[noreturn]] void ExitAfterShutdown(int status);
+
+namespace detail {
+
+using SigactionFn = int (*)(int, const struct sigaction*, struct sigaction*);
+
+// Test seam: InstallShutdownSignalHandlers() calls this with ::sigaction.
+bool InstallShutdownSignalHandlersWith(SigactionFn install);
+
+}  // namespace detail
+
+}  // namespace mooncake

mooncake-store/src/CMakeLists.txt

@@ -65,6 +65,7 @@ set(MOONCAKE_STORE_MASTER_SOURCES
     ha/common/redis/redis_connection.cpp
     ha/leadership/backends/redis/redis_leader_coordinator.cpp
     ha/leadership/master_service_supervisor.cpp
+    shutdown_request.cpp
     ha/kv/etcd_ha_kv_backend.cpp
     ha/kv/redis_ha_kv_backend.cpp
     ha/kv/ha_kv_backend_factory.cpp