Init manager checking unready targets with target-aware watchers

api/envoy/admin/v3/config_dump.proto

@@ -381,3 +381,18 @@ message EndpointsConfigDump {
   // The dynamically loaded endpoint configs.
   repeated DynamicEndpointConfig dynamic_endpoint_configs = 3;
 }
+
+// Message of unready targets information of an init manager.
+message UnreadyTargetsConfigDump {
+  // Name of the init manager.
+  string name = 1;
+
+  // Names of unready targets in the hash map of the init manager.
+  repeated string target_names = 2;
+}
+
+// Envoy's admin fills this message with init managers, which provides the information of their unready targets.
+message UnreadyTargetsConfigDumpList {
+  // The list of each init manager's unready targets.
+  repeated UnreadyTargetsConfigDump unready_targets_configs = 1;
+}

include/envoy/init/target.h

@@ -25,6 +25,11 @@ struct TargetHandle {
    * @return true if the target received this call, false if the target was already destroyed.
    */
   virtual bool initialize(const Watcher& watcher) const PURE;
+
+  /**
+   * @return a human-readable target name, for logging / debugging.
+   */
+  virtual absl::string_view name() const PURE;
 };
 using TargetHandlePtr = std::unique_ptr<TargetHandle>;
 

include/envoy/server/listener_manager.h

@@ -167,6 +167,13 @@ class ListenerManager {
    */
   virtual std::vector<std::reference_wrapper<Network::ListenerConfig>> listeners() PURE;
 
+  /**
+   * @return std::vector<std::reference_wrapper<Network::ListenerConfig>> a list of the currently
+   * warming listeners. Note that this routine returns references to the existing listeners. The
+   * references are only valid in the context of the current call stack and should not be stored.
+   */
+  virtual std::vector<std::reference_wrapper<Network::ListenerConfig>> warmingListeners() PURE;
+
   /**
    * @return uint64_t the total number of connections owned by all listeners across all workers.
    */
@@ -221,6 +228,8 @@ class ListenerManager {
    * @return the server's API Listener if it exists, nullopt if it does not.
    */
   virtual ApiListenerOptRef apiListener() PURE;
+
+  virtual bool isWorkerStarted() PURE;
 };
 
 } // namespace Server

source/common/init/manager_impl.cc

@@ -1,13 +1,18 @@
 #include "common/init/manager_impl.h"
 
+#include <functional>
+
 #include "common/common/assert.h"
+#include "common/init/watcher_impl.h"
 
 namespace Envoy {
 namespace Init {
 
 ManagerImpl::ManagerImpl(absl::string_view name)
     : name_(fmt::format("init manager {}", name)), state_(State::Uninitialized), count_(0),
-      watcher_(name_, [this]() { onTargetReady(); }) {}
+      watcher_(name_, [this](absl::string_view target_name) {
+        onTargetReadySendTargetName(target_name);
+      }) {}
 
 Manager::State ManagerImpl::state() const { return state_; }
 
@@ -18,13 +23,15 @@ void ManagerImpl::add(const Target& target) {
   case State::Uninitialized:
     // If the manager isn't initialized yet, save the target handle to be initialized later.
     ENVOY_LOG(debug, "added {} to {}", target.name(), name_);
+    ++target_names_count_[target.name()];
     target_handles_.push_back(std::move(target_handle));
     return;
   case State::Initializing:
     // If the manager is already initializing, initialize the new target immediately. Note that
     // it's important in this case that count_ was incremented above before calling the target,
     // because if the target calls the init manager back immediately, count_ will be decremented
     // here (see the definition of watcher_ above).
+    ++target_names_count_[target.name()];
     target_handle->initialize(watcher_);
     return;
   case State::Initialized:
@@ -53,12 +60,16 @@ void ManagerImpl::initialize(const Watcher& watcher) {
     // completed immediately.
     for (const auto& target_handle : target_handles_) {
       if (!target_handle->initialize(watcher_)) {
-        onTargetReady();
+        onTargetReadySendTargetName(target_handle->name());
       }
     }
   }
 }
 
+const absl::flat_hash_map<std::string, uint32_t>& ManagerImpl::unreadyTargets() const {
+  return target_names_count_;
+}
+
 void ManagerImpl::onTargetReady() {
   // If there are no remaining targets and one mysteriously calls us back, this manager is haunted.
   ASSERT(count_ != 0, fmt::format("{} called back by target after initialization complete"));
@@ -70,6 +81,23 @@ void ManagerImpl::onTargetReady() {
   }
 }
 
+void ManagerImpl::onTargetReadySendTargetName(absl::string_view target_name) {
+  // If there are no remaining targets and one mysteriously calls us back, this manager is haunted.
+  ASSERT(count_ != 0,
+         fmt::format("{} called back by target after initialization complete", target_name));
+
+  // Decrease target_name count by 1.
+  if (--target_names_count_[target_name] == 0) {
+    target_names_count_.erase(target_name);
+  }
+
+  // If there are no uninitialized targets remaining when called back by a target, that means it was
+  // the last. Signal `ready` to the handle we saved in `initialize`.
+  if (--count_ == 0) {
+    ready();
+  }
+}
+
 void ManagerImpl::ready() {
   state_ = State::Initialized;
   watcher_handle_->ready();

source/common/init/manager_impl.h

@@ -7,6 +7,8 @@
 #include "common/common/logger.h"
 #include "common/init/watcher_impl.h"
 
+#include "absl/container/flat_hash_map.h"
+
 namespace Envoy {
 namespace Init {
 
@@ -35,27 +37,42 @@ class ManagerImpl : public Manager, Logger::Loggable<Logger::Id::init> {
   void add(const Target& target) override;
   void initialize(const Watcher& watcher) override;
 
+  // Expose the const reference of target_names_count_ hash map to public.
+  const absl::flat_hash_map<std::string, uint32_t>& unreadyTargets() const;
+
 private:
+  // Callback function without parameters, decrease unready targets count by 1.
   void onTargetReady();
+
+  // Callback function with an additional target_name parameter, decrease unready targets count by
+  // 1, update target_names_count_ hash map.
+  void onTargetReadySendTargetName(absl::string_view target_name);
+
   void ready();
 
-  // Human-readable name for logging
+  // Human-readable name for logging.
   const std::string name_;
 
-  // Current state
+  // Current state.
   State state_;
 
-  // Current number of registered targets that have not yet initialized
+  // Current number of registered targets that have not yet initialized.
   uint32_t count_;
 
-  // Handle to the watcher passed in `initialize`, to be called when initialization completes
+  // Handle to the watcher passed in `initialize`, to be called when initialization completes.
   WatcherHandlePtr watcher_handle_;
 
-  // Watcher to receive ready notifications from each target
+  // Watcher to receive ready notifications from each target. We restrict the watcher_ inside
+  // ManagerImpl to be constructed with the 'TargetAwareReadyFn' fn so that the init manager will
+  // get target name information when the watcher_ calls 'onTargetSendName(target_name)' For any
+  // other purpose, a watcher can be constructed with either TargetAwareReadyFn or ReadyFn.
   const WatcherImpl watcher_;
 
-  // All registered targets
+  // All registered targets.
   std::list<TargetHandlePtr> target_handles_;
+
+  // Count of target_name of unready targets.
+  absl::flat_hash_map<std::string, uint32_t> target_names_count_;
 };
 
 } // namespace Init

source/common/init/target_impl.cc

@@ -22,6 +22,8 @@ bool TargetHandleImpl::initialize(const Watcher& watcher) const {
   }
 }
 
+absl::string_view TargetHandleImpl::name() const { return name_; }
+
 TargetImpl::TargetImpl(absl::string_view name, InitializeFn fn)
     : name_(fmt::format("target {}", name)),
       fn_(std::make_shared<InternalInitalizeFn>([this, fn](WatcherHandlePtr watcher_handle) {

source/common/init/target_impl.h

@@ -38,6 +38,8 @@ class TargetHandleImpl : public TargetHandle, Logger::Loggable<Logger::Id::init>
   // Init::TargetHandle
   bool initialize(const Watcher& watcher) const override;
 
+  absl::string_view name() const override;
+
 private:
   // Name of the handle (almost always the name of the ManagerImpl calling the target)
   const std::string handle_name_;

source/common/init/watcher_impl.cc

@@ -4,34 +4,38 @@ namespace Envoy {
 namespace Init {
 
 WatcherHandleImpl::WatcherHandleImpl(absl::string_view handle_name, absl::string_view name,
-                                     std::weak_ptr<ReadyFn> fn)
+                                     std::weak_ptr<TargetAwareReadyFn> fn)
     : handle_name_(handle_name), name_(name), fn_(std::move(fn)) {}
 
 bool WatcherHandleImpl::ready() const {
   auto locked_fn(fn_.lock());
   if (locked_fn) {
     // If we can "lock" a shared pointer to the watcher's callback function, call it.
     ENVOY_LOG(debug, "{} initialized, notifying {}", handle_name_, name_);
-    (*locked_fn)();
+    (*locked_fn)(handle_name_);
     return true;
   } else {
     // If not, the watcher was already destroyed.
-    ENVOY_LOG(debug, "{} initialized, but can't notify {} (unavailable)", handle_name_, name_);
+    ENVOY_LOG(debug, "{} initialized, but can't notify {}", handle_name_, name_);
     return false;
   }
 }
 
 WatcherImpl::WatcherImpl(absl::string_view name, ReadyFn fn)
-    : name_(name), fn_(std::make_shared<ReadyFn>(std::move(fn))) {}
+    : name_(name), fn_(std::make_shared<TargetAwareReadyFn>(
+                       [callback = std::move(fn)](absl::string_view) { callback(); })) {}
+
+WatcherImpl::WatcherImpl(absl::string_view name, TargetAwareReadyFn fn)
+    : name_(name), fn_(std::make_shared<TargetAwareReadyFn>(std::move(fn))) {}
 
 WatcherImpl::~WatcherImpl() { ENVOY_LOG(debug, "{} destroyed", name_); }
 
 absl::string_view WatcherImpl::name() const { return name_; }
 
 WatcherHandlePtr WatcherImpl::createHandle(absl::string_view handle_name) const {
-  // Note: can't use std::make_unique because WatcherHandleImpl ctor is private
+  // Note: can't use std::make_unique because WatcherHandleImpl ctor is private.
   return std::unique_ptr<WatcherHandle>(
-      new WatcherHandleImpl(handle_name, name_, std::weak_ptr<ReadyFn>(fn_)));
+      new WatcherHandleImpl(handle_name, name_, std::weak_ptr<TargetAwareReadyFn>(fn_)));
 }
 
 } // namespace Init

source/common/init/watcher_impl.h

@@ -14,6 +14,7 @@ namespace Init {
  * initialization completes.
  */
 using ReadyFn = std::function<void()>;
+using TargetAwareReadyFn = std::function<void(absl::string_view)>;
 
 /**
  * A WatcherHandleImpl functions as a weak reference to a Watcher. It is how a TargetImpl safely
@@ -24,23 +25,25 @@ using ReadyFn = std::function<void()>;
 class WatcherHandleImpl : public WatcherHandle, Logger::Loggable<Logger::Id::init> {
 private:
   friend class WatcherImpl;
+
+  // Ctor with std::function<void()> callback function.
   WatcherHandleImpl(absl::string_view handle_name, absl::string_view name,
-                    std::weak_ptr<ReadyFn> fn);
+                    std::weak_ptr<TargetAwareReadyFn> fn);
 
 public:
-  // Init::WatcherHandle
+  // Init::WatcherHandle.
   bool ready() const override;
 
 private:
   // Name of the handle (either the name of the target calling the manager, or the name of the
-  // manager calling the client)
+  // manager calling the client).
   const std::string handle_name_;
 
-  // Name of the watcher (either the name of the manager, or the name of the client)
+  // Name of the watcher (either the name of the manager, or the name of the client).
   const std::string name_;
 
-  // The watcher's callback function, only called if the weak pointer can be "locked"
-  const std::weak_ptr<ReadyFn> fn_;
+  // The watcher's callback function, only called if the weak pointer can be "locked".
+  const std::weak_ptr<TargetAwareReadyFn> fn_;
 };
 
 /**
@@ -51,22 +54,23 @@ class WatcherHandleImpl : public WatcherHandle, Logger::Loggable<Logger::Id::ini
 class WatcherImpl : public Watcher, Logger::Loggable<Logger::Id::init> {
 public:
   /**
-   * @param name a human-readable watcher name, for logging / debugging
-   * @param fn a callback function to invoke when `ready` is called on the handle
+   * @param name a human-readable watcher name, for logging / debugging.
+   * @param fn a callback function to invoke when `ready` is called on the handle.
    */
   WatcherImpl(absl::string_view name, ReadyFn fn);
+  WatcherImpl(absl::string_view name, TargetAwareReadyFn fn);
   ~WatcherImpl() override;
 
-  // Init::Watcher
+  // Init::Watcher.
   absl::string_view name() const override;
   WatcherHandlePtr createHandle(absl::string_view handle_name) const override;
 
 private:
-  // Human-readable name for logging
+  // Human-readable name for logging.
   const std::string name_;
 
-  // The callback function, called via WatcherHandleImpl by either the target or the manager
-  const std::shared_ptr<ReadyFn> fn_;
+  // The callback function, called via WatcherHandleImpl by either the target or the manager.
+  const std::shared_ptr<TargetAwareReadyFn> fn_;
 };
 
 } // namespace Init

source/server/admin/BUILD

@@ -65,6 +65,7 @@ envoy_cc_library(
         "//source/common/stats:isolated_store_lib",
         "//source/common/upstream:host_utility_lib",
         "//source/extensions/access_loggers/file:file_access_log_lib",
+        "//source/server:listener_manager_lib",
         "@envoy_api//envoy/admin/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto",