diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc
index e7d1e6d0a0cd2..6faf57ac090d9 100644
--- a/source/common/network/address_impl.cc
+++ b/source/common/network/address_impl.cc
@@ -21,19 +21,10 @@ namespace Address {
 
 namespace {
 
-// Check if an IP family is supported on this machine.
-bool ipFamilySupported(int domain) {
-  const int fd = ::socket(domain, SOCK_STREAM, 0);
-  if (fd >= 0) {
-    RELEASE_ASSERT(::close(fd) == 0, "");
-  }
-  return fd != -1;
-}
-
 // Validate that IPv4 is supported on this platform, raise an exception for the
 // given address if not.
 void validateIpv4Supported(const std::string& address) {
-  static const bool supported = ipFamilySupported(AF_INET);
+  static const bool supported = Network::Address::ipFamilySupported(AF_INET);
   if (!supported) {
     throw EnvoyException(
         fmt::format("IPv4 addresses are not supported on this machine: {}", address));
@@ -43,7 +34,7 @@ void validateIpv4Supported(const std::string& address) {
 // Validate that IPv6 is supported on this platform, raise an exception for the
 // given address if not.
 void validateIpv6Supported(const std::string& address) {
-  static const bool supported = ipFamilySupported(AF_INET6);
+  static const bool supported = Network::Address::ipFamilySupported(AF_INET6);
   if (!supported) {
     throw EnvoyException(
         fmt::format("IPv6 addresses are not supported on this machine: {}", address));
@@ -52,6 +43,15 @@ void validateIpv6Supported(const std::string& address) {
 
 } // namespace
 
+// Check if an IP family is supported on this machine.
+bool ipFamilySupported(int domain) {
+  const int fd = ::socket(domain, SOCK_STREAM, 0);
+  if (fd >= 0) {
+    RELEASE_ASSERT(::close(fd) == 0, "");
+  }
+  return fd != -1;
+}
+
 Address::InstanceConstSharedPtr addressFromSockAddr(const sockaddr_storage& ss, socklen_t ss_len,
                                                     bool v6only) {
   RELEASE_ASSERT(ss_len == 0 || ss_len >= sizeof(sa_family_t), "");
diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h
index c48003624c137..0bb41d26e422a 100644
--- a/source/common/network/address_impl.h
+++ b/source/common/network/address_impl.h
@@ -15,6 +15,12 @@ namespace Envoy {
 namespace Network {
 namespace Address {
 
+/**
+ * Returns true if the given family is supported on this machine.
+ * @param domain the IP family.
+ */
+bool ipFamilySupported(int domain);
+
 /**
  * Convert an address in the form of the socket address struct defined by Posix, Linux, etc. into
  * a Network::Address::Instance and return a pointer to it. Raises an EnvoyException on failure.
diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc
index 393fdfe1e35e5..e69bebe4ba3ff 100644
--- a/source/server/listener_manager_impl.cc
+++ b/source/server/listener_manager_impl.cc
@@ -380,17 +380,19 @@ void ListenerImpl::convertDestinationIPsMapToTrie() {
     for (const auto& entry : destination_ips_map) {
       std::vector<Network::Address::CidrRange> subnets;
       if (entry.first == EMPTY_STRING) {
-        list.push_back(
-            std::make_pair<ServerNamesMapSharedPtr, std::vector<Network::Address::CidrRange>>(
-                std::make_shared<ServerNamesMap>(entry.second),
-                {Network::Address::CidrRange::create("0.0.0.0/0"),
-                 Network::Address::CidrRange::create("::/0")}));
+        if (Network::Address::ipFamilySupported(AF_INET)) {
+          subnets.push_back(Network::Address::CidrRange::create("0.0.0.0/0"));
+        }
+        if (Network::Address::ipFamilySupported(AF_INET6)) {
+          subnets.push_back(Network::Address::CidrRange::create("::/0"));
+        }
       } else {
-        list.push_back(
-            std::make_pair<ServerNamesMapSharedPtr, std::vector<Network::Address::CidrRange>>(
-                std::make_shared<ServerNamesMap>(entry.second),
-                {Network::Address::CidrRange::create(entry.first)}));
+        subnets.push_back(Network::Address::CidrRange::create(entry.first));
       }
+      list.push_back(
+          std::make_pair<ServerNamesMapSharedPtr, std::vector<Network::Address::CidrRange>>(
+              std::make_shared<ServerNamesMap>(entry.second),
+              std::vector<Network::Address::CidrRange>(subnets)));
     }
     destination_ips_pair.second = std::make_unique<DestinationIPsTrie>(list, true);
   }