CDRIVER-4097 Implement srvMaxHosts for initial DNS seedlist and SRV polling

CONTRIBUTING.md

@@ -222,6 +222,8 @@ NOT tested, enable them with:
 
 * `MONGOC_TEST_DNS_LOADBALANCED=on` assumes a load balanced sharded cluster is running with mongoses on ports 27017 and 27018 and TLS enabled. The load balancer can be listening on any port.
 
+* `MONGOC_TEST_DNS_SRV_POLLING=on` assumes a sharded cluster is running with mongoses on ports 27017, 27018, 27019, and 27020 and TLS enabled.
+
 The mock server timeout threshold for future functions can be set with:
 
 * `MONGOC_TEST_FUTURE_TIMEOUT_MS=<int>`

src/libmongoc/doc/mongoc_uri_t.rst

@@ -101,6 +101,7 @@ MONGOC_URI_SOCKETTIMEOUTMS                 sockettimeoutms                   300
 MONGOC_URI_REPLICASET                      replicaset                        Empty (no replicaset)             The name of the Replica Set that the driver should connect to.
 MONGOC_URI_ZLIBCOMPRESSIONLEVEL            zlibcompressionlevel              -1                                When the MONGOC_URI_COMPRESSORS includes "zlib" this options configures the zlib compression level, when the zlib compressor is used to compress client data.
 MONGOC_URI_LOADBALANCED                    loadbalanced                      false                             If true, this indicates the driver is connecting to a MongoDB cluster behind a load balancer.
+MONGOC_URI_SRVMAXHOSTS                     srvmaxhosts                       0                                 If zero, the number of hosts in DNS results is unlimited. If greater than zero, the number of hosts in DNS results is limited to being less than or equal to the given value.
 ========================================== ================================= ================================= ============================================================================================================================================================================================================================================
 
 Setting any of the \*timeoutMS options above to ``0`` will be interpreted as "use the default value".

src/libmongoc/src/mongoc/mongoc-host-list-private.h

@@ -53,7 +53,7 @@ _mongoc_host_list_from_hostport_with_err (mongoc_host_list_t *host_list,
                                           bson_error_t *error);
 
 int
-_mongoc_host_list_length (mongoc_host_list_t *list);
+_mongoc_host_list_length (const mongoc_host_list_t *list);
 
 bool
 _mongoc_host_list_compare_one (const mongoc_host_list_t *host_a,

src/libmongoc/src/mongoc/mongoc-host-list.c

@@ -131,9 +131,9 @@ _mongoc_host_list_copy_all (const mongoc_host_list_t *src)
 }
 
 int
-_mongoc_host_list_length (mongoc_host_list_t *list)
+_mongoc_host_list_length (const mongoc_host_list_t *list)
 {
-   mongoc_host_list_t *tmp;
+   const mongoc_host_list_t *tmp;
    int counter = 0;
 
    tmp = list;
@@ -240,8 +240,7 @@ _mongoc_host_list_from_string_with_err (mongoc_host_list_t *link_,
                          MONGOC_ERROR_COMMAND,
                          MONGOC_ERROR_COMMAND_INVALID_ARG,
                          "If present, port should immediately follow the \"]\""
-                         "in an IPv6 address"
-                         );
+                         "in an IPv6 address");
          return false;
       }
 
@@ -259,8 +258,7 @@ _mongoc_host_list_from_string_with_err (mongoc_host_list_t *link_,
          bson_set_error (error,
                          MONGOC_ERROR_COMMAND,
                          MONGOC_ERROR_COMMAND_INVALID_ARG,
-                         "Missing matching bracket \"[\""
-                         );
+                         "Missing matching bracket \"[\"");
          return false;
       }
 
@@ -278,17 +276,15 @@ _mongoc_host_list_from_string_with_err (mongoc_host_list_t *link_,
          bson_set_error (error,
                          MONGOC_ERROR_COMMAND,
                          MONGOC_ERROR_COMMAND_INVALID_ARG,
-                         "Bad address, \":\" should not be first character"
-                        );
+                         "Bad address, \":\" should not be first character");
          return false;
       }
 
       if (!mongoc_parse_port (&port, sport + 1)) {
          bson_set_error (error,
                          MONGOC_ERROR_COMMAND,
                          MONGOC_ERROR_COMMAND_INVALID_ARG,
-                         "Port could not be parsed"
-                        );
+                         "Port could not be parsed");
          return false;
       }
 

src/libmongoc/src/mongoc/mongoc-topology-description-private.h

@@ -47,6 +47,7 @@ struct _mongoc_topology_description_t {
    bson_oid_t max_election_id;
    bson_error_t compatibility_error;
    uint32_t max_server_id;
+   int32_t max_hosts; /* srvMaxHosts */
    bool stale;
    unsigned int rand_seed;
 

src/libmongoc/src/mongoc/mongoc-topology-description.c

@@ -150,6 +150,7 @@ _mongoc_topology_description_copy_to (const mongoc_topology_description_t *src,
            &src->compatibility_error,
            sizeof (bson_error_t));
    dst->max_server_id = src->max_server_id;
+   dst->max_hosts = src->max_hosts;
    dst->stale = src->stale;
    memcpy (&dst->apm_callbacks,
            &src->apm_callbacks,
@@ -2282,12 +2283,35 @@ mongoc_topology_description_get_servers (
    return sds;
 }
 
+typedef struct {
+   mongoc_host_list_t *host_list;
+   size_t num_missing;
+} _count_num_hosts_to_remove_ctx_t;
+
+static bool
+_count_num_hosts_to_remove (void *sd_void, void *ctx_void)
+{
+   mongoc_server_description_t *sd;
+   _count_num_hosts_to_remove_ctx_t *ctx;
+   mongoc_host_list_t *host_list;
+
+   sd = sd_void;
+   ctx = ctx_void;
+   host_list = ctx->host_list;
+
+   if (!_mongoc_host_list_contains_one (host_list, &sd->host)) {
+      ++ctx->num_missing;
+   }
+
+   return true;
+}
+
 typedef struct {
    mongoc_host_list_t *host_list;
    mongoc_topology_description_t *td;
 } _remove_if_not_in_host_list_ctx_t;
 
-bool
+static bool
 _remove_if_not_in_host_list_cb (void *sd_void, void *ctx_void)
 {
    _remove_if_not_in_host_list_ctx_t *ctx;
@@ -2311,19 +2335,93 @@ void
 mongoc_topology_description_reconcile (mongoc_topology_description_t *td,
                                        mongoc_host_list_t *host_list)
 {
-   mongoc_host_list_t *host;
-   _remove_if_not_in_host_list_ctx_t ctx;
+   mongoc_set_t *servers;
+   size_t host_list_length;
+   size_t num_missing;
+
+   BSON_ASSERT_PARAM (td);
+
+   servers = mc_tpld_servers (td);
+   host_list_length = _mongoc_host_list_length (host_list);
+
+   /* Avoid removing all servers in topology, even temporarily, by deferring
+    * actual removal until after new hosts have been added. */
+   {
+      _count_num_hosts_to_remove_ctx_t ctx;
+
+      ctx.host_list = host_list;
+      ctx.num_missing = 0u;
+
+      mongoc_set_for_each (servers, _count_num_hosts_to_remove, &ctx);
+
+      num_missing = ctx.num_missing;
+   }
+
+   /* Polling SRV Records for mongos Discovery Spec: If srvMaxHosts is zero or
+    * greater than or equal to the number of valid hosts, each valid new host
+    * MUST be added to the topology as Unknown. */
+   if (td->max_hosts == 0 || (size_t) td->max_hosts >= host_list_length) {
+      mongoc_host_list_t *host;
+
+      LL_FOREACH (host_list, host)
+      {
+         /* "add" is really an "upsert" */
+         mongoc_topology_description_add_server (td, host->host_and_port, NULL);
+      }
+   }
+
+   /* Polling SRV Records for mongos Discovery Spec: If srvMaxHosts is greater
+    * than zero and less than the number of valid hosts, valid new hosts MUST be
+    * randomly selected and added to the topology as Unknown until the topology
+    * has srvMaxHosts hosts. */
+   else {
+      const size_t max_with_missing = td->max_hosts + num_missing;
+
+      size_t idx = 0u;
+      size_t hl_array_size = 0u;
 
-   LL_FOREACH (host_list, host)
+      /* Polling SRV Records for mongos Discovery Spec: Drivers MUST use the
+       * same randomization algorithm as they do for initial selection.
+       * Do not limit size of results yet (pass host_list_length) as we want to
+       * update any existing hosts in the topology, but add new hosts.
+       */
+      const mongoc_host_list_t *const *hl_array = _mongoc_apply_srv_max_hosts (
+         host_list, host_list_length, &hl_array_size);
+
+      for (idx = 0u;
+           servers->items_len < max_with_missing && idx < hl_array_size;
+           ++idx) {
+         const mongoc_host_list_t *const elem = hl_array[idx];
+
+         /* "add" is really an "upsert" */
+         mongoc_topology_description_add_server (td, elem->host_and_port, NULL);
+      }
+
+      /* There should not be a situation where all items in the valid host list
+       * were traversed without the number of hosts in the topology reaching
+       * srvMaxHosts. */
+      BSON_ASSERT (servers->items_len == max_with_missing);
+
+      bson_free ((void *) hl_array);
+   }
+
+   /* Polling SRV Records for mongos Discovery Spec: For all verified host
+    * names, as returned through the DNS SRV query, the driver MUST remove
+    * all hosts that are part of the topology, but are no longer in the
+    * returned set of valid hosts. */
    {
-      /* "add" is really an "upsert" */
-      mongoc_topology_description_add_server (td, host->host_and_port, NULL);
+      _remove_if_not_in_host_list_ctx_t ctx;
+
+      ctx.host_list = host_list;
+      ctx.td = td;
+
+      mongoc_set_for_each (servers, _remove_if_not_in_host_list_cb, &ctx);
    }
 
-   ctx.host_list = host_list;
-   ctx.td = td;
-   mongoc_set_for_each (
-      mc_tpld_servers (td), _remove_if_not_in_host_list_cb, &ctx);
+   /* At this point, the number of hosts in the host list should not exceed
+    * srvMaxHosts. */
+   BSON_ASSERT (td->max_hosts == 0 ||
+                servers->items_len <= (size_t) td->max_hosts);
 }
 
 

src/libmongoc/src/mongoc/mongoc-topology-private.h

@@ -588,4 +588,16 @@ _mongoc_topology_invalidate_server (mongoc_topology_t *td, uint32_t server_id)
    mc_tpld_modify_commit (tdmod);
 }
 
+/* Return an array view to `max_hosts` or fewer elements of `hl`, or NULL if
+ * `hl` is empty. The size of the returned array is written to `hl_array_size`
+ * even if `hl` is empty.
+ *
+ * The returned array must be freed with `bson_free()`. The elements of the
+ * array must not be freed, as they are still owned by `hl`.
+ */
+const mongoc_host_list_t **
+_mongoc_apply_srv_max_hosts (const mongoc_host_list_t *hl,
+                             int32_t max_hosts,
+                             size_t *hl_array_size);
+
 #endif