UCT/TCP: Bound negative connect test timeout

src/uct/tcp/tcp.h

@@ -411,6 +411,7 @@ typedef struct uct_tcp_iface {
         unsigned                  syn_cnt;           /* Number of SYN retransmits that TCP should send
                                                       * before aborting the attempt to connect.
                                                       * It cannot exceed 255. */
+        ucs_time_t                user_timeout;      /* TCP_USER_TIMEOUT */
         double                    max_bw;            /* Upper bound to TCP iface bandwidth */
         struct {
             ucs_time_t            idle;              /* The time the connection needs to remain
@@ -449,6 +450,7 @@ typedef struct uct_tcp_iface_config {
     int                            sockopt_nodelay;
     uct_tcp_send_recv_buf_config_t sockopt;
     unsigned                       syn_cnt;
+    ucs_time_t                     user_timeout;
     uct_iface_mpool_config_t       tx_mpool;
     uct_iface_mpool_config_t       rx_mpool;
     ucs_range_spec_t               port_range;

src/uct/tcp/tcp_base.c

@@ -12,13 +12,49 @@
 
 #include <ucs/sys/string.h>
 
+#include <limits.h>
+
 ucs_status_t ucs_tcp_base_set_syn_cnt(int fd, int tcp_syn_cnt)
 {
     if (tcp_syn_cnt != UCS_ULUNITS_AUTO) {
-        ucs_socket_setopt(fd, IPPROTO_TCP, TCP_SYNCNT, (const void*)&tcp_syn_cnt,
-                          sizeof(int));
+        return ucs_socket_setopt(fd, IPPROTO_TCP, TCP_SYNCNT,
+                                 (const void*)&tcp_syn_cnt,
+                                 sizeof(tcp_syn_cnt));
     }
 
-    /* return UCS_OK anyway since setting TCP_SYNCNT is done on best effort */
     return UCS_OK;
 }
+
+ucs_status_t ucs_tcp_base_set_user_timeout(int fd, ucs_time_t user_timeout)
+{
+#ifdef TCP_USER_TIMEOUT
+    int user_timeout_ms;
+    double user_timeout_ms_d;
+#endif
+
+    if (user_timeout == UCS_TIME_AUTO) {
+        return UCS_OK;
+    }
+
+#ifdef TCP_USER_TIMEOUT
+    if (user_timeout == UCS_TIME_INFINITY) {
+        user_timeout_ms = 0;
+    } else {
+        user_timeout_ms_d = ucs_time_to_msec(user_timeout);
+        if (user_timeout_ms_d < 1.0) {
+            user_timeout_ms = 1;
+        } else if (user_timeout_ms_d > INT_MAX) {
+            user_timeout_ms = INT_MAX;
+        } else {
+            user_timeout_ms = user_timeout_ms_d;
+        }
+    }
+
+    return ucs_socket_setopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT,
+                             (const void*)&user_timeout_ms,
+                             sizeof(user_timeout_ms));
+#else
+    ucs_error("TCP_USER_TIMEOUT is not supported");
+    return UCS_ERR_UNSUPPORTED;
+#endif
+}

src/uct/tcp/tcp_base.h

@@ -12,6 +12,7 @@
 #include <ucs/type/status.h>
 #include <ucs/sys/sock.h>
 #include <ucs/debug/log.h>
+#include <ucs/time/time.h>
 
 
 /**
@@ -43,6 +44,14 @@ typedef struct uct_tcp_send_recv_buf_config {
      (_offset) , UCS_CONFIG_TYPE_ULUNITS}
 
 
+#define UCT_TCP_USER_TIMEOUT(_offset) \
+    {"USER_TIMEOUT", "auto", \
+     "Maximum time that transmitted data may remain unacknowledged before TCP\n" \
+     "forcibly closes the socket. auto means to use the system default.", \
+     (_offset), UCS_CONFIG_TYPE_TIME_UNITS}
+
+
 ucs_status_t ucs_tcp_base_set_syn_cnt(int fd, int tcp_syn_cnt);
+ucs_status_t ucs_tcp_base_set_user_timeout(int fd, ucs_time_t user_timeout);
 
 #endif /* UCT_TCP_BASE_H */

src/uct/tcp/tcp_iface.c

@@ -78,6 +78,8 @@ static ucs_config_field_t uct_tcp_iface_config_table[] = {
 
   UCT_TCP_SYN_CNT(ucs_offsetof(uct_tcp_iface_config_t, syn_cnt)),
 
+  UCT_TCP_USER_TIMEOUT(ucs_offsetof(uct_tcp_iface_config_t, user_timeout)),
+
   UCT_IFACE_MPOOL_CONFIG_FIELDS("TX_", -1, 8, 128m, 1.0, "send",
                                 ucs_offsetof(uct_tcp_iface_config_t, tx_mpool), ""),
 
@@ -533,7 +535,12 @@ ucs_status_t uct_tcp_iface_set_sockopt(uct_tcp_iface_t *iface, int fd,
         return status;
     }
 
-    return ucs_tcp_base_set_syn_cnt(fd, iface->config.syn_cnt);
+    status = ucs_tcp_base_set_syn_cnt(fd, iface->config.syn_cnt);
+    if (status != UCS_OK) {
+        return status;
+    }
+
+    return ucs_tcp_base_set_user_timeout(fd, iface->config.user_timeout);
 }
 
 static uct_iface_ops_t uct_tcp_iface_ops = {
@@ -762,6 +769,7 @@ static UCS_CLASS_INIT_FUNC(uct_tcp_iface_t, uct_md_h md, uct_worker_h worker,
     self->config.max_poll          = config->max_poll;
     self->config.max_conn_retries  = config->max_conn_retries;
     self->config.syn_cnt           = config->syn_cnt;
+    self->config.user_timeout      = config->user_timeout;
     self->sockopt.nodelay          = config->sockopt_nodelay;
     self->sockopt.sndbuf           = config->sockopt.sndbuf;
     self->sockopt.rcvbuf           = config->sockopt.rcvbuf;

src/uct/tcp/tcp_sockcm.c

@@ -26,6 +26,8 @@ ucs_config_field_t uct_tcp_sockcm_config_table[] = {
 
    UCT_TCP_SYN_CNT(ucs_offsetof(uct_tcp_sockcm_config_t, syn_cnt)),
 
+   UCT_TCP_USER_TIMEOUT(ucs_offsetof(uct_tcp_sockcm_config_t, user_timeout)),
+
   {NULL}
 };
 
@@ -221,6 +223,7 @@ UCS_CLASS_INIT_FUNC(uct_tcp_sockcm_t, uct_component_h component,
     self->sockopt_sndbuf = cm_config->sockopt.sndbuf;
     self->sockopt_rcvbuf = cm_config->sockopt.rcvbuf;
     self->syn_cnt        = cm_config->syn_cnt;
+    self->user_timeout   = cm_config->user_timeout;
 
     ucs_list_head_init(&self->ep_list);
 

src/uct/tcp/tcp_sockcm.h

@@ -20,6 +20,7 @@ typedef struct uct_tcp_sockcm {
     size_t              sockopt_sndbuf;  /** SO_SNDBUF */
     size_t              sockopt_rcvbuf;  /** SO_RCVBUF */
     unsigned            syn_cnt;         /** TCP_SYNCNT */
+    ucs_time_t          user_timeout;    /** TCP_USER_TIMEOUT */
     ucs_list_link_t     ep_list;         /** List of endpoints */
 } uct_tcp_sockcm_t;
 
@@ -31,6 +32,7 @@ typedef struct uct_tcp_sockcm_config {
     size_t                          priv_data_len;
     uct_tcp_send_recv_buf_config_t  sockopt;
     unsigned                        syn_cnt;
+    ucs_time_t                      user_timeout;
 } uct_tcp_sockcm_config_t;
 
 

src/uct/tcp/tcp_sockcm_ep.c

@@ -849,7 +849,12 @@ ucs_status_t uct_tcp_sockcm_ep_set_sockopt(uct_tcp_sockcm_ep_t *ep)
         return status;
     }
 
-    return ucs_tcp_base_set_syn_cnt(ep->fd, tcp_sockcm->syn_cnt);
+    status = ucs_tcp_base_set_syn_cnt(ep->fd, tcp_sockcm->syn_cnt);
+    if (status != UCS_OK) {
+        return status;
+    }
+
+    return ucs_tcp_base_set_user_timeout(ep->fd, tcp_sockcm->user_timeout);
 }
 
 static ucs_status_t uct_tcp_sockcm_ep_client_init(uct_tcp_sockcm_ep_t *cep,

test/gtest/uct/test_sockaddr.cc

@@ -1142,7 +1142,12 @@ class test_uct_sockaddr_err_handle_non_exist_ip : public test_uct_sockaddr {
          * for the connect() to fail when connecting to a non-existing ip.
          * A transport for which this value is not configurable, like rdmacm,
          * will have no effect. */
-        modify_config("SYN_CNT", "1", SETENV_IF_NOT_EXIST);
+        modify_config("TCP_CM_SYN_CNT", "1", IGNORE_IF_NOT_EXIST);
+
+        /* TCP_SYNCNT may still leave SYN retransmission timeouts in the
+         * minutes range on some systems. Bound TCP_USER_TIMEOUT as well when
+         * the transport supports it. */
+        modify_config("TCP_CM_USER_TIMEOUT", "1s", IGNORE_IF_NOT_EXIST);
 
         test_uct_sockaddr::init();
     }