diff --git a/src/uct/tcp/tcp.h b/src/uct/tcp/tcp.h index e96591b7cee..68c3f394b12 100644 --- a/src/uct/tcp/tcp.h +++ b/src/uct/tcp/tcp.h @@ -411,6 +411,7 @@ typedef struct uct_tcp_iface { unsigned syn_cnt; /* Number of SYN retransmits that TCP should send * before aborting the attempt to connect. * It cannot exceed 255. */ + ucs_time_t user_timeout; /* TCP_USER_TIMEOUT */ double max_bw; /* Upper bound to TCP iface bandwidth */ struct { ucs_time_t idle; /* The time the connection needs to remain @@ -449,6 +450,7 @@ typedef struct uct_tcp_iface_config { int sockopt_nodelay; uct_tcp_send_recv_buf_config_t sockopt; unsigned syn_cnt; + ucs_time_t user_timeout; uct_iface_mpool_config_t tx_mpool; uct_iface_mpool_config_t rx_mpool; ucs_range_spec_t port_range; diff --git a/src/uct/tcp/tcp_base.c b/src/uct/tcp/tcp_base.c index f3b7e752a88..7534795702d 100644 --- a/src/uct/tcp/tcp_base.c +++ b/src/uct/tcp/tcp_base.c @@ -12,13 +12,49 @@ #include +#include + ucs_status_t ucs_tcp_base_set_syn_cnt(int fd, int tcp_syn_cnt) { if (tcp_syn_cnt != UCS_ULUNITS_AUTO) { - ucs_socket_setopt(fd, IPPROTO_TCP, TCP_SYNCNT, (const void*)&tcp_syn_cnt, - sizeof(int)); + return ucs_socket_setopt(fd, IPPROTO_TCP, TCP_SYNCNT, + (const void*)&tcp_syn_cnt, + sizeof(tcp_syn_cnt)); } - /* return UCS_OK anyway since setting TCP_SYNCNT is done on best effort */ return UCS_OK; } + +ucs_status_t ucs_tcp_base_set_user_timeout(int fd, ucs_time_t user_timeout) +{ +#ifdef TCP_USER_TIMEOUT + int user_timeout_ms; + double user_timeout_ms_d; +#endif + + if (user_timeout == UCS_TIME_AUTO) { + return UCS_OK; + } + +#ifdef TCP_USER_TIMEOUT + if (user_timeout == UCS_TIME_INFINITY) { + user_timeout_ms = 0; + } else { + user_timeout_ms_d = ucs_time_to_msec(user_timeout); + if (user_timeout_ms_d < 1.0) { + user_timeout_ms = 1; + } else if (user_timeout_ms_d > INT_MAX) { + user_timeout_ms = INT_MAX; + } else { + user_timeout_ms = user_timeout_ms_d; + } + } + + return ucs_socket_setopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, + (const void*)&user_timeout_ms, + sizeof(user_timeout_ms)); +#else + ucs_error("TCP_USER_TIMEOUT is not supported"); + return UCS_ERR_UNSUPPORTED; +#endif +} diff --git a/src/uct/tcp/tcp_base.h b/src/uct/tcp/tcp_base.h index 7875b574284..9484f3095f6 100644 --- a/src/uct/tcp/tcp_base.h +++ b/src/uct/tcp/tcp_base.h @@ -12,6 +12,7 @@ #include #include #include +#include /** @@ -43,6 +44,14 @@ typedef struct uct_tcp_send_recv_buf_config { (_offset) , UCS_CONFIG_TYPE_ULUNITS} +#define UCT_TCP_USER_TIMEOUT(_offset) \ + {"USER_TIMEOUT", "auto", \ + "Maximum time that transmitted data may remain unacknowledged before TCP\n" \ + "forcibly closes the socket. auto means to use the system default.", \ + (_offset), UCS_CONFIG_TYPE_TIME_UNITS} + + ucs_status_t ucs_tcp_base_set_syn_cnt(int fd, int tcp_syn_cnt); +ucs_status_t ucs_tcp_base_set_user_timeout(int fd, ucs_time_t user_timeout); #endif /* UCT_TCP_BASE_H */ diff --git a/src/uct/tcp/tcp_iface.c b/src/uct/tcp/tcp_iface.c index 3a97b22d9dd..ee26c4923fb 100644 --- a/src/uct/tcp/tcp_iface.c +++ b/src/uct/tcp/tcp_iface.c @@ -78,6 +78,8 @@ static ucs_config_field_t uct_tcp_iface_config_table[] = { UCT_TCP_SYN_CNT(ucs_offsetof(uct_tcp_iface_config_t, syn_cnt)), + UCT_TCP_USER_TIMEOUT(ucs_offsetof(uct_tcp_iface_config_t, user_timeout)), + UCT_IFACE_MPOOL_CONFIG_FIELDS("TX_", -1, 8, 128m, 1.0, "send", ucs_offsetof(uct_tcp_iface_config_t, tx_mpool), ""), @@ -533,7 +535,12 @@ ucs_status_t uct_tcp_iface_set_sockopt(uct_tcp_iface_t *iface, int fd, return status; } - return ucs_tcp_base_set_syn_cnt(fd, iface->config.syn_cnt); + status = ucs_tcp_base_set_syn_cnt(fd, iface->config.syn_cnt); + if (status != UCS_OK) { + return status; + } + + return ucs_tcp_base_set_user_timeout(fd, iface->config.user_timeout); } static uct_iface_ops_t uct_tcp_iface_ops = { @@ -762,6 +769,7 @@ static UCS_CLASS_INIT_FUNC(uct_tcp_iface_t, uct_md_h md, uct_worker_h worker, self->config.max_poll = config->max_poll; self->config.max_conn_retries = config->max_conn_retries; self->config.syn_cnt = config->syn_cnt; + self->config.user_timeout = config->user_timeout; self->sockopt.nodelay = config->sockopt_nodelay; self->sockopt.sndbuf = config->sockopt.sndbuf; self->sockopt.rcvbuf = config->sockopt.rcvbuf; diff --git a/src/uct/tcp/tcp_sockcm.c b/src/uct/tcp/tcp_sockcm.c index c24e6634073..752a6901ce7 100644 --- a/src/uct/tcp/tcp_sockcm.c +++ b/src/uct/tcp/tcp_sockcm.c @@ -26,6 +26,8 @@ ucs_config_field_t uct_tcp_sockcm_config_table[] = { UCT_TCP_SYN_CNT(ucs_offsetof(uct_tcp_sockcm_config_t, syn_cnt)), + UCT_TCP_USER_TIMEOUT(ucs_offsetof(uct_tcp_sockcm_config_t, user_timeout)), + {NULL} }; @@ -221,6 +223,7 @@ UCS_CLASS_INIT_FUNC(uct_tcp_sockcm_t, uct_component_h component, self->sockopt_sndbuf = cm_config->sockopt.sndbuf; self->sockopt_rcvbuf = cm_config->sockopt.rcvbuf; self->syn_cnt = cm_config->syn_cnt; + self->user_timeout = cm_config->user_timeout; ucs_list_head_init(&self->ep_list); diff --git a/src/uct/tcp/tcp_sockcm.h b/src/uct/tcp/tcp_sockcm.h index 208e9818f28..b36f3121c25 100644 --- a/src/uct/tcp/tcp_sockcm.h +++ b/src/uct/tcp/tcp_sockcm.h @@ -20,6 +20,7 @@ typedef struct uct_tcp_sockcm { size_t sockopt_sndbuf; /** SO_SNDBUF */ size_t sockopt_rcvbuf; /** SO_RCVBUF */ unsigned syn_cnt; /** TCP_SYNCNT */ + ucs_time_t user_timeout; /** TCP_USER_TIMEOUT */ ucs_list_link_t ep_list; /** List of endpoints */ } uct_tcp_sockcm_t; @@ -31,6 +32,7 @@ typedef struct uct_tcp_sockcm_config { size_t priv_data_len; uct_tcp_send_recv_buf_config_t sockopt; unsigned syn_cnt; + ucs_time_t user_timeout; } uct_tcp_sockcm_config_t; diff --git a/src/uct/tcp/tcp_sockcm_ep.c b/src/uct/tcp/tcp_sockcm_ep.c index 900ae11b4ed..45cce15ecae 100644 --- a/src/uct/tcp/tcp_sockcm_ep.c +++ b/src/uct/tcp/tcp_sockcm_ep.c @@ -849,7 +849,12 @@ ucs_status_t uct_tcp_sockcm_ep_set_sockopt(uct_tcp_sockcm_ep_t *ep) return status; } - return ucs_tcp_base_set_syn_cnt(ep->fd, tcp_sockcm->syn_cnt); + status = ucs_tcp_base_set_syn_cnt(ep->fd, tcp_sockcm->syn_cnt); + if (status != UCS_OK) { + return status; + } + + return ucs_tcp_base_set_user_timeout(ep->fd, tcp_sockcm->user_timeout); } static ucs_status_t uct_tcp_sockcm_ep_client_init(uct_tcp_sockcm_ep_t *cep, diff --git a/test/gtest/uct/test_sockaddr.cc b/test/gtest/uct/test_sockaddr.cc index a13a2ad3606..7c58ca8ce1e 100644 --- a/test/gtest/uct/test_sockaddr.cc +++ b/test/gtest/uct/test_sockaddr.cc @@ -1142,7 +1142,12 @@ class test_uct_sockaddr_err_handle_non_exist_ip : public test_uct_sockaddr { * for the connect() to fail when connecting to a non-existing ip. * A transport for which this value is not configurable, like rdmacm, * will have no effect. */ - modify_config("SYN_CNT", "1", SETENV_IF_NOT_EXIST); + modify_config("TCP_CM_SYN_CNT", "1", IGNORE_IF_NOT_EXIST); + + /* TCP_SYNCNT may still leave SYN retransmission timeouts in the + * minutes range on some systems. Bound TCP_USER_TIMEOUT as well when + * the transport supports it. */ + modify_config("TCP_CM_USER_TIMEOUT", "1s", IGNORE_IF_NOT_EXIST); test_uct_sockaddr::init(); }