Add a congestionWindowInBytes method to Envoy::Network::Connection

contrib/vcl/source/vcl_io_handle.cc

@@ -765,6 +765,8 @@ IoHandlePtr VclIoHandle::duplicate() {
 
 absl::optional<std::chrono::milliseconds> VclIoHandle::lastRoundTripTime() { return {}; }
 
+absl::optional<uint64_t> VclIoHandle::congestionWindowInBytes() const { return {}; }
+
 } // namespace Vcl
 } // namespace Network
 } // namespace Extensions

contrib/vcl/source/vcl_io_handle.h

@@ -45,6 +45,7 @@ class VclIoHandle : public Envoy::Network::IoHandle,
   Api::IoCallUint64Result recvmmsg(RawSliceArrays& slices, uint32_t self_port,
                                    RecvMsgOutput& output) override;
   absl::optional<std::chrono::milliseconds> lastRoundTripTime() override;
+  absl::optional<uint64_t> congestionWindowInBytes() const override;
 
   bool supportsMmsg() const override;
   bool supportsUdpGro() const override { return false; }

envoy/api/os_sys_calls.h

@@ -19,6 +19,9 @@ namespace Api {
 
 struct EnvoyTcpInfo {
   std::chrono::microseconds tcpi_rtt;
+  // Congestion window, in bytes. Note that posix's TCP_INFO socket option returns cwnd in packets,
+  // we multiply it by MSS to get bytes.
+  uint32_t tcpi_snd_cwnd = 0;
 };
 
 // Small struct to avoid exposing ifaddrs -- which is not defined in all platforms -- to the

envoy/network/connection.h

@@ -341,6 +341,14 @@ class Connection : public Event::DeferredDeletable,
    */
   virtual void configureInitialCongestionWindow(uint64_t bandwidth_bits_per_sec,
                                                 std::chrono::microseconds rtt) PURE;
+
+  /**
+   * @return the current congestion window in bytes, or unset if not available or not
+   * congestion-controlled.
+   * @note some congestion controller's cwnd is measured in number of packets, in that case the
+   * return value is cwnd(in packets) times the connection's MSS.
+   */
+  virtual absl::optional<uint64_t> congestionWindowInBytes() const PURE;
 };
 
 using ConnectionPtr = std::unique_ptr<Connection>;

envoy/network/io_handle.h

@@ -327,6 +327,14 @@ class IoHandle {
    */
   virtual absl::optional<std::chrono::milliseconds> lastRoundTripTime() PURE;
 
+  /**
+   * @return the current congestion window in bytes, or unset if not available or not
+   * congestion-controlled.
+   * @note some congestion controller's cwnd is measured in number of packets, in that case the
+   * return value is cwnd(in packets) times the connection's MSS.
+   */
+  virtual absl::optional<uint64_t> congestionWindowInBytes() const PURE;
+
   /**
    * @return the interface name for the socket, if the OS supports it. Otherwise, absl::nullopt.
    */

envoy/network/listen_socket.h

@@ -75,6 +75,14 @@ class ConnectionSocket : public virtual Socket, public virtual ScopeTrackedObjec
    *  returned.
    */
   virtual absl::optional<std::chrono::milliseconds> lastRoundTripTime() PURE;
+
+  /**
+   * @return the current congestion window in bytes, or unset if not available or not
+   * congestion-controlled.
+   * @note some congestion controller's cwnd is measured in number of packets, in that case the
+   * return value is cwnd(in packets) times the connection's MSS.
+   */
+  virtual absl::optional<uint64_t> congestionWindowInBytes() const PURE;
 };
 
 using ConnectionSocketPtr = std::unique_ptr<ConnectionSocket>;

source/common/api/posix/os_sys_calls_impl.cc

@@ -290,6 +290,10 @@ SysCallBoolResult OsSysCallsImpl::socketTcpInfo([[maybe_unused]] os_fd_t sockfd,
   auto result = ::getsockopt(sockfd, IPPROTO_TCP, TCP_INFO, &unix_tcp_info, &len);
   if (!SOCKET_FAILURE(result)) {
     tcp_info->tcpi_rtt = std::chrono::microseconds(unix_tcp_info.tcpi_rtt);
+
+    const uint64_t mss = (unix_tcp_info.tcpi_snd_mss > 0) ? unix_tcp_info.tcpi_snd_mss : 1460;
+    // Convert packets to bytes.
+    tcp_info->tcpi_snd_cwnd = unix_tcp_info.tcpi_snd_cwnd * mss;
   }
   return {!SOCKET_FAILURE(result), !SOCKET_FAILURE(result) ? 0 : errno};
 #endif

source/common/api/win32/os_sys_calls_impl.cc

@@ -403,6 +403,7 @@ SysCallBoolResult OsSysCallsImpl::socketTcpInfo([[maybe_unused]] os_fd_t sockfd,
 
   if (!SOCKET_FAILURE(rc)) {
     tcp_info->tcpi_rtt = std::chrono::microseconds(win_tcpinfo.RttUs);
+    tcp_info->tcpi_snd_cwnd = win_tcpinfo.Cwnd;
   }
   return {!SOCKET_FAILURE(rc), !SOCKET_FAILURE(rc) ? 0 : ::WSAGetLastError()};
 #endif

source/common/network/connection_impl.cc

@@ -788,6 +788,10 @@ void ConnectionImpl::configureInitialCongestionWindow(uint64_t bandwidth_bits_pe
   return transport_socket_->configureInitialCongestionWindow(bandwidth_bits_per_sec, rtt);
 }
 
+absl::optional<uint64_t> ConnectionImpl::congestionWindowInBytes() const {
+  return socket_->congestionWindowInBytes();
+}
+
 void ConnectionImpl::flushWriteBuffer() {
   if (state() == State::Open && write_buffer_->length() > 0) {
     onWriteReady();

source/common/network/connection_impl.h

@@ -101,6 +101,7 @@ class ConnectionImpl : public ConnectionImplBase, public TransportSocketCallback
   absl::optional<std::chrono::milliseconds> lastRoundTripTime() const override;
   void configureInitialCongestionWindow(uint64_t bandwidth_bits_per_sec,
                                         std::chrono::microseconds rtt) override;
+  absl::optional<uint64_t> congestionWindowInBytes() const override;
 
   // Network::FilterManagerConnection
   void rawWrite(Buffer::Instance& data, bool end_stream) override;

source/common/network/happy_eyeballs_connection_impl.cc

@@ -282,6 +282,11 @@ absl::optional<std::chrono::milliseconds> HappyEyeballsConnectionImpl::lastRound
   return connections_[0]->lastRoundTripTime();
 }
 
+absl::optional<uint64_t> HappyEyeballsConnectionImpl::congestionWindowInBytes() const {
+  // Note, this value changes constantly even within the same connection.
+  return connections_[0]->congestionWindowInBytes();
+}
+
 void HappyEyeballsConnectionImpl::addConnectionCallbacks(ConnectionCallbacks& cb) {
   if (connect_finished_) {
     connections_[0]->addConnectionCallbacks(cb);

source/common/network/happy_eyeballs_connection_impl.h

@@ -69,6 +69,7 @@ class HappyEyeballsConnectionImpl : public ClientConnection,
   bool startSecureTransport() override;
   absl::optional<std::chrono::milliseconds> lastRoundTripTime() const override;
   void configureInitialCongestionWindow(uint64_t, std::chrono::microseconds) override {}
+  absl::optional<uint64_t> congestionWindowInBytes() const override;
 
   // Simple getters which always delegate to the first connection in connections_.
   bool isHalfCloseEnabled() override;

source/common/network/io_socket_handle_impl.cc

@@ -576,6 +576,15 @@ absl::optional<std::chrono::milliseconds> IoSocketHandleImpl::lastRoundTripTime(
   return std::chrono::duration_cast<std::chrono::milliseconds>(info.tcpi_rtt);
 }
 
+absl::optional<uint64_t> IoSocketHandleImpl::congestionWindowInBytes() const {
+  Api::EnvoyTcpInfo info;
+  auto result = Api::OsSysCallsSingleton::get().socketTcpInfo(fd_, &info);
+  if (!result.return_value_) {
+    return {};
+  }
+  return info.tcpi_snd_cwnd;
+}
+
 absl::optional<std::string> IoSocketHandleImpl::interfaceName() {
   auto& os_syscalls_singleton = Api::OsSysCallsSingleton::get();
   if (!os_syscalls_singleton.supportsGetifaddrs()) {

source/common/network/io_socket_handle_impl.h

@@ -80,6 +80,7 @@ class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable<Logger::I
 
   Api::SysCallIntResult shutdown(int how) override;
   absl::optional<std::chrono::milliseconds> lastRoundTripTime() override;
+  absl::optional<uint64_t> congestionWindowInBytes() const override;
   absl::optional<std::string> interfaceName() override;
 
 protected:

source/common/network/listen_socket_impl.h

@@ -228,6 +228,10 @@ class ConnectionSocketImpl : public SocketImpl, public ConnectionSocket {
     return ioHandle().lastRoundTripTime();
   }
 
+  absl::optional<uint64_t> congestionWindowInBytes() const override {
+    return ioHandle().congestionWindowInBytes();
+  }
+
   void dumpState(std::ostream& os, int indent_level) const override {
     const char* spaces = spacesForLevel(indent_level);
     os << spaces << "ListenSocketImpl " << this << DUMP_MEMBER(transport_protocol_) << "\n";

source/common/quic/quic_filter_manager_connection_impl.cc

@@ -217,6 +217,20 @@ void QuicFilterManagerConnectionImpl::onSendBufferLowWatermark() {
   }
 }
 
+absl::optional<std::chrono::milliseconds>
+QuicFilterManagerConnectionImpl::lastRoundTripTime() const {
+  if (quicConnection() == nullptr) {
+    return {};
+  }
+
+  const auto* rtt_stats = quicConnection()->sent_packet_manager().GetRttStats();
+  if (!rtt_stats->latest_rtt().IsZero()) {
+    return std::chrono::milliseconds(rtt_stats->latest_rtt().ToMilliseconds());
+  }
+
+  return std::chrono::milliseconds(rtt_stats->initial_rtt().ToMilliseconds());
+}
+
 void QuicFilterManagerConnectionImpl::configureInitialCongestionWindow(
     uint64_t bandwidth_bits_per_sec, std::chrono::microseconds rtt) {
   if (quicConnection() != nullptr) {
@@ -231,5 +245,18 @@ void QuicFilterManagerConnectionImpl::configureInitialCongestionWindow(
   }
 }
 
+absl::optional<uint64_t> QuicFilterManagerConnectionImpl::congestionWindowInBytes() const {
+  if (quicConnection() == nullptr) {
+    return {};
+  }
+
+  uint64_t cwnd = quicConnection()->sent_packet_manager().GetCongestionWindowInBytes();
+  if (cwnd == 0) {
+    return {};
+  }
+
+  return cwnd;
+}
+
 } // namespace Quic
 } // namespace Envoy

source/common/quic/quic_filter_manager_connection_impl.h

@@ -114,10 +114,10 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase,
   const StreamInfo::StreamInfo& streamInfo() const override { return stream_info_; }
   absl::string_view transportFailureReason() const override { return transport_failure_reason_; }
   bool startSecureTransport() override { return false; }
-  // TODO(#2557) Implement this.
-  absl::optional<std::chrono::milliseconds> lastRoundTripTime() const override { return {}; }
+  absl::optional<std::chrono::milliseconds> lastRoundTripTime() const override;
   void configureInitialCongestionWindow(uint64_t bandwidth_bits_per_sec,
                                         std::chrono::microseconds rtt) override;
+  absl::optional<uint64_t> congestionWindowInBytes() const override;
 
   // Network::FilterManagerConnection
   void rawWrite(Buffer::Instance& data, bool end_stream) override;

source/common/quic/quic_io_handle_wrapper.h

@@ -5,6 +5,7 @@
 
 #include "envoy/network/io_handle.h"
 
+#include "source/common/common/assert.h"
 #include "source/common/network/io_socket_error_impl.h"
 
 namespace Envoy {
@@ -142,6 +143,12 @@ class QuicIoHandleWrapper : public Network::IoHandle {
 
   Api::SysCallIntResult shutdown(int how) override { return io_handle_.shutdown(how); }
   absl::optional<std::chrono::milliseconds> lastRoundTripTime() override { return {}; }
+  absl::optional<uint64_t> congestionWindowInBytes() const override {
+    // QUIC should get congestion window from QuicFilterManagerConnectionImpl, which implements the
+    // Envoy::Network::Connection::congestionWindowInBytes interface.
+    IS_ENVOY_BUG("QuicIoHandleWrapper does not implement congestionWindowInBytes.");
+    return {};
+  }
 
 private:
   Network::IoHandle& io_handle_;

source/extensions/io_socket/user_space/io_handle_impl.h

@@ -88,6 +88,7 @@ class IoHandleImpl final : public Network::IoHandle,
 
   Api::SysCallIntResult shutdown(int how) override;
   absl::optional<std::chrono::milliseconds> lastRoundTripTime() override { return absl::nullopt; }
+  absl::optional<uint64_t> congestionWindowInBytes() const override { return absl::nullopt; }
   absl::optional<std::string> interfaceName() override { return absl::nullopt; }
 
   void setWatermarks(uint32_t watermark) { pending_received_data_.setWatermarks(watermark); }

source/server/api_listener_impl.h

@@ -160,8 +160,9 @@ class ApiListenerImplBase : public ApiListener,
         IS_ENVOY_BUG("Unexpected function call");
         return false;
       }
-      absl::optional<std::chrono::milliseconds> lastRoundTripTime() const override { return {}; };
+      absl::optional<std::chrono::milliseconds> lastRoundTripTime() const override { return {}; }
       void configureInitialCongestionWindow(uint64_t, std::chrono::microseconds) override {}
+      absl::optional<uint64_t> congestionWindowInBytes() const override { return {}; }
       // ScopeTrackedObject
       void dumpState(std::ostream& os, int) const override { os << "SyntheticConnection"; }
 

test/common/network/connection_impl_test.cc

@@ -303,6 +303,23 @@ TEST_P(ConnectionImplTest, UniqueId) {
   disconnect(false);
 }
 
+TEST_P(ConnectionImplTest, GetCongestionWindow) {
+  setUpBasicConnection();
+  connect();
+
+// Congestion window is available on Posix(guarded by TCP_INFO) and Windows(guarded by
+// SIO_TCP_INFO).
+#if defined(TCP_INFO) || defined(SIO_TCP_INFO)
+  EXPECT_GT(client_connection_->congestionWindowInBytes().value(), 500);
+  EXPECT_GT(server_connection_->congestionWindowInBytes().value(), 500);
+#else
+  EXPECT_FALSE(client_connection_->congestionWindowInBytes().has_value());
+  EXPECT_FALSE(server_connection_->congestionWindowInBytes().has_value());
+#endif
+
+  disconnect(true);
+}
+
 TEST_P(ConnectionImplTest, CloseDuringConnectCallback) {
   setUpBasicConnection();
 

test/common/quic/client_connection_factory_impl_test.cc

@@ -64,7 +64,7 @@ TEST_P(QuicNetworkConnectionTest, BufferLimits) {
   ASSERT(session != nullptr);
   EXPECT_EQ(highWatermark(session), 45);
   EXPECT_EQ(absl::nullopt, session->unixSocketPeerCredentials());
-  EXPECT_EQ(absl::nullopt, session->lastRoundTripTime());
+  EXPECT_NE(absl::nullopt, session->lastRoundTripTime());
   client_connection->close(Network::ConnectionCloseType::NoFlush);
 }
 

test/common/quic/envoy_quic_client_session_test.cc

@@ -348,5 +348,16 @@ TEST_P(EnvoyQuicClientSessionTest, IncomingUnidirectionalReadStream) {
   envoy_quic_session_.OnStreamFrame(stream_frame2);
 }
 
+TEST_P(EnvoyQuicClientSessionTest, GetRttAndCwnd) {
+  EXPECT_GT(envoy_quic_session_.lastRoundTripTime().value(), std::chrono::microseconds(0));
+  // Just make sure the CWND is non-zero. We don't want to make strong assertions on what the value
+  // should be in this test, that is the job the congestion controllers' tests.
+  EXPECT_GT(envoy_quic_session_.congestionWindowInBytes().value(), 500);
+
+  envoy_quic_session_.configureInitialCongestionWindow(8000000, std::chrono::microseconds(1000000));
+  EXPECT_GT(envoy_quic_session_.congestionWindowInBytes().value(),
+            quic::kInitialCongestionWindow * quic::kDefaultTCPMSS);
+}
+
 } // namespace Quic
 } // namespace Envoy

test/common/quic/envoy_quic_server_session_test.cc

@@ -1033,5 +1033,17 @@ TEST_F(EnvoyQuicServerSessionTest, IncomingUnidirectionalReadStream) {
   envoy_quic_session_.OnStreamFrame(stream_frame);
 }
 
+TEST_F(EnvoyQuicServerSessionTest, GetRttAndCwnd) {
+  installReadFilter();
+  EXPECT_GT(envoy_quic_session_.lastRoundTripTime().value(), std::chrono::microseconds(0));
+  // Just make sure the CWND is non-zero. We don't want to make strong assertions on what the value
+  // should be in this test, that is the job the congestion controllers' tests.
+  EXPECT_GT(envoy_quic_session_.congestionWindowInBytes().value(), 500);
+
+  envoy_quic_session_.configureInitialCongestionWindow(8000000, std::chrono::microseconds(1000000));
+  EXPECT_GT(envoy_quic_session_.congestionWindowInBytes().value(),
+            quic::kInitialCongestionWindow * quic::kDefaultTCPMSS);
+}
+
 } // namespace Quic
 } // namespace Envoy

test/mocks/network/connection.h

@@ -88,6 +88,7 @@ class MockConnectionBase {
   MOCK_METHOD(absl::optional<std::chrono::milliseconds>, lastRoundTripTime, (), (const));          \
   MOCK_METHOD(void, configureInitialCongestionWindow,                                              \
               (uint64_t bandwidth_bits_per_sec, std::chrono::microseconds rtt), ());               \
+  MOCK_METHOD(absl::optional<uint64_t>, congestionWindowInBytes, (), (const));                     \
   MOCK_METHOD(void, dumpState, (std::ostream&, int), (const));
 
 class MockConnection : public Connection, public MockConnectionBase {

test/mocks/network/io_handle.h

@@ -61,6 +61,7 @@ class MockIoHandle : public IoHandle {
   MOCK_METHOD(void, resetFileEvents, ());
   MOCK_METHOD(Api::SysCallIntResult, shutdown, (int how));
   MOCK_METHOD(absl::optional<std::chrono::milliseconds>, lastRoundTripTime, ());
+  MOCK_METHOD(absl::optional<uint64_t>, congestionWindowInBytes, (), (const));
   MOCK_METHOD(Api::SysCallIntResult, ioctl,
               (unsigned long, void*, unsigned long, void*, unsigned long, unsigned long*));
   MOCK_METHOD(absl::optional<std::string>, interfaceName, ());

test/mocks/network/mocks.h

@@ -355,6 +355,7 @@ class MockConnectionSocket : public ConnectionSocket {
               (unsigned long, void*, unsigned long, void*, unsigned long, unsigned long*));
   MOCK_METHOD(Api::SysCallIntResult, setBlockingForTest, (bool));
   MOCK_METHOD(absl::optional<std::chrono::milliseconds>, lastRoundTripTime, ());
+  MOCK_METHOD(absl::optional<uint64_t>, congestionWindowInBytes, (), (const));
   MOCK_METHOD(void, dumpState, (std::ostream&, int), (const));
 
   IoHandlePtr io_handle_;

test/server/filter_chain_benchmark_test.cc

@@ -128,6 +128,7 @@ class MockConnectionSocket : public Network::ConnectionSocket {
   }
   Api::SysCallIntResult setBlockingForTest(bool) override { return {0, 0}; }
   absl::optional<std::chrono::milliseconds> lastRoundTripTime() override { return {}; }
+  absl::optional<uint64_t> congestionWindowInBytes() const override { return {}; }
   void dumpState(std::ostream&, int) const override {}
 
 private:

tools/spelling/spelling_dictionary.txt

@@ -320,6 +320,7 @@ SIGINT
 SIGPIPE
 SIGSEGV
 SIGTERM
+SIO
 SMTP
 SNI
 SOTW