feat(runtime): add TLS support to TCP request plane

components/src/dynamo/common/configuration/groups/runtime_args.py

@@ -58,6 +58,12 @@ class DynamoRuntimeConfig(ConfigBase):
     # default; when set, these surface env vars that the Rust runtime reads
     # directly (see lib/runtime/src/pipeline/network/ingress/shared_tcp_endpoint.rs).
     engine_request_limit: Optional[int] = None
+    tcp_tls_cert_path: Optional[str] = None
+    tcp_tls_key_path: Optional[str] = None
+    tcp_tls_ca_cert_path: Optional[str] = None
+    tcp_tls_insecure: bool = False
+    tcp_tls_server_name: Optional[str] = None
+    tcp_tls_handshake_timeout_secs: Optional[int] = None
 
     def validate(self) -> None:
         self.namespace = get_worker_namespace(self.namespace)
@@ -92,6 +98,28 @@ def validate(self) -> None:
                 f"--engine-request-limit must be a positive integer, got {self.engine_request_limit}"
             )
 
+        # Propagate TCP TLS CLI flags to env vars so the Rust runtime picks them up.
+        if self.tcp_tls_cert_path:
+            os.environ["DYN_TCP_TLS_CERT_PATH"] = self.tcp_tls_cert_path
+        if self.tcp_tls_key_path:
+            os.environ["DYN_TCP_TLS_KEY_PATH"] = self.tcp_tls_key_path
+        if self.tcp_tls_ca_cert_path:
+            os.environ["DYN_TCP_TLS_CA_CERT_PATH"] = self.tcp_tls_ca_cert_path
+        if self.tcp_tls_insecure:
+            os.environ["DYN_TCP_TLS_INSECURE"] = "1"
+        else:
+            os.environ.pop("DYN_TCP_TLS_INSECURE", None)
+        if self.tcp_tls_server_name:
+            os.environ["DYN_TCP_TLS_SERVER_NAME"] = self.tcp_tls_server_name
+        if self.tcp_tls_handshake_timeout_secs is not None:
+            if self.tcp_tls_handshake_timeout_secs <= 0:
+                raise ValueError(
+                    f"--tcp-tls-handshake-timeout must be a positive integer, got {self.tcp_tls_handshake_timeout_secs}"
+                )
+            os.environ["DYN_TCP_TLS_HANDSHAKE_TIMEOUT_SECS"] = str(
+                self.tcp_tls_handshake_timeout_secs
+            )
+
     def _validate_output_modalities(self) -> None:
         """Validate --output-modalities values."""
         if not self.output_modalities:
@@ -347,3 +375,53 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
             "semaphore size). Enables worker-side request rejection when set. "
             "Disabled by default.",
         )
+
+        add_argument(
+            g,
+            flag_name="--tcp-tls-cert-path",
+            env_var="DYN_TCP_TLS_CERT_PATH",
+            default=None,
+            help="Path to PEM certificate for the TCP server.",
+        )
+
+        add_argument(
+            g,
+            flag_name="--tcp-tls-key-path",
+            env_var="DYN_TCP_TLS_KEY_PATH",
+            default=None,
+            help="Path to PEM private key for the TCP server certificate.",
+        )
+
+        add_argument(
+            g,
+            flag_name="--tcp-tls-ca-cert-path",
+            env_var="DYN_TCP_TLS_CA_CERT_PATH",
+            default=None,
+            help="Path to PEM CA certificate used by this node to verify the TCP peer's certificate.",
+        )
+
+        add_negatable_bool_argument(
+            g,
+            flag_name="--tcp-tls-insecure",
+            env_var="DYN_TCP_TLS_INSECURE",
+            default=False,
+            help="Disable TCP TLS certificate verification. For local development only.",
+        )
+
+        add_argument(
+            g,
+            flag_name="--tcp-tls-server-name",
+            env_var="DYN_TCP_TLS_SERVER_NAME",
+            default=None,
+            help="Override TLS SNI server name for TCP connections (useful when connecting by IP).",
+        )
+
+        add_argument(
+            g,
+            flag_name="--tcp-tls-handshake-timeout",
+            env_var="DYN_TCP_TLS_HANDSHAKE_TIMEOUT_SECS",
+            default=None,
+            arg_type=int,
+            dest="tcp_tls_handshake_timeout_secs",
+            help="TLS handshake timeout in seconds (default: 3).",
+        )

components/src/dynamo/frontend/frontend_args.py

@@ -59,6 +59,9 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AicPerfConfigBase):
     http_port: int
     tls_cert_path: Optional[pathlib.Path]
     tls_key_path: Optional[pathlib.Path]
+    tcp_tls_cert_path: Optional[str] = None
+    tcp_tls_key_path: Optional[str] = None
+    tcp_tls_ca_cert_path: Optional[str] = None
 
     namespace: Optional[str] = None
     namespace_prefix: Optional[str] = None
@@ -248,6 +251,30 @@ def add_arguments(self, parser) -> None:
             arg_type=pathlib.Path,
         )
 
+        add_argument(
+            g,
+            flag_name="--tcp-tls-cert-path",
+            env_var="DYN_TCP_TLS_CERT_PATH",
+            default=None,
+            help="Path to PEM certificate for the TCP server.",
+        )
+
+        add_argument(
+            g,
+            flag_name="--tcp-tls-key-path",
+            env_var="DYN_TCP_TLS_KEY_PATH",
+            default=None,
+            help="Path to PEM private key for the TCP server certificate.",
+        )
+
+        add_argument(
+            g,
+            flag_name="--tcp-tls-ca-cert-path",
+            env_var="DYN_TCP_TLS_CA_CERT_PATH",
+            default=None,
+            help="Path to PEM CA certificate used to verify the TCP peer's certificate.",
+        )
+
         # Router options (shared with dynamo.router)
         RouterArgGroup().add_arguments(parser)
 

components/src/dynamo/frontend/main.py

@@ -423,6 +423,12 @@ def signal_handler():
         kwargs["tls_cert_path"] = config.tls_cert_path
     if config.tls_key_path:
         kwargs["tls_key_path"] = config.tls_key_path
+    if config.tcp_tls_cert_path:
+        os.environ["DYN_TCP_TLS_CERT_PATH"] = config.tcp_tls_cert_path
+    if config.tcp_tls_key_path:
+        os.environ["DYN_TCP_TLS_KEY_PATH"] = config.tcp_tls_key_path
+    if config.tcp_tls_ca_cert_path:
+        os.environ["DYN_TCP_TLS_CA_CERT_PATH"] = config.tcp_tls_ca_cert_path
     if config.namespace:
         kwargs["namespace"] = config.namespace
     if config.namespace_prefix:

docs/fern/kubernetes/tls.md

@@ -0,0 +1,124 @@
+---
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+title: TCP TLS
+subtitle: Encrypt TCP streaming connections between frontend and workers
+---
+
+Dynamo supports opt-in TLS encryption on the TCP call-home streaming transport
+(implemented by `TcpStreamServer` and `TcpClient`, handling both response
+streams and request streams between frontends and workers). When enabled, all TCP
+connections on this path are upgraded to TLS using
+[rustls](https://github.com/rustls/rustls) with the `ring` cryptographic
+provider. When no TLS configuration is provided, the transport operates in
+plaintext exactly as before.
+
+> **Note:** The shared TCP request plane (`egress/tcp_client` and
+> `ingress/shared_tcp_endpoint`) is a separate transport path not yet covered
+> by this TLS implementation. This is planned for a future PR.
+
+## Environment variables
+
+All TLS configuration is driven by environment variables. The Rust runtime
+reads these directly at first connection (lazy initialization).
+
+Both frontends and workers act as TCP server and client depending on the
+stream direction (response streams: worker dials frontend; request streams:
+frontend dials worker). All TLS env vars should be set on every pod.
+
+### Server role (accepting connections)
+
+| Variable | Description |
+|---|---|
+| `DYN_TCP_TLS_CERT_PATH` | Path to the PEM certificate file. When set together with `DYN_TCP_TLS_KEY_PATH`, TLS is enabled on the TCP server. |
+| `DYN_TCP_TLS_KEY_PATH` | Path to the PEM private key for the server certificate. |
+
+### Client role (dialing connections)
+
+| Variable | Description |
+|---|---|
+| `DYN_TCP_TLS_CA_CERT_PATH` | Path to the PEM CA certificate used to verify the peer's server certificate. |
+| `DYN_TCP_TLS_INSECURE` | Set to `1` or `true` to skip certificate verification. For local development only. |
+| `DYN_TCP_TLS_SERVER_NAME` | Override the TLS SNI hostname. Useful when connecting by IP to a server whose certificate has a DNS SAN. |
+| `DYN_TCP_TLS_HANDSHAKE_TIMEOUT_SECS` | TLS handshake timeout in seconds (default: 3). |
+
+## CLI flags
+
+The same configuration is available via command-line flags on all backends
+(vllm, sglang, trtllm, tokenspeed) through `DynamoRuntimeArgGroup`:
+
+```
+--tcp-tls-cert-path PATH      Server certificate (PEM)
+--tcp-tls-key-path PATH       Server private key (PEM)
+--tcp-tls-ca-cert-path PATH   CA certificate for server verification (PEM)
+--tcp-tls-insecure             Disable certificate verification
+--tcp-tls-server-name NAME     Override TLS SNI hostname
+--tcp-tls-handshake-timeout N  Handshake timeout in seconds (default: 3)
+```
+
+The frontend (`dynamo.frontend`) also accepts `--tcp-tls-cert-path`,
+`--tcp-tls-key-path`, and `--tcp-tls-ca-cert-path`.
+
+## Quick start
+
+Generate a self-signed certificate for local testing:
+
+```bash
+# Generate CA
+openssl req -x509 -newkey rsa:2048 -keyout ca-key.pem -out ca-cert.pem \
+  -days 365 -nodes -subj "/CN=DynamoCA"
+
+# Generate server cert with SAN
+openssl req -newkey rsa:2048 -keyout server-key.pem -out server-csr.pem \
+  -nodes -subj "/CN=localhost" \
+  -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
+
+openssl x509 -req -in server-csr.pem -CA ca-cert.pem -CAkey ca-key.pem \
+  -CAcreateserial -out server-cert.pem -days 365 -copy_extensions copyall
+```
+
+Both frontend and worker need the same flags (both act as server and client):
+
+```bash
+python -m dynamo.vllm \
+  --tcp-tls-cert-path server-cert.pem \
+  --tcp-tls-key-path server-key.pem \
+  --tcp-tls-ca-cert-path ca-cert.pem \
+  --tcp-tls-server-name localhost \
+  ...
+
+python -m dynamo.frontend \
+  --tcp-tls-cert-path server-cert.pem \
+  --tcp-tls-key-path server-key.pem \
+  --tcp-tls-ca-cert-path ca-cert.pem \
+  --tcp-tls-server-name localhost \
+  ...
+```
+
+## Kubernetes deployment
+
+In Kubernetes, TLS certificates are typically delivered by a certificate
+management system (e.g., cert-manager) and mounted into pods. Set the
+environment variables in the pod spec or via the Dynamo operator's
+`InfrastructureConfiguration`:
+
+```yaml
+env:
+  - name: DYN_TCP_TLS_CERT_PATH
+    value: /etc/certs/server/cert.pem
+  - name: DYN_TCP_TLS_KEY_PATH
+    value: /etc/certs/server/key.pem
+  - name: DYN_TCP_TLS_CA_CERT_PATH
+    value: /etc/certs/ca/ca.pem
+```
+
+## Design notes
+
+- TLS configuration is cached after the first TCP connection via `OnceCell`.
+  Certificate rotation requires a process restart.
+- The TLS handshake is spawned per-connection on the server side so the accept
+  loop is never blocked by a slow handshake.
+- When server and client TLS configurations are mismatched (e.g., server has TLS
+  but client does not), a warning is logged at startup.
+- An empty CA certificate file is detected at load time and rejected with a
+  clear error message.

lib/runtime/Cargo.toml

@@ -55,8 +55,11 @@ rmp-serde = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
+tokio-rustls = { workspace = true }
 tokio-stream = { workspace = true }
 tokio-util = { workspace = true }
+rustls = { workspace = true }
+rustls-pemfile = { workspace = true }
 tower-http = { workspace = true }
 tracing = { workspace = true }
 mio = { workspace = true }
@@ -111,6 +114,7 @@ k8s-openapi = { version = "0.26.0", features = ["v1_32"] }
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["async_tokio"] }
+rcgen = { workspace = true }
 rstest = { version = "0.23.0" }
 temp-env = { version = "0.3.6" , features=["async_closure"] }
 stdio-override = {version= "0.2.0"}