feat(request-trace): native S3 sink for request-trace records (cherry-pick #11806)

container/README.md

@@ -484,6 +484,8 @@ container/run.sh --image dynamo:latest-sglang-xpu-local-dev --device=xpu \
 sudo chown -R dynamo:0 /opt/miniforge3/envs/sglang
 cargo build --locked --features dynamo-llm/block-manager --workspace
 # 3a. ai_dynamo_runtime (Rust bindings: dynamo._core)
+# Add `--features request-trace-s3` to enable the S3 request-trace sink
+# (DYN_REQUEST_TRACE_SINKS=s3); it is off by default to keep the local build lean.
 cd lib/bindings/python && maturin develop --uv && cd -
 # 3b. ai-dynamo (Python namespace packages: dynamo.frontend, dynamo.sglang, ...)
 uv pip install --no-deps -e /workspace
@@ -544,6 +546,8 @@ etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0
 
 # 4. Compile code
 cargo build --locked --features dynamo-llm/block-manager --workspace
+# Add `--features request-trace-s3` to enable the S3 request-trace sink
+# (DYN_REQUEST_TRACE_SINKS=s3); it is off by default to keep the local build lean.
 cd lib/bindings/python && maturin develop --uv && cd -
 
 # 5. Sanity check (optional but recommended)

container/templates/wheel_builder.Dockerfile

@@ -524,9 +524,9 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token
     uv build --wheel --out-dir /opt/dynamo/dist && \
     cd /opt/dynamo/lib/bindings/python && \
     if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \
-        maturin build --release --features "media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass{% if target == "planner" %},mocker-kvbm-offload{% endif %}" --out /opt/dynamo/dist; \
+        maturin build --release --features "media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass,request-trace-s3{% if target == "planner" %},mocker-kvbm-offload{% endif %}" --out /opt/dynamo/dist; \
     else \
-        maturin build --release --features "kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass{% if target == "planner" %},mocker-kvbm-offload{% endif %}" --out /opt/dynamo/dist; \
+        maturin build --release --features "kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass,request-trace-s3{% if target == "planner" %},mocker-kvbm-offload{% endif %}" --out /opt/dynamo/dist; \
     fi && \
     /tmp/use-sccache.sh show-stats "Dynamo Runtime"
 

docs/fern/reference/observability/environment-variables.mdx

@@ -275,7 +275,7 @@ See [Forward Pass Metrics Trace Reference](forward-pass-metrics-tracing.mdx) for
 </ParamField>
 
 <ParamField path="DYN_REQUEST_TRACE_SINKS" type="string" default="file">
-  Comma-separated sinks: `file`, `stderr`, `nats`, and `otel`.
+  Comma-separated sinks: `file`, `stderr`, `nats`, `otel`, and `s3`.
 </ParamField>
 
 <ParamField path="DYN_REQUEST_TRACE_FILE_PATH" type="string" default="/tmp/dynamo-request-trace">
@@ -314,6 +314,26 @@ See [Forward Pass Metrics Trace Reference](forward-pass-metrics-tracing.mdx) for
   Optional gzip roll threshold in records.
 </ParamField>
 
+<ParamField path="DYN_REQUEST_TRACE_S3_BUCKET" type="string">
+  Destination bucket for the `s3` sink. Required when `DYN_REQUEST_TRACE_SINKS` includes `s3`; startup fails if unset.
+</ParamField>
+
+<ParamField path="DYN_REQUEST_TRACE_S3_REGION" type="string">
+  Region override for the `s3` sink. When unset the AWS SDK resolves the region from the environment, profile, or IMDS.
+</ParamField>
+
+<ParamField path="DYN_REQUEST_TRACE_S3_PREFIX" type="string">
+  Object key prefix for the `s3` sink. When unset, records land at the bucket root. Keys are `{prefix}/{yyyy}/{mm}/{dd}/{host}-{HHMMSS}-{run_id}-{seq}.jsonl.gz`.
+</ParamField>
+
+<ParamField path="DYN_REQUEST_TRACE_S3_ROLL_UNCOMPRESSED_BYTES" type="integer" default="67108864">
+  Batch roll threshold for the `s3` sink in uncompressed bytes. When the pending batch reaches this size it is gzipped and uploaded as one object.
+</ParamField>
+
+<ParamField path="DYN_REQUEST_TRACE_S3_FLUSH_INTERVAL_MS" type="integer" default="10000">
+  Periodic flush interval for the `s3` sink in milliseconds. A partial batch is uploaded when this elapses so low-volume traces still reach S3.
+</ParamField>
+
 <ParamField path="DYN_REQUEST_TRACE_TOOL_EVENTS_ZMQ_ENDPOINT" type="string">
   Optional ZMQ PULL bind address for harness tool events. Configure it on only one process.
 </ParamField>

docs/fern/reference/observability/request-tracing.mdx

@@ -46,6 +46,30 @@ transport settings in this order:
 Setting `DYN_REQUEST_TRACE_SINKS=stderr` does not enable OTLP export. Include `otel`, for example
 `file,otel` or `stderr,otel`.
 
+The `s3` sink writes records directly to an S3 bucket as gzipped JSONL objects, one object per rolled
+batch. Records are batched in-process and uploaded when the batch reaches
+`DYN_REQUEST_TRACE_S3_ROLL_UNCOMPRESSED_BYTES` or when `DYN_REQUEST_TRACE_S3_FLUSH_INTERVAL_MS`
+elapses. Object keys are `{prefix}/{yyyy}/{mm}/{dd}/{host}-{HHMMSS}-{run_id}-{seq}.jsonl.gz`, where
+`run_id` is a per-process UUID that keeps container restarts and hostname collisions from overwriting
+earlier batches. Credentials come from the AWS SDK default provider chain (environment variables,
+IMDS, IRSA, Pod Identity, shared profiles); how the frontend pod is credentialed is a deployment
+concern. On terminal upload failure the batch is dropped and a warning is logged; no on-disk retry
+queue is kept.
+
+```bash
+export DYN_REQUEST_TRACE=1
+export DYN_REQUEST_TRACE_SINKS=s3
+export DYN_REQUEST_TRACE_S3_BUCKET=my-org-request-traces
+export DYN_REQUEST_TRACE_S3_REGION=us-west-2
+export DYN_REQUEST_TRACE_S3_PREFIX=frontend-a/prod
+```
+
+<Note>
+The `s3` sink is compiled in only when `dynamo-llm` is built with the `request-trace-s3` cargo
+feature. Shipped Dynamo Python wheels enable it; local source builds pass
+`--features request-trace-s3` to `maturin develop`.
+</Note>
+
 ## Record Types
 
 ### `request_end`

lib/bindings/python/Cargo.toml

@@ -42,6 +42,11 @@ nvtx = ["dynamo-runtime/nvtx"]
 # recipes pass this flag explicitly; the dev container's
 # `.devcontainer/post-create.sh` also enables it for local workflows.
 mm-routing = ["dynamo-llm/mm-routing"]
+# Enable the native S3 destination for the request-trace sink. Pulls in
+# aws-sdk-s3 + aws-config so the default wheel stays lean; production wheel
+# builds enable this flag in container/templates/wheel_builder.Dockerfile so
+# `DYN_REQUEST_TRACE_SINKS=s3` works on shipped artifacts.
+request-trace-s3 = ["dynamo-llm/request-trace-s3"]
 
 [dependencies]
 # AIC perf model: pure-Rust hot-path latency engine. Pinned to the matching

lib/llm/Cargo.toml

@@ -35,6 +35,8 @@ ckf-diagnostics = []
 kv-router-stress = ["dep:clap", "dep:indicatif", "bench"]
 mm-routing = ["dep:llm-multimodal", "dep:llm-tokenizer"]
 request-trace-bench = []
+# S3 destination for the request-trace sink. Pulls in aws-sdk-s3 + aws-config.
+request-trace-s3 = ["dep:aws-sdk-s3", "dep:aws-config"]
 
 [[bench]]
 name = "tokenizer_simple"
@@ -142,6 +144,10 @@ nix = { version = "0.26", optional = true }
 # media (zlib compression for NIXL metadata)
 flate2 = { version = "1" }
 
+# request-trace-s3 (optional S3 sink)
+aws-sdk-s3 = { version = "1.120.0", optional = true }
+aws-config = { version = "1.8.11", optional = true }
+
 # block_manager_bench
 clap = { version = "4.5.49", features = ["derive"], optional = true }
 indicatif = { version = "0.18.0", optional = true }

lib/llm/src/request_trace/config.rs

@@ -21,6 +21,8 @@ const DEFAULT_FILE_PATH: &str = "/tmp/dynamo-request-trace";
 const DEFAULT_NATS_SUBJECT: &str = "dynamo.request_trace.v1";
 const DEFAULT_LEGACY_AUDIT_NATS_SUBJECT: &str = "dynamo.audit.v1";
 const DEFAULT_OTEL_MAX_PAYLOAD_BYTES: usize = 4 * 1024 * 1024;
+const DEFAULT_S3_ROLL_UNCOMPRESSED_BYTES: u64 = 64 * 1024 * 1024;
+const DEFAULT_S3_FLUSH_INTERVAL_MS: u64 = 10_000;
 
 const CAPTURE_UNINITIALIZED: u8 = 0;
 const CAPTURE_ACTIVE: u8 = 1;
@@ -32,6 +34,7 @@ pub enum RequestTraceSinkKind {
     Stderr,
     Nats,
     Otel,
+    S3,
 }
 
 impl RequestTraceSinkKind {
@@ -41,6 +44,7 @@ impl RequestTraceSinkKind {
             Self::Stderr => "stderr",
             Self::Nats => "nats",
             Self::Otel => "otel",
+            Self::S3 => "s3",
         }
     }
 }
@@ -94,6 +98,11 @@ pub struct RequestTracePolicy {
     pub http_header_capture_list: Vec<String>,
     pub tool_events_zmq_endpoint: Option<String>,
     pub tool_events_zmq_topic: Option<String>,
+    pub s3_bucket: Option<String>,
+    pub s3_region: Option<String>,
+    pub s3_prefix: Option<String>,
+    pub s3_roll_uncompressed_bytes: u64,
+    pub s3_flush_interval_ms: u64,
 }
 
 impl RequestTracePolicy {
@@ -214,6 +223,17 @@ fn load_from_env() -> RequestTracePolicy {
             .filter(|value| !value.is_empty())
             .unwrap_or_else(|| DEFAULT_TOOL_EVENTS_TOPIC.to_string())
     });
+    let s3_bucket = env_trimmed(env_request_trace::DYN_REQUEST_TRACE_S3_BUCKET);
+    let s3_region = env_trimmed(env_request_trace::DYN_REQUEST_TRACE_S3_REGION);
+    let s3_prefix = env_trimmed(env_request_trace::DYN_REQUEST_TRACE_S3_PREFIX);
+    let s3_roll_uncompressed_bytes =
+        env_u64(&[env_request_trace::DYN_REQUEST_TRACE_S3_ROLL_UNCOMPRESSED_BYTES])
+            .filter(|value| *value > 0)
+            .unwrap_or(DEFAULT_S3_ROLL_UNCOMPRESSED_BYTES);
+    let s3_flush_interval_ms =
+        env_u64(&[env_request_trace::DYN_REQUEST_TRACE_S3_FLUSH_INTERVAL_MS])
+            .filter(|value| *value > 0)
+            .unwrap_or(DEFAULT_S3_FLUSH_INTERVAL_MS);
 
     RequestTracePolicy {
         enabled,
@@ -231,6 +251,11 @@ fn load_from_env() -> RequestTracePolicy {
         http_header_capture_list,
         tool_events_zmq_endpoint,
         tool_events_zmq_topic,
+        s3_bucket,
+        s3_region,
+        s3_prefix,
+        s3_roll_uncompressed_bytes,
+        s3_flush_interval_ms,
     }
 }
 
@@ -314,6 +339,7 @@ fn parse_sink_kind_names(
             "stderr" => push_sink(&mut sinks, RequestTraceSinkKind::Stderr),
             "nats" => push_sink(&mut sinks, RequestTraceSinkKind::Nats),
             "otel" => push_sink(&mut sinks, RequestTraceSinkKind::Otel),
+            "s3" => push_sink(&mut sinks, RequestTraceSinkKind::S3),
             "jsonl" => {
                 legacy_jsonl = true;
                 push_sink(&mut sinks, RequestTraceSinkKind::File);

lib/llm/src/request_trace/mod.rs

@@ -9,6 +9,8 @@ pub mod payload;
 pub(crate) mod payload_stream;
 mod record;
 mod replay;
+#[cfg(feature = "request-trace-s3")]
+mod s3_sink;
 pub mod sink;
 mod tool_relay;
 pub mod types;