From 097e2116421c9d4606cfa5a34f5bb43a678608e2 Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:27:22 +0000 Subject: [PATCH 01/11] feat(sglang): upload sidecar metadata with OpenDAL Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- Cargo.lock | 521 ++++++++++++++++- Cargo.toml | 8 + .../src/dynamo/common/metadata_upload.py | 49 +- .../common/tests/test_metadata_upload.py | 167 ++++++ .../tests/test_sglang_processor_unit.py | 2 + .../tests/test_sglang_decode_handler.py | 82 --- .../nvidia-request-extensions-nvext.md | 10 +- .../backends/sglang/sidecar.md | 37 ++ .../integration-reference.md | 2 +- lib/bindings/python/Cargo.lock | 535 +++++++++++++++++- lib/llm/src/preprocessor.rs | 6 +- lib/llm/src/protocols/common/extensions.rs | 47 +- lib/sidecar/sglang/Cargo.toml | 5 + lib/sidecar/sglang/README.md | 51 ++ lib/sidecar/sglang/src/engine.rs | 50 +- lib/sidecar/sglang/src/lib.rs | 5 +- lib/sidecar/sglang/src/metadata_upload.rs | 402 +++++++++++++ lib/sidecar/sglang/src/native_http.rs | 92 ++- 18 files changed, 1926 insertions(+), 145 deletions(-) create mode 100644 components/src/dynamo/common/tests/test_metadata_upload.py create mode 100644 lib/sidecar/sglang/src/metadata_upload.rs diff --git a/Cargo.lock b/Cargo.lock index 051e4dd378de..a325c8678054 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -342,6 +353,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "atomic" version = "0.6.1" @@ -689,9 +710,9 @@ dependencies = [ "http 1.4.1", "http-body 1.0.1", "http-body-util", - "md-5", + "md-5 0.10.6", "pin-project-lite", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "tracing", ] @@ -927,7 +948,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper 1.0.2", "tokio", "tokio-tungstenite", @@ -1044,6 +1065,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1242,6 +1269,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -1352,6 +1388,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cbindgen" version = "0.27.0" @@ -1466,6 +1511,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -1683,6 +1738,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2150,7 +2211,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] @@ -2160,7 +2221,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -2285,6 +2346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", + "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -2296,6 +2358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", ] @@ -3045,16 +3108,21 @@ dependencies = [ "dynamo-runtime", "dynamo-sidecar-common", "futures", + "opendal", "prost 0.13.5", "rand 0.9.4", "reqwest 0.12.28", + "rmp-serde", + "serde", "serde_json", + "tempfile", "tokio", "tokio-util", "tonic 0.13.1", "tonic-build 0.13.1", "tracing", "url", + "zstd", ] [[package]] @@ -4841,6 +4909,16 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "insta" version = "1.47.2" @@ -4967,10 +5045,12 @@ checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", "windows-link 0.2.1", ] @@ -5234,7 +5314,7 @@ dependencies = [ "jsonpath-rust", "k8s-openapi", "kube-core", - "pem", + "pem 3.0.6", "rustls", "secrecy", "serde", @@ -5472,6 +5552,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.8", +] [[package]] name = "leb128fmt" @@ -5659,9 +5742,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.30" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loop9" @@ -5786,12 +5869,32 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "md5" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" +[[package]] +name = "mea" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c709842c4ce65cb91e2666ad5319dfc1efc3af0d34f02075eddca9000d9f8afb" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "memchr" version = "2.8.1" @@ -6326,6 +6429,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.6", + "smallvec", + "zeroize", +] + [[package]] name = "num-cmp" version = "0.1.0" @@ -6610,10 +6729,10 @@ dependencies = [ "humantime", "hyper 1.10.1", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", - "quick-xml", + "quick-xml 0.38.4", "rand 0.9.4", "reqwest 0.12.28", "ring", @@ -6705,11 +6824,153 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "thiserror 2.0.18", ] +[[package]] +name = "opendal" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" +dependencies = [ + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-service-azblob", + "opendal-service-fs", + "opendal-service-gcs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a43405d217dfdfb543f58847336d3af672897dd1939bb7dcf314b63cf364f1c9" +dependencies = [ + "anyhow", + "asyncband", + "base64 0.23.1", + "bytes", + "futures", + "http 1.4.1", + "jiff", + "log", + "md-5 0.11.0", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06" +dependencies = [ + "bytes", + "futures", + "http 1.4.1", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + +[[package]] +name = "opendal-service-azblob" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba0d2662ddf0de1f838db5dc48fafb8212ed1a5c7980bc164aa67809696c309c" +dependencies = [ + "base64 0.23.1", + "bytes", + "http 1.4.1", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.11.0", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" +dependencies = [ + "http 1.4.1", + "opendal-core", +] + +[[package]] +name = "opendal-service-fs" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb9caf04d6d38713299dd4abac984b95ab16ee20e1ff09160f495c5a64644083" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.1", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "opendal-service-s3" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388b1d39b62535c62803754ebef89808859558697366dbedd0299345887ba461" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http 1.4.1", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -6954,6 +7215,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pcre2" version = "0.2.11" @@ -7009,6 +7280,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -7170,6 +7451,32 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der 0.7.10", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki 0.7.3", +] + [[package]] name = "pkcs8" version = "0.9.0" @@ -7187,6 +7494,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der 0.7.10", + "pkcs5", + "rand_core 0.6.4", "spki 0.7.3", ] @@ -7626,6 +7935,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.7.0" @@ -7664,6 +7983,7 @@ version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.2", "lru-slab", @@ -7934,7 +8254,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ - "pem", + "pem 3.0.6", "ring", "rustls-pki-types", "time", @@ -8022,6 +8342,118 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "reqsign-aws-core" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac4749b7dfa7bfaccd01eb03e9dc795ed37e3f20d6f0f38e2c67ee85ad6bc86" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http 1.4.1", + "log", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff250f0fd0b913fbd565e405acc553da0f13bde30bfb5403178c9d0313cdc15f" +dependencies = [ + "bytes", + "http 1.4.1", + "log", + "quick-xml 0.41.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-azure-storage" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f1fbc9add082f54e51e3bd9730f18d850a1f09d246baff5cd612c74ae4290e" +dependencies = [ + "anyhow", + "base64 0.23.1", + "bytes", + "form_urlencoded", + "http 1.4.1", + "log", + "pem 4.0.0", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff052daffb0599681c50f85c59e7236438976efe991ab864edd9f3b235501a0f" +dependencies = [ + "anyhow", + "base64 0.23.1", + "bytes", + "futures", + "hex", + "hmac 0.13.0", + "http 1.4.1", + "jiff", + "log", + "mea", + "percent-encoding", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3235df90a6bca681aa47dd86f2393d122a6d77042aa8a7c81e218cd45c5bfc0" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "272a5813571885c1455ffe106f0c768577e6ce69313446d4476e8d41dcf6ac5a" +dependencies = [ + "bytes", + "form_urlencoded", + "http 1.4.1", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "reqwest" version = "0.11.27" @@ -8126,6 +8558,7 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project-lite", + "quinn", "rustls", "rustls-pki-types", "rustls-platform-verifier", @@ -8210,6 +8643,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "zeroize", +] + [[package]] name = "rstest" version = "0.18.2" @@ -8595,6 +9049,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -8656,6 +9119,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sec1" version = "0.3.0" @@ -8952,6 +9426,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -10486,7 +10971,7 @@ dependencies = [ "httparse", "log", "rand 0.9.4", - "sha1", + "sha1 0.10.6", "thiserror 2.0.18", "utf-8", ] @@ -11628,6 +12113,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index ee9e06eb0945..a49a8df50a84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,13 @@ ipnet = { version = "2" } libc = { version = "0.2" } oneshot = { version = "0.1.13", features = ["std", "async"] } ordered-float = "4" +opendal = { version = "0.59.1", default-features = false, features = [ + "http-transport-reqwest", + "services-azblob", + "services-fs", + "services-gcs", + "services-s3", +] } parking_lot = "0.12.5" prometheus = { version = "0.14"} rand = { version = "0.9.2" } @@ -188,6 +195,7 @@ uuid = { version = "1.18.1", features = ["v4", "serde"] } url = { version = "2.5", features = ["serde"] } xxhash-rust = { version = "0.8", features = ["xxh3", "const_xxh3"] } zeroize = { version = "1" } +zstd = "0.13" # ext-proc / Envoy gRPC prost = { version = "0.13.5" } diff --git a/components/src/dynamo/common/metadata_upload.py b/components/src/dynamo/common/metadata_upload.py index a29f21a9cbbb..88a93c887457 100644 --- a/components/src/dynamo/common/metadata_upload.py +++ b/components/src/dynamo/common/metadata_upload.py @@ -4,11 +4,15 @@ from __future__ import annotations import asyncio +import logging from dataclasses import dataclass from functools import cache from typing import Any, cast +logger = logging.getLogger(__name__) + + def _backend_metadata_upload_settings(request: dict[str, Any]) -> dict[str, Any] | None: nvext = request.get("nvext") extra_args = request.get("extra_args") or {} @@ -138,29 +142,47 @@ def _normalize_for_upload(value: Any) -> Any: return value +def _normalize_metadata_upload_url(value: object, field: str) -> str: + if not isinstance(value, str): + raise ValueError(f"metadata_upload.{field} must be a string") + value = value.strip() + if not value: + raise ValueError(f"metadata_upload.{field} must not be empty") + return value + + @dataclass(frozen=True) class MetadataUploader: url: str + fallback_url: str | None = None def __post_init__(self) -> None: - if not isinstance(self.url, str): - raise ValueError("metadata_upload.url must be a string") - url = self.url.strip() - if not url: - raise ValueError("metadata_upload.url must not be empty") - object.__setattr__(self, "url", url) + object.__setattr__( + self, + "url", + _normalize_metadata_upload_url(self.url, "url"), + ) + if self.fallback_url is not None: + object.__setattr__( + self, + "fallback_url", + _normalize_metadata_upload_url(self.fallback_url, "fallback_url"), + ) @classmethod def from_settings(cls, settings: dict[str, Any] | None) -> MetadataUploader | None: if settings is None: return None - unexpected = settings.keys() - {"url"} + unexpected = settings.keys() - {"url", "fallback_url"} if unexpected: field = min(unexpected) raise ValueError(f"metadata_upload.{field} is not supported") if "url" not in settings: raise ValueError("metadata_upload.url is required") - return cls(url=settings["url"]) + return cls( + url=settings["url"], + fallback_url=settings.get("fallback_url"), + ) @classmethod def from_backend_request(cls, request: dict[str, Any]) -> MetadataUploader | None: @@ -173,6 +195,15 @@ async def upload_choice(self, choice_index: int, metadata: dict[str, Any]) -> No storage_path = f"choice_{choice_index}.msgpack.zst" data = await asyncio.to_thread(_serialize_metadata, metadata) try: - await _upload_bytes(self.url, storage_path, data) + try: + await _upload_bytes(self.url, storage_path, data) + except Exception: + if self.fallback_url is None: + raise + logger.warning( + "Primary metadata upload failed; attempting configured fallback", + exc_info=True, + ) + await _upload_bytes(self.fallback_url, storage_path, data) finally: del data diff --git a/components/src/dynamo/common/tests/test_metadata_upload.py b/components/src/dynamo/common/tests/test_metadata_upload.py new file mode 100644 index 000000000000..4452cac067fd --- /dev/null +++ b/components/src/dynamo/common/tests/test_metadata_upload.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for out-of-band metadata uploads.""" + +import msgspec +import pytest +import zstandard as zstd + +from dynamo.common.metadata_upload import MetadataUploader + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +def _read_zstd_payload(path): + raw = zstd.ZstdDecompressor().decompress(path.read_bytes()) + return msgspec.msgpack.decode(raw) + + +def test_metadata_uploader_parses_backend_request(): + uploader = MetadataUploader.from_backend_request( + { + "extra_args": { + "nvext": { + "metadata_upload": { + "url": "s3://bucket/root/rollouts", + "fallback_url": " file:///var/tmp/rollouts ", + } + } + } + } + ) + + assert uploader is not None + assert uploader.url == "s3://bucket/root/rollouts" + assert uploader.fallback_url == "file:///var/tmp/rollouts" + + uploader = MetadataUploader.from_backend_request( + {"nvext": {"metadata_upload": {"url": "s3://bucket/root/rollouts"}}} + ) + assert uploader is not None + assert uploader.fallback_url is None + + +@pytest.mark.parametrize( + ("settings", "message"), + [ + ({}, "metadata_upload.url is required"), + ({"url": ""}, "metadata_upload.url must not be empty"), + ({"url": 1}, "metadata_upload.url must be a string"), + ( + {"url": "s3://bucket/root/rollouts", "fallback_url": ""}, + "metadata_upload.fallback_url must not be empty", + ), + ( + {"url": "s3://bucket/root/rollouts", "fallback_url": 1}, + "metadata_upload.fallback_url must be a string", + ), + ("invalid", "metadata_upload must be an object"), + ( + {"url": "s3://bucket/root/rollouts", "format": "json"}, + "metadata_upload.format is not supported", + ), + ], +) +def test_metadata_uploader_rejects_invalid_settings(settings, message): + with pytest.raises(ValueError, match=message): + MetadataUploader.from_backend_request({"nvext": {"metadata_upload": settings}}) + + +@pytest.mark.asyncio +async def test_metadata_upload_normalizes_tensor_values(tmp_path): + torch = pytest.importorskip("torch") + uploader = MetadataUploader(url=(tmp_path / "metadata/tensors").as_uri()) + tensor = torch.tensor([[1, 2], [3, 4]], dtype=torch.int32) + + await uploader.upload_choice(0, {"router_tensor": tensor}) + + payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_0.msgpack.zst") + uploaded_tensor = payload["metadata"]["router_tensor"] + assert uploaded_tensor["type"] == "tensor" + assert uploaded_tensor["dtype"] == "int32" + assert uploaded_tensor["shape"] == [2, 2] + assert uploaded_tensor["data"] == tensor.numpy().tobytes() + + bfloat_tensor = torch.tensor([1, 2], dtype=torch.bfloat16) + await uploader.upload_choice(1, {"router_tensor": bfloat_tensor}) + + payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_1.msgpack.zst") + uploaded_tensor = payload["metadata"]["router_tensor"] + assert uploaded_tensor["dtype"] == "bfloat16" + assert uploaded_tensor["shape"] == [2] + assert uploaded_tensor["data"] == bfloat_tensor.view(torch.uint8).numpy().tobytes() + + +@pytest.mark.asyncio +async def test_metadata_upload_normalizes_numpy_values(tmp_path): + np = pytest.importorskip("numpy") + uploader = MetadataUploader(url=(tmp_path / "metadata/arrays").as_uri()) + array = np.array([[1.5, 2.5], [3.5, 4.5]], dtype=np.float32) + + await uploader.upload_choice(0, {"scores": array[:, ::-1]}) + + payload = _read_zstd_payload(tmp_path / "metadata/arrays/choice_0.msgpack.zst") + uploaded_array = payload["metadata"]["scores"] + expected = np.ascontiguousarray(array[:, ::-1]) + assert uploaded_array["type"] == "ndarray" + assert uploaded_array["dtype"] == "float32" + assert uploaded_array["shape"] == [2, 2] + assert uploaded_array["data"] == expected.tobytes() + + +@pytest.mark.asyncio +async def test_metadata_upload_does_not_write_fallback_when_primary_succeeds(tmp_path): + primary_path = tmp_path / "metadata/primary" + fallback_path = tmp_path / "metadata/fallback" + uploader = MetadataUploader( + url=primary_path.as_uri(), + fallback_url=fallback_path.as_uri(), + ) + + await uploader.upload_choice(0, {"id": "request-1"}) + + assert (primary_path / "choice_0.msgpack.zst").is_file() + assert not fallback_path.exists() + + +@pytest.mark.asyncio +async def test_metadata_upload_uses_fallback_after_primary_failure(tmp_path, caplog): + primary_path = tmp_path / "primary-is-a-file" + primary_path.write_bytes(b"primary sentinel") + fallback_path = tmp_path / "metadata/fallback" + uploader = MetadataUploader( + url=primary_path.as_uri(), + fallback_url=fallback_path.as_uri(), + ) + + with caplog.at_level("WARNING"): + await uploader.upload_choice(2, {"id": "request-2"}) + + assert primary_path.read_bytes() == b"primary sentinel" + assert "Primary metadata upload failed" in caplog.text + payload = _read_zstd_payload(fallback_path / "choice_2.msgpack.zst") + assert payload["metadata"] == {"id": "request-2"} + + +@pytest.mark.asyncio +async def test_metadata_upload_propagates_fallback_failure(tmp_path): + primary_path = tmp_path / "primary-is-a-file" + fallback_path = tmp_path / "fallback-is-a-file" + primary_path.write_bytes(b"primary sentinel") + fallback_path.write_bytes(b"fallback sentinel") + uploader = MetadataUploader( + url=primary_path.as_uri(), + fallback_url=fallback_path.as_uri(), + ) + + with pytest.raises(OSError) as exc_info: + await uploader.upload_choice(0, {"id": "request-3"}) + + assert str(fallback_path) in str(exc_info.value) + assert primary_path.read_bytes() == b"primary sentinel" + assert fallback_path.read_bytes() == b"fallback sentinel" diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index 3fe41e89a161..5ccc5ec10e6f 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -530,6 +530,7 @@ def test_metadata_upload_nvext_is_forwarded_to_backend(self): "nvext": { "metadata_upload": { "url": "s3://bucket/root/rollouts", + "fallback_url": "file:///var/tmp/rollouts", }, }, }, @@ -540,6 +541,7 @@ def test_metadata_upload_nvext_is_forwarded_to_backend(self): assert result["extra_args"]["nvext"]["metadata_upload"] == { "url": "s3://bucket/root/rollouts", + "fallback_url": "file:///var/tmp/rollouts", } def test_model_name_and_token_ids(self): diff --git a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py index cd741b741572..9da4d43218cf 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py +++ b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py @@ -1061,46 +1061,6 @@ def test_extract_logprobs_formats_top_tokens_as_token_ids(): ] -def test_metadata_uploader_parses_extra_args_nvext(): - uploader = MetadataUploader.from_backend_request( - { - "extra_args": { - "nvext": { - "metadata_upload": { - "url": "s3://bucket/root/rollouts", - } - } - } - } - ) - - assert uploader is not None - assert uploader.url == "s3://bucket/root/rollouts" - - uploader = MetadataUploader.from_backend_request( - {"nvext": {"metadata_upload": {"url": "s3://bucket/root/rollouts"}}} - ) - assert uploader is not None - - -@pytest.mark.parametrize( - ("settings", "message"), - [ - ({}, "metadata_upload.url is required"), - ({"url": ""}, "metadata_upload.url must not be empty"), - ({"url": 1}, "metadata_upload.url must be a string"), - ("invalid", "metadata_upload must be an object"), - ( - {"url": "s3://bucket/root/rollouts", "format": "json"}, - "metadata_upload.format is not supported", - ), - ], -) -def test_metadata_uploader_rejects_invalid_settings(settings, message): - with pytest.raises(ValueError, match=message): - MetadataUploader.from_backend_request({"nvext": {"metadata_upload": settings}}) - - def test_nvext_extra_field_requested_supports_raw_and_preprocessed_shapes(): assert _nvext_extra_field_requested( {"nvext": {"extra_fields": ["stop_reason"]}}, "stop_reason" @@ -1131,48 +1091,6 @@ def test_metadata_upload_requires_enable_rl(): assert uploader.url == "s3://bucket/root/rollouts/run-1" -@pytest.mark.asyncio -async def test_metadata_upload_normalizes_tensor_values(tmp_path): - torch = pytest.importorskip("torch") - uploader = MetadataUploader(url=(tmp_path / "metadata/tensors").as_uri()) - tensor = torch.tensor([[1, 2], [3, 4]], dtype=torch.int32) - - await uploader.upload_choice(0, {"router_tensor": tensor}) - - payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_0.msgpack.zst") - uploaded_tensor = payload["metadata"]["router_tensor"] - assert uploaded_tensor["type"] == "tensor" - assert uploaded_tensor["dtype"] == "int32" - assert uploaded_tensor["shape"] == [2, 2] - assert uploaded_tensor["data"] == tensor.numpy().tobytes() - - bfloat_tensor = torch.tensor([1, 2], dtype=torch.bfloat16) - await uploader.upload_choice(1, {"router_tensor": bfloat_tensor}) - - payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_1.msgpack.zst") - uploaded_tensor = payload["metadata"]["router_tensor"] - assert uploaded_tensor["dtype"] == "bfloat16" - assert uploaded_tensor["shape"] == [2] - assert uploaded_tensor["data"] == bfloat_tensor.view(torch.uint8).numpy().tobytes() - - -@pytest.mark.asyncio -async def test_metadata_upload_normalizes_numpy_values(tmp_path): - np = pytest.importorskip("numpy") - uploader = MetadataUploader(url=(tmp_path / "metadata/arrays").as_uri()) - array = np.array([[1.5, 2.5], [3.5, 4.5]], dtype=np.float32) - - await uploader.upload_choice(0, {"scores": array[:, ::-1]}) - - payload = _read_zstd_payload(tmp_path / "metadata/arrays/choice_0.msgpack.zst") - uploaded_array = payload["metadata"]["scores"] - expected = np.ascontiguousarray(array[:, ::-1]) - assert uploaded_array["type"] == "ndarray" - assert uploaded_array["dtype"] == "float32" - assert uploaded_array["shape"] == [2, 2] - assert uploaded_array["data"] == expected.tobytes() - - @pytest.mark.asyncio async def test_process_token_stream_treats_completion_usage_as_optional(): handler = _new_decode_handler() diff --git a/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md b/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md index 28ef0272733b..db14d53c222f 100644 --- a/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md +++ b/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md @@ -38,7 +38,7 @@ Include `nvext` as a top-level field alongside standard OpenAI-compatible fields | `max_thinking_tokens` | `u32` | `None` | Backend | Maximum thinking tokens allowed (passed through to backends). | | `cache_salt` | `string` | `None` | Router / supported backends | Namespaces Dynamo KV routing. vLLM and TensorRT-LLM also isolate backend KV-cache reuse; see [Backend support](#backend-support). This is the recommended cache-isolation input. | | `extra_fields` | `string[]` | `None` | Response builder | Fields to include in the response `nvext`. Supported: `"worker_id"`, `"timing"`, `"routed_experts"`, `"engine_data"`, `"stop_reason"`, `"detailed_finish_reason"`, `"prompt_token_ids"`, `"completion_token_ids"`, `"prompt_logprobs"`. | -| `metadata_upload` | object | `None` | SGLang backend | Uploads final cumulative SGLang `meta_info` out of band. The object accepts one required `url` field. Requires an RL-enabled SGLang worker. | +| `metadata_upload` | object | `None` | SGLang backend | Uploads final cumulative SGLang `meta_info` out of band. Accepts required `url` and optional `fallback_url` strings. Requires an RL-enabled SGLang worker. | | `prefill_worker_id` | `u64` | `None` | Router | Routes the request to a specific prefill worker (disaggregated serving). | | `decode_worker_id` | `u64` | `None` | Router | Routes the request to a specific decode worker (disaggregated serving). | | `dp_rank` | `u32` | `None` | Router/backend | Data-parallel rank for the decode worker. Typically set by EPP routing headers. | @@ -56,6 +56,14 @@ token IDs, pass integer IDs in the normal `stop` array, for example `"stop": [576]`. Strings such as `"token_id:576"` remain literal string stop sequences and are not parsed as token IDs. +For `metadata_upload`, the Python SGLang worker uses the installed fsspec +backends. The SGLang sidecar uses OpenDAL and includes `file://`, `s3://`, +`gs://`, and `az://` support. Both implementations try `fallback_url` only +after the primary upload fails. The sidecar supports this field on +`/v1/chat/completions`, `/v1/completions`, and `/v1/responses`, but not on the +native SGLang `/generate` frontend API. In disaggregated serving, only the +decode worker uploads metadata. + ### Header Overrides Routing fields can also be set via HTTP headers, which take priority over `nvext` values: diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md index 4b6e28460b71..3e88d49cc861 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md @@ -28,6 +28,43 @@ in-process backend; sidecar feature parity is still under evaluation. See the [SGLang sidecar README](https://github.com/ai-dynamo/dynamo/blob/main/lib/sidecar/sglang/README.md) for current protocol details. +## Upload Response Metadata + +Start the sidecar with `--enable-rl` or `DYN_ENABLE_RL=true` to allow +out-of-band metadata uploads. Then set `nvext.metadata_upload` on each +OpenAI-compatible generation request that requires an artifact: + +```json +{ + "nvext": { + "metadata_upload": { + "url": "s3://bucket/root/rollout-1", + "fallback_url": "file:///var/tmp/rollout-1" + } + } +} +``` + +The sidecar writes the final cumulative SGLang `meta_info` to +`choice_0.msgpack.zst`. It tries `fallback_url` only when the primary upload +fails. The packaged sidecar supports `file://`, `s3://`, `gs://`, and `az://` +URLs through OpenDAL. + +| API or worker role | Support | +|---|---| +| `POST /v1/chat/completions` | Supported | +| `POST /v1/completions` | Supported | +| `POST /v1/responses` | Supported | +| Native SGLang `POST` or `PUT /generate` | Not supported | +| Aggregated worker | Uploads final metadata | +| Disaggregated prefill worker | Does not upload | +| Disaggregated decode worker | Uploads final metadata | + +The native `/generate` adapter does not currently project +`nvext.metadata_upload` into Dynamo request metadata. See +[NVIDIA Request Extensions](../../../../additional-resources/nvidia-request-extensions-nvext.md) +for the field contract. + ## Launch Locally From a Dynamo source checkout, build or install Dynamo so diff --git a/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md b/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md index 2b7f6839752e..3c00d9a41cc0 100644 --- a/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md +++ b/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md @@ -72,7 +72,7 @@ Before admitting a sample to training, verify: | Generated token IDs | Named `nvext.completion_token_ids`, or the native SGLang stream | Use the engine-returned sequence; exact placement depends on the selected interface. | | Selected and prompt log probabilities | Standard completion log probabilities, named `nvext.prompt_logprobs`, or native SGLang metadata | Check support and alignment on the exact backend and response path. | | Routed experts and raw engine data | Opt-in `nvext.routed_experts` or `nvext.engine_data` | Backend-specific. Prefer named fields over the raw engine payload. | -| Large SGLang `meta_info` | `nvext.metadata_upload.url` on the OpenAI-compatible path | Uploaded out of band by an RL-enabled SGLang worker; the destination is trusted control input. | +| Large SGLang `meta_info` | `nvext.metadata_upload` on `/v1/chat/completions`, `/v1/completions`, or `/v1/responses` | Uploaded out of band by an RL-enabled SGLang worker. Set required `url` and optional `fallback_url`; the destination is trusted control input. The native `/generate` frontend API does not support this extension. | | Masks, rewards, advantages, tool or environment state, and trajectory objects | Not a generic Dynamo response contract | The framework derives, stores, validates, and accepts these values. | See [NVIDIA Request Extensions](../../developer-guide/additional-resources/nvidia-request-extensions-nvext.md) for the complete `nvext` request and response shapes. diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index 8b8da101d6ee..87134fa0d453 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -324,6 +335,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "atomic" version = "0.6.1" @@ -441,7 +462,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.2", - "sha1", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -600,7 +621,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.4.2", "percent-encoding", @@ -842,7 +863,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -927,6 +948,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1125,6 +1152,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -1223,6 +1259,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.64" @@ -1291,6 +1336,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -1554,6 +1609,16 @@ dependencies = [ "libc", ] +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + [[package]] name = "crc32c" version = "0.6.8" @@ -1972,7 +2037,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", + "const-oid 0.9.6", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -2548,9 +2615,12 @@ dependencies = [ "dynamo-runtime", "dynamo-sidecar-common", "futures", + "opendal", "prost 0.13.5", "rand 0.9.4", "reqwest 0.12.28", + "rmp-serde", + "serde", "serde_json", "tokio", "tokio-util", @@ -2558,6 +2628,7 @@ dependencies = [ "tonic-build 0.13.1", "tracing", "url", + "zstd", ] [[package]] @@ -3044,7 +3115,7 @@ dependencies = [ "fastrand", "futures-core", "futures-sink", - "spin", + "spin 0.9.8", ] [[package]] @@ -3335,7 +3406,7 @@ dependencies = [ "chrono", "google-cloud-gax", "hex", - "hmac", + "hmac 0.13.0", "http 1.4.2", "reqwest 0.13.4", "rustc_version", @@ -3685,6 +3756,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -4141,6 +4221,16 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -4229,10 +4319,12 @@ checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", "windows-link", ] @@ -4465,7 +4557,7 @@ dependencies = [ "jsonpath-rust", "k8s-openapi", "kube-core", - "pem", + "pem 3.0.6", "rustls", "secrecy", "serde", @@ -4544,6 +4636,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.8", +] [[package]] name = "leb128fmt" @@ -4826,12 +4921,32 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "md5" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" +[[package]] +name = "mea" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c709842c4ce65cb91e2666ad5319dfc1efc3af0d34f02075eddca9000d9f8afb" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "memchr" version = "2.8.2" @@ -5329,6 +5444,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.6", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -5598,10 +5729,10 @@ dependencies = [ "humantime", "hyper", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", - "quick-xml", + "quick-xml 0.38.4", "rand 0.9.4", "reqwest 0.12.28", "ring", @@ -5687,11 +5818,153 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "thiserror 2.0.18", ] +[[package]] +name = "opendal" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" +dependencies = [ + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-service-azblob", + "opendal-service-fs", + "opendal-service-gcs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a43405d217dfdfb543f58847336d3af672897dd1939bb7dcf314b63cf364f1c9" +dependencies = [ + "anyhow", + "asyncband", + "base64 0.23.1", + "bytes", + "futures", + "http 1.4.2", + "jiff", + "log", + "md-5 0.11.0", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + +[[package]] +name = "opendal-service-azblob" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba0d2662ddf0de1f838db5dc48fafb8212ed1a5c7980bc164aa67809696c309c" +dependencies = [ + "base64 0.23.1", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.11.0", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" +dependencies = [ + "http 1.4.2", + "opendal-core", +] + +[[package]] +name = "opendal-service-fs" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb9caf04d6d38713299dd4abac984b95ab16ee20e1ff09160f495c5a64644083" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "opendal-service-s3" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388b1d39b62535c62803754ebef89808859558697366dbedd0299345887ba461" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http 1.4.2", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -5925,6 +6198,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pcre2" version = "0.2.11" @@ -5980,6 +6263,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -6141,6 +6434,32 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -6148,6 +6467,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core 0.6.4", "spki", ] @@ -6607,6 +6928,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.7.0" @@ -6645,6 +6976,7 @@ version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.2", "lru-slab", @@ -6896,7 +7228,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ - "pem", + "pem 3.0.6", "ring", "rustls-pki-types", "time", @@ -6978,6 +7310,118 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqsign-aws-core" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac4749b7dfa7bfaccd01eb03e9dc795ed37e3f20d6f0f38e2c67ee85ad6bc86" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http 1.4.2", + "log", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff250f0fd0b913fbd565e405acc553da0f13bde30bfb5403178c9d0313cdc15f" +dependencies = [ + "bytes", + "http 1.4.2", + "log", + "quick-xml 0.41.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-azure-storage" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f1fbc9add082f54e51e3bd9730f18d850a1f09d246baff5cd612c74ae4290e" +dependencies = [ + "anyhow", + "base64 0.23.1", + "bytes", + "form_urlencoded", + "http 1.4.2", + "log", + "pem 4.0.0", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff052daffb0599681c50f85c59e7236438976efe991ab864edd9f3b235501a0f" +dependencies = [ + "anyhow", + "base64 0.23.1", + "bytes", + "futures", + "hex", + "hmac 0.13.0", + "http 1.4.2", + "jiff", + "log", + "mea", + "percent-encoding", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3235df90a6bca681aa47dd86f2393d122a6d77042aa8a7c81e218cd45c5bfc0" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "272a5813571885c1455ffe106f0c768577e6ce69313446d4476e8d41dcf6ac5a" +dependencies = [ + "bytes", + "form_urlencoded", + "http 1.4.2", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -7046,6 +7490,7 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project-lite", + "quinn", "rustls", "rustls-pki-types", "rustls-platform-verifier", @@ -7130,6 +7575,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-embed" version = "8.11.0" @@ -7404,6 +7870,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -7465,6 +7940,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "secrecy" version = "0.10.3" @@ -7731,6 +8217,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -7916,6 +8413,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -9080,7 +9583,7 @@ dependencies = [ "httparse", "log", "rand 0.9.4", - "sha1", + "sha1 0.10.6", "thiserror 2.0.18", "utf-8", ] @@ -10113,6 +10616,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index a5b2f89460ac..c997fdd1ac89 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -9025,7 +9025,8 @@ mod tests { "cache_salt": "step_7", "extra_fields": ["completion_token_ids"], "metadata_upload": { - "url": "s3://bucket/root/rollouts" + "url": "s3://bucket/root/rollouts", + "fallback_url": "file:///var/tmp/rollouts" } } })) @@ -9041,7 +9042,8 @@ mod tests { assert_eq!( extra_args["nvext"]["metadata_upload"], serde_json::json!({ - "url": "s3://bucket/root/rollouts" + "url": "s3://bucket/root/rollouts", + "fallback_url": "file:///var/tmp/rollouts" }) ); assert_eq!(extra_args["sampling_options"]["detokenize"], false); diff --git a/lib/llm/src/protocols/common/extensions.rs b/lib/llm/src/protocols/common/extensions.rs index 527e85095018..171a62ef9279 100644 --- a/lib/llm/src/protocols/common/extensions.rs +++ b/lib/llm/src/protocols/common/extensions.rs @@ -46,6 +46,13 @@ pub struct RouterParams { pub struct MetadataUpload { #[serde(deserialize_with = "deserialize_metadata_upload_url")] pub url: String, + + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_metadata_upload_fallback_url" + )] + pub fallback_url: Option, } fn deserialize_metadata_upload_url<'de, D>(deserializer: D) -> Result @@ -53,11 +60,27 @@ where D: serde::Deserializer<'de>, { let url = String::deserialize(deserializer)?; + normalize_metadata_upload_url(url, "metadata_upload.url") +} + +fn deserialize_metadata_upload_fallback_url<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Option::::deserialize(deserializer)? + .map(|url| normalize_metadata_upload_url(url, "metadata_upload.fallback_url")) + .transpose() +} + +fn normalize_metadata_upload_url(url: String, field: &str) -> Result +where + E: serde::de::Error, +{ let url = url.trim(); if url.is_empty() { - return Err(serde::de::Error::custom( - "metadata_upload.url must not be empty", - )); + return Err(E::custom(format!("{field} must not be empty"))); } Ok(url.to_string()) } @@ -992,16 +1015,21 @@ mod tests { } #[test] - fn metadata_upload_parses_url() { + fn metadata_upload_parses_urls() { let nvext: NvExt = serde_json::from_value(serde_json::json!({ "metadata_upload": { - "url": " s3://bucket/root/rollouts " + "url": " s3://bucket/root/rollouts ", + "fallback_url": " file:///var/tmp/rollouts " } })) .unwrap(); let upload = nvext.metadata_upload.as_ref().unwrap(); assert_eq!(upload.url, "s3://bucket/root/rollouts"); + assert_eq!( + upload.fallback_url.as_deref(), + Some("file:///var/tmp/rollouts") + ); assert!(!NvExtResponseFieldSelection::from_nvext(Some(&nvext)).engine_data); assert!( @@ -1018,6 +1046,15 @@ mod tests { })) .is_err() ); + assert!( + serde_json::from_value::(serde_json::json!({ + "metadata_upload": { + "url": "s3://bucket/root/rollouts", + "fallback_url": "" + } + })) + .is_err() + ); assert!( serde_json::from_value::(serde_json::json!({ "metadata_upload": { diff --git a/lib/sidecar/sglang/Cargo.toml b/lib/sidecar/sglang/Cargo.toml index 596192eecfe9..f2b242ba2c4c 100644 --- a/lib/sidecar/sglang/Cargo.toml +++ b/lib/sidecar/sglang/Cargo.toml @@ -29,13 +29,17 @@ async-stream = { workspace = true } async-trait = { workspace = true } clap = { version = "4", features = ["derive", "env"] } futures = { workspace = true } +opendal = { workspace = true } rand = { workspace = true } reqwest = { workspace = true } +rmp-serde = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } tracing = { workspace = true } url = { workspace = true } +zstd = { workspace = true } prost = { workspace = true } tonic = { workspace = true } @@ -45,3 +49,4 @@ tonic-build = { workspace = true } [dev-dependencies] dynamo-backend-common = { workspace = true, features = ["testing"] } +tempfile = { workspace = true } diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md index cba07fd3c117..c2397ae33036 100644 --- a/lib/sidecar/sglang/README.md +++ b/lib/sidecar/sglang/README.md @@ -28,6 +28,57 @@ follow-up. Use `DYN_SIDECAR_GRPC_ENDPOINT` instead of `--grpc-endpoint` when the endpoint is provided through the environment. +## Upload response metadata + +Start the sidecar with `--enable-rl` or `DYN_ENABLE_RL=true` to allow response +metadata uploads. Uploads remain disabled for requests that omit +`nvext.metadata_upload`. + +Set `nvext.metadata_upload.url` in an OpenAI-compatible generation request: + +```json +{ + "nvext": { + "metadata_upload": { + "url": "s3://bucket/root/rollout-1", + "fallback_url": "file:///var/tmp/rollout-1" + } + } +} +``` + +The sidecar writes `choice_0.msgpack.zst` under this URL. +The artifact uses the same MessagePack and Zstandard schema as the Python SGLang bindings. +The supported URL schemes are `file://`, `s3://`, `gs://`, and `az://`. +OpenDAL loads credentials from the standard provider configuration for the selected service. +The optional `fallback_url` can use a different supported scheme. +If the primary upload fails, the sidecar writes the same artifact to the fallback URL. + +OpenDAL selects the storage service from each URL scheme. +The packaged sidecar includes the file, S3, Google Cloud Storage, and Azure Blob services. +A downstream binary can register a custom OpenDAL service before it starts the sidecar. +The custom service must use the compatible OpenDAL registry and link into that binary. +The metadata upload code does not need a change for the custom scheme. +The packaged executable does not load service plug-ins at run time. + +The sidecar uploads metadata only for aggregated and decode requests. +It does not return the uploaded metadata inline. +If the final upload fails, the sidecar stops the terminal response. +Thus, the caller cannot accept a rollout without its metadata. + +| Frontend API or worker role | Metadata upload support | +|---|---| +| `POST /v1/chat/completions` | Supported | +| `POST /v1/completions` | Supported | +| `POST /v1/responses` | Supported | +| Native SGLang `POST` or `PUT /generate` | Not supported; the native request adapter does not project `nvext.metadata_upload` into Dynamo request metadata | +| Aggregated worker | Uploads final `meta_info` | +| Disaggregated prefill worker | Does not upload | +| Disaggregated decode worker | Uploads final `meta_info` | + +The sidecar supports one output choice, so each request writes +`choice_0.msgpack.zst`. + Native Dynamo `/generate` requests are forwarded opaquely to SGLang's HTTP endpoint using the gRPC host and the HTTP port returned by `GetServerInfo`. The sidecar advertises this capability only after the HTTP health probe passes diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 17c2dae87056..7b6e40b77d69 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -22,6 +22,7 @@ use tokio_util::sync::CancellationToken; use crate::args::Args; use crate::client::{self, Client, Discovery, Pool}; +use crate::metadata_upload::{MetadataUploader, grpc_metadata}; use crate::native_http::{self, NativeHttp}; use crate::proto as pb; use crate::protocol::{ @@ -35,6 +36,7 @@ pub struct SglangSidecarEngine { disaggregation_mode: DisaggregationMode, bootstrap_host: Option, bootstrap_port: Option, + metadata_upload_enabled: bool, state: OnceCell, cancel: CancellationToken, } @@ -102,6 +104,7 @@ impl SglangSidecarEngine { ); let common = args.sidecar.common; + let metadata_upload_enabled = common.enable_rl; let config = WorkerConfig { namespace: common.namespace, component: if disaggregation_mode == DisaggregationMode::Aggregated { @@ -135,6 +138,7 @@ impl SglangSidecarEngine { disaggregation_mode, bootstrap_host, bootstrap_port, + metadata_upload_enabled, state: OnceCell::new(), cancel: CancellationToken::new(), }, @@ -249,6 +253,10 @@ impl LLMEngine for SglangSidecarEngine { .state .get() .ok_or_else(|| client::engine_shutdown("generate called before start"))?; + let metadata_uploader = MetadataUploader::from_request( + &request, + self.metadata_upload_enabled && !self.disaggregation_mode.is_prefill(), + )?; if let Some(native_request) = native_http::request( &request, ctx.id(), @@ -261,7 +269,12 @@ impl LLMEngine for SglangSidecarEngine { "native SGLang Generate is unavailable because no ready incremental HTTP endpoint was discovered", ) })?; - return Ok(native_http.generate(native_request, ctx, self.cancel.clone())); + return Ok(native_http.generate( + native_request, + ctx, + self.cancel.clone(), + metadata_uploader, + )); } let mut grpc_client = state.pool.stream_client(); @@ -353,14 +366,17 @@ impl LLMEngine for SglangSidecarEngine { break; } }; - let (log_probs, top_logprobs) = + let (log_probs, top_logprobs) = if metadata_uploader.is_some() { + (None, None) + } else { match extract_logprobs(&response.meta_info, return_tokens_as_ids) { Ok(values) => values, Err(err) => { yield Err(err); break; } - }; + } + }; if is_prefill { if response.finished { @@ -395,12 +411,20 @@ impl LLMEngine for SglangSidecarEngine { break; } }; - let engine_data = match engine_data_from_meta(&response.meta_info, true) { - Ok(engine_data) => engine_data, - Err(error) => { + let engine_data = if let Some(uploader) = metadata_uploader.as_ref() { + if let Err(error) = uploader.upload(grpc_metadata(&response.meta_info)).await { yield Err(error); break; } + None + } else { + match engine_data_from_meta(&response.meta_info, true) { + Ok(engine_data) => engine_data, + Err(error) => { + yield Err(error); + break; + } + } }; terminal.token_ids = token_ids; terminal.log_probs = log_probs; @@ -411,11 +435,15 @@ impl LLMEngine for SglangSidecarEngine { } if !token_ids.is_empty() { - let engine_data = match engine_data_from_meta(&response.meta_info, false) { - Ok(engine_data) => engine_data, - Err(error) => { - yield Err(error); - break; + let engine_data = if metadata_uploader.is_some() { + None + } else { + match engine_data_from_meta(&response.meta_info, false) { + Ok(engine_data) => engine_data, + Err(error) => { + yield Err(error); + break; + } } }; yield Ok(LLMEngineOutput { diff --git a/lib/sidecar/sglang/src/lib.rs b/lib/sidecar/sglang/src/lib.rs index 7e8968fc3207..1e2e0681ff56 100644 --- a/lib/sidecar/sglang/src/lib.rs +++ b/lib/sidecar/sglang/src/lib.rs @@ -9,12 +9,13 @@ //! role, parallelism, KV block sizing, and context length are discovered from //! the engine's gRPC metadata RPCs. //! -//! The crate never depends on `sglang` or any engine crate — only -//! `dynamo-backend-common`, `tonic`/`prost`, `clap`, and tokio. +//! The crate never depends on `sglang` or another engine crate. +//! It uses the generated protocol contract and OpenDAL storage services. pub mod args; pub mod client; pub mod engine; +mod metadata_upload; mod native_http; /// Generated SGLang gRPC types, temporarily exposed for the Mocker server diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs new file mode 100644 index 000000000000..fcf007f777f9 --- /dev/null +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -0,0 +1,402 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Out-of-band storage for SGLang response metadata. + +use std::collections::HashMap; +use std::io::Write; + +use dynamo_backend_common::{DynamoError, PreprocessedRequest}; +use opendal::{Buffer, Operator}; +use serde::Serialize; +use serde_json::{Map, Value}; + +use crate::client; + +const OUTPUT_PATH: &str = "choice_0.msgpack.zst"; + +/// The SGLang sidecar supports one output choice, so one uploader owns the +/// primary and optional fallback destinations for that choice. +#[derive(Clone)] +pub(crate) struct MetadataUploader { + primary: Operator, + fallback: Option, +} + +#[derive(Serialize)] +struct MetadataPayload<'a> { + schema_version: u8, + metadata: &'a Value, +} + +impl MetadataUploader { + pub(crate) fn from_request( + request: &PreprocessedRequest, + enabled: bool, + ) -> Result, DynamoError> { + if !enabled { + return Ok(None); + } + let Some(settings) = request + .extra_args + .as_ref() + .and_then(Value::as_object) + .and_then(|extra| extra.get("nvext")) + .and_then(Value::as_object) + .and_then(|nvext| nvext.get("metadata_upload")) + else { + return Ok(None); + }; + if settings.is_null() { + return Ok(None); + } + let settings = settings.as_object().ok_or_else(|| { + client::invalid_arg("extra_args.nvext.metadata_upload must be an object") + })?; + if let Some(unexpected) = settings + .keys() + .find(|key| !matches!(key.as_str(), "url" | "fallback_url")) + { + return Err(client::invalid_arg(format!( + "extra_args.nvext.metadata_upload.{unexpected} is not supported" + ))); + } + let url = settings + .get("url") + .ok_or_else(|| client::invalid_arg("extra_args.nvext.metadata_upload.url is required"))? + .as_str() + .ok_or_else(|| { + client::invalid_arg("extra_args.nvext.metadata_upload.url must be a string") + })?; + // The sidecar is also linked into a Python static library, where + // OpenDAL's constructor-based automatic registration is not reliable. + opendal::install_default(); + let primary = operator_from_url(url, "url")?; + let fallback = settings + .get("fallback_url") + .filter(|value| !value.is_null()) + .map(|value| { + value.as_str().ok_or_else(|| { + client::invalid_arg( + "extra_args.nvext.metadata_upload.fallback_url must be a string", + ) + }) + }) + .transpose()? + .map(|url| operator_from_url(url, "fallback_url")) + .transpose()?; + Ok(Some(Self { primary, fallback })) + } + + pub(crate) async fn upload(&self, metadata: Value) -> Result<(), DynamoError> { + if metadata.as_object().is_some_and(Map::is_empty) { + return Ok(()); + } + let compressed: Buffer = tokio::task::spawn_blocking(move || encode(metadata)) + .await + .map_err(|error| { + client::protocol_error(format!("SGLang metadata encoding task failed: {error}")) + })?? + .into(); + match self.primary.write(OUTPUT_PATH, compressed.clone()).await { + Ok(_) => Ok(()), + Err(primary_error) => { + let Some(fallback) = self.fallback.as_ref() else { + return Err(upload_error(&self.primary, primary_error)); + }; + tracing::warn!("primary metadata upload failed; attempting configured fallback"); + fallback + .write(OUTPUT_PATH, compressed) + .await + .map_err(|error| upload_error(fallback, error))?; + Ok(()) + } + } + } +} + +fn operator_from_url(raw: &str, field: &str) -> Result { + let raw = raw.trim(); + if raw.is_empty() { + return Err(client::invalid_arg(format!( + "extra_args.nvext.metadata_upload.{field} must not be empty" + ))); + } + let uri = normalize_fsspec_uri(raw, field)?; + Operator::from_uri(uri.as_str()).map_err(|error| { + client::invalid_arg(format!( + "could not configure `{}` metadata upload destination: {error}", + uri.split_once(':') + .map(|(scheme, _)| scheme) + .unwrap_or("unknown") + )) + }) +} + +fn upload_error(operator: &Operator, error: opendal::Error) -> DynamoError { + client::protocol_error(format!( + "could not upload SGLang metadata to {}: {error}", + operator.info().root() + )) +} + +fn encode(metadata: Value) -> Result, DynamoError> { + let payload = MetadataPayload { + schema_version: 1, + metadata: &metadata, + }; + let msgpack = rmp_serde::to_vec_named(&payload).map_err(|error| { + client::protocol_error(format!("could not serialize SGLang metadata: {error}")) + })?; + let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).map_err(|error| { + client::protocol_error(format!( + "could not initialize SGLang metadata compression: {error}" + )) + })?; + // Python's ZstdDecompressor.decompress(), used by the existing fsspec + // reader, requires the uncompressed size in the frame header. + encoder + .set_pledged_src_size(Some(msgpack.len() as u64)) + .map_err(|error| { + client::protocol_error(format!( + "could not configure SGLang metadata compression: {error}" + )) + })?; + encoder.include_contentsize(true).map_err(|error| { + client::protocol_error(format!( + "could not configure SGLang metadata compression: {error}" + )) + })?; + encoder.write_all(&msgpack).map_err(|error| { + client::protocol_error(format!("could not compress SGLang metadata: {error}")) + })?; + encoder.finish().map_err(|error| { + client::protocol_error(format!( + "could not finish SGLang metadata compression: {error}" + )) + }) +} + +/// OpenDAL uses canonical service names in URIs. Preserve the fsspec spellings +/// accepted by the Python implementation at the public request boundary. +fn normalize_fsspec_uri(raw: &str, field: &str) -> Result { + let uri = url::Url::parse(raw).map_err(|error| { + client::invalid_arg(format!( + "extra_args.nvext.metadata_upload.{field} is not a valid URL: {error}" + )) + })?; + let canonical_scheme = match uri.scheme() { + "file" => "fs", + "gs" => "gcs", + "az" => "azblob", + scheme => scheme, + }; + Ok(format!("{canonical_scheme}{}", &raw[uri.scheme().len()..])) +} + +pub(crate) fn grpc_metadata(meta: &HashMap) -> Value { + Value::Object( + meta.iter() + .map(|(key, raw)| { + let value = + serde_json::from_str(raw).unwrap_or_else(|_| Value::String(raw.clone())); + (key.clone(), value) + }) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use dynamo_backend_common::{OutputOptions, SamplingOptions, StopConditions}; + use serde_json::{Value, json}; + + use super::{MetadataUploader, OUTPUT_PATH, grpc_metadata, normalize_fsspec_uri}; + + fn request(extra_args: Value) -> dynamo_backend_common::PreprocessedRequest { + dynamo_backend_common::PreprocessedRequest::builder() + .model("model".to_string()) + .token_ids(vec![1]) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .stop_conditions(StopConditions::default()) + .extra_args(Some(extra_args)) + .build() + .unwrap() + } + + #[test] + fn normalizes_fsspec_scheme_aliases() { + assert_eq!( + normalize_fsspec_uri("file:///tmp/metadata", "url") + .unwrap() + .as_str(), + "fs:///tmp/metadata" + ); + assert_eq!( + normalize_fsspec_uri("gs://bucket/metadata", "url") + .unwrap() + .as_str(), + "gcs://bucket/metadata" + ); + assert_eq!( + normalize_fsspec_uri("az://container/metadata", "url") + .unwrap() + .as_str(), + "azblob://container/metadata" + ); + } + + #[test] + fn metadata_upload_is_rl_gated_and_strict() { + let configured = request(json!({ + "nvext": {"metadata_upload": {"url": "file:///tmp/metadata"}} + })); + assert!( + MetadataUploader::from_request(&configured, false) + .unwrap() + .is_none() + ); + assert!( + MetadataUploader::from_request(&configured, true) + .unwrap() + .is_some() + ); + + let invalid = request(json!({ + "nvext": {"metadata_upload": {"url": "file:///tmp", "format": "json"}} + })); + assert!(MetadataUploader::from_request(&invalid, true).is_err()); + } + + #[test] + fn grpc_metadata_decodes_json_values_and_preserves_plain_strings() { + let metadata = grpc_metadata(&std::collections::HashMap::from([ + ("finish_reason".into(), r#"{"type":"stop"}"#.into()), + ("legacy".into(), "plain".into()), + ])); + assert_eq!(metadata["finish_reason"]["type"], "stop"); + assert_eq!(metadata["legacy"], "plain"); + } + + #[tokio::test] + async fn uploads_python_compatible_msgpack_zstd_payload() { + let directory = tempfile::tempdir().unwrap(); + let url = url::Url::from_directory_path(directory.path()) + .unwrap() + .to_string(); + let configured = request(json!({ + "nvext": {"metadata_upload": {"url": url}} + })); + let uploader = MetadataUploader::from_request(&configured, true) + .unwrap() + .unwrap(); + uploader + .upload(json!({ + "id": "sglang-1", + "finish_reason": {"type": "stop"}, + "output_token_logprobs": [[-0.1, 101, "a"]] + })) + .await + .unwrap(); + + let compressed = std::fs::read(directory.path().join("choice_0.msgpack.zst")).unwrap(); + assert!( + zstd::zstd_safe::get_frame_content_size(&compressed) + .unwrap() + .is_some(), + "the frame must advertise its content size for Python compatibility" + ); + let msgpack = zstd::stream::decode_all(Cursor::new(compressed)).unwrap(); + let payload: Value = rmp_serde::from_slice(&msgpack).unwrap(); + assert_eq!(payload["schema_version"], 1); + assert_eq!(payload["metadata"]["id"], "sglang-1"); + assert_eq!(payload["metadata"]["output_token_logprobs"][0][1], 101); + } + + #[tokio::test] + async fn uses_fallback_only_after_primary_failure() { + let directory = tempfile::tempdir().unwrap(); + let primary = directory.path().join("primary-is-a-file"); + std::fs::write(&primary, b"primary sentinel").unwrap(); + let fallback = directory.path().join("fallback"); + let configured = request(json!({ + "nvext": {"metadata_upload": { + "url": url::Url::from_file_path(&primary).unwrap(), + "fallback_url": url::Url::from_directory_path(&fallback).unwrap() + }} + })); + let uploader = MetadataUploader::from_request(&configured, true) + .unwrap() + .unwrap(); + + uploader.upload(json!({"id": "sglang-2"})).await.unwrap(); + + assert_eq!(std::fs::read(primary).unwrap(), b"primary sentinel"); + assert!(fallback.join(OUTPUT_PATH).is_file()); + } + + #[tokio::test] + async fn does_not_write_fallback_when_primary_succeeds() { + let directory = tempfile::tempdir().unwrap(); + let primary = directory.path().join("primary"); + let fallback = directory.path().join("fallback"); + let configured = request(json!({ + "nvext": {"metadata_upload": { + "url": url::Url::from_directory_path(&primary).unwrap(), + "fallback_url": url::Url::from_directory_path(&fallback).unwrap() + }} + })); + let uploader = MetadataUploader::from_request(&configured, true) + .unwrap() + .unwrap(); + + uploader.upload(json!({"id": "sglang-3"})).await.unwrap(); + + assert!(primary.join(OUTPUT_PATH).is_file()); + assert!(!fallback.join(OUTPUT_PATH).exists()); + } + + #[tokio::test] + async fn returns_fallback_error_when_both_destinations_fail() { + let directory = tempfile::tempdir().unwrap(); + let primary = directory.path().join("primary-is-a-file"); + let fallback = directory.path().join("fallback-is-a-file"); + std::fs::write(&primary, b"primary sentinel").unwrap(); + std::fs::write(&fallback, b"fallback sentinel").unwrap(); + let configured = request(json!({ + "nvext": {"metadata_upload": { + "url": url::Url::from_file_path(&primary).unwrap(), + "fallback_url": url::Url::from_file_path(&fallback).unwrap() + }} + })); + let uploader = MetadataUploader::from_request(&configured, true) + .unwrap() + .unwrap(); + + let error = uploader + .upload(json!({"id": "sglang-4"})) + .await + .unwrap_err(); + + assert!(error.to_string().contains("fallback-is-a-file")); + assert_eq!(std::fs::read(primary).unwrap(), b"primary sentinel"); + assert_eq!(std::fs::read(fallback).unwrap(), b"fallback sentinel"); + } + + #[test] + fn accepts_a_registered_custom_scheme() { + opendal::OperatorRegistry::get().register::("custom-metadata"); + let configured = request(json!({ + "nvext": {"metadata_upload": {"url": "custom-metadata://rollout"}} + })); + + assert!( + MetadataUploader::from_request(&configured, true) + .unwrap() + .is_some() + ); + } +} diff --git a/lib/sidecar/sglang/src/native_http.rs b/lib/sidecar/sglang/src/native_http.rs index 5c1000fb9f63..6d2b972ae3a3 100644 --- a/lib/sidecar/sglang/src/native_http.rs +++ b/lib/sidecar/sglang/src/native_http.rs @@ -19,7 +19,7 @@ use tokio_util::{ sync::CancellationToken, }; -use crate::{client, client::Discovery, protocol}; +use crate::{client, client::Discovery, metadata_upload::MetadataUploader, protocol}; const PAYLOAD_KEY: &str = "sglang_tito"; const MAX_EVENT_BYTES: usize = 64 * 1024 * 1024; @@ -242,6 +242,7 @@ impl NativeHttp { request: NativeRequest, ctx: GenerateContext, cancel: CancellationToken, + metadata_uploader: Option, ) -> BoxStream<'static, Result> { Box::pin(async_stream::stream! { let is_prefill = request.is_prefill; @@ -319,7 +320,7 @@ impl NativeHttp { )); return; } - let response = match serde_json::from_str(data) { + let mut response = match serde_json::from_str(data) { Ok(response) => response, Err(error) => { yield Err(client::protocol_error(format!( @@ -328,8 +329,27 @@ impl NativeHttp { return; } }; + let terminal = response_is_terminal(&response); + if let Some(uploader) = metadata_uploader.as_ref() { + let metadata = response.get("meta_info").cloned(); + if terminal && let Some(metadata) = metadata { + if !metadata.is_object() { + yield Err(client::protocol_error( + "SGLang /generate meta_info must be an object", + )); + return; + } + if let Err(error) = uploader.upload(metadata).await { + yield Err(error); + return; + } + } + if let Some(metadata) = response.get_mut("meta_info") { + *metadata = Value::Object(Map::new()); + } + } let has_output = response_has_output(&response); - let (mut output, terminal) = output(response, &mut prefill_handoff); + let mut output = output(response, &mut prefill_handoff, terminal); if !first_output_seen && has_output && (!is_prefill || terminal) { ctx.notify_first_token(); first_output_seen = true; @@ -360,12 +380,15 @@ fn response_has_output(response: &Value) -> bool { }) } -fn output(response: Value, prefill_handoff: &mut Option) -> (LLMEngineOutput, bool) { - let error = response.get("error"); - let finished = error.is_some() +fn response_is_terminal(response: &Value) -> bool { + response.get("error").is_some() || response .pointer("/meta_info/finish_reason") - .is_some_and(|reason| !reason.is_null()); + .is_some_and(|reason| !reason.is_null()) +} + +fn output(response: Value, prefill_handoff: &mut Option, finished: bool) -> LLMEngineOutput { + let error = response.get("error"); let mut output = match error { Some(error) => LLMEngineOutput::error( error @@ -381,7 +404,7 @@ fn output(response: Value, prefill_handoff: &mut Option) -> (LLMEngineOut if finished { output.disaggregated_params = prefill_handoff.take(); } - (output, finished) + output } fn request_error(error: reqwest::Error) -> DynamoError { @@ -437,6 +460,7 @@ mod tests { response_has_output, }; use crate::client::Discovery; + use crate::metadata_upload::MetadataUploader; fn canonical_request() -> PreprocessedRequest { PreprocessedRequest::builder() @@ -582,6 +606,7 @@ mod tests { }, ctx, CancellationToken::new(), + None, ); assert!(stream.next().await.unwrap().is_ok()); @@ -591,6 +616,56 @@ mod tests { server.await.unwrap(); } + #[tokio::test] + async fn metadata_upload_writes_only_terminal_metadata_and_suppresses_it_inline() { + let body = concat!( + "data:{\"output_ids\":[101],\"meta_info\":{\"finish_reason\":null,\"step\":1}}\n\n", + "data:{\"output_ids\":[102],\"meta_info\":{\"finish_reason\":{\"type\":\"stop\"},\"step\":2}}\n\n" + ) + .to_string(); + let (port, server) = serve_once(body, "200 OK").await; + let directory = tempfile::tempdir().unwrap(); + let mut configured = canonical_request(); + configured.extra_args = Some(json!({ + "nvext": { + "metadata_upload": { + "url": url::Url::from_directory_path(directory.path()).unwrap() + } + } + })); + let uploader = MetadataUploader::from_request(&configured, true) + .unwrap() + .unwrap(); + let ctx = GenerateContext::new(dynamo_backend_common::testing::mock_context(), None); + let mut stream = native_http(port).generate( + NativeRequest { + body: json!({"input_ids": [1], "stream": true}), + is_prefill: false, + prefill_handoff: None, + }, + ctx, + CancellationToken::new(), + Some(uploader), + ); + + let incremental = stream.next().await.unwrap().unwrap(); + assert_eq!( + incremental.engine_data.unwrap()["sglang_response"]["meta_info"], + json!({}) + ); + assert!(!directory.path().join("choice_0.msgpack.zst").exists()); + + let terminal = stream.next().await.unwrap().unwrap(); + assert!(terminal.finish_reason.is_some()); + assert_eq!( + terminal.engine_data.unwrap()["sglang_response"]["meta_info"], + json!({}) + ); + assert!(directory.path().join("choice_0.msgpack.zst").exists()); + assert!(stream.next().await.is_none()); + server.await.unwrap(); + } + #[tokio::test] async fn cancellation_is_a_typed_stream_error() { let context = dynamo_backend_common::testing::mock_context(); @@ -604,6 +679,7 @@ mod tests { }, ctx, CancellationToken::new(), + None, ); let error = stream.next().await.unwrap().unwrap_err(); From 79e4ad84cc20ca4572fb2ae1c262faa2ec45d9b7 Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:59:23 +0000 Subject: [PATCH 02/11] refactor(sglang): scope upload dependencies to sidecar Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- Cargo.toml | 8 - .../src/dynamo/common/metadata_upload.py | 49 +---- .../common/tests/test_metadata_upload.py | 167 ------------------ .../tests/test_sglang_processor_unit.py | 2 - .../tests/test_sglang_decode_handler.py | 82 +++++++++ lib/sidecar/sglang/Cargo.toml | 10 +- 6 files changed, 99 insertions(+), 219 deletions(-) delete mode 100644 components/src/dynamo/common/tests/test_metadata_upload.py diff --git a/Cargo.toml b/Cargo.toml index a49a8df50a84..ee9e06eb0945 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,13 +132,6 @@ ipnet = { version = "2" } libc = { version = "0.2" } oneshot = { version = "0.1.13", features = ["std", "async"] } ordered-float = "4" -opendal = { version = "0.59.1", default-features = false, features = [ - "http-transport-reqwest", - "services-azblob", - "services-fs", - "services-gcs", - "services-s3", -] } parking_lot = "0.12.5" prometheus = { version = "0.14"} rand = { version = "0.9.2" } @@ -195,7 +188,6 @@ uuid = { version = "1.18.1", features = ["v4", "serde"] } url = { version = "2.5", features = ["serde"] } xxhash-rust = { version = "0.8", features = ["xxh3", "const_xxh3"] } zeroize = { version = "1" } -zstd = "0.13" # ext-proc / Envoy gRPC prost = { version = "0.13.5" } diff --git a/components/src/dynamo/common/metadata_upload.py b/components/src/dynamo/common/metadata_upload.py index 88a93c887457..a29f21a9cbbb 100644 --- a/components/src/dynamo/common/metadata_upload.py +++ b/components/src/dynamo/common/metadata_upload.py @@ -4,15 +4,11 @@ from __future__ import annotations import asyncio -import logging from dataclasses import dataclass from functools import cache from typing import Any, cast -logger = logging.getLogger(__name__) - - def _backend_metadata_upload_settings(request: dict[str, Any]) -> dict[str, Any] | None: nvext = request.get("nvext") extra_args = request.get("extra_args") or {} @@ -142,47 +138,29 @@ def _normalize_for_upload(value: Any) -> Any: return value -def _normalize_metadata_upload_url(value: object, field: str) -> str: - if not isinstance(value, str): - raise ValueError(f"metadata_upload.{field} must be a string") - value = value.strip() - if not value: - raise ValueError(f"metadata_upload.{field} must not be empty") - return value - - @dataclass(frozen=True) class MetadataUploader: url: str - fallback_url: str | None = None def __post_init__(self) -> None: - object.__setattr__( - self, - "url", - _normalize_metadata_upload_url(self.url, "url"), - ) - if self.fallback_url is not None: - object.__setattr__( - self, - "fallback_url", - _normalize_metadata_upload_url(self.fallback_url, "fallback_url"), - ) + if not isinstance(self.url, str): + raise ValueError("metadata_upload.url must be a string") + url = self.url.strip() + if not url: + raise ValueError("metadata_upload.url must not be empty") + object.__setattr__(self, "url", url) @classmethod def from_settings(cls, settings: dict[str, Any] | None) -> MetadataUploader | None: if settings is None: return None - unexpected = settings.keys() - {"url", "fallback_url"} + unexpected = settings.keys() - {"url"} if unexpected: field = min(unexpected) raise ValueError(f"metadata_upload.{field} is not supported") if "url" not in settings: raise ValueError("metadata_upload.url is required") - return cls( - url=settings["url"], - fallback_url=settings.get("fallback_url"), - ) + return cls(url=settings["url"]) @classmethod def from_backend_request(cls, request: dict[str, Any]) -> MetadataUploader | None: @@ -195,15 +173,6 @@ async def upload_choice(self, choice_index: int, metadata: dict[str, Any]) -> No storage_path = f"choice_{choice_index}.msgpack.zst" data = await asyncio.to_thread(_serialize_metadata, metadata) try: - try: - await _upload_bytes(self.url, storage_path, data) - except Exception: - if self.fallback_url is None: - raise - logger.warning( - "Primary metadata upload failed; attempting configured fallback", - exc_info=True, - ) - await _upload_bytes(self.fallback_url, storage_path, data) + await _upload_bytes(self.url, storage_path, data) finally: del data diff --git a/components/src/dynamo/common/tests/test_metadata_upload.py b/components/src/dynamo/common/tests/test_metadata_upload.py deleted file mode 100644 index 4452cac067fd..000000000000 --- a/components/src/dynamo/common/tests/test_metadata_upload.py +++ /dev/null @@ -1,167 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for out-of-band metadata uploads.""" - -import msgspec -import pytest -import zstandard as zstd - -from dynamo.common.metadata_upload import MetadataUploader - -pytestmark = [ - pytest.mark.unit, - pytest.mark.gpu_0, - pytest.mark.pre_merge, -] - - -def _read_zstd_payload(path): - raw = zstd.ZstdDecompressor().decompress(path.read_bytes()) - return msgspec.msgpack.decode(raw) - - -def test_metadata_uploader_parses_backend_request(): - uploader = MetadataUploader.from_backend_request( - { - "extra_args": { - "nvext": { - "metadata_upload": { - "url": "s3://bucket/root/rollouts", - "fallback_url": " file:///var/tmp/rollouts ", - } - } - } - } - ) - - assert uploader is not None - assert uploader.url == "s3://bucket/root/rollouts" - assert uploader.fallback_url == "file:///var/tmp/rollouts" - - uploader = MetadataUploader.from_backend_request( - {"nvext": {"metadata_upload": {"url": "s3://bucket/root/rollouts"}}} - ) - assert uploader is not None - assert uploader.fallback_url is None - - -@pytest.mark.parametrize( - ("settings", "message"), - [ - ({}, "metadata_upload.url is required"), - ({"url": ""}, "metadata_upload.url must not be empty"), - ({"url": 1}, "metadata_upload.url must be a string"), - ( - {"url": "s3://bucket/root/rollouts", "fallback_url": ""}, - "metadata_upload.fallback_url must not be empty", - ), - ( - {"url": "s3://bucket/root/rollouts", "fallback_url": 1}, - "metadata_upload.fallback_url must be a string", - ), - ("invalid", "metadata_upload must be an object"), - ( - {"url": "s3://bucket/root/rollouts", "format": "json"}, - "metadata_upload.format is not supported", - ), - ], -) -def test_metadata_uploader_rejects_invalid_settings(settings, message): - with pytest.raises(ValueError, match=message): - MetadataUploader.from_backend_request({"nvext": {"metadata_upload": settings}}) - - -@pytest.mark.asyncio -async def test_metadata_upload_normalizes_tensor_values(tmp_path): - torch = pytest.importorskip("torch") - uploader = MetadataUploader(url=(tmp_path / "metadata/tensors").as_uri()) - tensor = torch.tensor([[1, 2], [3, 4]], dtype=torch.int32) - - await uploader.upload_choice(0, {"router_tensor": tensor}) - - payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_0.msgpack.zst") - uploaded_tensor = payload["metadata"]["router_tensor"] - assert uploaded_tensor["type"] == "tensor" - assert uploaded_tensor["dtype"] == "int32" - assert uploaded_tensor["shape"] == [2, 2] - assert uploaded_tensor["data"] == tensor.numpy().tobytes() - - bfloat_tensor = torch.tensor([1, 2], dtype=torch.bfloat16) - await uploader.upload_choice(1, {"router_tensor": bfloat_tensor}) - - payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_1.msgpack.zst") - uploaded_tensor = payload["metadata"]["router_tensor"] - assert uploaded_tensor["dtype"] == "bfloat16" - assert uploaded_tensor["shape"] == [2] - assert uploaded_tensor["data"] == bfloat_tensor.view(torch.uint8).numpy().tobytes() - - -@pytest.mark.asyncio -async def test_metadata_upload_normalizes_numpy_values(tmp_path): - np = pytest.importorskip("numpy") - uploader = MetadataUploader(url=(tmp_path / "metadata/arrays").as_uri()) - array = np.array([[1.5, 2.5], [3.5, 4.5]], dtype=np.float32) - - await uploader.upload_choice(0, {"scores": array[:, ::-1]}) - - payload = _read_zstd_payload(tmp_path / "metadata/arrays/choice_0.msgpack.zst") - uploaded_array = payload["metadata"]["scores"] - expected = np.ascontiguousarray(array[:, ::-1]) - assert uploaded_array["type"] == "ndarray" - assert uploaded_array["dtype"] == "float32" - assert uploaded_array["shape"] == [2, 2] - assert uploaded_array["data"] == expected.tobytes() - - -@pytest.mark.asyncio -async def test_metadata_upload_does_not_write_fallback_when_primary_succeeds(tmp_path): - primary_path = tmp_path / "metadata/primary" - fallback_path = tmp_path / "metadata/fallback" - uploader = MetadataUploader( - url=primary_path.as_uri(), - fallback_url=fallback_path.as_uri(), - ) - - await uploader.upload_choice(0, {"id": "request-1"}) - - assert (primary_path / "choice_0.msgpack.zst").is_file() - assert not fallback_path.exists() - - -@pytest.mark.asyncio -async def test_metadata_upload_uses_fallback_after_primary_failure(tmp_path, caplog): - primary_path = tmp_path / "primary-is-a-file" - primary_path.write_bytes(b"primary sentinel") - fallback_path = tmp_path / "metadata/fallback" - uploader = MetadataUploader( - url=primary_path.as_uri(), - fallback_url=fallback_path.as_uri(), - ) - - with caplog.at_level("WARNING"): - await uploader.upload_choice(2, {"id": "request-2"}) - - assert primary_path.read_bytes() == b"primary sentinel" - assert "Primary metadata upload failed" in caplog.text - payload = _read_zstd_payload(fallback_path / "choice_2.msgpack.zst") - assert payload["metadata"] == {"id": "request-2"} - - -@pytest.mark.asyncio -async def test_metadata_upload_propagates_fallback_failure(tmp_path): - primary_path = tmp_path / "primary-is-a-file" - fallback_path = tmp_path / "fallback-is-a-file" - primary_path.write_bytes(b"primary sentinel") - fallback_path.write_bytes(b"fallback sentinel") - uploader = MetadataUploader( - url=primary_path.as_uri(), - fallback_url=fallback_path.as_uri(), - ) - - with pytest.raises(OSError) as exc_info: - await uploader.upload_choice(0, {"id": "request-3"}) - - assert str(fallback_path) in str(exc_info.value) - assert primary_path.read_bytes() == b"primary sentinel" - assert fallback_path.read_bytes() == b"fallback sentinel" diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index 5ccc5ec10e6f..3fe41e89a161 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -530,7 +530,6 @@ def test_metadata_upload_nvext_is_forwarded_to_backend(self): "nvext": { "metadata_upload": { "url": "s3://bucket/root/rollouts", - "fallback_url": "file:///var/tmp/rollouts", }, }, }, @@ -541,7 +540,6 @@ def test_metadata_upload_nvext_is_forwarded_to_backend(self): assert result["extra_args"]["nvext"]["metadata_upload"] == { "url": "s3://bucket/root/rollouts", - "fallback_url": "file:///var/tmp/rollouts", } def test_model_name_and_token_ids(self): diff --git a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py index 9da4d43218cf..cd741b741572 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py +++ b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py @@ -1061,6 +1061,46 @@ def test_extract_logprobs_formats_top_tokens_as_token_ids(): ] +def test_metadata_uploader_parses_extra_args_nvext(): + uploader = MetadataUploader.from_backend_request( + { + "extra_args": { + "nvext": { + "metadata_upload": { + "url": "s3://bucket/root/rollouts", + } + } + } + } + ) + + assert uploader is not None + assert uploader.url == "s3://bucket/root/rollouts" + + uploader = MetadataUploader.from_backend_request( + {"nvext": {"metadata_upload": {"url": "s3://bucket/root/rollouts"}}} + ) + assert uploader is not None + + +@pytest.mark.parametrize( + ("settings", "message"), + [ + ({}, "metadata_upload.url is required"), + ({"url": ""}, "metadata_upload.url must not be empty"), + ({"url": 1}, "metadata_upload.url must be a string"), + ("invalid", "metadata_upload must be an object"), + ( + {"url": "s3://bucket/root/rollouts", "format": "json"}, + "metadata_upload.format is not supported", + ), + ], +) +def test_metadata_uploader_rejects_invalid_settings(settings, message): + with pytest.raises(ValueError, match=message): + MetadataUploader.from_backend_request({"nvext": {"metadata_upload": settings}}) + + def test_nvext_extra_field_requested_supports_raw_and_preprocessed_shapes(): assert _nvext_extra_field_requested( {"nvext": {"extra_fields": ["stop_reason"]}}, "stop_reason" @@ -1091,6 +1131,48 @@ def test_metadata_upload_requires_enable_rl(): assert uploader.url == "s3://bucket/root/rollouts/run-1" +@pytest.mark.asyncio +async def test_metadata_upload_normalizes_tensor_values(tmp_path): + torch = pytest.importorskip("torch") + uploader = MetadataUploader(url=(tmp_path / "metadata/tensors").as_uri()) + tensor = torch.tensor([[1, 2], [3, 4]], dtype=torch.int32) + + await uploader.upload_choice(0, {"router_tensor": tensor}) + + payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_0.msgpack.zst") + uploaded_tensor = payload["metadata"]["router_tensor"] + assert uploaded_tensor["type"] == "tensor" + assert uploaded_tensor["dtype"] == "int32" + assert uploaded_tensor["shape"] == [2, 2] + assert uploaded_tensor["data"] == tensor.numpy().tobytes() + + bfloat_tensor = torch.tensor([1, 2], dtype=torch.bfloat16) + await uploader.upload_choice(1, {"router_tensor": bfloat_tensor}) + + payload = _read_zstd_payload(tmp_path / "metadata/tensors/choice_1.msgpack.zst") + uploaded_tensor = payload["metadata"]["router_tensor"] + assert uploaded_tensor["dtype"] == "bfloat16" + assert uploaded_tensor["shape"] == [2] + assert uploaded_tensor["data"] == bfloat_tensor.view(torch.uint8).numpy().tobytes() + + +@pytest.mark.asyncio +async def test_metadata_upload_normalizes_numpy_values(tmp_path): + np = pytest.importorskip("numpy") + uploader = MetadataUploader(url=(tmp_path / "metadata/arrays").as_uri()) + array = np.array([[1.5, 2.5], [3.5, 4.5]], dtype=np.float32) + + await uploader.upload_choice(0, {"scores": array[:, ::-1]}) + + payload = _read_zstd_payload(tmp_path / "metadata/arrays/choice_0.msgpack.zst") + uploaded_array = payload["metadata"]["scores"] + expected = np.ascontiguousarray(array[:, ::-1]) + assert uploaded_array["type"] == "ndarray" + assert uploaded_array["dtype"] == "float32" + assert uploaded_array["shape"] == [2, 2] + assert uploaded_array["data"] == expected.tobytes() + + @pytest.mark.asyncio async def test_process_token_stream_treats_completion_usage_as_optional(): handler = _new_decode_handler() diff --git a/lib/sidecar/sglang/Cargo.toml b/lib/sidecar/sglang/Cargo.toml index f2b242ba2c4c..1763d2702217 100644 --- a/lib/sidecar/sglang/Cargo.toml +++ b/lib/sidecar/sglang/Cargo.toml @@ -29,7 +29,13 @@ async-stream = { workspace = true } async-trait = { workspace = true } clap = { version = "4", features = ["derive", "env"] } futures = { workspace = true } -opendal = { workspace = true } +opendal = { version = "0.59.1", default-features = false, features = [ + "http-transport-reqwest", + "services-azblob", + "services-fs", + "services-gcs", + "services-s3", +] } rand = { workspace = true } reqwest = { workspace = true } rmp-serde = { workspace = true } @@ -39,7 +45,7 @@ tokio = { workspace = true } tokio-util = { workspace = true } tracing = { workspace = true } url = { workspace = true } -zstd = { workspace = true } +zstd = "0.13" prost = { workspace = true } tonic = { workspace = true } From 6ba7c4adc22da11eeb0b37148fcef1bea9740ef1 Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:04:00 +0000 Subject: [PATCH 03/11] docs: defer sidecar metadata upload documentation Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- .../nvidia-request-extensions-nvext.md | 10 +---- .../backends/sglang/sidecar.md | 37 ------------------- .../integration-reference.md | 2 +- 3 files changed, 2 insertions(+), 47 deletions(-) diff --git a/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md b/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md index db14d53c222f..28ef0272733b 100644 --- a/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md +++ b/docs/fern/pages/developer-guide/additional-resources/nvidia-request-extensions-nvext.md @@ -38,7 +38,7 @@ Include `nvext` as a top-level field alongside standard OpenAI-compatible fields | `max_thinking_tokens` | `u32` | `None` | Backend | Maximum thinking tokens allowed (passed through to backends). | | `cache_salt` | `string` | `None` | Router / supported backends | Namespaces Dynamo KV routing. vLLM and TensorRT-LLM also isolate backend KV-cache reuse; see [Backend support](#backend-support). This is the recommended cache-isolation input. | | `extra_fields` | `string[]` | `None` | Response builder | Fields to include in the response `nvext`. Supported: `"worker_id"`, `"timing"`, `"routed_experts"`, `"engine_data"`, `"stop_reason"`, `"detailed_finish_reason"`, `"prompt_token_ids"`, `"completion_token_ids"`, `"prompt_logprobs"`. | -| `metadata_upload` | object | `None` | SGLang backend | Uploads final cumulative SGLang `meta_info` out of band. Accepts required `url` and optional `fallback_url` strings. Requires an RL-enabled SGLang worker. | +| `metadata_upload` | object | `None` | SGLang backend | Uploads final cumulative SGLang `meta_info` out of band. The object accepts one required `url` field. Requires an RL-enabled SGLang worker. | | `prefill_worker_id` | `u64` | `None` | Router | Routes the request to a specific prefill worker (disaggregated serving). | | `decode_worker_id` | `u64` | `None` | Router | Routes the request to a specific decode worker (disaggregated serving). | | `dp_rank` | `u32` | `None` | Router/backend | Data-parallel rank for the decode worker. Typically set by EPP routing headers. | @@ -56,14 +56,6 @@ token IDs, pass integer IDs in the normal `stop` array, for example `"stop": [576]`. Strings such as `"token_id:576"` remain literal string stop sequences and are not parsed as token IDs. -For `metadata_upload`, the Python SGLang worker uses the installed fsspec -backends. The SGLang sidecar uses OpenDAL and includes `file://`, `s3://`, -`gs://`, and `az://` support. Both implementations try `fallback_url` only -after the primary upload fails. The sidecar supports this field on -`/v1/chat/completions`, `/v1/completions`, and `/v1/responses`, but not on the -native SGLang `/generate` frontend API. In disaggregated serving, only the -decode worker uploads metadata. - ### Header Overrides Routing fields can also be set via HTTP headers, which take priority over `nvext` values: diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md index 3e88d49cc861..4b6e28460b71 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/sidecar.md @@ -28,43 +28,6 @@ in-process backend; sidecar feature parity is still under evaluation. See the [SGLang sidecar README](https://github.com/ai-dynamo/dynamo/blob/main/lib/sidecar/sglang/README.md) for current protocol details. -## Upload Response Metadata - -Start the sidecar with `--enable-rl` or `DYN_ENABLE_RL=true` to allow -out-of-band metadata uploads. Then set `nvext.metadata_upload` on each -OpenAI-compatible generation request that requires an artifact: - -```json -{ - "nvext": { - "metadata_upload": { - "url": "s3://bucket/root/rollout-1", - "fallback_url": "file:///var/tmp/rollout-1" - } - } -} -``` - -The sidecar writes the final cumulative SGLang `meta_info` to -`choice_0.msgpack.zst`. It tries `fallback_url` only when the primary upload -fails. The packaged sidecar supports `file://`, `s3://`, `gs://`, and `az://` -URLs through OpenDAL. - -| API or worker role | Support | -|---|---| -| `POST /v1/chat/completions` | Supported | -| `POST /v1/completions` | Supported | -| `POST /v1/responses` | Supported | -| Native SGLang `POST` or `PUT /generate` | Not supported | -| Aggregated worker | Uploads final metadata | -| Disaggregated prefill worker | Does not upload | -| Disaggregated decode worker | Uploads final metadata | - -The native `/generate` adapter does not currently project -`nvext.metadata_upload` into Dynamo request metadata. See -[NVIDIA Request Extensions](../../../../additional-resources/nvidia-request-extensions-nvext.md) -for the field contract. - ## Launch Locally From a Dynamo source checkout, build or install Dynamo so diff --git a/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md b/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md index 3c00d9a41cc0..2b7f6839752e 100644 --- a/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md +++ b/docs/fern/pages/use-cases/reinforcement-learning/integration-reference.md @@ -72,7 +72,7 @@ Before admitting a sample to training, verify: | Generated token IDs | Named `nvext.completion_token_ids`, or the native SGLang stream | Use the engine-returned sequence; exact placement depends on the selected interface. | | Selected and prompt log probabilities | Standard completion log probabilities, named `nvext.prompt_logprobs`, or native SGLang metadata | Check support and alignment on the exact backend and response path. | | Routed experts and raw engine data | Opt-in `nvext.routed_experts` or `nvext.engine_data` | Backend-specific. Prefer named fields over the raw engine payload. | -| Large SGLang `meta_info` | `nvext.metadata_upload` on `/v1/chat/completions`, `/v1/completions`, or `/v1/responses` | Uploaded out of band by an RL-enabled SGLang worker. Set required `url` and optional `fallback_url`; the destination is trusted control input. The native `/generate` frontend API does not support this extension. | +| Large SGLang `meta_info` | `nvext.metadata_upload.url` on the OpenAI-compatible path | Uploaded out of band by an RL-enabled SGLang worker; the destination is trusted control input. | | Masks, rewards, advantages, tool or environment state, and trajectory objects | Not a generic Dynamo response contract | The framework derives, stores, validates, and accepts these values. | See [NVIDIA Request Extensions](../../developer-guide/additional-resources/nvidia-request-extensions-nvext.md) for the complete `nvext` request and response shapes. From 2042c3f4ed9162e5d94a9ba3d38f9f6bfa8c56a3 Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:15:20 +0000 Subject: [PATCH 04/11] fix(sglang): address metadata upload review Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- lib/sidecar/sglang/README.md | 4 +- lib/sidecar/sglang/src/engine.rs | 20 ++++++- lib/sidecar/sglang/src/lib.rs | 4 +- lib/sidecar/sglang/src/metadata_upload.rs | 71 ++++++----------------- lib/sidecar/sglang/src/native_http.rs | 24 ++++++-- 5 files changed, 58 insertions(+), 65 deletions(-) diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md index c2397ae33036..46a1f288b4df 100644 --- a/lib/sidecar/sglang/README.md +++ b/lib/sidecar/sglang/README.md @@ -41,7 +41,7 @@ Set `nvext.metadata_upload.url` in an OpenAI-compatible generation request: "nvext": { "metadata_upload": { "url": "s3://bucket/root/rollout-1", - "fallback_url": "file:///var/tmp/rollout-1" + "fallback_url": "fs:///var/tmp/rollout-1" } } } @@ -49,7 +49,7 @@ Set `nvext.metadata_upload.url` in an OpenAI-compatible generation request: The sidecar writes `choice_0.msgpack.zst` under this URL. The artifact uses the same MessagePack and Zstandard schema as the Python SGLang bindings. -The supported URL schemes are `file://`, `s3://`, `gs://`, and `az://`. +The supported OpenDAL URI schemes are `fs://`, `s3://`, `gcs://`, and `azblob://`. OpenDAL loads credentials from the standard provider configuration for the selected service. The optional `fallback_url` can use a different supported scheme. If the primary upload fails, the sidecar writes the same artifact to the fallback URL. diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 7b6e40b77d69..51da7660c101 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -412,9 +412,23 @@ impl LLMEngine for SglangSidecarEngine { } }; let engine_data = if let Some(uploader) = metadata_uploader.as_ref() { - if let Err(error) = uploader.upload(grpc_metadata(&response.meta_info)).await { - yield Err(error); - break; + let uploaded = tokio::select! { + biased; + _ = ctx.stopped() => None, + _ = cancel.cancelled() => None, + result = uploader.upload(grpc_metadata(&response.meta_info)) => Some(result), + }; + match uploaded { + Some(Ok(())) => {} + Some(Err(error)) => { + yield Err(error); + break; + } + None => { + yield Ok(LLMEngineOutput::cancelled() + .with_usage(usage(observed_prompt_tokens, generated))); + break; + } } None } else { diff --git a/lib/sidecar/sglang/src/lib.rs b/lib/sidecar/sglang/src/lib.rs index 1e2e0681ff56..578b0e3929ea 100644 --- a/lib/sidecar/sglang/src/lib.rs +++ b/lib/sidecar/sglang/src/lib.rs @@ -9,8 +9,8 @@ //! role, parallelism, KV block sizing, and context length are discovered from //! the engine's gRPC metadata RPCs. //! -//! The crate never depends on `sglang` or another engine crate. -//! It uses the generated protocol contract and OpenDAL storage services. +//! The crate never depends on `sglang` or any engine crate — only +//! `dynamo-backend-common`, `tonic`/`prost`, `clap`, and tokio. pub mod args; pub mod client; diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs index fcf007f777f9..afe8de5b9295 100644 --- a/lib/sidecar/sglang/src/metadata_upload.rs +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -122,11 +122,10 @@ fn operator_from_url(raw: &str, field: &str) -> Result { "extra_args.nvext.metadata_upload.{field} must not be empty" ))); } - let uri = normalize_fsspec_uri(raw, field)?; - Operator::from_uri(uri.as_str()).map_err(|error| { + Operator::from_uri(raw).map_err(|error| { client::invalid_arg(format!( "could not configure `{}` metadata upload destination: {error}", - uri.split_once(':') + raw.split_once(':') .map(|(scheme, _)| scheme) .unwrap_or("unknown") )) @@ -177,23 +176,6 @@ fn encode(metadata: Value) -> Result, DynamoError> { }) } -/// OpenDAL uses canonical service names in URIs. Preserve the fsspec spellings -/// accepted by the Python implementation at the public request boundary. -fn normalize_fsspec_uri(raw: &str, field: &str) -> Result { - let uri = url::Url::parse(raw).map_err(|error| { - client::invalid_arg(format!( - "extra_args.nvext.metadata_upload.{field} is not a valid URL: {error}" - )) - })?; - let canonical_scheme = match uri.scheme() { - "file" => "fs", - "gs" => "gcs", - "az" => "azblob", - scheme => scheme, - }; - Ok(format!("{canonical_scheme}{}", &raw[uri.scheme().len()..])) -} - pub(crate) fn grpc_metadata(meta: &HashMap) -> Value { Value::Object( meta.iter() @@ -209,11 +191,16 @@ pub(crate) fn grpc_metadata(meta: &HashMap) -> Value { #[cfg(test)] mod tests { use std::io::Cursor; + use std::path::Path; use dynamo_backend_common::{OutputOptions, SamplingOptions, StopConditions}; use serde_json::{Value, json}; - use super::{MetadataUploader, OUTPUT_PATH, grpc_metadata, normalize_fsspec_uri}; + use super::{MetadataUploader, OUTPUT_PATH, grpc_metadata}; + + fn fs_uri(path: &Path) -> String { + format!("fs://{}", path.display()) + } fn request(extra_args: Value) -> dynamo_backend_common::PreprocessedRequest { dynamo_backend_common::PreprocessedRequest::builder() @@ -227,32 +214,10 @@ mod tests { .unwrap() } - #[test] - fn normalizes_fsspec_scheme_aliases() { - assert_eq!( - normalize_fsspec_uri("file:///tmp/metadata", "url") - .unwrap() - .as_str(), - "fs:///tmp/metadata" - ); - assert_eq!( - normalize_fsspec_uri("gs://bucket/metadata", "url") - .unwrap() - .as_str(), - "gcs://bucket/metadata" - ); - assert_eq!( - normalize_fsspec_uri("az://container/metadata", "url") - .unwrap() - .as_str(), - "azblob://container/metadata" - ); - } - #[test] fn metadata_upload_is_rl_gated_and_strict() { let configured = request(json!({ - "nvext": {"metadata_upload": {"url": "file:///tmp/metadata"}} + "nvext": {"metadata_upload": {"url": "fs:///tmp/metadata"}} })); assert!( MetadataUploader::from_request(&configured, false) @@ -266,7 +231,7 @@ mod tests { ); let invalid = request(json!({ - "nvext": {"metadata_upload": {"url": "file:///tmp", "format": "json"}} + "nvext": {"metadata_upload": {"url": "fs:///tmp", "format": "json"}} })); assert!(MetadataUploader::from_request(&invalid, true).is_err()); } @@ -284,9 +249,7 @@ mod tests { #[tokio::test] async fn uploads_python_compatible_msgpack_zstd_payload() { let directory = tempfile::tempdir().unwrap(); - let url = url::Url::from_directory_path(directory.path()) - .unwrap() - .to_string(); + let url = fs_uri(directory.path()); let configured = request(json!({ "nvext": {"metadata_upload": {"url": url}} })); @@ -324,8 +287,8 @@ mod tests { let fallback = directory.path().join("fallback"); let configured = request(json!({ "nvext": {"metadata_upload": { - "url": url::Url::from_file_path(&primary).unwrap(), - "fallback_url": url::Url::from_directory_path(&fallback).unwrap() + "url": fs_uri(&primary), + "fallback_url": fs_uri(&fallback) }} })); let uploader = MetadataUploader::from_request(&configured, true) @@ -345,8 +308,8 @@ mod tests { let fallback = directory.path().join("fallback"); let configured = request(json!({ "nvext": {"metadata_upload": { - "url": url::Url::from_directory_path(&primary).unwrap(), - "fallback_url": url::Url::from_directory_path(&fallback).unwrap() + "url": fs_uri(&primary), + "fallback_url": fs_uri(&fallback) }} })); let uploader = MetadataUploader::from_request(&configured, true) @@ -368,8 +331,8 @@ mod tests { std::fs::write(&fallback, b"fallback sentinel").unwrap(); let configured = request(json!({ "nvext": {"metadata_upload": { - "url": url::Url::from_file_path(&primary).unwrap(), - "fallback_url": url::Url::from_file_path(&fallback).unwrap() + "url": fs_uri(&primary), + "fallback_url": fs_uri(&fallback) }} })); let uploader = MetadataUploader::from_request(&configured, true) diff --git a/lib/sidecar/sglang/src/native_http.rs b/lib/sidecar/sglang/src/native_http.rs index 6d2b972ae3a3..a15fac606faf 100644 --- a/lib/sidecar/sglang/src/native_http.rs +++ b/lib/sidecar/sglang/src/native_http.rs @@ -339,9 +339,25 @@ impl NativeHttp { )); return; } - if let Err(error) = uploader.upload(metadata).await { - yield Err(error); - return; + let uploaded = tokio::select! { + biased; + _ = ctx.stopped() => None, + _ = cancel.cancelled() => None, + result = uploader.upload(metadata) => Some(result), + }; + match uploaded { + Some(Ok(())) => {} + Some(Err(error)) => { + yield Err(error); + return; + } + None => { + yield Err(client::cancelled(format!( + "SGLang native request {} was cancelled", + ctx.id() + ))); + return; + } } } if let Some(metadata) = response.get_mut("meta_info") { @@ -629,7 +645,7 @@ mod tests { configured.extra_args = Some(json!({ "nvext": { "metadata_upload": { - "url": url::Url::from_directory_path(directory.path()).unwrap() + "url": format!("fs://{}", directory.path().display()) } } })); From 09ffe194dbc0c52113a444869f306b3a6527fa4e Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:58:29 +0000 Subject: [PATCH 05/11] refactor(sglang): reduce metadata upload scope Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- Cargo.lock | 290 ++------------------ lib/bindings/python/Cargo.lock | 300 ++------------------- lib/llm/src/preprocessor.rs | 6 +- lib/llm/src/protocols/common/extensions.rs | 41 +-- lib/sidecar/sglang/Cargo.toml | 2 - lib/sidecar/sglang/README.md | 51 ---- lib/sidecar/sglang/src/engine.rs | 15 +- lib/sidecar/sglang/src/metadata_upload.rs | 128 ++------- lib/sidecar/sglang/src/native_http.rs | 108 +------- 9 files changed, 80 insertions(+), 861 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a325c8678054..82aaafcc9308 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,17 +8,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures 0.2.17", -] - [[package]] name = "ahash" version = "0.8.12" @@ -1269,15 +1258,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "block2" version = "0.6.2" @@ -1388,15 +1368,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "cbc" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" -dependencies = [ - "cipher", -] - [[package]] name = "cbindgen" version = "0.27.0" @@ -1511,16 +1482,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -1611,7 +1572,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -2346,7 +2307,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -2392,7 +2352,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3413,7 +3373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4643,7 +4603,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.5.10", "system-configuration 0.6.1", "tokio", "tower-service", @@ -4909,16 +4869,6 @@ dependencies = [ "libc", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "insta" version = "1.47.2" @@ -4986,7 +4936,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5314,7 +5264,7 @@ dependencies = [ "jsonpath-rust", "k8s-openapi", "kube-core", - "pem 3.0.6", + "pem", "rustls", "secrecy", "serde", @@ -5552,9 +5502,6 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin 0.9.8", -] [[package]] name = "leb128fmt" @@ -6393,7 +6340,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6429,22 +6376,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-bigint-dig" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" -dependencies = [ - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.6", - "smallvec", - "zeroize", -] - [[package]] name = "num-cmp" version = "0.1.0" @@ -6837,9 +6768,7 @@ checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" dependencies = [ "opendal-core", "opendal-http-transport-reqwest", - "opendal-service-azblob", "opendal-service-fs", - "opendal-service-gcs", "opendal-service-s3", ] @@ -6883,37 +6812,6 @@ dependencies = [ "reqwest 0.13.4", ] -[[package]] -name = "opendal-service-azblob" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba0d2662ddf0de1f838db5dc48fafb8212ed1a5c7980bc164aa67809696c309c" -dependencies = [ - "base64 0.23.1", - "bytes", - "http 1.4.1", - "log", - "opendal-core", - "opendal-service-azure-common", - "quick-xml 0.41.0", - "reqsign-azure-storage", - "reqsign-core", - "reqsign-file-read-tokio", - "serde", - "sha2 0.11.0", - "uuid", -] - -[[package]] -name = "opendal-service-azure-common" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" -dependencies = [ - "http 1.4.1", - "opendal-core", -] - [[package]] name = "opendal-service-fs" version = "0.59.1" @@ -6928,28 +6826,6 @@ dependencies = [ "xattr", ] -[[package]] -name = "opendal-service-gcs" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" -dependencies = [ - "async-trait", - "bytes", - "http 1.4.1", - "log", - "opendal-core", - "percent-encoding", - "quick-xml 0.41.0", - "reqsign-core", - "reqsign-file-read-tokio", - "reqsign-google", - "serde", - "serde_json", - "tokio", - "uuid", -] - [[package]] name = "opendal-service-s3" version = "0.59.1" @@ -7215,16 +7091,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest 0.10.7", - "hmac 0.12.1", -] - [[package]] name = "pcre2" version = "0.2.11" @@ -7280,16 +7146,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "pem" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" -dependencies = [ - "base64 0.23.1", - "serde_core", -] - [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -7451,32 +7307,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der 0.7.10", - "pkcs8 0.10.2", - "spki 0.7.3", -] - -[[package]] -name = "pkcs5" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" -dependencies = [ - "aes", - "cbc", - "der 0.7.10", - "pbkdf2", - "scrypt", - "sha2 0.10.9", - "spki 0.7.3", -] - [[package]] name = "pkcs8" version = "0.9.0" @@ -7494,8 +7324,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der 0.7.10", - "pkcs5", - "rand_core 0.6.4", "spki 0.7.3", ] @@ -7766,7 +7594,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "itertools 0.14.0", "log", "multimap", @@ -7786,7 +7614,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "itertools 0.14.0", "log", "multimap", @@ -7970,7 +7798,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls", - "socket2 0.6.4", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -8009,9 +7837,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.5.10", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -8254,7 +8082,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ - "pem 3.0.6", + "pem", "ring", "rustls-pki-types", "time", @@ -8378,27 +8206,6 @@ dependencies = [ "serde", ] -[[package]] -name = "reqsign-azure-storage" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3f1fbc9add082f54e51e3bd9730f18d850a1f09d246baff5cd612c74ae4290e" -dependencies = [ - "anyhow", - "base64 0.23.1", - "bytes", - "form_urlencoded", - "http 1.4.1", - "log", - "pem 4.0.0", - "percent-encoding", - "reqsign-core", - "rsa", - "serde", - "serde_json", - "sha1 0.11.0", -] - [[package]] name = "reqsign-core" version = "3.3.1" @@ -8416,9 +8223,6 @@ dependencies = [ "log", "mea", "percent-encoding", - "rsa", - "serde", - "serde_json", "sha1 0.11.0", "sha2 0.11.0", "windows-sys 0.61.2", @@ -8435,25 +8239,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "reqsign-google" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "272a5813571885c1455ffe106f0c768577e6ce69313446d4476e8d41dcf6ac5a" -dependencies = [ - "bytes", - "form_urlencoded", - "http 1.4.1", - "log", - "percent-encoding", - "reqsign-aws-v4", - "reqsign-core", - "rsa", - "serde", - "serde_json", - "tokio", -] - [[package]] name = "reqwest" version = "0.11.27" @@ -8643,27 +8428,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rsa" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" -dependencies = [ - "const-oid 0.9.6", - "digest 0.10.7", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sha2 0.10.9", - "signature 2.2.0", - "spki 0.7.3", - "subtle", - "zeroize", -] - [[package]] name = "rstest" version = "0.18.2" @@ -8865,7 +8629,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8933,7 +8697,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9049,15 +8813,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "salsa20" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" -dependencies = [ - "cipher", -] - [[package]] name = "same-file" version = "1.0.6" @@ -9119,17 +8874,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scrypt" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" -dependencies = [ - "pbkdf2", - "salsa20", - "sha2 0.10.9", -] - [[package]] name = "sec1" version = "0.3.0" @@ -10075,7 +9819,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11645,7 +11389,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index 87134fa0d453..ff062171e324 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -8,17 +8,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures 0.2.17", -] - [[package]] name = "ahash" version = "0.8.12" @@ -621,7 +610,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac 0.13.0", + "hmac", "http 0.2.12", "http 1.4.2", "percent-encoding", @@ -1152,15 +1141,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "block2" version = "0.6.2" @@ -1259,15 +1239,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "cbc" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" -dependencies = [ - "cipher", -] - [[package]] name = "cc" version = "1.2.64" @@ -1336,16 +1307,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -1436,7 +1397,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -2037,9 +1998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.7", - "subtle", ] [[package]] @@ -2083,7 +2042,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2860,7 +2819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3406,7 +3365,7 @@ dependencies = [ "chrono", "google-cloud-gax", "hex", - "hmac 0.13.0", + "hmac", "http 1.4.2", "reqwest 0.13.4", "rustc_version", @@ -3756,15 +3715,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "hmac" version = "0.13.0" @@ -3946,7 +3896,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -4221,16 +4171,6 @@ dependencies = [ "libc", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "integer-encoding" version = "3.0.4" @@ -4557,7 +4497,7 @@ dependencies = [ "jsonpath-rust", "k8s-openapi", "kube-core", - "pem 3.0.6", + "pem", "rustls", "secrecy", "serde", @@ -4636,9 +4576,6 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin 0.9.8", -] [[package]] name = "leb128fmt" @@ -5409,7 +5346,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5444,22 +5381,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-bigint-dig" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" -dependencies = [ - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.6", - "smallvec", - "zeroize", -] - [[package]] name = "num-complex" version = "0.4.6" @@ -5831,9 +5752,7 @@ checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" dependencies = [ "opendal-core", "opendal-http-transport-reqwest", - "opendal-service-azblob", "opendal-service-fs", - "opendal-service-gcs", "opendal-service-s3", ] @@ -5877,37 +5796,6 @@ dependencies = [ "reqwest 0.13.4", ] -[[package]] -name = "opendal-service-azblob" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba0d2662ddf0de1f838db5dc48fafb8212ed1a5c7980bc164aa67809696c309c" -dependencies = [ - "base64 0.23.1", - "bytes", - "http 1.4.2", - "log", - "opendal-core", - "opendal-service-azure-common", - "quick-xml 0.41.0", - "reqsign-azure-storage", - "reqsign-core", - "reqsign-file-read-tokio", - "serde", - "sha2 0.11.0", - "uuid", -] - -[[package]] -name = "opendal-service-azure-common" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" -dependencies = [ - "http 1.4.2", - "opendal-core", -] - [[package]] name = "opendal-service-fs" version = "0.59.1" @@ -5922,28 +5810,6 @@ dependencies = [ "xattr", ] -[[package]] -name = "opendal-service-gcs" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" -dependencies = [ - "async-trait", - "bytes", - "http 1.4.2", - "log", - "opendal-core", - "percent-encoding", - "quick-xml 0.41.0", - "reqsign-core", - "reqsign-file-read-tokio", - "reqsign-google", - "serde", - "serde_json", - "tokio", - "uuid", -] - [[package]] name = "opendal-service-s3" version = "0.59.1" @@ -6198,16 +6064,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest 0.10.7", - "hmac 0.12.1", -] - [[package]] name = "pcre2" version = "0.2.11" @@ -6263,16 +6119,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "pem" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" -dependencies = [ - "base64 0.23.1", - "serde_core", -] - [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -6434,32 +6280,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs5" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" -dependencies = [ - "aes", - "cbc", - "der", - "pbkdf2", - "scrypt", - "sha2 0.10.9", - "spki", -] - [[package]] name = "pkcs8" version = "0.10.2" @@ -6467,8 +6287,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", - "pkcs5", - "rand_core 0.6.4", "spki", ] @@ -6963,7 +6781,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls", - "socket2 0.6.4", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -7002,9 +6820,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.5.10", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7228,7 +7046,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ - "pem 3.0.6", + "pem", "ring", "rustls-pki-types", "time", @@ -7346,27 +7164,6 @@ dependencies = [ "serde", ] -[[package]] -name = "reqsign-azure-storage" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3f1fbc9add082f54e51e3bd9730f18d850a1f09d246baff5cd612c74ae4290e" -dependencies = [ - "anyhow", - "base64 0.23.1", - "bytes", - "form_urlencoded", - "http 1.4.2", - "log", - "pem 4.0.0", - "percent-encoding", - "reqsign-core", - "rsa", - "serde", - "serde_json", - "sha1 0.11.0", -] - [[package]] name = "reqsign-core" version = "3.3.1" @@ -7378,15 +7175,12 @@ dependencies = [ "bytes", "futures", "hex", - "hmac 0.13.0", + "hmac", "http 1.4.2", "jiff", "log", "mea", "percent-encoding", - "rsa", - "serde", - "serde_json", "sha1 0.11.0", "sha2 0.11.0", "windows-sys 0.61.2", @@ -7403,25 +7197,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "reqsign-google" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "272a5813571885c1455ffe106f0c768577e6ce69313446d4476e8d41dcf6ac5a" -dependencies = [ - "bytes", - "form_urlencoded", - "http 1.4.2", - "log", - "percent-encoding", - "reqsign-aws-v4", - "reqsign-core", - "rsa", - "serde", - "serde_json", - "tokio", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -7575,27 +7350,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rsa" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" -dependencies = [ - "const-oid 0.9.6", - "digest 0.10.7", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "sha2 0.10.9", - "signature", - "spki", - "subtle", - "zeroize", -] - [[package]] name = "rust-embed" version = "8.11.0" @@ -7698,7 +7452,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7766,7 +7520,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7870,15 +7624,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "salsa20" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" -dependencies = [ - "cipher", -] - [[package]] name = "same-file" version = "1.0.6" @@ -7940,17 +7685,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scrypt" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" -dependencies = [ - "pbkdf2", - "salsa20", - "sha2 0.10.9", -] - [[package]] name = "secrecy" version = "0.10.3" @@ -8793,7 +8527,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10182,7 +9916,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index c997fdd1ac89..a5b2f89460ac 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -9025,8 +9025,7 @@ mod tests { "cache_salt": "step_7", "extra_fields": ["completion_token_ids"], "metadata_upload": { - "url": "s3://bucket/root/rollouts", - "fallback_url": "file:///var/tmp/rollouts" + "url": "s3://bucket/root/rollouts" } } })) @@ -9042,8 +9041,7 @@ mod tests { assert_eq!( extra_args["nvext"]["metadata_upload"], serde_json::json!({ - "url": "s3://bucket/root/rollouts", - "fallback_url": "file:///var/tmp/rollouts" + "url": "s3://bucket/root/rollouts" }) ); assert_eq!(extra_args["sampling_options"]["detokenize"], false); diff --git a/lib/llm/src/protocols/common/extensions.rs b/lib/llm/src/protocols/common/extensions.rs index 171a62ef9279..ef22c08bc6f8 100644 --- a/lib/llm/src/protocols/common/extensions.rs +++ b/lib/llm/src/protocols/common/extensions.rs @@ -47,11 +47,7 @@ pub struct MetadataUpload { #[serde(deserialize_with = "deserialize_metadata_upload_url")] pub url: String, - #[serde( - default, - skip_serializing_if = "Option::is_none", - deserialize_with = "deserialize_metadata_upload_fallback_url" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub fallback_url: Option, } @@ -60,27 +56,11 @@ where D: serde::Deserializer<'de>, { let url = String::deserialize(deserializer)?; - normalize_metadata_upload_url(url, "metadata_upload.url") -} - -fn deserialize_metadata_upload_fallback_url<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - Option::::deserialize(deserializer)? - .map(|url| normalize_metadata_upload_url(url, "metadata_upload.fallback_url")) - .transpose() -} - -fn normalize_metadata_upload_url(url: String, field: &str) -> Result -where - E: serde::de::Error, -{ let url = url.trim(); if url.is_empty() { - return Err(E::custom(format!("{field} must not be empty"))); + return Err(serde::de::Error::custom( + "metadata_upload.url must not be empty", + )); } Ok(url.to_string()) } @@ -1019,7 +999,7 @@ mod tests { let nvext: NvExt = serde_json::from_value(serde_json::json!({ "metadata_upload": { "url": " s3://bucket/root/rollouts ", - "fallback_url": " file:///var/tmp/rollouts " + "fallback_url": "fs:///var/tmp/rollouts" } })) .unwrap(); @@ -1028,7 +1008,7 @@ mod tests { assert_eq!(upload.url, "s3://bucket/root/rollouts"); assert_eq!( upload.fallback_url.as_deref(), - Some("file:///var/tmp/rollouts") + Some("fs:///var/tmp/rollouts") ); assert!(!NvExtResponseFieldSelection::from_nvext(Some(&nvext)).engine_data); @@ -1046,15 +1026,6 @@ mod tests { })) .is_err() ); - assert!( - serde_json::from_value::(serde_json::json!({ - "metadata_upload": { - "url": "s3://bucket/root/rollouts", - "fallback_url": "" - } - })) - .is_err() - ); assert!( serde_json::from_value::(serde_json::json!({ "metadata_upload": { diff --git a/lib/sidecar/sglang/Cargo.toml b/lib/sidecar/sglang/Cargo.toml index 1763d2702217..d9a9986278d9 100644 --- a/lib/sidecar/sglang/Cargo.toml +++ b/lib/sidecar/sglang/Cargo.toml @@ -31,9 +31,7 @@ clap = { version = "4", features = ["derive", "env"] } futures = { workspace = true } opendal = { version = "0.59.1", default-features = false, features = [ "http-transport-reqwest", - "services-azblob", "services-fs", - "services-gcs", "services-s3", ] } rand = { workspace = true } diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md index 46a1f288b4df..cba07fd3c117 100644 --- a/lib/sidecar/sglang/README.md +++ b/lib/sidecar/sglang/README.md @@ -28,57 +28,6 @@ follow-up. Use `DYN_SIDECAR_GRPC_ENDPOINT` instead of `--grpc-endpoint` when the endpoint is provided through the environment. -## Upload response metadata - -Start the sidecar with `--enable-rl` or `DYN_ENABLE_RL=true` to allow response -metadata uploads. Uploads remain disabled for requests that omit -`nvext.metadata_upload`. - -Set `nvext.metadata_upload.url` in an OpenAI-compatible generation request: - -```json -{ - "nvext": { - "metadata_upload": { - "url": "s3://bucket/root/rollout-1", - "fallback_url": "fs:///var/tmp/rollout-1" - } - } -} -``` - -The sidecar writes `choice_0.msgpack.zst` under this URL. -The artifact uses the same MessagePack and Zstandard schema as the Python SGLang bindings. -The supported OpenDAL URI schemes are `fs://`, `s3://`, `gcs://`, and `azblob://`. -OpenDAL loads credentials from the standard provider configuration for the selected service. -The optional `fallback_url` can use a different supported scheme. -If the primary upload fails, the sidecar writes the same artifact to the fallback URL. - -OpenDAL selects the storage service from each URL scheme. -The packaged sidecar includes the file, S3, Google Cloud Storage, and Azure Blob services. -A downstream binary can register a custom OpenDAL service before it starts the sidecar. -The custom service must use the compatible OpenDAL registry and link into that binary. -The metadata upload code does not need a change for the custom scheme. -The packaged executable does not load service plug-ins at run time. - -The sidecar uploads metadata only for aggregated and decode requests. -It does not return the uploaded metadata inline. -If the final upload fails, the sidecar stops the terminal response. -Thus, the caller cannot accept a rollout without its metadata. - -| Frontend API or worker role | Metadata upload support | -|---|---| -| `POST /v1/chat/completions` | Supported | -| `POST /v1/completions` | Supported | -| `POST /v1/responses` | Supported | -| Native SGLang `POST` or `PUT /generate` | Not supported; the native request adapter does not project `nvext.metadata_upload` into Dynamo request metadata | -| Aggregated worker | Uploads final `meta_info` | -| Disaggregated prefill worker | Does not upload | -| Disaggregated decode worker | Uploads final `meta_info` | - -The sidecar supports one output choice, so each request writes -`choice_0.msgpack.zst`. - Native Dynamo `/generate` requests are forwarded opaquely to SGLang's HTTP endpoint using the gRPC host and the HTTP port returned by `GetServerInfo`. The sidecar advertises this capability only after the HTTP health probe passes diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 51da7660c101..3bad458cdb3d 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -253,10 +253,6 @@ impl LLMEngine for SglangSidecarEngine { .state .get() .ok_or_else(|| client::engine_shutdown("generate called before start"))?; - let metadata_uploader = MetadataUploader::from_request( - &request, - self.metadata_upload_enabled && !self.disaggregation_mode.is_prefill(), - )?; if let Some(native_request) = native_http::request( &request, ctx.id(), @@ -269,13 +265,12 @@ impl LLMEngine for SglangSidecarEngine { "native SGLang Generate is unavailable because no ready incremental HTTP endpoint was discovered", ) })?; - return Ok(native_http.generate( - native_request, - ctx, - self.cancel.clone(), - metadata_uploader, - )); + return Ok(native_http.generate(native_request, ctx, self.cancel.clone())); } + let metadata_uploader = MetadataUploader::from_request( + &request, + self.metadata_upload_enabled && !self.disaggregation_mode.is_prefill(), + )?; let mut grpc_client = state.pool.stream_client(); let prompt_tokens = request.token_ids.len() as u32; diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs index afe8de5b9295..0b693285815c 100644 --- a/lib/sidecar/sglang/src/metadata_upload.rs +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -8,16 +8,13 @@ use std::io::Write; use dynamo_backend_common::{DynamoError, PreprocessedRequest}; use opendal::{Buffer, Operator}; -use serde::Serialize; -use serde_json::{Map, Value}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; use crate::client; const OUTPUT_PATH: &str = "choice_0.msgpack.zst"; -/// The SGLang sidecar supports one output choice, so one uploader owns the -/// primary and optional fallback destinations for that choice. -#[derive(Clone)] pub(crate) struct MetadataUploader { primary: Operator, fallback: Option, @@ -29,6 +26,13 @@ struct MetadataPayload<'a> { metadata: &'a Value, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MetadataUploadConfig { + url: String, + fallback_url: Option, +} + impl MetadataUploader { pub(crate) fn from_request( request: &PreprocessedRequest, @@ -37,7 +41,7 @@ impl MetadataUploader { if !enabled { return Ok(None); } - let Some(settings) = request + let Some(config) = request .extra_args .as_ref() .and_then(Value::as_object) @@ -47,49 +51,25 @@ impl MetadataUploader { else { return Ok(None); }; - if settings.is_null() { + if config.is_null() { return Ok(None); } - let settings = settings.as_object().ok_or_else(|| { - client::invalid_arg("extra_args.nvext.metadata_upload must be an object") + let config = MetadataUploadConfig::deserialize(config).map_err(|error| { + client::invalid_arg(format!("invalid extra_args.nvext.metadata_upload: {error}")) })?; - if let Some(unexpected) = settings - .keys() - .find(|key| !matches!(key.as_str(), "url" | "fallback_url")) - { - return Err(client::invalid_arg(format!( - "extra_args.nvext.metadata_upload.{unexpected} is not supported" - ))); - } - let url = settings - .get("url") - .ok_or_else(|| client::invalid_arg("extra_args.nvext.metadata_upload.url is required"))? - .as_str() - .ok_or_else(|| { - client::invalid_arg("extra_args.nvext.metadata_upload.url must be a string") - })?; - // The sidecar is also linked into a Python static library, where - // OpenDAL's constructor-based automatic registration is not reliable. + // Ensure built-in services are registered when this crate is linked statically. opendal::install_default(); - let primary = operator_from_url(url, "url")?; - let fallback = settings - .get("fallback_url") - .filter(|value| !value.is_null()) - .map(|value| { - value.as_str().ok_or_else(|| { - client::invalid_arg( - "extra_args.nvext.metadata_upload.fallback_url must be a string", - ) - }) - }) - .transpose()? + let primary = operator_from_url(&config.url, "url")?; + let fallback = config + .fallback_url + .as_deref() .map(|url| operator_from_url(url, "fallback_url")) .transpose()?; Ok(Some(Self { primary, fallback })) } pub(crate) async fn upload(&self, metadata: Value) -> Result<(), DynamoError> { - if metadata.as_object().is_some_and(Map::is_empty) { + if metadata.as_object().is_some_and(|value| value.is_empty()) { return Ok(()); } let compressed: Buffer = tokio::task::spawn_blocking(move || encode(metadata)) @@ -104,7 +84,7 @@ impl MetadataUploader { let Some(fallback) = self.fallback.as_ref() else { return Err(upload_error(&self.primary, primary_error)); }; - tracing::warn!("primary metadata upload failed; attempting configured fallback"); + tracing::warn!(error = %primary_error, "primary metadata upload failed; attempting fallback"); fallback .write(OUTPUT_PATH, compressed) .await @@ -236,16 +216,6 @@ mod tests { assert!(MetadataUploader::from_request(&invalid, true).is_err()); } - #[test] - fn grpc_metadata_decodes_json_values_and_preserves_plain_strings() { - let metadata = grpc_metadata(&std::collections::HashMap::from([ - ("finish_reason".into(), r#"{"type":"stop"}"#.into()), - ("legacy".into(), "plain".into()), - ])); - assert_eq!(metadata["finish_reason"]["type"], "stop"); - assert_eq!(metadata["legacy"], "plain"); - } - #[tokio::test] async fn uploads_python_compatible_msgpack_zstd_payload() { let directory = tempfile::tempdir().unwrap(); @@ -257,11 +227,11 @@ mod tests { .unwrap() .unwrap(); uploader - .upload(json!({ - "id": "sglang-1", - "finish_reason": {"type": "stop"}, - "output_token_logprobs": [[-0.1, 101, "a"]] - })) + .upload(grpc_metadata(&std::collections::HashMap::from([ + ("id".into(), "sglang-1".into()), + ("finish_reason".into(), r#"{"type":"stop"}"#.into()), + ("output_token_logprobs".into(), r#"[[-0.1,101,"a"]]"#.into()), + ]))) .await .unwrap(); @@ -301,54 +271,6 @@ mod tests { assert!(fallback.join(OUTPUT_PATH).is_file()); } - #[tokio::test] - async fn does_not_write_fallback_when_primary_succeeds() { - let directory = tempfile::tempdir().unwrap(); - let primary = directory.path().join("primary"); - let fallback = directory.path().join("fallback"); - let configured = request(json!({ - "nvext": {"metadata_upload": { - "url": fs_uri(&primary), - "fallback_url": fs_uri(&fallback) - }} - })); - let uploader = MetadataUploader::from_request(&configured, true) - .unwrap() - .unwrap(); - - uploader.upload(json!({"id": "sglang-3"})).await.unwrap(); - - assert!(primary.join(OUTPUT_PATH).is_file()); - assert!(!fallback.join(OUTPUT_PATH).exists()); - } - - #[tokio::test] - async fn returns_fallback_error_when_both_destinations_fail() { - let directory = tempfile::tempdir().unwrap(); - let primary = directory.path().join("primary-is-a-file"); - let fallback = directory.path().join("fallback-is-a-file"); - std::fs::write(&primary, b"primary sentinel").unwrap(); - std::fs::write(&fallback, b"fallback sentinel").unwrap(); - let configured = request(json!({ - "nvext": {"metadata_upload": { - "url": fs_uri(&primary), - "fallback_url": fs_uri(&fallback) - }} - })); - let uploader = MetadataUploader::from_request(&configured, true) - .unwrap() - .unwrap(); - - let error = uploader - .upload(json!({"id": "sglang-4"})) - .await - .unwrap_err(); - - assert!(error.to_string().contains("fallback-is-a-file")); - assert_eq!(std::fs::read(primary).unwrap(), b"primary sentinel"); - assert_eq!(std::fs::read(fallback).unwrap(), b"fallback sentinel"); - } - #[test] fn accepts_a_registered_custom_scheme() { opendal::OperatorRegistry::get().register::("custom-metadata"); diff --git a/lib/sidecar/sglang/src/native_http.rs b/lib/sidecar/sglang/src/native_http.rs index a15fac606faf..5c1000fb9f63 100644 --- a/lib/sidecar/sglang/src/native_http.rs +++ b/lib/sidecar/sglang/src/native_http.rs @@ -19,7 +19,7 @@ use tokio_util::{ sync::CancellationToken, }; -use crate::{client, client::Discovery, metadata_upload::MetadataUploader, protocol}; +use crate::{client, client::Discovery, protocol}; const PAYLOAD_KEY: &str = "sglang_tito"; const MAX_EVENT_BYTES: usize = 64 * 1024 * 1024; @@ -242,7 +242,6 @@ impl NativeHttp { request: NativeRequest, ctx: GenerateContext, cancel: CancellationToken, - metadata_uploader: Option, ) -> BoxStream<'static, Result> { Box::pin(async_stream::stream! { let is_prefill = request.is_prefill; @@ -320,7 +319,7 @@ impl NativeHttp { )); return; } - let mut response = match serde_json::from_str(data) { + let response = match serde_json::from_str(data) { Ok(response) => response, Err(error) => { yield Err(client::protocol_error(format!( @@ -329,43 +328,8 @@ impl NativeHttp { return; } }; - let terminal = response_is_terminal(&response); - if let Some(uploader) = metadata_uploader.as_ref() { - let metadata = response.get("meta_info").cloned(); - if terminal && let Some(metadata) = metadata { - if !metadata.is_object() { - yield Err(client::protocol_error( - "SGLang /generate meta_info must be an object", - )); - return; - } - let uploaded = tokio::select! { - biased; - _ = ctx.stopped() => None, - _ = cancel.cancelled() => None, - result = uploader.upload(metadata) => Some(result), - }; - match uploaded { - Some(Ok(())) => {} - Some(Err(error)) => { - yield Err(error); - return; - } - None => { - yield Err(client::cancelled(format!( - "SGLang native request {} was cancelled", - ctx.id() - ))); - return; - } - } - } - if let Some(metadata) = response.get_mut("meta_info") { - *metadata = Value::Object(Map::new()); - } - } let has_output = response_has_output(&response); - let mut output = output(response, &mut prefill_handoff, terminal); + let (mut output, terminal) = output(response, &mut prefill_handoff); if !first_output_seen && has_output && (!is_prefill || terminal) { ctx.notify_first_token(); first_output_seen = true; @@ -396,15 +360,12 @@ fn response_has_output(response: &Value) -> bool { }) } -fn response_is_terminal(response: &Value) -> bool { - response.get("error").is_some() +fn output(response: Value, prefill_handoff: &mut Option) -> (LLMEngineOutput, bool) { + let error = response.get("error"); + let finished = error.is_some() || response .pointer("/meta_info/finish_reason") - .is_some_and(|reason| !reason.is_null()) -} - -fn output(response: Value, prefill_handoff: &mut Option, finished: bool) -> LLMEngineOutput { - let error = response.get("error"); + .is_some_and(|reason| !reason.is_null()); let mut output = match error { Some(error) => LLMEngineOutput::error( error @@ -420,7 +381,7 @@ fn output(response: Value, prefill_handoff: &mut Option, finished: bool) if finished { output.disaggregated_params = prefill_handoff.take(); } - output + (output, finished) } fn request_error(error: reqwest::Error) -> DynamoError { @@ -476,7 +437,6 @@ mod tests { response_has_output, }; use crate::client::Discovery; - use crate::metadata_upload::MetadataUploader; fn canonical_request() -> PreprocessedRequest { PreprocessedRequest::builder() @@ -622,7 +582,6 @@ mod tests { }, ctx, CancellationToken::new(), - None, ); assert!(stream.next().await.unwrap().is_ok()); @@ -632,56 +591,6 @@ mod tests { server.await.unwrap(); } - #[tokio::test] - async fn metadata_upload_writes_only_terminal_metadata_and_suppresses_it_inline() { - let body = concat!( - "data:{\"output_ids\":[101],\"meta_info\":{\"finish_reason\":null,\"step\":1}}\n\n", - "data:{\"output_ids\":[102],\"meta_info\":{\"finish_reason\":{\"type\":\"stop\"},\"step\":2}}\n\n" - ) - .to_string(); - let (port, server) = serve_once(body, "200 OK").await; - let directory = tempfile::tempdir().unwrap(); - let mut configured = canonical_request(); - configured.extra_args = Some(json!({ - "nvext": { - "metadata_upload": { - "url": format!("fs://{}", directory.path().display()) - } - } - })); - let uploader = MetadataUploader::from_request(&configured, true) - .unwrap() - .unwrap(); - let ctx = GenerateContext::new(dynamo_backend_common::testing::mock_context(), None); - let mut stream = native_http(port).generate( - NativeRequest { - body: json!({"input_ids": [1], "stream": true}), - is_prefill: false, - prefill_handoff: None, - }, - ctx, - CancellationToken::new(), - Some(uploader), - ); - - let incremental = stream.next().await.unwrap().unwrap(); - assert_eq!( - incremental.engine_data.unwrap()["sglang_response"]["meta_info"], - json!({}) - ); - assert!(!directory.path().join("choice_0.msgpack.zst").exists()); - - let terminal = stream.next().await.unwrap().unwrap(); - assert!(terminal.finish_reason.is_some()); - assert_eq!( - terminal.engine_data.unwrap()["sglang_response"]["meta_info"], - json!({}) - ); - assert!(directory.path().join("choice_0.msgpack.zst").exists()); - assert!(stream.next().await.is_none()); - server.await.unwrap(); - } - #[tokio::test] async fn cancellation_is_a_typed_stream_error() { let context = dynamo_backend_common::testing::mock_context(); @@ -695,7 +604,6 @@ mod tests { }, ctx, CancellationToken::new(), - None, ); let error = stream.next().await.unwrap().unwrap_err(); From e818e977548729f21e3c1a5cff55551f9bfcb23e Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Wed, 16 Sep 2026 06:43:30 +0000 Subject: [PATCH 06/11] fix(sglang): bound metadata upload latency Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- .../src/dynamo/common/metadata_upload.py | 10 ++- .../tests/test_sglang_decode_handler.py | 14 +++- lib/llm/src/protocols/common/extensions.rs | 37 ++++++++- lib/sidecar/sglang/src/engine.rs | 9 +- lib/sidecar/sglang/src/metadata_upload.rs | 83 +++++++++++++------ 5 files changed, 118 insertions(+), 35 deletions(-) diff --git a/components/src/dynamo/common/metadata_upload.py b/components/src/dynamo/common/metadata_upload.py index a29f21a9cbbb..6168d05f1c5b 100644 --- a/components/src/dynamo/common/metadata_upload.py +++ b/components/src/dynamo/common/metadata_upload.py @@ -154,12 +154,20 @@ def __post_init__(self) -> None: def from_settings(cls, settings: dict[str, Any] | None) -> MetadataUploader | None: if settings is None: return None - unexpected = settings.keys() - {"url"} + # The Rust sidecar consumes fallback_url. Accept it here so the shared + # request contract does not break the Python SGLang worker. + unexpected = settings.keys() - {"url", "fallback_url"} if unexpected: field = min(unexpected) raise ValueError(f"metadata_upload.{field} is not supported") if "url" not in settings: raise ValueError("metadata_upload.url is required") + fallback_url = settings.get("fallback_url") + if fallback_url is not None: + if not isinstance(fallback_url, str): + raise ValueError("metadata_upload.fallback_url must be a string") + if not fallback_url.strip(): + raise ValueError("metadata_upload.fallback_url must not be empty") return cls(url=settings["url"]) @classmethod diff --git a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py index cd741b741572..c0dad0ff0485 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py +++ b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py @@ -1078,9 +1078,17 @@ def test_metadata_uploader_parses_extra_args_nvext(): assert uploader.url == "s3://bucket/root/rollouts" uploader = MetadataUploader.from_backend_request( - {"nvext": {"metadata_upload": {"url": "s3://bucket/root/rollouts"}}} + { + "nvext": { + "metadata_upload": { + "url": "s3://bucket/root/rollouts", + "fallback_url": "fs:///var/tmp/rollouts", + } + } + } ) assert uploader is not None + assert uploader.url == "s3://bucket/root/rollouts" @pytest.mark.parametrize( @@ -1094,6 +1102,10 @@ def test_metadata_uploader_parses_extra_args_nvext(): {"url": "s3://bucket/root/rollouts", "format": "json"}, "metadata_upload.format is not supported", ), + ( + {"url": "s3://bucket/root/rollouts", "fallback_url": ""}, + "metadata_upload.fallback_url must not be empty", + ), ], ) def test_metadata_uploader_rejects_invalid_settings(settings, message): diff --git a/lib/llm/src/protocols/common/extensions.rs b/lib/llm/src/protocols/common/extensions.rs index ef22c08bc6f8..e198cd60c180 100644 --- a/lib/llm/src/protocols/common/extensions.rs +++ b/lib/llm/src/protocols/common/extensions.rs @@ -47,7 +47,11 @@ pub struct MetadataUpload { #[serde(deserialize_with = "deserialize_metadata_upload_url")] pub url: String, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_metadata_upload_fallback_url" + )] pub fallback_url: Option, } @@ -65,6 +69,26 @@ where Ok(url.to_string()) } +fn deserialize_metadata_upload_fallback_url<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Option::::deserialize(deserializer)? + .map(|url| { + let url = url.trim(); + if url.is_empty() { + Err(serde::de::Error::custom( + "metadata_upload.fallback_url must not be empty", + )) + } else { + Ok(url.to_string()) + } + }) + .transpose() +} + /// Causal trigger that produced an incoming agent request. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] @@ -999,7 +1023,7 @@ mod tests { let nvext: NvExt = serde_json::from_value(serde_json::json!({ "metadata_upload": { "url": " s3://bucket/root/rollouts ", - "fallback_url": "fs:///var/tmp/rollouts" + "fallback_url": " fs:///var/tmp/rollouts " } })) .unwrap(); @@ -1026,6 +1050,15 @@ mod tests { })) .is_err() ); + assert!( + serde_json::from_value::(serde_json::json!({ + "metadata_upload": { + "url": "s3://bucket/root/rollouts", + "fallback_url": "" + } + })) + .is_err() + ); assert!( serde_json::from_value::(serde_json::json!({ "metadata_upload": { diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 3bad458cdb3d..78c9f708cefd 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -267,10 +267,6 @@ impl LLMEngine for SglangSidecarEngine { })?; return Ok(native_http.generate(native_request, ctx, self.cancel.clone())); } - let metadata_uploader = MetadataUploader::from_request( - &request, - self.metadata_upload_enabled && !self.disaggregation_mode.is_prefill(), - )?; let mut grpc_client = state.pool.stream_client(); let prompt_tokens = request.token_ids.len() as u32; @@ -285,6 +281,11 @@ impl LLMEngine for SglangSidecarEngine { self.bootstrap_host.as_deref(), self.bootstrap_port, )?; + let metadata_uploader = MetadataUploader::from_request( + &request, + self.metadata_upload_enabled && !self.disaggregation_mode.is_prefill(), + ) + .await?; let prefill_handoff = if self.disaggregation_mode.is_prefill() { grpc_request .disaggregated_params diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs index 0b693285815c..a1a92a0b43f8 100644 --- a/lib/sidecar/sglang/src/metadata_upload.rs +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use std::io::Write; +use std::time::Duration; use dynamo_backend_common::{DynamoError, PreprocessedRequest}; use opendal::{Buffer, Operator}; @@ -14,10 +15,11 @@ use serde_json::Value; use crate::client; const OUTPUT_PATH: &str = "choice_0.msgpack.zst"; +const WRITE_TIMEOUT: Duration = Duration::from_secs(10); pub(crate) struct MetadataUploader { primary: Operator, - fallback: Option, + fallback: Option, } #[derive(Serialize)] @@ -34,7 +36,7 @@ struct MetadataUploadConfig { } impl MetadataUploader { - pub(crate) fn from_request( + pub(crate) async fn from_request( request: &PreprocessedRequest, enabled: bool, ) -> Result, DynamoError> { @@ -59,11 +61,10 @@ impl MetadataUploader { })?; // Ensure built-in services are registered when this crate is linked statically. opendal::install_default(); - let primary = operator_from_url(&config.url, "url")?; + let primary = operator_from_url(config.url, "url").await?; let fallback = config .fallback_url - .as_deref() - .map(|url| operator_from_url(url, "fallback_url")) + .map(|url| normalize_url(url, "fallback_url")) .transpose()?; Ok(Some(Self { primary, fallback })) } @@ -78,38 +79,57 @@ impl MetadataUploader { client::protocol_error(format!("SGLang metadata encoding task failed: {error}")) })?? .into(); - match self.primary.write(OUTPUT_PATH, compressed.clone()).await { - Ok(_) => Ok(()), + match write(&self.primary, compressed.clone()).await { + Ok(()) => Ok(()), Err(primary_error) => { - let Some(fallback) = self.fallback.as_ref() else { - return Err(upload_error(&self.primary, primary_error)); + let Some(fallback_url) = self.fallback.as_ref() else { + return Err(primary_error); }; tracing::warn!(error = %primary_error, "primary metadata upload failed; attempting fallback"); - fallback - .write(OUTPUT_PATH, compressed) - .await - .map_err(|error| upload_error(fallback, error))?; - Ok(()) + let fallback = operator_from_url(fallback_url.clone(), "fallback_url").await?; + write(&fallback, compressed).await } } } } -fn operator_from_url(raw: &str, field: &str) -> Result { +fn normalize_url(raw: String, field: &str) -> Result { let raw = raw.trim(); if raw.is_empty() { return Err(client::invalid_arg(format!( "extra_args.nvext.metadata_upload.{field} must not be empty" ))); } - Operator::from_uri(raw).map_err(|error| { - client::invalid_arg(format!( - "could not configure `{}` metadata upload destination: {error}", - raw.split_once(':') - .map(|(scheme, _)| scheme) - .unwrap_or("unknown") - )) + Ok(raw.to_string()) +} + +async fn operator_from_url(raw: String, field: &'static str) -> Result { + tokio::task::spawn_blocking(move || { + let raw = normalize_url(raw, field)?; + Operator::from_uri(raw.as_str()).map_err(|error| { + client::invalid_arg(format!( + "could not configure `{}` metadata upload destination: {error}", + raw.split_once(':') + .map(|(scheme, _)| scheme) + .unwrap_or("unknown") + )) + }) }) + .await + .map_err(|error| { + client::protocol_error(format!("metadata upload setup task failed: {error}")) + })? +} + +async fn write(operator: &Operator, data: Buffer) -> Result<(), DynamoError> { + match tokio::time::timeout(WRITE_TIMEOUT, operator.write(OUTPUT_PATH, data)).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(error)) => Err(upload_error(operator, error)), + Err(_) => Err(client::protocol_error(format!( + "SGLang metadata upload to {} timed out after {WRITE_TIMEOUT:?}", + operator.info().root() + ))), + } } fn upload_error(operator: &Operator, error: opendal::Error) -> DynamoError { @@ -194,18 +214,20 @@ mod tests { .unwrap() } - #[test] - fn metadata_upload_is_rl_gated_and_strict() { + #[tokio::test] + async fn metadata_upload_is_rl_gated_and_strict() { let configured = request(json!({ "nvext": {"metadata_upload": {"url": "fs:///tmp/metadata"}} })); assert!( MetadataUploader::from_request(&configured, false) + .await .unwrap() .is_none() ); assert!( MetadataUploader::from_request(&configured, true) + .await .unwrap() .is_some() ); @@ -213,7 +235,11 @@ mod tests { let invalid = request(json!({ "nvext": {"metadata_upload": {"url": "fs:///tmp", "format": "json"}} })); - assert!(MetadataUploader::from_request(&invalid, true).is_err()); + assert!( + MetadataUploader::from_request(&invalid, true) + .await + .is_err() + ); } #[tokio::test] @@ -224,6 +250,7 @@ mod tests { "nvext": {"metadata_upload": {"url": url}} })); let uploader = MetadataUploader::from_request(&configured, true) + .await .unwrap() .unwrap(); uploader @@ -262,6 +289,7 @@ mod tests { }} })); let uploader = MetadataUploader::from_request(&configured, true) + .await .unwrap() .unwrap(); @@ -271,8 +299,8 @@ mod tests { assert!(fallback.join(OUTPUT_PATH).is_file()); } - #[test] - fn accepts_a_registered_custom_scheme() { + #[tokio::test] + async fn accepts_a_registered_custom_scheme() { opendal::OperatorRegistry::get().register::("custom-metadata"); let configured = request(json!({ "nvext": {"metadata_upload": {"url": "custom-metadata://rollout"}} @@ -280,6 +308,7 @@ mod tests { assert!( MetadataUploader::from_request(&configured, true) + .await .unwrap() .is_some() ); From 2e5f4c8a697506b08f60250f432cac890f888887 Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Wed, 16 Sep 2026 07:06:59 +0000 Subject: [PATCH 07/11] feat(sglang): cache and retry metadata uploads Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- Cargo.lock | 24 +++ lib/bindings/python/Cargo.lock | 24 +++ lib/sidecar/sglang/Cargo.toml | 3 + lib/sidecar/sglang/src/args.rs | 147 +++++++++++++++++ lib/sidecar/sglang/src/engine.rs | 6 +- lib/sidecar/sglang/src/metadata_upload.rs | 190 ++++++++++++++++++---- 6 files changed, 363 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82aaafcc9308..7ccf9f089a60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3068,6 +3068,7 @@ dependencies = [ "dynamo-runtime", "dynamo-sidecar-common", "futures", + "lru 0.16.4", "opendal", "prost 0.13.5", "rand 0.9.4", @@ -6768,6 +6769,8 @@ checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" dependencies = [ "opendal-core", "opendal-http-transport-reqwest", + "opendal-layer-retry", + "opendal-layer-timeout", "opendal-service-fs", "opendal-service-s3", ] @@ -6812,6 +6815,27 @@ dependencies = [ "reqwest 0.13.4", ] +[[package]] +name = "opendal-layer-retry" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06306202c97c54fb41bdbbdcb854c8798823f0022ae7aac7a40c8a1023aa83a6" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd334cbd0a0bc934146733e74a80a5faf8db8e014781a25f6d9d80d7b87c981" +dependencies = [ + "opendal-core", + "tokio", +] + [[package]] name = "opendal-service-fs" version = "0.59.1" diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index ff062171e324..a98cb902f8a8 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2574,6 +2574,7 @@ dependencies = [ "dynamo-runtime", "dynamo-sidecar-common", "futures", + "lru", "opendal", "prost 0.13.5", "rand 0.9.4", @@ -5752,6 +5753,8 @@ checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" dependencies = [ "opendal-core", "opendal-http-transport-reqwest", + "opendal-layer-retry", + "opendal-layer-timeout", "opendal-service-fs", "opendal-service-s3", ] @@ -5796,6 +5799,27 @@ dependencies = [ "reqwest 0.13.4", ] +[[package]] +name = "opendal-layer-retry" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06306202c97c54fb41bdbbdcb854c8798823f0022ae7aac7a40c8a1023aa83a6" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd334cbd0a0bc934146733e74a80a5faf8db8e014781a25f6d9d80d7b87c981" +dependencies = [ + "opendal-core", + "tokio", +] + [[package]] name = "opendal-service-fs" version = "0.59.1" diff --git a/lib/sidecar/sglang/Cargo.toml b/lib/sidecar/sglang/Cargo.toml index d9a9986278d9..31ac7166cb04 100644 --- a/lib/sidecar/sglang/Cargo.toml +++ b/lib/sidecar/sglang/Cargo.toml @@ -29,8 +29,11 @@ async-stream = { workspace = true } async-trait = { workspace = true } clap = { version = "4", features = ["derive", "env"] } futures = { workspace = true } +lru = { workspace = true } opendal = { version = "0.59.1", default-features = false, features = [ "http-transport-reqwest", + "layers-retry", + "layers-timeout", "services-fs", "services-s3", ] } diff --git a/lib/sidecar/sglang/src/args.rs b/lib/sidecar/sglang/src/args.rs index 153bec1549a7..c6ed61560f9d 100644 --- a/lib/sidecar/sglang/src/args.rs +++ b/lib/sidecar/sglang/src/args.rs @@ -3,8 +3,78 @@ //! Command-line arguments for the SGLang sidecar. +use std::num::{NonZeroU64, NonZeroUsize}; + use dynamo_sidecar_common::SidecarArgs; +#[derive(clap::Args, Debug, Clone)] +pub struct MetadataUploadArgs { + /// Maximum number of OpenDAL operators retained by the metadata upload LRU. + #[arg( + long = "metadata-upload-cache-capacity", + env = "DYN_SGLANG_METADATA_UPLOAD_CACHE_CAPACITY", + default_value = "64" + )] + pub cache_capacity: NonZeroUsize, + + /// OpenDAL control-operation timeout in seconds. + #[arg( + long = "metadata-upload-timeout-secs", + env = "DYN_SGLANG_METADATA_UPLOAD_TIMEOUT_SECS", + default_value = "3" + )] + pub timeout_secs: NonZeroU64, + + /// Timeout in seconds for each OpenDAL I/O attempt. + #[arg( + long = "metadata-upload-io-timeout-secs", + env = "DYN_SGLANG_METADATA_UPLOAD_IO_TIMEOUT_SECS", + default_value = "3" + )] + pub io_timeout_secs: NonZeroU64, + + /// Maximum attempts for a retryable OpenDAL operation. + #[arg( + long = "metadata-upload-retry-max-times", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_TIMES", + default_value = "3" + )] + pub retry_max_times: NonZeroUsize, + + /// Initial exponential retry delay in milliseconds. + #[arg( + long = "metadata-upload-retry-min-delay-ms", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MIN_DELAY_MS", + default_value_t = 100 + )] + pub retry_min_delay_ms: u64, + + /// Maximum exponential retry delay in milliseconds. + #[arg( + long = "metadata-upload-retry-max-delay-ms", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_DELAY_MS", + default_value_t = 1_000 + )] + pub retry_max_delay_ms: u64, + + /// Exponential retry multiplier; must be at least 1.0. + #[arg( + long = "metadata-upload-retry-factor", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_FACTOR", + default_value_t = 2.0 + )] + pub retry_factor: f32, + + /// Add random jitter to exponential retry delays. + #[arg( + long = "metadata-upload-retry-jitter", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_JITTER", + default_value_t = true, + action = clap::ArgAction::Set + )] + pub retry_jitter: bool, +} + /// Parsed sidecar arguments. #[derive(clap::Parser, Debug, Clone)] #[command( @@ -22,4 +92,81 @@ pub struct Args { /// wildcard addresses. #[arg(long, env = "SGLANG_DISAGGREGATION_BOOTSTRAP_HOST")] pub bootstrap_host: Option, + + #[command(flatten)] + pub metadata_upload: MetadataUploadArgs, +} + +#[cfg(test)] +mod tests { + use clap::{CommandFactory, Parser}; + + use super::Args; + + #[test] + fn metadata_upload_policy_is_configurable_by_cli_and_env() { + let args = Args::try_parse_from([ + "dynamo-sglang-sidecar", + "--grpc-endpoint", + "127.0.0.1:30000", + "--metadata-upload-cache-capacity", + "8", + "--metadata-upload-timeout-secs", + "11", + "--metadata-upload-io-timeout-secs", + "12", + "--metadata-upload-retry-max-times", + "4", + "--metadata-upload-retry-min-delay-ms", + "20", + "--metadata-upload-retry-max-delay-ms", + "200", + "--metadata-upload-retry-factor", + "1.5", + "--metadata-upload-retry-jitter=false", + ]) + .unwrap(); + let policy = args.metadata_upload; + assert_eq!(policy.cache_capacity.get(), 8); + assert_eq!(policy.timeout_secs.get(), 11); + assert_eq!(policy.io_timeout_secs.get(), 12); + assert_eq!(policy.retry_max_times.get(), 4); + assert_eq!(policy.retry_min_delay_ms, 20); + assert_eq!(policy.retry_max_delay_ms, 200); + assert_eq!(policy.retry_factor, 1.5); + assert!(!policy.retry_jitter); + + let command = Args::command(); + for (id, env) in [ + ( + "cache_capacity", + "DYN_SGLANG_METADATA_UPLOAD_CACHE_CAPACITY", + ), + ("timeout_secs", "DYN_SGLANG_METADATA_UPLOAD_TIMEOUT_SECS"), + ( + "io_timeout_secs", + "DYN_SGLANG_METADATA_UPLOAD_IO_TIMEOUT_SECS", + ), + ( + "retry_max_times", + "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_TIMES", + ), + ( + "retry_min_delay_ms", + "DYN_SGLANG_METADATA_UPLOAD_RETRY_MIN_DELAY_MS", + ), + ( + "retry_max_delay_ms", + "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_DELAY_MS", + ), + ("retry_factor", "DYN_SGLANG_METADATA_UPLOAD_RETRY_FACTOR"), + ("retry_jitter", "DYN_SGLANG_METADATA_UPLOAD_RETRY_JITTER"), + ] { + let arg = command + .get_arguments() + .find(|arg| arg.get_id() == id) + .unwrap(); + assert_eq!(arg.get_env().unwrap(), env); + } + } } diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 78c9f708cefd..4ef5fc21c7f8 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -22,7 +22,7 @@ use tokio_util::sync::CancellationToken; use crate::args::Args; use crate::client::{self, Client, Discovery, Pool}; -use crate::metadata_upload::{MetadataUploader, grpc_metadata}; +use crate::metadata_upload::{MetadataUploader, OperatorCache, grpc_metadata}; use crate::native_http::{self, NativeHttp}; use crate::proto as pb; use crate::protocol::{ @@ -37,6 +37,7 @@ pub struct SglangSidecarEngine { bootstrap_host: Option, bootstrap_port: Option, metadata_upload_enabled: bool, + metadata_upload_operators: Arc, state: OnceCell, cancel: CancellationToken, } @@ -77,6 +78,7 @@ impl SglangSidecarEngine { "route-to-encoder is not supported by the SGLang sidecar", )); } + let metadata_upload_operators = Arc::new(OperatorCache::from_args(&args.metadata_upload)?); let endpoint = args.sidecar.grpc_endpoint; let transport = args.sidecar.grpc.config(); @@ -139,6 +141,7 @@ impl SglangSidecarEngine { bootstrap_host, bootstrap_port, metadata_upload_enabled, + metadata_upload_operators, state: OnceCell::new(), cancel: CancellationToken::new(), }, @@ -284,6 +287,7 @@ impl LLMEngine for SglangSidecarEngine { let metadata_uploader = MetadataUploader::from_request( &request, self.metadata_upload_enabled && !self.disaggregation_mode.is_prefill(), + self.metadata_upload_operators.clone(), ) .await?; let prefill_handoff = if self.disaggregation_mode.is_prefill() { diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs index a1a92a0b43f8..7a32679c699e 100644 --- a/lib/sidecar/sglang/src/metadata_upload.rs +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -5,21 +5,43 @@ use std::collections::HashMap; use std::io::Write; +use std::num::NonZeroUsize; +use std::sync::Arc; use std::time::Duration; use dynamo_backend_common::{DynamoError, PreprocessedRequest}; +use lru::LruCache; +use opendal::layers::{RetryLayer, TimeoutLayer}; use opendal::{Buffer, Operator}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use tokio::sync::Mutex; +use crate::args::MetadataUploadArgs; use crate::client; const OUTPUT_PATH: &str = "choice_0.msgpack.zst"; -const WRITE_TIMEOUT: Duration = Duration::from_secs(10); pub(crate) struct MetadataUploader { primary: Operator, fallback: Option, + operators: Arc, +} + +pub(crate) struct OperatorCache { + operators: Mutex>, + config: OperatorConfig, +} + +#[derive(Clone, Copy)] +struct OperatorConfig { + timeout: Duration, + io_timeout: Duration, + retry_max_times: usize, + retry_min_delay: Duration, + retry_max_delay: Duration, + retry_factor: f32, + retry_jitter: bool, } #[derive(Serialize)] @@ -35,10 +57,69 @@ struct MetadataUploadConfig { fallback_url: Option, } +impl OperatorCache { + pub(crate) fn from_args(args: &MetadataUploadArgs) -> Result { + if !args.retry_factor.is_finite() || args.retry_factor < 1.0 { + return Err(client::invalid_arg( + "metadata-upload-retry-factor must be finite and at least 1.0", + )); + } + if args.retry_max_delay_ms < args.retry_min_delay_ms { + return Err(client::invalid_arg( + "metadata-upload-retry-max-delay-ms must be greater than or equal to metadata-upload-retry-min-delay-ms", + )); + } + Ok(Self::new( + args.cache_capacity, + OperatorConfig { + timeout: Duration::from_secs(args.timeout_secs.get()), + io_timeout: Duration::from_secs(args.io_timeout_secs.get()), + retry_max_times: args.retry_max_times.get(), + retry_min_delay: Duration::from_millis(args.retry_min_delay_ms), + retry_max_delay: Duration::from_millis(args.retry_max_delay_ms), + retry_factor: args.retry_factor, + retry_jitter: args.retry_jitter, + }, + )) + } + + fn new(capacity: NonZeroUsize, config: OperatorConfig) -> Self { + // Ensure built-in services are registered when this crate is linked statically. + opendal::install_default(); + Self { + operators: Mutex::new(LruCache::new(capacity)), + config, + } + } + + async fn get(&self, raw: String, field: &'static str) -> Result { + let url = normalize_url(raw, field)?; + if let Some(operator) = self.operators.lock().await.get(&url).cloned() { + return Ok(operator); + } + + let build_url = url.clone(); + let config = self.config; + let operator = tokio::task::spawn_blocking(move || build_operator(&build_url, config)) + .await + .map_err(|error| { + client::protocol_error(format!("metadata upload setup task failed: {error}")) + })??; + + let mut operators = self.operators.lock().await; + if let Some(operator) = operators.get(&url).cloned() { + return Ok(operator); + } + operators.put(url, operator.clone()); + Ok(operator) + } +} + impl MetadataUploader { pub(crate) async fn from_request( request: &PreprocessedRequest, enabled: bool, + operators: Arc, ) -> Result, DynamoError> { if !enabled { return Ok(None); @@ -59,14 +140,16 @@ impl MetadataUploader { let config = MetadataUploadConfig::deserialize(config).map_err(|error| { client::invalid_arg(format!("invalid extra_args.nvext.metadata_upload: {error}")) })?; - // Ensure built-in services are registered when this crate is linked statically. - opendal::install_default(); - let primary = operator_from_url(config.url, "url").await?; + let primary = operators.get(config.url, "url").await?; let fallback = config .fallback_url .map(|url| normalize_url(url, "fallback_url")) .transpose()?; - Ok(Some(Self { primary, fallback })) + Ok(Some(Self { + primary, + fallback, + operators, + })) } pub(crate) async fn upload(&self, metadata: Value) -> Result<(), DynamoError> { @@ -86,7 +169,10 @@ impl MetadataUploader { return Err(primary_error); }; tracing::warn!(error = %primary_error, "primary metadata upload failed; attempting fallback"); - let fallback = operator_from_url(fallback_url.clone(), "fallback_url").await?; + let fallback = self + .operators + .get(fallback_url.clone(), "fallback_url") + .await?; write(&fallback, compressed).await } } @@ -103,10 +189,25 @@ fn normalize_url(raw: String, field: &str) -> Result { Ok(raw.to_string()) } -async fn operator_from_url(raw: String, field: &'static str) -> Result { - tokio::task::spawn_blocking(move || { - let raw = normalize_url(raw, field)?; - Operator::from_uri(raw.as_str()).map_err(|error| { +fn build_operator(raw: &str, config: OperatorConfig) -> Result { + let retry = RetryLayer::new() + .with_factor(config.retry_factor) + .with_min_delay(config.retry_min_delay) + .with_max_delay(config.retry_max_delay) + .with_max_times(config.retry_max_times); + let retry = if config.retry_jitter { + retry.with_jitter() + } else { + retry + }; + let timeout = TimeoutLayer::new() + .with_timeout(config.timeout) + .with_io_timeout(config.io_timeout); + Operator::from_uri(raw) + // Timeout must be inside RetryLayer so a timeout cannot drop a + // stateful retry future before OpenDAL restores its body state. + .map(|operator| operator.layer(timeout).layer(retry)) + .map_err(|error| { client::invalid_arg(format!( "could not configure `{}` metadata upload destination: {error}", raw.split_once(':') @@ -114,22 +215,14 @@ async fn operator_from_url(raw: String, field: &'static str) -> Result Result<(), DynamoError> { - match tokio::time::timeout(WRITE_TIMEOUT, operator.write(OUTPUT_PATH, data)).await { - Ok(Ok(_)) => Ok(()), - Ok(Err(error)) => Err(upload_error(operator, error)), - Err(_) => Err(client::protocol_error(format!( - "SGLang metadata upload to {} timed out after {WRITE_TIMEOUT:?}", - operator.info().root() - ))), - } + operator + .write(OUTPUT_PATH, data) + .await + .map(|_| ()) + .map_err(|error| upload_error(operator, error)) } fn upload_error(operator: &Operator, error: opendal::Error) -> DynamoError { @@ -191,12 +284,30 @@ pub(crate) fn grpc_metadata(meta: &HashMap) -> Value { #[cfg(test)] mod tests { use std::io::Cursor; + use std::num::NonZeroUsize; use std::path::Path; + use std::sync::Arc; + use std::time::Duration; use dynamo_backend_common::{OutputOptions, SamplingOptions, StopConditions}; use serde_json::{Value, json}; - use super::{MetadataUploader, OUTPUT_PATH, grpc_metadata}; + use super::{MetadataUploader, OUTPUT_PATH, OperatorCache, OperatorConfig, grpc_metadata}; + + fn operator_cache(capacity: usize) -> Arc { + Arc::new(OperatorCache::new( + NonZeroUsize::new(capacity).unwrap(), + OperatorConfig { + timeout: Duration::from_secs(10), + io_timeout: Duration::from_secs(10), + retry_max_times: 3, + retry_min_delay: Duration::from_millis(1), + retry_max_delay: Duration::from_millis(10), + retry_factor: 2.0, + retry_jitter: false, + }, + )) + } fn fs_uri(path: &Path) -> String { format!("fs://{}", path.display()) @@ -220,13 +331,13 @@ mod tests { "nvext": {"metadata_upload": {"url": "fs:///tmp/metadata"}} })); assert!( - MetadataUploader::from_request(&configured, false) + MetadataUploader::from_request(&configured, false, operator_cache(2)) .await .unwrap() .is_none() ); assert!( - MetadataUploader::from_request(&configured, true) + MetadataUploader::from_request(&configured, true, operator_cache(2)) .await .unwrap() .is_some() @@ -236,7 +347,7 @@ mod tests { "nvext": {"metadata_upload": {"url": "fs:///tmp", "format": "json"}} })); assert!( - MetadataUploader::from_request(&invalid, true) + MetadataUploader::from_request(&invalid, true, operator_cache(2)) .await .is_err() ); @@ -249,7 +360,7 @@ mod tests { let configured = request(json!({ "nvext": {"metadata_upload": {"url": url}} })); - let uploader = MetadataUploader::from_request(&configured, true) + let uploader = MetadataUploader::from_request(&configured, true, operator_cache(2)) .await .unwrap() .unwrap(); @@ -288,7 +399,7 @@ mod tests { "fallback_url": fs_uri(&fallback) }} })); - let uploader = MetadataUploader::from_request(&configured, true) + let uploader = MetadataUploader::from_request(&configured, true, operator_cache(2)) .await .unwrap() .unwrap(); @@ -307,10 +418,29 @@ mod tests { })); assert!( - MetadataUploader::from_request(&configured, true) + MetadataUploader::from_request(&configured, true, operator_cache(2)) .await .unwrap() .is_some() ); } + + #[tokio::test] + async fn operator_cache_evicts_least_recently_used_url() { + let directory = tempfile::tempdir().unwrap(); + let first = fs_uri(&directory.path().join("first")); + let second = fs_uri(&directory.path().join("second")); + let third = fs_uri(&directory.path().join("third")); + let cache = operator_cache(2); + + cache.get(first.clone(), "url").await.unwrap(); + cache.get(second.clone(), "url").await.unwrap(); + cache.get(first.clone(), "url").await.unwrap(); + cache.get(third.clone(), "url").await.unwrap(); + + let operators = cache.operators.lock().await; + assert!(operators.peek(&first).is_some()); + assert!(operators.peek(&second).is_none()); + assert!(operators.peek(&third).is_some()); + } } From fa1ba56707238afed52c94addbf87e9963f089ba Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Wed, 16 Sep 2026 07:09:32 +0000 Subject: [PATCH 08/11] fix(sglang): clarify operator cache setting Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- lib/sidecar/sglang/src/args.rs | 14 +++++++------- lib/sidecar/sglang/src/metadata_upload.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/sidecar/sglang/src/args.rs b/lib/sidecar/sglang/src/args.rs index c6ed61560f9d..a3c981be1e1e 100644 --- a/lib/sidecar/sglang/src/args.rs +++ b/lib/sidecar/sglang/src/args.rs @@ -11,11 +11,11 @@ use dynamo_sidecar_common::SidecarArgs; pub struct MetadataUploadArgs { /// Maximum number of OpenDAL operators retained by the metadata upload LRU. #[arg( - long = "metadata-upload-cache-capacity", - env = "DYN_SGLANG_METADATA_UPLOAD_CACHE_CAPACITY", + long = "metadata-upload-operator-cache-capacity", + env = "DYN_SGLANG_METADATA_UPLOAD_OPERATOR_CACHE_CAPACITY", default_value = "64" )] - pub cache_capacity: NonZeroUsize, + pub operator_cache_capacity: NonZeroUsize, /// OpenDAL control-operation timeout in seconds. #[arg( @@ -109,7 +109,7 @@ mod tests { "dynamo-sglang-sidecar", "--grpc-endpoint", "127.0.0.1:30000", - "--metadata-upload-cache-capacity", + "--metadata-upload-operator-cache-capacity", "8", "--metadata-upload-timeout-secs", "11", @@ -127,7 +127,7 @@ mod tests { ]) .unwrap(); let policy = args.metadata_upload; - assert_eq!(policy.cache_capacity.get(), 8); + assert_eq!(policy.operator_cache_capacity.get(), 8); assert_eq!(policy.timeout_secs.get(), 11); assert_eq!(policy.io_timeout_secs.get(), 12); assert_eq!(policy.retry_max_times.get(), 4); @@ -139,8 +139,8 @@ mod tests { let command = Args::command(); for (id, env) in [ ( - "cache_capacity", - "DYN_SGLANG_METADATA_UPLOAD_CACHE_CAPACITY", + "operator_cache_capacity", + "DYN_SGLANG_METADATA_UPLOAD_OPERATOR_CACHE_CAPACITY", ), ("timeout_secs", "DYN_SGLANG_METADATA_UPLOAD_TIMEOUT_SECS"), ( diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs index 7a32679c699e..871b6e210213 100644 --- a/lib/sidecar/sglang/src/metadata_upload.rs +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -70,7 +70,7 @@ impl OperatorCache { )); } Ok(Self::new( - args.cache_capacity, + args.operator_cache_capacity, OperatorConfig { timeout: Duration::from_secs(args.timeout_secs.get()), io_timeout: Duration::from_secs(args.io_timeout_secs.get()), From ffd7f9c1e73504078e2a894e8ef4b7e142c91996 Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Wed, 16 Sep 2026 07:34:15 +0000 Subject: [PATCH 09/11] refactor(sglang): harden metadata upload resilience Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- lib/sidecar/sglang/src/args.rs | 144 +-------- lib/sidecar/sglang/src/engine.rs | 79 ++--- lib/sidecar/sglang/src/metadata_upload.rs | 289 ++++++++++-------- .../sglang/src/metadata_upload/config.rs | 185 +++++++++++ .../src/metadata_upload/resilience_tests.rs | 285 +++++++++++++++++ 5 files changed, 666 insertions(+), 316 deletions(-) create mode 100644 lib/sidecar/sglang/src/metadata_upload/config.rs create mode 100644 lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs diff --git a/lib/sidecar/sglang/src/args.rs b/lib/sidecar/sglang/src/args.rs index a3c981be1e1e..705d13dd5191 100644 --- a/lib/sidecar/sglang/src/args.rs +++ b/lib/sidecar/sglang/src/args.rs @@ -3,77 +3,9 @@ //! Command-line arguments for the SGLang sidecar. -use std::num::{NonZeroU64, NonZeroUsize}; - use dynamo_sidecar_common::SidecarArgs; -#[derive(clap::Args, Debug, Clone)] -pub struct MetadataUploadArgs { - /// Maximum number of OpenDAL operators retained by the metadata upload LRU. - #[arg( - long = "metadata-upload-operator-cache-capacity", - env = "DYN_SGLANG_METADATA_UPLOAD_OPERATOR_CACHE_CAPACITY", - default_value = "64" - )] - pub operator_cache_capacity: NonZeroUsize, - - /// OpenDAL control-operation timeout in seconds. - #[arg( - long = "metadata-upload-timeout-secs", - env = "DYN_SGLANG_METADATA_UPLOAD_TIMEOUT_SECS", - default_value = "3" - )] - pub timeout_secs: NonZeroU64, - - /// Timeout in seconds for each OpenDAL I/O attempt. - #[arg( - long = "metadata-upload-io-timeout-secs", - env = "DYN_SGLANG_METADATA_UPLOAD_IO_TIMEOUT_SECS", - default_value = "3" - )] - pub io_timeout_secs: NonZeroU64, - - /// Maximum attempts for a retryable OpenDAL operation. - #[arg( - long = "metadata-upload-retry-max-times", - env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_TIMES", - default_value = "3" - )] - pub retry_max_times: NonZeroUsize, - - /// Initial exponential retry delay in milliseconds. - #[arg( - long = "metadata-upload-retry-min-delay-ms", - env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MIN_DELAY_MS", - default_value_t = 100 - )] - pub retry_min_delay_ms: u64, - - /// Maximum exponential retry delay in milliseconds. - #[arg( - long = "metadata-upload-retry-max-delay-ms", - env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_DELAY_MS", - default_value_t = 1_000 - )] - pub retry_max_delay_ms: u64, - - /// Exponential retry multiplier; must be at least 1.0. - #[arg( - long = "metadata-upload-retry-factor", - env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_FACTOR", - default_value_t = 2.0 - )] - pub retry_factor: f32, - - /// Add random jitter to exponential retry delays. - #[arg( - long = "metadata-upload-retry-jitter", - env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_JITTER", - default_value_t = true, - action = clap::ArgAction::Set - )] - pub retry_jitter: bool, -} +use crate::metadata_upload::MetadataUploadArgs; /// Parsed sidecar arguments. #[derive(clap::Parser, Debug, Clone)] @@ -96,77 +28,3 @@ pub struct Args { #[command(flatten)] pub metadata_upload: MetadataUploadArgs, } - -#[cfg(test)] -mod tests { - use clap::{CommandFactory, Parser}; - - use super::Args; - - #[test] - fn metadata_upload_policy_is_configurable_by_cli_and_env() { - let args = Args::try_parse_from([ - "dynamo-sglang-sidecar", - "--grpc-endpoint", - "127.0.0.1:30000", - "--metadata-upload-operator-cache-capacity", - "8", - "--metadata-upload-timeout-secs", - "11", - "--metadata-upload-io-timeout-secs", - "12", - "--metadata-upload-retry-max-times", - "4", - "--metadata-upload-retry-min-delay-ms", - "20", - "--metadata-upload-retry-max-delay-ms", - "200", - "--metadata-upload-retry-factor", - "1.5", - "--metadata-upload-retry-jitter=false", - ]) - .unwrap(); - let policy = args.metadata_upload; - assert_eq!(policy.operator_cache_capacity.get(), 8); - assert_eq!(policy.timeout_secs.get(), 11); - assert_eq!(policy.io_timeout_secs.get(), 12); - assert_eq!(policy.retry_max_times.get(), 4); - assert_eq!(policy.retry_min_delay_ms, 20); - assert_eq!(policy.retry_max_delay_ms, 200); - assert_eq!(policy.retry_factor, 1.5); - assert!(!policy.retry_jitter); - - let command = Args::command(); - for (id, env) in [ - ( - "operator_cache_capacity", - "DYN_SGLANG_METADATA_UPLOAD_OPERATOR_CACHE_CAPACITY", - ), - ("timeout_secs", "DYN_SGLANG_METADATA_UPLOAD_TIMEOUT_SECS"), - ( - "io_timeout_secs", - "DYN_SGLANG_METADATA_UPLOAD_IO_TIMEOUT_SECS", - ), - ( - "retry_max_times", - "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_TIMES", - ), - ( - "retry_min_delay_ms", - "DYN_SGLANG_METADATA_UPLOAD_RETRY_MIN_DELAY_MS", - ), - ( - "retry_max_delay_ms", - "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_DELAY_MS", - ), - ("retry_factor", "DYN_SGLANG_METADATA_UPLOAD_RETRY_FACTOR"), - ("retry_jitter", "DYN_SGLANG_METADATA_UPLOAD_RETRY_JITTER"), - ] { - let arg = command - .get_arguments() - .find(|arg| arg.get_id() == id) - .unwrap(); - assert_eq!(arg.get_env().unwrap(), env); - } - } -} diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 4ef5fc21c7f8..296670212c06 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -22,12 +22,12 @@ use tokio_util::sync::CancellationToken; use crate::args::Args; use crate::client::{self, Client, Discovery, Pool}; -use crate::metadata_upload::{MetadataUploader, OperatorCache, grpc_metadata}; +use crate::metadata_upload::{MetadataDelivery, MetadataUploadService, grpc_metadata}; use crate::native_http::{self, NativeHttp}; use crate::proto as pb; use crate::protocol::{ - build_generate_request, disaggregated_params_to_json, engine_data_from_meta, extract_logprobs, - meta_u32, output_ids_to_u32, terminal_from_meta, + build_generate_request, disaggregated_params_to_json, meta_u32, output_ids_to_u32, + terminal_from_meta, }; pub struct SglangSidecarEngine { @@ -36,8 +36,7 @@ pub struct SglangSidecarEngine { disaggregation_mode: DisaggregationMode, bootstrap_host: Option, bootstrap_port: Option, - metadata_upload_enabled: bool, - metadata_upload_operators: Arc, + metadata_upload: Option>, state: OnceCell, cancel: CancellationToken, } @@ -78,8 +77,6 @@ impl SglangSidecarEngine { "route-to-encoder is not supported by the SGLang sidecar", )); } - let metadata_upload_operators = Arc::new(OperatorCache::from_args(&args.metadata_upload)?); - let endpoint = args.sidecar.grpc_endpoint; let transport = args.sidecar.grpc.config(); let discovery = bootstrap_discover(&endpoint, &transport)?; @@ -106,7 +103,11 @@ impl SglangSidecarEngine { ); let common = args.sidecar.common; - let metadata_upload_enabled = common.enable_rl; + let metadata_upload = if common.enable_rl && !disaggregation_mode.is_prefill() { + Some(Arc::new(MetadataUploadService::new(args.metadata_upload)?)) + } else { + None + }; let config = WorkerConfig { namespace: common.namespace, component: if disaggregation_mode == DisaggregationMode::Aggregated { @@ -140,8 +141,7 @@ impl SglangSidecarEngine { disaggregation_mode, bootstrap_host, bootstrap_port, - metadata_upload_enabled, - metadata_upload_operators, + metadata_upload, state: OnceCell::new(), cancel: CancellationToken::new(), }, @@ -284,12 +284,10 @@ impl LLMEngine for SglangSidecarEngine { self.bootstrap_host.as_deref(), self.bootstrap_port, )?; - let metadata_uploader = MetadataUploader::from_request( - &request, - self.metadata_upload_enabled && !self.disaggregation_mode.is_prefill(), - self.metadata_upload_operators.clone(), - ) - .await?; + let metadata_delivery = match &self.metadata_upload { + Some(service) => service.delivery_for(&request).await?, + None => MetadataDelivery::Inline, + }; let prefill_handoff = if self.disaggregation_mode.is_prefill() { grpc_request .disaggregated_params @@ -366,18 +364,6 @@ impl LLMEngine for SglangSidecarEngine { break; } }; - let (log_probs, top_logprobs) = if metadata_uploader.is_some() { - (None, None) - } else { - match extract_logprobs(&response.meta_info, return_tokens_as_ids) { - Ok(values) => values, - Err(err) => { - yield Err(err); - break; - } - } - }; - if is_prefill { if response.finished { let mut terminal = match terminal_from_meta( @@ -398,6 +384,19 @@ impl LLMEngine for SglangSidecarEngine { continue; } + let ((log_probs, top_logprobs), engine_data) = match metadata_delivery + .response_fields( + &response.meta_info, + return_tokens_as_ids, + response.finished, + ) + { + Ok(values) => values, + Err(err) => { + yield Err(err); + break; + } + }; generated = generated.saturating_add(token_ids.len() as u32); if response.finished { let mut terminal = match terminal_from_meta( @@ -411,7 +410,7 @@ impl LLMEngine for SglangSidecarEngine { break; } }; - let engine_data = if let Some(uploader) = metadata_uploader.as_ref() { + if let Some(uploader) = metadata_delivery.uploader() { let uploaded = tokio::select! { biased; _ = ctx.stopped() => None, @@ -430,16 +429,7 @@ impl LLMEngine for SglangSidecarEngine { break; } } - None - } else { - match engine_data_from_meta(&response.meta_info, true) { - Ok(engine_data) => engine_data, - Err(error) => { - yield Err(error); - break; - } - } - }; + } terminal.token_ids = token_ids; terminal.log_probs = log_probs; terminal.top_logprobs = top_logprobs; @@ -449,17 +439,6 @@ impl LLMEngine for SglangSidecarEngine { } if !token_ids.is_empty() { - let engine_data = if metadata_uploader.is_some() { - None - } else { - match engine_data_from_meta(&response.meta_info, false) { - Ok(engine_data) => engine_data, - Err(error) => { - yield Err(error); - break; - } - } - }; yield Ok(LLMEngineOutput { token_ids, log_probs, diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs index 871b6e210213..52cdd2d84ad0 100644 --- a/lib/sidecar/sglang/src/metadata_upload.rs +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -5,9 +5,7 @@ use std::collections::HashMap; use std::io::Write; -use std::num::NonZeroUsize; use std::sync::Arc; -use std::time::Duration; use dynamo_backend_common::{DynamoError, PreprocessedRequest}; use lru::LruCache; @@ -15,13 +13,26 @@ use opendal::layers::{RetryLayer, TimeoutLayer}; use opendal::{Buffer, Operator}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, OnceCell}; -use crate::args::MetadataUploadArgs; use crate::client; +use crate::protocol::{ExtractedLogprobs, engine_data_from_meta, extract_logprobs}; + +mod config; +pub(crate) use config::MetadataUploadArgs; +use config::OperatorPolicy; const OUTPUT_PATH: &str = "choice_0.msgpack.zst"; +pub(crate) struct MetadataUploadService { + operators: Arc, +} + +pub(crate) enum MetadataDelivery { + Inline, + Upload(MetadataUploader), +} + pub(crate) struct MetadataUploader { primary: Operator, fallback: Option, @@ -29,19 +40,10 @@ pub(crate) struct MetadataUploader { } pub(crate) struct OperatorCache { - operators: Mutex>, - config: OperatorConfig, -} - -#[derive(Clone, Copy)] -struct OperatorConfig { - timeout: Duration, - io_timeout: Duration, - retry_max_times: usize, - retry_min_delay: Duration, - retry_max_delay: Duration, - retry_factor: f32, - retry_jitter: bool, + operators: Mutex>>>, + policy: OperatorPolicy, + #[cfg(test)] + build_count: std::sync::atomic::AtomicUsize, } #[derive(Serialize)] @@ -58,72 +60,70 @@ struct MetadataUploadConfig { } impl OperatorCache { - pub(crate) fn from_args(args: &MetadataUploadArgs) -> Result { - if !args.retry_factor.is_finite() || args.retry_factor < 1.0 { - return Err(client::invalid_arg( - "metadata-upload-retry-factor must be finite and at least 1.0", - )); - } - if args.retry_max_delay_ms < args.retry_min_delay_ms { - return Err(client::invalid_arg( - "metadata-upload-retry-max-delay-ms must be greater than or equal to metadata-upload-retry-min-delay-ms", - )); - } - Ok(Self::new( - args.operator_cache_capacity, - OperatorConfig { - timeout: Duration::from_secs(args.timeout_secs.get()), - io_timeout: Duration::from_secs(args.io_timeout_secs.get()), - retry_max_times: args.retry_max_times.get(), - retry_min_delay: Duration::from_millis(args.retry_min_delay_ms), - retry_max_delay: Duration::from_millis(args.retry_max_delay_ms), - retry_factor: args.retry_factor, - retry_jitter: args.retry_jitter, - }, - )) - } - - fn new(capacity: NonZeroUsize, config: OperatorConfig) -> Self { + fn new(policy: OperatorPolicy) -> Self { // Ensure built-in services are registered when this crate is linked statically. opendal::install_default(); Self { - operators: Mutex::new(LruCache::new(capacity)), - config, + operators: Mutex::new(LruCache::new(policy.capacity)), + policy, + #[cfg(test)] + build_count: std::sync::atomic::AtomicUsize::new(0), } } async fn get(&self, raw: String, field: &'static str) -> Result { let url = normalize_url(raw, field)?; - if let Some(operator) = self.operators.lock().await.get(&url).cloned() { - return Ok(operator); - } + let cell = { + let mut operators = self.operators.lock().await; + if let Some(cell) = operators.get(&url) { + cell.clone() + } else { + let cell = Arc::new(OnceCell::new()); + operators.put(url.clone(), cell.clone()); + cell + } + }; + let policy = self.policy; + let operator = cell + .get_or_try_init(|| async { + #[cfg(test)] + self.build_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let build_url = url.clone(); + tokio::task::spawn_blocking(move || build_operator(&build_url, policy)) + .await + .map_err(|error| { + client::protocol_error(format!( + "metadata upload setup task failed: {error}" + )) + })? + }) + .await?; + Ok(operator.clone()) + } - let build_url = url.clone(); - let config = self.config; - let operator = tokio::task::spawn_blocking(move || build_operator(&build_url, config)) + #[cfg(test)] + async fn insert(&self, url: &str, operator: Operator) { + let cell = OnceCell::new(); + cell.set(operator).expect("new OnceCell must be empty"); + self.operators + .lock() .await - .map_err(|error| { - client::protocol_error(format!("metadata upload setup task failed: {error}")) - })??; - - let mut operators = self.operators.lock().await; - if let Some(operator) = operators.get(&url).cloned() { - return Ok(operator); - } - operators.put(url, operator.clone()); - Ok(operator) + .put(url.to_string(), Arc::new(cell)); } } -impl MetadataUploader { - pub(crate) async fn from_request( +impl MetadataUploadService { + pub(crate) fn new(args: MetadataUploadArgs) -> Result { + Ok(Self { + operators: Arc::new(OperatorCache::new(args.try_into()?)), + }) + } + + pub(crate) async fn delivery_for( + &self, request: &PreprocessedRequest, - enabled: bool, - operators: Arc, - ) -> Result, DynamoError> { - if !enabled { - return Ok(None); - } + ) -> Result { let Some(config) = request .extra_args .as_ref() @@ -132,26 +132,52 @@ impl MetadataUploader { .and_then(Value::as_object) .and_then(|nvext| nvext.get("metadata_upload")) else { - return Ok(None); + return Ok(MetadataDelivery::Inline); }; if config.is_null() { - return Ok(None); + return Ok(MetadataDelivery::Inline); } let config = MetadataUploadConfig::deserialize(config).map_err(|error| { client::invalid_arg(format!("invalid extra_args.nvext.metadata_upload: {error}")) })?; - let primary = operators.get(config.url, "url").await?; + let primary = self.operators.get(config.url, "url").await?; let fallback = config .fallback_url .map(|url| normalize_url(url, "fallback_url")) .transpose()?; - Ok(Some(Self { + Ok(MetadataDelivery::Upload(MetadataUploader { primary, fallback, - operators, + operators: self.operators.clone(), })) } +} + +impl MetadataDelivery { + pub(crate) fn response_fields( + &self, + meta: &HashMap, + return_tokens_as_ids: bool, + terminal: bool, + ) -> Result<(ExtractedLogprobs, Option), DynamoError> { + match self { + Self::Inline => Ok(( + extract_logprobs(meta, return_tokens_as_ids)?, + engine_data_from_meta(meta, terminal)?, + )), + Self::Upload(_) => Ok(((None, None), None)), + } + } + pub(crate) fn uploader(&self) -> Option<&MetadataUploader> { + match self { + Self::Inline => None, + Self::Upload(uploader) => Some(uploader), + } + } +} + +impl MetadataUploader { pub(crate) async fn upload(&self, metadata: Value) -> Result<(), DynamoError> { if metadata.as_object().is_some_and(|value| value.is_empty()) { return Ok(()); @@ -189,24 +215,9 @@ fn normalize_url(raw: String, field: &str) -> Result { Ok(raw.to_string()) } -fn build_operator(raw: &str, config: OperatorConfig) -> Result { - let retry = RetryLayer::new() - .with_factor(config.retry_factor) - .with_min_delay(config.retry_min_delay) - .with_max_delay(config.retry_max_delay) - .with_max_times(config.retry_max_times); - let retry = if config.retry_jitter { - retry.with_jitter() - } else { - retry - }; - let timeout = TimeoutLayer::new() - .with_timeout(config.timeout) - .with_io_timeout(config.io_timeout); +fn build_operator(raw: &str, policy: OperatorPolicy) -> Result { Operator::from_uri(raw) - // Timeout must be inside RetryLayer so a timeout cannot drop a - // stateful retry future before OpenDAL restores its body state. - .map(|operator| operator.layer(timeout).layer(retry)) + .map(|operator| configure_operator(operator, policy)) .map_err(|error| { client::invalid_arg(format!( "could not configure `{}` metadata upload destination: {error}", @@ -217,6 +228,25 @@ fn build_operator(raw: &str, config: OperatorConfig) -> Result Operator { + let retry = RetryLayer::new() + .with_factor(policy.retry_factor) + .with_min_delay(policy.retry_min_delay) + .with_max_delay(policy.retry_max_delay) + .with_max_times(policy.retry_max_times); + let retry = if policy.retry_jitter { + retry.with_jitter() + } else { + retry + }; + let timeout = TimeoutLayer::new() + .with_timeout(policy.timeout) + .with_io_timeout(policy.io_timeout); + // Timeout must be inside RetryLayer so a timeout cannot drop a stateful + // retry future before OpenDAL restores its body state. + operator.layer(timeout).layer(retry) +} + async fn write(operator: &Operator, data: Buffer) -> Result<(), DynamoError> { operator .write(OUTPUT_PATH, data) @@ -292,21 +322,30 @@ mod tests { use dynamo_backend_common::{OutputOptions, SamplingOptions, StopConditions}; use serde_json::{Value, json}; - use super::{MetadataUploader, OUTPUT_PATH, OperatorCache, OperatorConfig, grpc_metadata}; + use super::config::OperatorPolicy; + use super::{MetadataUploadService, OUTPUT_PATH, OperatorCache, grpc_metadata}; + + fn policy(capacity: usize) -> OperatorPolicy { + OperatorPolicy { + capacity: NonZeroUsize::new(capacity).unwrap(), + timeout: Duration::from_secs(10), + io_timeout: Duration::from_secs(10), + retry_max_times: 3, + retry_min_delay: Duration::from_millis(1), + retry_max_delay: Duration::from_millis(10), + retry_factor: 2.0, + retry_jitter: false, + } + } fn operator_cache(capacity: usize) -> Arc { - Arc::new(OperatorCache::new( - NonZeroUsize::new(capacity).unwrap(), - OperatorConfig { - timeout: Duration::from_secs(10), - io_timeout: Duration::from_secs(10), - retry_max_times: 3, - retry_min_delay: Duration::from_millis(1), - retry_max_delay: Duration::from_millis(10), - retry_factor: 2.0, - retry_jitter: false, - }, - )) + Arc::new(OperatorCache::new(policy(capacity))) + } + + fn service(capacity: usize) -> MetadataUploadService { + MetadataUploadService { + operators: operator_cache(capacity), + } } fn fs_uri(path: &Path) -> String { @@ -326,31 +365,33 @@ mod tests { } #[tokio::test] - async fn metadata_upload_is_rl_gated_and_strict() { + async fn request_configuration_is_optional_and_strict() { let configured = request(json!({ "nvext": {"metadata_upload": {"url": "fs:///tmp/metadata"}} })); assert!( - MetadataUploader::from_request(&configured, false, operator_cache(2)) + service(2) + .delivery_for(&configured) .await .unwrap() - .is_none() + .uploader() + .is_some() ); + + let unconfigured = request(json!({"nvext": {}})); assert!( - MetadataUploader::from_request(&configured, true, operator_cache(2)) + service(2) + .delivery_for(&unconfigured) .await .unwrap() - .is_some() + .uploader() + .is_none() ); let invalid = request(json!({ "nvext": {"metadata_upload": {"url": "fs:///tmp", "format": "json"}} })); - assert!( - MetadataUploader::from_request(&invalid, true, operator_cache(2)) - .await - .is_err() - ); + assert!(service(2).delivery_for(&invalid).await.is_err()); } #[tokio::test] @@ -360,10 +401,8 @@ mod tests { let configured = request(json!({ "nvext": {"metadata_upload": {"url": url}} })); - let uploader = MetadataUploader::from_request(&configured, true, operator_cache(2)) - .await - .unwrap() - .unwrap(); + let delivery = service(2).delivery_for(&configured).await.unwrap(); + let uploader = delivery.uploader().unwrap(); uploader .upload(grpc_metadata(&std::collections::HashMap::from([ ("id".into(), "sglang-1".into()), @@ -399,10 +438,8 @@ mod tests { "fallback_url": fs_uri(&fallback) }} })); - let uploader = MetadataUploader::from_request(&configured, true, operator_cache(2)) - .await - .unwrap() - .unwrap(); + let delivery = service(2).delivery_for(&configured).await.unwrap(); + let uploader = delivery.uploader().unwrap(); uploader.upload(json!({"id": "sglang-2"})).await.unwrap(); @@ -418,9 +455,11 @@ mod tests { })); assert!( - MetadataUploader::from_request(&configured, true, operator_cache(2)) + service(2) + .delivery_for(&configured) .await .unwrap() + .uploader() .is_some() ); } @@ -444,3 +483,7 @@ mod tests { assert!(operators.peek(&third).is_some()); } } + +#[cfg(test)] +#[path = "metadata_upload/resilience_tests.rs"] +mod resilience_tests; diff --git a/lib/sidecar/sglang/src/metadata_upload/config.rs b/lib/sidecar/sglang/src/metadata_upload/config.rs new file mode 100644 index 000000000000..d3ee1e23b1e9 --- /dev/null +++ b/lib/sidecar/sglang/src/metadata_upload/config.rs @@ -0,0 +1,185 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::num::{NonZeroU64, NonZeroUsize}; +use std::time::Duration; + +use dynamo_backend_common::DynamoError; + +use crate::client; + +#[derive(clap::Args, Debug, Clone)] +pub struct MetadataUploadArgs { + /// Maximum number of OpenDAL operators retained by the metadata upload LRU. + #[arg( + long = "metadata-upload-operator-cache-capacity", + env = "DYN_SGLANG_METADATA_UPLOAD_OPERATOR_CACHE_CAPACITY", + default_value = "64" + )] + pub operator_cache_capacity: NonZeroUsize, + + /// OpenDAL control-operation timeout in seconds. + #[arg( + long = "metadata-upload-timeout-secs", + env = "DYN_SGLANG_METADATA_UPLOAD_TIMEOUT_SECS", + default_value = "3" + )] + pub timeout_secs: NonZeroU64, + + /// Timeout in seconds for each OpenDAL I/O attempt. + #[arg( + long = "metadata-upload-io-timeout-secs", + env = "DYN_SGLANG_METADATA_UPLOAD_IO_TIMEOUT_SECS", + default_value = "3" + )] + pub io_timeout_secs: NonZeroU64, + + /// Maximum retries after the initial OpenDAL operation attempt. + #[arg( + long = "metadata-upload-retry-max-times", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_TIMES", + default_value = "3" + )] + pub retry_max_times: NonZeroUsize, + + /// Initial exponential retry delay in milliseconds. + #[arg( + long = "metadata-upload-retry-min-delay-ms", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MIN_DELAY_MS", + default_value_t = 100 + )] + pub retry_min_delay_ms: u64, + + /// Maximum exponential retry delay in milliseconds. + #[arg( + long = "metadata-upload-retry-max-delay-ms", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_DELAY_MS", + default_value_t = 1_000 + )] + pub retry_max_delay_ms: u64, + + /// Exponential retry multiplier; must be at least 1.0. + #[arg( + long = "metadata-upload-retry-factor", + env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_FACTOR", + default_value_t = 2.0 + )] + pub retry_factor: f32, + + /// Add random jitter to exponential retry delays. + #[arg(long = "metadata-upload-retry-jitter", env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_JITTER", default_value_t = true, action = clap::ArgAction::Set)] + pub retry_jitter: bool, +} + +#[derive(Clone, Copy)] +pub(super) struct OperatorPolicy { + pub capacity: NonZeroUsize, + pub timeout: Duration, + pub io_timeout: Duration, + pub retry_max_times: usize, + pub retry_min_delay: Duration, + pub retry_max_delay: Duration, + pub retry_factor: f32, + pub retry_jitter: bool, +} + +impl TryFrom for OperatorPolicy { + type Error = DynamoError; + + fn try_from(args: MetadataUploadArgs) -> Result { + if !args.retry_factor.is_finite() || args.retry_factor < 1.0 { + return Err(client::invalid_arg( + "metadata-upload-retry-factor must be finite and at least 1.0", + )); + } + if args.retry_max_delay_ms < args.retry_min_delay_ms { + return Err(client::invalid_arg( + "metadata-upload-retry-max-delay-ms must be greater than or equal to metadata-upload-retry-min-delay-ms", + )); + } + Ok(Self { + capacity: args.operator_cache_capacity, + timeout: Duration::from_secs(args.timeout_secs.get()), + io_timeout: Duration::from_secs(args.io_timeout_secs.get()), + retry_max_times: args.retry_max_times.get(), + retry_min_delay: Duration::from_millis(args.retry_min_delay_ms), + retry_max_delay: Duration::from_millis(args.retry_max_delay_ms), + retry_factor: args.retry_factor, + retry_jitter: args.retry_jitter, + }) + } +} + +#[cfg(test)] +mod tests { + use clap::{CommandFactory, Parser}; + + use crate::args::Args; + + #[test] + fn policy_is_configurable_by_cli_and_env() { + let args = Args::try_parse_from([ + "dynamo-sglang-sidecar", + "--grpc-endpoint", + "127.0.0.1:30000", + "--metadata-upload-operator-cache-capacity", + "8", + "--metadata-upload-timeout-secs", + "11", + "--metadata-upload-io-timeout-secs", + "12", + "--metadata-upload-retry-max-times", + "4", + "--metadata-upload-retry-min-delay-ms", + "20", + "--metadata-upload-retry-max-delay-ms", + "200", + "--metadata-upload-retry-factor", + "1.5", + "--metadata-upload-retry-jitter=false", + ]) + .unwrap(); + let policy = args.metadata_upload; + assert_eq!(policy.operator_cache_capacity.get(), 8); + assert_eq!(policy.timeout_secs.get(), 11); + assert_eq!(policy.io_timeout_secs.get(), 12); + assert_eq!(policy.retry_max_times.get(), 4); + assert_eq!(policy.retry_min_delay_ms, 20); + assert_eq!(policy.retry_max_delay_ms, 200); + assert_eq!(policy.retry_factor, 1.5); + assert!(!policy.retry_jitter); + + let command = Args::command(); + for (id, env) in [ + ( + "operator_cache_capacity", + "DYN_SGLANG_METADATA_UPLOAD_OPERATOR_CACHE_CAPACITY", + ), + ("timeout_secs", "DYN_SGLANG_METADATA_UPLOAD_TIMEOUT_SECS"), + ( + "io_timeout_secs", + "DYN_SGLANG_METADATA_UPLOAD_IO_TIMEOUT_SECS", + ), + ( + "retry_max_times", + "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_TIMES", + ), + ( + "retry_min_delay_ms", + "DYN_SGLANG_METADATA_UPLOAD_RETRY_MIN_DELAY_MS", + ), + ( + "retry_max_delay_ms", + "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_DELAY_MS", + ), + ("retry_factor", "DYN_SGLANG_METADATA_UPLOAD_RETRY_FACTOR"), + ("retry_jitter", "DYN_SGLANG_METADATA_UPLOAD_RETRY_JITTER"), + ] { + let arg = command + .get_arguments() + .find(|arg| arg.get_id() == id) + .unwrap(); + assert_eq!(arg.get_env().unwrap(), env); + } + } +} diff --git a/lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs b/lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs new file mode 100644 index 000000000000..dfb6126d7afe --- /dev/null +++ b/lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs @@ -0,0 +1,285 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use dynamo_backend_common::{OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions}; +use opendal::raw::*; +use opendal::{ + Buffer, Builder, Capability, Error, ErrorKind, Metadata, MetadataBuilder, OperationContext, + Operator, Result, +}; +use serde_json::json; + +use super::config::OperatorPolicy; +use super::{MetadataUploadService, OperatorCache, configure_operator}; + +fn policy(capacity: usize) -> OperatorPolicy { + OperatorPolicy { + capacity: NonZeroUsize::new(capacity).unwrap(), + timeout: Duration::from_secs(10), + io_timeout: Duration::from_secs(10), + retry_max_times: 3, + retry_min_delay: Duration::from_millis(1), + retry_max_delay: Duration::from_millis(10), + retry_factor: 2.0, + retry_jitter: false, + } +} + +fn operator_cache(capacity: usize) -> Arc { + Arc::new(OperatorCache::new(policy(capacity))) +} + +fn request() -> PreprocessedRequest { + PreprocessedRequest::builder() + .model("model".to_string()) + .token_ids(vec![1]) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .stop_conditions(StopConditions::default()) + .extra_args(Some(json!({ + "nvext": {"metadata_upload": { + "url": "test://primary", + "fallback_url": "test://fallback" + }} + }))) + .build() + .unwrap() +} + +#[tokio::test] +async fn concurrent_cache_misses_build_one_operator() { + let directory = tempfile::tempdir().unwrap(); + let url = format!("fs://{}", directory.path().display()); + let cache = operator_cache(2); + let mut tasks = Vec::new(); + for _ in 0..32 { + let cache = cache.clone(); + let url = url.clone(); + tasks.push(tokio::spawn(async move { cache.get(url, "url").await })); + } + for task in tasks { + task.await.unwrap().unwrap(); + } + assert_eq!(cache.build_count.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn retries_temporary_errors_until_success() { + let (operator, attempts) = test_operator(WriteBehavior::FailUntil(2), policy(2)); + + operator + .write("metadata", Buffer::from("data")) + .await + .unwrap(); + + assert_eq!(attempts.load(Ordering::Relaxed), 3); +} + +#[tokio::test] +async fn stops_after_configured_retry_limit() { + let mut retry_policy = policy(2); + retry_policy.retry_max_times = 2; + let (operator, attempts) = test_operator(WriteBehavior::FailUntil(usize::MAX), retry_policy); + + assert!( + operator + .write("metadata", Buffer::from("data")) + .await + .is_err() + ); + assert_eq!(attempts.load(Ordering::Relaxed), 3); +} + +#[tokio::test] +async fn timeout_exhausts_retries_before_fallback() { + let cache = operator_cache(2); + let mut timeout_policy = policy(2); + timeout_policy.io_timeout = Duration::from_millis(5); + timeout_policy.retry_max_times = 1; + let (primary, primary_attempts) = test_operator( + WriteBehavior::Sleep(Duration::from_millis(50)), + timeout_policy, + ); + let (fallback, fallback_attempts) = test_operator(WriteBehavior::Success, policy(2)); + cache.insert("test://primary", primary).await; + cache.insert("test://fallback", fallback).await; + let service = MetadataUploadService { operators: cache }; + let delivery = service.delivery_for(&request()).await.unwrap(); + + delivery + .uploader() + .unwrap() + .upload(json!({"id": "timeout"})) + .await + .unwrap(); + + assert_eq!(primary_attempts.load(Ordering::Relaxed), 2); + assert_eq!(fallback_attempts.load(Ordering::Relaxed), 1); +} + +#[derive(Clone, Copy, Debug)] +enum WriteBehavior { + Success, + FailUntil(usize), + Sleep(Duration), +} + +#[derive(Clone, Debug)] +struct TestBuilder { + behavior: WriteBehavior, + attempts: Arc, +} + +impl Default for TestBuilder { + fn default() -> Self { + Self { + behavior: WriteBehavior::Success, + attempts: Arc::new(AtomicUsize::new(0)), + } + } +} + +impl Builder for TestBuilder { + type Config = (); + + fn build(self) -> Result { + Ok(TestService { + behavior: self.behavior, + attempts: self.attempts, + }) + } +} + +#[derive(Clone, Debug)] +struct TestService { + behavior: WriteBehavior, + attempts: Arc, +} + +impl Service for TestService { + type Reader = (); + type Writer = TestWriter; + type Lister = (); + type Deleter = (); + type Copier = (); + type Composer = (); + + fn info(&self) -> ServiceInfo { + ServiceInfo::with_scheme("metadata-test") + } + + fn capability(&self) -> Capability { + Capability { + write: true, + write_can_multi: true, + ..Default::default() + } + } + + async fn create_dir( + &self, + _: &OperationContext, + _: &str, + _: OpCreateDir, + ) -> Result { + unsupported() + } + + async fn stat(&self, _: &OperationContext, _: &str, _: OpStat) -> Result { + unsupported() + } + + fn read(&self, _: &OperationContext, _: &str, _: OpRead) -> Result { + unsupported() + } + + fn write(&self, _: &OperationContext, _: &str, _: OpWrite) -> Result { + Ok(TestWriter { + behavior: self.behavior, + attempts: self.attempts.clone(), + size: 0, + }) + } + + fn delete(&self, _: &OperationContext) -> Result { + unsupported() + } + + fn list(&self, _: &OperationContext, _: &str, _: OpList) -> Result { + unsupported() + } + + fn copy(&self, _: &OperationContext, _: &str, _: &str, _: OpCopy) -> Result { + unsupported() + } + + async fn rename( + &self, + _: &OperationContext, + _: &str, + _: &str, + _: OpRename, + ) -> Result { + unsupported() + } + + async fn presign(&self, _: &OperationContext, _: &str, _: OpPresign) -> Result { + unsupported() + } +} + +#[derive(Debug)] +struct TestWriter { + behavior: WriteBehavior, + attempts: Arc, + size: usize, +} + +impl oio::Write for TestWriter { + async fn write(&mut self, data: Buffer) -> Result<()> { + self.size += data.len(); + Ok(()) + } + + async fn close(&mut self) -> Result { + let attempt = self.attempts.fetch_add(1, Ordering::Relaxed) + 1; + match self.behavior { + WriteBehavior::Success => Ok(file_metadata(self.size)), + WriteBehavior::FailUntil(limit) if attempt <= limit => { + Err(Error::new(ErrorKind::Unexpected, "injected temporary failure").set_temporary()) + } + WriteBehavior::FailUntil(_) => Ok(file_metadata(self.size)), + WriteBehavior::Sleep(duration) => { + tokio::time::sleep(duration).await; + Ok(file_metadata(self.size)) + } + } + } + + async fn abort(&mut self) -> Result<()> { + Ok(()) + } +} + +fn test_operator(behavior: WriteBehavior, policy: OperatorPolicy) -> (Operator, Arc) { + let attempts = Arc::new(AtomicUsize::new(0)); + let operator = Operator::new(TestBuilder { + behavior, + attempts: attempts.clone(), + }) + .unwrap(); + (configure_operator(operator, policy), attempts) +} + +fn unsupported() -> Result { + Err(Error::new(ErrorKind::Unsupported, "not used by this test")) +} + +fn file_metadata(size: usize) -> Metadata { + MetadataBuilder::file(size as u64).build() +} From 47ec91869f9ff67ac0d84419d42803148368e20f Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Wed, 16 Sep 2026 07:50:08 +0000 Subject: [PATCH 10/11] refactor(sglang): simplify metadata delivery Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- lib/sidecar/sglang/src/engine.rs | 67 ++++--- lib/sidecar/sglang/src/metadata_upload.rs | 176 +++++++++++------- .../sglang/src/metadata_upload/config.rs | 20 +- .../src/metadata_upload/resilience_tests.rs | 53 ++++-- 4 files changed, 193 insertions(+), 123 deletions(-) diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 296670212c06..066e73024113 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -22,7 +22,7 @@ use tokio_util::sync::CancellationToken; use crate::args::Args; use crate::client::{self, Client, Discovery, Pool}; -use crate::metadata_upload::{MetadataDelivery, MetadataUploadService, grpc_metadata}; +use crate::metadata_upload::{MetadataDelivery, MetadataUploadService}; use crate::native_http::{self, NativeHttp}; use crate::proto as pb; use crate::protocol::{ @@ -36,7 +36,7 @@ pub struct SglangSidecarEngine { disaggregation_mode: DisaggregationMode, bootstrap_host: Option, bootstrap_port: Option, - metadata_upload: Option>, + metadata_upload: Option, state: OnceCell, cancel: CancellationToken, } @@ -104,7 +104,7 @@ impl SglangSidecarEngine { let common = args.sidecar.common; let metadata_upload = if common.enable_rl && !disaggregation_mode.is_prefill() { - Some(Arc::new(MetadataUploadService::new(args.metadata_upload)?)) + Some(MetadataUploadService::new(args.metadata_upload)?) } else { None }; @@ -384,19 +384,6 @@ impl LLMEngine for SglangSidecarEngine { continue; } - let ((log_probs, top_logprobs), engine_data) = match metadata_delivery - .response_fields( - &response.meta_info, - return_tokens_as_ids, - response.finished, - ) - { - Ok(values) => values, - Err(err) => { - yield Err(err); - break; - } - }; generated = generated.saturating_add(token_ids.len() as u32); if response.finished { let mut terminal = match terminal_from_meta( @@ -410,26 +397,27 @@ impl LLMEngine for SglangSidecarEngine { break; } }; - if let Some(uploader) = metadata_delivery.uploader() { - let uploaded = tokio::select! { - biased; - _ = ctx.stopped() => None, - _ = cancel.cancelled() => None, - result = uploader.upload(grpc_metadata(&response.meta_info)) => Some(result), - }; - match uploaded { - Some(Ok(())) => {} - Some(Err(error)) => { - yield Err(error); - break; - } - None => { - yield Ok(LLMEngineOutput::cancelled() - .with_usage(usage(observed_prompt_tokens, generated))); - break; - } + let fields = tokio::select! { + biased; + _ = ctx.stopped() => None, + _ = cancel.cancelled() => None, + result = metadata_delivery.finish( + &response.meta_info, + return_tokens_as_ids, + ) => Some(result), + }; + let ((log_probs, top_logprobs), engine_data) = match fields { + Some(Ok(fields)) => fields, + Some(Err(error)) => { + yield Err(error); + break; } - } + None => { + yield Ok(LLMEngineOutput::cancelled() + .with_usage(usage(observed_prompt_tokens, generated))); + break; + } + }; terminal.token_ids = token_ids; terminal.log_probs = log_probs; terminal.top_logprobs = top_logprobs; @@ -439,6 +427,15 @@ impl LLMEngine for SglangSidecarEngine { } if !token_ids.is_empty() { + let ((log_probs, top_logprobs), engine_data) = match metadata_delivery + .chunk_fields(&response.meta_info, return_tokens_as_ids) + { + Ok(fields) => fields, + Err(error) => { + yield Err(error); + break; + } + }; yield Ok(LLMEngineOutput { token_ids, log_probs, diff --git a/lib/sidecar/sglang/src/metadata_upload.rs b/lib/sidecar/sglang/src/metadata_upload.rs index 52cdd2d84ad0..b02a7bcb0346 100644 --- a/lib/sidecar/sglang/src/metadata_upload.rs +++ b/lib/sidecar/sglang/src/metadata_upload.rs @@ -23,24 +23,25 @@ pub(crate) use config::MetadataUploadArgs; use config::OperatorPolicy; const OUTPUT_PATH: &str = "choice_0.msgpack.zst"; +type CachedOperator = Arc>>; +type ResponseFields = (ExtractedLogprobs, Option); pub(crate) struct MetadataUploadService { - operators: Arc, + operators: OperatorCache, } pub(crate) enum MetadataDelivery { Inline, - Upload(MetadataUploader), + Upload(Box), } pub(crate) struct MetadataUploader { primary: Operator, - fallback: Option, - operators: Arc, + fallback: Option, } pub(crate) struct OperatorCache { - operators: Mutex>>>, + operators: Mutex>, policy: OperatorPolicy, #[cfg(test)] build_count: std::sync::atomic::AtomicUsize, @@ -85,7 +86,7 @@ impl OperatorCache { }; let policy = self.policy; let operator = cell - .get_or_try_init(|| async { + .get_or_init(|| async { #[cfg(test)] self.build_count .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -98,14 +99,14 @@ impl OperatorCache { )) })? }) - .await?; - Ok(operator.clone()) + .await; + operator.clone() } #[cfg(test)] async fn insert(&self, url: &str, operator: Operator) { let cell = OnceCell::new(); - cell.set(operator).expect("new OnceCell must be empty"); + cell.set(Ok(operator)).expect("new OnceCell must be empty"); self.operators .lock() .await @@ -116,7 +117,7 @@ impl OperatorCache { impl MetadataUploadService { pub(crate) fn new(args: MetadataUploadArgs) -> Result { Ok(Self { - operators: Arc::new(OperatorCache::new(args.try_into()?)), + operators: OperatorCache::new(args.try_into()?), }) } @@ -140,39 +141,52 @@ impl MetadataUploadService { let config = MetadataUploadConfig::deserialize(config).map_err(|error| { client::invalid_arg(format!("invalid extra_args.nvext.metadata_upload: {error}")) })?; - let primary = self.operators.get(config.url, "url").await?; - let fallback = config - .fallback_url - .map(|url| normalize_url(url, "fallback_url")) - .transpose()?; - Ok(MetadataDelivery::Upload(MetadataUploader { + let primary = self.operators.get(config.url, "url"); + let fallback = async { + match config.fallback_url { + Some(url) => self.operators.get(url, "fallback_url").await.map(Some), + None => Ok(None), + } + }; + let (primary, fallback) = tokio::join!(primary, fallback); + let primary = primary?; + let fallback = fallback?; + Ok(MetadataDelivery::Upload(Box::new(MetadataUploader { primary, fallback, - operators: self.operators.clone(), - })) + }))) } } impl MetadataDelivery { - pub(crate) fn response_fields( + pub(crate) fn chunk_fields( &self, meta: &HashMap, return_tokens_as_ids: bool, - terminal: bool, - ) -> Result<(ExtractedLogprobs, Option), DynamoError> { + ) -> Result { match self { Self::Inline => Ok(( extract_logprobs(meta, return_tokens_as_ids)?, - engine_data_from_meta(meta, terminal)?, + engine_data_from_meta(meta, false)?, )), Self::Upload(_) => Ok(((None, None), None)), } } - pub(crate) fn uploader(&self) -> Option<&MetadataUploader> { + pub(crate) async fn finish( + &self, + meta: &HashMap, + return_tokens_as_ids: bool, + ) -> Result { match self { - Self::Inline => None, - Self::Upload(uploader) => Some(uploader), + Self::Inline => Ok(( + extract_logprobs(meta, return_tokens_as_ids)?, + engine_data_from_meta(meta, true)?, + )), + Self::Upload(uploader) => { + uploader.upload(grpc_metadata(meta)).await?; + Ok(((None, None), None)) + } } } } @@ -191,15 +205,11 @@ impl MetadataUploader { match write(&self.primary, compressed.clone()).await { Ok(()) => Ok(()), Err(primary_error) => { - let Some(fallback_url) = self.fallback.as_ref() else { + let Some(fallback) = self.fallback.as_ref() else { return Err(primary_error); }; tracing::warn!(error = %primary_error, "primary metadata upload failed; attempting fallback"); - let fallback = self - .operators - .get(fallback_url.clone(), "fallback_url") - .await?; - write(&fallback, compressed).await + write(fallback, compressed).await } } } @@ -316,14 +326,13 @@ mod tests { use std::io::Cursor; use std::num::NonZeroUsize; use std::path::Path; - use std::sync::Arc; use std::time::Duration; use dynamo_backend_common::{OutputOptions, SamplingOptions, StopConditions}; use serde_json::{Value, json}; use super::config::OperatorPolicy; - use super::{MetadataUploadService, OUTPUT_PATH, OperatorCache, grpc_metadata}; + use super::{MetadataDelivery, MetadataUploadService, OUTPUT_PATH, OperatorCache}; fn policy(capacity: usize) -> OperatorPolicy { OperatorPolicy { @@ -338,8 +347,8 @@ mod tests { } } - fn operator_cache(capacity: usize) -> Arc { - Arc::new(OperatorCache::new(policy(capacity))) + fn operator_cache(capacity: usize) -> OperatorCache { + OperatorCache::new(policy(capacity)) } fn service(capacity: usize) -> MetadataUploadService { @@ -369,29 +378,56 @@ mod tests { let configured = request(json!({ "nvext": {"metadata_upload": {"url": "fs:///tmp/metadata"}} })); - assert!( - service(2) - .delivery_for(&configured) - .await - .unwrap() - .uploader() - .is_some() - ); + assert!(matches!( + service(2).delivery_for(&configured).await.unwrap(), + MetadataDelivery::Upload(_) + )); let unconfigured = request(json!({"nvext": {}})); - assert!( - service(2) - .delivery_for(&unconfigured) - .await - .unwrap() - .uploader() - .is_none() - ); + assert!(matches!( + service(2).delivery_for(&unconfigured).await.unwrap(), + MetadataDelivery::Inline + )); let invalid = request(json!({ "nvext": {"metadata_upload": {"url": "fs:///tmp", "format": "json"}} })); assert!(service(2).delivery_for(&invalid).await.is_err()); + + let invalid_fallback = request(json!({ + "nvext": {"metadata_upload": { + "url": "fs:///tmp", + "fallback_url": "unknown://destination" + }} + })); + assert!(service(2).delivery_for(&invalid_fallback).await.is_err()); + } + + #[tokio::test] + async fn delivery_mode_owns_inline_response_fields() { + let meta = std::collections::HashMap::from([ + ("output_token_logprobs".into(), r#"[[-0.1,101,"a"]]"#.into()), + ("routed_experts".into(), "[1,2]".into()), + ]); + let inline = service(2) + .delivery_for(&request(json!({"nvext": {}}))) + .await + .unwrap(); + let ((logprobs, _), engine_data) = inline.chunk_fields(&meta, false).unwrap(); + assert_eq!(logprobs.unwrap(), vec![-0.1]); + assert_eq!(engine_data.unwrap()["routed_experts"], json!([1, 2])); + + opendal::OperatorRegistry::get().register::("metadata-fields"); + let remote = service(2) + .delivery_for(&request(json!({ + "nvext": {"metadata_upload": {"url": "metadata-fields://rollout"}} + }))) + .await + .unwrap(); + assert_eq!( + remote.chunk_fields(&meta, false).unwrap(), + ((None, None), None) + ); } #[tokio::test] @@ -402,13 +438,15 @@ mod tests { "nvext": {"metadata_upload": {"url": url}} })); let delivery = service(2).delivery_for(&configured).await.unwrap(); - let uploader = delivery.uploader().unwrap(); - uploader - .upload(grpc_metadata(&std::collections::HashMap::from([ - ("id".into(), "sglang-1".into()), - ("finish_reason".into(), r#"{"type":"stop"}"#.into()), - ("output_token_logprobs".into(), r#"[[-0.1,101,"a"]]"#.into()), - ]))) + delivery + .finish( + &std::collections::HashMap::from([ + ("id".into(), "sglang-1".into()), + ("finish_reason".into(), r#"{"type":"stop"}"#.into()), + ("output_token_logprobs".into(), r#"[[-0.1,101,"a"]]"#.into()), + ]), + false, + ) .await .unwrap(); @@ -439,9 +477,13 @@ mod tests { }} })); let delivery = service(2).delivery_for(&configured).await.unwrap(); - let uploader = delivery.uploader().unwrap(); - - uploader.upload(json!({"id": "sglang-2"})).await.unwrap(); + delivery + .finish( + &std::collections::HashMap::from([("id".into(), "sglang-2".into())]), + false, + ) + .await + .unwrap(); assert_eq!(std::fs::read(primary).unwrap(), b"primary sentinel"); assert!(fallback.join(OUTPUT_PATH).is_file()); @@ -454,14 +496,10 @@ mod tests { "nvext": {"metadata_upload": {"url": "custom-metadata://rollout"}} })); - assert!( - service(2) - .delivery_for(&configured) - .await - .unwrap() - .uploader() - .is_some() - ); + assert!(matches!( + service(2).delivery_for(&configured).await.unwrap(), + MetadataDelivery::Upload(_) + )); } #[tokio::test] diff --git a/lib/sidecar/sglang/src/metadata_upload/config.rs b/lib/sidecar/sglang/src/metadata_upload/config.rs index d3ee1e23b1e9..c38945b9dd0a 100644 --- a/lib/sidecar/sglang/src/metadata_upload/config.rs +++ b/lib/sidecar/sglang/src/metadata_upload/config.rs @@ -40,7 +40,7 @@ pub struct MetadataUploadArgs { env = "DYN_SGLANG_METADATA_UPLOAD_RETRY_MAX_TIMES", default_value = "3" )] - pub retry_max_times: NonZeroUsize, + pub retry_max_times: usize, /// Initial exponential retry delay in milliseconds. #[arg( @@ -101,7 +101,7 @@ impl TryFrom for OperatorPolicy { capacity: args.operator_cache_capacity, timeout: Duration::from_secs(args.timeout_secs.get()), io_timeout: Duration::from_secs(args.io_timeout_secs.get()), - retry_max_times: args.retry_max_times.get(), + retry_max_times: args.retry_max_times, retry_min_delay: Duration::from_millis(args.retry_min_delay_ms), retry_max_delay: Duration::from_millis(args.retry_max_delay_ms), retry_factor: args.retry_factor, @@ -143,7 +143,7 @@ mod tests { assert_eq!(policy.operator_cache_capacity.get(), 8); assert_eq!(policy.timeout_secs.get(), 11); assert_eq!(policy.io_timeout_secs.get(), 12); - assert_eq!(policy.retry_max_times.get(), 4); + assert_eq!(policy.retry_max_times, 4); assert_eq!(policy.retry_min_delay_ms, 20); assert_eq!(policy.retry_max_delay_ms, 200); assert_eq!(policy.retry_factor, 1.5); @@ -182,4 +182,18 @@ mod tests { assert_eq!(arg.get_env().unwrap(), env); } } + + #[test] + fn zero_retries_is_valid() { + let args = Args::try_parse_from([ + "dynamo-sglang-sidecar", + "--grpc-endpoint", + "127.0.0.1:30000", + "--metadata-upload-retry-max-times", + "0", + ]) + .unwrap(); + + assert_eq!(args.metadata_upload.retry_max_times, 0); + } } diff --git a/lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs b/lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs index dfb6126d7afe..c38516a0c7b3 100644 --- a/lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs +++ b/lib/sidecar/sglang/src/metadata_upload/resilience_tests.rs @@ -30,8 +30,8 @@ fn policy(capacity: usize) -> OperatorPolicy { } } -fn operator_cache(capacity: usize) -> Arc { - Arc::new(OperatorCache::new(policy(capacity))) +fn operator_cache(capacity: usize) -> OperatorCache { + OperatorCache::new(policy(capacity)) } fn request() -> PreprocessedRequest { @@ -55,19 +55,45 @@ fn request() -> PreprocessedRequest { async fn concurrent_cache_misses_build_one_operator() { let directory = tempfile::tempdir().unwrap(); let url = format!("fs://{}", directory.path().display()); - let cache = operator_cache(2); + let cache = Arc::new(operator_cache(2)); + let barrier = Arc::new(tokio::sync::Barrier::new(33)); let mut tasks = Vec::new(); for _ in 0..32 { let cache = cache.clone(); let url = url.clone(); - tasks.push(tokio::spawn(async move { cache.get(url, "url").await })); + let barrier = barrier.clone(); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + cache.get(url, "url").await + })); } + barrier.wait().await; for task in tasks { task.await.unwrap().unwrap(); } assert_eq!(cache.build_count.load(Ordering::Relaxed), 1); } +#[tokio::test] +async fn concurrent_cache_misses_share_construction_failure() { + let cache = Arc::new(operator_cache(2)); + let barrier = Arc::new(tokio::sync::Barrier::new(33)); + let mut tasks = Vec::new(); + for _ in 0..32 { + let cache = cache.clone(); + let barrier = barrier.clone(); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + cache.get("unknown://destination".to_string(), "url").await + })); + } + barrier.wait().await; + for task in tasks { + assert!(task.await.unwrap().is_err()); + } + assert_eq!(cache.build_count.load(Ordering::Relaxed), 1); +} + #[tokio::test] async fn retries_temporary_errors_until_success() { let (operator, attempts) = test_operator(WriteBehavior::FailUntil(2), policy(2)); @@ -101,10 +127,7 @@ async fn timeout_exhausts_retries_before_fallback() { let mut timeout_policy = policy(2); timeout_policy.io_timeout = Duration::from_millis(5); timeout_policy.retry_max_times = 1; - let (primary, primary_attempts) = test_operator( - WriteBehavior::Sleep(Duration::from_millis(50)), - timeout_policy, - ); + let (primary, primary_attempts) = test_operator(WriteBehavior::Pending, timeout_policy); let (fallback, fallback_attempts) = test_operator(WriteBehavior::Success, policy(2)); cache.insert("test://primary", primary).await; cache.insert("test://fallback", fallback).await; @@ -112,9 +135,10 @@ async fn timeout_exhausts_retries_before_fallback() { let delivery = service.delivery_for(&request()).await.unwrap(); delivery - .uploader() - .unwrap() - .upload(json!({"id": "timeout"})) + .finish( + &std::collections::HashMap::from([("id".into(), "timeout".into())]), + false, + ) .await .unwrap(); @@ -126,7 +150,7 @@ async fn timeout_exhausts_retries_before_fallback() { enum WriteBehavior { Success, FailUntil(usize), - Sleep(Duration), + Pending, } #[derive(Clone, Debug)] @@ -254,10 +278,7 @@ impl oio::Write for TestWriter { Err(Error::new(ErrorKind::Unexpected, "injected temporary failure").set_temporary()) } WriteBehavior::FailUntil(_) => Ok(file_metadata(self.size)), - WriteBehavior::Sleep(duration) => { - tokio::time::sleep(duration).await; - Ok(file_metadata(self.size)) - } + WriteBehavior::Pending => std::future::pending().await, } } From 66f12025293ac05523e844ae8a2cb05d99c8f6e8 Mon Sep 17 00:00:00 2001 From: William Arnold <7565007+Aphoh@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:13:29 +0000 Subject: [PATCH 11/11] refactor(sglang): keep fallback sidecar-only Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com> --- components/src/dynamo/common/metadata_upload.py | 10 +--------- .../sglang/tests/test_sglang_decode_handler.py | 14 +------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/components/src/dynamo/common/metadata_upload.py b/components/src/dynamo/common/metadata_upload.py index 6168d05f1c5b..a29f21a9cbbb 100644 --- a/components/src/dynamo/common/metadata_upload.py +++ b/components/src/dynamo/common/metadata_upload.py @@ -154,20 +154,12 @@ def __post_init__(self) -> None: def from_settings(cls, settings: dict[str, Any] | None) -> MetadataUploader | None: if settings is None: return None - # The Rust sidecar consumes fallback_url. Accept it here so the shared - # request contract does not break the Python SGLang worker. - unexpected = settings.keys() - {"url", "fallback_url"} + unexpected = settings.keys() - {"url"} if unexpected: field = min(unexpected) raise ValueError(f"metadata_upload.{field} is not supported") if "url" not in settings: raise ValueError("metadata_upload.url is required") - fallback_url = settings.get("fallback_url") - if fallback_url is not None: - if not isinstance(fallback_url, str): - raise ValueError("metadata_upload.fallback_url must be a string") - if not fallback_url.strip(): - raise ValueError("metadata_upload.fallback_url must not be empty") return cls(url=settings["url"]) @classmethod diff --git a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py index c0dad0ff0485..cd741b741572 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py +++ b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py @@ -1078,17 +1078,9 @@ def test_metadata_uploader_parses_extra_args_nvext(): assert uploader.url == "s3://bucket/root/rollouts" uploader = MetadataUploader.from_backend_request( - { - "nvext": { - "metadata_upload": { - "url": "s3://bucket/root/rollouts", - "fallback_url": "fs:///var/tmp/rollouts", - } - } - } + {"nvext": {"metadata_upload": {"url": "s3://bucket/root/rollouts"}}} ) assert uploader is not None - assert uploader.url == "s3://bucket/root/rollouts" @pytest.mark.parametrize( @@ -1102,10 +1094,6 @@ def test_metadata_uploader_parses_extra_args_nvext(): {"url": "s3://bucket/root/rollouts", "format": "json"}, "metadata_upload.format is not supported", ), - ( - {"url": "s3://bucket/root/rollouts", "fallback_url": ""}, - "metadata_upload.fallback_url must not be empty", - ), ], ) def test_metadata_uploader_rejects_invalid_settings(settings, message):