Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion container/templates/dynamo_base.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ ENV PATH=/usr/local/bin/etcd/:$PATH
ENV RUSTUP_HOME=/usr/local/rustup \
CARGO_HOME=/usr/local/cargo \
PATH=/usr/local/cargo/bin:$PATH \
RUST_VERSION=1.93.1
RUST_VERSION=1.96.1

# Install Rust — ARCH_ALT (x86_64/aarch64) is derived from TARGETARCH at build time
RUN ARCH_ALT=$([ "${TARGETARCH}" = "amd64" ] && echo "x86_64" || echo "aarch64") && \
Expand Down
2 changes: 1 addition & 1 deletion deploy/inference-gateway/epp/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
# --build-context dynamo=../../.. \
# -t dynamo/dynamo-epp:dev .

ARG RUST_IMAGE=rust:1.93.1
ARG RUST_IMAGE=rust:1.96.1
ARG BUILDER_IMAGE=golang:1.26.3
ARG BASE_IMAGE=ubuntu:24.04

Expand Down
2 changes: 1 addition & 1 deletion deploy/inference-gateway/epp/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ SCCACHE_REGION ?=

DOCKER_BUILDX_CMD ?= docker buildx
IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build
RUST_IMAGE ?= $(DOCKER_PROXY)library/rust:1.93.1
RUST_IMAGE ?= $(DOCKER_PROXY)library/rust:1.96.1
BUILDER_IMAGE ?= $(DOCKER_PROXY)library/golang:1.26.3
BASE_IMAGE ?= $(DOCKER_PROXY)library/ubuntu:24.04

Expand Down
2 changes: 1 addition & 1 deletion deploy/inference-gateway/ext-proc/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# --build-context dynamo=../../.. \
# -t dynamo/dynamo-rust-epp:dev .

ARG RUST_IMAGE=rust:1.93.1
ARG RUST_IMAGE=rust:1.96.1
ARG BASE_IMAGE=ubuntu:24.04

# =============================================================================
Expand Down
2 changes: 1 addition & 1 deletion deploy/inference-gateway/ext-proc/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ SCCACHE_REGION ?=

DOCKER_BUILDX_CMD ?= docker buildx
IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build
RUST_IMAGE ?= $(DOCKER_PROXY)rust:1.93.1
RUST_IMAGE ?= $(DOCKER_PROXY)rust:1.96.1
BASE_IMAGE ?= $(DOCKER_PROXY)ubuntu:24.04

# Container tool
Expand Down
2 changes: 1 addition & 1 deletion docs/development/custom-containers.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ ENTRYPOINT ["my-backend"]
For a Rust custom backend, build the backend binary in your own builder stage and copy it into the Dynamo runtime image:

```Dockerfile
FROM rust:1.93 AS backend-builder
FROM rust:1.96.1 AS backend-builder
WORKDIR /src
COPY . .
RUN cargo build --release
Expand Down
2 changes: 1 addition & 1 deletion docs/development/unified-backends.md
Original file line number Diff line number Diff line change
Expand Up @@ -1090,7 +1090,7 @@ inside `dynamo-runtime`:

```toml
[toolchain]
channel = "1.93.1"
channel = "1.96.1"
```

Older toolchains fail with `feature edition2024 is required`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1723,7 +1723,7 @@ where
let block_pairs: Vec<(usize, usize)> = offload_req
.block_ids
.into_iter()
.zip(allocated_block_ids.into_iter())
.zip(allocated_block_ids)
.collect();

tracing::debug!(
Expand All @@ -1737,10 +1737,8 @@ where
let mut blocks_to_register = Vec::new();
let priorities = offload_req.priorities;

for ((mut mutable_block, token_block), priority) in blocks
.into_iter()
.zip(token_blocks.into_iter())
.zip(priorities.into_iter())
for ((mut mutable_block, token_block), priority) in
blocks.into_iter().zip(token_blocks).zip(priorities)
{
mutable_block
.apply_token_block(token_block.clone())
Expand Down
25 changes: 15 additions & 10 deletions lib/kv-router/src/indexer/branch_sharded.rs
Original file line number Diff line number Diff line change
Expand Up @@ -961,16 +961,21 @@ impl<S: AsyncShardHandle> KvIndexerInterface for BranchShardedIndexer<S> {

let timing = {
let calls = self.metrics.timing.calls.load(Ordering::Relaxed);
let avg_routing_ns = if calls > 0 {
self.metrics.timing.routing_ns.load(Ordering::Relaxed) / calls
} else {
0
};
let avg_shard_us = if calls > 0 {
self.metrics.timing.shard_ns.load(Ordering::Relaxed) / calls / 1000
} else {
0
};
let avg_routing_ns = self
.metrics
.timing
.routing_ns
.load(Ordering::Relaxed)
.checked_div(calls)
.unwrap_or(0);
let avg_shard_us = self
.metrics
.timing
.shard_ns
.load(Ordering::Relaxed)
.checked_div(calls)
.unwrap_or(0)
/ 1000;
format!("\n avg routing = {avg_routing_ns}ns\n avg shard = {avg_shard_us}µs")
};

Expand Down
25 changes: 12 additions & 13 deletions lib/kv-router/src/indexer/pruning.rs
Original file line number Diff line number Diff line change
Expand Up @@ -580,21 +580,20 @@ impl WorkerPruneManagerInner {
let mut expired = Vec::new();

loop {
let Some((Reverse(expiry), worker)) = ({
let mut expiries = self
.next_expiries
.lock()
.expect("worker expiry index mutex poisoned");
let Some((Reverse(expiry), _)) = expiries.peek().copied() else {
break;
};
if expiry > now {
break;
}
expiries.pop()
}) else {
let mut expiries = self
.next_expiries
.lock()
.expect("worker expiry index mutex poisoned");
let Some((Reverse(expiry), _)) = expiries.peek().copied() else {
break;
};
if expiry > now {
break;
}
let Some((Reverse(expiry), worker)) = ({ expiries.pop() }) else {
break;
};
Comment thread
grahamking marked this conversation as resolved.
Resolved
drop(expiries);

let (mut worker_expired, next_expiry) = {
let Some(state) = self.workers.get(&worker) else {
Expand Down
2 changes: 1 addition & 1 deletion lib/kvbm-engine/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co

## Build & Test

This is a Rust crate (`kvbm-engine`) in the dynamo workspace. Rust edition 2024, requires rustc 1.93.1+.
This is a Rust crate (`kvbm-engine`) in the dynamo workspace. Rust edition 2024, requires rustc 1.96.1+.

```bash
# Build
Expand Down
8 changes: 3 additions & 5 deletions lib/kvbm-engine/src/leader/session/initiator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -928,7 +928,7 @@ impl InitiatorSession {
// Sort (block_id, seq_hash) pairs by position to ensure correct transfer order
// This is a safety net in case responder sent blocks in wrong order
let mut pairs: Vec<(BlockId, SequenceHash)> =
block_ids.into_iter().zip(seq_hashes.into_iter()).collect();
block_ids.into_iter().zip(seq_hashes).collect();
pairs.sort_by_key(|(_, hash)| hash.position());

let block_ids: Vec<BlockId> = pairs.iter().map(|(id, _)| *id).collect();
Expand Down Expand Up @@ -1259,10 +1259,8 @@ impl InitiatorSession {
let mut blocks = Vec::new();

// Iterate over results alongside the dst_blocks and hashes
for ((result, dst_block), seq_hash) in results
.into_iter()
.zip(dst_blocks.into_iter())
.zip(hashes.iter())
for ((result, dst_block), seq_hash) in
results.into_iter().zip(dst_blocks).zip(hashes.iter())
{
match result {
Ok(hash) => {
Expand Down
4 changes: 1 addition & 3 deletions lib/kvbm-engine/src/worker/group/spmd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,7 @@ impl WorkerTransfers for SpmdParallelWorkers {
let mut new_handles = Vec::new();
let mut import_responses = Vec::new();

for (worker_idx, (worker, meta)) in
self.workers.iter().zip(metadata.into_iter()).enumerate()
{
for (worker_idx, (worker, meta)) in self.workers.iter().zip(metadata).enumerate() {
// Unpack to extract logical type info
let unpacked = meta.unpack()?;

Expand Down
6 changes: 1 addition & 5 deletions lib/kvbm-physical/src/layout/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,7 @@ impl LayoutConfig {
pub fn head_dim(&self) -> Option<usize> {
self.num_heads.map(|nh| {
let divisor = self.page_size * nh;
if divisor > 0 {
self.inner_dim / divisor
} else {
0
}
self.inner_dim.checked_div(divisor).unwrap_or(0)
})
}

Expand Down
4 changes: 2 additions & 2 deletions lib/llm/src/block_manager/pool/managed/inactive.rs
Original file line number Diff line number Diff line change
Expand Up @@ -711,7 +711,7 @@ pub(crate) mod tests {

// Iterate through the generated TokenBlocks and the template Blocks,
// setting the state and registering each one.
for (block, token_block) in blocks.iter_mut().zip(token_blocks.into_iter()) {
for (block, token_block) in blocks.iter_mut().zip(token_blocks) {
assert!(block.state().is_reset()); // Start with empty blocks
block.update_state(BlockState::Complete(CompleteState::new(token_block)));
block
Expand Down Expand Up @@ -779,7 +779,7 @@ pub(crate) mod tests {
assert!(unmatched.state().is_reset());
}

for (unmatched, token_block) in unmatched_blocks.iter_mut().zip(token_blocks.into_iter()) {
for (unmatched, token_block) in unmatched_blocks.iter_mut().zip(token_blocks) {
assert!(unmatched.state().is_reset());
unmatched.update_state(BlockState::Complete(CompleteState::new(token_block)));
unmatched.register(&mut registry).unwrap();
Expand Down
4 changes: 1 addition & 3 deletions lib/llm/src/preprocessor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1124,9 +1124,7 @@ impl OpenAIPreprocessor {
}))
.await;

for ((type_str, _content_part), result) in
fetch_tasks.into_iter().zip(results.into_iter())
{
for ((type_str, _content_part), result) in fetch_tasks.into_iter().zip(results) {
// if one item fails, errors the whole request, other items will be cleaned up by Drop
let rdma_descriptor = result?;

Expand Down
14 changes: 6 additions & 8 deletions lib/llm/src/protocols/openai/chat_completions/aggregator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ impl DeltaAggregator {
.map(dynamo_protocols::types::ChatChoice::from)
.collect();

choices.sort_by(|a, b| a.index.cmp(&b.index));
choices.sort_by_key(|a| a.index);

// Construct the final response object.
let response = NvCreateChatCompletionResponse {
Expand Down Expand Up @@ -568,8 +568,8 @@ mod tests {
let tool_calls: Option<serde_json::Value> =
tool_calls.map(|tool_calls| serde_json::from_str(tool_calls).unwrap());

let tool_call_chunks = if let Some(tool_calls) = tool_calls {
Some(vec![
let tool_call_chunks = tool_calls.map(|tool_calls| {
vec![
dynamo_protocols::types::ChatCompletionMessageToolCallChunk {
index: 0,
id: Some("test_id".to_string()),
Expand All @@ -579,10 +579,8 @@ mod tests {
arguments: Some(serde_json::to_string(&tool_calls["arguments"]).unwrap()),
}),
},
])
} else {
None
};
]
});

let delta = dynamo_protocols::types::ChatCompletionStreamResponseDelta {
content: Some(ChatCompletionMessageContent::Text(text.to_string())),
Expand Down Expand Up @@ -1210,7 +1208,7 @@ mod tests {

// Verify the response fields
assert_eq!(response.inner.choices.len(), 2);
response.inner.choices.sort_by(|a, b| a.index.cmp(&b.index)); // Ensure the choices are ordered
response.inner.choices.sort_by_key(|a| a.index); // Ensure the choices are ordered
let choice0 = &response.inner.choices[0];
assert_eq!(choice0.index, 0);
assert_eq!(
Expand Down
4 changes: 2 additions & 2 deletions lib/llm/src/protocols/openai/completions/aggregator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ impl DeltaAggregator {
.map(dynamo_protocols::types::Choice::from)
.collect();

choices.sort_by(|a, b| a.index.cmp(&b.index));
choices.sort_by_key(|a| a.index);

let inner = dynamo_protocols::types::CreateCompletionResponse {
id: aggregator.id,
Expand Down Expand Up @@ -438,7 +438,7 @@ mod tests {

// Verify the response fields
assert_eq!(response.inner.choices.len(), 2);
response.inner.choices.sort_by(|a, b| a.index.cmp(&b.index)); // Ensure the choices are ordered
response.inner.choices.sort_by_key(|a| a.index); // Ensure the choices are ordered
let choice0 = &response.inner.choices[0];
assert_eq!(choice0.index, 0);
assert_eq!(choice0.text, "Choice 0".to_string());
Expand Down
42 changes: 19 additions & 23 deletions lib/mocker/src/replay/offline/single.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,25 +114,24 @@ impl SingleRuntime {
fn enqueue_trace_arrivals(&mut self) {
let mut ready_requests = Vec::new();
match &mut self.admission {
AdmissionSource::Requests(pending) => loop {
let Some(next_arrival_ms) = pending
AdmissionSource::Requests(pending) => {
while let Some(next_arrival_ms) = pending
.front()
.and_then(|request| request.arrival_timestamp_ms)
else {
break;
};
if next_arrival_ms > self.current_time_ms {
break;
}
{
if next_arrival_ms > self.current_time_ms {
break;
}

let request = pending
.pop_front()
.expect("front request must exist when arrival is available");
let arrival_ms = request
.arrival_timestamp_ms
.expect("trace replay requests must have an arrival timestamp");
ready_requests.push((request, arrival_ms));
},
let request = pending
.pop_front()
.expect("front request must exist when arrival is available");
let arrival_ms = request
.arrival_timestamp_ms
.expect("trace replay requests must have an arrival timestamp");
ready_requests.push((request, arrival_ms));
}
}
AdmissionSource::Workload(driver) => {
ready_requests.extend(
driver
Expand Down Expand Up @@ -356,13 +355,10 @@ mod tests {
collector: &mut TraceCollector,
current_time_ms: f64,
) {
loop {
let Some(next_arrival_ms) = pending
.front()
.and_then(|request| request.arrival_timestamp_ms)
else {
break;
};
while let Some(next_arrival_ms) = pending
.front()
.and_then(|request| request.arrival_timestamp_ms)
{
if next_arrival_ms > current_time_ms {
break;
}
Expand Down
12 changes: 2 additions & 10 deletions lib/mocker/src/scheduler/sglang/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -600,16 +600,8 @@ impl SglangCore {
let prefill_fpm = admit.prefill_fpm;

let batch_size = admit.can_run.len();
let mean_isl = if batch_size > 0 {
admit.total_isl / batch_size
} else {
0
};
let mean_prefix = if batch_size > 0 {
admit.total_prefix / batch_size
} else {
0
};
let mean_isl = admit.total_isl.checked_div(batch_size).unwrap_or(0);
let mean_prefix = admit.total_prefix.checked_div(batch_size).unwrap_or(0);
let prefill_time =
simulate_prefill_duration(batch_size, mean_isl, mean_prefix, &self.config, true);

Expand Down
6 changes: 1 addition & 5 deletions lib/mocker/src/scheduler/sglang/decode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -276,11 +276,7 @@ pub(super) fn simulate_decode_step_with_sampler(
let mut output_signals = Vec::with_capacity(sampled_bursts.iter().copied().sum::<usize>());
let mut completed_indices = Vec::new();

for (idx, (req, burst)) in running
.iter_mut()
.zip(sampled_bursts.into_iter())
.enumerate()
{
for (idx, (req, burst)) in running.iter_mut().zip(sampled_bursts).enumerate() {
for _ in 0..burst {
let crossing_page_boundary = req.current_sequence_len() + 1 > req.allocated_tokens;
let last_idx = req.kv_indices.last().copied();
Expand Down
2 changes: 1 addition & 1 deletion lib/runtime/examples/rust-toolchain.toml
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
[toolchain]
channel = "1.93.1"
channel = "1.96.1"
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ use dynamo_runtime::{
};
use futures::StreamExt;
use rand::Rng;
use reqwest;
use system_metrics::{DEFAULT_COMPONENT, DEFAULT_ENDPOINT, DEFAULT_NAMESPACE, backend};
use tokio::time::{Duration, sleep};

Expand Down
Loading
Loading