Skip to content
Merged
4 changes: 3 additions & 1 deletion crates/onnx-genai-cli/src/transcribe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,9 @@ impl<R: BufRead> PcmStreamReader<R> {
.chunks_exact(frame_bytes)
.map(|frame| {
let sum: f32 = frame
.chunks_exact(2)
.as_chunks::<2>()
.0
.iter()
.map(|pair| f32::from(i16::from_le_bytes([pair[0], pair[1]])))
.sum();
sum / f32::from(self.channels) / 32768.0
Expand Down
6 changes: 4 additions & 2 deletions crates/onnx-genai-engine/src/engine/speculative_load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@ pub(crate) fn read_f32_weights(path: &Path) -> anyhow::Result<Vec<f32>> {
);
}
Ok(bytes
.chunks_exact(4)
.map(|bytes| f32::from_le_bytes(bytes.try_into().expect("four-byte chunk")))
.as_chunks::<4>()
.0
.iter()
.map(|bytes| f32::from_le_bytes(*bytes))
.collect())
}

Expand Down
4 changes: 3 additions & 1 deletion crates/onnx-genai-engine/src/native_component.rs
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,9 @@ mod tests {
fn as_f32(tensor: &ComponentTensor) -> Vec<f32> {
tensor
.as_bytes()
.chunks_exact(4)
.as_chunks::<4>()
.0
.iter()
.map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]))
.collect()
}
Expand Down
6 changes: 4 additions & 2 deletions crates/onnx-genai-engine/src/native_decode/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1597,8 +1597,10 @@ mod prefill_chunk_tests {
assert_eq!(tensor.shape, vec![1, 2, hidden]);
let taken: Vec<f32> = tensor
.as_bytes()
.chunks_exact(4)
.map(|word| f32::from_le_bytes(word.try_into().expect("f32")))
.as_chunks::<4>()
.0
.iter()
.map(|word| f32::from_le_bytes(*word))
.collect();
// Rows 1 and 2 of a [1, 4, 3] tensor numbered 0..12.
assert_eq!(taken, vec![3.0, 4.0, 5.0, 6.0, 7.0, 8.0]);
Expand Down
50 changes: 33 additions & 17 deletions crates/onnx-genai-engine/src/native_decode/tensor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,14 @@ pub(crate) fn argmax_logits_tensor(tensor: &Tensor) -> anyhow::Result<TokenId> {
}
return Ok(sample_greedy(&values[row_start..]) as TokenId);
}
for bytes in tensor.as_bytes().chunks_exact(4).take(value_count) {
visit(f32::from_le_bytes(
bytes.try_into().expect("four-byte chunk"),
))?;
for bytes in tensor
.as_bytes()
.as_chunks::<4>()
.0
.iter()
.take(value_count)
{
visit(f32::from_le_bytes(*bytes))?;
}
}
DataType::Float16 => {
Expand All @@ -102,10 +106,14 @@ pub(crate) fn argmax_logits_tensor(tensor: &Tensor) -> anyhow::Result<TokenId> {
return argmax_finite_half_values(halves, value_count, row_start)
.map(|index| index as TokenId);
}
for bytes in tensor.as_bytes().chunks_exact(2).take(value_count) {
visit(f16_to_f32(u16::from_le_bytes(
bytes.try_into().expect("two-byte chunk"),
)))?;
for bytes in tensor
.as_bytes()
.as_chunks::<2>()
.0
.iter()
.take(value_count)
{
visit(f16_to_f32(u16::from_le_bytes(*bytes)))?;
}
}
DataType::BFloat16 => {
Expand All @@ -115,12 +123,14 @@ pub(crate) fn argmax_logits_tensor(tensor: &Tensor) -> anyhow::Result<TokenId> {
return argmax_finite_half_values(halves, value_count, row_start)
.map(|index| index as TokenId);
}
for bytes in tensor.as_bytes().chunks_exact(2).take(value_count) {
visit(f32::from_bits(
u32::from(u16::from_le_bytes(
bytes.try_into().expect("two-byte chunk"),
)) << 16,
))?;
for bytes in tensor
.as_bytes()
.as_chunks::<2>()
.0
.iter()
.take(value_count)
{
visit(f32::from_bits(u32::from(u16::from_le_bytes(*bytes)) << 16))?;
}
}
dtype => bail!("native logits must be Float32, Float16, or BFloat16, got {dtype:?}"),
Expand Down Expand Up @@ -203,12 +213,16 @@ fn tensor_to_f32(tensor: &Tensor) -> anyhow::Result<Vec<f32>> {
DataType::Float32 => Ok(tensor.to_vec_f32()),
DataType::Float16 => Ok(tensor
.as_bytes()
.chunks_exact(2)
.as_chunks::<2>()
.0
.iter()
.map(|bytes| f16_to_f32(u16::from_le_bytes([bytes[0], bytes[1]])))
.collect()),
DataType::BFloat16 => Ok(tensor
.as_bytes()
.chunks_exact(2)
.as_chunks::<2>()
.0
.iter()
.map(|bytes| f32::from_bits(u32::from(u16::from_le_bytes([bytes[0], bytes[1]])) << 16))
.collect()),
dtype => bail!("native logits must be Float32, Float16, or BFloat16, got {dtype:?}"),
Expand Down Expand Up @@ -281,7 +295,9 @@ pub(crate) fn kv_dtype_to_f32(tensor: &Tensor) -> anyhow::Result<Vec<f32>> {
} else {
Ok(tensor
.as_bytes()
.chunks_exact(2)
.as_chunks::<2>()
.0
.iter()
.map(|bytes| {
half::f16::from_bits(u16::from_le_bytes([bytes[0], bytes[1]])).to_f32()
})
Expand Down
8 changes: 6 additions & 2 deletions crates/onnx-genai-engine/src/native_decode/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3938,7 +3938,9 @@ fn growing_the_cpu_kv_cache_relocates_every_head() {

let raw = grown.read_bytes().expect("read grown cache");
let values: Vec<f32> = raw
.chunks_exact(4)
.as_chunks::<4>()
.0
.iter()
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect();
assert_eq!(values.len(), heads * grown_capacity * head_dim);
Expand Down Expand Up @@ -4032,7 +4034,9 @@ fn assert_kv_prefix_intact(
let capacity = binding.physical_shape()[2];
let raw = binding.read_bytes().expect("read cache");
let values: Vec<f32> = raw
.chunks_exact(4)
.as_chunks::<4>()
.0
.iter()
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect();
for block in 0..heads {
Expand Down
9 changes: 3 additions & 6 deletions crates/onnx-genai-engine/src/pipeline_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,14 +106,11 @@ impl DigestBuilder {
/// collide with a single field holding their concatenation.
pub fn absorb(&mut self, bytes: &[u8]) {
self.absorb_u64(bytes.len() as u64);
let mut chunks = bytes.chunks_exact(8);
for chunk in &mut chunks {
let word =
u64::from_le_bytes(chunk.try_into().expect("chunks_exact(8) yields 8 bytes"));
self.mix(u128::from(word));
let (chunks, remainder) = bytes.as_chunks::<8>();
for chunk in chunks {
self.mix(u128::from(u64::from_le_bytes(*chunk)));
}
let mut tail = [0u8; 8];
let remainder = chunks.remainder();
tail[..remainder.len()].copy_from_slice(remainder);
self.mix(u128::from(u64::from_le_bytes(tail)));
}
Expand Down
12 changes: 8 additions & 4 deletions crates/onnx-genai-engine/src/speculative/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2515,8 +2515,10 @@ mod tests {
embedder.embed(7, &mut embedded)?;
let raw_embedding = std::fs::read(fixture.join("embedding.f32"))?;
let expected_embedding = raw_embedding
.chunks_exact(4)
.map(|bytes| f32::from_le_bytes(bytes.try_into().expect("four bytes")))
.as_chunks::<4>()
.0
.iter()
.map(|bytes| f32::from_le_bytes(*bytes))
.collect::<Vec<_>>();
assert_eq!(embedded, expected_embedding[7 * 16..8 * 16]);

Expand All @@ -2526,8 +2528,10 @@ mod tests {
let raw_lm_head = std::fs::read(fixture.join("lm_head.f32"))?;
let linear_lm_head = LinearLmHead::new(
raw_lm_head
.chunks_exact(4)
.map(|bytes| f32::from_le_bytes(bytes.try_into().expect("four bytes")))
.as_chunks::<4>()
.0
.iter()
.map(|bytes| f32::from_le_bytes(*bytes))
.collect(),
16,
32,
Expand Down
8 changes: 6 additions & 2 deletions crates/onnx-genai-engine/tests/codec_pipeline_e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ fn composite_codec_pipeline_runs_encoder_then_vocoder() -> anyhow::Result<()> {

// Intermediate encoder codes are visible in the shared pool: mean of pairs.
let expected_codes: Vec<f32> = waveform
.chunks_exact(2)
.as_chunks::<2>()
.0
.iter()
.map(|pair| (pair[0] + pair[1]) / 2.0)
.collect();
let codes = outputs
Expand All @@ -58,7 +60,9 @@ fn composite_codec_pipeline_runs_encoder_then_vocoder() -> anyhow::Result<()> {
// Vocoder reconstructs the waveform: each code doubled into an adjacent pair,
// so audio[2i] == audio[2i+1] == w[2i] + w[2i+1].
let expected_audio: Vec<f32> = waveform
.chunks_exact(2)
.as_chunks::<2>()
.0
.iter()
.flat_map(|pair| {
let sum = pair[0] + pair[1];
[sum, sum]
Expand Down
4 changes: 3 additions & 1 deletion crates/onnx-genai-preprocess/src/image/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1562,7 +1562,9 @@ preprocessing:
};
(
values
.chunks_exact(12)
.as_chunks::<12>()
.0
.iter()
.map(|patch| patch[0])
.collect::<Vec<_>>(),
bundle.images[0].expansion_count,
Expand Down
6 changes: 4 additions & 2 deletions crates/onnx-runtime-capi/tests/capi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,10 @@ fn full_roundtrip_matches_reference() {
assert_eq!(data_len, 4 * 4); // 4 f32
let out_bytes = unsafe { std::slice::from_raw_parts(data_ptr as *const u8, data_len) };
let got: Vec<f32> = out_bytes
.chunks_exact(4)
.map(|c| f32::from_le_bytes(c.try_into().unwrap()))
.as_chunks::<4>()
.0
.iter()
.map(|c| f32::from_le_bytes(*c))
.collect();
let want = reference(&x_data);
for (g, w) in got.iter().zip(&want) {
Expand Down
19 changes: 14 additions & 5 deletions crates/onnx-runtime-comm/src/inprocess.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1172,8 +1172,10 @@ mod tests {
fn read_f32(buffer: &DeviceBuffer) -> Vec<f32> {
buffer
.as_slice()
.chunks_exact(4)
.map(|bytes| f32::from_le_bytes(bytes.try_into().unwrap()))
.as_chunks::<4>()
.0
.iter()
.map(|bytes| f32::from_le_bytes(*bytes))
.collect()
}

Expand Down Expand Up @@ -1447,7 +1449,12 @@ mod tests {
*value += *partial;
}
}
let expected: Vec<Vec<u8>> = reduced.chunks_exact(2).map(f32_bytes).collect();
let expected: Vec<Vec<u8>> = reduced
.as_chunks::<2>()
.0
.iter()
.map(|c| f32_bytes(c))
.collect();
let world = InProcessCommunicator::world(shards.len());
let handles: Vec<_> = world
.into_iter()
Expand Down Expand Up @@ -1714,8 +1721,10 @@ mod tests {
))
.unwrap();
recv.as_slice()
.chunks_exact(4)
.map(|bytes| i32::from_le_bytes(bytes.try_into().unwrap()))
.as_chunks::<4>()
.0
.iter()
.map(|bytes| i32::from_le_bytes(*bytes))
.collect::<Vec<_>>()
})
})
Expand Down
33 changes: 24 additions & 9 deletions crates/onnx-runtime-comm/src/reduction.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,14 @@ fn float_combine(lhs: f32, rhs: f32, op: ReduceOp) -> f32 {
fn reduce_f32(inputs: &[Vec<u8>], op: ReduceOp) -> CommResult<Vec<u8>> {
let mut output = inputs[0].clone();
for input in &inputs[1..] {
for (out, rhs) in output.chunks_exact_mut(4).zip(input.chunks_exact(4)) {
let lhs = f32::from_le_bytes(out.try_into().expect("four-byte chunk"));
let rhs = f32::from_le_bytes(rhs.try_into().expect("four-byte chunk"));
for (out, rhs) in output
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(input.as_chunks::<4>().0.iter())
{
let lhs = f32::from_le_bytes(*out);
let rhs = f32::from_le_bytes(*rhs);
out.copy_from_slice(&float_combine(lhs, rhs, op).to_le_bytes());
}
}
Expand All @@ -70,9 +75,14 @@ fn reduce_f32(inputs: &[Vec<u8>], op: ReduceOp) -> CommResult<Vec<u8>> {
fn reduce_f16(inputs: &[Vec<u8>], op: ReduceOp) -> CommResult<Vec<u8>> {
let mut output = inputs[0].clone();
for input in &inputs[1..] {
for (out, rhs) in output.chunks_exact_mut(2).zip(input.chunks_exact(2)) {
let lhs = f16::from_le_bytes(out.try_into().expect("two-byte chunk")).to_f32();
let rhs = f16::from_le_bytes(rhs.try_into().expect("two-byte chunk")).to_f32();
for (out, rhs) in output
.as_chunks_mut::<2>()
.0
.iter_mut()
.zip(input.as_chunks::<2>().0.iter())
{
let lhs = f16::from_le_bytes(*out).to_f32();
let rhs = f16::from_le_bytes(*rhs).to_f32();
out.copy_from_slice(&f16::from_f32(float_combine(lhs, rhs, op)).to_le_bytes());
}
}
Expand All @@ -82,9 +92,14 @@ fn reduce_f16(inputs: &[Vec<u8>], op: ReduceOp) -> CommResult<Vec<u8>> {
fn reduce_bf16(inputs: &[Vec<u8>], op: ReduceOp) -> CommResult<Vec<u8>> {
let mut output = inputs[0].clone();
for input in &inputs[1..] {
for (out, rhs) in output.chunks_exact_mut(2).zip(input.chunks_exact(2)) {
let lhs = bf16::from_le_bytes(out.try_into().expect("two-byte chunk")).to_f32();
let rhs = bf16::from_le_bytes(rhs.try_into().expect("two-byte chunk")).to_f32();
for (out, rhs) in output
.as_chunks_mut::<2>()
.0
.iter_mut()
.zip(input.as_chunks::<2>().0.iter())
{
let lhs = bf16::from_le_bytes(*out).to_f32();
let rhs = bf16::from_le_bytes(*rhs).to_f32();
out.copy_from_slice(&bf16::from_f32(float_combine(lhs, rhs, op)).to_le_bytes());
}
}
Expand Down
10 changes: 7 additions & 3 deletions crates/onnx-runtime-eager/src/tensor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,9 @@ impl Tensor {
"to_vec_f32 on non-f32 tensor"
);
self.as_bytes()
.chunks_exact(4)
.as_chunks::<4>()
.0
.iter()
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect()
}
Expand All @@ -281,8 +283,10 @@ impl Tensor {
pub fn to_vec_i64(&self) -> Vec<i64> {
assert_eq!(self.dtype, DataType::Int64, "to_vec_i64 on non-i64 tensor");
self.as_bytes()
.chunks_exact(8)
.map(|c| i64::from_le_bytes(c.try_into().unwrap()))
.as_chunks::<8>()
.0
.iter()
.map(|c| i64::from_le_bytes(*c))
.collect()
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/onnx-runtime-ep-cpu/src/kernels/block_dequant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ pub(crate) fn quantize_fp4_e2m1_block(
.ok_or_else(|| error("FP4 E2M1 E8M0 scale exponent is out of range"))?;
*scale_exponent = exponent as u8;
let scale = 2.0f32.powi(scale_power);
for (pair, destination) in input.chunks_exact(2).zip(packed.iter_mut()) {
for (pair, destination) in input.as_chunks::<2>().0.iter().zip(packed.iter_mut()) {
let low = encode_e2m1((pair[0] / scale).clamp(-6.0, 6.0));
let high = encode_e2m1((pair[1] / scale).clamp(-6.0, 6.0));
*destination = low | (high << 4);
Expand Down
Loading
Loading