Support YUY2-encoded images (#4877)

<!--
Open the PR up as a draft until you feel it is ready for a proper
review.

Do not make PR:s from your own `main` branch, as that makes it difficult
for reviewers to add their own fixes.

Add any improvements to the branch as new commits to make it easier for
reviewers to follow the progress. All commits will be squashed to a
single commit once the PR is merged into `main`.

Make sure you mention any issues that this PR closes in the description,
as well as any other related issues.

To get an auto-generated PR description you can put "copilot:summary" or
"copilot:walkthrough" anywhere.
-->

### What
This PR Introduces support for YUV422/YUY2 encoded images based on
#3541.

<img width="1117" alt="Image showcasing RGB, NV12 and YUV422 encoded
images"
src="https://github.com/rerun-io/rerun/assets/14833076/5ead1748-f765-4d68-b349-b920a89a6512">

The raw (encoded) image data is stored in an R8Uint texture. The
contents of the texture get decoded in the `rectangle_fs.wgsl` via the
decoder written in `decodings.wgsl`.

Other
[YUV](https://gist.github.com/Jim-Bar/3cbba684a71d1a9d468a6711a6eddbeb)
image formats could be added easily, following the NV12 implementation
with slight modifications to the decoder.

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using newly built examples:
[app.rerun.io](https://app.rerun.io/pr/4877/index.html)
* Using examples from latest `main` build:
[app.rerun.io](https://app.rerun.io/pr/4877/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[app.rerun.io](https://app.rerun.io/pr/4877/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG

- [PR Build Summary](https://build.rerun.io/pr/4877)
- [Docs
preview](https://rerun.io/preview/3f787b44ebb4a566224aba821024b27214245575/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/3f787b44ebb4a566224aba821024b27214245575/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

---------

Co-authored-by: Harold <[email protected]>
Co-authored-by: Harold Ruiter <[email protected]>
Co-authored-by: Andreas Reich <[email protected]>

crates/re_data_ui/src/image.rs

@@ -381,6 +381,11 @@ pub fn tensor_summary_ui_grid_contents(
             ui.label("NV12");
             ui.end_row();
         }
+        TensorBuffer::Yuy2(_) => {
+            re_ui.grid_left_hand_label(ui, "Encoding");
+            ui.label("YUY2");
+            ui.end_row();
+        }
     }
 
     let TensorStats {
@@ -713,6 +718,7 @@ fn tensor_pixel_value_ui(
                 // TODO(jleibs): Track RGB ordering somehow -- don't just assume it
                 if let Some([r, g, b]) = match &tensor.buffer {
                     TensorBuffer::Nv12(_) => tensor.get_nv12_pixel(x, y),
+                    TensorBuffer::Yuy2(_) => tensor.get_yuy2_pixel(x, y),
                     _ => {
                         if let [Some(r), Some(g), Some(b)] = [
                             tensor.get_with_image_coords(x, y, 0),

crates/re_renderer/shader/decodings.wgsl

@@ -8,18 +8,58 @@ fn decode_nv12(texture: texture_2d<u32>, coords: vec2i) -> vec4f {
     let uv_row = u32(coords.y / 2);
     var uv_col = u32(coords.x / 2) * 2u;
 
-    let y = max(0.0, (f32(textureLoad(texture, vec2u(coords), 0).r) - 16.0)) / 219.0;
-    let u = (f32(textureLoad(texture, vec2u(u32(uv_col), uv_offset + uv_row), 0).r) - 128.0) / 224.0;
-    let v = (f32(textureLoad(texture, vec2u((u32(uv_col) + 1u), uv_offset + uv_row), 0).r) - 128.0) / 224.0;
+    let y = f32(textureLoad(texture, vec2u(coords), 0).r);
+    let u = f32(textureLoad(texture, vec2u(u32(uv_col), uv_offset + uv_row), 0).r);
+    let v = f32(textureLoad(texture, vec2u((u32(uv_col) + 1u), uv_offset + uv_row), 0).r);
+
+    let rgb = set_color_standard(vec3f(y, u, v));
+
+    return vec4f(rgb, 1.0);
+}
+
+/// Loads an RGBA texel from a texture holding an YUY2 encoded image at the given screen space coordinates.
+fn decode_yuy2(texture: texture_2d<u32>, coords: vec2i) -> vec4f {
+    // texture is 2 * width * height
+    // every 4 bytes is 2 pixels
+    let uv_row = u32(coords.y);
+    // multiply by 2 because the width is multiplied by 2
+    let y_col = u32(coords.x) * 2u;
+    let y = f32(textureLoad(texture, vec2u(y_col, uv_row), 0).r);
+
+    // at odd pixels we're in the second half of the yuyu block, offset back by 2
+    let uv_col = y_col - u32(coords.x % 2) * 2u;
+    let u = f32(textureLoad(texture, vec2u(uv_col + 1u, uv_row), 0).r);
+    let v = f32(textureLoad(texture, vec2u(uv_col + 3u, uv_row), 0).r);
+
+    let rgb = set_color_standard(vec3f(y, u, v));
+
+    return vec4f(rgb, 1.0);
+}
+
+/// Sets the color standard for the given YUV color.
+///
+/// This conversion mirrors the function in `crates/re_types/src/datatypes/tensor_data_ext.rs`
+///
+/// Specifying the color standard should be exposed in the future [#3541](https://github.com/rerun-io/rerun/pull/3541)
+fn set_color_standard(yuv: vec3f) -> vec3f {
+    // rescale YUV values
+    let y = (yuv.x - 16.0) / 219.0;
+    let u = (yuv.y - 128.0) / 224.0;
+    let v = (yuv.z - 128.0) / 224.0;
 
-    // Specifying the color standard should be exposed in the future (https://github.com/rerun-io/rerun/pull/3541)
     // BT.601 (aka. SDTV, aka. Rec.601). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
-    let r = clamp(y + 1.402 * v, 0.0, 1.0);
-    let g = clamp(y  - (0.344 * u + 0.714 * v), 0.0, 1.0);
-    let b = clamp(y + 1.772 * u, 0.0, 1.0);
+    let r = y + 1.402 * v;
+    let g = y - 0.344 * u - 0.714 * v;
+    let b = y + 1.772 * u;
+
     // BT.709 (aka. HDTV, aka. Rec.709). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion
-    // let r = clamp(y + 1.5748 * v, 0.0, 1.0);
-    // let g = clamp(y + u * -0.1873 + v * -0.4681, 0.0, 1.0);
-    // let b = clamp(y + u * 1.8556, 0.0 , 1.0);
-    return vec4f(r, g, b, 1.0);
+    // let r = y + 1.575 * v;
+    // let g = y - 0.187 * u - 0.468 * v;
+    // let b = y + 1.856 * u;
+
+    return vec3f(
+        clamp(r, 0.0, 1.0),
+        clamp(g, 0.0, 1.0),
+        clamp(b, 0.0, 1.0)
+    );
 }

crates/re_renderer/shader/rectangle.wgsl

@@ -7,6 +7,7 @@ const SAMPLE_TYPE_FLOAT = 1u;
 const SAMPLE_TYPE_SINT  = 2u;
 const SAMPLE_TYPE_UINT  = 3u;
 const SAMPLE_TYPE_NV12  = 4u;
+const SAMPLE_TYPE_YUY2  = 5u;
 
 // How do we do colormapping?
 const COLOR_MAPPER_OFF_GRAYSCALE = 1u;

crates/re_renderer/shader/rectangle_fs.wgsl

@@ -75,14 +75,16 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
         texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
     } else if rect_info.sample_type == SAMPLE_TYPE_NV12 {
         texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
+    } else if rect_info.sample_type == SAMPLE_TYPE_YUY2 {
+        texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
     }
 
     let coord = in.texcoord * texture_dimensions;
     let clamped_coord = clamp_to_edge_nearest_neighbor(coord, texture_dimensions);
     let v00_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(-0.5, -0.5), texture_dimensions);
-    let v01_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(-0.5,  0.5), texture_dimensions);
-    let v10_coord = clamp_to_edge_nearest_neighbor(coord + vec2f( 0.5, -0.5), texture_dimensions);
-    let v11_coord = clamp_to_edge_nearest_neighbor(coord + vec2f( 0.5,  0.5), texture_dimensions);
+    let v01_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(-0.5, 0.5), texture_dimensions);
+    let v10_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(0.5, -0.5), texture_dimensions);
+    let v11_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(0.5, 0.5), texture_dimensions);
 
     if rect_info.sample_type == SAMPLE_TYPE_FLOAT {
         if tex_filter(coord) == FILTER_NEAREST {
@@ -97,8 +99,7 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
             let v11 = textureLoad(texture_float, v11_coord, 0);
             normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
         }
-    }
-    else if rect_info.sample_type == SAMPLE_TYPE_SINT {
+    } else if rect_info.sample_type == SAMPLE_TYPE_SINT {
         if tex_filter(coord) == FILTER_NEAREST {
             // nearest
             normalized_value = decode_color(vec4f(textureLoad(texture_sint, clamped_coord, 0)));
@@ -110,8 +111,7 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
             let v11 = vec4f(textureLoad(texture_sint, v11_coord, 0));
             normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
         }
-    }
-    else if rect_info.sample_type == SAMPLE_TYPE_UINT {
+    } else if rect_info.sample_type == SAMPLE_TYPE_UINT {
         if tex_filter(coord) == FILTER_NEAREST {
             // nearest
             normalized_value = decode_color(vec4f(textureLoad(texture_uint, clamped_coord, 0)));
@@ -135,8 +135,19 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
             let v11 = decode_nv12(texture_uint, v11_coord);
             normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
         }
-    }
-    else {
+    } else if rect_info.sample_type == SAMPLE_TYPE_YUY2 {
+        if tex_filter(coord) == FILTER_NEAREST {
+            // nearest
+            normalized_value = decode_color(vec4f(decode_yuy2(texture_uint, clamped_coord)));
+        } else {
+            // bilinear
+            let v00 = decode_yuy2(texture_uint, v00_coord);
+            let v01 = decode_yuy2(texture_uint, v01_coord);
+            let v10 = decode_yuy2(texture_uint, v10_coord);
+            let v11 = decode_yuy2(texture_uint, v11_coord);
+            normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
+        }
+    } else {
         return ERROR_RGBA; // unknown sample type
     }
 

crates/re_renderer/shader/rectangle_vs.wgsl

@@ -4,15 +4,17 @@
 @vertex
 fn vs_main(@builtin(vertex_index) v_idx: u32) -> VertexOut {
     let texcoord = vec2f(f32(v_idx / 2u), f32(v_idx % 2u));
-    let pos = texcoord.x * rect_info.extent_u + texcoord.y * rect_info.extent_v +
-                rect_info.top_left_corner_position;
+    let pos = texcoord.x * rect_info.extent_u + texcoord.y * rect_info.extent_v + rect_info.top_left_corner_position;
 
     var out: VertexOut;
     out.position = apply_depth_offset(frame.projection_from_world * vec4f(pos, 1.0), rect_info.depth_offset);
     out.texcoord = texcoord;
     if rect_info.sample_type == SAMPLE_TYPE_NV12 {
         out.texcoord.y /= 1.5;
     }
+    if rect_info.sample_type == SAMPLE_TYPE_YUY2 {
+        out.texcoord.x /= 2.0;
+    }
 
     return out;
 }

crates/re_renderer/src/renderer/rectangles.rs

@@ -50,6 +50,7 @@ pub enum TextureFilterMin {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum ShaderDecoding {
     Nv12,
+    Yuy2,
 }
 
 /// Describes a texture and how to map it to a color.
@@ -146,6 +147,10 @@ impl ColormappedTexture {
                 let [width, height] = self.texture.width_height();
                 [width, height * 2 / 3]
             }
+            Some(ShaderDecoding::Yuy2) => {
+                let [width, height] = self.texture.width_height();
+                [width / 2, height]
+            }
             _ => self.texture.width_height(),
         }
     }
@@ -233,6 +238,7 @@ mod gpu_data {
     const SAMPLE_TYPE_SINT: u32 = 2;
     const SAMPLE_TYPE_UINT: u32 = 3;
     const SAMPLE_TYPE_NV12: u32 = 4;
+    const SAMPLE_TYPE_YUY2: u32 = 5;
 
     // How do we do colormapping?
     const COLOR_MAPPER_OFF_GRAYSCALE: u32 = 1;
@@ -314,6 +320,8 @@ mod gpu_data {
                 Some(wgpu::TextureSampleType::Uint) => {
                     if shader_decoding == &Some(super::ShaderDecoding::Nv12) {
                         SAMPLE_TYPE_NV12
+                    } else if shader_decoding == &Some(super::ShaderDecoding::Yuy2) {
+                        SAMPLE_TYPE_YUY2
                     } else {
                         SAMPLE_TYPE_UINT
                     }

crates/re_space_view_bar_chart/src/space_view_class.rs

@@ -234,6 +234,13 @@ impl SpaceViewClass for BarChartSpaceView {
                             );
                             continue;
                         }
+                        TensorBuffer::Yuy2(_) => {
+                            re_log::warn_once!(
+                                "trying to display YUY2 data as a bar chart ({:?})",
+                                ent_path
+                            );
+                            continue;
+                        }
                     };
 
                     plot_ui.bar_chart(chart);

crates/re_space_view_tensor/src/tensor_slice_to_gpu.rs

@@ -49,8 +49,9 @@ pub fn colormapped_texture(
         multiply_rgb_with_alpha: false,
         gamma: color_mapping.gamma,
         color_mapper: re_renderer::renderer::ColorMapper::Function(color_mapping.map),
-        shader_decoding: match &tensor.buffer {
-            &TensorBuffer::Nv12(_) => Some(ShaderDecoding::Nv12),
+        shader_decoding: match tensor.buffer {
+            TensorBuffer::Nv12(_) => Some(ShaderDecoding::Nv12),
+            TensorBuffer::Yuy2(_) => Some(ShaderDecoding::Yuy2),
             _ => None,
         },
     })

crates/re_types/definitions/rerun/datatypes/tensor_buffer.fbs

@@ -55,6 +55,9 @@ table JPEGBuffer(order: 100, transparent) {
 table NV12Buffer(order: 100, transparent) {
   data: [ubyte] (order: 100);
 }
+table YUY2Buffer(order: 100, transparent) {
+  data: [ubyte] (order: 100);
+}
 
 
 /// The underlying storage for a `Tensor`.
@@ -76,4 +79,5 @@ union TensorBuffer (
   F64: F64Buffer (transparent, order:1200),
   JPEG: JPEGBuffer (transparent, order:1300),
   NV12: NV12Buffer (transparent, order:1400),
+  YUY2: YUY2Buffer (transparent, order:1500),
 }