Support YUY2-encoded images

crates/re_data_ui/src/image.rs

@@ -381,6 +381,11 @@ pub fn tensor_summary_ui_grid_contents(
             ui.label("NV12");
             ui.end_row();
         }
+        TensorBuffer::Yuy2(_) => {
+            re_ui.grid_left_hand_label(ui, "Encoding");
+            ui.label("YUY2");
+            ui.end_row();
+        }
     }
 
     let TensorStats {
@@ -713,6 +718,7 @@ fn tensor_pixel_value_ui(
                 // TODO(jleibs): Track RGB ordering somehow -- don't just assume it
                 if let Some([r, g, b]) = match &tensor.buffer {
                     TensorBuffer::Nv12(_) => tensor.get_nv12_pixel(x, y),
+                    TensorBuffer::Yuy2(_) => tensor.get_yuy2_pixel(x, y),
                     _ => {
                         if let [Some(r), Some(g), Some(b)] = [
                             tensor.get_with_image_coords(x, y, 0),

crates/re_renderer/shader/decodings.wgsl

@@ -8,18 +8,58 @@ fn decode_nv12(texture: texture_2d<u32>, coords: vec2i) -> vec4f {
     let uv_row = u32(coords.y / 2);
     var uv_col = u32(coords.x / 2) * 2u;
 
-    let y = max(0.0, (f32(textureLoad(texture, vec2u(coords), 0).r) - 16.0)) / 219.0;
-    let u = (f32(textureLoad(texture, vec2u(u32(uv_col), uv_offset + uv_row), 0).r) - 128.0) / 224.0;
-    let v = (f32(textureLoad(texture, vec2u((u32(uv_col) + 1u), uv_offset + uv_row), 0).r) - 128.0) / 224.0;
+    let y = f32(textureLoad(texture, vec2u(coords), 0).r);
+    let u = f32(textureLoad(texture, vec2u(u32(uv_col), uv_offset + uv_row), 0).r);
+    let v = f32(textureLoad(texture, vec2u((u32(uv_col) + 1u), uv_offset + uv_row), 0).r);
+
+    let rgb = set_color_standard(vec3f(y, u, v));
+
+    return vec4f(rgb, 1.0);
+}
+
+/// Loads an RGBA texel from a texture holding an YUY2 encoded image at the given screen space coordinates.
+fn decode_yuy2(texture: texture_2d<u32>, coords: vec2i) -> vec4f {
+    // texture is 2 * width * height
+    // every 4 bytes is 2 pixels
+    let uv_row = u32(coords.y);
+    // multiply by 2 because the width is multiplied by 2
+    let y_col = u32(coords.x) * 2u;
+    let y = f32(textureLoad(texture, vec2u(y_col, uv_row), 0).r);
+
+    // at odd pixels we're in the second half of the yuyu block, offset back by 2
+    let uv_col = y_col - u32(coords.x % 2) * 2u;
+    let u = f32(textureLoad(texture, vec2u(uv_col + 1u, uv_row), 0).r);
+    let v = f32(textureLoad(texture, vec2u(uv_col + 3u, uv_row), 0).r);
+
+    let rgb = set_color_standard(vec3f(y, u, v));
+
+    return vec4f(rgb, 1.0);
+}
+
+/// Sets the color standard for the given YUV color.
+///
+/// This conversion mirrors the function in `crates/re_types/src/datatypes/tensor_data_ext.rs`
+///
+/// Specifying the color standard should be exposed in the future [#3541](https://github.com/rerun-io/rerun/pull/3541)
+fn set_color_standard(yuv: vec3f) -> vec3f {
+    // rescale YUV values
+    let y = (yuv.x - 16.0) / 219.0;
+    let u = (yuv.y - 128.0) / 224.0;
+    let v = (yuv.z - 128.0) / 224.0;
 
-    // Specifying the color standard should be exposed in the future (https://github.com/rerun-io/rerun/pull/3541)
     // BT.601 (aka. SDTV, aka. Rec.601). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
-    let r = clamp(y + 1.402 * v, 0.0, 1.0);
-    let g = clamp(y  - (0.344 * u + 0.714 * v), 0.0, 1.0);
-    let b = clamp(y + 1.772 * u, 0.0, 1.0);
+    let r = y + 1.402 * v;
+    let g = y - 0.344 * u - 0.714 * v;
+    let b = y + 1.772 * u;
+
     // BT.709 (aka. HDTV, aka. Rec.709). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion
-    // let r = clamp(y + 1.5748 * v, 0.0, 1.0);
-    // let g = clamp(y + u * -0.1873 + v * -0.4681, 0.0, 1.0);
-    // let b = clamp(y + u * 1.8556, 0.0 , 1.0);
-    return vec4f(r, g, b, 1.0);
+    // let r = y + 1.575 * v;
+    // let g = y - 0.187 * u - 0.468 * v;
+    // let b = y + 1.856 * u;
+
+    return vec3f(
+        clamp(r, 0.0, 1.0),
+        clamp(g, 0.0, 1.0),
+        clamp(b, 0.0, 1.0)
+    );
 }

crates/re_renderer/shader/rectangle.wgsl

@@ -7,6 +7,7 @@ const SAMPLE_TYPE_FLOAT = 1u;
 const SAMPLE_TYPE_SINT  = 2u;
 const SAMPLE_TYPE_UINT  = 3u;
 const SAMPLE_TYPE_NV12  = 4u;
+const SAMPLE_TYPE_YUY2  = 5u;
 
 // How do we do colormapping?
 const COLOR_MAPPER_OFF_GRAYSCALE = 1u;

crates/re_renderer/shader/rectangle_fs.wgsl

@@ -75,14 +75,16 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
         texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
     } else if rect_info.sample_type == SAMPLE_TYPE_NV12 {
         texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
+    } else if rect_info.sample_type == SAMPLE_TYPE_YUY2 {
+        texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
     }
 
     let coord = in.texcoord * texture_dimensions;
     let clamped_coord = clamp_to_edge_nearest_neighbor(coord, texture_dimensions);
     let v00_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(-0.5, -0.5), texture_dimensions);
-    let v01_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(-0.5,  0.5), texture_dimensions);
-    let v10_coord = clamp_to_edge_nearest_neighbor(coord + vec2f( 0.5, -0.5), texture_dimensions);
-    let v11_coord = clamp_to_edge_nearest_neighbor(coord + vec2f( 0.5,  0.5), texture_dimensions);
+    let v01_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(-0.5, 0.5), texture_dimensions);
+    let v10_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(0.5, -0.5), texture_dimensions);
+    let v11_coord = clamp_to_edge_nearest_neighbor(coord + vec2f(0.5, 0.5), texture_dimensions);
 
     if rect_info.sample_type == SAMPLE_TYPE_FLOAT {
         if tex_filter(coord) == FILTER_NEAREST {
@@ -97,8 +99,7 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
             let v11 = textureLoad(texture_float, v11_coord, 0);
             normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
         }
-    }
-    else if rect_info.sample_type == SAMPLE_TYPE_SINT {
+    } else if rect_info.sample_type == SAMPLE_TYPE_SINT {
         if tex_filter(coord) == FILTER_NEAREST {
             // nearest
             normalized_value = decode_color(vec4f(textureLoad(texture_sint, clamped_coord, 0)));
@@ -110,8 +111,7 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
             let v11 = vec4f(textureLoad(texture_sint, v11_coord, 0));
             normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
         }
-    }
-    else if rect_info.sample_type == SAMPLE_TYPE_UINT {
+    } else if rect_info.sample_type == SAMPLE_TYPE_UINT {
         if tex_filter(coord) == FILTER_NEAREST {
             // nearest
             normalized_value = decode_color(vec4f(textureLoad(texture_uint, clamped_coord, 0)));
@@ -135,8 +135,19 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
             let v11 = decode_nv12(texture_uint, v11_coord);
             normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
         }
-    }
-    else {
+    } else if rect_info.sample_type == SAMPLE_TYPE_YUY2 {
+        if tex_filter(coord) == FILTER_NEAREST {
+            // nearest
+            normalized_value = decode_color(vec4f(decode_yuy2(texture_uint, clamped_coord)));
+        } else {
+            // bilinear
+            let v00 = decode_yuy2(texture_uint, v00_coord);
+            let v01 = decode_yuy2(texture_uint, v01_coord);
+            let v10 = decode_yuy2(texture_uint, v10_coord);
+            let v11 = decode_yuy2(texture_uint, v11_coord);
+            normalized_value = decode_color_and_filter_bilinear(coord, v00, v01, v10, v11);
+        }
+    } else {
         return ERROR_RGBA; // unknown sample type
     }
 

crates/re_renderer/shader/rectangle_vs.wgsl

@@ -4,15 +4,17 @@
 @vertex
 fn vs_main(@builtin(vertex_index) v_idx: u32) -> VertexOut {
     let texcoord = vec2f(f32(v_idx / 2u), f32(v_idx % 2u));
-    let pos = texcoord.x * rect_info.extent_u + texcoord.y * rect_info.extent_v +
-                rect_info.top_left_corner_position;
+    let pos = texcoord.x * rect_info.extent_u + texcoord.y * rect_info.extent_v + rect_info.top_left_corner_position;
 
     var out: VertexOut;
     out.position = apply_depth_offset(frame.projection_from_world * vec4f(pos, 1.0), rect_info.depth_offset);
     out.texcoord = texcoord;
     if rect_info.sample_type == SAMPLE_TYPE_NV12 {
         out.texcoord.y /= 1.5;
     }
+    if rect_info.sample_type == SAMPLE_TYPE_YUY2 {
+        out.texcoord.x /= 2.0;
+    }
 
     return out;
 }

crates/re_renderer/src/renderer/rectangles.rs

@@ -50,6 +50,7 @@ pub enum TextureFilterMin {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum ShaderDecoding {
     Nv12,
+    Yuy2,
 }
 
 /// Describes a texture and how to map it to a color.
@@ -146,6 +147,10 @@ impl ColormappedTexture {
                 let [width, height] = self.texture.width_height();
                 [width, height * 2 / 3]
             }
+            Some(ShaderDecoding::Yuy2) => {
+                let [width, height] = self.texture.width_height();
+                [width / 2, height]
+            }
             _ => self.texture.width_height(),
         }
     }
@@ -233,6 +238,7 @@ mod gpu_data {
     const SAMPLE_TYPE_SINT: u32 = 2;
     const SAMPLE_TYPE_UINT: u32 = 3;
     const SAMPLE_TYPE_NV12: u32 = 4;
+    const SAMPLE_TYPE_YUY2: u32 = 5;
 
     // How do we do colormapping?
     const COLOR_MAPPER_OFF_GRAYSCALE: u32 = 1;
@@ -314,6 +320,8 @@ mod gpu_data {
                 Some(wgpu::TextureSampleType::Uint) => {
                     if shader_decoding == &Some(super::ShaderDecoding::Nv12) {
                         SAMPLE_TYPE_NV12
+                    } else if shader_decoding == &Some(super::ShaderDecoding::Yuy2) {
+                        SAMPLE_TYPE_YUY2
                     } else {
                         SAMPLE_TYPE_UINT
                     }

crates/re_space_view_bar_chart/src/space_view_class.rs

@@ -234,6 +234,13 @@ impl SpaceViewClass for BarChartSpaceView {
                             );
                             continue;
                         }
+                        TensorBuffer::Yuy2(_) => {
+                            re_log::warn_once!(
+                                "trying to display YUY2 data as a bar chart ({:?})",
+                                ent_path
+                            );
+                            continue;
+                        }
                     };
 
                     plot_ui.bar_chart(chart);

crates/re_space_view_tensor/src/tensor_slice_to_gpu.rs

@@ -49,8 +49,9 @@ pub fn colormapped_texture(
         multiply_rgb_with_alpha: false,
         gamma: color_mapping.gamma,
         color_mapper: re_renderer::renderer::ColorMapper::Function(color_mapping.map),
-        shader_decoding: match &tensor.buffer {
-            &TensorBuffer::Nv12(_) => Some(ShaderDecoding::Nv12),
+        shader_decoding: match tensor.buffer {
+            TensorBuffer::Nv12(_) => Some(ShaderDecoding::Nv12),
+            TensorBuffer::Yuy2(_) => Some(ShaderDecoding::Yuy2),
             _ => None,
         },
     })

crates/re_types/definitions/rerun/datatypes/tensor_buffer.fbs

@@ -55,6 +55,9 @@ table JPEGBuffer(order: 100, transparent) {
 table NV12Buffer(order: 100, transparent) {
   data: [ubyte] (order: 100);
 }
+table YUY2Buffer(order: 100, transparent) {
+  data: [ubyte] (order: 100);
+}
 
 
 /// The underlying storage for a `Tensor`.
@@ -76,4 +79,5 @@ union TensorBuffer (
   F64: F64Buffer (transparent, order:1200),
   JPEG: JPEGBuffer (transparent, order:1300),
   NV12: NV12Buffer (transparent, order:1400),
+  YUY2: YUY2Buffer (transparent, order:1500),
 }