Fix some videos having offsetted (incorrect) timestamps

crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs

@@ -19,6 +19,10 @@ table VideoFrameReference (
     ///
     /// Note that this uses the closest video frame instead of the latest at this timestamp
     /// in order to be more forgiving of rounding errors for inprecise timestamp types.
+    ///
+    /// Timestamps are relative to the start of the video, i.e. a timestamp of 0 always corresponds to the first frame.
+    /// This is oftentimes equivalent to presentation timestamps (known as PTS), but in the presence of B-frames
+    /// (bidirectionally predicted frames) there may be an offset on the first presentation timestamp in the video.
     timestamp: rerun.components.VideoTimestamp ("attr.rerun.component_required", required, order: 1000);
 
     // --- Optional ---

crates/store/re_video/src/decode/mod.rs

@@ -282,7 +282,7 @@ impl Default for FrameInfo {
 
 impl FrameInfo {
     /// Presentation timestamp range in which this frame is valid.
-    pub fn time_range(&self) -> std::ops::Range<Time> {
+    pub fn presentation_time_range(&self) -> std::ops::Range<Time> {
         self.presentation_timestamp..self.presentation_timestamp + self.duration
     }
 }

crates/store/re_video/src/demux/mod.rs

@@ -60,12 +60,57 @@ pub struct VideoData {
     /// and should be presented in composition-timestamp order.
     pub samples: Vec<Sample>,
 
+    /// Meta information about the samples.
+    pub sample_statistics: SampleStatistics,
+
     /// All the tracks in the mp4; not just the video track.
     ///
     /// Can be nice to show in a UI.
     pub mp4_tracks: BTreeMap<TrackId, Option<TrackKind>>,
 }
 
+/// Meta informationa about the video samples.
+#[derive(Clone, Debug)]
+pub struct SampleStatistics {
+    /// The smallest presentation timestamp observed in this video.
+    ///
+    /// This is typically 0, but in the presence of B-frames, it may be non-zero.
+    /// In fact, many formats don't require this to be zero, but video players typically
+    /// normalize the shown time to start at zero.
+    /// Note that timestamps in the [`Sample`]s are *not* automatically adjusted with this value.
+    // This is roughly equivalent to FFmpeg's internal `min_corrected_pts`
+    // https://github.com/FFmpeg/FFmpeg/blob/4047b887fc44b110bccb1da09bcb79d6e454b88b/libavformat/isom.h#L202
+    // (unlike us, this handles a bunch more edge cases but it fulfills the same role)
+    // To learn more about this I recommend reading the patch that introduced this in FFmpeg:
+    // https://patchwork.ffmpeg.org/project/ffmpeg/patch/[email protected]/#12592
+    pub minimum_presentation_timestamp: Time,
+
+    /// Whether all decode timestamps are equal to presentation timestamps.
+    ///
+    /// If true, the video typically has no B-frames as those require frame reordering.
+    pub dts_always_equal_pts: bool,
+}
+
+impl SampleStatistics {
+    pub fn new(samples: &[Sample]) -> Self {
+        re_tracing::profile_function!();
+
+        let minimum_presentation_timestamp = samples
+            .iter()
+            .map(|s| s.presentation_timestamp)
+            .min()
+            .unwrap_or_default();
+        let dts_always_equal_pts = samples
+            .iter()
+            .all(|s| s.decode_timestamp == s.presentation_timestamp);
+
+        Self {
+            minimum_presentation_timestamp,
+            dts_always_equal_pts,
+        }
+    }
+}
+
 impl VideoData {
     /// Loads a video from the given data.
     ///
@@ -229,17 +274,25 @@ impl VideoData {
         }
     }
 
-    /// Determines the presentation timestamps of all frames inside a video, returning raw time values.
+    /// Determines the video timestamps of all frames inside a video, returning raw time values.
     ///
     /// Returned timestamps are in nanoseconds since start and are guaranteed to be monotonically increasing.
+    /// These are *not* necessarily the same as the presentation timestamps, as the returned timestamps are
+    /// normalized respect to the start of the video, see [`SampleStatistics::minimum_presentation_timestamp`].
     pub fn frame_timestamps_ns(&self) -> impl Iterator<Item = i64> + '_ {
+        re_tracing::profile_function!();
+
         // Segments are guaranteed to be sorted among each other, but within a segment,
         // presentation timestamps may not be sorted since this is sorted by decode timestamps.
         self.gops.iter().flat_map(|seg| {
             self.samples[seg.range()]
                 .iter()
-                .map(|sample| sample.presentation_timestamp.into_nanos(self.timescale))
+                .map(|sample| sample.presentation_timestamp)
                 .sorted()
+                .map(|pts| {
+                    (pts - self.sample_statistics.minimum_presentation_timestamp)
+                        .into_nanos(self.timescale)
+                })
         })
     }
 }

crates/store/re_video/src/demux/mp4.rs

@@ -2,7 +2,7 @@
 
 use super::{Config, GroupOfPictures, Sample, VideoData, VideoLoadError};
 
-use crate::{Time, Timescale};
+use crate::{demux::SampleStatistics, Time, Timescale};
 
 impl VideoData {
     pub fn load_mp4(bytes: &[u8]) -> Result<Self, VideoLoadError> {
@@ -42,32 +42,36 @@ impl VideoData {
         let mut gops = Vec::<GroupOfPictures>::new();
         let mut gop_sample_start_index = 0;
 
-        for sample in &track.samples {
-            if sample.is_sync && !samples.is_empty() {
-                let start = samples[gop_sample_start_index].decode_timestamp;
-                let sample_range = gop_sample_start_index as u32..samples.len() as u32;
-                gops.push(GroupOfPictures {
-                    start,
-                    sample_range,
+        {
+            re_tracing::profile_scope!("copy samples & build gops");
+
+            for sample in &track.samples {
+                if sample.is_sync && !samples.is_empty() {
+                    let start = samples[gop_sample_start_index].decode_timestamp;
+                    let sample_range = gop_sample_start_index as u32..samples.len() as u32;
+                    gops.push(GroupOfPictures {
+                        start,
+                        sample_range,
+                    });
+                    gop_sample_start_index = samples.len();
+                }
+
+                let decode_timestamp = Time::new(sample.decode_timestamp as i64);
+                let presentation_timestamp = Time::new(sample.composition_timestamp as i64);
+                let duration = Time::new(sample.duration as i64);
+
+                let byte_offset = sample.offset as u32;
+                let byte_length = sample.size as u32;
+
+                samples.push(Sample {
+                    is_sync: sample.is_sync,
+                    decode_timestamp,
+                    presentation_timestamp,
+                    duration,
+                    byte_offset,
+                    byte_length,
                 });
-                gop_sample_start_index = samples.len();
             }
-
-            let decode_timestamp = Time::new(sample.decode_timestamp as i64);
-            let presentation_timestamp = Time::new(sample.composition_timestamp as i64);
-            let duration = Time::new(sample.duration as i64);
-
-            let byte_offset = sample.offset as u32;
-            let byte_length = sample.size as u32;
-
-            samples.push(Sample {
-                is_sync: sample.is_sync,
-                decode_timestamp,
-                presentation_timestamp,
-                duration,
-                byte_offset,
-                byte_length,
-            });
         }
 
         if !samples.is_empty() {
@@ -79,10 +83,13 @@ impl VideoData {
             });
         }
 
+        let sample_statistics = SampleStatistics::new(&samples);
+
         Ok(Self {
             config,
             timescale,
             duration,
+            sample_statistics,
             gops,
             samples,
             mp4_tracks,

crates/store/re_video/src/lib.rs

@@ -9,7 +9,7 @@ pub use re_mp4::{TrackId, TrackKind};
 
 pub use self::{
     decode::{Chunk, Frame, PixelFormat},
-    demux::{Config, Sample, VideoData, VideoLoadError},
+    demux::{Config, Sample, SampleStatistics, VideoData, VideoLoadError},
     time::{Time, Timescale},
 };
 

crates/viewer/re_data_ui/src/blob.rs

@@ -2,7 +2,11 @@ use re_types::components::{Blob, MediaType, VideoTimestamp};
 use re_ui::{list_item::PropertyContent, UiExt};
 use re_viewer_context::UiLayout;
 
-use crate::{image::image_preview_ui, video::show_video_blob_info, EntityDataUi};
+use crate::{
+    image::image_preview_ui,
+    video::{show_decoded_frame_info, show_video_blob_info},
+    EntityDataUi,
+};
 
 impl EntityDataUi for Blob {
     fn entity_data_ui(
@@ -105,9 +109,11 @@ pub fn blob_preview_and_save_ui(
             .entry(|c: &mut re_viewer_context::ImageDecodeCache| c.entry(row_id, blob, media_type))
             .ok()
     });
-    if let Some(image) = &image {
+
+    let video_result_for_frame_preview = if let Some(image) = &image {
         let colormap = None; // TODO(andreas): Rely on default here for now.
         image_preview_ui(ctx, ui, ui_layout, query, entity_path, image, colormap);
+        None
     }
     // Try to treat it as a video if treating it as image didn't work:
     else if let Some(blob_row_id) = blob_row_id {
@@ -122,15 +128,12 @@ pub fn blob_preview_and_save_ui(
             )
         });
 
-        show_video_blob_info(
-            ctx.render_ctx,
-            ui,
-            ui_layout,
-            &video_result,
-            video_timestamp,
-            blob,
-        );
-    }
+        show_video_blob_info(ui, ui_layout, &video_result);
+
+        Some(video_result)
+    } else {
+        None
+    };
 
     if !ui_layout.is_single_line() && ui_layout != UiLayout::Tooltip {
         ui.horizontal(|ui| {
@@ -167,4 +170,13 @@ pub fn blob_preview_and_save_ui(
             }
         });
     }
+
+    // Show a mini video player for video blobs:
+    if let Some(video_result) = &video_result_for_frame_preview {
+        if let Ok(video) = video_result.as_ref() {
+            ui.separator();
+
+            show_decoded_frame_info(ctx.render_ctx, ui, ui_layout, video, video_timestamp, blob);
+        }
+    }
 }