fix: only clear buffer before last I-frame

When removing buffered video data, there needs to be
at least an I-frame left in the remaining buffer, otherwise
the video playback freezes (both chrome and firefox).

With this change, we mark I-frames and make sure that there
is at least one I-frame in the remaining buffer when clearing
old buffered data.

examples/node/player.js

@@ -1,6 +1,6 @@
 const yargs = require('yargs')
 
-const { pipelines } = require('../../lib/index.node.js')
+const { pipelines } = require('../../dist/cjs/index.node.js')
 
 /**
  * Stream live from camera (to be used from Node CLI).
@@ -59,5 +59,5 @@ const config = {
 }
 
 // Setup a new pipeline
-const pipeline = new pipelines.TcpRtspMp4Pipeline(config)
+const pipeline = new pipelines.CliMp4Pipeline(config)
 pipeline.rtsp.play()

lib/components/h264depay/parser.test.ts

@@ -25,7 +25,7 @@ describe('h264 handler', () => {
     const msg = callback.mock.calls[0][0]
 
     expect(msg.timestamp).toEqual(547056949)
-    expect(msg.type).toEqual(MessageType.ELEMENTARY)
+    expect(msg.type).toEqual(MessageType.H264)
     expect(msg.data.length).toEqual(14)
     expect(msg.payloadType).toEqual(96)
   })
@@ -58,7 +58,7 @@ describe('h264 handler', () => {
     const msg = callback.mock.calls[0][0]
 
     expect(msg.timestamp).toEqual(153026579)
-    expect(msg.type).toEqual(MessageType.ELEMENTARY)
+    expect(msg.type).toEqual(MessageType.H264)
     expect(msg.data.length).toEqual(535)
     expect(msg.payloadType).toEqual(96)
   })

lib/components/h264depay/parser.ts

@@ -1,7 +1,15 @@
-import { RtpMessage, ElementaryMessage, MessageType } from '../message'
+import { RtpMessage, MessageType, H264Message } from '../message'
 import { payload, timestamp, payloadType } from '../../utils/protocols/rtp'
 import debug from 'debug'
 
+export enum NAL_TYPES {
+  UNSPECIFIED = 0,
+  NON_IDR_PICTURE = 1, // P-frame
+  IDR_PICTURE = 5, // I-frame
+  SPS = 7,
+  PPS = 8,
+}
+
 /*
 First byte in payload (rtp payload header):
       +---------------+
@@ -23,7 +31,7 @@ const h264Debug = debug('msl:h264depay')
 export function h264depay(
   buffered: Buffer,
   rtp: RtpMessage,
-  callback: (msg: ElementaryMessage) => void,
+  callback: (msg: H264Message) => void,
 ) {
   const rtpPayload = payload(rtp.data)
   const type = rtpPayload[0] & 0x1f
@@ -46,31 +54,36 @@ export function h264depay(
         rtpPayload.slice(2),
       ])
       h264frame.writeUInt32BE(h264frame.length - 4, 0)
-      const msg: ElementaryMessage = {
+      const msg: H264Message = {
         data: h264frame,
-        type: MessageType.ELEMENTARY,
+        type: MessageType.H264,
         timestamp: timestamp(rtp.data),
         ntpTimestamp: rtp.ntpTimestamp,
         payloadType: payloadType(rtp.data),
+        nalType: nalType,
       }
       callback(msg)
       return Buffer.alloc(0)
     } else {
       // Put the received data on the buffer and cut the header bytes
       return Buffer.concat([buffered, rtpPayload.slice(2)])
     }
-  } else if ((type === 1 || type === 5) && buffered.length === 0) {
+  } else if (
+    (type === NAL_TYPES.NON_IDR_PICTURE || type === NAL_TYPES.IDR_PICTURE) &&
+    buffered.length === 0
+  ) {
     /* Single NALU */ const h264frame = Buffer.concat([
       Buffer.from([0, 0, 0, 0]),
       rtpPayload,
     ])
     h264frame.writeUInt32BE(h264frame.length - 4, 0)
-    const msg: ElementaryMessage = {
+    const msg: H264Message = {
       data: h264frame,
-      type: MessageType.ELEMENTARY,
+      type: MessageType.H264,
       timestamp: timestamp(rtp.data),
       ntpTimestamp: rtp.ntpTimestamp,
       payloadType: payloadType(rtp.data),
+      nalType: type,
     }
     callback(msg)
     return Buffer.alloc(0)

lib/components/message.ts

@@ -14,6 +14,7 @@ export enum MessageType {
   RTSP,
   SDP,
   ELEMENTARY,
+  H264,
   ISOM,
   XML,
   JPEG,
@@ -52,8 +53,16 @@ export interface ElementaryMessage extends GenericMessage {
   readonly timestamp: number
 }
 
+export interface H264Message extends GenericMessage {
+  readonly type: MessageType.H264
+  readonly payloadType: number
+  readonly timestamp: number
+  readonly nalType: number
+}
+
 export interface IsomMessage extends GenericMessage {
   readonly type: MessageType.ISOM
+  readonly checkpointTime?: number // presentation time of last I-frame (s)
 }
 
 export interface XmlMessage extends GenericMessage {
@@ -79,6 +88,7 @@ export type Message =
   | RtspMessage
   | SdpMessage
   | ElementaryMessage
+  | H264Message
   | IsomMessage
   | XmlMessage
   | JpegMessage

lib/components/mp4muxer/index.ts

@@ -4,6 +4,7 @@ import { Box } from './helpers/isom'
 import { BoxBuilder } from './helpers/boxbuilder'
 import { Transform } from 'stream'
 import { Tube } from '../component'
+import { NAL_TYPES } from '../h264depay/parser'
 
 /**
  * Component that converts elementary stream data into MP4 boxes honouring
@@ -44,7 +45,10 @@ export class Mp4Muxer extends Tube {
 
           this.push(msg) // Pass on the original SDP message
           this.push({ type: MessageType.ISOM, data, ftyp, moov })
-        } else if (msg.type === MessageType.ELEMENTARY) {
+        } else if (
+          msg.type === MessageType.ELEMENTARY ||
+          msg.type === MessageType.H264
+        ) {
           /**
            * Otherwise we are getting some elementary stream data.
            * Set up the moof and mdat boxes.
@@ -61,6 +65,20 @@ export class Mp4Muxer extends Tube {
               }
             }
 
+            let checkpointTime: number | undefined = undefined
+            const idrPicture =
+              msg.type === MessageType.H264
+                ? msg.nalType === NAL_TYPES.IDR_PICTURE
+                : undefined
+            if (
+              boxBuilder.ntpPresentationTime &&
+              idrPicture &&
+              msg.ntpTimestamp !== undefined
+            ) {
+              checkpointTime =
+                (msg.ntpTimestamp - boxBuilder.ntpPresentationTime) / 1000
+            }
+
             const byteLength = msg.data.byteLength
             const moof = boxBuilder.moof({ trackId, timestamp, byteLength })
             const mdat = boxBuilder.mdat(msg.data)
@@ -75,6 +93,7 @@ export class Mp4Muxer extends Tube {
               moof,
               mdat,
               ntpTimestamp,
+              checkpointTime,
             })
           }
         } else {

lib/components/mse/index.ts

@@ -15,6 +15,7 @@ export interface MediaTrack {
 export class MseSink extends Sink {
   private _videoEl: HTMLVideoElement
   private _done?: () => void
+  private _lastCheckpointTime: number
 
   public onSourceOpen?: (mse: MediaSource, tracks: MediaTrack[]) => void
 
@@ -42,6 +43,7 @@ export class MseSink extends Sink {
       write: (msg: Message, encoding, callback) => {
         if (msg.type === MessageType.SDP) {
           // Start a new movie (new SDP info available)
+          this._lastCheckpointTime = 0
 
           // Set up a list of tracks that contain info about
           // the type of media, encoding, and codec are present.
@@ -75,6 +77,10 @@ export class MseSink extends Sink {
           }
           mse.addEventListener('sourceopen', handler)
         } else if (msg.type === MessageType.ISOM) {
+          this._lastCheckpointTime =
+            msg.checkpointTime !== undefined
+              ? msg.checkpointTime
+              : this._lastCheckpointTime
           // ISO BMFF Byte Stream data to be added to the source buffer
           this._done = callback
           try {
@@ -130,6 +136,7 @@ export class MseSink extends Sink {
     super(incoming, outgoing)
 
     this._videoEl = el
+    this._lastCheckpointTime = 0
   }
 
   /**
@@ -150,7 +157,7 @@ export class MseSink extends Sink {
 
         const index = sourceBuffer.buffered.length - 1
         const start = sourceBuffer.buffered.start(index)
-        const end = el.currentTime - 10
+        const end = Math.min(el.currentTime, this._lastCheckpointTime) - 10
         try {
           // remove all material up to 10 seconds before current time
           if (end > start) {