feat: Session Replay preload optimizations (#982)

src/features/session_replay/aggregate/index.js

@@ -54,7 +54,6 @@ export class Aggregate extends AggregateBase {
     this.timeKeeper = undefined
 
     this.recorder = args?.recorder
-    this.preloaded = !!this.recorder
     this.errorNoticed = args?.errorNoticed || false
 
     handle(SUPPORTABILITY_METRIC_CHANNEL, ['Config/SessionReplay/Enabled'], undefined, FEATURE_NAMES.metrics, this.ee)
@@ -112,6 +111,10 @@ export class Aggregate extends AggregateBase {
       this.forceStop(this.mode !== MODE.ERROR)
     }, this.featureName, this.ee)
 
+    registerHandler(SR_EVENT_EMITTER_TYPES.ERROR_DURING_REPLAY, e => {
+      this.handleError(e)
+    }, this.featureName, this.ee)
+
     const { error_sampling_rate, sampling_rate, autoStart, block_selector, mask_text_selector, mask_all_inputs, inline_stylesheet, inline_images, collect_fonts } = getConfigurationValue(this.agentIdentifier, 'session_replay')
 
     this.waitForFlags(['sr']).then(([flagOn]) => {
@@ -156,15 +159,15 @@ export class Aggregate extends AggregateBase {
   }
 
   switchToFull () {
+    if (!this.entitled || this.blocked) return
     this.mode = MODE.FULL
     // if the error was noticed AFTER the recorder was already imported....
     if (this.recorder && this.initialized) {
-      this.recorder.stopRecording()
-      this.recorder.startRecording()
-
+      if (!this.recorder.recording) this.recorder.startRecording()
       this.scheduler.startTimer(this.harvestTimeSeconds)
-
       this.syncWithSessionManager({ sessionReplayMode: this.mode })
+    } else {
+      this.initializeRecording(false, true, true)
     }
   }
 
@@ -221,7 +224,6 @@ export class Aggregate extends AggregateBase {
 
     // If an error was noticed before the mode could be set (like in the early lifecycle of the page), immediately set to FULL mode
     if (this.mode === MODE.ERROR && this.errorNoticed) this.mode = MODE.FULL
-    if (!this.preloaded) this.ee.on('err', e => this.handleError(e))
 
     // FULL mode records AND reports from the beginning, while ERROR mode only records (but does not report).
     // ERROR mode will do this until an error is thrown, and then switch into FULL mode.
@@ -301,7 +303,7 @@ export class Aggregate extends AggregateBase {
   }
 
   getCorrectedTimestamp (node) {
-    if (!node.timestamp) return
+    if (!node?.timestamp) return
     if (node.__newrelic) return node.timestamp
     return this.timeKeeper.correctAbsoluteTimestamp(node.timestamp)
   }

src/features/session_replay/constants.js

@@ -6,7 +6,8 @@ export const FEATURE_NAME = FEATURE_NAMES.sessionReplay
 export const SR_EVENT_EMITTER_TYPES = {
   RECORD: 'recordReplay',
   PAUSE: 'pauseReplay',
-  REPLAY_RUNNING: 'replayRunning'
+  REPLAY_RUNNING: 'replayRunning',
+  ERROR_DURING_REPLAY: 'errorDuringReplay'
 }
 
 export const AVG_COMPRESSION = 0.12

src/features/session_replay/instrument/index.js

@@ -9,30 +9,40 @@
  * It is not production ready, and is not intended to be imported or implemented in any build of the browser agent until
  * functionality is validated and a full user experience is curated.
  */
+import { handle } from '../../../common/event-emitter/handle'
 import { DEFAULT_KEY, MODE, PREFIX } from '../../../common/session/constants'
 import { InstrumentBase } from '../../utils/instrument-base'
-import { FEATURE_NAME } from '../constants'
+import { FEATURE_NAME, SR_EVENT_EMITTER_TYPES } from '../constants'
 import { isPreloadAllowed } from '../shared/utils'
 
 export class Instrument extends InstrumentBase {
   static featureName = FEATURE_NAME
   constructor (agentIdentifier, aggregator, auto = true) {
     super(agentIdentifier, aggregator, FEATURE_NAME, auto)
     let session
+    this.replayRunning = false
     try {
       session = JSON.parse(localStorage.getItem(`${PREFIX}_${DEFAULT_KEY}`))
     } catch (err) { }
 
     if (this.#canPreloadRecorder(session)) {
-      /** If this is preloaded, set up a buffer, if not, later when sampling we will set up a .on for live events */
-      this.ee.on('err', (e) => {
-        this.errorNoticed = true
-        if (this.featAggregate) this.featAggregate.handleError()
-      })
       this.#startRecording(session?.sessionReplayMode)
     } else {
       this.importAggregator()
     }
+
+    /** If the recorder is running, we can pass error events on to the agg to help it switch to full mode later */
+    this.ee.on('err', (e) => {
+      if (this.replayRunning) {
+        this.errorNoticed = true
+        handle(SR_EVENT_EMITTER_TYPES.ERROR_DURING_REPLAY, [e], undefined, this.featureName, this.ee)
+      }
+    })
+
+    /** Emitted by the recorder when it starts capturing data, used to determine if we should pass errors on to the agg */
+    this.ee.on(SR_EVENT_EMITTER_TYPES.REPLAY_RUNNING, (isRunning) => {
+      this.replayRunning = isRunning
+    })
   }
 
   // At this point wherein session state exists already but we haven't init SessionEntity aka verify timers.

src/features/session_replay/shared/recorder.js

@@ -89,12 +89,10 @@ export class Recorder {
       checkoutEveryNms: CHECKOUT_MS[this.parent.mode]
     })
 
-    this.parent.ee.emit(SR_EVENT_EMITTER_TYPES.REPLAY_RUNNING, [true, this.parent.mode])
-
     this.stopRecording = () => {
       this.recording = false
       this.parent.ee.emit(SR_EVENT_EMITTER_TYPES.REPLAY_RUNNING, [false, this.parent.mode])
-      stop()
+      stop?.()
     }
   }
 
@@ -141,6 +139,11 @@ export class Recorder {
 
     if (this.parent.blocked) return
 
+    if (!this.notified) {
+      this.parent.ee.emit(SR_EVENT_EMITTER_TYPES.REPLAY_RUNNING, [true, this.parent.mode])
+      this.notified = true
+    }
+
     if (this.parent.timeKeeper?.ready && !event.__newrelic) {
       event.__newrelic = buildNRMetaNode(event.timestamp, this.parent.timeKeeper)
       event.timestamp = this.parent.timeKeeper.correctAbsoluteTimestamp(event.timestamp)
@@ -172,7 +175,7 @@ export class Recorder {
 
     // We are making an effort to try to keep payloads manageable for unloading.  If they reach the unload limit before their interval,
     // it will send immediately.  This often happens on the first snapshot, which can be significantly larger than the other payloads.
-    if (payloadSize > IDEAL_PAYLOAD_SIZE && this.parent.mode !== MODE.ERROR) {
+    if (((event.type === RRWEB_EVENT_TYPES.FullSnapshot && this.currentBufferTarget.hasMeta) || payloadSize > IDEAL_PAYLOAD_SIZE) && this.parent.mode === MODE.FULL) {
       // if we've made it to the ideal size of ~64kb before the interval timer, we should send early.
       if (this.parent.scheduler) {
         this.parent.scheduler.runHarvest()

src/features/utils/nr1-debugger.js

@@ -0,0 +1,22 @@
+import { gosCDN } from '../../common/window/nreum'
+
+const debugId = 1
+const newrelic = gosCDN()
+export function debugNR1 (agentIdentifier, location, event, otherprops = {}, debugName = 'SR') {
+  const api = agentIdentifier ? newrelic.initializedAgents[agentIdentifier].api.addPageAction : newrelic.addPageAction
+  let url
+  try {
+    const locURL = new URL(window.location)
+    url = locURL.pathname
+  } catch (err) {
+
+  }
+  api(debugName, {
+    debugId,
+    url,
+    location,
+    event,
+    now: performance.now(),
+    ...otherprops
+  })
+}

tests/components/session_replay/aggregate.test.js

@@ -6,7 +6,6 @@ import { configure } from '../../../src/loaders/configure/configure'
 import { Recorder } from '../../../src/features/session_replay/shared/recorder'
 import { MODE, SESSION_EVENTS } from '../../../src/common/session/constants'
 import { setNREUMInitializedAgent } from '../../../src/common/window/nreum'
-import { handle } from '../../../src/common/event-emitter/handle'
 import { FEATURE_NAMES } from '../../../src/loaders/features/features'
 import { ee } from '../../../src/common/event-emitter/contextual-ee'
 import { TimeKeeper } from '../../../src/common/timing/time-keeper'
@@ -214,7 +213,7 @@ describe('Session Replay', () => {
   describe('Session Replay Error Mode Behaviors', () => {
     test('An error BEFORE rrweb import starts running in ERROR from beginning (when not preloaded)', async () => {
       setConfiguration(agentIdentifier, { session_replay: { preload: false, error_sampling_rate: 100, sampling_rate: 0 } })
-      handle('err', ['test1'], undefined, FEATURE_NAMES.sessionReplay, ee.get(agentIdentifier))
+      ee.get(agentIdentifier).emit('errorDuringReplay', ['test1'], undefined, FEATURE_NAMES.sessionReplay, ee.get(agentIdentifier))
       sr = new SessionReplayAgg(agentIdentifier, new Aggregator({}))
       sr.ee.emit('rumresp', [{ sr: 1 }])
       await wait(100)
@@ -229,7 +228,7 @@ describe('Session Replay', () => {
       await wait(1)
       expect(sr.mode).toEqual(MODE.ERROR)
       expect(sr.scheduler.started).toEqual(false)
-      sr.ee.emit('err', ['test2'])
+      sr.ee.emit('errorDuringReplay', ['test2'])
       expect(sr.mode).toEqual(MODE.FULL)
       expect(sr.scheduler.started).toEqual(true)
     })
@@ -243,7 +242,9 @@ describe('Session Replay', () => {
       setConfiguration(agentIdentifier, { ...init })
       sr = new SessionReplayAgg(agentIdentifier, new Aggregator({}))
       sr.ee.emit('rumresp', [{ sr: 1 }])
+      sr.scheduler.runHarvest = jest.fn()
       await wait(1)
+      expect(sr.scheduler.runHarvest).toHaveBeenCalledTimes(1)
       const harvestContents = sr.getHarvestContents()
       // query attrs
       expect(harvestContents.qs).toMatchObject(anyQuery)
@@ -264,6 +265,7 @@ describe('Session Replay', () => {
       setConfiguration(agentIdentifier, { ...init })
       sr = new SessionReplayAgg(agentIdentifier, new Aggregator({}))
       sr.ee.emit('rumresp', [{ sr: 1 }])
+      sr.scheduler.runHarvest = jest.fn()
       await wait(1)
       const [harvestContents] = sr.prepareHarvest()
       expect(harvestContents.qs).toMatchObject(anyQuery)
@@ -282,6 +284,7 @@ describe('Session Replay', () => {
       setConfiguration(agentIdentifier, { ...init })
       sr = new SessionReplayAgg(agentIdentifier, new Aggregator({}))
       sr.ee.emit('rumresp', [{ sr: 1 }])
+      sr.scheduler.runHarvest = jest.fn()
       await wait(1)
 
       sr.gzipper = undefined

tests/components/soft_navigations/api.test.js

@@ -42,7 +42,7 @@ describe('soft navigations API', () => {
     importAggregatorFn()
     await expect(softNavInstrument.onAggregateImported).resolves.toEqual(true)
     softNavAggregate = softNavInstrument.featAggregate
-    softNavAggregate.ee.emit('rumresp', [{ spa: 1 }])
+    softNavAggregate.drain()
   })
   beforeEach(() => {
     softNavAggregate.initialPageLoadInteraction = null

tests/specs/session-replay/payload.e2e.js

@@ -23,10 +23,14 @@ describe.withBrowsersMatching(notIE)('Session Replay Payload Validation', () =>
   })
 
   it('should allow for gzip', async () => {
-    await browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
-      .then(() => browser.waitForAgentLoad())
-
-    const { request: harvestContents } = await browser.testHandle.expectBlob()
+    const [{ request: harvestContents }] = await Promise.all([
+      browser.testHandle.expectBlob(),
+      browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
+        .then(() => browser.waitForAgentLoad())
+        .then(() => browser.execute(function () {
+          newrelic.noticeError(new Error('test'))
+        }))
+    ])
 
     expect((
       harvestContents.query.attributes.includes('content_encoding') &&
@@ -65,62 +69,46 @@ describe.withBrowsersMatching(notIE)('Session Replay Payload Validation', () =>
   })
 
   it('should match expected payload - standard', async () => {
-    await browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
-      .then(() => browser.waitForAgentLoad())
+    const [{ request: harvestContents }] = await Promise.all([
+      browser.testHandle.expectBlob(),
+      browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
+        .then(() => browser.waitForAgentLoad())
+    ])
 
-    const { request: harvestContents } = await browser.testHandle.expectBlob()
     const { localStorage } = await browser.getAgentSessionInfo()
 
     testExpectedReplay({ data: harvestContents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: true })
   })
 
   it('should match expected payload - error', async () => {
-    await browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
-      .then(() => browser.waitForAgentLoad())
-
-    const [{ request: harvestContents }] = await Promise.all([
+    const [{ request: harvestContents1 }, { request: harvestContents2 }] = await Promise.all([
       browser.testHandle.expectBlob(),
-      browser.execute(function () {
-        newrelic.noticeError(new Error('test'))
-      })
-    ])
-    const { localStorage } = await browser.getAgentSessionInfo()
-
-    testExpectedReplay({ data: harvestContents, session: localStorage.value, hasError: true, hasMeta: true, hasSnapshot: true, isFirstChunk: true })
-  })
-
-  it('should handle meta if separated', async () => {
-    await browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
-      .then(() => browser.waitForAgentLoad())
-
-    const events = await browser.execute(function () {
-      var instance = Object.values(newrelic.initializedAgents)[0]
-      return instance.features.session_replay.featAggregate.recorder.getEvents().events.filter(x => x.type !== 4)
-    })
-
-    expect(events.find(x => x.type === 4)).toEqual(undefined)
-
-    const [{ request: harvestContents }] = await Promise.all([
       browser.testHandle.expectBlob(),
-      browser.execute(function () {
-        newrelic.noticeError(new Error('test'))
-      })
+      browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
+        .then(() => browser.waitForAgentLoad())
+        .then(() => browser.execute(function () {
+          newrelic.noticeError(new Error('test'))
+        }))
     ])
     const { localStorage } = await browser.getAgentSessionInfo()
 
-    testExpectedReplay({ data: harvestContents, session: localStorage.value, hasError: true, hasMeta: true, hasSnapshot: true, isFirstChunk: true })
+    testExpectedReplay({ data: harvestContents1, session: localStorage.value, hasMeta: true, hasSnapshot: true, isFirstChunk: true })
+    testExpectedReplay({ data: harvestContents2, session: localStorage.value, hasMeta: false, hasSnapshot: false, isFirstChunk: false })
+    const hasError = decodeAttributes(harvestContents1.query.attributes).hasError || decodeAttributes(harvestContents2.query.attributes).hasError
+    expect(hasError).toBeTruthy()
   })
 
   /**
    * auto-inlining broken stylesheets does not work in safari browsers < 16.3
    * current mitigation strategy is defined as informing customers to add `crossOrigin: anonymous` tags to cross-domain stylesheets
   */
   it.withBrowsersMatching([notSafari, notIOS])('should place inlined css for cross origin stylesheets even if no crossOrigin tag', async () => {
-    await browser.url(await browser.testHandle.assetURL('rrweb-invalid-stylesheet.html', config()))
-      .then(() => browser.waitForFeatureAggregate('session_replay'))
-
     /** snapshot and mutation payloads */
-    const { request: { body: snapshot1, query: snapshot1Query } } = await browser.testHandle.expectSessionReplaySnapshot(10000)
+    const [{ request: { body: snapshot1, query: snapshot1Query } }] = await Promise.all([
+      browser.testHandle.expectSessionReplaySnapshot(10000),
+      browser.url(await browser.testHandle.assetURL('rrweb-invalid-stylesheet.html', config()))
+        .then(() => browser.waitForFeatureAggregate('session_replay'))
+    ])
     const snapshot1Nodes = snapshot1.filter(x => x.type === 2)
     expect(decodeAttributes(snapshot1Query.attributes).inlinedAllStylesheets).toEqual(true)
     snapshot1Nodes.forEach(snapshotNode => {