feat: Standardize Feature Buffering Behavior (#1155)

src/features/ajax/aggregate/chunk.js

@@ -1,5 +1,6 @@
 import { nullable, numeric, getAddStringContext, addCustomAttributes } from '../../../common/serialize/bel-serializer'
 import { getInfo } from '../../../common/config/info'
+import { MAX_PAYLOAD_SIZE } from '../../../common/constants/agent-constants'
 
 export default class Chunk {
   constructor (events, aggregateInstance) {
@@ -46,6 +47,6 @@ export default class Chunk {
       this.payload += insert
     }
 
-    this.tooBig = this.payload.length * 2 > aggregateInstance.MAX_PAYLOAD_SIZE
+    this.tooBig = this.payload.length * 2 > MAX_PAYLOAD_SIZE
   }
 }

src/features/ajax/aggregate/index.js

@@ -17,6 +17,7 @@ import { AggregateBase } from '../../utils/aggregate-base'
 import { parseGQL } from './gql'
 import { getNREUMInitializedAgent } from '../../../common/window/nreum'
 import Chunk from './chunk'
+import { EventBuffer } from '../../utils/event-buffer'
 
 export class Aggregate extends AggregateBase {
   static featureName = FEATURE_NAME
@@ -32,26 +33,24 @@ export class Aggregate extends AggregateBase {
     this.#agentInit = getConfiguration(agentIdentifier)
 
     const harvestTimeSeconds = this.#agentInit.ajax.harvestTimeSeconds || 10
-    this.MAX_PAYLOAD_SIZE = this.#agentInit.ajax.maxPayloadSize || 1000000
     setDenyList(this.#agentRuntime.denyList)
 
-    this.ajaxEvents = []
+    this.ajaxEvents = new EventBuffer()
     this.spaAjaxEvents = {}
-    this.sentAjaxEvents = []
     const classThis = this
 
     // --- v Used by old spa feature
     this.ee.on('interactionDone', (interaction, wasSaved) => {
-      if (!this.spaAjaxEvents[interaction.id]) return
+      if (!this.spaAjaxEvents[interaction.id]?.hasData) return
 
       if (!wasSaved) { // if the ixn was saved, then its ajax reqs are part of the payload whereas if it was discarded, it should still be harvested in the ajax feature itself
-        this.spaAjaxEvents[interaction.id].forEach((item) => this.ajaxEvents.push(item))
+        this.ajaxEvents.merge(this.spaAjaxEvents[interaction.id])
       }
       delete this.spaAjaxEvents[interaction.id]
     })
     // --- ^
     // --- v Used by new soft nav
-    registerHandler('returnAjax', event => this.ajaxEvents.push(event), this.featureName, this.ee)
+    registerHandler('returnAjax', event => this.ajaxEvents.add(event), this.featureName, this.ee)
     // --- ^
     registerHandler('xhr', function () { // the EE-drain system not only switches "this" but also passes a new EventContext with info. Should consider platform refactor to another system which passes a mutable context around separately and predictably to avoid problems like this.
       classThis.storeXhr(...arguments, this) // this switches the context back to the class instance while passing the NR context as an argument -- see "ctx" in storeXhr
@@ -135,33 +134,31 @@ export class Aggregate extends AggregateBase {
       handle('ajax', [event], undefined, FEATURE_NAMES.softNav, this.ee)
     } else if (ctx.spaNode) { // For old spa (when running), if the ajax happened inside an interaction, hold it until the interaction finishes
       const interactionId = ctx.spaNode.interaction.id
-      this.spaAjaxEvents[interactionId] = this.spaAjaxEvents[interactionId] || []
-      this.spaAjaxEvents[interactionId].push(event)
+      this.spaAjaxEvents[interactionId] ??= new EventBuffer()
+      this.spaAjaxEvents[interactionId].add(event)
     } else {
-      this.ajaxEvents.push(event)
+      this.ajaxEvents.add(event)
     }
   }
 
   prepareHarvest (options) {
     options = options || {}
-    if (this.ajaxEvents.length === 0) return null
+    if (this.ajaxEvents.buffer.length === 0) return null
 
-    const payload = this.#getPayload(this.ajaxEvents)
+    const payload = this.#getPayload(this.ajaxEvents.buffer)
     const payloadObjs = []
 
     for (let i = 0; i < payload.length; i++) payloadObjs.push({ body: { e: payload[i] } })
 
-    if (options.retry) this.sentAjaxEvents = this.ajaxEvents
-    this.ajaxEvents = []
+    if (options.retry) this.ajaxEvents.hold()
+    else this.ajaxEvents.clear()
 
     return payloadObjs
   }
 
   onEventsHarvestFinished (result) {
-    if (result.retry && this.sentAjaxEvents.length > 0) {
-      this.ajaxEvents.unshift(...this.sentAjaxEvents)
-      this.sentAjaxEvents = []
-    }
+    if (result.retry && this.ajaxEvents.held.hasData) this.ajaxEvents.unhold()
+    else this.ajaxEvents.held.clear()
   }
 
   #getPayload (events, numberOfChunks) {

src/features/ajax/constants.js

@@ -1,5 +1,3 @@
 import { FEATURE_NAMES } from '../../loaders/features/features'
 
 export const FEATURE_NAME = FEATURE_NAMES.ajax
-
-export const MAX_PAYLOAD_SIZE = 1000000

src/features/generic_events/aggregate/index.js

@@ -8,16 +8,17 @@ import { cleanURL } from '../../../common/url/clean-url'
 import { getInfo } from '../../../common/config/info'
 import { getConfigurationValue } from '../../../common/config/init'
 import { getRuntime } from '../../../common/config/runtime'
-import { FEATURE_NAME, IDEAL_PAYLOAD_SIZE } from '../constants'
+import { FEATURE_NAME } from '../constants'
 import { isBrowserScope } from '../../../common/constants/runtime'
 import { AggregateBase } from '../../utils/aggregate-base'
 import { warn } from '../../../common/util/console'
 import { now } from '../../../common/timing/now'
 import { registerHandler } from '../../../common/event-emitter/register-handler'
 import { deregisterDrain } from '../../../common/drain/drain'
 import { SUPPORTABILITY_METRIC_CHANNEL } from '../../metrics/constants'
-import { EventBuffer } from './event-buffer'
+import { EventBuffer } from '../../utils/event-buffer'
 import { applyFnToProps } from '../../../common/util/traverse'
+import { IDEAL_PAYLOAD_SIZE } from '../../../common/constants/agent-constants'
 
 export class Aggregate extends AggregateBase {
   #agentRuntime
@@ -31,7 +32,6 @@ export class Aggregate extends AggregateBase {
     this.referrerUrl = (isBrowserScope && document.referrer) ? cleanURL(document.referrer) : undefined
 
     this.events = new EventBuffer()
-    this.retryEvents = new EventBuffer()
 
     this.#agentRuntime = getRuntime(this.agentIdentifier)
 
@@ -120,17 +120,13 @@ export class Aggregate extends AggregateBase {
       )
     })
 
-    if (options.retry) this.retryEvents = this.events
-    this.events = new EventBuffer()
-
+    if (options.retry) this.events.hold()
     return payload
   }
 
   onHarvestFinished (result) {
-    if (result && result?.sent && result?.retry && this.retryEvents.hasData) {
-      this.events.merge(this.retryEvents, true)
-      this.retryEvents = new EventBuffer()
-    }
+    if (result && result?.sent && result?.retry && this.events.held.hasData) this.events.unhold()
+    else this.events.held.clear()
   }
 
   checkEventLimits () {

src/features/logging/constants.js

@@ -11,5 +11,3 @@ export const LOG_LEVELS = {
 export const LOGGING_EVENT_EMITTER_CHANNEL = 'log'
 
 export const FEATURE_NAME = FEATURE_NAMES.logging
-
-export const MAX_PAYLOAD_SIZE = 1000000

src/features/page_view_timing/aggregate/index.js

@@ -22,6 +22,7 @@ import { timeToFirstByte } from '../../../common/vitals/time-to-first-byte'
 import { longTask } from '../../../common/vitals/long-task'
 import { subscribeToVisibilityChange } from '../../../common/window/page-visibility'
 import { VITAL_NAMES } from '../../../common/vitals/constants'
+import { EventBuffer } from '../../utils/event-buffer'
 
 export class Aggregate extends AggregateBase {
   static featureName = FEATURE_NAME
@@ -33,8 +34,7 @@ export class Aggregate extends AggregateBase {
   constructor (agentIdentifier, aggregator) {
     super(agentIdentifier, aggregator, FEATURE_NAME)
 
-    this.timings = []
-    this.timingsSent = []
+    this.timings = new EventBuffer()
     this.curSessEndRecorded = false
 
     if (getConfigurationValue(this.agentIdentifier, 'page_view_timing.long_task') === true) longTask.subscribe(this.#handleVitalMetric)
@@ -116,7 +116,7 @@ export class Aggregate extends AggregateBase {
       attrs.cls = cumulativeLayoutShift.current.value
     }
 
-    this.timings.push({
+    this.timings.add({
       name,
       value,
       attrs
@@ -126,10 +126,8 @@ export class Aggregate extends AggregateBase {
   }
 
   onHarvestFinished (result) {
-    if (result.retry && this.timingsSent.length > 0) {
-      this.timings.unshift(...this.timingsSent)
-      this.timingsSent = []
-    }
+    if (result.retry && this.timings.held.hasData) this.timings.unhold()
+    else this.timings.held.clear()
   }
 
   appendGlobalCustomAttributes (timing) {
@@ -147,15 +145,12 @@ export class Aggregate extends AggregateBase {
 
   // serialize and return current timing data, clear and save current data for retry
   prepareHarvest (options) {
-    if (this.timings.length === 0) return
+    if (!this.timings.hasData) return
+
+    var payload = this.getPayload(this.timings.buffer)
+    if (options.retry) this.timings.hold()
+    else this.timings.clear()
 
-    var payload = this.getPayload(this.timings)
-    if (options.retry) {
-      for (var i = 0; i < this.timings.length; i++) {
-        this.timingsSent.push(this.timings[i])
-      }
-    }
-    this.timings = []
     return {
       body: { e: payload }
     }

src/features/session_replay/aggregate/index.js

@@ -8,7 +8,7 @@
 
 import { registerHandler } from '../../../common/event-emitter/register-handler'
 import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler'
-import { ABORT_REASONS, FEATURE_NAME, MAX_PAYLOAD_SIZE, QUERY_PARAM_PADDING, RRWEB_EVENT_TYPES, SR_EVENT_EMITTER_TYPES, TRIGGERS } from '../constants'
+import { ABORT_REASONS, FEATURE_NAME, QUERY_PARAM_PADDING, RRWEB_EVENT_TYPES, SR_EVENT_EMITTER_TYPES, TRIGGERS } from '../constants'
 import { getInfo } from '../../../common/config/info'
 import { getConfigurationValue } from '../../../common/config/init'
 import { getRuntime } from '../../../common/config/runtime'
@@ -27,6 +27,7 @@ import { stylesheetEvaluator } from '../shared/stylesheet-evaluator'
 import { deregisterDrain } from '../../../common/drain/drain'
 import { now } from '../../../common/timing/now'
 import { buildNRMetaNode } from '../shared/utils'
+import { MAX_PAYLOAD_SIZE } from '../../../common/constants/agent-constants'
 
 export class Aggregate extends AggregateBase {
   static featureName = FEATURE_NAME

src/features/session_replay/constants.js

@@ -19,10 +19,6 @@ export const RRWEB_EVENT_TYPES = {
   Meta: 4,
   Custom: 5
 }
-/** Vortex caps payload sizes at 1MB */
-export const MAX_PAYLOAD_SIZE = 1000000
-/** Unloading caps around 64kb */
-export const IDEAL_PAYLOAD_SIZE = 64000
 /** Interval between forcing new full snapshots -- 15 seconds in error mode (x2), 5 minutes in full mode */
 export const CHECKOUT_MS = { [MODE.ERROR]: 15000, [MODE.FULL]: 300000, [MODE.OFF]: 0 }
 export const ABORT_REASONS = {

src/features/session_replay/shared/recorder-events.js

@@ -1,27 +1,34 @@
+import { EventBuffer } from '../../utils/event-buffer'
+
 export class RecorderEvents {
-  constructor () {
-    /** The buffer to hold recorder event nodes */
-    this.events = []
-    /** Payload metadata -- Should indicate when a replay blob started recording.  Resets each time a harvest occurs.
+  /** The buffer to hold recorder event nodes */
+  #events = new EventBuffer(Infinity)
+  /** Payload metadata -- Should indicate when a replay blob started recording.  Resets each time a harvest occurs.
      * cycle timestamps are used as fallbacks if event timestamps cannot be used
      */
-    this.cycleTimestamp = Date.now()
-    /** A value which increments with every new mutation node reported. Resets after a harvest is sent */
-    this.payloadBytesEstimation = 0
-    /** Payload metadata -- Should indicate that the payload being sent has a full DOM snapshot. This can happen
-     * -- When the recording library begins recording, it starts by taking a DOM snapshot
-     * -- When visibility changes from "hidden" -> "visible", it must capture a full snapshot for the replay to work correctly across tabs
-    */
-    this.hasSnapshot = false
-    /** Payload metadata -- Should indicate that the payload being sent has a meta node. The meta node should always precede a snapshot node. */
-    this.hasMeta = false
-    /** Payload metadata -- Should indicate that the payload being sent contains an error.  Used for query/filter purposes in UI */
-    this.hasError = false
-    /** Payload metadata -- Denotes whether all stylesheet elements were able to be inlined */
-    this.inlinedAllStylesheets = true
-  }
+  cycleTimestamp = Date.now()
+  /** Payload metadata -- Should indicate that the payload being sent has a full DOM snapshot. This can happen
+   * -- When the recording library begins recording, it starts by taking a DOM snapshot
+   * -- When visibility changes from "hidden" -> "visible", it must capture a full snapshot for the replay to work correctly across tabs
+  */
+  hasSnapshot = false
+  /** Payload metadata -- Should indicate that the payload being sent has a meta node. The meta node should always precede a snapshot node. */
+  hasMeta = false
+  /** Payload metadata -- Should indicate that the payload being sent contains an error.  Used for query/filter purposes in UI */
+  hasError = false
+  /** Payload metadata -- Denotes whether all stylesheet elements were able to be inlined */
+  inlinedAllStylesheets = true
 
   add (event) {
-    this.events.push(event)
+    this.#events.add(event)
+  }
+
+  get events () {
+    return this.#events.buffer
+  }
+
+  /** A value which increments with every new mutation node reported. Resets after a harvest is sent */
+  get payloadBytesEstimation () {
+    return this.#events.bytes
   }
 }