feat: Capture Page Resource Assets (#1257)

docs/supportability-metrics.md

@@ -124,6 +124,12 @@ A timeslice metric is harvested to the JSE/XHR consumer. An aggregation service
 * Generic/Performance/Mark/Seen
 <!--- A Performance.measure event was observed --->
 * Generic/Performance/Measure/Seen
+<!--- A Performance.resource event was observed --->
+* Generic/Performance/Resource/Seen
+<!--- A first party Performance.resource event was observed --->
+* Generic/Performance/FirstPartyResource/Seen
+<!--- A first party Performance.resource event was observed --->
+* Generic/Performance/NrResource/Seen
 <!--- A resource timing API Ajax event was observed that matches the Agent beacon --->
 * Generic/Resources/Ajax/Internal
 <!--- A resource timing API Non-Ajax (other assets like scripts, etc) event was observed that matches the Agent beacon --->
@@ -186,6 +192,18 @@ A timeslice metric is harvested to the JSE/XHR consumer. An aggregation service
 * Config/AssetsUrl/Changed
 <!--- init.proxy.beacon was Changed from the default --->
 * Config/BeaconUrl/Changed
+<!--- init.performance.capture_marks was Enabled --->
+* Config/Performance/CaptureMarks/Enabled
+<!--- init.performance.capture_measures was Enabled --->
+* Config/Performance/CaptureMeasures/Enabled
+<!--- init.performance.resources was Enabled --->
+* Config/Performance/Resources/Enabled
+<!--- init.performance.resources.asset_types was changed --->
+* Config/Performance/Resources/AssetTypes/Changed
+<!--- init.performance.resources.first_party_domains was changed --->
+* Config/Performance/Resources/FirstPartyDomains/Changed
+<!--- init.performance.resources.ignore_newrelic was changed --->
+* Config/Performance/Resources/IgnoreNewrelic/Changed
 <!--- init.Session_replay.Enabled was Enabled --->
 * Config/SessionReplay/Enabled
 <!--- init.Session_replay.autoStart was Changed from the default --->

src/common/config/init.js

@@ -52,7 +52,13 @@ const model = () => {
     page_view_timing: { enabled: true, harvestTimeSeconds: 30, autoStart: true },
     performance: {
       capture_marks: false,
-      capture_measures: false // false by default through experimental phase, but flipped to true once GA'd
+      capture_measures: false, // false by default through experimental phase, but flipped to true once GA'd
+      resources: {
+        enabled: false, // whether to run this subfeature or not in the generic_events feature. false by default through experimental phase, but flipped to true once GA'd
+        asset_types: [], // MDN types to collect, empty array will collect all types
+        first_party_domains: [], // when included, will decorate the resource as first party if matching
+        ignore_newrelic: true // ignore capturing internal agent scripts and harvest calls
+      }
     },
     privacy: { cookies_enabled: true }, // *cli - per discussion, default should be true
     proxy: {

src/features/generic_events/aggregate/index.js

@@ -6,7 +6,7 @@ import { stringify } from '../../../common/util/stringify'
 import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler'
 import { cleanURL } from '../../../common/url/clean-url'
 import { FEATURE_NAME } from '../constants'
-import { initialLocation, isBrowserScope } from '../../../common/constants/runtime'
+import { globalScope, initialLocation, isBrowserScope } from '../../../common/constants/runtime'
 import { AggregateBase } from '../../utils/aggregate-base'
 import { warn } from '../../../common/util/console'
 import { now } from '../../../common/timing/now'
@@ -16,6 +16,7 @@ import { applyFnToProps } from '../../../common/util/traverse'
 import { FEATURE_TO_ENDPOINT } from '../../../loaders/features/features'
 import { UserActionsAggregator } from './user-actions/user-actions-aggregator'
 import { isIFrameWindow } from '../../../common/dom/iframe'
+import { handle } from '../../../common/event-emitter/handle'
 
 export class Aggregate extends AggregateBase {
   static featureName = FEATURE_NAME
@@ -34,6 +35,8 @@ export class Aggregate extends AggregateBase {
         return
       }
 
+      this.trackSupportabilityMetrics()
+
       if (agentRef.init.page_action.enabled) {
         registerHandler('api-addPageAction', (timestamp, name, attributes) => {
           this.addEvent({
@@ -103,10 +106,11 @@ export class Aggregate extends AggregateBase {
               const observer = new PerformanceObserver((list) => {
                 list.getEntries().forEach(entry => {
                   try {
+                    handle(SUPPORTABILITY_METRIC_CHANNEL, ['Generic/Performance/' + type + '/Seen'])
                     this.addEvent({
                       eventType: 'BrowserPerformance',
                       timestamp: Math.floor(agentRef.runtime.timeKeeper.correctRelativeTimestamp(entry.startTime)),
-                      entryName: entry.name,
+                      entryName: cleanURL(entry.name),
                       entryDuration: entry.duration,
                       entryType: type,
                       ...(entry.detail && { entryDetail: entry.detail })
@@ -123,6 +127,47 @@ export class Aggregate extends AggregateBase {
         }
       }
 
+      if (isBrowserScope && agentRef.init.performance.resources.enabled) {
+        registerHandler('browserPerformance.resource', (entry) => {
+          try {
+            // convert the entry to a plain object and separate the name and duration from the object
+            // you need to do this to be able to spread it into the addEvent call later, and name and duration
+            // would be duplicative of entryName and entryDuration and are protected keys in NR1
+            const { name, duration, ...entryObject } = entry.toJSON()
+
+            let firstParty = false
+            try {
+              const entryDomain = new URL(name).hostname
+              const isNr = (entryDomain.includes('newrelic.com') || entryDomain.includes('nr-data.net') || entryDomain.includes('nr-local.net'))
+              /** decide if we should ignore nr-specific assets */
+              if (this.agentRef.init.performance.resources.ignore_newrelic && isNr) return
+              /** decide if we should ignore the asset type (empty means allow everything, which is the default) */
+              if (this.agentRef.init.performance.resources.asset_types.length && !this.agentRef.init.performance.resources.asset_types.includes(entryObject.initiatorType)) return
+              /** decide if the entryDomain is a first party domain */
+              firstParty = entryDomain === globalScope?.location.hostname || agentRef.init.performance.resources.first_party_domains.includes(entryDomain)
+              if (firstParty) handle(SUPPORTABILITY_METRIC_CHANNEL, ['Generic/Performance/FirstPartyResource/Seen'])
+              if (isNr) handle(SUPPORTABILITY_METRIC_CHANNEL, ['Generic/Performance/NrResource/Seen'])
+            } catch (err) {
+            // couldnt parse the URL, so firstParty will just default to false
+            }
+
+            handle(SUPPORTABILITY_METRIC_CHANNEL, ['Generic/Performance/Resource/Seen'])
+            const event = {
+              ...entryObject,
+              eventType: 'BrowserPerformance',
+              timestamp: Math.floor(agentRef.runtime.timeKeeper.correctRelativeTimestamp(entryObject.startTime)),
+              entryName: name,
+              entryDuration: duration,
+              firstParty
+            }
+
+            this.addEvent(event)
+          } catch (err) {
+            this.ee.emit('internal-error', [err, 'GenericEvents-Resource'])
+          }
+        }, this.featureName, this.ee)
+      }
+
       this.harvestScheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], {
         onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry),
         onUnload: () => addUserAction?.(this.userActionAggregator.aggregationEvent)
@@ -195,4 +240,15 @@ export class Aggregate extends AggregateBase {
   queryStringsBuilder () {
     return { ua: this.agentRef.info.userAttributes, at: this.agentRef.info.atts }
   }
+
+  trackSupportabilityMetrics () {
+    /** track usage SMs to improve these experimental features */
+    const configPerfTag = 'Config/Performance/'
+    if (this.agentRef.init.performance.capture_marks) handle(SUPPORTABILITY_METRIC_CHANNEL, [configPerfTag + 'CaptureMarks/Enabled'])
+    if (this.agentRef.init.performance.capture_measures) handle(SUPPORTABILITY_METRIC_CHANNEL, [configPerfTag + 'CaptureMeasures/Enabled'])
+    if (this.agentRef.init.performance.resources.enabled) handle(SUPPORTABILITY_METRIC_CHANNEL, [configPerfTag + 'Resources/Enabled'])
+    if (this.agentRef.init.performance.resources.asset_types?.length !== 0) handle(SUPPORTABILITY_METRIC_CHANNEL, [configPerfTag + 'Resources/AssetTypes/Changed'])
+    if (this.agentRef.init.performance.resources.first_party_domains?.length !== 0) handle(SUPPORTABILITY_METRIC_CHANNEL, [configPerfTag + 'Resources/FirstPartyDomains/Changed'])
+    if (this.agentRef.init.performance.resources.ignore_newrelic === false) handle(SUPPORTABILITY_METRIC_CHANNEL, [configPerfTag + 'Resources/IgnoreNewrelic/Changed'])
+  }
 }

src/features/generic_events/instrument/index.js

@@ -2,7 +2,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { isBrowserScope } from '../../../common/constants/runtime'
+import { globalScope, isBrowserScope } from '../../../common/constants/runtime'
 import { handle } from '../../../common/event-emitter/handle'
 import { windowAddEventListener } from '../../../common/event-listener/event-listener-opts'
 import { InstrumentBase } from '../../utils/instrument-base'
@@ -12,22 +12,33 @@ export class Instrument extends InstrumentBase {
   static featureName = FEATURE_NAME
   constructor (agentRef, auto = true) {
     super(agentRef, FEATURE_NAME, auto)
+    /** config values that gate whether the generic events aggregator should be imported at all */
     const genericEventSourceConfigs = [
       agentRef.init.page_action.enabled,
       agentRef.init.performance.capture_marks,
       agentRef.init.performance.capture_measures,
-      agentRef.init.user_actions.enabled
-      // other future generic event source configs to go here, like M&Ms, PageResouce, etc.
+      agentRef.init.user_actions.enabled,
+      agentRef.init.performance.resources.enabled
     ]
 
-    if (isBrowserScope && agentRef.init.user_actions.enabled) {
-      OBSERVED_EVENTS.forEach(eventType =>
-        windowAddEventListener(eventType, (evt) => handle('ua', [evt], undefined, this.featureName, this.ee), true)
-      )
-      OBSERVED_WINDOW_EVENTS.forEach(eventType =>
-        windowAddEventListener(eventType, (evt) => handle('ua', [evt], undefined, this.featureName, this.ee))
+    if (isBrowserScope) {
+      if (agentRef.init.user_actions.enabled) {
+        OBSERVED_EVENTS.forEach(eventType =>
+          windowAddEventListener(eventType, (evt) => handle('ua', [evt], undefined, this.featureName, this.ee), true)
+        )
+        OBSERVED_WINDOW_EVENTS.forEach(eventType =>
+          windowAddEventListener(eventType, (evt) => handle('ua', [evt], undefined, this.featureName, this.ee))
         // Capture is not used here so that we don't get element focus/blur events, only the window's as they do not bubble. They are also not cancellable, so no worries about being front of line.
-      )
+        )
+      }
+      if (agentRef.init.performance.resources.enabled && globalScope.PerformanceObserver?.supportedEntryTypes.includes('resource')) {
+        const observer = new PerformanceObserver((list) => {
+          list.getEntries().forEach(entry => {
+            handle('browserPerformance.resource', [entry], undefined, this.featureName, this.ee)
+          })
+        })
+        observer.observe({ type: 'resource', buffered: true })
+      }
     }
 
     /** If any of the sources are active, import the aggregator. otherwise deregister */

src/features/jserrors/aggregate/index.js

@@ -114,9 +114,10 @@ export class Aggregate extends AggregateBase {
    * @param {boolean=} internal if the error was "caught" and deemed "internal" before reporting to the jserrors feature
    * @param {object=} customAttributes  any custom attributes to be included in the error payload
    * @param {boolean=} hasReplay a flag indicating if the error occurred during a replay session
+   * @param {string=} swallowReason a string indicating pre-defined reason if swallowing the error.  Mainly used by the internal error SMs.
    * @returns
    */
-  storeError (err, time, internal, customAttributes, hasReplay) {
+  storeError (err, time, internal, customAttributes, hasReplay, swallowReason) {
     if (!err) return
     // are we in an interaction
     time = time || now()
@@ -134,7 +135,7 @@ export class Aggregate extends AggregateBase {
 
     var stackInfo = computeStackTrace(err)
 
-    const { shouldSwallow, reason } = evaluateInternalError(stackInfo, internal)
+    const { shouldSwallow, reason } = evaluateInternalError(stackInfo, internal, swallowReason)
     if (shouldSwallow) {
       handle(SUPPORTABILITY_METRIC_CHANNEL, ['Internal/Error/' + reason], undefined, FEATURE_NAMES.metrics, this.ee)
       return

src/features/jserrors/aggregate/internal-errors.js

@@ -5,8 +5,8 @@ const REASON_SECURITY_POLICY = 'Security-Policy'
  * @param {Object} stackInfo - The error stack information.
  * @returns {boolean} - Whether the error should be swallowed or not.
  */
-export function evaluateInternalError (stackInfo, internal) {
-  const output = { shouldSwallow: internal || false, reason: 'Other' }
+export function evaluateInternalError (stackInfo, internal, reason) {
+  const output = { shouldSwallow: internal || false, reason: reason || 'Other' }
   const leadingFrame = stackInfo.frames?.[0]
   /** If we cant otherwise determine from the frames and message, the default of internal + reason will be the fallback */
   if (!leadingFrame || typeof stackInfo?.message !== 'string') return output

src/features/jserrors/instrument/index.js

@@ -24,9 +24,9 @@ export class Instrument extends InstrumentBase {
       this.removeOnAbort = new AbortController()
     } catch (e) {}
 
-    this.ee.on('internal-error', (error) => {
+    this.ee.on('internal-error', (error, reason) => {
       if (!this.abortHandler) return
-      handle('ierr', [castError(error), now(), true, {}, this.#replayRunning], undefined, this.featureName, this.ee)
+      handle('ierr', [castError(error), now(), true, {}, this.#replayRunning, reason], undefined, this.featureName, this.ee)
     })
 
     this.ee.on(SR_EVENT_EMITTER_TYPES.REPLAY_RUNNING, (isRunning) => {

src/features/metrics/aggregate/index.js

@@ -134,38 +134,6 @@ export class Aggregate extends AggregateBase {
   }
 
   unload () {
-    try {
-      if (this.resourcesSent) return
-      this.resourcesSent = true // make sure this only gets sent once
-
-      // Capture SMs around network resources using the performance API to assess
-      // work to split this out from the ST nodes
-      // differentiate between internal+external and ajax+non-ajax
-      const ajaxResources = ['beacon', 'fetch', 'xmlhttprequest']
-      const internalUrls = ['nr-data.net', 'newrelic.com', 'nr-local.net', 'localhost']
-      function isInternal (x) { return internalUrls.some(y => x.name.indexOf(y) >= 0) }
-      function isAjax (x) { return ajaxResources.includes(x.initiatorType) }
-      const allResources = performance?.getEntriesByType('resource') || []
-      allResources.forEach((entry) => {
-        if (isInternal(entry)) {
-          if (isAjax(entry)) this.storeSupportabilityMetrics('Generic/Resources/Ajax/Internal')
-          else this.storeSupportabilityMetrics('Generic/Resources/Non-Ajax/Internal')
-        } else {
-          if (isAjax(entry)) this.storeSupportabilityMetrics('Generic/Resources/Ajax/External')
-          else this.storeSupportabilityMetrics('Generic/Resources/Non-Ajax/External')
-        }
-      })
-
-      // Capture SMs for performance markers and measures to assess the usage and possible inclusion of this
-      // data in the agent for use in NR
-      if (typeof performance !== 'undefined') {
-        const markers = performance.getEntriesByType('mark')
-        const measures = performance.getEntriesByType('measure')
-        if (markers.length) this.storeSupportabilityMetrics('Generic/Performance/Mark/Seen', markers.length)
-        if (measures.length) this.storeSupportabilityMetrics('Generic/Performance/Measure/Seen', measures.length)
-      }
-    } catch (e) {
-      // do nothing
-    }
+    // do nothing for now, marks and measures and resources stats are now being captured by the ge feature
   }
 }

tests/assets/page-resources.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<!--
+  Copyright 2020 New Relic Corporation.
+  PDX-License-Identifier: Apache-2.0
+-->
+<html>
+  <head>
+    <title>RUM Unit Test</title>
+    <link rel="stylesheet" type="text/css" href="style.css" /> 
+    {init} {config} {loader}
+  </head>
+  <body>
+    this is a page that provides several types of page assets
+    <img src="http://upload.wikimedia.org/wikipedia/commons/d/d7/House_of_Commons_Chamber_1.png" />
+  </body>
+</html>

tests/components/generic_events/aggregate/index.test.js

@@ -284,7 +284,7 @@ describe('sub-features', () => {
   })
 
   test('should record measures when enabled', async () => {
-    agentSetup.init.performance = { capture_measures: true }
+    agentSetup.init.performance = { capture_measures: true, resources: { enabled: false, asset_types: [], first_party_domains: [], ignore_newrelic: true } }
     getInfo(agentSetup.agentIdentifier).jsAttributes = { globalFoo: 'globalBar' }
     const mockPerformanceObserver = jest.fn(cb => ({
       observe: () => {