From 9ab8e2835ec2d2ac9b2cc2a4560a628b7dba6d83 Mon Sep 17 00:00:00 2001 From: John Pham Date: Tue, 4 Jan 2022 15:17:02 -0800 Subject: [PATCH] Add webgl recording and playback (#57) --- package.json | 6 +- src/record/observer.ts | 91 ++------- src/record/observers/canvas/2d.ts | 94 +++++++++ src/record/observers/canvas/canvas.ts | 35 ++++ src/record/observers/canvas/serialize-args.ts | 125 ++++++++++++ src/record/observers/canvas/webgl.ts | 190 ++++++++++++++++++ src/replay/canvas/2d.ts | 48 +++++ src/replay/canvas/index.ts | 34 ++++ src/replay/canvas/webgl.ts | 163 +++++++++++++++ src/replay/index.ts | 91 +++++---- src/replay/machine.ts | 10 +- src/replay/timer.ts | 32 ++- src/snapshot/snapshot.ts | 20 +- src/snapshot/types.ts | 4 + src/types.ts | 32 +++ yarn.lock | 25 ++- 16 files changed, 883 insertions(+), 117 deletions(-) create mode 100644 src/record/observers/canvas/2d.ts create mode 100644 src/record/observers/canvas/canvas.ts create mode 100644 src/record/observers/canvas/serialize-args.ts create mode 100644 src/record/observers/canvas/webgl.ts create mode 100644 src/replay/canvas/2d.ts create mode 100644 src/replay/canvas/index.ts create mode 100644 src/replay/canvas/webgl.ts diff --git a/package.json b/package.json index d0155eac..2ff17f34 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@highlight-run/rrweb", - "version": "1.0.8", + "version": "1.1.0", "description": "record and replay the web", "scripts": { "test": "npm run bundle:browser && cross-env TS_NODE_CACHE=false TS_NODE_FILES=true mocha -r ts-node/register -r ignore-styles -r jsdom-global/register test/**.test.ts", @@ -43,13 +43,13 @@ "chai": "^4.2.0", "cross-env": "^5.2.0", "css-loader": "^5.0.1", - "mini-css-extract-plugin": "^1.3.8", "fast-mhtml": "^1.1.9", "ignore-styles": "^5.0.1", "inquirer": "^6.2.1", "jest-snapshot": "^23.6.0", "jsdom": "^16.6.0", "jsdom-global": "^3.0.2", + "mini-css-extract-plugin": "^1.3.8", "mocha": "^5.2.0", "node-libtidy": "^0.4.0", "prettier": "2.2.1", @@ -76,7 +76,9 @@ "dependencies": { "@types/css-font-loading-module": "0.0.4", "@xstate/fsm": "^1.4.0", + "base64-arraybuffer": "^1.0.1", "fflate": "^0.4.4", + "identity-obj-proxy": "^3.0.0", "mitt": "^1.1.3" } } diff --git a/src/record/observer.ts b/src/record/observer.ts index a2566413..645f6cc6 100644 --- a/src/record/observer.ts +++ b/src/record/observer.ts @@ -50,6 +50,9 @@ import { import MutationBuffer from './mutation'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; +import initCanvasContextObserver from './observers/canvas/canvas'; +import initCanvas2DMutationObserver from './observers/canvas/2d'; +import initCanvasWebGLMutationObserver from './observers/canvas/webgl'; type WindowWithStoredMutationObserver = IWindow & { __rrMutationObserver?: MutationObserver; @@ -715,79 +718,25 @@ function initCanvasMutationObserver( blockClass: blockClass, mirror: Mirror, ): listenerHandler { - const props = Object.getOwnPropertyNames( - (win as any).CanvasRenderingContext2D.prototype, + const canvasContextReset = initCanvasContextObserver(win, blockClass); + const canvas2DReset = initCanvas2DMutationObserver( + cb, + win, + blockClass, + mirror, ); - const handlers: listenerHandler[] = []; - for (const prop of props) { - try { - if ( - typeof (win as any).CanvasRenderingContext2D.prototype[ - prop as keyof CanvasRenderingContext2D - ] !== 'function' - ) { - continue; - } - const restoreHandler = patch( - (win as any).CanvasRenderingContext2D.prototype, - prop, - function (original) { - return function ( - this: CanvasRenderingContext2D, - ...args: Array - ) { - if (!isBlocked(this.canvas, blockClass)) { - setTimeout(() => { - const recordArgs = [...args]; - if (prop === 'drawImage') { - if ( - recordArgs[0] && - recordArgs[0] instanceof HTMLCanvasElement - ) { - const canvas = recordArgs[0]; - const ctx = canvas.getContext('2d'); - let imgd = ctx?.getImageData( - 0, - 0, - canvas.width, - canvas.height, - ); - let pix = imgd?.data; - recordArgs[0] = JSON.stringify(pix); - } - } - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - property: prop, - args: recordArgs, - }); - }, 0); - } - return original.apply(this, args); - }; - }, - ); - handlers.push(restoreHandler); - } catch { - const hookHandler = hookSetter( - (win as any).CanvasRenderingContext2D.prototype, - prop, - { - set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - property: prop, - args: [v], - setter: true, - }); - }, - }, - ); - handlers.push(hookHandler); - } - } + + const canvasWebGL1and2Reset = initCanvasWebGLMutationObserver( + cb, + win, + blockClass, + mirror, + ); + return () => { - handlers.forEach((h) => h()); + canvasContextReset(); + canvas2DReset(); + canvasWebGL1and2Reset(); }; } diff --git a/src/record/observers/canvas/2d.ts b/src/record/observers/canvas/2d.ts new file mode 100644 index 00000000..613ec618 --- /dev/null +++ b/src/record/observers/canvas/2d.ts @@ -0,0 +1,94 @@ +import { INode } from '../../../snapshot'; +import { + blockClass, + CanvasContext, + canvasMutationCallback, + IWindow, + listenerHandler, + Mirror, +} from '../../../types'; +import { hookSetter, isBlocked, patch } from '../../../utils'; + +export default function initCanvas2DMutationObserver( + cb: canvasMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + const props2D = Object.getOwnPropertyNames( + win.CanvasRenderingContext2D.prototype, + ); + for (const prop of props2D) { + try { + if ( + typeof win.CanvasRenderingContext2D.prototype[ + prop as keyof CanvasRenderingContext2D + ] !== 'function' + ) { + continue; + } + const restoreHandler = patch( + win.CanvasRenderingContext2D.prototype, + prop, + function (original) { + return function ( + this: CanvasRenderingContext2D, + ...args: Array + ) { + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + setTimeout(() => { + const recordArgs = [...args]; + if (prop === 'drawImage') { + if ( + recordArgs[0] && + recordArgs[0] instanceof HTMLCanvasElement + ) { + const canvas = recordArgs[0]; + const ctx = canvas.getContext('2d'); + let imgd = ctx?.getImageData( + 0, + 0, + canvas.width, + canvas.height, + ); + let pix = imgd?.data; + recordArgs[0] = JSON.stringify(pix); + } + } + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext['2D'], + property: prop, + args: recordArgs, + }); + }, 0); + } + return original.apply(this, args); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter( + win.CanvasRenderingContext2D.prototype, + prop, + { + set(v) { + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext['2D'], + property: prop, + args: [v], + setter: true, + }); + }, + }, + ); + handlers.push(hookHandler); + } + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/src/record/observers/canvas/canvas.ts b/src/record/observers/canvas/canvas.ts new file mode 100644 index 00000000..429788b9 --- /dev/null +++ b/src/record/observers/canvas/canvas.ts @@ -0,0 +1,35 @@ +import { ICanvas, INode } from '../../../snapshot'; +import { blockClass, IWindow, listenerHandler } from '../../../types'; +import { isBlocked, patch } from '../../../utils'; + +export default function initCanvasContextObserver( + win: IWindow, + blockClass: blockClass, +): listenerHandler { + const handlers: listenerHandler[] = []; + try { + const restoreHandler = patch( + win.HTMLCanvasElement.prototype, + 'getContext', + function (original) { + return function ( + this: ICanvas, + contextType: string, + ...args: Array + ) { + if (!isBlocked((this as unknown) as INode, blockClass)) { + if (!('__context' in this)) + (this as ICanvas).__context = contextType; + } + return original.apply(this, [contextType, ...args]); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + console.error('failed to patch HTMLCanvasElement.prototype.getContext'); + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/src/record/observers/canvas/serialize-args.ts b/src/record/observers/canvas/serialize-args.ts new file mode 100644 index 00000000..29d4dba4 --- /dev/null +++ b/src/record/observers/canvas/serialize-args.ts @@ -0,0 +1,125 @@ +import { encode } from 'base64-arraybuffer'; +import { SerializedWebGlArg } from '../../../types'; + +// from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77 +const webGLVars: Record> = {}; +export function serializeArg(value: any): SerializedWebGlArg { + if (value instanceof Array) { + return value.map(serializeArg); + } else if (value === null) { + return value; + } else if ( + value instanceof Float32Array || + value instanceof Float64Array || + value instanceof Int32Array || + value instanceof Uint32Array || + value instanceof Uint8Array || + value instanceof Uint16Array || + value instanceof Int16Array || + value instanceof Int8Array || + value instanceof Uint8ClampedArray + ) { + const name = value.constructor.name; + return { + rr_type: name, + args: [Object.values(value)], + }; + } else if ( + // SharedArrayBuffer disabled on most browsers due to spectre. + // More info: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer/SharedArrayBuffer + // value instanceof SharedArrayBuffer || + value instanceof ArrayBuffer + ) { + const name = value.constructor.name as 'ArrayBuffer'; + const base64 = encode(value); + + return { + rr_type: name, + base64, + }; + } else if (value instanceof DataView) { + const name = value.constructor.name; + return { + rr_type: name, + args: [serializeArg(value.buffer), value.byteOffset, value.byteLength], + }; + } else if (value instanceof HTMLImageElement) { + const name = value.constructor.name; + const { src } = value; + return { + rr_type: name, + src, + }; + } else if (value instanceof ImageData) { + const name = value.constructor.name; + return { + rr_type: name, + args: [serializeArg(value.data), value.width, value.height], + }; + } else if ( + value instanceof WebGLActiveInfo || + value instanceof WebGLBuffer || + value instanceof WebGLFramebuffer || + value instanceof WebGLProgram || + value instanceof WebGLRenderbuffer || + value instanceof WebGLShader || + value instanceof WebGLShaderPrecisionFormat || + value instanceof WebGLTexture || + value instanceof WebGLUniformLocation || + value instanceof WebGLVertexArrayObject || + // In Chrome, value won't be an instanceof WebGLVertexArrayObject. + (value && value.constructor.name == 'WebGLVertexArrayObjectOES') || + typeof value === 'object' + ) { + const name = value.constructor.name; + const list = webGLVars[name] || (webGLVars[name] = []); + let index = list.indexOf(value); + + if (index === -1) { + index = list.length; + list.push(value); + } + + return { + rr_type: name, + index, + }; + } + + return value; +} + +export const serializeArgs = (args: Array) => { + return [...args].map(serializeArg); +}; + +export const saveWebGLVar = (value: any): number | void => { + if ( + !( + value instanceof WebGLActiveInfo || + value instanceof WebGLBuffer || + value instanceof WebGLFramebuffer || + value instanceof WebGLProgram || + value instanceof WebGLRenderbuffer || + value instanceof WebGLShader || + value instanceof WebGLShaderPrecisionFormat || + value instanceof WebGLTexture || + value instanceof WebGLUniformLocation || + value instanceof WebGLVertexArrayObject || + // In Chrome, value won't be an instanceof WebGLVertexArrayObject. + (value && value.constructor.name == 'WebGLVertexArrayObjectOES') || + typeof value === 'object' + ) + ) + return; + + const name = value.constructor.name; + const list = webGLVars[name] || (webGLVars[name] = []); + let index = list.indexOf(value); + + if (index === -1) { + index = list.length; + list.push(value); + } + return index; +}; diff --git a/src/record/observers/canvas/webgl.ts b/src/record/observers/canvas/webgl.ts new file mode 100644 index 00000000..77c877e6 --- /dev/null +++ b/src/record/observers/canvas/webgl.ts @@ -0,0 +1,190 @@ +import { INode } from '../../../snapshot'; +import { + blockClass, + CanvasContext, + canvasMutationCallback, + canvasMutationParam, + IWindow, + listenerHandler, + Mirror, +} from '../../../types'; +import { hookSetter, isBlocked, patch } from '../../../utils'; +import { saveWebGLVar, serializeArgs } from './serialize-args'; + +type pendingCanvasMutationsMap = Map; +type RafStamps = { latestId: number; invokeId: number | null }; + +// FIXME: total hack here, we need to find a better way to do this +function flushPendingCanvasMutations( + pendingCanvasMutations: pendingCanvasMutationsMap, + cb: canvasMutationCallback, + mirror: Mirror, +) { + pendingCanvasMutations.forEach( + (values: canvasMutationParam[], canvas: HTMLCanvasElement) => { + const id = mirror.getId((canvas as unknown) as INode); + flushPendingCanvasMutationFor(canvas, pendingCanvasMutations, id, cb); + }, + ); + requestAnimationFrame(() => + flushPendingCanvasMutations(pendingCanvasMutations, cb, mirror), + ); +} + +function flushPendingCanvasMutationFor( + canvas: HTMLCanvasElement, + pendingCanvasMutations: pendingCanvasMutationsMap, + id: number, + cb: canvasMutationCallback, +) { + const values = pendingCanvasMutations.get(canvas); + if (!values || id === -1) return; + + values.forEach((p) => cb({ ...p, id })); + pendingCanvasMutations.delete(canvas); +} + +function patchGLPrototype( + prototype: WebGLRenderingContext | WebGL2RenderingContext, + type: CanvasContext, + cb: canvasMutationCallback, + blockClass: blockClass, + mirror: Mirror, + pendingCanvasMutations: pendingCanvasMutationsMap, + rafStamps: RafStamps, +): listenerHandler[] { + const handlers: listenerHandler[] = []; + + const props = Object.getOwnPropertyNames(prototype); + + for (const prop of props) { + try { + if (typeof prototype[prop as keyof typeof prototype] !== 'function') { + continue; + } + const restoreHandler = patch(prototype, prop, function (original) { + return function (this: typeof prototype, ...args: Array) { + const newFrame = + rafStamps.invokeId && rafStamps.latestId !== rafStamps.invokeId; + if (newFrame || !rafStamps.invokeId) + rafStamps.invokeId = rafStamps.latestId; + + const result = original.apply(this, args); + saveWebGLVar(result); + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + const id = mirror.getId((this.canvas as unknown) as INode); + + const recordArgs = serializeArgs([...args]); + const mutation: canvasMutationParam = { + id, + type, + property: prop, + args: recordArgs, + }; + if (newFrame) mutation.newFrame = true; + + if (id === -1) { + // FIXME! THIS COULD MAYBE BE AN OFFSCREEN CANVAS + if ( + !pendingCanvasMutations.has(this.canvas as HTMLCanvasElement) + ) { + pendingCanvasMutations.set( + this.canvas as HTMLCanvasElement, + [], + ); + } + + pendingCanvasMutations + .get(this.canvas as HTMLCanvasElement)! + .push(mutation); + } else { + // flush all pending mutations + flushPendingCanvasMutationFor( + this.canvas as HTMLCanvasElement, + pendingCanvasMutations, + id, + cb, + ); + cb(mutation); + } + } + + return result; + }; + }); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter(prototype, prop, { + set(v) { + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type, + property: prop, + args: [v], + setter: true, + }); + }, + }); + handlers.push(hookHandler); + } + } + + return handlers; +} + +export default function initCanvasWebGLMutationObserver( + cb: canvasMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + const pendingCanvasMutations: pendingCanvasMutationsMap = new Map(); + + const rafStamps: RafStamps = { + latestId: 0, + invokeId: null, + }; + + const setLatestRAFTimestamp = (timestamp: DOMHighResTimeStamp) => { + rafStamps.latestId = timestamp; + requestAnimationFrame(setLatestRAFTimestamp); + }; + requestAnimationFrame(setLatestRAFTimestamp); + + // TODO: replace me + requestAnimationFrame(() => + flushPendingCanvasMutations(pendingCanvasMutations, cb, mirror), + ); + + handlers.push( + ...patchGLPrototype( + win.WebGLRenderingContext.prototype, + CanvasContext.WebGL, + cb, + blockClass, + mirror, + pendingCanvasMutations, + rafStamps, + ), + ); + + if (typeof win.WebGL2RenderingContext !== 'undefined') { + handlers.push( + ...patchGLPrototype( + win.WebGL2RenderingContext.prototype, + CanvasContext.WebGL2, + cb, + blockClass, + mirror, + pendingCanvasMutations, + rafStamps, + ), + ); + } + + return () => { + pendingCanvasMutations.clear(); + handlers.forEach((h) => h()); + }; +} diff --git a/src/replay/canvas/2d.ts b/src/replay/canvas/2d.ts new file mode 100644 index 00000000..7daa7e42 --- /dev/null +++ b/src/replay/canvas/2d.ts @@ -0,0 +1,48 @@ +import { Replayer } from '../'; +import { canvasMutationData } from '../../types'; + +export default function canvasMutation({ + event, + mutation, + target, + imageMap, + errorHandler, +}: { + event: Parameters[0]; + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + const ctx = ((target as unknown) as HTMLCanvasElement).getContext('2d')!; + + if (mutation.setter) { + // skip some read-only type checks + // tslint:disable-next-line:no-any + (ctx as any)[mutation.property] = mutation.args[0]; + return; + } + const original = ctx[ + mutation.property as Exclude + ] as Function; + + /** + * We have serialized the image source into base64 string during recording, + * which has been preloaded before replay. + * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast. + */ + if ( + mutation.property === 'drawImage' && + typeof mutation.args[0] === 'string' + ) { + const image = imageMap.get(event); + mutation.args[0] = image; + original.apply(ctx, mutation.args); + } else { + original.apply(ctx, mutation.args); + } + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/src/replay/canvas/index.ts b/src/replay/canvas/index.ts new file mode 100644 index 00000000..783f0bde --- /dev/null +++ b/src/replay/canvas/index.ts @@ -0,0 +1,34 @@ +import { Replayer } from '..'; +import { CanvasContext, canvasMutationData } from '../../types'; +import webglMutation from './webgl'; +import canvas2DMutation from './2d'; + +export default function canvasMutation({ + event, + mutation, + target, + imageMap, + errorHandler, +}: { + event: Parameters[0]; + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + if ([CanvasContext.WebGL, CanvasContext.WebGL2].includes(mutation.type)) { + return webglMutation({ mutation, target, imageMap, errorHandler }); + } + // default is '2d' for backwards compatibility (rrweb below 1.1.x) + return canvas2DMutation({ + event, + mutation, + target, + imageMap, + errorHandler, + }); + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/src/replay/canvas/webgl.ts b/src/replay/canvas/webgl.ts new file mode 100644 index 00000000..68b4babf --- /dev/null +++ b/src/replay/canvas/webgl.ts @@ -0,0 +1,163 @@ +import { decode } from 'base64-arraybuffer'; +import { Replayer } from '../'; +import { + CanvasContext, + canvasMutationData, + SerializedWebGlArg, +} from '../../types'; + +// TODO: add ability to wipe this list +const webGLVarMap: Map = new Map(); +export function variableListFor(ctor: string) { + if (!webGLVarMap.has(ctor)) { + webGLVarMap.set(ctor, []); + } + return webGLVarMap.get(ctor) as any[]; +} + +function getContext( + target: HTMLCanvasElement, + type: CanvasContext, +): WebGLRenderingContext | WebGL2RenderingContext | null { + // Note to whomever is going to implement support for `contextAttributes`: + // if `preserveDrawingBuffer` is set to true, + // you have to do `ctx.flush()` before every `newFrame: true` + try { + if (type === CanvasContext.WebGL) { + return ( + target.getContext('webgl')! || target.getContext('experimental-webgl') + ); + } + return target.getContext('webgl2')!; + } catch (e) { + return null; + } +} + +const WebGLVariableConstructors = [ + WebGLActiveInfo, + WebGLBuffer, + WebGLFramebuffer, + WebGLProgram, + WebGLRenderbuffer, + WebGLShader, + WebGLShaderPrecisionFormat, + WebGLTexture, + WebGLUniformLocation, + WebGLVertexArrayObject, +]; +const WebGLVariableConstructorsNames = WebGLVariableConstructors.map( + (ctor) => ctor.name, +); + +function saveToWebGLVarMap(result: any) { + if (!result?.constructor) return; // probably null or undefined + + const { name } = result.constructor; + if (!WebGLVariableConstructorsNames.includes(name)) return; // not a WebGL variable + + const variables = variableListFor(name); + if (!variables.includes(result)) variables.push(result); +} + +export function deserializeArg( + imageMap: Replayer['imageMap'], +): (arg: SerializedWebGlArg) => any { + return (arg: SerializedWebGlArg): any => { + if (arg && typeof arg === 'object' && 'rr_type' in arg) { + if ('index' in arg) { + const { rr_type: name, index } = arg; + return variableListFor(name)[index]; + } else if ('args' in arg) { + const { rr_type: name, args } = arg; + + // @ts-ignore + const ctor = window[name] as unknown; + + // @ts-ignore + return new ctor(...args.map(deserializeArg(imageMap))); + } else if ('base64' in arg) { + return decode(arg.base64); + } else if ('src' in arg) { + const image = imageMap.get(arg.src); + if (image) { + return image; + } else { + const image = new Image(); + image.src = arg.src; + imageMap.set(arg.src, image); + return image; + } + } + } else if (Array.isArray(arg)) { + return arg.map(deserializeArg(imageMap)); + } + return arg; + }; +} + +export default function webglMutation({ + mutation, + target, + imageMap, + errorHandler, +}: { + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + const ctx = getContext(target, mutation.type); + if (!ctx) return; + + // NOTE: if `preserveDrawingBuffer` is set to true, + // we must flush the buffers on every newFrame: true + // if (mutation.newFrame) ctx.flush(); + + if (mutation.setter) { + // skip some read-only type checks + // tslint:disable-next-line:no-any + (ctx as any)[mutation.property] = mutation.args[0]; + return; + } + const original = ctx[ + mutation.property as Exclude + ] as Function; + + const args = mutation.args.map(deserializeArg(imageMap)); + const result = original.apply(ctx, args); + saveToWebGLVarMap(result); + + const debugMode = false; + // const debugMode = true; + if (debugMode) { + if (mutation.property === 'compileShader') { + if (!ctx.getShaderParameter(args[0], ctx.COMPILE_STATUS)) + console.warn( + 'something went wrong in replay', + ctx.getShaderInfoLog(args[0]), + ); + } else if (mutation.property === 'linkProgram') { + ctx.validateProgram(args[0]); + if (!ctx.getProgramParameter(args[0], ctx.LINK_STATUS)) + console.warn( + 'something went wrong in replay', + ctx.getProgramInfoLog(args[0]), + ); + } + const webglError = ctx.getError(); + if (webglError !== ctx.NO_ERROR) { + console.warn( + 'WEBGL ERROR', + webglError, + 'on command:', + mutation.property, + ...args, + ); + } + } + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/src/replay/index.ts b/src/replay/index.ts index 316a52a9..9d722d18 100644 --- a/src/replay/index.ts +++ b/src/replay/index.ts @@ -39,6 +39,7 @@ import { mouseMovePos, styleAttributeValue, styleValueWithPriority, + CanvasContext, } from '../types'; import { createMirror, @@ -62,6 +63,7 @@ import { VirtualStyleRules, VirtualStyleRulesMap, } from './virtual-styles'; +import canvasMutation from './canvas'; const SKIP_TIME_THRESHOLD = 10 * 1000; const SKIP_TIME_INTERVAL = 2 * 1000; @@ -123,7 +125,8 @@ export class Replayer { // The replayer uses the cache to speed up replay and scrubbing. private cache: BuildCache = createCache(); - private imageMap: Map = new Map(); + private imageMap: Map = new Map(); + /** The first time the player is playing. */ private nextUserInteractionEvent: eventWithTime | null; @@ -891,6 +894,37 @@ export class Replayer { } } + private hasImageArg(args: any[]): boolean { + for (const arg of args) { + if (!arg || typeof arg !== 'object') { + // do nothing + } else if ('rr_type' in arg && 'args' in arg) { + if (this.hasImageArg(arg.args)) return true; + } else if ('rr_type' in arg && arg.rr_type === 'HTMLImageElement') { + return true; // has image! + } else if (arg instanceof Array) { + if (this.hasImageArg(arg)) return true; + } + } + return false; + } + + private getImageArgs(args: any[]): string[] { + const images: string[] = []; + for (const arg of args) { + if (!arg || typeof arg !== 'object') { + // do nothing + } else if ('rr_type' in arg && 'args' in arg) { + images.push(...this.getImageArgs(arg.args)); + } else if ('rr_type' in arg && arg.rr_type === 'HTMLImageElement') { + images.push(arg.src); + } else if (arg instanceof Array) { + images.push(...this.getImageArgs(arg)); + } + } + return images; + } + /** * pause when loading style sheet, resume when loaded all timeout exceed */ @@ -974,6 +1008,16 @@ export class Replayer { let d = imgd?.data; d = JSON.parse(event.data.args[0]); ctx?.putImageData(imgd!, 0, 0); + } else if ( + event.type === EventType.IncrementalSnapshot && + event.data.source === IncrementalSource.CanvasMutation && + this.hasImageArg(event.data.args) + ) { + this.getImageArgs(event.data.args).forEach((url) => { + const image = new Image(); + image.src = url; // this preloads the image + this.imageMap.set(url, image); + }); } } if (count !== resolved) { @@ -1022,6 +1066,7 @@ export class Replayer { p.timeOffset + e.timestamp - this.service.state.context.baselineTime, + newFrame: false, }; this.timer.addAction(action); }); @@ -1029,6 +1074,7 @@ export class Replayer { this.timer.addAction({ doAction() {}, delay: e.delay! - d.positions[0]?.timeOffset, + newFrame: false, }); } break; @@ -1347,34 +1393,13 @@ export class Replayer { if (!target) { return this.debugNodeNotFound(d, d.id); } - try { - const ctx = ((target as unknown) as HTMLCanvasElement).getContext( - '2d', - )!; - if (d.setter) { - // skip some read-only type checks - // tslint:disable-next-line:no-any - (ctx as any)[d.property] = d.args[0]; - return; - } - const original = ctx[ - d.property as keyof CanvasRenderingContext2D - ] as Function; - /** - * We have serialized the image source into base64 string during recording, - * which has been preloaded before replay. - * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast. - */ - if (d.property === 'drawImage' && typeof d.args[0] === 'string') { - const image = this.imageMap.get(e); - d.args[0] = image; - original.apply(ctx, d.args); - } else { - original.apply(ctx, d.args); - } - } catch (error) { - this.warnCanvasMutationFailed(d, d.id, error); - } + canvasMutation({ + event: e, + mutation: d, + target: (target as unknown) as HTMLCanvasElement, + imageMap: this.imageMap, + errorHandler: this.warnCanvasMutationFailed.bind(this), + }); break; } case IncrementalSource.Font: { @@ -1974,12 +1999,8 @@ export class Replayer { } } - private warnCanvasMutationFailed( - d: canvasMutationData, - id: number, - error: unknown, - ) { - this.warn(`Has error on update canvas '${id}'`, d, error); + private warnCanvasMutationFailed(d: canvasMutationData, error: unknown) { + this.warn(`Has error on canvas update`, error, 'canvas mutation:', d); } private debugNodeNotFound(d: incrementalData, id: number) { diff --git a/src/replay/machine.ts b/src/replay/machine.ts index 2dbc512b..49dd96ef 100644 --- a/src/replay/machine.ts +++ b/src/replay/machine.ts @@ -8,7 +8,7 @@ import { Emitter, IncrementalSource, } from '../types'; -import { Timer, addDelay } from './timer'; +import { Timer, addDelay, LastDelay } from './timer'; export type PlayerContext = { events: eventWithTime[]; @@ -167,9 +167,11 @@ export function createPlayerService( play(ctx) { const { timer, events, baselineTime, lastPlayedEvent } = ctx; timer.clear(); + + const lastDelay: LastDelay = { at: null }; for (const event of events) { // TODO: improve this API - addDelay(event, baselineTime); + addDelay(event, baselineTime, lastDelay); } const neededEvents = discardPriorSnapshots(events, baselineTime); @@ -207,6 +209,8 @@ export function createPlayerService( emitter.emit(ReplayerEvents.EventCast, event); }, delay: event.delay!, + newFrame: + ('newFrame' in event.data && event.data.newFrame) || false, }); } } @@ -272,6 +276,8 @@ export function createPlayerService( emitter.emit(ReplayerEvents.EventCast, event); }, delay: event.delay!, + newFrame: + ('newFrame' in event.data && event.data.newFrame) || false, }); } } diff --git a/src/replay/timer.ts b/src/replay/timer.ts index 684c3ff0..9799a2e2 100644 --- a/src/replay/timer.ts +++ b/src/replay/timer.ts @@ -44,6 +44,11 @@ export class Timer { lastTimestamp = time; while (actions.length) { const action = actions[0]; + if (action.newFrame) { + action.newFrame = false; + break; + } + if (self.timeOffset >= action.delay) { actions.shift(); action.doAction(); @@ -97,8 +102,14 @@ export class Timer { } } +export type LastDelay = { at: number | null }; + // TODO: add speed to mouse move timestamp calculation -export function addDelay(event: eventWithTime, baselineTime: number): number { +export function addDelay( + event: eventWithTime, + baselineTime: number, + lastDelay?: LastDelay, +): number { // Mouse move events was recorded in a throttle function, // so we need to find the real timestamp by traverse the time offsets. if ( @@ -112,5 +123,24 @@ export function addDelay(event: eventWithTime, baselineTime: number): number { return firstTimestamp - baselineTime; } event.delay = event.timestamp - baselineTime; + + if (lastDelay) { + // WebGL events need to be bundled together as much as possible so they don't + // accidentally get split over multiple animation frames. + if ( + event.type === EventType.IncrementalSnapshot && + event.data.source === IncrementalSource.CanvasMutation && + // `newFrame: true` is used to indicate the start of an new animation frame in the recording, + // and that the event shouldn't be bundled with the previous events. + !event.data.newFrame && + lastDelay.at + ) { + // Override the current delay with the last delay + event.delay = lastDelay.at; + } else { + lastDelay.at = event.delay!; + } + } + return event.delay; } diff --git a/src/snapshot/snapshot.ts b/src/snapshot/snapshot.ts index e09d70fd..945bda85 100644 --- a/src/snapshot/snapshot.ts +++ b/src/snapshot/snapshot.ts @@ -10,6 +10,7 @@ import { MaskTextFn, MaskInputFn, KeepIframeSrcFn, + ICanvas, } from './types'; import { isElement, isShadowRoot, maskInputValue } from './utils'; @@ -488,8 +489,23 @@ function serializeNode( } } // canvas image data - if (tagName === 'canvas' && recordCanvas) { - attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL(); + if ( + tagName === 'canvas' && + recordCanvas && + (!('__context' in n) || (n as ICanvas).__context === '2d') // only record this on 2d canvas + ) { + const canvasDataURL = (n as HTMLCanvasElement).toDataURL(); + + // create blank canvas of same dimensions + const blankCanvas = document.createElement('canvas'); + blankCanvas.width = (n as HTMLCanvasElement).width; + blankCanvas.height = (n as HTMLCanvasElement).height; + const blankCanvasDataURL = blankCanvas.toDataURL(); + + // no need to save dataURL if it's the same as blank canvas + if (canvasDataURL !== blankCanvasDataURL) { + attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL(); + } } // media elements if (tagName === 'audio' || tagName === 'video') { diff --git a/src/snapshot/types.ts b/src/snapshot/types.ts index 8720783e..f239213d 100644 --- a/src/snapshot/types.ts +++ b/src/snapshot/types.ts @@ -71,6 +71,10 @@ export interface INode extends Node { __sn: serializedNodeWithId; } +export interface ICanvas extends HTMLCanvasElement { + __context: string; +} + export type idNodeMap = { [key: number]: INode; }; diff --git a/src/types.ts b/src/types.ts index a109698f..7104d037 100644 --- a/src/types.ts +++ b/src/types.ts @@ -390,6 +390,35 @@ export enum MouseInteractions { TouchCancel, } +export enum CanvasContext { + '2D', + WebGL, + WebGL2, +} + +export type SerializedWebGlArg = + | { + rr_type: 'ArrayBuffer'; + base64: string; // base64 + } + | { + rr_type: string; + src: string; // url of image + } + | { + rr_type: string; + args: Array; + } + | { + rr_type: string; + index: number; + } + | string + | number + | boolean + | null + | SerializedWebGlArg[]; + type mouseInteractionParam = { type: MouseInteractions; id: number; @@ -443,9 +472,11 @@ export type canvasMutationCallback = (p: canvasMutationParam) => void; export type canvasMutationParam = { id: number; + type: CanvasContext; property: string; args: Array; setter?: true; + newFrame?: true; }; export type fontParam = { @@ -569,6 +600,7 @@ export type missingNodeMap = { export type actionWithDelay = { doAction: () => void; delay: number; + newFrame: boolean; }; export type Handler = (event?: unknown) => void; diff --git a/yarn.lock b/yarn.lock index 0c137ffd..874dc555 100644 --- a/yarn.lock +++ b/yarn.lock @@ -704,6 +704,11 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= +base64-arraybuffer@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.1.tgz#87bd13525626db4a9838e00a508c2b73efcf348c" + integrity sha512-vFIUq7FdLtjZMhATwDul5RZWv2jpXQ09Pd6jcVEOvIsqCWTRFD/ONHNfyOS8dA/Ippi5dsIgpyKWKZaAKZltbA== + base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -2521,10 +2526,10 @@ fsevents@^1.2.7: bindings "^1.5.0" nan "^2.12.1" -fsevents@~2.1.2: - version "2.1.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" - integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== +fsevents@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== function-bind@^1.1.1: version "1.1.1" @@ -2720,6 +2725,11 @@ handle-thing@^2.0.0: resolved "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz" integrity sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg== +harmony-reflect@^1.4.6: + version "1.6.2" + resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.2.tgz#31ecbd32e648a34d030d86adb67d4d47547fe710" + integrity sha512-HIp/n38R9kQjDEziXyDTuW3vvoxxyxjxFzXLrBr18uB47GnSt+G9D29fqrpM5ZkspMcPICud3XsBJQ4Y2URg8g== + has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz" @@ -2947,6 +2957,13 @@ icss-utils@^5.0.0, icss-utils@^5.1.0: resolved "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz" integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== +identity-obj-proxy@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz#94d2bda96084453ef36fbc5aaec37e0f79f1fc14" + integrity sha1-lNK9qWCERT7zb7xarsN+D3nx/BQ= + dependencies: + harmony-reflect "^1.4.6" + ieee754@^1.1.13: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"