From 4f8b326d55bd42dfd491532bf06c802b2ef3bfc2 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Wed, 25 Dec 2024 15:08:15 +0700 Subject: [PATCH 01/15] decoder issue detector test --- src/WebRTCIssueDetector.ts | 2 + src/detectors/VideoDecoderIssueDetector.ts | 71 ++++++++++++++++++++++ src/detectors/index.ts | 1 + 3 files changed, 74 insertions(+) create mode 100644 src/detectors/VideoDecoderIssueDetector.ts diff --git a/src/WebRTCIssueDetector.ts b/src/WebRTCIssueDetector.ts index 3ba3cd3..baa4b6a 100644 --- a/src/WebRTCIssueDetector.ts +++ b/src/WebRTCIssueDetector.ts @@ -24,6 +24,7 @@ import { QualityLimitationsIssueDetector, UnknownVideoDecoderImplementationDetector, FrozenVideoTrackDetector, + VideoDecoderIssueDetector, } from './detectors'; import { CompositeRTCStatsParser, RTCStatsParser } from './parser'; import createLogger from './utils/logger'; @@ -67,6 +68,7 @@ class WebRTCIssueDetector { new AvailableOutgoingBitrateIssueDetector(), new UnknownVideoDecoderImplementationDetector(), new FrozenVideoTrackDetector(), + new VideoDecoderIssueDetector(), ]; this.networkScoresCalculator = params.networkScoresCalculator ?? new DefaultNetworkScoresCalculator(); diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts new file mode 100644 index 0000000..4461d79 --- /dev/null +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -0,0 +1,71 @@ +import { + IssueDetectorResult, + WebRTCStatsParsed, +} from '../types'; +import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; + +interface VideoDecoderIssueDetectorParams extends BaseIssueDetectorParams { + decodeTimePerFrameIncreaseSpeedThreshold?: number; + minDecodeTimePerFrameIncreaseCases?: number; +} + +class VideoDecoderIssueDetector extends BaseIssueDetector { + #decodeTimePerFrameIncreaseSpeedThreshold: number; + + #minDecodeTimePerFrameIncreaseCases: number; + + constructor(params: VideoDecoderIssueDetectorParams = {}) { + super(params); + this.#decodeTimePerFrameIncreaseSpeedThreshold = params.decodeTimePerFrameIncreaseSpeedThreshold ?? 1.05; + this.#minDecodeTimePerFrameIncreaseCases = params.minDecodeTimePerFrameIncreaseCases ?? 3; + } + + performDetection(data: WebRTCStatsParsed): IssueDetectorResult { + const { connection: { id: connectionId } } = data; + const issues = this.processData(data); + this.setLastProcessedStats(connectionId, data); + return issues; + } + + private processData(data: WebRTCStatsParsed): IssueDetectorResult { + const currentIncomeVideoStreams = data.video.inbound; + const allLastProcessedStats = this + .getAllLastProcessedStats(data.connection.id); + + const issues: IssueDetectorResult = []; + + currentIncomeVideoStreams.forEach((incomeVideoStream) => { + const lastIncomeVideoStreamStats = allLastProcessedStats + .map((connectionStats) => connectionStats.video.inbound.find( + (videoStreamStats) => videoStreamStats.id === incomeVideoStream.id, + )) + .filter((stats) => (stats?.framesDecoded || 0) > 0 && (stats?.totalDecodeTime || 0) > 0); + + if (lastIncomeVideoStreamStats.length < this.#minDecodeTimePerFrameIncreaseCases) { + return; + } + + const decodeTimePerFrame = lastIncomeVideoStreamStats + .map((stats) => (stats!.totalDecodeTime * 1000) / stats!.framesDecoded); + + const currentDecodeTimePerFrame = (incomeVideoStream.totalDecodeTime * 1000) / incomeVideoStream.framesDecoded; + decodeTimePerFrame.push(currentDecodeTimePerFrame); + + const mean = decodeTimePerFrame.reduce((acc, val) => acc + val, 0) / decodeTimePerFrame.length; + const squaredDiffs = decodeTimePerFrame.map((val) => (val - mean) ** 2); + const variance = squaredDiffs.reduce((acc, val) => acc + val, 0) / squaredDiffs.length; + const volatility = Math.sqrt(variance); + + console.log({ + decodeTimePerFrame, + mean, + variance, + volatility, + }); + }); + + return issues; + } +} + +export default VideoDecoderIssueDetector; diff --git a/src/detectors/index.ts b/src/detectors/index.ts index d797b85..07d66f7 100644 --- a/src/detectors/index.ts +++ b/src/detectors/index.ts @@ -8,3 +8,4 @@ export { default as OutboundNetworkIssueDetector } from './OutboundNetworkIssueD export { default as QualityLimitationsIssueDetector } from './QualityLimitationsIssueDetector'; export { default as UnknownVideoDecoderImplementationDetector } from './UnknownVideoDecoderImplementationDetector'; export { default as FrozenVideoTrackDetector } from './FrozenVideoTrackDetector'; +export { default as VideoDecoderIssueDetector } from './VideoDecoderIssueDetector'; From a0b59aa86d45860dd8a9dbd4aec82e220504f939 Mon Sep 17 00:00:00 2001 From: vlprojects-bot Date: Wed, 25 Dec 2024 08:10:13 +0000 Subject: [PATCH 02/15] chore(release): 1.14.0-decoder-issue-detector.1 [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 45b1419..322afdb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webrtc-issue-detector", - "version": "1.13.0", + "version": "1.14.0-decoder-issue-detector.1", "description": "WebRTC diagnostic tool that detects issues with network or user devices", "repository": "git@github.com:VLprojects/webrtc-issue-detector.git", "author": "Roman Kuzakov ", From eca85ecb3e4531d74bf2d8e35d914b361b74a746 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Thu, 26 Dec 2024 12:14:54 +0700 Subject: [PATCH 03/15] fix: refactor detector --- .eslintrc | 3 +- src/detectors/VideoDecoderIssueDetector.ts | 120 ++++++++++++++------- 2 files changed, 84 insertions(+), 39 deletions(-) diff --git a/.eslintrc b/.eslintrc index e93eaff..cebe930 100644 --- a/.eslintrc +++ b/.eslintrc @@ -15,6 +15,7 @@ "no-underscore-dangle": "off", "max-len": ["error", { "code": 120 }], "import/extensions": "off", - "import/no-cycle": "off" + "import/no-cycle": "off", + "no-continue": "off" } } diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts index 4461d79..c99ab47 100644 --- a/src/detectors/VideoDecoderIssueDetector.ts +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -1,23 +1,25 @@ import { IssueDetectorResult, + IssueReason, + IssueType, WebRTCStatsParsed, } from '../types'; import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; interface VideoDecoderIssueDetectorParams extends BaseIssueDetectorParams { - decodeTimePerFrameIncreaseSpeedThreshold?: number; - minDecodeTimePerFrameIncreaseCases?: number; + volatilityThreshold?: number; + affectedStreamsPercentThreshold?: number; } class VideoDecoderIssueDetector extends BaseIssueDetector { - #decodeTimePerFrameIncreaseSpeedThreshold: number; + #volatilityThreshold: number; - #minDecodeTimePerFrameIncreaseCases: number; + #affectedStreamsPercentThreshold: number; constructor(params: VideoDecoderIssueDetectorParams = {}) { super(params); - this.#decodeTimePerFrameIncreaseSpeedThreshold = params.decodeTimePerFrameIncreaseSpeedThreshold ?? 1.05; - this.#minDecodeTimePerFrameIncreaseCases = params.minDecodeTimePerFrameIncreaseCases ?? 3; + this.#volatilityThreshold = params.volatilityThreshold ?? 1.5; + this.#affectedStreamsPercentThreshold = params.affectedStreamsPercentThreshold ?? 50; } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { @@ -28,41 +30,83 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { } private processData(data: WebRTCStatsParsed): IssueDetectorResult { - const currentIncomeVideoStreams = data.video.inbound; - const allLastProcessedStats = this - .getAllLastProcessedStats(data.connection.id); - const issues: IssueDetectorResult = []; - currentIncomeVideoStreams.forEach((incomeVideoStream) => { - const lastIncomeVideoStreamStats = allLastProcessedStats - .map((connectionStats) => connectionStats.video.inbound.find( - (videoStreamStats) => videoStreamStats.id === incomeVideoStream.id, - )) - .filter((stats) => (stats?.framesDecoded || 0) > 0 && (stats?.totalDecodeTime || 0) > 0); - - if (lastIncomeVideoStreamStats.length < this.#minDecodeTimePerFrameIncreaseCases) { - return; - } - - const decodeTimePerFrame = lastIncomeVideoStreamStats - .map((stats) => (stats!.totalDecodeTime * 1000) / stats!.framesDecoded); - - const currentDecodeTimePerFrame = (incomeVideoStream.totalDecodeTime * 1000) / incomeVideoStream.framesDecoded; - decodeTimePerFrame.push(currentDecodeTimePerFrame); - - const mean = decodeTimePerFrame.reduce((acc, val) => acc + val, 0) / decodeTimePerFrame.length; - const squaredDiffs = decodeTimePerFrame.map((val) => (val - mean) ** 2); - const variance = squaredDiffs.reduce((acc, val) => acc + val, 0) / squaredDiffs.length; - const volatility = Math.sqrt(variance); - - console.log({ - decodeTimePerFrame, - mean, - variance, - volatility, + const allProcessedStats = [ + ...this.getAllLastProcessedStats(data.connection.id), + data, + ]; + + const throtthedStreams = data.video.inbound + .map((incomeVideoStream) => { + const allDecodeTimePerFrame: number[] = []; + + // We need at least 4 elements to have enough representation + if (allProcessedStats.length < 4) { + return; + } + + // exclude first element to calculate accurate delta + for (let i = 1; i < allProcessedStats.length; i += 1) { + let deltaFramesDecoded = 0; + let deltaTotalDecodeTime = 0; + let decodeTimePerFrame = 0; + + const videoStreamStats = allProcessedStats[i].video.inbound.find( + (stream) => stream.id === incomeVideoStream.id, + ); + + if (!videoStreamStats) { + continue; + } + + const prevVideoStreamStats = allProcessedStats[i - 1].video.inbound.find( + (stream) => stream.id === incomeVideoStream.id, + ); + + if (prevVideoStreamStats) { + deltaFramesDecoded = videoStreamStats.framesDecoded - prevVideoStreamStats.framesDecoded; + deltaTotalDecodeTime = videoStreamStats.totalDecodeTime - prevVideoStreamStats.totalDecodeTime; + } + + if (deltaTotalDecodeTime > 0 && deltaFramesDecoded > 0) { + decodeTimePerFrame = deltaTotalDecodeTime * 1000 / deltaFramesDecoded; + } + + allDecodeTimePerFrame.push(decodeTimePerFrame); + } + + // Calculate volatility + const mean = allDecodeTimePerFrame.reduce((acc, val) => acc + val, 0) / allDecodeTimePerFrame.length; + const squaredDiffs = allDecodeTimePerFrame.map((val) => (val - mean) ** 2); + const variance = squaredDiffs.reduce((acc, val) => acc + val, 0) / squaredDiffs.length; + const volatility = Math.sqrt(variance); + + const isDecodeTimePerFrameIncrease = allDecodeTimePerFrame.every( + (decodeTimePerFrame, index) => { + return index === 0 || decodeTimePerFrame > allDecodeTimePerFrame[index - 1]; + }, + ); + + console.log({ + allDecodeTimePerFrame, + isDecodeTimePerFrameIncrease, + mean, + volatility, + }); + + return volatility > this.#volatilityThreshold && isDecodeTimePerFrameIncrease; + }) + .filter((throttled) => throttled); + + + const affectedStreamsPercent = throtthedStreams.length / (data.video.inbound.length / 100); + if (affectedStreamsPercent > this.#affectedStreamsPercentThreshold) { + issues.push({ + type: IssueType.CPU, + reason: IssueReason.DecoderCPUThrottling, }); - }); + } return issues; } From d85eb3ace10fabd576a1b50983167bea4eea6459 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Thu, 26 Dec 2024 12:20:50 +0700 Subject: [PATCH 04/15] fix: refactor detector --- README.md | 13 +--- src/WebRTCIssueDetector.ts | 2 - src/detectors/FramesDroppedIssueDetector.ts | 80 --------------------- src/detectors/FrozenVideoTrackDetector.ts | 2 +- src/detectors/VideoDecoderIssueDetector.ts | 12 +++- src/detectors/index.ts | 1 - 6 files changed, 14 insertions(+), 96 deletions(-) delete mode 100644 src/detectors/FramesDroppedIssueDetector.ts diff --git a/README.md b/README.md index 6cba1f4..9f9ac88 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,6 @@ By default, WebRTCIssueDetector can be created with minimum of mandatory constru ```typescript import WebRTCIssueDetector, { QualityLimitationsIssueDetector, - FramesDroppedIssueDetector, FramesEncodedSentIssueDetector, InboundNetworkIssueDetector, OutboundNetworkIssueDetector, @@ -59,6 +58,7 @@ import WebRTCIssueDetector, { AvailableOutgoingBitrateIssueDetector, UnknownVideoDecoderImplementationDetector, FrozenVideoTrackDetector, + VideoDecoderIssueDetector, } from 'webrtc-issue-detector'; const widWithDefaultConstructorArgs = new WebRTCIssueDetector(); @@ -68,7 +68,6 @@ const widWithDefaultConstructorArgs = new WebRTCIssueDetector(); const widWithCustomConstructorArgs = new WebRTCIssueDetector({ detectors: [ // you are free to change the detectors list according to your needs new QualityLimitationsIssueDetector(), - new FramesDroppedIssueDetector(), new FramesEncodedSentIssueDetector(), new InboundNetworkIssueDetector(), new OutboundNetworkIssueDetector(), @@ -76,6 +75,7 @@ const widWithCustomConstructorArgs = new WebRTCIssueDetector({ new AvailableOutgoingBitrateIssueDetector(), new UnknownVideoDecoderImplementationDetector(), new FrozenVideoTrackDetector(), + new VideoDecoderIssueDetector(), ], getStatsInterval: 10_000, // set custom stats parsing interval onIssues: (payload: IssueDetectorResult) => { @@ -106,19 +106,12 @@ const exampleIssue = { } ``` -### FramesDroppedIssueDetector +### VideoDecoderIssueDetector Detects issues with decoder. ```js const exampleIssue = { type: 'cpu', reason: 'decoder-cpu-throttling', - statsSample: { - deltaFramesDropped: 100, - deltaFramesReceived: 1000, - deltaFramesDecoded: 900, - framesDroppedPct: 10, - }, - ssrc: 1234, } ``` diff --git a/src/WebRTCIssueDetector.ts b/src/WebRTCIssueDetector.ts index baa4b6a..fb072b6 100644 --- a/src/WebRTCIssueDetector.ts +++ b/src/WebRTCIssueDetector.ts @@ -16,7 +16,6 @@ import PeriodicWebRTCStatsReporter from './parser/PeriodicWebRTCStatsReporter'; import DefaultNetworkScoresCalculator from './NetworkScoresCalculator'; import { AvailableOutgoingBitrateIssueDetector, - FramesDroppedIssueDetector, FramesEncodedSentIssueDetector, InboundNetworkIssueDetector, NetworkMediaSyncIssueDetector, @@ -60,7 +59,6 @@ class WebRTCIssueDetector { this.detectors = params.detectors ?? [ new QualityLimitationsIssueDetector(), - new FramesDroppedIssueDetector(), new FramesEncodedSentIssueDetector(), new InboundNetworkIssueDetector(), new OutboundNetworkIssueDetector(), diff --git a/src/detectors/FramesDroppedIssueDetector.ts b/src/detectors/FramesDroppedIssueDetector.ts deleted file mode 100644 index 9512d42..0000000 --- a/src/detectors/FramesDroppedIssueDetector.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - IssueDetectorResult, - IssueReason, - IssueType, - WebRTCStatsParsed, -} from '../types'; -import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; - -interface FramesDroppedIssueDetectorParams extends BaseIssueDetectorParams { - framesDroppedThreshold?: number; -} - -class FramesDroppedIssueDetector extends BaseIssueDetector { - readonly #framesDroppedThreshold: number; - - constructor(params: FramesDroppedIssueDetectorParams = {}) { - super(params); - this.#framesDroppedThreshold = params.framesDroppedThreshold ?? 0.5; - } - - performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; - } - - private processData(data: WebRTCStatsParsed): IssueDetectorResult { - const streamsWithDroppedFrames = data.video.inbound.filter((stats) => stats.framesDropped > 0); - const issues: IssueDetectorResult = []; - const previousInboundRTPVideoStreamsStats = this.getLastProcessedStats(data.connection.id)?.video.inbound; - - if (!previousInboundRTPVideoStreamsStats) { - return issues; - } - - streamsWithDroppedFrames.forEach((streamStats) => { - const previousStreamStats = previousInboundRTPVideoStreamsStats.find((item) => item.ssrc === streamStats.ssrc); - if (!previousStreamStats) { - return; - } - - if (streamStats.framesDropped === previousStreamStats.framesDropped) { - // stream is decoded correctly - return; - } - - const deltaFramesReceived = streamStats.framesReceived - previousStreamStats.framesReceived; - const deltaFramesDecoded = streamStats.framesDecoded - previousStreamStats.framesDecoded; - const deltaFramesDropped = streamStats.framesDropped - previousStreamStats.framesDropped; - const framesDropped = deltaFramesDropped / deltaFramesReceived; - - if (deltaFramesReceived === 0 || deltaFramesDecoded === 0) { - // looks like stream is stopped, skip checking framesDropped - return; - } - - const statsSample = { - deltaFramesDropped, - deltaFramesReceived, - deltaFramesDecoded, - framesDroppedPct: Math.round(framesDropped * 100), - }; - - if (framesDropped >= this.#framesDroppedThreshold) { - // more than half of the received frames were dropped - issues.push({ - statsSample, - type: IssueType.CPU, - reason: IssueReason.DecoderCPUThrottling, - ssrc: streamStats.ssrc, - }); - } - }); - - return issues; - } -} - -export default FramesDroppedIssueDetector; diff --git a/src/detectors/FrozenVideoTrackDetector.ts b/src/detectors/FrozenVideoTrackDetector.ts index 7fae653..d87e67d 100644 --- a/src/detectors/FrozenVideoTrackDetector.ts +++ b/src/detectors/FrozenVideoTrackDetector.ts @@ -69,7 +69,7 @@ class FrozenVideoTrackDetector extends BaseIssueDetector { return; } - // We skip it when ratio is too low because it should be handled by FramesDroppedIssueDetector + // We skip it when ratio is too low because it should be handled by VideoDecoderIssueDetector if (ratioFramesDropped >= this.#framesDroppedThreshold) { return; } diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts index c99ab47..9f71759 100644 --- a/src/detectors/VideoDecoderIssueDetector.ts +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -95,9 +95,13 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { volatility, }); - return volatility > this.#volatilityThreshold && isDecodeTimePerFrameIncrease; + if (volatility > this.#volatilityThreshold && isDecodeTimePerFrameIncrease) { + return { ssrc: incomeVideoStream.ssrc, allDecodeTimePerFrame, volatility, }; + } else { + return undefined; + } }) - .filter((throttled) => throttled); + .filter((throttledVideoStream) => Boolean(throttledVideoStream)); const affectedStreamsPercent = throtthedStreams.length / (data.video.inbound.length / 100); @@ -105,6 +109,10 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { issues.push({ type: IssueType.CPU, reason: IssueReason.DecoderCPUThrottling, + statsSample: { + affectedStreamsPercent, + throtthedStreams: throtthedStreams, + } }); } diff --git a/src/detectors/index.ts b/src/detectors/index.ts index 07d66f7..534dc6c 100644 --- a/src/detectors/index.ts +++ b/src/detectors/index.ts @@ -1,6 +1,5 @@ export { default as BaseIssueDetector } from './BaseIssueDetector'; export { default as AvailableOutgoingBitrateIssueDetector } from './AvailableOutgoingBitrateIssueDetector'; -export { default as FramesDroppedIssueDetector } from './FramesDroppedIssueDetector'; export { default as FramesEncodedSentIssueDetector } from './FramesEncodedSentIssueDetector'; export { default as InboundNetworkIssueDetector } from './InboundNetworkIssueDetector'; export { default as NetworkMediaSyncIssueDetector } from './NetworkMediaSyncIssueDetector'; From 93aa5553063e98e738656582926b9bea3e3b0d25 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Thu, 26 Dec 2024 12:28:02 +0700 Subject: [PATCH 05/15] refactor --- src/detectors/VideoDecoderIssueDetector.ts | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts index 9f71759..543902c 100644 --- a/src/detectors/VideoDecoderIssueDetector.ts +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -38,12 +38,12 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { ]; const throtthedStreams = data.video.inbound - .map((incomeVideoStream) => { + .map((incomeVideoStream): { ssrc: number, allDecodeTimePerFrame: number[], volatility: number } | undefined => { const allDecodeTimePerFrame: number[] = []; // We need at least 4 elements to have enough representation if (allProcessedStats.length < 4) { - return; + return undefined; } // exclude first element to calculate accurate delta @@ -72,7 +72,7 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { if (deltaTotalDecodeTime > 0 && deltaFramesDecoded > 0) { decodeTimePerFrame = deltaTotalDecodeTime * 1000 / deltaFramesDecoded; } - + allDecodeTimePerFrame.push(decodeTimePerFrame); } @@ -83,9 +83,7 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { const volatility = Math.sqrt(variance); const isDecodeTimePerFrameIncrease = allDecodeTimePerFrame.every( - (decodeTimePerFrame, index) => { - return index === 0 || decodeTimePerFrame > allDecodeTimePerFrame[index - 1]; - }, + (decodeTimePerFrame, index) => index === 0 || decodeTimePerFrame > allDecodeTimePerFrame[index - 1], ); console.log({ @@ -96,23 +94,24 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { }); if (volatility > this.#volatilityThreshold && isDecodeTimePerFrameIncrease) { - return { ssrc: incomeVideoStream.ssrc, allDecodeTimePerFrame, volatility, }; - } else { - return undefined; + console.log('CPU THROTTLE SUSPECTED FOR STREAM', incomeVideoStream.ssrc); + return { ssrc: incomeVideoStream.ssrc, allDecodeTimePerFrame, volatility }; } + + return undefined; }) .filter((throttledVideoStream) => Boolean(throttledVideoStream)); - const affectedStreamsPercent = throtthedStreams.length / (data.video.inbound.length / 100); if (affectedStreamsPercent > this.#affectedStreamsPercentThreshold) { + console.log('CPU THROTTLE DETECTED'); issues.push({ type: IssueType.CPU, reason: IssueReason.DecoderCPUThrottling, statsSample: { affectedStreamsPercent, - throtthedStreams: throtthedStreams, - } + throtthedStreams, + }, }); } From aa2660159654ed9c4ff788de67e5dca1322eff91 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Thu, 26 Dec 2024 12:28:51 +0700 Subject: [PATCH 06/15] fix linter errors --- src/detectors/VideoDecoderIssueDetector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts index 543902c..6f6136a 100644 --- a/src/detectors/VideoDecoderIssueDetector.ts +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -70,7 +70,7 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { } if (deltaTotalDecodeTime > 0 && deltaFramesDecoded > 0) { - decodeTimePerFrame = deltaTotalDecodeTime * 1000 / deltaFramesDecoded; + decodeTimePerFrame = (deltaTotalDecodeTime * 1000) / deltaFramesDecoded; } allDecodeTimePerFrame.push(decodeTimePerFrame); From 84420d235b79c9a1c68248af0e7b7adcf7bece4b Mon Sep 17 00:00:00 2001 From: vlprojects-bot Date: Thu, 26 Dec 2024 05:29:50 +0000 Subject: [PATCH 07/15] chore(release): 1.14.0-decoder-issue-detector.2 [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 322afdb..b01418e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webrtc-issue-detector", - "version": "1.14.0-decoder-issue-detector.1", + "version": "1.14.0-decoder-issue-detector.2", "description": "WebRTC diagnostic tool that detects issues with network or user devices", "repository": "git@github.com:VLprojects/webrtc-issue-detector.git", "author": "Roman Kuzakov ", From e3f20cb9e33167cc1764eb799f16a10da2f82ccb Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Thu, 26 Dec 2024 12:41:24 +0700 Subject: [PATCH 08/15] fix: readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 9f9ac88..5fe834a 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,12 @@ Detects issues with decoder. const exampleIssue = { type: 'cpu', reason: 'decoder-cpu-throttling', + statsSample: { + affectedStreamsPercent: 67, + throtthedStreams: [ + { ssrc: 123, allDecodeTimePerFrame: [1.2, 1.6, 1.9, 2.4, 2.9], volatility: 1.7 }, + ] + }, } ``` From de773e458c59ac2aa60584153e58cd4cd234ebc8 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Wed, 15 Jan 2025 15:15:04 +0700 Subject: [PATCH 09/15] feat: Improve FrozenVideo and VideoDecoderCPU detectors --- .eslintrc | 3 +- src/WebRTCIssueDetector.ts | 17 +- src/detectors/BaseIssueDetector.ts | 32 +++- .../FramesEncodedSentIssueDetector.ts | 5 +- src/detectors/FrozenVideoTrackDetector.ts | 178 ++++++++---------- src/detectors/InboundNetworkIssueDetector.ts | 5 +- .../NetworkMediaSyncIssueDetector.ts | 5 +- src/detectors/OutboundNetworkIssueDetector.ts | 5 +- .../QualityLimitationsIssueDetector.ts | 5 +- ...knownVideoDecoderImplementationDetector.ts | 5 +- src/detectors/VideoDecoderIssueDetector.ts | 51 +++-- src/types.ts | 14 +- src/utils/video.ts | 25 +++ 13 files changed, 185 insertions(+), 165 deletions(-) create mode 100644 src/utils/video.ts diff --git a/.eslintrc b/.eslintrc index cebe930..8b3d8cf 100644 --- a/.eslintrc +++ b/.eslintrc @@ -16,6 +16,7 @@ "max-len": ["error", { "code": 120 }], "import/extensions": "off", "import/no-cycle": "off", - "no-continue": "off" + "no-continue": "off", + "import/prefer-default-export": "off" } } diff --git a/src/WebRTCIssueDetector.ts b/src/WebRTCIssueDetector.ts index fb072b6..deda166 100644 --- a/src/WebRTCIssueDetector.ts +++ b/src/WebRTCIssueDetector.ts @@ -7,6 +7,7 @@ import { IssueDetector, IssuePayload, Logger, + NetworkScores, StatsReportItem, WebRTCIssueDetectorConstructorParams, WebRTCStatsParsed, @@ -88,11 +89,8 @@ class WebRTCIssueDetector { } this.statsReporter.on(PeriodicWebRTCStatsReporter.STATS_REPORT_READY_EVENT, (report: StatsReportItem) => { - this.detectIssues({ - data: report.stats, - }); - - this.calculateNetworkScores(report.stats); + const networkScores = this.calculateNetworkScores(report.stats); + this.detectIssues({ data: report.stats }, networkScores); }); this.statsReporter.on(PeriodicWebRTCStatsReporter.STATS_REPORTS_PARSED, (data: { @@ -161,16 +159,19 @@ class WebRTCIssueDetector { this.eventEmitter.emit(EventType.Issue, issues); } - private detectIssues({ data }: DetectIssuesPayload): void { - const issues = this.detectors.reduce((acc, detector) => [...acc, ...detector.detect(data)], []); + private detectIssues({ data }: DetectIssuesPayload, networkScores: NetworkScores): void { + const issues = this.detectors + .reduce((acc, detector) => [...acc, ...detector.detect(data, networkScores)], []); + if (issues.length > 0) { this.emitIssues(issues); } } - private calculateNetworkScores(data: WebRTCStatsParsed): void { + private calculateNetworkScores(data: WebRTCStatsParsed): NetworkScores { const networkScores = this.networkScoresCalculator.calculate(data); this.eventEmitter.emit(EventType.NetworkScoresUpdated, networkScores); + return networkScores; } private wrapRTCPeerConnection(): void { diff --git a/src/detectors/BaseIssueDetector.ts b/src/detectors/BaseIssueDetector.ts index 2ff4d10..cfe05a5 100644 --- a/src/detectors/BaseIssueDetector.ts +++ b/src/detectors/BaseIssueDetector.ts @@ -1,4 +1,10 @@ -import { IssueDetector, IssueDetectorResult, WebRTCStatsParsed } from '../types'; +import { + IssueDetector, + IssueDetectorResult, + NetworkScores, + WebRTCStatsParsed, + WebRTCStatsParsedWithNetworkScores, +} from '../types'; import { scheduleTask } from '../utils/tasks'; import { CLEANUP_PREV_STATS_TTL_MS, MAX_PARSED_STATS_STORAGE_SIZE } from '../utils/constants'; @@ -13,7 +19,7 @@ export interface BaseIssueDetectorParams { } abstract class BaseIssueDetector implements IssueDetector { - readonly #parsedStatsStorage: Map = new Map(); + readonly #parsedStatsStorage: Map = new Map(); readonly #statsCleanupDelayMs: number; @@ -24,11 +30,19 @@ abstract class BaseIssueDetector implements IssueDetector { this.#maxParsedStatsStorageSize = params.maxParsedStatsStorageSize ?? MAX_PARSED_STATS_STORAGE_SIZE; } - abstract performDetection(data: WebRTCStatsParsed): IssueDetectorResult; + abstract performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult; - detect(data: WebRTCStatsParsed): IssueDetectorResult { - const result = this.performDetection(data); + detect(data: WebRTCStatsParsed, networkScores?: NetworkScores): IssueDetectorResult { + const parsedStatsWithNetworkScores = { + ...data, + networkScores: { + ...networkScores, + statsSamples: networkScores?.statsSamples || {}, + }, + }; + const result = this.performDetection(parsedStatsWithNetworkScores); + this.setLastProcessedStats(data.connection.id, parsedStatsWithNetworkScores); this.performPrevStatsCleanup({ connectionId: data.connection.id, }); @@ -56,7 +70,7 @@ abstract class BaseIssueDetector implements IssueDetector { }); } - protected setLastProcessedStats(connectionId: string, parsedStats: WebRTCStatsParsed): void { + protected setLastProcessedStats(connectionId: string, parsedStats: WebRTCStatsParsedWithNetworkScores): void { if (!connectionId || parsedStats.connection.id !== connectionId) { return; } @@ -71,16 +85,16 @@ abstract class BaseIssueDetector implements IssueDetector { this.#parsedStatsStorage.set(connectionId, connectionStats); } - protected getLastProcessedStats(connectionId: string): WebRTCStatsParsed | undefined { + protected getLastProcessedStats(connectionId: string): WebRTCStatsParsedWithNetworkScores | undefined { const connectionStats = this.#parsedStatsStorage.get(connectionId); return connectionStats?.[connectionStats.length - 1]; } - protected getAllLastProcessedStats(connectionId: string): WebRTCStatsParsed[] { + protected getAllLastProcessedStats(connectionId: string): WebRTCStatsParsedWithNetworkScores[] { return this.#parsedStatsStorage.get(connectionId) ?? []; } - private deleteLastProcessedStats(connectionId: string): void { + protected deleteLastProcessedStats(connectionId: string): void { this.#parsedStatsStorage.delete(connectionId); } } diff --git a/src/detectors/FramesEncodedSentIssueDetector.ts b/src/detectors/FramesEncodedSentIssueDetector.ts index 87a4a9c..14aa8d5 100644 --- a/src/detectors/FramesEncodedSentIssueDetector.ts +++ b/src/detectors/FramesEncodedSentIssueDetector.ts @@ -19,10 +19,7 @@ class FramesEncodedSentIssueDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/FrozenVideoTrackDetector.ts b/src/detectors/FrozenVideoTrackDetector.ts index d87e67d..6ac28c5 100644 --- a/src/detectors/FrozenVideoTrackDetector.ts +++ b/src/detectors/FrozenVideoTrackDetector.ts @@ -2,134 +2,108 @@ import { IssueDetectorResult, IssueReason, IssueType, - ParsedInboundVideoStreamStats, + MosQuality, WebRTCStatsParsed, + WebRTCStatsParsedWithNetworkScores, } from '../types'; +import { isSvcSpatialLayerChanged } from '../utils/video'; import BaseIssueDetector from './BaseIssueDetector'; interface FrozenVideoTrackDetectorParams { - timeoutMs?: number; - framesDroppedThreshold?: number; + avgFreezeDurationThresholdMs?: number; + frozenDurationThresholdPct?: number; } -class FrozenVideoTrackDetector extends BaseIssueDetector { - readonly #lastMarkedAt = new Map(); +interface FrozenStream { + ssrc: number; + avgFreezeDurationMs: number; + frozenDurationPct: number; +} - readonly #timeoutMs: number; +class FrozenVideoTrackDetector extends BaseIssueDetector { + #avgFreezeDurationThresholdMs: number; - readonly #framesDroppedThreshold: number; + #frozenDurationThresholdPct: number; constructor(params: FrozenVideoTrackDetectorParams = {}) { super(); - this.#timeoutMs = params.timeoutMs ?? 10_000; - this.#framesDroppedThreshold = params.framesDroppedThreshold ?? 0.5; + this.#avgFreezeDurationThresholdMs = params.avgFreezeDurationThresholdMs ?? 1_000; + this.#frozenDurationThresholdPct = params.frozenDurationThresholdPct ?? 30; } - performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { + const inboundScore = data.networkScores.inbound; + if (inboundScore !== undefined && inboundScore <= MosQuality.BAD) { + // do not execute detection on stats based on poor network quality + // to avoid false positives + return []; + } + + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const previousStats = this.getLastProcessedStats(connectionId); const issues: IssueDetectorResult = []; - - if (!previousStats) { - return issues; + const allLastProcessedStats = this.getAllLastProcessedStats(data.connection.id); + if (allLastProcessedStats.length === 0) { + return []; } - const { video: { inbound: newInbound } } = data; - const { video: { inbound: prevInbound } } = previousStats; - - const mapByTrackId = (items: ParsedInboundVideoStreamStats[]) => new Map( - items.map((item) => [item.track.trackIdentifier, item] as const), - ); - - const newInboundByTrackId = mapByTrackId(newInbound); - const prevInboundByTrackId = mapByTrackId(prevInbound); - const unvisitedTrackIds = new Set(this.#lastMarkedAt.keys()); - - Array.from(newInboundByTrackId.entries()).forEach(([trackId, newInboundItem]) => { - unvisitedTrackIds.delete(trackId); - - const prevInboundItem = prevInboundByTrackId.get(trackId); - if (!prevInboundItem) { - return; - } - - const deltaFramesReceived = newInboundItem.framesReceived - prevInboundItem.framesReceived; - const deltaFramesDropped = newInboundItem.framesDropped - prevInboundItem.framesDropped; - const deltaFramesDecoded = newInboundItem.framesDecoded - prevInboundItem.framesDecoded; - const ratioFramesDropped = deltaFramesDropped / deltaFramesReceived; - - if (deltaFramesReceived === 0) { - return; - } - - // We skip it when ratio is too low because it should be handled by VideoDecoderIssueDetector - if (ratioFramesDropped >= this.#framesDroppedThreshold) { - return; - } - - // It seems that track is alive and we can remove mark if it was marked - if (deltaFramesDecoded > 0) { - this.removeMarkIssue(trackId); - return; - } - - const hasIssue = this.markIssue(trackId); - - if (!hasIssue) { - return; - } - - const statsSample = { - framesReceived: newInboundItem.framesReceived, - framesDropped: newInboundItem.framesDropped, - framesDecoded: newInboundItem.framesDecoded, - deltaFramesReceived, - deltaFramesDropped, - deltaFramesDecoded, - }; - + const frozenStreams = data.video.inbound + .map((videoStream): FrozenStream | undefined => { + const prevStat = allLastProcessedStats[allLastProcessedStats.length - 1] + .video.inbound.find((stream) => stream.ssrc === videoStream.ssrc); + + if (!prevStat) { + return undefined; + } + + const isSpatialLayerChanged = isSvcSpatialLayerChanged(videoStream.ssrc, [ + allLastProcessedStats[allLastProcessedStats.length - 1], + data, + ]); + + if (isSpatialLayerChanged) { + return undefined; + } + + const deltaFreezeCount = videoStream.freezeCount - (prevStat.freezeCount ?? 0); + const deltaFreezesTimeMs = (videoStream.totalFreezesDuration - (prevStat.totalFreezesDuration ?? 0)) * 1000; + const avgFreezeDurationMs = deltaFreezeCount > 0 ? deltaFreezesTimeMs / deltaFreezeCount : 0; + + const statsTimeDiff = videoStream.timestamp - prevStat.timestamp; + const frozenDurationPct = (deltaFreezesTimeMs / statsTimeDiff) * 100; + if (frozenDurationPct > this.#frozenDurationThresholdPct) { + return { + ssrc: videoStream.ssrc, + avgFreezeDurationMs, + frozenDurationPct, + }; + } + + if (avgFreezeDurationMs > this.#avgFreezeDurationThresholdMs) { + return { + ssrc: videoStream.ssrc, + avgFreezeDurationMs, + frozenDurationPct, + }; + } + + return undefined; + }) + .filter((stream) => stream !== undefined) as FrozenStream[]; + + if (frozenStreams.length > 0) { issues.push({ - statsSample, type: IssueType.Stream, reason: IssueReason.FrozenVideoTrack, - trackIdentifier: trackId, + statsSample: { + ssrcs: frozenStreams.map((stream) => stream.ssrc), + }, }); - }); - - // just clear unvisited tracks from memory - unvisitedTrackIds.forEach((trackId) => { - this.removeMarkIssue(trackId); - }); - - return issues; - } - - private markIssue(trackId: string): boolean { - const now = Date.now(); - - const lastMarkedAt = this.#lastMarkedAt.get(trackId); - - if (!lastMarkedAt) { - this.#lastMarkedAt.set(trackId, now); - return false; - } - - if (now - lastMarkedAt < this.#timeoutMs) { - return false; } - return true; - } - - private removeMarkIssue(trackId: string): void { - this.#lastMarkedAt.delete(trackId); + return issues; } } diff --git a/src/detectors/InboundNetworkIssueDetector.ts b/src/detectors/InboundNetworkIssueDetector.ts index f1736a3..bbb4ed2 100644 --- a/src/detectors/InboundNetworkIssueDetector.ts +++ b/src/detectors/InboundNetworkIssueDetector.ts @@ -31,10 +31,7 @@ class InboundNetworkIssueDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/NetworkMediaSyncIssueDetector.ts b/src/detectors/NetworkMediaSyncIssueDetector.ts index 8bacd94..7fa36c9 100644 --- a/src/detectors/NetworkMediaSyncIssueDetector.ts +++ b/src/detectors/NetworkMediaSyncIssueDetector.ts @@ -19,10 +19,7 @@ class NetworkMediaSyncIssueDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/OutboundNetworkIssueDetector.ts b/src/detectors/OutboundNetworkIssueDetector.ts index 95a637a..399ca55 100644 --- a/src/detectors/OutboundNetworkIssueDetector.ts +++ b/src/detectors/OutboundNetworkIssueDetector.ts @@ -23,10 +23,7 @@ class OutboundNetworkIssueDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/QualityLimitationsIssueDetector.ts b/src/detectors/QualityLimitationsIssueDetector.ts index c78e9f5..8a25a80 100644 --- a/src/detectors/QualityLimitationsIssueDetector.ts +++ b/src/detectors/QualityLimitationsIssueDetector.ts @@ -8,10 +8,7 @@ import BaseIssueDetector from './BaseIssueDetector'; class QualityLimitationsIssueDetector extends BaseIssueDetector { performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/UnknownVideoDecoderImplementationDetector.ts b/src/detectors/UnknownVideoDecoderImplementationDetector.ts index 139dd39..18793e5 100644 --- a/src/detectors/UnknownVideoDecoderImplementationDetector.ts +++ b/src/detectors/UnknownVideoDecoderImplementationDetector.ts @@ -14,10 +14,7 @@ class UnknownVideoDecoderImplementationDetector extends BaseIssueDetector { } = {}; performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } protected performPrevStatsCleanup(payload: PrevStatsCleanupPayload) { diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts index 6f6136a..03f55cf 100644 --- a/src/detectors/VideoDecoderIssueDetector.ts +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -2,8 +2,10 @@ import { IssueDetectorResult, IssueReason, IssueType, - WebRTCStatsParsed, + MosQuality, + WebRTCStatsParsedWithNetworkScores, } from '../types'; +import { isSvcSpatialLayerChanged } from '../utils/video'; import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; interface VideoDecoderIssueDetectorParams extends BaseIssueDetectorParams { @@ -22,14 +24,25 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { this.#affectedStreamsPercentThreshold = params.affectedStreamsPercentThreshold ?? 50; } - performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { + const allHistoricalStats = [ + ...this.getAllLastProcessedStats(data.connection.id), + data, + ]; + + const isBadNetworkHappened = allHistoricalStats + .find((stat) => stat.networkScores.inbound !== undefined && stat.networkScores.inbound <= MosQuality.BAD); + + if (isBadNetworkHappened) { + // do not execute detection on historical stats based on bad network quality + // to avoid false positives + return []; + } + + return this.processData(data); } - private processData(data: WebRTCStatsParsed): IssueDetectorResult { + private processData(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { const issues: IssueDetectorResult = []; const allProcessedStats = [ @@ -40,9 +53,13 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { const throtthedStreams = data.video.inbound .map((incomeVideoStream): { ssrc: number, allDecodeTimePerFrame: number[], volatility: number } | undefined => { const allDecodeTimePerFrame: number[] = []; + const isSpatialLayerChanged = isSvcSpatialLayerChanged(incomeVideoStream.ssrc, allProcessedStats); + if (isSpatialLayerChanged) { + return undefined; + } - // We need at least 4 elements to have enough representation - if (allProcessedStats.length < 4) { + // We need at least 5 elements to have enough representation + if (allProcessedStats.length < 5) { return undefined; } @@ -53,7 +70,7 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { let decodeTimePerFrame = 0; const videoStreamStats = allProcessedStats[i].video.inbound.find( - (stream) => stream.id === incomeVideoStream.id, + (stream) => stream.ssrc === incomeVideoStream.ssrc, ); if (!videoStreamStats) { @@ -61,7 +78,7 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { } const prevVideoStreamStats = allProcessedStats[i - 1].video.inbound.find( - (stream) => stream.id === incomeVideoStream.id, + (stream) => stream.ssrc === incomeVideoStream.ssrc, ); if (prevVideoStreamStats) { @@ -86,15 +103,7 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { (decodeTimePerFrame, index) => index === 0 || decodeTimePerFrame > allDecodeTimePerFrame[index - 1], ); - console.log({ - allDecodeTimePerFrame, - isDecodeTimePerFrameIncrease, - mean, - volatility, - }); - if (volatility > this.#volatilityThreshold && isDecodeTimePerFrameIncrease) { - console.log('CPU THROTTLE SUSPECTED FOR STREAM', incomeVideoStream.ssrc); return { ssrc: incomeVideoStream.ssrc, allDecodeTimePerFrame, volatility }; } @@ -104,7 +113,6 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { const affectedStreamsPercent = throtthedStreams.length / (data.video.inbound.length / 100); if (affectedStreamsPercent > this.#affectedStreamsPercentThreshold) { - console.log('CPU THROTTLE DETECTED'); issues.push({ type: IssueType.CPU, reason: IssueReason.DecoderCPUThrottling, @@ -113,6 +121,9 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { throtthedStreams, }, }); + + // clear all processed stats for this connection to avoid duplicate issues + this.deleteLastProcessedStats(data.connection.id); } return issues; diff --git a/src/types.ts b/src/types.ts index 7f6e7d6..1f1de93 100644 --- a/src/types.ts +++ b/src/types.ts @@ -11,7 +11,7 @@ export interface WIDWindow { export type IssueDetectorResult = IssuePayload[]; export interface IssueDetector { - detect(data: WebRTCStatsParsed): IssueDetectorResult; + detect(data: WebRTCStatsParsed, networkScores: NetworkScores): IssueDetectorResult; } export interface INetworkScoresCalculator { @@ -251,6 +251,8 @@ export type ParsedInboundVideoStreamStats = { totalDecodeTime: number, totalInterFrameDelay: number, totalSquaredInterFrameDelay: number, + freezeCount: number, + totalFreezesDuration: number, track: { detached: boolean, ended: boolean, @@ -433,3 +435,13 @@ export interface Logger { warn: (msg: any, ...meta: any[]) => void; error: (msg: any, ...meta: any[]) => void; } + +export enum MosQuality { + BAD = 2.1, + POOR = 2.6, + FAIR = 3.1, + GOOD = 3.8, + EXCELLENT = 4.3, +} + +export type WebRTCStatsParsedWithNetworkScores = WebRTCStatsParsed & { networkScores: NetworkScores }; diff --git a/src/utils/video.ts b/src/utils/video.ts new file mode 100644 index 0000000..0dbb5b1 --- /dev/null +++ b/src/utils/video.ts @@ -0,0 +1,25 @@ +import { WebRTCStatsParsed } from '../types'; + +export const isSvcSpatialLayerChanged = (ssrc: number, allProcessedStats: WebRTCStatsParsed[]): boolean => { + for (let i = 1; i < allProcessedStats.length; i += 1) { + const videoStreamStats = allProcessedStats[i].video.inbound.find( + (stream) => stream.ssrc === ssrc, + ); + + if (!videoStreamStats) { + continue; + } + + const prevVideoStreamStats = allProcessedStats[i - 1].video.inbound.find( + (stream) => stream.ssrc === ssrc, + ); + + const widthChanged = videoStreamStats.frameWidth !== prevVideoStreamStats?.frameWidth; + const heightChanged = videoStreamStats.frameHeight !== prevVideoStreamStats?.frameHeight; + if (widthChanged || heightChanged) { + return true; + } + } + + return false; +}; From 2c15658b10b36e9666e0708d532e340a3ee57e2a Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Wed, 15 Jan 2025 15:22:55 +0700 Subject: [PATCH 10/15] feat: Delete FramesEncodedSentIssueDetector --- README.md | 17 ---- src/WebRTCIssueDetector.ts | 2 - .../FramesEncodedSentIssueDetector.ts | 80 ------------------- src/detectors/index.ts | 1 - 4 files changed, 100 deletions(-) delete mode 100644 src/detectors/FramesEncodedSentIssueDetector.ts diff --git a/README.md b/README.md index 5fe834a..8ef2acf 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,6 @@ By default, WebRTCIssueDetector can be created with minimum of mandatory constru ```typescript import WebRTCIssueDetector, { QualityLimitationsIssueDetector, - FramesEncodedSentIssueDetector, InboundNetworkIssueDetector, OutboundNetworkIssueDetector, NetworkMediaSyncIssueDetector, @@ -68,7 +67,6 @@ const widWithDefaultConstructorArgs = new WebRTCIssueDetector(); const widWithCustomConstructorArgs = new WebRTCIssueDetector({ detectors: [ // you are free to change the detectors list according to your needs new QualityLimitationsIssueDetector(), - new FramesEncodedSentIssueDetector(), new InboundNetworkIssueDetector(), new OutboundNetworkIssueDetector(), new NetworkMediaSyncIssueDetector(), @@ -121,21 +119,6 @@ const exampleIssue = { } ``` -### FramesEncodedSentIssueDetector -Detects issues with outbound network throughput. -```js -const exampleIssue = { - type: 'network', - reason: 'outbound-network-throughput', - statsSample: { - deltaFramesSent: 900, - deltaFramesEncoded: 1000, - missedFramesPct: 10, - }, - ssrc: 1234, -} -``` - ### InboundNetworkIssueDetector Detects issues with inbound network connection. ```js diff --git a/src/WebRTCIssueDetector.ts b/src/WebRTCIssueDetector.ts index deda166..8406cb2 100644 --- a/src/WebRTCIssueDetector.ts +++ b/src/WebRTCIssueDetector.ts @@ -17,7 +17,6 @@ import PeriodicWebRTCStatsReporter from './parser/PeriodicWebRTCStatsReporter'; import DefaultNetworkScoresCalculator from './NetworkScoresCalculator'; import { AvailableOutgoingBitrateIssueDetector, - FramesEncodedSentIssueDetector, InboundNetworkIssueDetector, NetworkMediaSyncIssueDetector, OutboundNetworkIssueDetector, @@ -60,7 +59,6 @@ class WebRTCIssueDetector { this.detectors = params.detectors ?? [ new QualityLimitationsIssueDetector(), - new FramesEncodedSentIssueDetector(), new InboundNetworkIssueDetector(), new OutboundNetworkIssueDetector(), new NetworkMediaSyncIssueDetector(), diff --git a/src/detectors/FramesEncodedSentIssueDetector.ts b/src/detectors/FramesEncodedSentIssueDetector.ts deleted file mode 100644 index 14aa8d5..0000000 --- a/src/detectors/FramesEncodedSentIssueDetector.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - IssueDetectorResult, - IssueReason, - IssueType, - WebRTCStatsParsed, -} from '../types'; -import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; - -interface FramesEncodedSentIssueDetectorParams extends BaseIssueDetectorParams { - missedFramesThreshold?: number; -} - -class FramesEncodedSentIssueDetector extends BaseIssueDetector { - readonly #missedFramesThreshold: number; - - constructor(params: FramesEncodedSentIssueDetectorParams = {}) { - super(params); - this.#missedFramesThreshold = params.missedFramesThreshold ?? 0.15; - } - - performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - return this.processData(data); - } - - private processData(data: WebRTCStatsParsed): IssueDetectorResult { - const streamsWithEncodedFrames = data.video.outbound.filter((stats) => stats.framesEncoded > 0); - const issues: IssueDetectorResult = []; - const previousOutboundRTPVideoStreamsStats = this.getLastProcessedStats(data.connection.id)?.video.outbound; - - if (!previousOutboundRTPVideoStreamsStats) { - return issues; - } - - streamsWithEncodedFrames.forEach((streamStats) => { - const previousStreamStats = previousOutboundRTPVideoStreamsStats.find((item) => item.ssrc === streamStats.ssrc); - - if (!previousStreamStats) { - return; - } - - if (streamStats.framesEncoded === previousStreamStats.framesEncoded) { - // stream is paused - return; - } - - const deltaFramesEncoded = streamStats.framesEncoded - previousStreamStats.framesEncoded; - const deltaFramesSent = streamStats.framesSent - previousStreamStats.framesSent; - const missedFrames = 1 - deltaFramesSent / deltaFramesEncoded; - - if (deltaFramesEncoded === 0) { - // stream is paused - return; - } - - if (deltaFramesEncoded === deltaFramesSent) { - // stream is ok - return; - } - - const statsSample = { - deltaFramesSent, - deltaFramesEncoded, - missedFramesPct: Math.round(missedFrames * 100), - }; - - if (missedFrames >= this.#missedFramesThreshold) { - issues.push({ - statsSample, - type: IssueType.Network, - reason: IssueReason.OutboundNetworkThroughput, - ssrc: streamStats.ssrc, - }); - } - }); - - return issues; - } -} - -export default FramesEncodedSentIssueDetector; diff --git a/src/detectors/index.ts b/src/detectors/index.ts index 534dc6c..2c7e266 100644 --- a/src/detectors/index.ts +++ b/src/detectors/index.ts @@ -1,6 +1,5 @@ export { default as BaseIssueDetector } from './BaseIssueDetector'; export { default as AvailableOutgoingBitrateIssueDetector } from './AvailableOutgoingBitrateIssueDetector'; -export { default as FramesEncodedSentIssueDetector } from './FramesEncodedSentIssueDetector'; export { default as InboundNetworkIssueDetector } from './InboundNetworkIssueDetector'; export { default as NetworkMediaSyncIssueDetector } from './NetworkMediaSyncIssueDetector'; export { default as OutboundNetworkIssueDetector } from './OutboundNetworkIssueDetector'; From c6016fb17742afa560f6dc8aa873cf7e20673adc Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Wed, 22 Jan 2025 12:29:40 +0700 Subject: [PATCH 11/15] fix: refactor --- src/detectors/FrozenVideoTrackDetector.ts | 10 ++-- src/detectors/VideoDecoderIssueDetector.ts | 67 ++++++++-------------- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/src/detectors/FrozenVideoTrackDetector.ts b/src/detectors/FrozenVideoTrackDetector.ts index 6ac28c5..02a605f 100644 --- a/src/detectors/FrozenVideoTrackDetector.ts +++ b/src/detectors/FrozenVideoTrackDetector.ts @@ -14,16 +14,16 @@ interface FrozenVideoTrackDetectorParams { frozenDurationThresholdPct?: number; } -interface FrozenStream { +interface FrozenStreamStatsSample { ssrc: number; avgFreezeDurationMs: number; frozenDurationPct: number; } class FrozenVideoTrackDetector extends BaseIssueDetector { - #avgFreezeDurationThresholdMs: number; + readonly #avgFreezeDurationThresholdMs: number; - #frozenDurationThresholdPct: number; + readonly #frozenDurationThresholdPct: number; constructor(params: FrozenVideoTrackDetectorParams = {}) { super(); @@ -50,7 +50,7 @@ class FrozenVideoTrackDetector extends BaseIssueDetector { } const frozenStreams = data.video.inbound - .map((videoStream): FrozenStream | undefined => { + .map((videoStream): FrozenStreamStatsSample | undefined => { const prevStat = allLastProcessedStats[allLastProcessedStats.length - 1] .video.inbound.find((stream) => stream.ssrc === videoStream.ssrc); @@ -91,7 +91,7 @@ class FrozenVideoTrackDetector extends BaseIssueDetector { return undefined; }) - .filter((stream) => stream !== undefined) as FrozenStream[]; + .filter((stream) => stream !== undefined) as FrozenStreamStatsSample[]; if (frozenStreams.length > 0) { issues.push({ diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts index 03f55cf..8bf7aa2 100644 --- a/src/detectors/VideoDecoderIssueDetector.ts +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -1,3 +1,4 @@ +import { calculateVolatility } from '../helpers/calc'; import { IssueDetectorResult, IssueReason, @@ -14,14 +15,14 @@ interface VideoDecoderIssueDetectorParams extends BaseIssueDetectorParams { } class VideoDecoderIssueDetector extends BaseIssueDetector { - #volatilityThreshold: number; + readonly #volatilityThreshold: number; - #affectedStreamsPercentThreshold: number; + readonly #affectedStreamsPercentThreshold: number; constructor(params: VideoDecoderIssueDetectorParams = {}) { super(params); - this.#volatilityThreshold = params.volatilityThreshold ?? 1.5; - this.#affectedStreamsPercentThreshold = params.affectedStreamsPercentThreshold ?? 50; + this.#volatilityThreshold = params.volatilityThreshold ?? 8; + this.#affectedStreamsPercentThreshold = params.affectedStreamsPercentThreshold ?? 30; } performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { @@ -51,66 +52,46 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { ]; const throtthedStreams = data.video.inbound - .map((incomeVideoStream): { ssrc: number, allDecodeTimePerFrame: number[], volatility: number } | undefined => { - const allDecodeTimePerFrame: number[] = []; - const isSpatialLayerChanged = isSvcSpatialLayerChanged(incomeVideoStream.ssrc, allProcessedStats); - if (isSpatialLayerChanged) { + .map((incomeVideoStream): { ssrc: number, allFps: number[], volatility: number } | undefined => { + // At least 5 elements needed to have enough representation + if (allProcessedStats.length < 5) { return undefined; } - // We need at least 5 elements to have enough representation - if (allProcessedStats.length < 5) { + const isSpatialLayerChanged = isSvcSpatialLayerChanged(incomeVideoStream.ssrc, allProcessedStats); + if (isSpatialLayerChanged) { return undefined; } - // exclude first element to calculate accurate delta - for (let i = 1; i < allProcessedStats.length; i += 1) { - let deltaFramesDecoded = 0; - let deltaTotalDecodeTime = 0; - let decodeTimePerFrame = 0; - + const allFps: number[] = []; + for (let i = 0; i < allProcessedStats.length - 1; i += 1) { const videoStreamStats = allProcessedStats[i].video.inbound.find( (stream) => stream.ssrc === incomeVideoStream.ssrc, ); - if (!videoStreamStats) { - continue; + if (videoStreamStats?.framesPerSecond !== undefined) { + allFps.push(videoStreamStats.framesPerSecond); } - - const prevVideoStreamStats = allProcessedStats[i - 1].video.inbound.find( - (stream) => stream.ssrc === incomeVideoStream.ssrc, - ); - - if (prevVideoStreamStats) { - deltaFramesDecoded = videoStreamStats.framesDecoded - prevVideoStreamStats.framesDecoded; - deltaTotalDecodeTime = videoStreamStats.totalDecodeTime - prevVideoStreamStats.totalDecodeTime; - } - - if (deltaTotalDecodeTime > 0 && deltaFramesDecoded > 0) { - decodeTimePerFrame = (deltaTotalDecodeTime * 1000) / deltaFramesDecoded; - } - - allDecodeTimePerFrame.push(decodeTimePerFrame); } - // Calculate volatility - const mean = allDecodeTimePerFrame.reduce((acc, val) => acc + val, 0) / allDecodeTimePerFrame.length; - const squaredDiffs = allDecodeTimePerFrame.map((val) => (val - mean) ** 2); - const variance = squaredDiffs.reduce((acc, val) => acc + val, 0) / squaredDiffs.length; - const volatility = Math.sqrt(variance); + if (allFps.length === 0) { + return undefined; + } - const isDecodeTimePerFrameIncrease = allDecodeTimePerFrame.every( - (decodeTimePerFrame, index) => index === 0 || decodeTimePerFrame > allDecodeTimePerFrame[index - 1], - ); + const volatility = calculateVolatility(allFps); - if (volatility > this.#volatilityThreshold && isDecodeTimePerFrameIncrease) { - return { ssrc: incomeVideoStream.ssrc, allDecodeTimePerFrame, volatility }; + if (volatility > this.#volatilityThreshold) { + return { ssrc: incomeVideoStream.ssrc, allFps, volatility }; } return undefined; }) .filter((throttledVideoStream) => Boolean(throttledVideoStream)); + if (throtthedStreams.length === 0) { + return issues; + } + const affectedStreamsPercent = throtthedStreams.length / (data.video.inbound.length / 100); if (affectedStreamsPercent > this.#affectedStreamsPercentThreshold) { issues.push({ From 508c69e8df2ac443471171d3ecebd50100f22737 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Wed, 22 Jan 2025 12:42:43 +0700 Subject: [PATCH 12/15] fix: IssueDetector type --- src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index 1f1de93..2b57d02 100644 --- a/src/types.ts +++ b/src/types.ts @@ -11,7 +11,7 @@ export interface WIDWindow { export type IssueDetectorResult = IssuePayload[]; export interface IssueDetector { - detect(data: WebRTCStatsParsed, networkScores: NetworkScores): IssueDetectorResult; + detect(data: WebRTCStatsParsed, networkScores?: NetworkScores): IssueDetectorResult; } export interface INetworkScoresCalculator { From 3c901600a02ae91d43c78695b62971ce19b14520 Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Wed, 22 Jan 2025 12:43:46 +0700 Subject: [PATCH 13/15] add helpers --- src/helpers/calc.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/helpers/calc.ts diff --git a/src/helpers/calc.ts b/src/helpers/calc.ts new file mode 100644 index 0000000..29db3ed --- /dev/null +++ b/src/helpers/calc.ts @@ -0,0 +1,11 @@ +export const calculateMean = (values: number[]) => values.reduce((acc, val) => acc + val, 0) / values.length; + +export const calculateVolatility = (values: number[]) => { + if (values.length === 0) { + throw new Error('Cannot calculate volatility for empty array'); + } + + const mean = calculateMean(values); + const meanAbsoluteDeviationFps = values.reduce((acc, val) => acc + Math.abs(val - mean), 0) / values.length; + return (meanAbsoluteDeviationFps * 100) / mean; +}; From c8f1e85630e5dc2761ae2b5ed6140a73eef11b9f Mon Sep 17 00:00:00 2001 From: Roman Kuzakov Date: Wed, 22 Jan 2025 12:46:28 +0700 Subject: [PATCH 14/15] fix: added MOS quality threshold as a detector param --- src/detectors/FrozenVideoTrackDetector.ts | 6 +++++- src/detectors/VideoDecoderIssueDetector.ts | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/detectors/FrozenVideoTrackDetector.ts b/src/detectors/FrozenVideoTrackDetector.ts index 02a605f..7408645 100644 --- a/src/detectors/FrozenVideoTrackDetector.ts +++ b/src/detectors/FrozenVideoTrackDetector.ts @@ -12,6 +12,7 @@ import BaseIssueDetector from './BaseIssueDetector'; interface FrozenVideoTrackDetectorParams { avgFreezeDurationThresholdMs?: number; frozenDurationThresholdPct?: number; + minMosQuality?: number; } interface FrozenStreamStatsSample { @@ -25,15 +26,18 @@ class FrozenVideoTrackDetector extends BaseIssueDetector { readonly #frozenDurationThresholdPct: number; + readonly #minMosQuality: MosQuality; + constructor(params: FrozenVideoTrackDetectorParams = {}) { super(); this.#avgFreezeDurationThresholdMs = params.avgFreezeDurationThresholdMs ?? 1_000; this.#frozenDurationThresholdPct = params.frozenDurationThresholdPct ?? 30; + this.#minMosQuality = params.minMosQuality ?? MosQuality.BAD; } performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { const inboundScore = data.networkScores.inbound; - if (inboundScore !== undefined && inboundScore <= MosQuality.BAD) { + if (inboundScore !== undefined && inboundScore <= this.#minMosQuality) { // do not execute detection on stats based on poor network quality // to avoid false positives return []; diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts index 8bf7aa2..e879daa 100644 --- a/src/detectors/VideoDecoderIssueDetector.ts +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -12,6 +12,7 @@ import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector' interface VideoDecoderIssueDetectorParams extends BaseIssueDetectorParams { volatilityThreshold?: number; affectedStreamsPercentThreshold?: number; + minMosQuality?: number; } class VideoDecoderIssueDetector extends BaseIssueDetector { @@ -19,10 +20,13 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { readonly #affectedStreamsPercentThreshold: number; + readonly #minMosQuality: MosQuality; + constructor(params: VideoDecoderIssueDetectorParams = {}) { super(params); this.#volatilityThreshold = params.volatilityThreshold ?? 8; this.#affectedStreamsPercentThreshold = params.affectedStreamsPercentThreshold ?? 30; + this.#minMosQuality = params.minMosQuality ?? MosQuality.BAD; } performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { @@ -32,7 +36,7 @@ class VideoDecoderIssueDetector extends BaseIssueDetector { ]; const isBadNetworkHappened = allHistoricalStats - .find((stat) => stat.networkScores.inbound !== undefined && stat.networkScores.inbound <= MosQuality.BAD); + .find((stat) => stat.networkScores.inbound !== undefined && stat.networkScores.inbound <= this.#minMosQuality); if (isBadNetworkHappened) { // do not execute detection on historical stats based on bad network quality From ef04b4bdbed39af914b5754fdfb78d0cb858a62a Mon Sep 17 00:00:00 2001 From: vlprojects-bot Date: Wed, 22 Jan 2025 05:49:19 +0000 Subject: [PATCH 15/15] chore(release): 1.14.0-improve-base-detectors.1 [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b01418e..013532f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webrtc-issue-detector", - "version": "1.14.0-decoder-issue-detector.2", + "version": "1.14.0-improve-base-detectors.1", "description": "WebRTC diagnostic tool that detects issues with network or user devices", "repository": "git@github.com:VLprojects/webrtc-issue-detector.git", "author": "Roman Kuzakov ",