diff --git a/src/browser/provider/built-in/chrome/cdp.js b/src/browser/provider/built-in/chrome/cdp.js index ec85bb702fa..d9bfb7735a5 100644 --- a/src/browser/provider/built-in/chrome/cdp.js +++ b/src/browser/provider/built-in/chrome/cdp.js @@ -45,6 +45,20 @@ async function setEmulation (runtimeInfo) { await resizeWindow({ width: config.width, height: config.height }, runtimeInfo); } +async function getScreenshotData (client) { + const { visualViewport } = await client.Page.getLayoutMetrics(); + + const clipRegion = { + x: visualViewport.pageX, + y: visualViewport.pageY, + width: visualViewport.clientWidth, + height: visualViewport.clientHeight, + scale: visualViewport.scale + }; + + return await client.Page.captureScreenshot({ fromSurface: true, clip: clipRegion }); +} + export async function createClient (runtimeInfo) { const { browserId, config, cdpPort } = runtimeInfo; @@ -99,20 +113,16 @@ export async function updateMobileViewportSize (runtimeInfo) { runtimeInfo.viewportSize.height = windowDimensions.outerHeight; } -export async function takeScreenshot (path, { client }) { - const { visualViewport } = await client.Page.getLayoutMetrics(); +export async function getVideoFrameData ({ client }) { + const screenshotData = await getScreenshotData(client); - const clipRegion = { - x: visualViewport.pageX, - y: visualViewport.pageY, - width: visualViewport.clientWidth, - height: visualViewport.clientHeight, - scale: visualViewport.scale - }; + return Buffer.from(screenshotData.data, 'base64'); +} - const screenshot = await client.Page.captureScreenshot({ fromSurface: true, clip: clipRegion }); +export async function takeScreenshot (path, { client }) { + const screenshotData = await getScreenshotData(client); - await writeFile(path, screenshot.data, { encoding: 'base64' }); + await writeFile(path, screenshotData.data, { encoding: 'base64' }); } export async function resizeWindow (newDimensions, runtimeInfo) { diff --git a/src/browser/provider/built-in/chrome/index.js b/src/browser/provider/built-in/chrome/index.js index cb6c0754427..93d28bac8cb 100644 --- a/src/browser/provider/built-in/chrome/index.js +++ b/src/browser/provider/built-in/chrome/index.js @@ -88,6 +88,10 @@ export default { await this.resizeWindow(browserId, maximumSize.width, maximumSize.height, maximumSize.width, maximumSize.height); }, + async getVideoFrameData (browserId) { + return await cdp.getVideoFrameData(this.openedBrowsers[browserId]); + }, + async hasCustomActionForBrowser (browserId) { const { config, client } = this.openedBrowsers[browserId]; @@ -97,6 +101,7 @@ export default { hasMaximizeWindow: !!client && config.headless, hasTakeScreenshot: !!client, hasChromelessScreenshots: !!client, + hasGetVideoFrameData: !!client, hasCanResizeWindowToDimensions: false }; }, diff --git a/src/browser/provider/built-in/firefox/index.js b/src/browser/provider/built-in/firefox/index.js index 2cf7c33cde8..e7be2f39239 100644 --- a/src/browser/provider/built-in/firefox/index.js +++ b/src/browser/provider/built-in/firefox/index.js @@ -84,6 +84,12 @@ export default { await this.resizeWindow(browserId, maximumSize.width, maximumSize.height); }, + async getVideoFrameData (browserId) { + const { marionetteClient } = this.openedBrowsers[browserId]; + + return await marionetteClient.getVideoFrameData(); + }, + async hasCustomActionForBrowser (browserId) { const { config, marionetteClient } = this.openedBrowsers[browserId]; @@ -91,6 +97,7 @@ export default { hasCloseBrowser: true, hasTakeScreenshot: !!marionetteClient, hasChromelessScreenshots: !!marionetteClient, + hasGetVideoFrameData: !!marionetteClient, hasResizeWindow: !!marionetteClient && config.headless, hasMaximizeWindow: !!marionetteClient && config.headless, hasCanResizeWindowToDimensions: false diff --git a/src/browser/provider/built-in/firefox/marionette-client/index.js b/src/browser/provider/built-in/firefox/marionette-client/index.js index ef3a9e60d4b..d3e7f71a903 100644 --- a/src/browser/provider/built-in/firefox/marionette-client/index.js +++ b/src/browser/provider/built-in/firefox/marionette-client/index.js @@ -143,6 +143,10 @@ module.exports = class MarionetteClient { return responsePacket.body[3]; } + async _getScreenshotData () { + return await this._getResponse({ command: COMMANDS.takeScreenshot }); + } + async connect () { await this._connectSocket(this.port, this.host); @@ -169,9 +173,15 @@ module.exports = class MarionetteClient { } async takeScreenshot (path) { - const screenshot = await this._getResponse({ command: COMMANDS.takeScreenshot }); + const screenshotData = await this._getScreenshotData(); + + await writeFile(path, screenshotData.value, { encoding: 'base64' }); + } + + async getVideoFrameData () { + const screenshotData = await this._getScreenshotData(); - await writeFile(path, screenshot.value, { encoding: 'base64' }); + return Buffer.from(screenshotData.value, 'base64'); } async setWindowSize (width, height) { diff --git a/src/browser/provider/index.js b/src/browser/provider/index.js index bd9a4caf215..7112b693187 100644 --- a/src/browser/provider/index.js +++ b/src/browser/provider/index.js @@ -303,6 +303,10 @@ export default class BrowserProvider { await this.plugin.takeScreenshot(browserId, screenshotPath, pageWidth, pageHeight); } + async getVideoFrameData (browserId) { + return this.plugin.getVideoFrameData(browserId); + } + async hasCustomActionForBrowser (browserId) { return this.plugin.hasCustomActionForBrowser(browserId); } diff --git a/src/browser/provider/plugin-host.js b/src/browser/provider/plugin-host.js index 6608d7d696a..06580a4b527 100644 --- a/src/browser/provider/plugin-host.js +++ b/src/browser/provider/plugin-host.js @@ -109,6 +109,7 @@ export default class BrowserProviderPluginHost { hasCloseBrowser: this.hasOwnProperty('closeBrowser'), hasResizeWindow: this.hasOwnProperty('resizeWindow'), hasTakeScreenshot: this.hasOwnProperty('takeScreenshot'), + hasGetVideoFrameData: this.hasOwnProperty('getVideoFrameData'), hasCanResizeWindowToDimensions: this.hasOwnProperty('canResizeWindowToDimensions'), hasMaximizeWindow: this.hasOwnProperty('maximizeWindow'), hasChromelessScreenshots: false @@ -131,6 +132,10 @@ export default class BrowserProviderPluginHost { this.reportWarning(browserId, WARNING_MESSAGE.maximizeNotSupportedByBrowserProvider, this[name]); } + async getVideoFrameData (browserId) { + this.reportWarning(browserId, WARNING_MESSAGE.screenshotNotSupportedByBrowserProvider, this[name]); + } + async reportJobResult (/*browserId, status, data*/) { return; } diff --git a/src/cli/argument-parser.js b/src/cli/argument-parser.js index 6d9e2845300..ce8e8151e48 100644 --- a/src/cli/argument-parser.js +++ b/src/cli/argument-parser.js @@ -6,10 +6,11 @@ import MESSAGE from '../errors/runtime/message'; import { assertType, is } from '../errors/runtime/type-assertions'; import getViewPortWidth from '../utils/get-viewport-width'; import { wordWrap, splitQuotedText } from '../utils/string'; -import parseSslOptions from '../utils/parse-ssl-options'; +import { getSSLOptions, getVideoOptions } from '../utils/get-options'; import createFilterFn from '../utils/create-filter-fn'; import { optionValueToRegExp, optionValueToKeyValue } from '../configuration/option-conversion'; + const REMOTE_ALIAS_RE = /^remote(?::(\d*))?$/; const DESCRIPTION = dedent(` @@ -76,6 +77,8 @@ export default class CLIArgumentParser { .option('-F, --fixture-grep ', 'run only fixtures matching the specified pattern') .option('-a, --app ', 'launch the tested app using the specified command before running tests') .option('-c, --concurrency ', 'run tests concurrently') + .option('--video ', 'capture a video of the running tests') + .option('--video-options ', 'video capturing options') .option('--test-meta ', 'run only tests with matching metadata') .option('--fixture-meta ', 'run only fixtures with matching metadata') .option('--debug-on-fail', 'pause the test if it fails') @@ -182,7 +185,7 @@ export default class CLIArgumentParser { async _parseSslOptions () { if (this.opts.ssl) - this.opts.ssl = await parseSslOptions(this.opts.ssl); + this.opts.ssl = await getSSLOptions(this.opts.ssl); } async _parseReporters () { @@ -205,6 +208,11 @@ export default class CLIArgumentParser { this.src = this.program.args.slice(1); } + async _parseVideoOptions () { + this.opts.video = typeof this.opts.video === 'string' ? await getVideoOptions(this.opts.video) : {}; + this.opts.advancedVideoOptions = typeof this.opts.videoOptions === 'string' ? await getVideoOptions(this.opts.videoOptions) : {}; + } + _getProviderName () { this.opts.providerName = this.opts.listBrowsers === true ? void 0 : this.opts.listBrowsers; } @@ -232,6 +240,7 @@ export default class CLIArgumentParser { this._parseConcurrency(); this._parseFileList(); + await this._parseVideoOptions(); await this._parseSslOptions(); await this._parseReporters(); } diff --git a/src/cli/cli.js b/src/cli/cli.js index e783051cbc2..6a4ba45f116 100644 --- a/src/cli/cli.js +++ b/src/cli/cli.js @@ -85,6 +85,9 @@ async function runTests (argParser) { .screenshots(opts.screenshots, opts.screenshotsOnFails, opts.screenshotPathPattern) .startApp(opts.app, opts.appInitDelay); + if (opts.video) + runner.video(opts.video, opts.videoOptions); + runner.once('done-bootstrapping', () => log.hideSpinner()); try { diff --git a/src/configuration/index.js b/src/configuration/index.js index f0e7adba7fa..b845db155bc 100644 --- a/src/configuration/index.js +++ b/src/configuration/index.js @@ -1,9 +1,8 @@ -import Promise from 'pinkie'; import { fsObjectExists, readFile } from '../utils/promisified-functions'; import Option from './option'; import optionSource from './option-source'; import { cloneDeep, castArray } from 'lodash'; -import { ensureOptionValue as ensureSslOptionValue } from '../utils/parse-ssl-options'; +import { getSSLOptions } from '../utils/get-options'; import OPTION_NAMES from './option-names'; import { optionValueToRegExp } from './option-conversion'; import createFilterFn from '../utils/create-filter-fn'; @@ -100,14 +99,12 @@ export default class Configuration { } async _prepareSslOptions () { - const sslOptions = this._options[OPTION_NAMES.ssl]; + let sslOptions = this._options[OPTION_NAMES.ssl]; if (!sslOptions) return; - await Promise.all(Object.entries(sslOptions.value).map(async ([key, value]) => { - sslOptions.value[key] = await ensureSslOptionValue(key, value); - })); + sslOptions = await getSSLOptions(sslOptions); } _ensureOption (name, value, source) { diff --git a/src/configuration/option-names.js b/src/configuration/option-names.js index 1ad7ceb41af..2496e020675 100644 --- a/src/configuration/option-names.js +++ b/src/configuration/option-names.js @@ -13,5 +13,7 @@ export default { proxyBypass: 'proxyBypass', appCommand: 'appCommand', appInitDelay: 'appInitDelay', - disableTestSyntaxValidation: 'disableTestSyntaxValidation' + disableTestSyntaxValidation: 'disableTestSyntaxValidation', + video: 'video', + videoOptions: 'videoOptions' }; diff --git a/src/runner/browser-job.js b/src/runner/browser-job.js index ece388614f1..4521b195b62 100644 --- a/src/runner/browser-job.js +++ b/src/runner/browser-job.js @@ -36,6 +36,8 @@ export default class BrowserJob extends AsyncEventEmitter { const testRunController = new TestRunController(test, index + 1, this.proxy, this.screenshots, this.warningLog, this.fixtureHookController, this.opts); + testRunController.on('test-run-create', testRunInfo => this.emit('test-run-create', testRunInfo)); + testRunController.on('test-run-ready', () => this.emit('test-run-ready', testRunController.testRun)); testRunController.on('test-run-start', () => this.emit('test-run-start', testRunController.testRun)); testRunController.on('test-run-restart', () => this._onTestRunRestart(testRunController)); testRunController.on('test-run-done', () => this._onTestRunDone(testRunController)); diff --git a/src/runner/index.js b/src/runner/index.js index ea5baefd3c6..cc4518caa3d 100644 --- a/src/runner/index.js +++ b/src/runner/index.js @@ -13,6 +13,7 @@ import { GeneralError } from '../errors/runtime'; import MESSAGE from '../errors/runtime/message'; import { assertType, is } from '../errors/runtime/type-assertions'; import renderForbiddenCharsList from '../errors/render-forbidden-chars-list'; +import detectFFMPEG from '../utils/detect-ffmpeg'; import checkFilePath from '../utils/check-file-path'; import { addRunningTest, removeRunningTest, startHandlingTestErrors, stopHandlingTestErrors } from '../utils/handle-errors'; import OPTION_NAMES from '../configuration/option-names'; @@ -211,8 +212,23 @@ export default class Runner extends EventEmitter { throw new GeneralError(MESSAGE.cantUseScreenshotPathPatternWithoutBaseScreenshotPathSpecified); } - _validateRunOptions () { + async _validateVideoOptions () { + const videoConfig = this.configuration.getOption(OPTION_NAMES.video); + + if (!videoConfig) + return; + + videoConfig.basePath = resolvePath(videoConfig.basePath); + + if (videoConfig.ffmpegPath) + videoConfig.ffmpegPath = resolvePath(videoConfig.ffmpegPath); + else + videoConfig.ffmpegPath = await detectFFMPEG(); + } + + async _validateRunOptions () { this._validateScreenshotOptions(); + await this._validateVideoOptions(); this._validateSpeedOption(); this._validateConcurrencyOption(); this._validateProxyBypassOption(); @@ -345,6 +361,15 @@ export default class Runner extends EventEmitter { return this; } + video (options, advancedOptions) { + this.configuration.mergeOptions({ + [OPTION_NAMES.video]: options || {}, + [OPTION_NAMES.videoOptions]: advancedOptions + }); + + return this; + } + startApp (command, initDelay) { this.configuration.mergeOptions({ [OPTION_NAMES.appCommand]: command, @@ -375,11 +400,8 @@ export default class Runner extends EventEmitter { this._setBootstrapperOptions(); const runTaskPromise = Promise.resolve() - .then(() => { - this._validateRunOptions(); - - return this.bootstrapper.createRunnableConfiguration(); - }) + .then(() => this._validateRunOptions()) + .then(() => this.bootstrapper.createRunnableConfiguration()) .then(({ reporterPlugins, browserSet, tests, testedApp }) => { this.emit('done-bootstrapping'); diff --git a/src/runner/task.js b/src/runner/task.js index 7f166f4e6f2..bd1840b53ee 100644 --- a/src/runner/task.js +++ b/src/runner/task.js @@ -1,7 +1,9 @@ import { pull as remove } from 'lodash'; +import moment from 'moment'; import AsyncEventEmitter from '../utils/async-event-emitter'; import BrowserJob from './browser-job'; import Screenshots from '../screenshots'; +import VideoRecorder from '../video-recorder'; import WarningLog from '../notifications/warning-log'; import FixtureHookController from './fixture-hook-controller'; @@ -9,6 +11,7 @@ export default class Task extends AsyncEventEmitter { constructor (tests, browserConnectionGroups, proxy, opts) { super(); + this.timeStamp = moment(); this.running = false; this.browserConnectionGroups = browserConnectionGroups; this.tests = tests; @@ -18,6 +21,9 @@ export default class Task extends AsyncEventEmitter { this.fixtureHookController = new FixtureHookController(tests, browserConnectionGroups.length); this.pendingBrowserJobs = this._createBrowserJobs(proxy, this.opts); + + if (this.opts.video) + this.videoRecorders = this._createVideoRecorders(this.pendingBrowserJobs); } _assignBrowserJobEventHandlers (job) { @@ -60,6 +66,12 @@ export default class Task extends AsyncEventEmitter { }); } + _createVideoRecorders (browserJobs) { + const videoConfig = { timeStamp: this.timeStamp, ...this.opts.video }; + + return browserJobs.map(browserJob => new VideoRecorder(browserJob, videoConfig, this.opts.videoOptions)); + } + // API abort () { this.pendingBrowserJobs.forEach(job => job.abort()); diff --git a/src/runner/test-run-controller.js b/src/runner/test-run-controller.js index 783e62754c3..edc28a67886 100644 --- a/src/runner/test-run-controller.js +++ b/src/runner/test-run-controller.js @@ -55,7 +55,7 @@ export default class TestRunController extends AsyncEventEmitter { return test.isLegacy ? LegacyTestRun : TestRun; } - _createTestRun (connection) { + async _createTestRun (connection) { const screenshotCapturer = this.screenshots.createCapturerFor(this.test, this.index, this.quarantine, connection, this.warningLog); const TestRunCtor = this.TestRunCtor; @@ -64,6 +64,13 @@ export default class TestRunController extends AsyncEventEmitter { if (this.testRun.addQuarantineInfo) this.testRun.addQuarantineInfo(this.quarantine); + await this.emit('test-run-create', { + testRun: this.testRun, + test: this.test, + index: this.index, + quarantine: this.quarantine, + }); + return this.testRun; } @@ -139,7 +146,7 @@ export default class TestRunController extends AsyncEventEmitter { } async start (connection) { - const testRun = this._createTestRun(connection); + const testRun = await this._createTestRun(connection); const hookOk = await this.fixtureHookController.runFixtureBeforeHookIfNecessary(testRun); @@ -150,6 +157,7 @@ export default class TestRunController extends AsyncEventEmitter { } testRun.once('start', () => this.emit('test-run-start')); + testRun.once('ready', () => this.emit('test-run-ready')); testRun.once('done', () => this._testRunDone()); testRun.once('disconnected', () => this._testRunDisconnected(connection)); diff --git a/src/screenshots/index.js b/src/screenshots/index.js index c59bf18c7e2..cda15783f71 100644 --- a/src/screenshots/index.js +++ b/src/screenshots/index.js @@ -1,9 +1,12 @@ import { find } from 'lodash'; import moment from 'moment'; import Capturer from './capturer'; -import PathPattern from './path-pattern'; +import PathPattern from '../utils/path-pattern'; import getCommonPath from '../utils/get-common-path'; + +const SCREENSHOT_EXTENSION = 'png'; + export default class Screenshots { constructor (path, pattern) { this.enabled = !!path; @@ -54,7 +57,7 @@ export default class Screenshots { createCapturerFor (test, testIndex, quarantine, connection, warningLog) { const testEntry = this._ensureTestEntry(test); - const pathPattern = new PathPattern(this.screenshotsPattern, { + const pathPattern = new PathPattern(this.screenshotsPattern, SCREENSHOT_EXTENSION, { testIndex, quarantineAttempt: quarantine ? quarantine.getNextAttemptNumber() : null, now: this.now, diff --git a/src/test-run/index.js b/src/test-run/index.js index f2ca3e43fcb..f3c7a283be0 100644 --- a/src/test-run/index.js +++ b/src/test-run/index.js @@ -696,6 +696,8 @@ const ServiceMessages = TestRun.prototype; ServiceMessages[CLIENT_MESSAGES.ready] = function (msg) { this.debugLog.driverMessage(msg); + this.emit('ready'); + this._clearPendingRequest(); // NOTE: the driver sends the status for the second time if it didn't get a response at the diff --git a/src/utils/convert-to-best-fit-type.js b/src/utils/convert-to-best-fit-type.js new file mode 100644 index 00000000000..507ea51e9e2 --- /dev/null +++ b/src/utils/convert-to-best-fit-type.js @@ -0,0 +1,19 @@ +const NUMBER_REG_EX = /^[0-9-.,]+$/; +const BOOLEAN_STRING_VALUES = ['true', 'false']; + + +export default function (valueStr) { + if (typeof valueStr !== 'string') + return void 0; + + else if (NUMBER_REG_EX.test(valueStr)) + return parseFloat(valueStr); + + else if (BOOLEAN_STRING_VALUES.includes(valueStr)) + return valueStr === 'true'; + + else if (!valueStr.length) + return void 0; + + return valueStr; +} diff --git a/src/utils/detect-ffmpeg.js b/src/utils/detect-ffmpeg.js new file mode 100644 index 00000000000..f2ee172c096 --- /dev/null +++ b/src/utils/detect-ffmpeg.js @@ -0,0 +1,53 @@ +import { isWin } from 'os-family'; +import resolveCwd from 'resolve-cwd'; +import { exec } from './promisified-functions'; + +const FFMPEG_MODULE_NAME = '@ffmpeg-installer/ffmpeg'; +const FFMPEG_SEARCH_COMMAND = isWin ? 'where' : 'which'; +const FFMPEG_BINARY_NAME = 'ffmpeg'; + +async function findFFMPEGinPath () { + try { + const ffmpegPath = await exec(`${FFMPEG_SEARCH_COMMAND} ${FFMPEG_BINARY_NAME}`); + + return ffmpegPath.trim(); + } + catch (e) { + return ''; + } +} + +async function requireFFMPEGModuleFromCwd () { + try { + const ffmpegModulePath = resolveCwd(FFMPEG_MODULE_NAME); + + return require(ffmpegModulePath).path; + } + catch (e) { + return ''; + } +} + +async function requireFFMPEGModule () { + try { + return require(FFMPEG_MODULE_NAME).path; + } + catch (e) { + return ''; + } +} + +export default async function () { + let ffmpegPath = process.env.FFMPEG_PATH; + + if (!ffmpegPath) + ffmpegPath = await requireFFMPEGModuleFromCwd(); + + if (!ffmpegPath) + ffmpegPath = await requireFFMPEGModule(); + + if (!ffmpegPath) + ffmpegPath = await findFFMPEGinPath(); + + return ffmpegPath; +} diff --git a/src/utils/get-options/base.js b/src/utils/get-options/base.js new file mode 100644 index 00000000000..a36a2d7cd36 --- /dev/null +++ b/src/utils/get-options/base.js @@ -0,0 +1,50 @@ +import Promise from 'pinkie'; +import convertToBestFitType from '../convert-to-best-fit-type'; + + +const DEFAULT_OPTIONS_SEPARATOR = ','; +const DEFAULT_KEY_VALUE_SEPARATOR = '='; + +const DEFAULT_ON_OPTION_PARSED = (key, value) => value; + +function parseOptionsString (optionsStr, optionsSeparator, keyValueSeparator) { + const splittedOptions = optionsStr.split(optionsSeparator); + + if (!splittedOptions.length) + return null; + + const parsedOptions = {}; + + splittedOptions.forEach(item => { + const keyValuePair = item.split(keyValueSeparator); + const key = keyValuePair[0]; + let value = keyValuePair[1]; + + if (!value && keyValuePair.length === 1) + value = true; + + parsedOptions[key] = value; + }); + + return parsedOptions; +} + +export default async function (options = '', optionsConfig) { + const { + optionsSeparator = DEFAULT_OPTIONS_SEPARATOR, + keyValueSeparator = DEFAULT_KEY_VALUE_SEPARATOR, + onOptionParsed = DEFAULT_ON_OPTION_PARSED + } = optionsConfig; + + if (typeof options === 'string') + options = parseOptionsString(options, optionsSeparator, keyValueSeparator); + + await Promise.all(Object.entries(options).map(async ([key, value]) => { + value = convertToBestFitType(value); + + options[key] = await onOptionParsed(key, value); + })); + + return options; +} + diff --git a/src/utils/get-options/index.js b/src/utils/get-options/index.js new file mode 100644 index 00000000000..8397def68a8 --- /dev/null +++ b/src/utils/get-options/index.js @@ -0,0 +1,4 @@ +import getSSLOptions from './ssl'; +import getVideoOptions from './video'; + +export { getVideoOptions, getSSLOptions }; diff --git a/src/utils/get-options/ssl.js b/src/utils/get-options/ssl.js new file mode 100644 index 00000000000..acdb8395003 --- /dev/null +++ b/src/utils/get-options/ssl.js @@ -0,0 +1,32 @@ +import os from 'os'; +import baseGetOptions from './base'; +import { fsObjectExists, readFile } from '../promisified-functions'; + + +const MAX_PATH_LENGTH = { + 'Linux': 4096, + 'Windows_NT': 260, + 'Darwin': 1024 +}; + +const OS_MAX_PATH_LENGTH = MAX_PATH_LENGTH[os.type()]; + +const OPTIONS_SEPARATOR = ';'; +const FILE_OPTION_NAMES = ['cert', 'key', 'pfx']; + + +export default function (optionString) { + return baseGetOptions(optionString, { + optionsSeparator: OPTIONS_SEPARATOR, + + async onOptionParsed (key, value) { + const isFileOption = FILE_OPTION_NAMES.includes(key) && value.length < OS_MAX_PATH_LENGTH; + + if (isFileOption && await fsObjectExists(value)) + value = await readFile(value); + + return value; + } + }); +} + diff --git a/src/utils/get-options/video.js b/src/utils/get-options/video.js new file mode 100644 index 00000000000..48615906fc0 --- /dev/null +++ b/src/utils/get-options/video.js @@ -0,0 +1,6 @@ +import baseGetOptions from './base'; + + +export default function (options) { + return baseGetOptions(options, {}); +} diff --git a/src/screenshots/path-pattern.js b/src/utils/path-pattern.js similarity index 94% rename from src/screenshots/path-pattern.js rename to src/utils/path-pattern.js index 51c1f6e12ac..36ebd07b1a0 100644 --- a/src/screenshots/path-pattern.js +++ b/src/utils/path-pattern.js @@ -5,8 +5,6 @@ import escapeUserAgent from '../utils/escape-user-agent'; const DATE_FORMAT = 'YYYY-MM-DD'; const TIME_FORMAT = 'HH-mm-ss'; -const SCRENSHOT_EXTENTION = 'png'; - const ERRORS_FOLDER = 'errors'; const PLACEHOLDERS = { @@ -25,14 +23,15 @@ const PLACEHOLDERS = { }; const DEFAULT_PATH_PATTERN_FOR_REPORT = `${PLACEHOLDERS.DATE}_${PLACEHOLDERS.TIME}\\test-${PLACEHOLDERS.TEST_INDEX}`; -const DEFAULT_PATH_PATTERN = `${DEFAULT_PATH_PATTERN_FOR_REPORT}\\${PLACEHOLDERS.USERAGENT}\\${PLACEHOLDERS.FILE_INDEX}.${SCRENSHOT_EXTENTION}`; -const QUARANTINE_MODE_DEFAULT_PATH_PATTERN = `${DEFAULT_PATH_PATTERN_FOR_REPORT}\\run-${PLACEHOLDERS.QUARANTINE_ATTEMPT}\\${PLACEHOLDERS.USERAGENT}\\${PLACEHOLDERS.FILE_INDEX}.${SCRENSHOT_EXTENTION}`; +const DEFAULT_PATH_PATTERN = `${DEFAULT_PATH_PATTERN_FOR_REPORT}\\${PLACEHOLDERS.USERAGENT}\\${PLACEHOLDERS.FILE_INDEX}`; +const QUARANTINE_MODE_DEFAULT_PATH_PATTERN = `${DEFAULT_PATH_PATTERN_FOR_REPORT}\\run-${PLACEHOLDERS.QUARANTINE_ATTEMPT}\\${PLACEHOLDERS.USERAGENT}\\${PLACEHOLDERS.FILE_INDEX}`; export default class PathPattern { - constructor (pattern, data) { + constructor (pattern, fileExtension, data) { this.pattern = this._ensurePattern(pattern, data.quarantineAttempt); this.data = this._addDefaultFields(data); this.placeholderToDataMap = this._createPlaceholderToDataMap(); + this.fileExtension = fileExtension; } _ensurePattern (pattern, quarantineAttempt) { @@ -103,7 +102,7 @@ export default class PathPattern { getPath (forError) { const path = PathPattern._buildPath(this.pattern, this.placeholderToDataMap, forError); - return correctFilePath(path, SCRENSHOT_EXTENTION); + return correctFilePath(path, this.fileExtension); } // For testing purposes diff --git a/src/video-recorder/index.js b/src/video-recorder/index.js new file mode 100644 index 00000000000..4bc11f2e60b --- /dev/null +++ b/src/video-recorder/index.js @@ -0,0 +1,143 @@ +import { join, dirname } from 'path'; +import fs from 'fs'; +import { spawnSync } from 'child_process'; +import makeDir from 'make-dir'; +import VideoRecorderProcess from './process'; +import TempDirectory from '../utils/temp-directory'; +import PathPattern from '../utils/path-pattern'; + + +const VIDEO_EXTENSION = 'mp4'; + +const TEMP_DIR_PREFIX = 'video'; +const TEMP_VIDEO_FILE_PREFIX = 'tmp-video'; +const TEMP_MERGE_FILE_PREFIX = TEMP_VIDEO_FILE_PREFIX + '-merge'; + +const TEMP_MERGE_CONFIG_FILE_PREFIX = 'config'; +const TEMP_MERGE_CONFIG_FILE_EXTENSION = 'txt'; + +export default class VideoRecorder { + constructor (browserJob, opts, advancedOpts) { + this.failedOnly = opts.failedOnly; + this.singleFile = opts.singleFile; + this.ffmpegPath = opts.ffmpegPath; + this.basePath = opts.basePath; + this.customPathPattern = opts.pathPattern; + this.timeStamp = opts.timeStamp; + this.advancedOptions = advancedOpts; + + this.tempDirectory = new TempDirectory(TEMP_DIR_PREFIX); + this.tempVideoPath = ''; + this.tempMergeConfigPath = ''; + + this.firstFile = true; + + this.testRunInfo = {}; + + this._assignEventHandlers(browserJob); + } + + _assignEventHandlers (browserJob) { + browserJob.once('start', () => this._onBrowserJobStart()); + browserJob.once('done', () => this._onBrowserJobDone()); + browserJob.on('test-run-create', testRunInfo => this._onTestRunCreate(testRunInfo)); + browserJob.on('test-run-ready', testRun => this._onTestRunReady(testRun)); + browserJob.on('test-run-done', testRun => this._onTestRunDone(testRun)); + } + + _getTargetVideoPath (testRunInfo) { + if (this.singleFile) + return join(this.basePath, 'video.mp4'); + + const { quarantine, test, index, testRun } = testRunInfo; + + const connection = testRun.browserConnection; + + const pathPattern = new PathPattern(this.customPathPattern, VIDEO_EXTENSION, { + testIndex: index, + quarantineAttempt: quarantine ? quarantine.getNextAttemptNumber() : null, + now: this.timeStamp, + fixture: test.fixture.name, + test: test.name, + parsedUserAgent: connection.browserInfo.parsedUserAgent, + }); + + return join(this.basePath, pathPattern.getPath()); + } + + _generateTempNames (id) { + const tempFileNames = { + tempVideoPath: `${TEMP_VIDEO_FILE_PREFIX}-${id}.${VIDEO_EXTENSION}`, + tempMergeConfigPath: `${TEMP_MERGE_CONFIG_FILE_PREFIX}-${id}.${TEMP_MERGE_CONFIG_FILE_EXTENSION}`, + tmpMergeName: `${TEMP_MERGE_FILE_PREFIX}-${id}.${VIDEO_EXTENSION}` + }; + + for (const [tempFile, tempName] of Object.entries(tempFileNames)) + tempFileNames[tempFile] = join(this.tempDirectory.path, tempName); + + return tempFileNames; + } + + _concatVideo (targetVideoPath, { tempVideoPath, tempMergeConfigPath, tempMergePath }) { + if (this.firstFile) { + this.firstFile = false; + return; + } + + fs.writeFileSync(this.tempMergeConfigPath, ` + file '${targetVideoPath}' + file '${tempVideoPath}' + `); + + spawnSync(this.ffmpegPath, ['-y', '-f', 'concat', '-safe', '0', '-i', tempMergeConfigPath, '-c', 'copy', tempMergePath], { stdio: 'inherit' }); + fs.copyFileSync(tempMergePath, tempVideoPath); + } + + async _onBrowserJobStart () { + await this.tempDirectory.init(); + } + + async _onBrowserJobDone () { + await this.tempDirectory.dispose(); + } + + async _onTestRunCreate ({ testRun, quarantine, test, index }) { + const testRunInfo = { testRun, quarantine, test, index }; + + this.testRunInfo[testRun] = testRunInfo; + + const connection = testRun.browserConnection; + + testRunInfo.tempFiles = this._generateTempNames(connection.id); + + testRunInfo.videoRecorder = new VideoRecorderProcess(testRunInfo.tempFiles.tempVideoPath, this.ffmpegPath, connection, this.advancedOptions); + + await testRunInfo.videoRecorder.init(); + } + + async _onTestRunReady (testRun) { + const testRunInfo = this.testRunInfo[testRun]; + + await testRunInfo.videoRecorder.startCapturing(); + } + + async _onTestRunDone (testRun) { + const testRunInfo = this.testRunInfo[testRun]; + + delete this.testRunInfo[testRun]; + + await testRunInfo.videoRecorder.finishCapturing(); + + const videoPath = this._getTargetVideoPath(testRunInfo); + + if (this.failedOnly && !testRun.errs.length) + return; + + await makeDir(dirname(videoPath)); + + if (this.singleFile) + this._concatVideo(videoPath, testRunInfo.tempFiles); + + fs.copyFileSync(testRunInfo.tempFiles.tempVideoPath, videoPath); + } +} diff --git a/src/video-recorder/process.js b/src/video-recorder/process.js new file mode 100644 index 00000000000..3cbec34dbba --- /dev/null +++ b/src/video-recorder/process.js @@ -0,0 +1,104 @@ +import { spawn } from 'child_process'; +import { flatten } from 'lodash'; +import Promise from 'pinkie'; +import AsyncEmitter from '../utils/async-event-emitter'; +import delay from '../utils/delay'; + + +const DEFAULT_OPTIONS = { + 'y': true, + 'use_wallclock_as_timestamps': 1, + 'i': 'pipe:0', + 'c:v': 'libx264', + 'preset': 'ultrafast', + 'pix_fmt': 'yuv420p', + 'vf': 'scale=trunc(iw/2)*2:trunc(ih/2)*2', + 'r': 30 +}; + +const FFMPEG_START_DELAY = 500; + +export default class VideoRecorder extends AsyncEmitter { + constructor (basePath, ffmpegPath, connection, customOptions) { + super(); + + this.customOptions = customOptions; + this.videoPath = basePath; + this.connection = connection; + this.ffmpegPath = ffmpegPath; + this.ffmpegProcess = null; + + this.ffmpegClosingPromise = null; + + this.closed = false; + + this.optionsList = this._getOptionsList(); + + this.capturingPromise = null; + } + + static _filterOption ([key, value]) { + if (value === true) + return ['-' + key]; + + return ['-' + key, value]; + } + + _getOptionsList () { + const optionsObject = Object.assign({}, DEFAULT_OPTIONS, this.customOptions); + + const optionsList = flatten(Object.entries(optionsObject).map(VideoRecorder._filterOption)); + + optionsList.push(this.videoPath); + + return optionsList; + } + + async _addFrame (frameData) { + const writingFinished = this.ffmpegProcess.stdin.write(frameData); + + if (!writingFinished) + await new Promise(r => this.ffmpegProcess.stdin.once('drain', r)); + } + + async _capture () { + while (!this.closed) { + const frame = await this.connection.provider.getVideoFrameData(this.connection.id); + + if (frame) { + await this.emit('frame'); + await this._addFrame(frame); + } + } + } + + async init () { + this.ffmpegProcess = spawn(this.ffmpegPath, this.optionsList, { stdio: ['pipe', 'inherit', 'inherit' ] }); + + this.ffmpegClosingPromise = new Promise(r => { + this.ffmpegProcess.on('exit', r); + this.ffmpegProcess.on('error', r); + }); + + await delay(FFMPEG_START_DELAY); + } + + async startCapturing () { + this.capturingPromise = this._capture(); + + await this.once('frame'); + } + + async finishCapturing () { + if (this.closed) + return; + + this.closed = true; + + await this.capturingPromise; + + this.ffmpegProcess.stdin.end(); + + await this.ffmpegClosingPromise; + } +} diff --git a/test/server/path-pattern-test.js b/test/server/path-pattern-test.js index b90e8a99853..06013b9425a 100644 --- a/test/server/path-pattern-test.js +++ b/test/server/path-pattern-test.js @@ -2,9 +2,11 @@ const path = require('path'); const expect = require('chai').expect; const moment = require('moment'); const userAgent = require('useragent'); -const PathPattern = require('../../lib/screenshots/path-pattern'); +const PathPattern = require('../../lib/utils/path-pattern'); +const SCREENSHOT_EXTENSION = 'png'; + describe('Screenshot path pattern', () => { const parsedUserAgentMock = { toVersion: () => {}, @@ -17,7 +19,7 @@ describe('Screenshot path pattern', () => { data.parsedUserAgent = data.parsedUserAgent || parsedUserAgentMock; data.quarantineAttempt = data.quarantineAttempt || null; - return new PathPattern(pattern, data); + return new PathPattern(pattern, SCREENSHOT_EXTENSION, data); }; describe('Default pattern', () => {