Skip to content

Commit

Permalink
Functional Test execution on Lamdatest (#4472)
Browse files Browse the repository at this point in the history
dsilhavy authored May 6, 2024
1 parent 8239d6a commit a7f983e
Showing 8 changed files with 266 additions and 224 deletions.
93 changes: 60 additions & 33 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -29,23 +29,36 @@ commands:
- run:
name: Check if preconditions are met for functional tests
command: |
if [ -z "$BROWSERSTACK_ACCESS_KEY" ]; then
echo "BrowserStack not configured, functional tests will not be executed."
if [ -z "LAMBDATEST_ACCESS_KEY" ]; then
echo "Lambdatest not configured, functional tests will not be executed."
circleci-agent step halt
fi
functional_test_setup:
steps:
- run:
name: Download the browserstack binary file to create a tunnel
command: wget "https://www.browserstack.com/browserstack-local/BrowserStackLocal-linux-x64.zip"
name: "Downloading tunnel binary"
command: |
wget https://downloads.lambdatest.com/tunnel/v3/linux/64bit/LT_Linux.zip
- run:
name: Unzip the browserstack binary file
command: unzip BrowserStackLocal-linux-x64.zip
name: "Extracting tunnel binary"
command: |
sudo apt-get install unzip
unzip LT_Linux.zip
- run:
name: Run browserstack with provided access key
command: ./BrowserStackLocal $BROWSERSTACK_ACCESS_KEY
name: "Executing tunnel binary"
background: true
command: |
./LT --user ${LAMBDATEST_EMAIL} --key ${LAMBDATEST_KEY}
sleep 40
- run:
name: "Setup custom environment variables"
command: |
echo 'export LT_USERNAME="${LAMBDATEST_USER}"' >> $BASH_ENV
echo 'export LT_ACCESS_KEY="${LAMBDATEST_ACCESS_KEY}"' >> $BASH_ENV
run_testbuild:
steps:
@@ -67,10 +80,11 @@ commands:
command: |
mkdir test/functional/results/test/karma/junit/adjusted
cd test/functional/results/test/karma/junit
for file in *.xml; do
for file in **/*.xml; do
[ -f "$file" ] || continue
xmlstarlet ed -L -u '//testcase[@time="NaN"]/@time' -v '0' "$file"
cp "$file" adjusted/$file
modified_file="${file//\//_}"
cp "$file" adjusted/$modified_file
done
- store_test_results:
path: test/functional/results/test/karma/junit/adjusted
@@ -79,6 +93,17 @@ commands:
steps:
- functional_precondition
- checkout
- run:
name: Virtual merge into development branch
command: |
if [ "${CIRCLE_BRANCH}" = "development" ]; then
echo "On development branch already, no merge needed"
else
git config --global user.email "circleci@example.com"
git config --global user.name "CircleCI"
git checkout development
git merge --no-edit --no-ff $CIRCLE_BRANCH
fi
- dependencies_setup
- run_testbuild
- functional_test_setup
@@ -89,9 +114,13 @@ commands:
default: "single"
type: string
configfile:
default: "browserstack"
default: "lambdatest"
type: string
steps:
- run:
name: Sleep
command: |
sleep 30
- run:
name: Run functional tests <<parameters.configfile>> - <<parameters.streamsfile>>
when: always
@@ -138,31 +167,22 @@ jobs:
- dependencies_setup
- run_testbuild

functional-tests-smoke:
functional-tests-single:
executor: dashjs-executor
steps:
- functional_precondition
- checkout
- run:
name: Virtual merge into development branch
command: |
if [ "${CIRCLE_BRANCH}" = "development" ]; then
echo "On development branch already, no merge needed"
else
git config --global user.email "circleci@example.com"
git config --global user.name "CircleCI"
git checkout development
git merge --no-edit --no-ff $CIRCLE_BRANCH
fi
- dependencies_setup
- run_testbuild
- functional_test_setup
- functional_steps
- run_test_suite:
streamsfile: single
configfile: browserstack
configfile: lambdatest
- process_test_results

functional-tests-smoke:
executor: dashjs-executor
steps:
- functional_steps
- run_test_suite:
streamsfile: smoke
configfile: browserstack
configfile: lambdatest-smoke
- process_test_results

functional-tests-full-part-1:
@@ -171,7 +191,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: drm_emsg_eptdelta_gaps
configfile: browserstack
configfile: lambdatest-full
- process_test_results

functional-tests-full-part-2:
@@ -180,7 +200,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: multiperiod_subtitle_vendor
configfile: browserstack
configfile: lambdatest-full
- process_test_results

functional-tests-full-part-3:
@@ -189,7 +209,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: vod_live_lowlatency_mss_multiaudio
configfile: browserstack
configfile: lambdatest-full
- process_test_results

workflows:
@@ -202,7 +222,14 @@ workflows:
branches:
ignore:
- development # skiping redundant job if already on development
- functional-tests-single:
filters:
branches:
ignore: # as creds are available only for non-forked branches
- /pull\/[0-9]+/
- functional-tests-smoke:
requires:
- functional-tests-single
filters:
branches:
ignore: # as creds are available only for non-forked branches
72 changes: 30 additions & 42 deletions test/functional/config/karma.functional.conf.cjs
Original file line number Diff line number Diff line change
@@ -13,18 +13,26 @@ module.exports = function (config) {
return
}

const testConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/${configFileName}.json`, 'utf-8'));
let testConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/${configFileName}.json`, 'utf-8'));
const streamsConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/streams/${streamsFileName}.json`, 'utf-8'));
const includedTestfiles = _getIncludedTestfiles(streamsConfiguration)
const excludedTestfiles = _getExcludedTestfiles(streamsConfiguration)
const customContextFile = testConfiguration.customContextFile ? testConfiguration.customContextFile : 'test/functional/view/index.html';
const testvectors = streamsConfiguration.testvectors

if (testConfiguration && testConfiguration.type && testConfiguration.type === 'lambdatest') {
testConfiguration = _adjustConfigurationForLambdatest(testConfiguration)
}

config.set({

// base path that will be used to resolve all patterns (eg. files, exclude)
basePath: '../../../',

// web server port
hostname: testConfiguration.hostname ? testConfiguration.hostname : 'localhost',
port: testConfiguration.port ? testConfiguration.port : 9876,
protocol: testConfiguration.protocol ? testConfiguration.protocol : 'http',

// frameworks to use
// available frameworks: https://npmjs.org/browse/keyword/karma-adapter
@@ -68,8 +76,7 @@ module.exports = function (config) {
outputDir: `test/functional/results/test/karma/junit`, // results will be saved as $outputDir/$browserName.xml
outputFile: `${Date.now()}.xml`, // if included, results will be saved as $outputDir/$browserName/$outputFile
suite: '', // suite will become the package name attribute in xml testsuite element
useBrowserName: false, // add browser name to report and classes names
nameFormatter: undefined, // function (browser, result) to customize the name attribute in xml testcase element
useBrowserName: true, // add browser name to report and classes names
classNameFormatter: undefined, // function (browser, result) to customize the classname attribute in xml testcase element
properties: {}, // key value pair of properties to add to the <properties> section of the report
xmlVersion: null // use '1' if reporting to be per SonarQube 6.2 XML format
@@ -101,17 +108,13 @@ module.exports = function (config) {
webpack: {},

client: {
useIframe: true,
useIframe: false,
mocha: {
timeout: 180000
timeout: 120000
},
testvectors
},

// web server port
port: 9876,


// enable / disable colors in the output (reporters and logs)
colors: true,

@@ -124,45 +127,15 @@ module.exports = function (config) {
// enable / disable watching file and executing tests whenever any file changes
autoWatch: false,

browserNoActivityTimeout: 180000,
browserNoActivityTimeout: 120000,
browserDisconnectTimeout: 20000,
browserDisconnectTolerance: 3,
browserDisconnectTolerance: 2,

// start these browsers
// available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
browsers: testConfiguration.browsers,

customLaunchers: {
bs_chrome_win_11: {
base: 'BrowserStack',
browser: 'chrome',
'browser_version': 'latest',
os: 'Windows',
'os_version': '11',
},
bs_firefox_win_11: {
base: 'BrowserStack',
browser: 'firefox',
'browser_version': 'latest',
os: 'Windows',
'os_version': '11',
},
bs_safari_mac: {
'base': 'BrowserStack',
'browser_version': 'latest',
'os': 'OS X',
'os_version': 'Ventura',
'browser': 'safari',
},
chrome_custom: {
base: 'Chrome',
flags: ['--disable-web-security', '--autoplay-policy=no-user-gesture-required', '--disable-popup-blocking']
},
firefox_custom: {
base: 'Firefox',
prefs: {}
}
},
customLaunchers: testConfiguration.customLaunchers,

// Continuous Integration mode
// if true, Karma captures browsers, runs the tests and exits
@@ -198,3 +171,18 @@ function _getExcludedTestfiles(testConfiguration) {
return `test/functional/test/${entry}.js`
})
}

function _adjustConfigurationForLambdatest(testConfiguration) {
if (testConfiguration && testConfiguration.customLaunchers) {
Object.keys(testConfiguration.customLaunchers).forEach((key) => {
testConfiguration.customLaunchers[key].user = process.env.LAMBDATEST_USER;
testConfiguration.customLaunchers[key].accessKey = process.env.LAMBDATEST_ACCESS_KEY;
testConfiguration.customLaunchers[key].config = {
hostname: 'hub.lambdatest.com',
port: 80
};
})
}

return testConfiguration
}
148 changes: 0 additions & 148 deletions test/functional/config/karma.functional.seleniumgrid.conf.cjs

This file was deleted.

23 changes: 23 additions & 0 deletions test/functional/config/test-configurations/browserstack.json
Original file line number Diff line number Diff line change
@@ -2,6 +2,29 @@
"browsers": [
"bs_chrome_win_11"
],
"customLaunchers": {
"bs_chrome_win_11": {
"base": "BrowserStack",
"browser": "chrome",
"browser_version": "latest",
"os": "Windows",
"os_version": "11"
},
"bs_firefox_win_11": {
"base": "BrowserStack",
"browser": "firefox",
"browser_version": "latest",
"os": "Windows",
"os_version": "11"
},
"bs_safari_mac": {
"base": "BrowserStack",
"browser_version": "latest",
"os": "OS X",
"os_version": "Ventura",
"browser": "safari"
}
},
"reporters": [
"mocha",
"junit",
28 changes: 28 additions & 0 deletions test/functional/config/test-configurations/lambdatest-full.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"type": "lambdatest",
"browsers": [
"chrome"
],
"hostname": "localhost",
"port": 9876,
"protocol": "http",
"customLaunchers": {
"chrome": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "windows 11",
"version": "latest",
"name": "Chrome Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
}
},
"reporters": [
"mocha",
"junit",
"coverage"
]
}
28 changes: 28 additions & 0 deletions test/functional/config/test-configurations/lambdatest-smoke.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"type": "lambdatest",
"browsers": [
"chrome_windows_11"
],
"hostname": "localhost",
"port": 9876,
"protocol": "http",
"customLaunchers": {
"chrome_windows_11": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "windows 11",
"version": "latest",
"name": "Chrome Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 5000,
"tunnel": true
}
},
"reporters": [
"mocha",
"junit",
"coverage"
]
}
77 changes: 77 additions & 0 deletions test/functional/config/test-configurations/lambdatest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{
"type": "lambdatest",
"browsers": [
"chrome_windows_11",
"firefox_windows_11",
"edge_windows_11",
"chrome_mac_sonoma"
],
"hostname": "localhost",
"port": 9876,
"protocol": "http",
"customLaunchers": {
"chrome_windows_11": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "windows 11",
"version": "latest",
"name": "Chrome Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
},
"firefox_windows_11": {
"base": "WebDriver",
"browserName": "firefox",
"platform": "windows 11",
"version": "latest",
"name": "Firefox Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true,
"moz:firefoxOptions": {
"prefs": {
"media.gmp-manager.updateEnabled": true,
"media.eme.enabled": true
}
},
"prefs": {
"media.gmp-manager.updateEnabled": true,
"media.eme.enabled": true
}
},
"edge_windows_11": {
"base": "WebDriver",
"browserName": "edge",
"platform": "windows 11",
"version": "latest",
"name": "Edge Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
},
"chrome_mac_sonoma": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "macOS Sonoma",
"version": "latest",
"name": "Chrome MacOS sonoma",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
}
},
"reporters": [
"mocha",
"junit",
"coverage"
]
}
21 changes: 20 additions & 1 deletion test/functional/config/test-configurations/local.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,27 @@
{
"browsers": [
"chrome_custom"
"chrome_custom",
"firefox_custom"
],
"hostname": "localhost",
"port": 4200,
"protocol": "http",
"customLaunchers": {
"chrome_custom": {
"base": "Chrome",
"flags": [
"--disable-web-security",
"--autoplay-policy=no-user-gesture-required",
"--disable-popup-blocking"
]
},
"firefox_custom": {
"base": "Firefox",
"prefs": {}
}
},
"reporters": [
"junit",
"mocha",
"html",
"coverage"

0 comments on commit a7f983e

Please sign in to comment.