Skip to content

Commit

Permalink
Functional Test execution on Lamdatest (#4472)
Browse files Browse the repository at this point in the history
dsilhavy authored May 6, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 8239d6a commit a7f983e
Showing 8 changed files with 266 additions and 224 deletions.
93 changes: 60 additions & 33 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -29,23 +29,36 @@ commands:
- run:
name: Check if preconditions are met for functional tests
command: |
if [ -z "$BROWSERSTACK_ACCESS_KEY" ]; then
echo "BrowserStack not configured, functional tests will not be executed."
if [ -z "LAMBDATEST_ACCESS_KEY" ]; then
echo "Lambdatest not configured, functional tests will not be executed."
circleci-agent step halt
fi
functional_test_setup:
steps:
- run:
name: Download the browserstack binary file to create a tunnel
command: wget "https://www.browserstack.com/browserstack-local/BrowserStackLocal-linux-x64.zip"
name: "Downloading tunnel binary"
command: |
wget https://downloads.lambdatest.com/tunnel/v3/linux/64bit/LT_Linux.zip
- run:
name: Unzip the browserstack binary file
command: unzip BrowserStackLocal-linux-x64.zip
name: "Extracting tunnel binary"
command: |
sudo apt-get install unzip
unzip LT_Linux.zip
- run:
name: Run browserstack with provided access key
command: ./BrowserStackLocal $BROWSERSTACK_ACCESS_KEY
name: "Executing tunnel binary"
background: true
command: |
./LT --user ${LAMBDATEST_EMAIL} --key ${LAMBDATEST_KEY}
sleep 40
- run:
name: "Setup custom environment variables"
command: |
echo 'export LT_USERNAME="${LAMBDATEST_USER}"' >> $BASH_ENV
echo 'export LT_ACCESS_KEY="${LAMBDATEST_ACCESS_KEY}"' >> $BASH_ENV
run_testbuild:
steps:
@@ -67,10 +80,11 @@ commands:
command: |
mkdir test/functional/results/test/karma/junit/adjusted
cd test/functional/results/test/karma/junit
for file in *.xml; do
for file in **/*.xml; do
[ -f "$file" ] || continue
xmlstarlet ed -L -u '//testcase[@time="NaN"]/@time' -v '0' "$file"
cp "$file" adjusted/$file
modified_file="${file//\//_}"
cp "$file" adjusted/$modified_file
done
- store_test_results:
path: test/functional/results/test/karma/junit/adjusted
@@ -79,6 +93,17 @@ commands:
steps:
- functional_precondition
- checkout
- run:
name: Virtual merge into development branch
command: |
if [ "${CIRCLE_BRANCH}" = "development" ]; then
echo "On development branch already, no merge needed"
else
git config --global user.email "circleci@example.com"
git config --global user.name "CircleCI"
git checkout development
git merge --no-edit --no-ff $CIRCLE_BRANCH
fi
- dependencies_setup
- run_testbuild
- functional_test_setup
@@ -89,9 +114,13 @@ commands:
default: "single"
type: string
configfile:
default: "browserstack"
default: "lambdatest"
type: string
steps:
- run:
name: Sleep
command: |
sleep 30
- run:
name: Run functional tests <<parameters.configfile>> - <<parameters.streamsfile>>
when: always
@@ -138,31 +167,22 @@ jobs:
- dependencies_setup
- run_testbuild

functional-tests-smoke:
functional-tests-single:
executor: dashjs-executor
steps:
- functional_precondition
- checkout
- run:
name: Virtual merge into development branch
command: |
if [ "${CIRCLE_BRANCH}" = "development" ]; then
echo "On development branch already, no merge needed"
else
git config --global user.email "circleci@example.com"
git config --global user.name "CircleCI"
git checkout development
git merge --no-edit --no-ff $CIRCLE_BRANCH
fi
- dependencies_setup
- run_testbuild
- functional_test_setup
- functional_steps
- run_test_suite:
streamsfile: single
configfile: browserstack
configfile: lambdatest
- process_test_results

functional-tests-smoke:
executor: dashjs-executor
steps:
- functional_steps
- run_test_suite:
streamsfile: smoke
configfile: browserstack
configfile: lambdatest-smoke
- process_test_results

functional-tests-full-part-1:
@@ -171,7 +191,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: drm_emsg_eptdelta_gaps
configfile: browserstack
configfile: lambdatest-full
- process_test_results

functional-tests-full-part-2:
@@ -180,7 +200,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: multiperiod_subtitle_vendor
configfile: browserstack
configfile: lambdatest-full
- process_test_results

functional-tests-full-part-3:
@@ -189,7 +209,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: vod_live_lowlatency_mss_multiaudio
configfile: browserstack
configfile: lambdatest-full
- process_test_results

workflows:
@@ -202,7 +222,14 @@ workflows:
branches:
ignore:
- development # skiping redundant job if already on development
- functional-tests-single:
filters:
branches:
ignore: # as creds are available only for non-forked branches
- /pull\/[0-9]+/
- functional-tests-smoke:
requires:
- functional-tests-single
filters:
branches:
ignore: # as creds are available only for non-forked branches
72 changes: 30 additions & 42 deletions test/functional/config/karma.functional.conf.cjs
Original file line number Diff line number Diff line change
@@ -13,18 +13,26 @@ module.exports = function (config) {
return
}

const testConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/${configFileName}.json`, 'utf-8'));
let testConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/${configFileName}.json`, 'utf-8'));
const streamsConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/streams/${streamsFileName}.json`, 'utf-8'));
const includedTestfiles = _getIncludedTestfiles(streamsConfiguration)
const excludedTestfiles = _getExcludedTestfiles(streamsConfiguration)
const customContextFile = testConfiguration.customContextFile ? testConfiguration.customContextFile : 'test/functional/view/index.html';
const testvectors = streamsConfiguration.testvectors

if (testConfiguration && testConfiguration.type && testConfiguration.type === 'lambdatest') {
testConfiguration = _adjustConfigurationForLambdatest(testConfiguration)
}

config.set({

// base path that will be used to resolve all patterns (eg. files, exclude)
basePath: '../../../',

// web server port
hostname: testConfiguration.hostname ? testConfiguration.hostname : 'localhost',
port: testConfiguration.port ? testConfiguration.port : 9876,
protocol: testConfiguration.protocol ? testConfiguration.protocol : 'http',

// frameworks to use
// available frameworks: https://npmjs.org/browse/keyword/karma-adapter
@@ -68,8 +76,7 @@ module.exports = function (config) {
outputDir: `test/functional/results/test/karma/junit`, // results will be saved as $outputDir/$browserName.xml
outputFile: `${Date.now()}.xml`, // if included, results will be saved as $outputDir/$browserName/$outputFile
suite: '', // suite will become the package name attribute in xml testsuite element
useBrowserName: false, // add browser name to report and classes names
nameFormatter: undefined, // function (browser, result) to customize the name attribute in xml testcase element
useBrowserName: true, // add browser name to report and classes names
classNameFormatter: undefined, // function (browser, result) to customize the classname attribute in xml testcase element
properties: {}, // key value pair of properties to add to the <properties> section of the report
xmlVersion: null // use '1' if reporting to be per SonarQube 6.2 XML format
@@ -101,17 +108,13 @@ module.exports = function (config) {
webpack: {},

client: {
useIframe: true,
useIframe: false,
mocha: {
timeout: 180000
timeout: 120000
},
testvectors
},

// web server port
port: 9876,


// enable / disable colors in the output (reporters and logs)
colors: true,

@@ -124,45 +127,15 @@ module.exports = function (config) {
// enable / disable watching file and executing tests whenever any file changes
autoWatch: false,

browserNoActivityTimeout: 180000,
browserNoActivityTimeout: 120000,
browserDisconnectTimeout: 20000,
browserDisconnectTolerance: 3,
browserDisconnectTolerance: 2,

// start these browsers
// available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
browsers: testConfiguration.browsers,

customLaunchers: {
bs_chrome_win_11: {
base: 'BrowserStack',
browser: 'chrome',
'browser_version': 'latest',
os: 'Windows',
'os_version': '11',
},
bs_firefox_win_11: {
base: 'BrowserStack',
browser: 'firefox',
'browser_version': 'latest',
os: 'Windows',
'os_version': '11',
},
bs_safari_mac: {
'base': 'BrowserStack',
'browser_version': 'latest',
'os': 'OS X',
'os_version': 'Ventura',
'browser': 'safari',
},
chrome_custom: {
base: 'Chrome',
flags: ['--disable-web-security', '--autoplay-policy=no-user-gesture-required', '--disable-popup-blocking']
},
firefox_custom: {
base: 'Firefox',
prefs: {}
}
},
customLaunchers: testConfiguration.customLaunchers,

// Continuous Integration mode
// if true, Karma captures browsers, runs the tests and exits
@@ -198,3 +171,18 @@ function _getExcludedTestfiles(testConfiguration) {
return `test/functional/test/${entry}.js`
})
}

function _adjustConfigurationForLambdatest(testConfiguration) {
if (testConfiguration && testConfiguration.customLaunchers) {
Object.keys(testConfiguration.customLaunchers).forEach((key) => {
testConfiguration.customLaunchers[key].user = process.env.LAMBDATEST_USER;
testConfiguration.customLaunchers[key].accessKey = process.env.LAMBDATEST_ACCESS_KEY;
testConfiguration.customLaunchers[key].config = {
hostname: 'hub.lambdatest.com',
port: 80
};
})
}

return testConfiguration
}
148 changes: 0 additions & 148 deletions test/functional/config/karma.functional.seleniumgrid.conf.cjs

This file was deleted.

23 changes: 23 additions & 0 deletions test/functional/config/test-configurations/browserstack.json
Original file line number Diff line number Diff line change
@@ -2,6 +2,29 @@
"browsers": [
"bs_chrome_win_11"
],
"customLaunchers": {
"bs_chrome_win_11": {
"base": "BrowserStack",
"browser": "chrome",
"browser_version": "latest",
"os": "Windows",
"os_version": "11"
},
"bs_firefox_win_11": {
"base": "BrowserStack",
"browser": "firefox",
"browser_version": "latest",
"os": "Windows",
"os_version": "11"
},
"bs_safari_mac": {
"base": "BrowserStack",
"browser_version": "latest",
"os": "OS X",
"os_version": "Ventura",
"browser": "safari"
}
},
"reporters": [
"mocha",
"junit",
28 changes: 28 additions & 0 deletions test/functional/config/test-configurations/lambdatest-full.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"type": "lambdatest",
"browsers": [
"chrome"
],
"hostname": "localhost",
"port": 9876,
"protocol": "http",
"customLaunchers": {
"chrome": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "windows 11",
"version": "latest",
"name": "Chrome Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
}
},
"reporters": [
"mocha",
"junit",
"coverage"
]
}
28 changes: 28 additions & 0 deletions test/functional/config/test-configurations/lambdatest-smoke.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"type": "lambdatest",
"browsers": [
"chrome_windows_11"
],
"hostname": "localhost",
"port": 9876,
"protocol": "http",
"customLaunchers": {
"chrome_windows_11": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "windows 11",
"version": "latest",
"name": "Chrome Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 5000,
"tunnel": true
}
},
"reporters": [
"mocha",
"junit",
"coverage"
]
}
77 changes: 77 additions & 0 deletions test/functional/config/test-configurations/lambdatest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{
"type": "lambdatest",
"browsers": [
"chrome_windows_11",
"firefox_windows_11",
"edge_windows_11",
"chrome_mac_sonoma"
],
"hostname": "localhost",
"port": 9876,
"protocol": "http",
"customLaunchers": {
"chrome_windows_11": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "windows 11",
"version": "latest",
"name": "Chrome Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
},
"firefox_windows_11": {
"base": "WebDriver",
"browserName": "firefox",
"platform": "windows 11",
"version": "latest",
"name": "Firefox Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true,
"moz:firefoxOptions": {
"prefs": {
"media.gmp-manager.updateEnabled": true,
"media.eme.enabled": true
}
},
"prefs": {
"media.gmp-manager.updateEnabled": true,
"media.eme.enabled": true
}
},
"edge_windows_11": {
"base": "WebDriver",
"browserName": "edge",
"platform": "windows 11",
"version": "latest",
"name": "Edge Win 11",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
},
"chrome_mac_sonoma": {
"base": "WebDriver",
"browserName": "chrome",
"platform": "macOS Sonoma",
"version": "latest",
"name": "Chrome MacOS sonoma",
"console": true,
"terminal": true,
"network": true,
"pseudoActivityInterval": 30000,
"tunnel": true
}
},
"reporters": [
"mocha",
"junit",
"coverage"
]
}
21 changes: 20 additions & 1 deletion test/functional/config/test-configurations/local.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,27 @@
{
"browsers": [
"chrome_custom"
"chrome_custom",
"firefox_custom"
],
"hostname": "localhost",
"port": 4200,
"protocol": "http",
"customLaunchers": {
"chrome_custom": {
"base": "Chrome",
"flags": [
"--disable-web-security",
"--autoplay-policy=no-user-gesture-required",
"--disable-popup-blocking"
]
},
"firefox_custom": {
"base": "Firefox",
"prefs": {}
}
},
"reporters": [
"junit",
"mocha",
"html",
"coverage"

0 comments on commit a7f983e

Please sign in to comment.