diff --git a/.github/workflows/peer-api.yaml b/.github/workflows/peer-api.yaml index ce642134d09..a1bf09c85ac 100644 --- a/.github/workflows/peer-api.yaml +++ b/.github/workflows/peer-api.yaml @@ -22,8 +22,8 @@ jobs: run: npm install semver - name: Check API dependency semantics (stable) - run: lerna exec --ignore propagation-validation-server "node ../../scripts/peer-api-check.js" + run: lerna exec --ignore propagation-validation-server,@opentelemetry/selenium-tests "node ../../scripts/peer-api-check.js" - name: Check API dependency semantics (experimental) working-directory: experimental - run: lerna exec --ignore propagation-validation-server "node ../../../scripts/peer-api-check.js" + run: lerna exec --ignore propagation-validation-server,@opentelemetry/selenium-tests "node ../../../scripts/peer-api-check.js" diff --git a/lerna.json b/lerna.json index b530144ffcd..2c0b8bd6648 100644 --- a/lerna.json +++ b/lerna.json @@ -5,6 +5,7 @@ "packages": [ "benchmark/*", "packages/*", - "integration-tests/*" + "integration-tests/*", + "selenium-tests" ] } diff --git a/selenium-tests/.gitignore b/selenium-tests/.gitignore new file mode 100644 index 00000000000..53b77116eaa --- /dev/null +++ b/selenium-tests/.gitignore @@ -0,0 +1,2 @@ +tests_output +tmp diff --git a/selenium-tests/README.md b/selenium-tests/README.md new file mode 100644 index 00000000000..3abbcb12333 --- /dev/null +++ b/selenium-tests/README.md @@ -0,0 +1,43 @@ +## Selenium Tests +Selenium tests are to help verify the working of opentelemetry in different browsers. For that the nightwatch is used. +This can be run either locally or in github actions with usage of browserstack. +Browser stack also gives possibility of running tests in browserstack on different browsers using our local environment. +This helps to test and debug things locally using any browser we want. + +## Running tests locally using local browser - this is also useful when adding new test +1. run server +```shell +npm run server +``` +2. Run local test for example for xhr +```shell +npm run local:xhr +``` + +Please wait a bit it should run selenium tests using our local version of chrome + +## Running tests locally using browser stack account - for that you need to have a browser stack account +If you have it please create a file ".env" based on "example.env" + +1. run server +```shell +npm run server +``` +2. Run local test for example for xhr +```shell +npm run local:bs:xhr +``` + +## Architecture +1. Folder pages contains all the pages that can be entered after running `npn run server` at http://localhost:8090/ . +These are fully functioning pages and can be run without running tests. +2. To be able to test it automatically instead of manually we have to create a test. Tests are kept in folder "tests". +For each page there is a corresponding folder with exactly the same name. There are additional 2 files + +- helper.js - this file keeps some helpers functions that are included in page automatically and they are available in tests only in section "execute". +This is because this section is being sent to the browser and executed automatically. This is the only way of "sending" and "reading" data back to test +When data is being sent between browser and selenium it needs to be stringified that's why we have one more helper for that called "JSONSafeStringify" +The last helper is the one that helps to verify test has finished successfully "OTELSeleniumDone". + +- tracing.js - this file contains the skeleton for tracing and export function "loadOtel" this is a helper function that accepts instrumentations as param. +This way it is easy to add different tests that has different lists of instrumentations. diff --git a/selenium-tests/babel.config.js b/selenium-tests/babel.config.js new file mode 100644 index 00000000000..26b15edca0b --- /dev/null +++ b/selenium-tests/babel.config.js @@ -0,0 +1,33 @@ +module.exports = function (api) { + api.cache(true); + const presets = [ + [ + '@babel/preset-env', + { + corejs: { + version: '3', + proposals: true, + }, + useBuiltIns: 'entry', + targets: { + // 'edge': 16, + // 'safari': 9, + // 'firefox': 57, + 'ie': 11, + // 'ios': 9, + // 'chrome': 49, + }, + }, + ], + ]; + const plugins = [ + ['@babel/plugin-proposal-decorators', { decoratorsBeforeExport: true }], + ['@babel/plugin-proposal-class-properties', { 'loose': true }], + ["@babel/plugin-proposal-private-methods", { "loose": true }], + ["@babel/plugin-proposal-private-property-in-object", { "loose": true }], + ]; + return { + presets, + plugins, + }; +}; \ No newline at end of file diff --git a/selenium-tests/example.env b/selenium-tests/example.env new file mode 100644 index 00000000000..28fc9739b03 --- /dev/null +++ b/selenium-tests/example.env @@ -0,0 +1,2 @@ +BROWSERSTACK_USER="YOUR USER" +BROWSERSTACK_KEY="YOUR BROWSER STACK KEY" \ No newline at end of file diff --git a/selenium-tests/nightwatch.conf.js b/selenium-tests/nightwatch.conf.js new file mode 100644 index 00000000000..8547bc24223 --- /dev/null +++ b/selenium-tests/nightwatch.conf.js @@ -0,0 +1,287 @@ +// Autogenerated by Nightwatch +// Refer to the online docs for more details: https://nightwatchjs.org/gettingstarted/configuration/ +const Services = {}; loadServices(); + +module.exports = { + // An array of folders (excluding subfolders) where your tests are located; + // if this is not specified, the test source must be passed as the second argument to the test runner. + src_folders: [], + + // See https://nightwatchjs.org/guide/working-with-page-objects/ + page_objects_path: '', + + // See https://nightwatchjs.org/guide/extending-nightwatch/#writing-custom-commands + custom_commands_path: '', + + // See https://nightwatchjs.org/guide/extending-nightwatch/#writing-custom-assertions + custom_assertions_path: '', + + // See https://nightwatchjs.org/guide/#external-globals + globals_path : '', + + webdriver: {}, + + test_settings: { + default: { + disable_error_log: false, + launch_url: 'https://nightwatchjs.org', + + screenshots: { + enabled: false, + path: 'screens', + on_failure: true + }, + + desiredCapabilities: { + browserName : 'firefox' + }, + + webdriver: { + start_process: true, + server_path: (Services.geckodriver ? Services.geckodriver.path : '') + } + }, + + safari: { + desiredCapabilities : { + browserName : 'safari', + alwaysMatch: { + acceptInsecureCerts: false + } + }, + webdriver: { + port: 4445, + start_process: true, + server_path: '/usr/bin/safaridriver' + } + }, + + firefox: { + desiredCapabilities : { + browserName : 'firefox', + alwaysMatch: { + acceptInsecureCerts: true, + 'moz:firefoxOptions': { + args: [ + // '-headless', + // '-verbose' + ] + } + } + + }, + webdriver: { + start_process: true, + port: 4444, + server_path: (Services.geckodriver ? Services.geckodriver.path : ''), + cli_args: [ + // very verbose geckodriver logs + // '-vv' + ] + } + }, + + chrome: { + desiredCapabilities : { + browserName : 'chrome', + 'goog:chromeOptions' : { + // More info on Chromedriver: https://sites.google.com/a/chromium.org/chromedriver/ + // + // This tells Chromedriver to run using the legacy JSONWire protocol (not required in Chrome 78) + w3c: false, + args: [ + //'--no-sandbox', + //'--ignore-certificate-errors', + //'--allow-insecure-localhost', + //'--headless' + ] + } + }, + + webdriver: { + start_process: true, + port: 9515, + server_path: (Services.chromedriver ? Services.chromedriver.path : ''), + cli_args: [ + // --verbose + ] + } + }, + + edge: { + desiredCapabilities : { + browserName : 'MicrosoftEdge', + 'ms:edgeOptions' : { + w3c: false, + // More info on EdgeDriver: https://docs.microsoft.com/en-us/microsoft-edge/webdriver-chromium/capabilities-edge-options + args: [ + //'--headless' + ] + } + }, + + webdriver: { + start_process: true, + // Download msedgedriver from https://docs.microsoft.com/en-us/microsoft-edge/webdriver-chromium/ + // and set the location below: + server_path: '', + cli_args: [ + // --verbose + ] + } + }, + + ////////////////////////////////////////////////////////////////////////////////// + // Configuration for when using the browserstack.com cloud service | + // | + // Please set the username and access key by setting the environment variables: | + // - BROWSERSTACK_USER | + // - BROWSERSTACK_KEY | + // .env files are supported | + ////////////////////////////////////////////////////////////////////////////////// + browserstack: { + selenium: { + host: 'hub-cloud.browserstack.com', + port: 443 + }, + // More info on configuring capabilities can be found on: + // https://www.browserstack.com/automate/capabilities?tag=selenium-4 + desiredCapabilities: { + 'bstack:options' : { + userName: '${BROWSERSTACK_USER}', + accessKey: '${BROWSERSTACK_KEY}', + } + }, + + disable_error_log: true, + webdriver: { + timeout_options: { + timeout: 15000, + retry_attempts: 3 + }, + keep_alive: true, + start_process: false + } + }, + + 'browserstack.local': { + extends: 'browserstack', + desiredCapabilities: { + 'browserstack.local': true, + 'browserstack.console': 'errors' + } + }, + + 'browserstack.chrome': { + extends: 'browserstack', + desiredCapabilities: { + browserName: 'chrome', + chromeOptions : { + w3c: false + } + } + }, + + 'browserstack.firefox': { + extends: 'browserstack', + desiredCapabilities: { + browserName: 'firefox' + } + }, + + 'browserstack.ie': { + extends: 'browserstack', + desiredCapabilities: { + browserName: 'internet explorer', + browserVersion: '11.0' + } + }, + + 'browserstack.safari': { + extends: 'browserstack', + desiredCapabilities: { + browserName: 'safari' + } + }, + + 'browserstack.local_chrome': { + extends: 'browserstack.local', + desiredCapabilities: { + browserName: 'chrome' + } + }, + + 'browserstack.local_firefox': { + extends: 'browserstack.local', + desiredCapabilities: { + browserName: 'firefox' + } + }, + 'browserstack.local_ie': { + extends: 'browserstack.local', + desiredCapabilities: { + browserName: 'internet explorer', + browserVersion: '11.0' + } + }, + 'browserstack.local_safari': { + extends: 'browserstack.local', + desiredCapabilities: { + browserName: 'safari', + } + }, + ////////////////////////////////////////////////////////////////////////////////// + // Configuration for when using the Selenium service, either locally or remote, | + // like Selenium Grid | + ////////////////////////////////////////////////////////////////////////////////// + selenium_server: { + // Selenium Server is running locally and is managed by Nightwatch + selenium: { + start_process: true, + port: 4444, + server_path: (Services.seleniumServer ? Services.seleniumServer.path : ''), + cli_args: { + 'webdriver.gecko.driver': (Services.geckodriver ? Services.geckodriver.path : ''), + 'webdriver.chrome.driver': (Services.chromedriver ? Services.chromedriver.path : '') + } + } + }, + + 'selenium.chrome': { + extends: 'selenium_server', + desiredCapabilities: { + browserName: 'chrome', + chromeOptions : { + w3c: false, + } + } + }, + + 'selenium.firefox': { + extends: 'selenium_server', + desiredCapabilities: { + browserName: 'firefox', + 'moz:firefoxOptions': { + args: [ + // '-headless', + // '-verbose' + ] + } + } + } + } +}; + +function loadServices() { + try { + Services.seleniumServer = require('selenium-server'); + } catch (err) {} + + try { + Services.chromedriver = require('chromedriver'); + } catch (err) {} + + try { + Services.geckodriver = require('geckodriver'); + } catch (err) {} +} diff --git a/selenium-tests/package.json b/selenium-tests/package.json new file mode 100644 index 00000000000..fd78e471051 --- /dev/null +++ b/selenium-tests/package.json @@ -0,0 +1,70 @@ +{ + "name": "@opentelemetry/selenium-tests", + "version": "0.0.1", + "description": "OpenTelemetry Selenium Tests", + "main": "index.js", + "repository": "open-telemetry/opentelemetry-js", + "scripts": { + "all:bs": "nightwatch ./tests --parallel --env browserstack.chrome,browserstack.safari,browserstack.firefox", + "all:local": "nightwatch ./tests --parallel --env selenium.chrome", + "local:bs:fetch": "node scripts/local.runner.js --test ./tests/fetch/fetch.js --parallel --env browserstack.local_chrome,browserstack.local_firefox,browserstack.local_safari", + "local:bs:xhr": "node scripts/local.runner.js --test ./tests/xhr/xhr.js --parallel --env browserstack.local_chrome,browserstack.local_firefox,browserstack.local_ie,browserstack.local_safari", + "local:fetch": "nightwatch ./tests/fetch/fetch.js --env selenium.chrome", + "local:xhr": "nightwatch ./tests/xhr/xhr.js --env selenium.chrome", + "server": "webpack serve --progress --port 8090 --config webpack.dev.js --hot --host 0.0.0.0" + }, + "keywords": [ + "opentelemetry", + "web", + "tracing", + "profiling", + "metrics", + "stats" + ], + "author": "OpenTelemetry Authors", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0.0" + }, + "publishConfig": { + "access": "private" + }, + "devDependencies": { + "@babel/core": "^7.15.8", + "@babel/plugin-proposal-class-properties": "^7.14.5", + "@babel/plugin-proposal-decorators": "^7.15.8", + "@babel/plugin-transform-runtime": "^7.15.8", + "@babel/preset-env": "^7.15.0", + "@opentelemetry/api": "^1.0.3", + "babel-loader": "^8.2.3", + "babel-polyfill": "^6.26.0", + "browserstack-local": "^1.4.8", + "chromedriver": "^95.0.0", + "dotenv": "^10.0.0", + "fast-safe-stringify": "^2.1.1", + "geckodriver": "^2.0.4", + "nightwatch": "^1.7.11", + "selenium-server": "^3.141.59", + "terser-webpack-plugin": "^5.2.4", + "webpack": "^5.60.0", + "webpack-cli": "^4.9.1", + "webpack-dev-server": "^4.3.1", + "webpack-merge": "^5.8.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.3" + }, + "dependencies": { + "@opentelemetry/context-zone-peer-dep": "^1.0.0", + "@opentelemetry/core": "^1.0.0", + "@opentelemetry/exporter-otlp-http": "^0.26.0", + "@opentelemetry/exporter-zipkin": "^1.0.0", + "@opentelemetry/instrumentation": "^0.26.0", + "@opentelemetry/instrumentation-fetch": "^0.26.0", + "@opentelemetry/instrumentation-xml-http-request": "^0.26.0", + "@opentelemetry/sdk-metrics-base": "^0.26.0", + "@opentelemetry/sdk-trace-base": "^1.0.0", + "@opentelemetry/sdk-trace-web": "^1.0.0", + "zone.js": "^0.10.3" + } +} diff --git a/selenium-tests/pages/fetch/index.html b/selenium-tests/pages/fetch/index.html new file mode 100644 index 00000000000..a43e52d2ef1 --- /dev/null +++ b/selenium-tests/pages/fetch/index.html @@ -0,0 +1,20 @@ + + + +
+ +