chore: adding selenium tests with browserstack

.github/workflows/peer-api.yaml

@@ -22,8 +22,8 @@ jobs:
         run: npm install semver
 
       - name: Check API dependency semantics (stable)
-        run: lerna exec --ignore propagation-validation-server "node ../../scripts/peer-api-check.js"
+        run: lerna exec --ignore propagation-validation-server,@opentelemetry/selenium-tests "node ../../scripts/peer-api-check.js"
 
       - name: Check API dependency semantics (experimental)
         working-directory: experimental
-        run: lerna exec --ignore propagation-validation-server "node ../../../scripts/peer-api-check.js"
+        run: lerna exec --ignore propagation-validation-server,@opentelemetry/selenium-tests "node ../../../scripts/peer-api-check.js"

lerna.json

@@ -5,6 +5,7 @@
   "packages": [
     "benchmark/*",
     "packages/*",
-    "integration-tests/*"
+    "integration-tests/*",
+    "selenium-tests"
   ]
 }

selenium-tests/.gitignore

@@ -0,0 +1,2 @@
+tests_output
+tmp

selenium-tests/README.md

@@ -0,0 +1,43 @@
+## Selenium Tests
+Selenium tests are to help verify the working of opentelemetry in different browsers. For that the nightwatch is used.
+This can be run either locally or in github actions with usage of browserstack.
+Browser stack also gives possibility of running tests in browserstack on different browsers using our local environment.
+This helps to test and debug things locally using any browser we want.  
+
+## Running tests locally using local browser - this is also useful when adding new test
+1. run server
+```shell
+npm run server
+```
+2. Run local test for example for xhr
+```shell
+npm run local:xhr
+```
+
+Please wait a bit it should run selenium tests using our local version of chrome
+
+## Running tests locally using browser stack account - for that you need to have a browser stack account
+If you have it please create a file ".env" based on "example.env"
+
+1. run server
+```shell
+npm run server
+```
+2. Run local test for example for xhr
+```shell
+npm run local:bs:xhr
+```
+
+## Architecture
+1. Folder pages contains all the pages that can be entered after running `npn run server` at http://localhost:8090/ . 
+These are fully functioning pages and can be run without running tests.
+2. To be able to test it automatically instead of manually we have to create a test. Tests are kept in folder "tests". 
+For each page there is a corresponding folder with exactly the same name. There are additional 2 files
+
+- helper.js - this file keeps some helpers functions that are included in page automatically and they are available in tests only in section "execute".
+This is because this section is being sent to the browser and executed automatically. This is the only way of "sending" and "reading" data back to test 
+When data is being sent between browser and selenium it needs to be stringified that's why we have one more helper for that called "JSONSafeStringify"
+The last helper is the one that helps to verify test has finished successfully "OTELSeleniumDone".
+
+- tracing.js - this file contains the skeleton for tracing and export function "loadOtel" this is a helper function that accepts instrumentations as param.
+This way it is easy to add different tests that has different lists of instrumentations.

selenium-tests/babel.config.js

@@ -0,0 +1,33 @@
+module.exports = function (api) {
+  api.cache(true);
+  const presets = [
+    [
+      '@babel/preset-env',
+      {
+        corejs: {
+          version: '3',
+          proposals: true,
+        },
+        useBuiltIns: 'entry',
+        targets: {
+            // 'edge': 16,
+            // 'safari': 9,
+            // 'firefox': 57,
+            'ie': 11,
+            // 'ios': 9,
+            // 'chrome': 49,
+        },
+      },
+    ],
+  ];
+  const plugins = [
+    ['@babel/plugin-proposal-decorators', { decoratorsBeforeExport: true }],
+    ['@babel/plugin-proposal-class-properties', { 'loose': true }],
+    ["@babel/plugin-proposal-private-methods", { "loose": true }],
+    ["@babel/plugin-proposal-private-property-in-object", { "loose": true }],
+  ];
+  return {
+    presets,
+    plugins,
+  };
+};

selenium-tests/example.env

@@ -0,0 +1,2 @@
+BROWSERSTACK_USER="YOUR USER"
+BROWSERSTACK_KEY="YOUR BROWSER STACK KEY"

selenium-tests/nightwatch.conf.js

@@ -0,0 +1,287 @@
+// Autogenerated by Nightwatch
+// Refer to the online docs for more details: https://nightwatchjs.org/gettingstarted/configuration/
+const Services = {}; loadServices();
+
+module.exports = {
+  // An array of folders (excluding subfolders) where your tests are located;
+  // if this is not specified, the test source must be passed as the second argument to the test runner.
+  src_folders: [],
+
+  // See https://nightwatchjs.org/guide/working-with-page-objects/
+  page_objects_path: '',
+
+  // See https://nightwatchjs.org/guide/extending-nightwatch/#writing-custom-commands
+  custom_commands_path:  '',
+
+  // See https://nightwatchjs.org/guide/extending-nightwatch/#writing-custom-assertions
+  custom_assertions_path: '',
+
+  // See https://nightwatchjs.org/guide/#external-globals
+  globals_path : '',
+
+  webdriver: {},
+
+  test_settings: {
+    default: {
+      disable_error_log: false,
+      launch_url: 'https://nightwatchjs.org',
+
+      screenshots: {
+        enabled: false,
+        path: 'screens',
+        on_failure: true
+      },
+
+      desiredCapabilities: {
+        browserName : 'firefox'
+      },
+
+      webdriver: {
+        start_process: true,
+        server_path: (Services.geckodriver ? Services.geckodriver.path : '')
+      }
+    },
+
+    safari: {
+      desiredCapabilities : {
+        browserName : 'safari',
+        alwaysMatch: {
+          acceptInsecureCerts: false
+        }
+      },
+      webdriver: {
+        port: 4445,
+        start_process: true,
+        server_path: '/usr/bin/safaridriver'
+      }
+    },
+
+    firefox: {
+      desiredCapabilities : {
+        browserName : 'firefox',
+        alwaysMatch: {
+          acceptInsecureCerts: true,
+          'moz:firefoxOptions': {
+            args: [
+              // '-headless',
+              // '-verbose'
+            ]
+          }
+        }
+
+      },
+      webdriver: {
+        start_process: true,
+        port: 4444,
+        server_path: (Services.geckodriver ? Services.geckodriver.path : ''),
+        cli_args: [
+          // very verbose geckodriver logs
+          // '-vv'
+        ]
+      }
+    },
+
+    chrome: {
+      desiredCapabilities : {
+        browserName : 'chrome',
+        'goog:chromeOptions' : {
+          // More info on Chromedriver: https://sites.google.com/a/chromium.org/chromedriver/
+          //
+          // This tells Chromedriver to run using the legacy JSONWire protocol (not required in Chrome 78)
+          w3c: false,
+          args: [
+            //'--no-sandbox',
+            //'--ignore-certificate-errors',
+            //'--allow-insecure-localhost',
+            //'--headless'
+          ]
+        }
+      },
+
+      webdriver: {
+        start_process: true,
+        port: 9515,
+        server_path: (Services.chromedriver ? Services.chromedriver.path : ''),
+        cli_args: [
+          // --verbose
+        ]
+      }
+    },
+
+    edge: {
+      desiredCapabilities : {
+        browserName : 'MicrosoftEdge',
+        'ms:edgeOptions' : {
+          w3c: false,
+          // More info on EdgeDriver: https://docs.microsoft.com/en-us/microsoft-edge/webdriver-chromium/capabilities-edge-options
+          args: [
+            //'--headless'
+          ]
+        }
+      },
+
+      webdriver: {
+        start_process: true,
+        // Download msedgedriver from https://docs.microsoft.com/en-us/microsoft-edge/webdriver-chromium/
+        //  and set the location below:
+        server_path: '',
+        cli_args: [
+          // --verbose
+        ]
+      }
+    },
+
+    //////////////////////////////////////////////////////////////////////////////////
+    // Configuration for when using the browserstack.com cloud service               |
+    //                                                                               |
+    // Please set the username and access key by setting the environment variables:  |
+    // - BROWSERSTACK_USER                                                           |
+    // - BROWSERSTACK_KEY                                                            |
+    // .env files are supported                                                      |
+    //////////////////////////////////////////////////////////////////////////////////
+    browserstack: {
+      selenium: {
+        host: 'hub-cloud.browserstack.com',
+        port: 443
+      },
+      // More info on configuring capabilities can be found on:
+      // https://www.browserstack.com/automate/capabilities?tag=selenium-4
+      desiredCapabilities: {
+        'bstack:options' : {
+          userName: '${BROWSERSTACK_USER}',
+          accessKey: '${BROWSERSTACK_KEY}',
+        }
+      },
+
+      disable_error_log: true,
+      webdriver: {
+        timeout_options: {
+          timeout: 15000,
+          retry_attempts: 3
+        },
+        keep_alive: true,
+        start_process: false
+      }
+    },
+
+    'browserstack.local': {
+      extends: 'browserstack',
+      desiredCapabilities: {
+        'browserstack.local': true,
+        'browserstack.console': 'errors'
+      }
+    },
+
+    'browserstack.chrome': {
+      extends: 'browserstack',
+      desiredCapabilities: {
+        browserName: 'chrome',
+        chromeOptions : {
+          w3c: false
+        }
+      }
+    },
+
+    'browserstack.firefox': {
+      extends: 'browserstack',
+      desiredCapabilities: {
+        browserName: 'firefox'
+      }
+    },
+
+    'browserstack.ie': {
+      extends: 'browserstack',
+      desiredCapabilities: {
+        browserName: 'internet explorer',
+        browserVersion: '11.0'
+      }
+    },
+
+    'browserstack.safari': {
+      extends: 'browserstack',
+      desiredCapabilities: {
+        browserName: 'safari'
+      }
+    },
+
+    'browserstack.local_chrome': {
+      extends: 'browserstack.local',
+      desiredCapabilities: {
+        browserName: 'chrome'
+      }
+    },
+
+    'browserstack.local_firefox': {
+      extends: 'browserstack.local',
+      desiredCapabilities: {
+        browserName: 'firefox'
+      }
+    },
+    'browserstack.local_ie': {
+      extends: 'browserstack.local',
+      desiredCapabilities: {
+        browserName: 'internet explorer',
+        browserVersion: '11.0'
+      }
+    },
+    'browserstack.local_safari': {
+      extends: 'browserstack.local',
+      desiredCapabilities: {
+        browserName: 'safari',
+      }
+    },
+    //////////////////////////////////////////////////////////////////////////////////
+    // Configuration for when using the Selenium service, either locally or remote,  |
+    //  like Selenium Grid                                                           |
+    //////////////////////////////////////////////////////////////////////////////////
+    selenium_server: {
+      // Selenium Server is running locally and is managed by Nightwatch
+      selenium: {
+        start_process: true,
+        port: 4444,
+        server_path: (Services.seleniumServer ? Services.seleniumServer.path : ''),
+        cli_args: {
+          'webdriver.gecko.driver': (Services.geckodriver ? Services.geckodriver.path : ''),
+          'webdriver.chrome.driver': (Services.chromedriver ? Services.chromedriver.path : '')
+        }
+      }
+    },
+
+    'selenium.chrome': {
+      extends: 'selenium_server',
+      desiredCapabilities: {
+        browserName: 'chrome',
+        chromeOptions : {
+          w3c: false,
+        }
+      }
+    },
+
+    'selenium.firefox': {
+      extends: 'selenium_server',
+      desiredCapabilities: {
+        browserName: 'firefox',
+        'moz:firefoxOptions': {
+          args: [
+            // '-headless',
+            // '-verbose'
+          ]
+        }
+      }
+    }
+  }
+};
+
+function loadServices() {
+  try {
+    Services.seleniumServer = require('selenium-server');
+  } catch (err) {}
+
+  try {
+    Services.chromedriver = require('chromedriver');
+  } catch (err) {}
+
+  try {
+    Services.geckodriver = require('geckodriver');
+  } catch (err) {}
+}