[AXON-37] Implement headless e2e testing

.gitignore

@@ -16,6 +16,6 @@ TEMP-*
 yalc.lock
 .env
 coverage
-.test-extensions/
 .generated/
 e2e/.resources/
+e2e/.test-extensions/

.vscodeignore

@@ -6,6 +6,7 @@
 
 .generated/
 e2e/
+.npm/
 
 .env
 .env.example
@@ -26,7 +27,6 @@ ui-test-settings.json
 vscode-stylenames.txt
 .devcontainer/
 .github/
-.test-extensions/
 coverage/
 scripts/
 

e2e/Dockerfile

@@ -0,0 +1,41 @@
+FROM node:20
+
+# Install dependencies and xvfb
+RUN apt-get update && apt-get install -y wget gpg xvfb xauth --no-install-recommends
+
+# Download vscode .deb package and install it (from https://go.microsoft.com/fwlink/?LinkID=760868)
+# Otherwise vscode fails to run, probably due to missing dependencies
+RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg \
+    && install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg \
+    && echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" | tee /etc/apt/sources.list.d/vscode.list > /dev/null \
+    && rm -f packages.microsoft.gpg \
+    && apt install -y apt-transport-https \
+    && apt update \
+    && apt install -y code \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install -g npm@latest
+
+# Running vscode as root is not recommended, so let's create a user.
+# We need that user's $HOME to exist for the test process to work properly.
+RUN useradd -m -s /bin/bash atlascode
+
+# Set the working directory
+# We will mount the whole project directory to WORKDIR
+# so it's best to match the WORKDIR folder name with the project directory name,
+# otherwise tsc might generate wrong paths for e2e tests
+WORKDIR /atlascode
+
+# Where our vscode, chromedriver, images etc. will go
+ENV TEST_RESOURCES=/atlascode/e2e/.resources
+
+# We want to run `xvfb-run` with the parameters passed to `docker run`
+# However, using `xvfb-run` as an entrypoint directly causes docker to hang (?!)
+# So, let's create an entrypoint to pass parameters to xvfb-run.
+# We'll do this in-place to not have any build context for this Dockerfile:
+RUN cat <<EOF > /usr/bin/entrypoint.sh && chmod +x /usr/bin/entrypoint.sh
+#!/usr/bin/env bash
+echo "Invoking xvfb-run with the following arguments: \$@"
+xvfb-run \$@
+EOF
+
+ENTRYPOINT ["/usr/bin/entrypoint.sh"]

e2e/README.md

@@ -27,3 +27,16 @@ Commands related to e2e tests:
     # Run the tests without rebuilding the extension
     # Use this one when iterating
     npm run e2e:rerun
+
+### Running E2E Tests Headless
+
+I'm sure we all love looking at the VSCode UI - but sometimes one might want to take a break from that, and run their tests in console only 😉 Unfortunately, [vscode-extension-tester](https://github.com/redhat-developer/vscode-extension-tester) doesn't have built-in support for [headless](https://en.wikipedia.org/wiki/Headless_browser) execution.
+
+However, we can work around that - by running our tests in Docker! We can mount our whole working directory, and run tests in [xvfb](https://en.wikipedia.org/wiki/Xvfb).
+
+In this folder, there's a `Dockerfile` with a rather lightweight image - `vscode`, `xvfb`, their dependencies, and `npm`/`node` - and some scripts to run it. The intended usage is as follows:
+
+* Build the docker image by running `npm run e2e:docker:build`. You only need to do it once
+* Run one of the two commands:
+    - `npm run e2e:docker` - this will do the full cycle of building the extension and setting up tests. You typically want to run this when you've just updated the extension
+    - `npm run e2e:docker:rerun` - a much faster command to rerun the tests against an already prepared setup. Use this when iterating on tests themselves

e2e/scripts/build-docker

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Build the docker iamge for e2e tests
+# Explicitly using empty context to avoid unnecessary rebuilds,
+# since we'll be mounting the whole work folder when using the image
+docker build --platform linux/amd64 --tag atlascode-e2e - <e2e/Dockerfile

e2e/scripts/in-docker

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Usage:
+# e2e/scripts/in-docker some-fancy-command --with-args
+
+docker run \
+    --platform linux/amd64 \
+    -v $(pwd):/atlascode \
+    --user atlascode \
+    -it \
+    atlascode-e2e $@

e2e/scripts/run.sh → e2e/scripts/run

@@ -6,10 +6,16 @@ if [ "$1" == "--rerun" ]; then
     # Skip rebuilding the extension and setting up the tests
     # Use this if you are iterating in the tests themselves
     command="run-tests"
+
+    # If you delete your e2e/.resources, or bump a version somewhere
+    # it could be useful to have these here. They are only ran if
+    # the binaries are not already present
+    extest get-vscode
+    extest get-chromedriver
 fi
 
 extest ${command} \
     './.generated/atlascode/e2e/tests/*.test.js' \
     --code_version max \
     --code_settings e2e/test-settings.json \
-    --extensions_dir .test-extensions
+    --extensions_dir e2e/.test-extensions

e2e/tests/dummy.test.ts

@@ -1,28 +1,62 @@
 // import the webdriver and the high level browser wrapper
-import { assert } from 'chai';
-import { before, VSBrowser, WebDriver } from 'vscode-extension-tester';
+import { expect } from 'chai';
+import { before, VSBrowser, WebDriver, ActivityBar, after } from 'vscode-extension-tester';
 import { describe, it } from 'mocha';
 
 // Let's make sure we can import something from the src folder if needed
 import { AuthInfoVersionKey } from '../../src/constants';
 
 // Create a Mocha suite
-describe('My Test Suite', () => {
+describe('Atlassian Extension', async () => {
     let browser: VSBrowser;
     let driver: WebDriver;
 
     console.log('Oh look, a constant from src!');
     console.log('AuthInfoVersionKey:', AuthInfoVersionKey);
 
+    const { log, debug, info, warn } = console;
     // initialize the browser and webdriver
     before(async () => {
+        // Mocking console.log
+        global.console = {
+            ...console,
+            log: () => {},
+            debug: () => {},
+            info: () => {},
+            warn: () => {},
+        };
+
         browser = VSBrowser.instance;
         driver = browser.driver;
     });
 
+    after(async () => {
+        // reset global.console
+        global.console = {
+            ...console,
+            log,
+            debug,
+            info,
+            warn,
+        };
+    });
+
     // test whatever we want using webdriver, here we are just checking the page title
     it('My Test Case', async () => {
         const title = await driver.getTitle();
-        assert.isTrue(title === 'Getting Started' || title === 'Walkthrough: Setup VS Code');
+        expect(title).to.be.oneOf([
+            'Getting Started',
+            'Walkthrough: Setup VS Code',
+            'Getting Started - Visual Studio Code',
+        ]);
+    });
+
+    it('should be installed', async () => {
+        const activityBar = new ActivityBar();
+        const controls = await activityBar.getViewControls();
+        // Get title from every control
+        const titles = await Promise.all(controls.map(async (control) => control.getTitle()));
+
+        expect('Atlassian').to.be.oneOf(titles);
     });
 });

package.json

@@ -62,9 +62,12 @@
         "extension:package": "npm run extension:clean && vsce package --baseContentUrl https://raw.githubusercontent.com/atlassian/atlascode/main/ --allow-star-activation",
         "extension:install": "npm run extension:package && code --install-extension ./atlascode-*.vsix --force",
         "test": "jest",
+        "e2e": "npm run e2e:compile && e2e/scripts/run",
         "e2e:compile": "tsc --project e2e/tsconfig.e2e.json",
-        "e2e": "npm run e2e:compile && e2e/scripts/run.sh",
-        "e2e:rerun": "npm run e2e:compile && e2e/scripts/run.sh --rerun",
+        "e2e:rerun": "npm run e2e:compile && e2e/scripts/run --rerun",
+        "e2e:docker": "e2e/scripts/in-docker npm run e2e",
+        "e2e:docker:build": "e2e/scripts/build-docker",
+        "e2e:docker:rerun": "e2e/scripts/in-docker npm run e2e:rerun",
         "devcompile": "npm-run-all --parallel devcompile:react devcompile:extension",
         "devcompile:react": "webpack --mode development --config webpack.react.dev.js",
         "devcompile:extension": "webpack --mode development --config webpack.extension.dev.js",