Add screenshot nightly test for wkorg

.circleci/config.yml

@@ -291,16 +291,43 @@ jobs:
       - store_artifacts:
           path: frontend/javascripts/test/snapshots/type-check
 
+  wkorg_nightly:
+    docker:
+      - image: scalableminds/puppeteer:master
+    resource_class: large
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            yarn install --frozen-lockfile
+      - run:
+          name: Run screenshot-tests
+          command: |
+            # CircleCI cancels the job after 60 minutes. To ensure that screenshots are still
+            # uploaded as artifacts, we define a timeout of 50 minutes for the screenshot tests.
+            timeout 3000 \
+            yarn test-wkorg-screenshot
+
+      - store_artifacts:
+          path: frontend/javascripts/test/screenshots
+
 workflows:
   version: 2
   circleci_build:
     jobs:
-      - build_test_deploy:
+      - wkorg_nightly:
           context:
             - DockerHub
           filters:
             tags:
               only: /.*/
+      # - build_test_deploy:
+      #     context:
+      #       - DockerHub
+      #     filters:
+      #       tags:
+      #         only: /.*/
   circleci_nightly:
     jobs:
       - nightly
@@ -312,3 +339,14 @@ workflows:
             branches:
               only:
                 - master
+  circleci_wkorg_nightly:
+    job:
+      - wkorg_nightly
+    triggers:
+      - schedule:
+          # 03:15 AM UTC
+          cron: "15 3 * * *"
+          filters:
+            branches:
+              only:
+                - master

frontend/javascripts/test/puppeteer/dataset_rendering.screenshot.ts

@@ -1,31 +1,26 @@
 import "test/mocks/lz4";
 import type { PartialDatasetConfiguration } from "oxalis/store";
-import type { TestInterface } from "ava";
-import anyTest from "ava";
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'node... Remove this comment to see the full error message
-import fetch, { Headers, Request, Response, FetchError } from "node-fetch";
 import path from "path";
-import type { Browser } from "puppeteer";
-import puppeteer from "puppeteer";
 import { compareScreenshot, isPixelEquivalent } from "./screenshot_helpers";
 import {
+  test,
+  getNewPage,
   screenshotAnnotation,
   screenshotDataset,
   screenshotDatasetWithMapping,
   screenshotDatasetWithMappingLink,
   screenshotSandboxWithMappingLink,
+  setupBeforeEachAndAfterEach,
+  withRetry,
   WK_AUTH_TOKEN,
+  checkBrowserstackCredentials,
 } from "./dataset_rendering_helpers";
 
 if (!WK_AUTH_TOKEN) {
   throw new Error("No WK_AUTH_TOKEN specified.");
 }
 
-if (process.env.BROWSERSTACK_USERNAME == null || process.env.BROWSERSTACK_ACCESS_KEY == null) {
-  throw new Error(
-    "BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY must be defined as env variables.",
-  );
-}
+checkBrowserstackCredentials();
 
 process.on("unhandledRejection", (err, promise) => {
   console.error("Unhandled rejection (promise: ", promise, ", reason: ", err, ").");
@@ -47,60 +42,8 @@ if (!process.env.URL) {
 }
 
 console.log(`[Info] Executing tests on URL ${URL}.`);
-// Ava's recommendation for Typescript types
-// https://github.com/avajs/ava/blob/main/docs/recipes/typescript.md#typing-tcontext
-const test: TestInterface<{
-  browser: Browser;
-}> = anyTest as any;
 
-async function getNewPage(browser: Browser) {
-  const page = await browser.newPage();
-  page.setViewport({
-    width: 1920,
-    height: 1080,
-  });
-  page.setExtraHTTPHeaders({
-    // @ts-expect-error ts-migrate(2322) FIXME: Type 'string | undefined' is not assignable to typ... Remove this comment to see the full error message
-    "X-Auth-Token": WK_AUTH_TOKEN,
-  });
-  return page;
-}
-
-test.beforeEach(async (t) => {
-  t.context.browser = await puppeteer.launch({
-    args: [
-      "--headless",
-      "--hide-scrollbars",
-      "--no-sandbox",
-      "--disable-setuid-sandbox",
-      "--disable-dev-shm-usage",
-      "--use-gl=swiftshader",
-    ],
-    dumpio: true,
-  });
-
-  const caps = {
-    browser: "chrome",
-    browser_version: "latest",
-    os: "os x",
-    os_version: "mojave",
-    "browserstack.username": process.env.BROWSERSTACK_USERNAME,
-    "browserstack.accessKey": process.env.BROWSERSTACK_ACCESS_KEY,
-  };
-  t.context.browser = await puppeteer.connect({
-    browserWSEndpoint: `ws://cdp.browserstack.com/puppeteer?caps=${encodeURIComponent(
-      JSON.stringify(caps),
-    )}`,
-  });
-
-  console.log(`\nRunning chrome version ${await t.context.browser.version()}\n`);
-  global.Headers = Headers;
-  global.fetch = fetch;
-  global.Request = Request;
-  global.Response = Response;
-  // @ts-expect-error ts-migrate(7017) FIXME: Element implicitly has an 'any' type because type ... Remove this comment to see the full error message
-  global.FetchError = FetchError;
-});
+setupBeforeEachAndAfterEach();
 
 // These datasets are available on our dev instance (e.g., master.webknossos.xyz)
 const datasetNames = [
@@ -166,26 +109,6 @@ const datasetConfigOverrides: Record<string, PartialDatasetConfiguration> = {
   },
 };
 
-async function withRetry(
-  retryCount: number,
-  testFn: () => Promise<boolean>,
-  resolveFn: (arg0: boolean) => void,
-) {
-  retryCount = 3;
-  for (let i = 0; i < retryCount; i++) {
-    // eslint-disable-next-line no-await-in-loop
-    const condition = await testFn();
-
-    if (condition || i === retryCount - 1) {
-      // Either the test passed or we executed the last attempt
-      resolveFn(condition);
-      return;
-    }
-
-    console.error(`Test failed, retrying. This will be attempt ${i + 2}/${retryCount}.`);
-  }
-}
-
 datasetNames.map(async (datasetName) => {
   test.serial(`it should render dataset ${datasetName} correctly`, async (t) => {
     await withRetry(
@@ -406,6 +329,3 @@ test.serial(
     );
   },
 );
-test.afterEach.always(async (t) => {
-  await t.context.browser.close();
-});

frontend/javascripts/test/puppeteer/dataset_rendering.wkorg_screenshot.ts

@@ -0,0 +1,56 @@
+import "test/mocks/lz4";
+import path from "path";
+import { compareScreenshot, isPixelEquivalent } from "./screenshot_helpers";
+import {
+  test,
+  getNewPage,
+  screenshotDatasetView,
+  setupBeforeEachAndAfterEach,
+  withRetry,
+  checkBrowserstackCredentials,
+} from "./dataset_rendering_helpers";
+
+checkBrowserstackCredentials();
+
+process.on("unhandledRejection", (err, promise) => {
+  console.error("Unhandled rejection (promise: ", promise, ", reason: ", err, ").");
+});
+const BASE_PATH = path.join(__dirname, "../../../../frontend/javascripts/test/screenshots-wkorg");
+const URL = "https://webknossos.org";
+
+console.log(`[Info] Executing tests on URL ${URL}.`);
+setupBeforeEachAndAfterEach();
+
+const demoDatasetName = "l4dense_motta_et_al_demo";
+const owningOrganization = "scalable_minds";
+
+test.serial(`it should render dataset ${demoDatasetName} correctly`, async (t) => {
+  await withRetry(
+    3,
+    async () => {
+      const datasetId = {
+        name: demoDatasetName,
+        owningOrganization,
+      };
+      const { screenshot, width, height } = await screenshotDatasetView(
+        await getNewPage(t.context.browser),
+        URL,
+        datasetId,
+      );
+      const changedPixels = await compareScreenshot(
+        screenshot,
+        width,
+        height,
+        BASE_PATH,
+        demoDatasetName,
+      );
+      return isPixelEquivalent(changedPixels, width, height);
+    },
+    (condition) => {
+      t.true(
+        condition,
+        `Dataset with name: "${demoDatasetName}" does not look the same, see ${demoDatasetName}.diff.png for the difference and ${demoDatasetName}.new.png for the new screenshot.`,
+      );
+    },
+  );
+});