Running CircleCI render tests on Windows, Mac and Firefox (#12268)

* Build + render tests in windows * Fixing issues with render tests running on Windows * Adding prepare-windows script * Adding prepare-windows to workflow * Installing chrome * Manually installing chrome * Take 2 * fix * Take 3 * Take 4 * Fix typo * Switching check for Chrome * Consolodating windows prepare script * Adding checkout and restore cache * Updating to latest version of browser-tools * Add testing for firefox on Linux * Adding missing prepare * Fix indent * Add test-render-firefox command * Modifying alloweds so firefox tests pass * Running tests on Chrome MacOS * Removing cache * Updating alloweds * Updating incorrect test * Adding overflow:scroll to test html to allow viewing difs with large images that were previously offscreen * Updating allowed to pass tests failing on windows with no perceptible difference due to antialiasing * Removing extra 0 * Regenerating expected for collision-pitched-wrapped * Fixing changed alloweds * Specifying browser in windows run * Adding windows test ignores * Fix broken test runs * Adding firefox ignore file and rearranging test ignores * Moving test triggering crash to skip * Increasing timeout on Windows and Mac * Fixing failures on windows * Bumping time out and some windows alloweds * Increasing browser_disconnect_timeout * Increasing timeouts * Increasing windows timeout to 45s * Adding timing out tests to Windows ignore * Adding MacOS ignores on CI * Fix lint * Raising alloweds to allow slightly flaky globe tests to pass on Windows * Raising allowed on 10 tests to pass in MacOS on CI * Adjusting timeout in tape() call * Test: removing setting chrome flags to disable angle on CI * Changing chrome flags to use llvmpipe driver on Linux instead of on CI * Forcing gl on MacOS CI * Moving tests from windows skip to TODO * Adding check for valid angle option and using metal on Mac CI * Updating CI MacOS to 12.5.1 * Test: Using swiftshader on MacOS * Adding windows ignores to mac * Always calling --disable-backgrounding-occluded-windows on Chrome * Simplifying testing scripts * Upping timeout to 1m for windows * Adding prepare as dependency to windows and macos tests Co-authored-by: Karim Naaji <[email protected]> * Undoing too-high alloweds * Raising allowed on one test for slight antialiasing difference * Removing debug tile visualization and allowed from globe transition tests * Updating existing render tests run names to include linux-chrome * Changing timeout on windows from 60s to 90s * Removing windows specific bug from Mac ignore * Increasing allowed to account for antialiasing with two globe transition tests * Adding -dev suffix to tests, removing prepare requirement from virtual machines * Adding documentation * Adding new test runs as dependencies for deploying benchmarks and release * Adding timout to firefox to reduce flakiness * Fix lint * Bumping allowed to allow test to pass on macos * Upping browser timeout * Bumping windows timeout * Adding links to issues * Adding yarn cache dir for debugging * Updating yarn cache location * Adding fallback cache keys for partial cache restore * Adding frozen lockfile to install, changing cache to just node modules * Testing caching yarn cache instead of node_modules * Removing exact cache check in windows run * Full directory of cache on Windows * Bumping browser timeout to 60 * Bumping windows timeout by 30s * Changing windows to use seperate cache * Bumping browser disconnect timeout to 90s * Experiment: add linux yarn cache to mac * Adding seperate mac cache * Fixing inconsistent cache directories * Renaming caches * Removing caching from windows * Attempt at parallelism on Windows * Change syntax to windows cmd and attempt adding parallelism to mac * Adding echos for debugging * Changing echo to cat * Removing cat * Fixing globe * Testing parallel runs on linux * Setting up render tests to accept a list of directories in arguments * More robust argument parsing * Attempt to change windows variable syntax to powershell * Fix lint * Try pipe in windows to solve command line too long error * Attempt at writing and reading test list from file * Fix typo * Adding debug artifact * Fix typo in debug path * Changing encoding to utf8 * Splitting on regex to work in windows * Converting Windows to posix paths * Changing to utf8NoBOM * Fix posix coversion * Revert "Changing to utf8NoBOM" This reverts commit 0cb12f9. * Attempt at removing BOM * Updating heatmap expecteds * Try writing test list file for macos parallelization too * Adding trim to remove newline at file end being read as extra test * Splitting mac run by timing * Removing console.logs * Refactor render tests (#12436) * Clean up directory parsing code * Removing test list artifacts added for debugging * Adding flaky tests to brawser-specific ignores * Adding one more test to mac ignore * Adding one more test to macos skip Co-authored-by: Karim Naaji <[email protected]> Co-authored-by: Stepan Kuzmin <[email protected]>
mapbox · Dec 13, 2022 · f0d57f5 · f0d57f5
1 parent 43e0db0
commit f0d57f5
Show file tree

Hide file tree

Showing 159 changed files with 982 additions and 451 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,7 +1,8 @@
 version: 2.1
 orbs:
   aws-cli: circleci/[email protected]
-  browser-tools: circleci/[email protected]
+  browser-tools: circleci/[email protected]
+  win: circleci/[email protected]
 
 workflows:
   version: 2
@@ -55,19 +56,33 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - test-render:
+      - test-render-linux-chrome-dev:
           requires:
             - prepare
           filters:
             tags:
               only: /.*/
-      - test-render-prod:
+      - test-render-linux-firefox-dev:
           requires:
             - prepare
           filters:
             tags:
               only: /.*/
-      - test-render-csp:
+      - test-render-macos-chrome-dev:
+          filters:
+            tags:
+              only: /.*/
+      - test-render-windows-chrome-dev:
+          filters:
+            tags:
+              only: /.*/
+      - test-render-linux-chrome-prod:
+          requires:
+            - prepare
+          filters:
+            tags:
+              only: /.*/
+      - test-render-linux-chrome-csp:
           requires:
             - prepare
           filters:
@@ -119,8 +134,11 @@ workflows:
             - build
             - test-flow
             - test-unit
-            - test-render
-            - test-render-prod
+            - test-render-linux-chrome-dev
+            - test-render-linux-chrome-prod
+            - test-render-linux-firefox-dev
+            - test-render-macos-chrome-dev
+            - test-render-windows-chrome-dev
             - test-query
             - test-expressions
           filters:
@@ -137,8 +155,11 @@ workflows:
             - build
             - test-flow
             - test-unit
-            - test-render
-            - test-render-prod
+            - test-render-linux-chrome-dev
+            - test-render-linux-chrome-prod
+            - test-render-linux-firefox-dev
+            - test-render-macos-chrome-dev
+            - test-render-windows-chrome-dev
             - test-query
             - test-expressions
           filters:
@@ -160,13 +181,14 @@ jobs:
       - checkout
       - restore_cache:
           keys:
-            - v4-yarn-{{ checksum "yarn.lock" }}
-      - run: yarn
+            - v0-linux-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }}
+            - v0-linux-yarn-{{ .Branch }}-
+            - v0-linux-yarn-
+      - run: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
       - save_cache:
-          key: v4-yarn-{{ checksum "yarn.lock" }}
+          key: v0-linux-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }}
           paths:
-            - '~/.yarn'
-            - 'node_modules'
+           - ~/.cache/yarn
       - persist_to_workspace:
           root: ~/
           paths:
@@ -253,19 +275,36 @@ jobs:
           at: ~/
       - run: yarn run test-unit
 
-  test-render:
+  test-render-linux-chrome-dev:
     <<: *defaults
     steps:
       - attach_workspace:
           at: ~/
       - browser-tools/install-chrome
-      - run: yarn run test-render
+      # - run: yarn run test-render
+      - run:
+          name: Running tests in parallel
+          command: |
+            yarn run test-render
       - store_test_results:
           path: test/integration/render-tests
       - store_artifacts:
           path: "test/integration/render-tests/index.html"
 
-  test-render-prod:
+  test-render-linux-firefox-dev:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
+      - browser-tools/install-firefox
+      - run: yarn run test-render-firefox
+      - store_test_results:
+          path: test/integration/render-tests
+      - store_artifacts:
+          path: "test/integration/render-tests/index.html"
+
+
+  test-render-linux-chrome-prod:
     <<: *defaults
     steps:
       - attach_workspace:
@@ -277,7 +316,7 @@ jobs:
       - store_artifacts:
           path: "test/integration/render-tests/index.html"
 
-  test-render-csp:
+  test-render-linux-chrome-csp:
     <<: *defaults
     steps:
       - attach_workspace:
@@ -401,3 +440,68 @@ jobs:
           name: Deploy release
           command: |
             bash ./build/upload.sh
+
+  test-render-macos-chrome-dev:
+    macos:
+      xcode: 14.0.1 # macOS version 12.5.1 (Monterey)
+    parallelism: 3
+    working_directory: ~/mapbox-gl-js
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ~/
+      - restore_cache:
+          keys:
+            - v0-mac-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }}
+            - v0-mac-yarn-{{ .Branch }}-
+            - v0-mac-yarn-
+      - browser-tools/install-chrome
+      - run: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
+      - save_cache:
+          key: v0-mac-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }}
+          paths:
+           - ~/.cache/yarn
+      - run:
+          name: Creating test list
+          command: |
+            circleci tests glob "test/integration/render-tests/**/*.json" | circleci tests split --split-by=timings > tests-to-run.txt
+      - run: yarn run test-render
+      - store_test_results:
+          path: test/integration/render-tests
+      - store_artifacts:
+          path: "test/integration/render-tests/index.html"
+
+
+  test-render-windows-chrome-dev:
+    executor:
+      name: win/default
+    parallelism: 5
+    working_directory: ~/mapbox-gl-js
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ~/
+      # The browser-tools orb doesn't work on Windows, so we install chrome manually.
+      - run:
+          name: Installing Chrome on Windows
+          command: |
+            $uri = "https://dl.google.com/chrome/install/latest/chrome_installer.exe";
+            $path = "$PSScriptRoot\ChromeSetup.exe";
+            Invoke-WebRequest -Uri $uri -OutFile $path;
+            Start-Process $path /install -NoNewWindow -Wait; 
+            Remove-Item $path;
+
+            $chromeInstalled = (Get-Item (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe').'(Default)').VersionInfo;
+            if ($chromeInstalled.FileName -eq $null) {
+              Write-Host "Chrome failed to install";
+            }
+      - run: yarn --frozen-lockfile
+      - run:
+          name: Creating test list
+          command: |
+            circleci tests glob "test/integration/render-tests/**/*.json" | circleci tests split | Out-File -Encoding utf8 -FilePath tests-to-run.txt
+      - run: yarn run test-render
+      - store_test_results:
+          path: test/integration/render-tests
+      - store_artifacts:
+          path: "test/integration/render-tests/index.html"
diff --git a/package.json b/package.json
@@ -49,6 +49,7 @@
     "browserify": "^17.0.0",
     "chalk": "^5.0.1",
     "chokidar": "^3.5.3",
+    "cross-env": "^7.0.3",
     "cssnano": "^5.1.13",
     "d3": "^7.6.1",
     "d3-queue": "^3.0.7",
@@ -130,12 +131,13 @@
     "test-unit": "build/run-tap --reporter classic --no-coverage test/unit",
     "test-build": "build/run-tap --no-coverage test/build/**/*.test.js",
     "test-browser": "build/run-tap --jobs=1 --reporter spec --no-coverage -- test/browser/**/*.test.js",
-    "watch-render": "SUITE_NAME=render testem -f test/integration/testem/testem.js",
+    "watch-render": "cross-env SUITE_NAME=render testem -f test/integration/testem/testem.js",
     "watch-query": "SUITE_NAME=query testem -f test/integration/testem/testem.js",
-    "test-render": "SUITE_NAME=render RENDER=true testem ci -f test/integration/testem/testem.js",
-    "test-render-prod": "BUILD=production SUITE_NAME=render RENDER=true testem ci -f test/integration/testem/testem.js",
-    "test-render-csp": "BUILD=csp SUITE_NAME=render RENDER=true testem ci -f test/integration/testem/testem.js",
-    "test-query": "SUITE_NAME=query RENDER=true testem ci -f test/integration/testem/testem.js",
+    "test-render": "cross-env SUITE_NAME=render testem ci -f test/integration/testem/testem.js",
+    "test-render-firefox": "cross-env BROWSER=Firefox SUITE_NAME=render testem ci -f test/integration/testem/testem.js",
+    "test-render-prod": "BUILD=production SUITE_NAME=render testem ci -f test/integration/testem/testem.js",
+    "test-render-csp": "BUILD=csp SUITE_NAME=render testem ci -f test/integration/testem/testem.js",
+    "test-query": "SUITE_NAME=query testem ci -f test/integration/testem/testem.js",
     "test-expressions": "build/run-node test/expression.test.js",
     "test-flow": "build/run-node build/generate-flow-typed-style-spec && flow .",
     "test-cov": "nyc --require=@mapbox/flow-remove-types/register --reporter=text-summary --reporter=lcov --cache run-s test-unit test-expressions test-query test-render",

diff --git a/test/expression.test.js b/test/expression.test.js
@@ -3,7 +3,7 @@ import {createPropertyExpression} from '../src/style-spec/expression/index.js';
 import {isFunction} from '../src/style-spec/function/index.js';
 import convertFunction from '../src/style-spec/function/convert.js';
 import {toString} from '../src/style-spec/expression/types.js';
-import ignores from './ignores.json';
+import ignores from './ignores/all.js';
 import {CanonicalTileID} from '../src/source/tile_id.js';
 import MercatorCoordinate from '../src/geo/mercator_coordinate.js';
 import tileTransform, {getTilePoint} from '../src/geo/projection/tile_transform.js';

diff --git a/test/ignores.json b/test/ignores.json