diff --git a/.circleci/config.yml b/.circleci/config.yml index fee8c159268..87f3e61ab30 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,8 @@ version: 2.1 orbs: aws-cli: circleci/aws-cli@1.4.0 - browser-tools: circleci/browser-tools@1.2.3 + browser-tools: circleci/browser-tools@1.4.0 + win: circleci/windows@4.1.1 workflows: version: 2 @@ -55,19 +56,33 @@ workflows: filters: tags: only: /.*/ - - test-render: + - test-render-linux-chrome-dev: requires: - prepare filters: tags: only: /.*/ - - test-render-prod: + - test-render-linux-firefox-dev: requires: - prepare filters: tags: only: /.*/ - - test-render-csp: + - test-render-macos-chrome-dev: + filters: + tags: + only: /.*/ + - test-render-windows-chrome-dev: + filters: + tags: + only: /.*/ + - test-render-linux-chrome-prod: + requires: + - prepare + filters: + tags: + only: /.*/ + - test-render-linux-chrome-csp: requires: - prepare filters: @@ -119,8 +134,11 @@ workflows: - build - test-flow - test-unit - - test-render - - test-render-prod + - test-render-linux-chrome-dev + - test-render-linux-chrome-prod + - test-render-linux-firefox-dev + - test-render-macos-chrome-dev + - test-render-windows-chrome-dev - test-query - test-expressions filters: @@ -137,8 +155,11 @@ workflows: - build - test-flow - test-unit - - test-render - - test-render-prod + - test-render-linux-chrome-dev + - test-render-linux-chrome-prod + - test-render-linux-firefox-dev + - test-render-macos-chrome-dev + - test-render-windows-chrome-dev - test-query - test-expressions filters: @@ -160,13 +181,14 @@ jobs: - checkout - restore_cache: keys: - - v4-yarn-{{ checksum "yarn.lock" }} - - run: yarn + - v0-linux-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }} + - v0-linux-yarn-{{ .Branch }}- + - v0-linux-yarn- + - run: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - save_cache: - key: v4-yarn-{{ checksum "yarn.lock" }} + key: v0-linux-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }} paths: - - '~/.yarn' - - 'node_modules' + - ~/.cache/yarn - persist_to_workspace: root: ~/ paths: @@ -253,19 +275,36 @@ jobs: at: ~/ - run: yarn run test-unit - test-render: + test-render-linux-chrome-dev: <<: *defaults steps: - attach_workspace: at: ~/ - browser-tools/install-chrome - - run: yarn run test-render + # - run: yarn run test-render + - run: + name: Running tests in parallel + command: | + yarn run test-render - store_test_results: path: test/integration/render-tests - store_artifacts: path: "test/integration/render-tests/index.html" - test-render-prod: + test-render-linux-firefox-dev: + <<: *defaults + steps: + - attach_workspace: + at: ~/ + - browser-tools/install-firefox + - run: yarn run test-render-firefox + - store_test_results: + path: test/integration/render-tests + - store_artifacts: + path: "test/integration/render-tests/index.html" + + + test-render-linux-chrome-prod: <<: *defaults steps: - attach_workspace: @@ -277,7 +316,7 @@ jobs: - store_artifacts: path: "test/integration/render-tests/index.html" - test-render-csp: + test-render-linux-chrome-csp: <<: *defaults steps: - attach_workspace: @@ -401,3 +440,68 @@ jobs: name: Deploy release command: | bash ./build/upload.sh + + test-render-macos-chrome-dev: + macos: + xcode: 14.0.1 # macOS version 12.5.1 (Monterey) + parallelism: 3 + working_directory: ~/mapbox-gl-js + steps: + - checkout + - attach_workspace: + at: ~/ + - restore_cache: + keys: + - v0-mac-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }} + - v0-mac-yarn-{{ .Branch }}- + - v0-mac-yarn- + - browser-tools/install-chrome + - run: yarn --frozen-lockfile --cache-folder ~/.cache/yarn + - save_cache: + key: v0-mac-yarn-{{ .Branch }}-{{ checksum "yarn.lock" }} + paths: + - ~/.cache/yarn + - run: + name: Creating test list + command: | + circleci tests glob "test/integration/render-tests/**/*.json" | circleci tests split --split-by=timings > tests-to-run.txt + - run: yarn run test-render + - store_test_results: + path: test/integration/render-tests + - store_artifacts: + path: "test/integration/render-tests/index.html" + + + test-render-windows-chrome-dev: + executor: + name: win/default + parallelism: 5 + working_directory: ~/mapbox-gl-js + steps: + - checkout + - attach_workspace: + at: ~/ + # The browser-tools orb doesn't work on Windows, so we install chrome manually. + - run: + name: Installing Chrome on Windows + command: | + $uri = "https://dl.google.com/chrome/install/latest/chrome_installer.exe"; + $path = "$PSScriptRoot\ChromeSetup.exe"; + Invoke-WebRequest -Uri $uri -OutFile $path; + Start-Process $path /install -NoNewWindow -Wait; + Remove-Item $path; + + $chromeInstalled = (Get-Item (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe').'(Default)').VersionInfo; + if ($chromeInstalled.FileName -eq $null) { + Write-Host "Chrome failed to install"; + } + - run: yarn --frozen-lockfile + - run: + name: Creating test list + command: | + circleci tests glob "test/integration/render-tests/**/*.json" | circleci tests split | Out-File -Encoding utf8 -FilePath tests-to-run.txt + - run: yarn run test-render + - store_test_results: + path: test/integration/render-tests + - store_artifacts: + path: "test/integration/render-tests/index.html" diff --git a/package.json b/package.json index 1a60a460e00..965bd8488e4 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "browserify": "^17.0.0", "chalk": "^5.0.1", "chokidar": "^3.5.3", + "cross-env": "^7.0.3", "cssnano": "^5.1.13", "d3": "^7.6.1", "d3-queue": "^3.0.7", @@ -130,12 +131,13 @@ "test-unit": "build/run-tap --reporter classic --no-coverage test/unit", "test-build": "build/run-tap --no-coverage test/build/**/*.test.js", "test-browser": "build/run-tap --jobs=1 --reporter spec --no-coverage -- test/browser/**/*.test.js", - "watch-render": "SUITE_NAME=render testem -f test/integration/testem/testem.js", + "watch-render": "cross-env SUITE_NAME=render testem -f test/integration/testem/testem.js", "watch-query": "SUITE_NAME=query testem -f test/integration/testem/testem.js", - "test-render": "SUITE_NAME=render RENDER=true testem ci -f test/integration/testem/testem.js", - "test-render-prod": "BUILD=production SUITE_NAME=render RENDER=true testem ci -f test/integration/testem/testem.js", - "test-render-csp": "BUILD=csp SUITE_NAME=render RENDER=true testem ci -f test/integration/testem/testem.js", - "test-query": "SUITE_NAME=query RENDER=true testem ci -f test/integration/testem/testem.js", + "test-render": "cross-env SUITE_NAME=render testem ci -f test/integration/testem/testem.js", + "test-render-firefox": "cross-env BROWSER=Firefox SUITE_NAME=render testem ci -f test/integration/testem/testem.js", + "test-render-prod": "BUILD=production SUITE_NAME=render testem ci -f test/integration/testem/testem.js", + "test-render-csp": "BUILD=csp SUITE_NAME=render testem ci -f test/integration/testem/testem.js", + "test-query": "SUITE_NAME=query testem ci -f test/integration/testem/testem.js", "test-expressions": "build/run-node test/expression.test.js", "test-flow": "build/run-node build/generate-flow-typed-style-spec && flow .", "test-cov": "nyc --require=@mapbox/flow-remove-types/register --reporter=text-summary --reporter=lcov --cache run-s test-unit test-expressions test-query test-render", diff --git a/test/expression.test.js b/test/expression.test.js index db741f604e2..57f9a5ff4f3 100644 --- a/test/expression.test.js +++ b/test/expression.test.js @@ -3,7 +3,7 @@ import {createPropertyExpression} from '../src/style-spec/expression/index.js'; import {isFunction} from '../src/style-spec/function/index.js'; import convertFunction from '../src/style-spec/function/convert.js'; import {toString} from '../src/style-spec/expression/types.js'; -import ignores from './ignores.json'; +import ignores from './ignores/all.js'; import {CanonicalTileID} from '../src/source/tile_id.js'; import MercatorCoordinate from '../src/geo/mercator_coordinate.js'; import tileTransform, {getTilePoint} from '../src/geo/projection/tile_transform.js'; diff --git a/test/ignores.json b/test/ignores.json deleted file mode 100644 index e76f05b4b41..00000000000 --- a/test/ignores.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "query-tests/regressions/mapbox-gl-js#4494": "https://github.com/mapbox/mapbox-gl-js/issues/2716", - "query-tests/globe/circle/opposite-side-over-north-pole": "skip - needs baseline update", - "query-tests/terrain/draped/lines/slope-occlusion": "skip - https://github.com/mapbox/mapbox-gl-js/issues/11726", - "render-tests/background-pattern/projected": "skip - patterns not working with projections", - "render-tests/fill-pattern/projected": "https://github.com/mapbox/mapbox-gl-js/issues/11221", - "render-tests/debug/tile": "skip - inconsistent text rendering with canvas on different platforms", - "render-tests/debug/tile-overscaled": "skip - inconsistent text rendering with canvas on different platforms", - "render-tests/fill-extrusion-pattern/1.5x-on-1x-add-image": "skip - non-deterministic on AMD graphics cards", - "render-tests/fill-extrusion-pattern/multiple-layers-flat": "skip - mapbox-gl-js-internal#223", - "render-tests/fill-extrusion-pattern/opacity-terrain-flat-on-border": "skip - mapbox-gl-js-internal#223", - "render-tests/fill-extrusion-pattern/tile-buffer": "skip - not rendering correctly on CI", - "render-tests/fill-pattern/update-feature-state": "https://github.com/mapbox/mapbox-gl-js/issues/7207", - "render-tests/geojson/inline-linestring-fill": "skip - current behavior is arbitrary", - "render-tests/icon-image/icon-sdf-non-sdf-one-layer": "skip - render sdf icon and normal icon in one layer", - "render-tests/icon-text-fit/text-variable-anchor-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/map-mode/static": "https://github.com/mapbox/mapbox-gl-js/issues/5649", - "render-tests/map-mode/tile": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/map-mode/tile-avoid-edges": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/mixed-zoom/z10-z11": "current behavior conflicts with https://github.com/mapbox/mapbox-gl-js/pull/6803. can be fixed when https://github.com/mapbox/api-maps/issues/1480 is done", - "render-tests/projection/axonometric": "axonometric rendering in gl-js tbd", - "render-tests/projection/axonometric-multiple": "axonometric rendering in gl-js tbd", - "render-tests/projection/skew": "axonometric rendering in gl-js tbd", - "render-tests/symbol-placement/line-center-buffer-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/symbol-placement/line-center-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/symbol-sort-key/text-ignore-placement": "skip - text drawn over icons", - "render-tests/text-variable-anchor/all-anchors-labels-priority-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/text-variable-anchor/all-anchors-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/text-variable-anchor/avoid-edges-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/text-variable-anchor/left-top-right-bottom-offset-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/text-variable-anchor/pitched-rotated-debug": "skip - non-deterministic when rendered in browser", - "render-tests/text-variable-anchor/pitched-with-map": "skip - non-deterministic when rendered in browser", - "render-tests/text-variable-anchor/remember-last-placement": "skip - not sure this is correct behavior", - "render-tests/text-size/zero": "https://github.com/mapbox/mapbox-gl-js/issues/9161", - "render-tests/tile-mode/streets-v11": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/within/paint-line": "https://github.com/mapbox/mapbox-gl-js/issues/7023", - "render-tests/distance/layout-text-size": "skip - distance expression is not implemented", - "render-tests/skybox/atmosphere-padding": "skip - https://github.com/mapbox/mapbox-gl-js/issues/10314", - "render-tests/terrain/symbol-draping/style.json": "skip - https://github.com/mapbox/mapbox-gl-js/issues/10365", - "render-tests/text-variable-anchor/pitched": "skip - non-deterministic symbol placement on tile boundaries", - "render-tests/video/projected": "https://github.com/mapbox/mapbox-gl-js/issues/11234", - "query-tests/terrain/draped/lines/slope-occlusion-box-query": "skip - non-deterministic", - "render-tests/fill-extrusion-pattern/opacity-terrain": "skip - deferred for globe-view", - "render-tests/globe/globe-transition/horizon-during-transition": "skip - pending globe-view implementation", - "render-tests/resize/mercator": "skip - implemented for mapbox-gl-native only", - "render-tests/resize/globe": "skip - implemented for mapbox-gl-native only" -} diff --git a/test/ignores/all.js b/test/ignores/all.js new file mode 100644 index 00000000000..6fbf8f17a7a --- /dev/null +++ b/test/ignores/all.js @@ -0,0 +1,115 @@ + +const todo = [ + // "https://github.com/mapbox/mapbox-gl-js/issues/2716 + "query-tests/regressions/mapbox-gl-js#4494", + + // https://github.com/mapbox/mapbox-gl-js/issues/7207 + "render-tests/fill-pattern/update-feature-state", + + // https://github.com/mapbox/mapbox-gl-js/issues/5649, + "render-tests/map-mode/static", + + // "current behavior conflicts with https://github.com/mapbox/mapbox-gl-js/pull/6803. can be fixed when https://github.com/mapbox/api-maps/issues/1480 is done" + "render-tests/mixed-zoom/z10-z11", + + // "axonometric rendering in gl-js tbd" + "render-tests/projection/axonometric", + "render-tests/projection/axonometric-multiple", + "render-tests/projection/skew", + + // https://github.com/mapbox/mapbox-gl-js/issues/9161 + "render-tests/text-size/zero", + + // https://github.com/mapbox/mapbox-gl-js/issues/7023 + "render-tests/within/paint-line", + + // Needs port from Native + // "https://github.com/mapbox/mapbox-gl-js/issues/10365" + "render-tests/terrain/symbol-draping/style.json", +]; + +const skip = [ + // Pattern should be seamless across tile edges + // https://github.com/mapbox/mapbox-gl-js/issues/11221 + "render-tests/background-pattern/projected", + "render-tests/fill-pattern/projected", + + // Needs baseline update + "query-tests/globe/circle/opposite-side-over-north-pole", + + // inconsistent text rendering with canvas on different platforms + "render-tests/debug/tile", + "render-tests/debug/tile-overscaled", + + // "skip - non-deterministic on AMD graphics cards + "render-tests/fill-extrusion-pattern/1.5x-on-1x-add-image", + + // Inconsistent fill extrusion height rendering across tile boundaries + // https://github.com/mapbox/mapbox-gl-js/issues/10181 + "render-tests/fill-extrusion-pattern/multiple-layers-flat", + "render-tests/fill-extrusion-pattern/opacity-terrain-flat-on-border", + + // Not rendering correctly on CI + "render-tests/fill-extrusion-pattern/tile-buffer", + + // Current behavior is arbitrary + "render-tests/geojson/inline-linestring-fill", + + // Render SDF icon and normal icon in one layer + "render-tests/icon-image/icon-sdf-non-sdf-one-layer", + + // Mapbox-gl-js does not support tile-mode + "render-tests/icon-text-fit/text-variable-anchor-tile-map-mode", + "render-tests/map-mode/tile", + "render-tests/map-mode/tile-avoid-edges", + "render-tests/symbol-placement/line-center-buffer-tile-map-mode", + "render-tests/symbol-placement/line-center-tile-map-mode", + "render-tests/text-variable-anchor/all-anchors-labels-priority-tile-map-mode", + "render-tests/text-variable-anchor/all-anchors-tile-map-mode", + "render-tests/text-variable-anchor/avoid-edges-tile-map-mode", + "render-tests/text-variable-anchor/left-top-right-bottom-offset-tile-map-mode", + "render-tests/tile-mode/streets-v11", + + // Text drawn over icons + "render-tests/symbol-sort-key/text-ignore-placement", + + // Non-deterministiic when rendered in browser + "render-tests/text-variable-anchor/pitched-rotated-debug", + "render-tests/text-variable-anchor/pitched-with-map", + + // Not sure this is correct behavior + "render-tests/text-variable-anchor/remember-last-placement", + + // Flaky + // https://github.com/mapbox/mapbox-gl-js/issues/10314" + "render-tests/skybox/atmosphere-padding", + + // non-deterministic symbol placement on tile boundaries" + "render-tests/text-variable-anchor/pitched", + + // Flaky + // https://github.com/mapbox/mapbox-gl-js/issues/11234 + "render-tests/video/projected", + + // Non-deterministic + "query-tests/terrain/draped/lines/slope-occlusion-box-query", + + // Deferred for globe view + "render-tests/fill-extrusion-pattern/opacity-terrain", + + // Pending globe view implementation + "render-tests/globe/globe-transition/horizon-during-transition", + + // Implemented for mapbox-gl-native only + "render-tests/resize/mercator", + "render-tests/resize/globe", + + // Distance expression is not implemented, test times out + "render-tests/distance/layout-text-size", + + // Extremely flaky: https://github.com/mapbox/mapbox-gl-js/issues/11726 + "query-tests/terrain/draped/lines/slope-occlusion" + +]; + +export default {todo, skip}; diff --git a/test/ignores/firefox.js b/test/ignores/firefox.js new file mode 100644 index 00000000000..f5d5ce0ca7b --- /dev/null +++ b/test/ignores/firefox.js @@ -0,0 +1,8 @@ +const todo = [ + // Test fails to render. + // https://github.com/mapbox/mapbox-gl-js/issues/12406 + "render-tests/debug/terrain/collision-occlusion", +]; +const skip = []; + +export default {todo, skip}; diff --git a/test/ignores/macos.js b/test/ignores/macos.js new file mode 100644 index 00000000000..aa3ed99c378 --- /dev/null +++ b/test/ignores/macos.js @@ -0,0 +1,64 @@ +const todo = [ + // Appearance is slightly offset and diff is > 0.01 + "render-tests/icon-image/stretchable-content", + + // Bug: missing icons. + // https://github.com/mapbox/mapbox-gl-js/issues/12409 + "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen", + "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen", + + // Bug: raster-resampling doesn't work on Windows and some Macs + // https://github.com/mapbox/mapbox-gl-js/issues/7331 + "render-tests/raster-resampling/function", + "render-tests/raster-resampling/literal", + + // Due to incorrect resampling or antialasing, shifted icons appear pixelated. + // Likely related to the above issue. + // Difs here are small, but the appearance is worse. + "render-tests/debug/collision-overscaled-fractional-zoom", + "render-tests/icon-opacity/default", + "render-tests/icon-opacity/function", + "render-tests/icon-opacity/icon-only", + "render-tests/icon-opacity/literal", + "render-tests/icon-opacity/text-and-icon", + "render-tests/icon-translate-anchor/map", + "render-tests/icon-translate-anchor/viewport", + "render-tests/icon-translate/default", + "render-tests/icon-translate/function", + "render-tests/icon-translate/literal", + "render-tests/icon-visibility/visible", + "render-tests/regressions/mapbox-gl-js#7172", + "render-tests/runtime-styling/set-style-sprite", + "render-tests/symbol-placement/point", + "render-tests/symbol-spacing/point-close", + "render-tests/symbol-spacing/point-far", + "render-tests/symbol-visibility/visible", + + // Antialiasing results in a slightly different appearance for icon pattern on globe. + // Appearance is still good but the dif is too high (this could use a platform-specific expected.png) + "render-tests/globe/globe-transforms/north-pole", + + // Debug rendering + "render-tests/terrain/wireframe" +]; + +const skip = [ + // Flaky tests: + + // Test is occasionally blank: + "render-tests/debug/collision-pitched-wrapped-fractional-zoom", + "render-tests/debug/collision-pitched-wrapped-fractional-zoom", + + "render-tests/debug/collision-overscaled", // Missing icons + + // Globe sometimes missing + "render-tests/globe/globe-antialiasing/horizon-blend", + "render-tests/globe/globe-antialiasing/default", + "render-tests/globe/globe-antialiasing/tilted", + "render-tests/globe/globe-camera/reset-north", + + "render-tests/free-camera/terrain", // Occasionally subtly different camera position + "render-tests/fog/terrain/equal-range" // Ground sometimes missing (sky renders) +]; + +export default {todo, skip}; diff --git a/test/ignores/windows.js b/test/ignores/windows.js new file mode 100644 index 00000000000..1f20bb0c8fe --- /dev/null +++ b/test/ignores/windows.js @@ -0,0 +1,71 @@ +const todo = [ + // Appearance is slightly offset and diff is > 0.01 + "render-tests/icon-image/stretchable-content", + + // Bug: missing icons. + // https://github.com/mapbox/mapbox-gl-js/issues/12409 + "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen", + "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen", + + // Bug: Inconsistent zoom + // https://github.com/mapbox/mapbox-gl-js/issues/12408 + "render-tests/fit-screen-coordinates/terrain", + + // Bug: raster-resampling doesn't work on Windows and some Macs + // https://github.com/mapbox/mapbox-gl-js/issues/7331 + "render-tests/raster-resampling/function", + "render-tests/raster-resampling/literal", + + // Due to incorrect resampling or antialasing, shifted icons appear pixelated. + // Likely related to the above issue. + // Difs here are small, but the appearance is worse. + "render-tests/debug/collision-overscaled-fractional-zoom", + "render-tests/icon-opacity/default", + "render-tests/icon-opacity/function", + "render-tests/icon-opacity/icon-only", + "render-tests/icon-opacity/literal", + "render-tests/icon-opacity/text-and-icon", + "render-tests/icon-translate-anchor/map", + "render-tests/icon-translate-anchor/viewport", + "render-tests/icon-translate/default", + "render-tests/icon-translate/function", + "render-tests/icon-translate/literal", + "render-tests/icon-visibility/visible", + "render-tests/regressions/mapbox-gl-js#7172", + "render-tests/runtime-styling/set-style-sprite", + "render-tests/symbol-placement/point", + "render-tests/symbol-spacing/point-close", + "render-tests/symbol-spacing/point-far", + "render-tests/symbol-visibility/visible", + + // Antialiasing results in a slightly different appearance for icon pattern on globe. + // Appearance is still good but the dif is too high (this could use a platform-specific expected.png) + "render-tests/globe/globe-transforms/north-pole", + + // Debug rendering + "render-tests/terrain/wireframe" +]; + +const skip = [ + // Timing out on CI. + // Tracked in https://github.com/mapbox/mapbox-gl-js/issues/12407 + "render-tests/skybox/atmosphere-terrain", + "render-tests/terrain/decrease-exaggeration-fog", + "render-tests/terrain/error-overlap/initializing-no-terrain-at-center", + + // Flaky tests: + + "render-tests/dynamic-filter/symbols/line/pitch-low-show", // Test is occasionally blank + "render-tests/fog/disable bottom", // Terrain sometimes doesn't load + "render-tests/globe/globe-antialiasing/high-exaggeration", // Globe is sometimes missing + + // Occasionally crashing with 'Error occured during ["wait"]': + "render-tests/debug/terrain/collision-pitch-with-map-text-and-icon", + "render-tests/globe/globe-antialiasing/default", + "render-tests/globe/globe-circle/change-projection/set-projection", + "render-tests/globe/globe-circle/change-projection/set-projection", + "render-tests/globe/globe-circle/vertical-viewport-scaled-viewport-aligned/style.json", + "render-tests/free-camera/default/style.json", + "render-tests/fit-screen-coordinates/terrain" +]; +export default {todo, skip}; diff --git a/test/integration/README.md b/test/integration/README.md index adcf38cb479..92734621a43 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -64,18 +64,6 @@ Results at: ./test/integration/render-tests/index.html Done in 2.32s. ``` - -### Enable ANGLE configuration on render tests - -Some devices (e.g. M1 Macs) seem to run test with significantly less failures when forcing the ANGLE backend to use OpenGL. - -To configure the ANGLE backend, you can set the `--use-angle` input value to `USE_ANGLE` in CLI like so: -``` -USE_ANGLE={INPUT} yarn run test-render -``` - -Accepted inputs for `USE_ANGLE` are `metal`, `gl`, `vulkan`, `swiftshader`, and `gles`. See `chrome://flags/#use-angle` for more information on the `--use-angle` flag. - ### Viewing test results During a test run, the test harness will use Mapbox GL JS to create an `actual.png` image from the given `style.json`, and will then use [pixelmatch](https://github.com/mapbox/pixelmatch) to compare that image to `expected.png`, generating a `diff.png` highlighting the mismatched pixels (if any) in red. @@ -119,6 +107,17 @@ You can run a specific test by as follows ?filter=circle-radius/antimeridian ``` +### Enable ANGLE configuration on render tests + +Some devices (e.g. M1 Macs) seem to run test with significantly less failures when forcing the ANGLE backend to use OpenGL. + +To configure the ANGLE backend, you can set the `--use-angle` input value to `USE_ANGLE` in CLI like so: +``` +USE_ANGLE={INPUT} yarn run test-render +``` + +Accepted inputs for `USE_ANGLE` are `metal`, `gl`, `vulkan`, `swiftshader`, and `gles`. See `chrome://flags/#use-angle` for more information on the `--use-angle` flag. + ### Build Notifications The terminal window can be very noisy with both the build and the test servers running in the same session. @@ -147,7 +146,25 @@ To add a new render test: 5. Commit the new `style.json` and `expected.png` :rocket: +## Tests on CircleCI + +Every pushed commit triggers test runs on the CircleCI server. These catch regressions and prevent platform-specific bugs. + +Render tests often fail due to minor antialiasing differences between platforms. In these cases, you can add an "allowed" property under "test" in the test's `style.json` to tell the test runner the degree of difference that is acceptable. This is the fraction of pixels that can differ between `expected.png` and `actual.png`, ignoring some antialiasing, that will still allow the test to pass. + +How much to adjust the "allowed" is acceptable depends on the test, but alloweds >= .01 are usually much too high. Especially with larger test images, alloweds should generally be negligable, since a too-high allowed will fail to catch regressions and significant rendering differences suggest a bug. + +Larger alloweds are acceptable for testing debug features that will not be directly used by customers. + +## Ignores + +If a test fails on a run with too large a difference to adjust the "allowed," it can be added to the corresponding [ignore file](../ignore) for the browser or operating system. + +Ignores include tests under `"todo"` and `"skip"`. `"todo"` tests show up in test results but do not trigger a failing run. Most tests failing on one pltaform should be marked as "ignore." This allows us to notice if the tests start passing. + +Tests under `"skip"` will not run at all. Tests should be skipped if they trigger crashes or if they are flaky (to prevent falsely concluding that the test is a non-issue). +Ignored tests should link to an issue explaining the reason for ignoring the test. ## Reading Vector Tile Fixtures Install `vt2geojson`, a command line utility which turns vector tiles into geojson, and `harp`, a simple file server. diff --git a/test/integration/lib/generate-fixture-json.js b/test/integration/lib/generate-fixture-json.js index 9c4dd02fdf4..82b0752cb8c 100644 --- a/test/integration/lib/generate-fixture-json.js +++ b/test/integration/lib/generate-fixture-json.js @@ -1,4 +1,5 @@ -import path from 'path'; +// Use linux 'path' syntax on all operating systems to preserve compatability with 'glob' +import {posix as path} from "path"; import fs from 'fs'; import glob from 'glob'; import localizeURLs from './localize-urls.js'; @@ -12,56 +13,58 @@ import localizeURLs from './localize-urls.js'; * @param {string} suiteDirectory * @param {boolean} includeImages */ -export function generateFixtureJson(rootDirectory, suiteDirectory, outputDirectory = 'test/integration/dist', includeImages = false) { - const globs = getAllFixtureGlobs(rootDirectory, suiteDirectory); - const jsonPaths = globs[0]; - const imagePaths = globs[1]; - //Extract the filedata into a flat dictionary - const allFiles = {}; - let allPaths = glob.sync(jsonPaths); - if (includeImages) { - allPaths = allPaths.concat(glob.sync(imagePaths)); +export function generateFixtureJson(rootDirectory, suiteDirectory, outputDirectory = 'test/integration/dist', includeImages = false, stylePaths = []) { + if (!stylePaths.length) { + const pathGlob = getAllFixtureGlobs(rootDirectory, suiteDirectory)[0]; + stylePaths = glob.sync(pathGlob); + if (!stylePaths.length) { + console.error(`Found no tests matching the pattern ${pathGlob}`); + } } - for (const fixturePath of allPaths) { - const fileName = path.basename(fixturePath); - const extension = path.extname(fixturePath); + const testCases = {}; + + for (const stylePath of stylePaths) { + const dirName = path.dirname(stylePath); + const testName = dirName.replace(rootDirectory, ''); try { - if (extension === '.json') { - const json = parseJsonFromFile(fixturePath); + const json = parseJsonFromFile(stylePath); - //Special case for style json which needs some preprocessing - if (fileName === 'style.json') { - // 7357 is testem's default port - localizeURLs(json, 7357); - } + // Special case for style json which needs some preprocessing + // 7357 is testem's default port + localizeURLs(json, 7357); - allFiles[fixturePath] = json; - } else if (extension === '.png') { - allFiles[fixturePath] = true; - } else { - throw new Error(`${extension} is incompatible , file path ${fixturePath}`); + const testObject = {}; + + const filenames = fs.readdirSync(dirName); + for (const file of filenames) { + const [name, extension] = file.split("."); + if (extension === 'json') { + const json = parseJsonFromFile(path.join(dirName, file)); + //Special case for style json which needs some preprocessing + if (file === 'style.json') { + // 7357 is testem's default port + localizeURLs(json, 7357); + } + testObject[name] = json; + } else if (extension === 'png') { + if (includeImages) { + testObject[name] = true; + } + } else { + throw new Error(`${extension} is incompatible , file path ${path.join(dirName, file)}`); + } } - } catch (e) { - console.log(`Error parsing file: ${fixturePath}`); - allFiles[fixturePath] = {PARSE_ERROR: true, message: e.message}; - } - } + testCases[testName] = testObject; - // Re-nest by directory path, each directory path defines a testcase. - const result = {}; - for (const fullPath in allFiles) { - const testName = path.dirname(fullPath).replace(rootDirectory, ''); - //Lazily initialize an object to store each file wihin a particular testName - if (result[testName] == null) { - result[testName] = {}; + } catch (e) { + console.log(`Error parsing file: ${stylePath}`); + console.log(e.message); + testCases[testName] = {PARSE_ERROR: true, message: e.message}; } - //Trim extension from filename - const fileName = path.basename(fullPath, path.extname(fullPath)); - result[testName][fileName] = allFiles[fullPath]; } - const outputStr = JSON.stringify(result, null, 4); + const outputStr = JSON.stringify(testCases, null, 4); const outputFile = `${suiteDirectory.split('-')[0]}-fixtures.json`; const outputPath = path.join(outputDirectory, outputFile); diff --git a/test/integration/lib/operation-handlers.js b/test/integration/lib/operation-handlers.js index bc6e0bcb61a..ec2a7774c90 100644 --- a/test/integration/lib/operation-handlers.js +++ b/test/integration/lib/operation-handlers.js @@ -160,7 +160,7 @@ export function applyOperations(map, options) { function updateCanvas(imagePath) { return new Promise((resolve) => { const canvas = window.document.getElementById('fake-canvas'); - const ctx = canvas.getContext('2d'); + const ctx = canvas.getContext('2d', {willReadFrequently: true}); const image = new Image(); image.src = imagePath.replace('./', ''); image.onload = () => { diff --git a/test/integration/lib/query.js b/test/integration/lib/query.js index 0b0e1e147f1..dbd34d9c7ff 100644 --- a/test/integration/lib/query.js +++ b/test/integration/lib/query.js @@ -4,7 +4,7 @@ // query-fixtures.json is automatically generated before this file gets built // refer testem.js#before_tests() import fixtures from '../dist/query-fixtures.json'; -import ignores from '../../ignores.json'; +import ignores from '../../ignores/all.js'; import {applyOperations} from './operation-handlers.js'; import {deepEqual, generateDiffLog} from './json-diff.js'; import {setupHTML, updateHTML} from '../../util/html_generator.js'; @@ -33,13 +33,13 @@ for (const testName in fixtures) { function ensureTeardown(t) { const testName = t.name; const options = {timeout: 5000}; - if (testName in ignores) { - const ignoreType = ignores[testName]; - if (/^skip/.test(ignoreType)) { - options.skip = true; - } else { - options.todo = true; - } + + if (ignores.skip.includes(testName)) { + options.skip = true; + } + + if (ignores.todo.includes(testName)) { + options.todo = true; } t.test(testName, options, runTest); diff --git a/test/integration/lib/render.js b/test/integration/lib/render.js index a3bd0ad7f9f..9c689be7ed6 100644 --- a/test/integration/lib/render.js +++ b/test/integration/lib/render.js @@ -4,7 +4,10 @@ // render-fixtures.json is automatically generated before this file gets built // refer testem.js#before_tests() import fixtures from '../dist/render-fixtures.json'; -import ignores from '../../ignores.json'; +import ignores from '../../ignores/all.js'; +import ignoreWindows from '../../ignores/windows.js'; +import ignoreMac from '../../ignores/macos.js'; +import ignoreFirefox from '../../ignores/firefox.js'; import config from '../../../src/util/config.js'; import {clamp} from '../../../src/util/util.js'; import {mercatorZfromAltitude} from '../../../src/geo/mercator_coordinate.js'; @@ -50,15 +53,39 @@ tape.onFinish(() => { mapboxgl.clearPrewarmedResources(); }); +let osIgnore; +let timeout = 30000; + +if (process.env.CI) { + // On CI, MacOS and Windows run on virtual machines. + // Windows runs are especially slow so we increase the timeout. + const os = navigator.appVersion; + if (os.includes("Macintosh")) { + osIgnore = ignoreMac; + } else if (os.includes("Linux")) { + osIgnore = null; + } else if (os.includes("Windows")) { + osIgnore = ignoreWindows; + timeout = 150000; // 2:30 + } else { console.warn("Unrecognized OS:", os); } +} + +function checkIgnore(ignoreConfig, testName, options) { + if (ignoreConfig.skip.includes(testName)) { + options.skip = true; + } else if (ignoreConfig.todo.includes(testName)) { + options.todo = true; + } +} + for (const testName in fixtures) { - const options = {timeout: 20000}; - if (testName in ignores) { - const ignoreType = ignores[testName]; - if (/^skip/.test(ignoreType)) { - options.skip = true; - } else { - options.todo = true; - } + const options = {timeout}; + checkIgnore(ignores, testName, options); + if (osIgnore) { + checkIgnore(osIgnore, testName, options); + } + if (navigator.userAgent.includes("Firefox")) { + checkIgnore(ignoreFirefox, testName, options); } tape(testName, options, runTest); } @@ -82,207 +109,252 @@ function ensureTeardown() { mapboxgl.restoreNow(); } -async function runTest(t) { - t.teardown(ensureTeardown); +function parseStyle(currentFixture) { + const style = currentFixture.style; + if (!style) { + throw new Error('style.json is missing'); + } - let style, options; - // This needs to be read from the `t` object because this function runs async in a closure. - const currentTestName = t.name; - const writeFileBasePath = `test/integration/${currentTestName}`; + if (style.PARSE_ERROR) { + throw new Error(`Error occured while parsing style.json: ${style.message}`); + } + + return style; +} + +function parseOptions(currentFixture, style) { + const options = { + width: 512, + height: 512, + pixelRatio: 1, + allowed: 0.00015, + ...((style.metadata && style.metadata.test) || {}) + }; + + return options; +} + +async function setupLayout(options) { + window.devicePixelRatio = options.pixelRatio; + container.style.width = `${options.width}px`; + container.style.height = `${options.height}px`; + + if (options.addFakeCanvas) { + const {canvas, ctx} = createCanvas(options.addFakeCanvas.id); + const src = options.addFakeCanvas.image.replace('./', ''); + await drawImage(canvas, ctx, src, false); + fakeCanvasContainer.appendChild(canvas); + } +} + +async function getExpectedImages(currentTestName) { const currentFixture = fixtures[currentTestName]; - try { - style = currentFixture.style; - if (!style) { - throw new Error('style.json is missing'); - } - if (style.PARSE_ERROR) { - throw new Error(`Error occured while parsing style.json: ${style.message}`); + // there may be multiple expected images, covering different platforms + const expectedPaths = []; + for (const prop in currentFixture) { + if (prop.indexOf('expected') > -1) { + let path = `${currentTestName}/${prop}.png`; + // regression tests with # in the name need to be sanitized + path = encodeURIComponent(path); + expectedPaths.push(path); } + } - options = { - width: 512, - height: 512, - pixelRatio: 1, - allowed: 0.00015, - ...((style.metadata && style.metadata.test) || {}) - }; + // if we have multiple expected images, we'll compare against each one and pick the one with + // the least amount of difference; this is useful for covering features that render differently + // depending on platform, i.e. heatmaps use half-float textures for improved rendering where supported + const expectedImages = await Promise.all(expectedPaths.map((path) => drawImage(expectedCanvas, expectedCtx, path))); + + if (!process.env.UPDATE && expectedImages.length === 0) { + throw new Error('No expected*.png files found; did you mean to run tests with UPDATE=true?'); + } + + return expectedImages; +} - // there may be multiple expected images, covering different platforms - const expectedPaths = []; - for (const prop in currentFixture) { - if (prop.indexOf('expected') > -1) { - let path = `${currentTestName}/${prop}.png`; - // regression tests with # in the name need to be sanitized - path = encodeURIComponent(path); - expectedPaths.push(path); +async function renderMap(style, options) { + mapboxgl.accessToken = 'pk.eyJ1IjoibWFwYm94LWdsLWpzIiwiYSI6ImNram9ybGI1ajExYjQyeGxlemppb2pwYjIifQ.LGy5UGNIsXUZdYMvfYRiAQ'; + map = new mapboxgl.Map({ + container, + style, + classes: options.classes, + interactive: false, + attributionControl: false, + preserveDrawingBuffer: true, + axonometric: options.axonometric || false, + skew: options.skew || [0, 0], + fadeDuration: options.fadeDuration || 0, + optimizeForTerrain: options.optimizeForTerrain || false, + localIdeographFontFamily: options.localIdeographFontFamily || false, + projection: options.projection, + crossSourceCollisions: typeof options.crossSourceCollisions === "undefined" ? true : options.crossSourceCollisions, + performanceMetricsCollection: false, + transformRequest: (url, resourceType) => { + // some tests have the port hardcoded to 2900 + // this makes that backwards compatible + if (resourceType === 'Tile') { + const transformedUrl = new URL(url); + transformedUrl.port = '7357'; + return { + url: transformedUrl.toString() + }; } } + }); - const expectedImagePromises = Promise.all(expectedPaths.map((path) => drawImage(expectedCanvas, expectedCtx, path))); + map._authenticate = () => {}; - window.devicePixelRatio = options.pixelRatio; + // override internal timing to enable precise wait operations + window._renderTestNow = 0; + mapboxgl.setNow(window._renderTestNow); - if (options.addFakeCanvas) { - const {canvas, ctx} = createCanvas(options.addFakeCanvas.id); - const src = options.addFakeCanvas.image.replace('./', ''); - await drawImage(canvas, ctx, src, false); - fakeCanvasContainer.appendChild(canvas); - } + if (options.debug) map.showTileBoundaries = true; + if (options.showOverdrawInspector) map.showOverdrawInspector = true; + if (options.showTerrainWireframe) map.showTerrainWireframe = true; + if (options.showPadding) map.showPadding = true; + if (options.collisionDebug) map.showCollisionBoxes = true; + if (options.fadeDuration) map._isInitialLoad = false; - container.style.width = `${options.width}px`; - container.style.height = `${options.height}px`; + // Disable anisotropic filtering on render tests + map.painter.context.extTextureFilterAnisotropicForceOff = true; + // Disable globe antialiasing on render tests expect for antialiasing test + map.painter.context.extStandardDerivativesForceOff = !options.standardDerivatives; - //2. Initialize the Map - mapboxgl.accessToken = 'pk.eyJ1IjoibWFwYm94LWdsLWpzIiwiYSI6ImNram9ybGI1ajExYjQyeGxlemppb2pwYjIifQ.LGy5UGNIsXUZdYMvfYRiAQ'; - map = new mapboxgl.Map({ - container, - style, - classes: options.classes, - interactive: false, - attributionControl: false, - preserveDrawingBuffer: true, - axonometric: options.axonometric || false, - skew: options.skew || [0, 0], - fadeDuration: options.fadeDuration || 0, - optimizeForTerrain: options.optimizeForTerrain || false, - localIdeographFontFamily: options.localIdeographFontFamily || false, - projection: options.projection, - crossSourceCollisions: typeof options.crossSourceCollisions === "undefined" ? true : options.crossSourceCollisions, - performanceMetricsCollection: false, - transformRequest: (url, resourceType) => { - // some tests have the port hardcoded to 2900 - // this makes that backwards compatible - if (resourceType === 'Tile') { - const transformedUrl = new URL(url); - transformedUrl.port = '7357'; - return { - url: transformedUrl.toString() - }; - } - } - }); + map.repaint = true; + await map.once('load'); - map.repaint = true; - map._authenticate = () => {}; - - // override internal timing to enable precise wait operations - window._renderTestNow = 0; - mapboxgl.setNow(window._renderTestNow); - - if (options.debug) map.showTileBoundaries = true; - if (options.showOverdrawInspector) map.showOverdrawInspector = true; - if (options.showTerrainWireframe) map.showTerrainWireframe = true; - if (options.showPadding) map.showPadding = true; - if (options.collisionDebug) map.showCollisionBoxes = true; - if (options.fadeDuration) map._isInitialLoad = false; - - // Disable anisotropic filtering on render tests - map.painter.context.extTextureFilterAnisotropicForceOff = true; - // Disable globe antialiasing on render tests expect for antialiasing test - map.painter.context.extStandardDerivativesForceOff = !options.standardDerivatives; - - const gl = map.painter.context.gl; - await map.once('load'); - // Disable vertex morphing by default - if (map.painter.terrain) { - map.painter.terrain.useVertexMorphing = false; - } + // Disable vertex morphing by default + if (map.painter.terrain) { + map.painter.terrain.useVertexMorphing = false; + } - //3. Run the operations on the map - await applyOperations(map, options); - map.repaint = false; - const viewport = gl.getParameter(gl.VIEWPORT); - const w = viewport[2]; - const h = viewport[3]; - let actualImageData; - - // 1. get pixel data from test canvas as Uint8Array - if (options.output === "terrainDepth") { - const pixels = drawTerrainDepth(map, w, h); - if (!pixels) { - throw new Error('Failed to render terrain depth, make sure that terrain is enabled on the render test'); - } - actualImageData = Uint8ClampedArray.from(pixels); - } else { - actualCanvas.width = w; - actualCanvas.height = h; - actualCtx.drawImage(map.getCanvas(), 0, 0); - actualImageData = actualCtx.getImageData(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight).data; + // 3. Run the operations on the map + await applyOperations(map, options); + + // 4. Wait until the map is idle + map.repaint = true; + await new Promise(resolve => map._requestDomTask(resolve)); + + return map; +} + +function getViewportSize(map) { + const gl = map.painter.context.gl; + const viewport = gl.getParameter(gl.VIEWPORT); + const w = viewport[2]; + const h = viewport[3]; + return {w, h}; +} + +function getActualImageData(map, {w, h}, options) { + const gl = map.painter.context.gl; + + // 1. get pixel data from test canvas as Uint8Array + if (options.output === "terrainDepth") { + const pixels = drawTerrainDepth(map, w, h); + if (!pixels) { + throw new Error('Failed to render terrain depth, make sure that terrain is enabled on the render test'); } + return Uint8ClampedArray.from(pixels); + } - // if we have multiple expected images, we'll compare against each one and pick the one with - // the least amount of difference; this is useful for covering features that render differently - // depending on platform, i.e. heatmaps use half-float textures for improved rendering where supported - const expectedImages = await expectedImagePromises; + actualCanvas.width = w; + actualCanvas.height = h; + actualCtx.drawImage(map.getCanvas(), 0, 0); + return actualCtx.getImageData(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight).data; +} - if (!process.env.UPDATE && expectedImages.length === 0) { - throw new Error('No expected*.png files found; did you mean to run tests with UPDATE=true?'); +function getActualImageDataURL(actualImageData, map, {w, h}, options) { + if (options.output === "terrainDepth") { + actualCanvas.width = w; + actualCanvas.height = h; + const terrainDepthData = new ImageData(actualImageData, w, h); + actualCtx.putImageData(terrainDepthData, 0, 0); + return actualCanvas.toDataURL(); + } + + return map.getCanvas().toDataURL(); +} + +function calculateDiff(map, {w, h}, actualImageData, expectedImages) { + // 2. draw expected.png into a canvas and extract ImageData + let minDiffImage; + let minExpectedCanvas; + let minDiff = Infinity; + + for (let i = 0; i < expectedImages.length; i++) { + // 3. set up Uint8ClampedArray to write diff into + const diffImage = new Uint8ClampedArray(w * h * 4); + + // 4. Use pixelmatch to compare actual and expected images and write diff + // all inputs must be Uint8Array or Uint8ClampedArray + const currentDiff = pixelmatch(actualImageData, expectedImages[i].data, diffImage, w, h, {threshold: 0.1285}) / (w * h); + + if (currentDiff < minDiff) { + minDiff = currentDiff; + minDiffImage = diffImage; + minExpectedCanvas = expectedCanvas; } + } - let fileInfo; + return {minDiff, minDiffImage, minExpectedCanvas}; +} - const getActual = () => { - if (options.output === "terrainDepth") { - actualCanvas.width = w; - actualCanvas.height = h; - const terrainDepthData = new ImageData(actualImageData, w, h); - actualCtx.putImageData(terrainDepthData, 0, 0); - return actualCanvas.toDataURL(); - } - return map.getCanvas().toDataURL(); - }; +async function runTest(t) { + t.teardown(ensureTeardown); + + // This needs to be read from the `t` object because this function runs async in a closure. + const currentTestName = t.name; + const writeFileBasePath = `test/integration/${currentTestName}`; + const currentFixture = fixtures[currentTestName]; + try { + const expectedImages = await getExpectedImages(currentTestName); + + const style = parseStyle(currentFixture); + const options = parseOptions(currentFixture, style); + await setupLayout(options); + + //2. Initialize the Map + map = await renderMap(style, options); + const {w, h} = getViewportSize(map); + const actualImageData = getActualImageData(map, {w, h}, options); if (process.env.UPDATE) { - fileInfo = [ - { - path: `${writeFileBasePath}/expected.png`, - data: getActual().split(',')[1] - } - ]; - } else { - // 2. draw expected.png into a canvas and extract ImageData - let minDiffImage; - let minExpectedCanvas; - let minDiff = Infinity; - - for (let i = 0; i < expectedImages.length; i++) { - // 3. set up Uint8ClampedArray to write diff into - const diffImage = new Uint8ClampedArray(w * h * 4); - - // 4. Use pixelmatch to compare actual and expected images and write diff - // all inputs must be Uint8Array or Uint8ClampedArray - const currentDiff = pixelmatch(actualImageData, expectedImages[i].data, diffImage, w, h, {threshold: 0.1285}) / (w * h); - - if (currentDiff < minDiff) { - minDiff = currentDiff; - minDiffImage = diffImage; - minExpectedCanvas = expectedCanvas; - } - } + browserWriteFile.postMessage([{ + path: `${writeFileBasePath}/expected.png`, + data: getActualImageDataURL(actualImageData, map, {w, h}, options).split(',')[1] + }]); - const pass = minDiff <= options.allowed; - const testMetaData = { - name: currentTestName, - minDiff: Math.round(100000 * minDiff) / 100000, - status: t._todo ? 'todo' : pass ? 'passed' : 'failed' - }; - t.ok(pass || t._todo, t.name); - - // only display results locally, or on CI if it's failing - if (!process.env.CI || !pass) { - // 5. Convert diff Uint8Array to ImageData and write to canvas - // so we can get a base64 string to display the diff in the browser - diffCanvas.width = w; - diffCanvas.height = h; - const diffImageData = new ImageData(minDiffImage, w, h); - diffCtx.putImageData(diffImageData, 0, 0); - - const actual = getActual(); - const imgDiff = diffCanvas.toDataURL(); - - // 6. use browserWriteFile to write actual and diff to disk (convert image back to base64) - fileInfo = process.env.CI ? null : [ + return; + } + + const {minDiff, minDiffImage, minExpectedCanvas} = calculateDiff(map, {w, h}, actualImageData, expectedImages); + const pass = minDiff <= options.allowed; + const testMetaData = { + name: currentTestName, + minDiff: Math.round(100000 * minDiff) / 100000, + status: t._todo ? 'todo' : pass ? 'passed' : 'failed', + style: map.getStyle(), + }; + + t.ok(pass || t._todo, t.name); + + // only display results locally, or on CI if it's failing + if (!process.env.CI || !pass) { + // 5. Convert diff Uint8Array to ImageData and write to canvas + // so we can get a base64 string to display the diff in the browser + diffCanvas.width = w; + diffCanvas.height = h; + const diffImageData = new ImageData(minDiffImage, w, h); + diffCtx.putImageData(diffImageData, 0, 0); + + const actual = getActualImageDataURL(actualImageData, map, {w, h}, options); + const imgDiff = diffCanvas.toDataURL(); + + // 6. use browserWriteFile to write actual and diff to disk (convert image back to base64) + if (!process.env.CI) { + browserWriteFile.postMessage([ { path: `${writeFileBasePath}/actual.png`, data: actual.split(',')[1] @@ -291,25 +363,20 @@ async function runTest(t) { path: `${writeFileBasePath}/diff.png`, data: imgDiff.split(',')[1] } - ]; - - // 7. pass image paths to testMetaData so the UI can load them from disk - testMetaData.actual = actual; - testMetaData.expected = minExpectedCanvas.toDataURL(); - testMetaData.imgDiff = imgDiff; + ]); } - updateHTML(testMetaData); + // 7. pass image paths to testMetaData so the UI can render them + testMetaData.actual = actual; + testMetaData.expected = minExpectedCanvas.toDataURL(); + testMetaData.imgDiff = imgDiff; } - if (!process.env.CI || process.env.UPDATE) browserWriteFile.postMessage(fileInfo); - + updateHTML(testMetaData); } catch (e) { t.error(e); updateHTML({name: t.name, status:'failed', jsonDiff: e.message}); } - - t.end(); } function drawImage(canvas, ctx, src, getImageData = true) { diff --git a/test/integration/lib/server.js b/test/integration/lib/server.js index 873c87eb2a2..8babe6e16dc 100644 --- a/test/integration/lib/server.js +++ b/test/integration/lib/server.js @@ -1,35 +1,52 @@ -import path from 'path'; import fs from 'fs'; -import st from 'st'; +import path from 'path'; import {createServer} from 'http'; +import {fileURLToPath} from 'url'; +import {createRequire} from 'module'; +import st from 'st'; import localizeURLs from './localize-urls.js'; -import {fileURLToPath} from 'url'; +const require = createRequire(import.meta.url); const __dirname = fileURLToPath(new URL('.', import.meta.url)); -import {createRequire} from 'module'; -const require = createRequire(import.meta.url); +const integrationMount = st({ + index: false, + path: path.join(__dirname, '..') +}); + +const mapboxGLStylesMount = st({ + url: '/mapbox-gl-styles', + path: path.dirname(require.resolve('mapbox-gl-styles')), + passthrough: true, + index: false +}); + +const mapboxMVTFixturesMount = st({ + url: '/mvt-fixtures', + path: path.dirname(require.resolve('@mapbox/mvt-fixtures')), + passthrough: true, + index: false +}); + +const port = 2900; export default function () { - const port = 2900; - const integrationMount = st({path: path.join(__dirname, '..')}); - const mapboxGLStylesMount = st({path: path.dirname(require.resolve('mapbox-gl-styles')), url: 'mapbox-gl-styles'}); - const mapboxMVTFixturesMount = st({path: path.dirname(require.resolve('@mapbox/mvt-fixtures')), url: 'mvt-fixtures'}); const server = createServer((req, res) => { + // Write data to disk if (req.method === 'POST' && req.url === '/write-file') { let body = ''; req.on('data', (data) => { body += data; }); - req.on('end', () => { - //Write data to disk + return req.on('end', () => { const {filePath, data} = JSON.parse(body); let encoding; if (filePath.split('.')[1] !== 'json') { encoding = 'base64'; } + fs.writeFile(path.join(process.cwd(), filePath), data, encoding, () => { res.writeHead(200, {'Content-Type': 'text/html'}); res.end('ok'); diff --git a/test/integration/render-tests/combinations/circle-translucent--fill-translucent/style.json b/test/integration/render-tests/combinations/circle-translucent--fill-translucent/style.json index 860f5e25fe2..a770b65f88f 100644 --- a/test/integration/render-tests/combinations/circle-translucent--fill-translucent/style.json +++ b/test/integration/render-tests/combinations/circle-translucent--fill-translucent/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0005 } }, "center": [ diff --git a/test/integration/render-tests/combinations/fill-opaque--heatmap-translucent/style.json b/test/integration/render-tests/combinations/fill-opaque--heatmap-translucent/style.json index 248bec2977a..a6e62ac4a40 100644 --- a/test/integration/render-tests/combinations/fill-opaque--heatmap-translucent/style.json +++ b/test/integration/render-tests/combinations/fill-opaque--heatmap-translucent/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.008 } }, "center": [ diff --git a/test/integration/render-tests/combinations/fill-opaque--hillshade-translucent/style.json b/test/integration/render-tests/combinations/fill-opaque--hillshade-translucent/style.json index 4636606b968..52369253d00 100644 --- a/test/integration/render-tests/combinations/fill-opaque--hillshade-translucent/style.json +++ b/test/integration/render-tests/combinations/fill-opaque--hillshade-translucent/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.002 } }, "center": [ diff --git a/test/integration/render-tests/combinations/fill-translucent--hillshade-translucent/style.json b/test/integration/render-tests/combinations/fill-translucent--hillshade-translucent/style.json index d3d62603b6e..d995d5bab48 100644 --- a/test/integration/render-tests/combinations/fill-translucent--hillshade-translucent/style.json +++ b/test/integration/render-tests/combinations/fill-translucent--hillshade-translucent/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0003 } }, "center": [ diff --git a/test/integration/render-tests/combinations/heatmap-translucent--fill-opaque/style.json b/test/integration/render-tests/combinations/heatmap-translucent--fill-opaque/style.json index 073955047f8..2c788d24d17 100644 --- a/test/integration/render-tests/combinations/heatmap-translucent--fill-opaque/style.json +++ b/test/integration/render-tests/combinations/heatmap-translucent--fill-opaque/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.008 } }, "center": [ diff --git a/test/integration/render-tests/combinations/heatmap-translucent--fill-translucent/style.json b/test/integration/render-tests/combinations/heatmap-translucent--fill-translucent/style.json index 0863f0426be..39cffab0af3 100644 --- a/test/integration/render-tests/combinations/heatmap-translucent--fill-translucent/style.json +++ b/test/integration/render-tests/combinations/heatmap-translucent--fill-translucent/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.008 } }, "center": [ diff --git a/test/integration/render-tests/combinations/hillshade-translucent--fill-opaque/style.json b/test/integration/render-tests/combinations/hillshade-translucent--fill-opaque/style.json index d741e845901..78cfbdb3560 100644 --- a/test/integration/render-tests/combinations/hillshade-translucent--fill-opaque/style.json +++ b/test/integration/render-tests/combinations/hillshade-translucent--fill-opaque/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.002 } }, "center": [ diff --git a/test/integration/render-tests/combinations/hillshade-translucent--fill-translucent/style.json b/test/integration/render-tests/combinations/hillshade-translucent--fill-translucent/style.json index dda0015709b..1fca31f1ec8 100644 --- a/test/integration/render-tests/combinations/hillshade-translucent--fill-translucent/style.json +++ b/test/integration/render-tests/combinations/hillshade-translucent--fill-translucent/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.004 } }, "center": [ diff --git a/test/integration/render-tests/combinations/raster-translucent--fill-translucent/style.json b/test/integration/render-tests/combinations/raster-translucent--fill-translucent/style.json index 55bb422f3e6..3e2d54ce13e 100644 --- a/test/integration/render-tests/combinations/raster-translucent--fill-translucent/style.json +++ b/test/integration/render-tests/combinations/raster-translucent--fill-translucent/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.008 } }, "center": [ diff --git a/test/integration/render-tests/custom-layer-js/tent-3d/style.json b/test/integration/render-tests/custom-layer-js/tent-3d/style.json index 835d307134b..66ed96b2afe 100644 --- a/test/integration/render-tests/custom-layer-js/tent-3d/style.json +++ b/test/integration/render-tests/custom-layer-js/tent-3d/style.json @@ -3,6 +3,7 @@ "metadata": { "test": { "height": 256, + "allowed": 0.00025, "operations": [ [ "addCustomLayer", diff --git a/test/integration/render-tests/debug/collision-fractional-zoom/style.json b/test/integration/render-tests/debug/collision-fractional-zoom/style.json index eb622768b11..dd8c2b5e1dd 100644 --- a/test/integration/render-tests/debug/collision-fractional-zoom/style.json +++ b/test/integration/render-tests/debug/collision-fractional-zoom/style.json @@ -4,7 +4,7 @@ "test": { "collisionDebug": true, "height": 256, - "allowed": 0.0004 + "allowed": 0.003 } }, "center": [ diff --git a/test/integration/render-tests/debug/collision-high-pitched-wrapped-fractional-zoom/style.json b/test/integration/render-tests/debug/collision-high-pitched-wrapped-fractional-zoom/style.json index 5937d27ef1e..2e673452801 100644 --- a/test/integration/render-tests/debug/collision-high-pitched-wrapped-fractional-zoom/style.json +++ b/test/integration/render-tests/debug/collision-high-pitched-wrapped-fractional-zoom/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "width": 500, "height": 500, - "allowed": 0.005 + "allowed": 0.01 } }, "center": [ diff --git a/test/integration/render-tests/debug/collision-overscaled-fractional-zoom/style.json b/test/integration/render-tests/debug/collision-overscaled-fractional-zoom/style.json index db23c8f3c93..59ecc1dcf0d 100644 --- a/test/integration/render-tests/debug/collision-overscaled-fractional-zoom/style.json +++ b/test/integration/render-tests/debug/collision-overscaled-fractional-zoom/style.json @@ -3,7 +3,9 @@ "metadata": { "test": { "collisionDebug": true, - "height": 256 + "height": 256, + "allowed": 0.00004 + } }, "center": [ diff --git a/test/integration/render-tests/debug/collision-pitched-wrapped-fractional-zoom/style.json b/test/integration/render-tests/debug/collision-pitched-wrapped-fractional-zoom/style.json index 941ee35553f..a0f1260cf07 100644 --- a/test/integration/render-tests/debug/collision-pitched-wrapped-fractional-zoom/style.json +++ b/test/integration/render-tests/debug/collision-pitched-wrapped-fractional-zoom/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "width": 500, "height": 500, - "allowed": 0.005 + "allowed": 0.01 } }, "center": [ diff --git a/test/integration/render-tests/debug/collision-pitched-wrapped/expected.png b/test/integration/render-tests/debug/collision-pitched-wrapped/expected.png index 2ffc01ff310..2bdc70b7809 100644 Binary files a/test/integration/render-tests/debug/collision-pitched-wrapped/expected.png and b/test/integration/render-tests/debug/collision-pitched-wrapped/expected.png differ diff --git a/test/integration/render-tests/debug/collision-pitched-wrapped/style.json b/test/integration/render-tests/debug/collision-pitched-wrapped/style.json index eb29afbe093..ae9a930e920 100644 --- a/test/integration/render-tests/debug/collision-pitched-wrapped/style.json +++ b/test/integration/render-tests/debug/collision-pitched-wrapped/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "width": 500, "height": 500, - "allowed": 0.005 + "allowed": 0.009 } }, "center": [ diff --git a/test/integration/render-tests/debug/collision/style.json b/test/integration/render-tests/debug/collision/style.json index 867fbc6483b..80555db2952 100644 --- a/test/integration/render-tests/debug/collision/style.json +++ b/test/integration/render-tests/debug/collision/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "collisionDebug": true, - "height": 256 + "height": 256, + "allowed": 0.017 } }, "center": [ diff --git a/test/integration/render-tests/debug/terrain/camera-under-terrain/style.json b/test/integration/render-tests/debug/terrain/camera-under-terrain/style.json index 1eafce26211..7cf3bbbb8c8 100644 --- a/test/integration/render-tests/debug/terrain/camera-under-terrain/style.json +++ b/test/integration/render-tests/debug/terrain/camera-under-terrain/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0005 } }, "center": [ diff --git a/test/integration/render-tests/debug/terrain/collision-occlusion/style.json b/test/integration/render-tests/debug/terrain/collision-occlusion/style.json index 2bc0e0b0dc2..6326826384f 100644 --- a/test/integration/render-tests/debug/terrain/collision-occlusion/style.json +++ b/test/integration/render-tests/debug/terrain/collision-occlusion/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "height": 160, "width": 160, - "allowed": 0.0005, + "allowed": 0.005, "description": "Collision debug and occlusion test for text and icons.", "operations": [ ["wait", 1500] diff --git a/test/integration/render-tests/debug/terrain/collision-pitch-with-map-text-and-icon/style.json b/test/integration/render-tests/debug/terrain/collision-pitch-with-map-text-and-icon/style.json index 52602ba5a53..0be2e0ec969 100644 --- a/test/integration/render-tests/debug/terrain/collision-pitch-with-map-text-and-icon/style.json +++ b/test/integration/render-tests/debug/terrain/collision-pitch-with-map-text-and-icon/style.json @@ -3,6 +3,7 @@ "metadata": { "test": { "height": 256, + "allowed": 0.0025, "collisionDebug": true, "operations": [ ["wait"], diff --git a/test/integration/render-tests/debug/terrain/collision-pitch-with-map/style.json b/test/integration/render-tests/debug/terrain/collision-pitch-with-map/style.json index a85aa58a791..ea4de443f08 100644 --- a/test/integration/render-tests/debug/terrain/collision-pitch-with-map/style.json +++ b/test/integration/render-tests/debug/terrain/collision-pitch-with-map/style.json @@ -3,6 +3,7 @@ "metadata": { "test": { "height": 256, + "allowed": 0.001, "collisionDebug": true, "operations": [ ["wait"], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-high-pitch/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-high-pitch/style.json index 6bba706be14..fe8aea720e5 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-high-pitch/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-high-pitch/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json index da6d72d3a80..2d53137ee9e 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, - "operations": [["wait"]] + "allowed": 0.0003 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/late-dynamic/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/late-dynamic/style.json index d2e832e11d6..1f9fad68a1a 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/late-dynamic/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance-data-driven/late-dynamic/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/high-pitch-far-hidden/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/high-pitch-far-hidden/style.json index ce2df982d3d..85ab10cb4ee 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/high-pitch-far-hidden/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/high-pitch-far-hidden/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/low-pitch-far-visible/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/low-pitch-far-visible/style.json index fd53adce3a1..218e3e9dee6 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/low-pitch-far-visible/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/combined-pitch-distance/low-pitch-far-visible/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/distance-far-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/distance-far-cull/style.json index 31c8877aee0..4dac16aa7b0 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/distance-far-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/distance-far-cull/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-and-far-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-and-far-cull/style.json index f18d9b2f8ee..9cc4828aa07 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-and-far-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-and-far-cull/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, - "operations": [["wait"]] + "allowed": 0.0012 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-cull/style.json index 33bc7bc6ea7..d20be851955 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/distance-near-cull/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, - "operations": [["wait"]] + "allowed": 0.00062 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/distance-nofilter/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/distance-nofilter/style.json index 6a7b9a8c537..4700a930f55 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/distance-nofilter/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/distance-nofilter/style.json @@ -5,7 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, - "operations": [["wait"]] + "allowed": 0.0007 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-cull/style.json index 1a287fbcfa8..45fc7ec144d 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-cull/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-show/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-show/style.json index 95bf64feb05..b8490dbb487 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-show/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-high-show/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-cull/style.json index 9124618ac7c..f32625989bf 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-cull/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-show/style.json b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-show/style.json index 3cb0846e8e1..e4eb6e71873 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-show/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/line/pitch-low-show/style.json @@ -4,8 +4,7 @@ "test": { "collisionDebug": true, "height": 264, - "width": 400, - "operations": [["wait"]] + "width": 400 } }, "center": [-120.30344797631889, 38.11726797649675], diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json index 35d27a015ea..ef6003497e7 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/early-dynamic-low-pitch/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.0005, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/late-dynamic/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/late-dynamic/style.json index f4e4b6d9998..0b759000824 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/late-dynamic/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance-data-driven/late-dynamic/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.0007, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/high-pitch-far-hidden/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/high-pitch-far-hidden/style.json index d1f34b01ab7..2ae69b720fd 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/high-pitch-far-hidden/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/high-pitch-far-hidden/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.00025, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/low-pitch-far-visible/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/low-pitch-far-visible/style.json index 42a5a8f1de3..f38e1079e09 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/low-pitch-far-visible/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/combined-pitch-distance/low-pitch-far-visible/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.0016, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/distance-far-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/distance-far-cull/style.json index 8b41822e30c..44184a7ae9f 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/distance-far-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/distance-far-cull/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.0008, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-and-far-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-and-far-cull/style.json index 52dbe81644c..9cdefd430c8 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-and-far-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-and-far-cull/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.006, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-cull/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-cull/style.json index 2e3b811999f..96c97350019 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-cull/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/distance-near-cull/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.006, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/distance-nofilter/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/distance-nofilter/style.json index 22fbb072400..e52baafd047 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/distance-nofilter/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/distance-nofilter/style.json @@ -5,6 +5,7 @@ "collisionDebug": true, "height": 264, "width": 400, + "allowed": 0.007, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/dynamic-filter/symbols/point/pitch-high-show/style.json b/test/integration/render-tests/dynamic-filter/symbols/point/pitch-high-show/style.json index 6203b9c0942..79e43ebd5fd 100644 --- a/test/integration/render-tests/dynamic-filter/symbols/point/pitch-high-show/style.json +++ b/test/integration/render-tests/dynamic-filter/symbols/point/pitch-high-show/style.json @@ -4,6 +4,7 @@ "test": { "height": 264, "width": 400, + "allowed": 0.0002, "operations": [["wait"]] } }, diff --git a/test/integration/render-tests/fill-extrusion-height/property-function/style.json b/test/integration/render-tests/fill-extrusion-height/property-function/style.json index ac194913524..cb7f8797497 100644 --- a/test/integration/render-tests/fill-extrusion-height/property-function/style.json +++ b/test/integration/render-tests/fill-extrusion-height/property-function/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.00016 } }, "sources": { diff --git a/test/integration/render-tests/fill-extrusion-height/z-fighting/style.json b/test/integration/render-tests/fill-extrusion-height/z-fighting/style.json index 786031bf55e..c0e5ec476ca 100644 --- a/test/integration/render-tests/fill-extrusion-height/z-fighting/style.json +++ b/test/integration/render-tests/fill-extrusion-height/z-fighting/style.json @@ -3,6 +3,7 @@ "metadata": { "test": { "height": 512, + "allowed": 0.008, "operations": [ ["wait"], ["setPaintProperty", "extrusion", "fill-extrusion-ambient-occlusion-intensity", 0.001], diff --git a/test/integration/render-tests/fill-extrusion-opacity/default/style.json b/test/integration/render-tests/fill-extrusion-opacity/default/style.json index 805dfadc462..8288b21a3fd 100644 --- a/test/integration/render-tests/fill-extrusion-opacity/default/style.json +++ b/test/integration/render-tests/fill-extrusion-opacity/default/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.0002 } }, "sources": { diff --git a/test/integration/render-tests/fill-extrusion-opacity/function/style.json b/test/integration/render-tests/fill-extrusion-opacity/function/style.json index e215f0d115a..165df75fb78 100644 --- a/test/integration/render-tests/fill-extrusion-opacity/function/style.json +++ b/test/integration/render-tests/fill-extrusion-opacity/function/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.0003 } }, "sources": { diff --git a/test/integration/render-tests/fill-extrusion-opacity/literal/style.json b/test/integration/render-tests/fill-extrusion-opacity/literal/style.json index 6b30980aba7..dcd15a93edd 100644 --- a/test/integration/render-tests/fill-extrusion-opacity/literal/style.json +++ b/test/integration/render-tests/fill-extrusion-opacity/literal/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.0003 } }, "sources": { diff --git a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-ao/style.json b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-ao/style.json index 29a4e229a5b..6d8543fed25 100644 --- a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-ao/style.json +++ b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-ao/style.json @@ -3,6 +3,7 @@ "metadata": { "test": { "height": 256, + "allowed": 0.0006, "operations": [ ["setLight", {"intensity": 0.2}], ["wait"] diff --git a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-gl-js#11516/style.json b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-gl-js#11516/style.json index 7d52410c254..de3ab25a7b1 100644 --- a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-gl-js#11516/style.json +++ b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-gl-js#11516/style.json @@ -5,6 +5,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0005, "operations": [ ["wait"] ] diff --git a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom-zoomin/style.json b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom-zoomin/style.json index a828a8f3ec1..62b869b13d7 100644 --- a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom-zoomin/style.json +++ b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom-zoomin/style.json @@ -5,6 +5,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0008, "operations": [ ["wait"], [ diff --git a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom/style.json b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom/style.json index 8471fe91944..6201037cec0 100644 --- a/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom/style.json +++ b/test/integration/render-tests/fill-extrusion-terrain/flat-roof-over-border-of-different-zoom/style.json @@ -5,6 +5,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0007, "operations": [ ["wait"] ] diff --git a/test/integration/render-tests/fill-extrusion-terrain/flat-roof/style.json b/test/integration/render-tests/fill-extrusion-terrain/flat-roof/style.json index 041e6099b17..cc5cf9314c6 100644 --- a/test/integration/render-tests/fill-extrusion-terrain/flat-roof/style.json +++ b/test/integration/render-tests/fill-extrusion-terrain/flat-roof/style.json @@ -4,6 +4,7 @@ "description": "Verifies there is no assert mapbox-gl-js-internal#134", "test": { "height": 256, + "allowed": 0.0003, "operations": [ ["wait"] ] diff --git a/test/integration/render-tests/fill-extrusion-terrain/globe-ao/style.json b/test/integration/render-tests/fill-extrusion-terrain/globe-ao/style.json index ef69f92c439..279b374cc91 100644 --- a/test/integration/render-tests/fill-extrusion-terrain/globe-ao/style.json +++ b/test/integration/render-tests/fill-extrusion-terrain/globe-ao/style.json @@ -5,6 +5,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0004, "operations": [ ["wait"] ] diff --git a/test/integration/render-tests/fill-extrusion-vertical-gradient/default/style.json b/test/integration/render-tests/fill-extrusion-vertical-gradient/default/style.json index 405797c646a..d5262899461 100644 --- a/test/integration/render-tests/fill-extrusion-vertical-gradient/default/style.json +++ b/test/integration/render-tests/fill-extrusion-vertical-gradient/default/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.006 } }, "light": { diff --git a/test/integration/render-tests/fill-extrusion-vertical-gradient/false/style.json b/test/integration/render-tests/fill-extrusion-vertical-gradient/false/style.json index 7b7d5a6af3b..854a09b759b 100644 --- a/test/integration/render-tests/fill-extrusion-vertical-gradient/false/style.json +++ b/test/integration/render-tests/fill-extrusion-vertical-gradient/false/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.006 } }, "light": { diff --git a/test/integration/render-tests/filter/in/style.json b/test/integration/render-tests/filter/in/style.json index 67c01c41248..bab94c1e52f 100644 --- a/test/integration/render-tests/filter/in/style.json +++ b/test/integration/render-tests/filter/in/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0005 } }, "sources": { diff --git a/test/integration/render-tests/fog/2d/fill-pattern/style.json b/test/integration/render-tests/fog/2d/fill-pattern/style.json index a544b4e2ee1..26518b3a68f 100644 --- a/test/integration/render-tests/fog/2d/fill-pattern/style.json +++ b/test/integration/render-tests/fog/2d/fill-pattern/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0003 } }, "pitch": 70, diff --git a/test/integration/render-tests/fog/2d/heatmap/expected.png b/test/integration/render-tests/fog/2d/heatmap/expected.png index 6a15018c94f..98424b95efc 100644 Binary files a/test/integration/render-tests/fog/2d/heatmap/expected.png and b/test/integration/render-tests/fog/2d/heatmap/expected.png differ diff --git a/test/integration/render-tests/globe/globe-antialiasing/high-exaggeration/style.json b/test/integration/render-tests/globe/globe-antialiasing/high-exaggeration/style.json index 9da846fbca2..3be9cdc17b0 100644 --- a/test/integration/render-tests/globe/globe-antialiasing/high-exaggeration/style.json +++ b/test/integration/render-tests/globe/globe-antialiasing/high-exaggeration/style.json @@ -5,6 +5,7 @@ "standardDerivatives": true, "width": 512, "height": 512, + "allowed": 0.0002, "operations": [ ["setProjection", "globe"], ["wait"] diff --git a/test/integration/render-tests/globe/globe-circle/change-projection/set-projection/style.json b/test/integration/render-tests/globe/globe-circle/change-projection/set-projection/style.json index 4a5d2d5e7ac..546f9ba1710 100644 --- a/test/integration/render-tests/globe/globe-circle/change-projection/set-projection/style.json +++ b/test/integration/render-tests/globe/globe-circle/change-projection/set-projection/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.00025, "operations": [ ["setProjection", "globe"], ["wait"], diff --git a/test/integration/render-tests/globe/globe-circle/change-projection/set-style/style.json b/test/integration/render-tests/globe/globe-circle/change-projection/set-style/style.json index c25f06e0348..191edd379b1 100644 --- a/test/integration/render-tests/globe/globe-circle/change-projection/set-style/style.json +++ b/test/integration/render-tests/globe/globe-circle/change-projection/set-style/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.00025, "operations": [ ["setStyle", { "version": 8, diff --git a/test/integration/render-tests/globe/globe-circle/near-transition/style.json b/test/integration/render-tests/globe/globe-circle/near-transition/style.json index 030606471a0..fe789490b1e 100644 --- a/test/integration/render-tests/globe/globe-circle/near-transition/style.json +++ b/test/integration/render-tests/globe/globe-circle/near-transition/style.json @@ -4,7 +4,7 @@ "test": { "height": 256, "width": 256, - "allowed": 0.0002, + "allowed": 0.00045, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-fill-extrusion/opacity/style.json b/test/integration/render-tests/globe/globe-fill-extrusion/opacity/style.json index be80be04da2..9e40174f268 100644 --- a/test/integration/render-tests/globe/globe-fill-extrusion/opacity/style.json +++ b/test/integration/render-tests/globe/globe-fill-extrusion/opacity/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.00017, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-fill-extrusion/stack-interleaved-layers/style.json b/test/integration/render-tests/globe/globe-fill-extrusion/stack-interleaved-layers/style.json index 8d26872348a..9774f92eedd 100644 --- a/test/integration/render-tests/globe/globe-fill-extrusion/stack-interleaved-layers/style.json +++ b/test/integration/render-tests/globe/globe-fill-extrusion/stack-interleaved-layers/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.0003, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-fill-extrusion/vertical_gradient/style.json b/test/integration/render-tests/globe/globe-fill-extrusion/vertical_gradient/style.json index 0e778595713..4c37b983178 100644 --- a/test/integration/render-tests/globe/globe-fill-extrusion/vertical_gradient/style.json +++ b/test/integration/render-tests/globe/globe-fill-extrusion/vertical_gradient/style.json @@ -3,6 +3,7 @@ "metadata": { "test": { "height": 256, + "allowed": 0.0004, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-angle-limit/style.json b/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-angle-limit/style.json index 8597e5d0e18..2407af3f968 100644 --- a/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-angle-limit/style.json +++ b/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-angle-limit/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.0004, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-complex/style.json b/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-complex/style.json index 0ca4f1249fc..9a8aece35ff 100644 --- a/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-complex/style.json +++ b/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-complex/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.002, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-simple/style.json b/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-simple/style.json index 7f8857c6c73..4c0326ee6fb 100644 --- a/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-simple/style.json +++ b/test/integration/render-tests/globe/globe-fill-extrusion/zero-height-clipping-simple/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.002, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-heatmap/near-transition/style.json b/test/integration/render-tests/globe/globe-heatmap/near-transition/style.json index e44d3babcb3..d4912ce89bd 100644 --- a/test/integration/render-tests/globe/globe-heatmap/near-transition/style.json +++ b/test/integration/render-tests/globe/globe-heatmap/near-transition/style.json @@ -4,6 +4,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.00025, "operations": [ ["setProjection", "globe"], [ "wait" ] diff --git a/test/integration/render-tests/globe/globe-set-style/default/style.json b/test/integration/render-tests/globe/globe-set-style/default/style.json index 6e5291e8537..0a3eb52143a 100644 --- a/test/integration/render-tests/globe/globe-set-style/default/style.json +++ b/test/integration/render-tests/globe/globe-set-style/default/style.json @@ -4,6 +4,7 @@ "test": { "height": 256, "width": 256, + "allowed": 0.0004, "operations": [ ["setStyle", { "version": 8, diff --git a/test/integration/render-tests/globe/globe-set-style/symbol/style.json b/test/integration/render-tests/globe/globe-set-style/symbol/style.json index 6798a0146b6..e1f9db4ca26 100644 --- a/test/integration/render-tests/globe/globe-set-style/symbol/style.json +++ b/test/integration/render-tests/globe/globe-set-style/symbol/style.json @@ -4,6 +4,7 @@ "test": { "width": 256, "height": 256, + "allowed": 0.0004, "operations": [ [ "setStyle", diff --git a/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping-allow-overlap/style.json b/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping-allow-overlap/style.json index fa713bdec4c..35d6b013cf3 100644 --- a/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping-allow-overlap/style.json +++ b/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping-allow-overlap/style.json @@ -4,6 +4,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0003, "operations": [ ["setProjection", "globe"], ["wait"] diff --git a/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping/style.json b/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping/style.json index 1a62164147d..4757bb8f8e4 100644 --- a/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping/style.json +++ b/test/integration/render-tests/globe/globe-symbols/globe-edge-clipping/style.json @@ -4,6 +4,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0003, "operations": [ ["setProjection", "globe"], ["wait"] diff --git a/test/integration/render-tests/globe/globe-symbols/icon-allow-overlap/style.json b/test/integration/render-tests/globe/globe-symbols/icon-allow-overlap/style.json index 53273a6dcac..1708c395599 100644 --- a/test/integration/render-tests/globe/globe-symbols/icon-allow-overlap/style.json +++ b/test/integration/render-tests/globe/globe-symbols/icon-allow-overlap/style.json @@ -4,6 +4,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0003, "operations": [ ["setProjection", "globe"], ["wait"] diff --git a/test/integration/render-tests/globe/globe-symbols/icon-viewport-align-rotate-with-map/style.json b/test/integration/render-tests/globe/globe-symbols/icon-viewport-align-rotate-with-map/style.json index 440dc4df30e..15b700e4653 100644 --- a/test/integration/render-tests/globe/globe-symbols/icon-viewport-align-rotate-with-map/style.json +++ b/test/integration/render-tests/globe/globe-symbols/icon-viewport-align-rotate-with-map/style.json @@ -4,6 +4,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0002, "operations": [ ["setProjection", "globe"], ["wait"] diff --git a/test/integration/render-tests/globe/globe-symbols/set-style/style.json b/test/integration/render-tests/globe/globe-symbols/set-style/style.json index a3d13e060a9..a6a8f053641 100644 --- a/test/integration/render-tests/globe/globe-symbols/set-style/style.json +++ b/test/integration/render-tests/globe/globe-symbols/set-style/style.json @@ -6,6 +6,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0003, "operations": [ ["setProjection", "globe"], ["wait"], diff --git a/test/integration/render-tests/globe/globe-symbols/text-allow-overlap/style.json b/test/integration/render-tests/globe/globe-symbols/text-allow-overlap/style.json index 46f7b74ec57..ce41386bfb6 100644 --- a/test/integration/render-tests/globe/globe-symbols/text-allow-overlap/style.json +++ b/test/integration/render-tests/globe/globe-symbols/text-allow-overlap/style.json @@ -4,6 +4,7 @@ "test": { "height": 512, "width": 512, + "allowed": 0.0003, "operations": [ ["setProjection", "globe"], ["wait"] diff --git a/test/integration/render-tests/globe/globe-transition/bearing/expected.png b/test/integration/render-tests/globe/globe-transition/bearing/expected.png index 0b9a27e6b11..a89fddf7fcb 100644 Binary files a/test/integration/render-tests/globe/globe-transition/bearing/expected.png and b/test/integration/render-tests/globe/globe-transition/bearing/expected.png differ diff --git a/test/integration/render-tests/globe/globe-transition/bearing/style.json b/test/integration/render-tests/globe/globe-transition/bearing/style.json index 951719216b3..4d8e9af0ec8 100644 --- a/test/integration/render-tests/globe/globe-transition/bearing/style.json +++ b/test/integration/render-tests/globe/globe-transition/bearing/style.json @@ -4,8 +4,7 @@ "test": { "width": 512, "height": 512, - "debug": true, - "allowed": 0.004 + "allowed": 0.0011 } }, "center": [3, 49], diff --git a/test/integration/render-tests/globe/globe-transition/ocean-artifact/expected.png b/test/integration/render-tests/globe/globe-transition/ocean-artifact/expected.png index 559c8500ce3..530971bdf9f 100644 Binary files a/test/integration/render-tests/globe/globe-transition/ocean-artifact/expected.png and b/test/integration/render-tests/globe/globe-transition/ocean-artifact/expected.png differ diff --git a/test/integration/render-tests/globe/globe-transition/ocean-artifact/style.json b/test/integration/render-tests/globe/globe-transition/ocean-artifact/style.json index c14a99070db..416dba088dd 100644 --- a/test/integration/render-tests/globe/globe-transition/ocean-artifact/style.json +++ b/test/integration/render-tests/globe/globe-transition/ocean-artifact/style.json @@ -4,8 +4,7 @@ "test": { "width": 512, "height": 512, - "debug": true, - "allowed": 0.002 + "allowed": 0.0003 } }, "center": [-80, 30], diff --git a/test/integration/render-tests/globe/globe-transition/pitch/expected.png b/test/integration/render-tests/globe/globe-transition/pitch/expected.png index dc293afb623..c0468d0c96e 100644 Binary files a/test/integration/render-tests/globe/globe-transition/pitch/expected.png and b/test/integration/render-tests/globe/globe-transition/pitch/expected.png differ diff --git a/test/integration/render-tests/globe/globe-transition/pitch/style.json b/test/integration/render-tests/globe/globe-transition/pitch/style.json index 935ce44d337..742be93b48a 100644 --- a/test/integration/render-tests/globe/globe-transition/pitch/style.json +++ b/test/integration/render-tests/globe/globe-transition/pitch/style.json @@ -5,7 +5,7 @@ "width": 512, "height": 768, "debug": true, - "allowed": 0.004 + "allowed": 0.016 } }, "center": [0, 48.7], diff --git a/test/integration/render-tests/globe/globe-transition/tile-corners-outside/expected.png b/test/integration/render-tests/globe/globe-transition/tile-corners-outside/expected.png index f2e88875ce1..cb74fa9d112 100644 Binary files a/test/integration/render-tests/globe/globe-transition/tile-corners-outside/expected.png and b/test/integration/render-tests/globe/globe-transition/tile-corners-outside/expected.png differ diff --git a/test/integration/render-tests/globe/globe-transition/tile-corners-outside/style.json b/test/integration/render-tests/globe/globe-transition/tile-corners-outside/style.json index 34565c66787..a5d6b64ed76 100644 --- a/test/integration/render-tests/globe/globe-transition/tile-corners-outside/style.json +++ b/test/integration/render-tests/globe/globe-transition/tile-corners-outside/style.json @@ -4,7 +4,7 @@ "test": { "width": 512, "height": 512, - "allowed": 0.001 + "allowed": 0.0015 } }, "center": [22, 78.9], diff --git a/test/integration/render-tests/globe/globe-transition/tile-edges/expected.png b/test/integration/render-tests/globe/globe-transition/tile-edges/expected.png index eaabcbb05bb..97dc51838b3 100644 Binary files a/test/integration/render-tests/globe/globe-transition/tile-edges/expected.png and b/test/integration/render-tests/globe/globe-transition/tile-edges/expected.png differ diff --git a/test/integration/render-tests/globe/globe-transition/tile-edges/style.json b/test/integration/render-tests/globe/globe-transition/tile-edges/style.json index aee36862ae1..80367cce46b 100644 --- a/test/integration/render-tests/globe/globe-transition/tile-edges/style.json +++ b/test/integration/render-tests/globe/globe-transition/tile-edges/style.json @@ -3,9 +3,7 @@ "metadata": { "test": { "width": 640, - "height": 512, - "debug": true, - "allowed": 0.004 + "height": 512 } }, "center": [5.44, 46.42], diff --git a/test/integration/render-tests/globe/globe-transition/transition-fill/expected.png b/test/integration/render-tests/globe/globe-transition/transition-fill/expected.png index f395df50169..eadf1463088 100644 Binary files a/test/integration/render-tests/globe/globe-transition/transition-fill/expected.png and b/test/integration/render-tests/globe/globe-transition/transition-fill/expected.png differ diff --git a/test/integration/render-tests/globe/globe-transition/transition-fill/style.json b/test/integration/render-tests/globe/globe-transition/transition-fill/style.json index 57b5609132f..a37783c122f 100644 --- a/test/integration/render-tests/globe/globe-transition/transition-fill/style.json +++ b/test/integration/render-tests/globe/globe-transition/transition-fill/style.json @@ -3,9 +3,7 @@ "metadata": { "test": { "width": 512, - "height": 512, - "debug": true, - "allowed": 0.004 + "height": 512 } }, "center": [0.236, 42.762], diff --git a/test/integration/render-tests/heatmap-color/default/expected.png b/test/integration/render-tests/heatmap-color/default/expected.png index deb0f101d73..345bf60d346 100644 Binary files a/test/integration/render-tests/heatmap-color/default/expected.png and b/test/integration/render-tests/heatmap-color/default/expected.png differ diff --git a/test/integration/render-tests/heatmap-color/expression/expected.png b/test/integration/render-tests/heatmap-color/expression/expected.png index deb0f101d73..345bf60d346 100644 Binary files a/test/integration/render-tests/heatmap-color/expression/expected.png and b/test/integration/render-tests/heatmap-color/expression/expected.png differ diff --git a/test/integration/render-tests/heatmap-intensity/default/expected.png b/test/integration/render-tests/heatmap-intensity/default/expected.png index 563ee0e8ab8..f54897bfbfb 100644 Binary files a/test/integration/render-tests/heatmap-intensity/default/expected.png and b/test/integration/render-tests/heatmap-intensity/default/expected.png differ diff --git a/test/integration/render-tests/heatmap-intensity/function/expected.png b/test/integration/render-tests/heatmap-intensity/function/expected.png index 51b027a8c95..07457392e3c 100644 Binary files a/test/integration/render-tests/heatmap-intensity/function/expected.png and b/test/integration/render-tests/heatmap-intensity/function/expected.png differ diff --git a/test/integration/render-tests/heatmap-intensity/literal/expected.png b/test/integration/render-tests/heatmap-intensity/literal/expected.png index 5891e360d5c..b17f23ae78f 100644 Binary files a/test/integration/render-tests/heatmap-intensity/literal/expected.png and b/test/integration/render-tests/heatmap-intensity/literal/expected.png differ diff --git a/test/integration/render-tests/heatmap-opacity/default/expected.png b/test/integration/render-tests/heatmap-opacity/default/expected.png index 7f0ba0a9a80..9c6a123ba8f 100644 Binary files a/test/integration/render-tests/heatmap-opacity/default/expected.png and b/test/integration/render-tests/heatmap-opacity/default/expected.png differ diff --git a/test/integration/render-tests/heatmap-opacity/function/expected.png b/test/integration/render-tests/heatmap-opacity/function/expected.png index 8b369583118..19062c59dc4 100644 Binary files a/test/integration/render-tests/heatmap-opacity/function/expected.png and b/test/integration/render-tests/heatmap-opacity/function/expected.png differ diff --git a/test/integration/render-tests/heatmap-opacity/literal/expected.png b/test/integration/render-tests/heatmap-opacity/literal/expected.png index 70cc004c240..fce28e13887 100644 Binary files a/test/integration/render-tests/heatmap-opacity/literal/expected.png and b/test/integration/render-tests/heatmap-opacity/literal/expected.png differ diff --git a/test/integration/render-tests/heatmap-radius/antimeridian/expected.png b/test/integration/render-tests/heatmap-radius/antimeridian/expected.png index 32b15d283d5..a97b1fba903 100644 Binary files a/test/integration/render-tests/heatmap-radius/antimeridian/expected.png and b/test/integration/render-tests/heatmap-radius/antimeridian/expected.png differ diff --git a/test/integration/render-tests/heatmap-radius/data-expression/expected.png b/test/integration/render-tests/heatmap-radius/data-expression/expected.png index 40a057f59da..70f69125e83 100644 Binary files a/test/integration/render-tests/heatmap-radius/data-expression/expected.png and b/test/integration/render-tests/heatmap-radius/data-expression/expected.png differ diff --git a/test/integration/render-tests/heatmap-radius/default/expected.png b/test/integration/render-tests/heatmap-radius/default/expected.png index 563ee0e8ab8..f54897bfbfb 100644 Binary files a/test/integration/render-tests/heatmap-radius/default/expected.png and b/test/integration/render-tests/heatmap-radius/default/expected.png differ diff --git a/test/integration/render-tests/heatmap-radius/function/expected.png b/test/integration/render-tests/heatmap-radius/function/expected.png index f05a3f3831b..73ffbb6d201 100644 Binary files a/test/integration/render-tests/heatmap-radius/function/expected.png and b/test/integration/render-tests/heatmap-radius/function/expected.png differ diff --git a/test/integration/render-tests/heatmap-radius/literal/expected.png b/test/integration/render-tests/heatmap-radius/literal/expected.png index 697980ed048..34c39fcb0f1 100644 Binary files a/test/integration/render-tests/heatmap-radius/literal/expected.png and b/test/integration/render-tests/heatmap-radius/literal/expected.png differ diff --git a/test/integration/render-tests/heatmap-radius/pitch30/expected.png b/test/integration/render-tests/heatmap-radius/pitch30/expected.png index 756e9836ea3..d2ffc18d01b 100644 Binary files a/test/integration/render-tests/heatmap-radius/pitch30/expected.png and b/test/integration/render-tests/heatmap-radius/pitch30/expected.png differ diff --git a/test/integration/render-tests/heatmap-radius/projected/expected.png b/test/integration/render-tests/heatmap-radius/projected/expected.png index e2ee3f1f51d..770ce2583ee 100644 Binary files a/test/integration/render-tests/heatmap-radius/projected/expected.png and b/test/integration/render-tests/heatmap-radius/projected/expected.png differ diff --git a/test/integration/render-tests/heatmap-weight/default/expected.png b/test/integration/render-tests/heatmap-weight/default/expected.png index 563ee0e8ab8..f54897bfbfb 100644 Binary files a/test/integration/render-tests/heatmap-weight/default/expected.png and b/test/integration/render-tests/heatmap-weight/default/expected.png differ diff --git a/test/integration/render-tests/heatmap-weight/identity-property-function/expected.png b/test/integration/render-tests/heatmap-weight/identity-property-function/expected.png index 86a1256d4d1..e3d8221d619 100644 Binary files a/test/integration/render-tests/heatmap-weight/identity-property-function/expected.png and b/test/integration/render-tests/heatmap-weight/identity-property-function/expected.png differ diff --git a/test/integration/render-tests/heatmap-weight/literal/expected.png b/test/integration/render-tests/heatmap-weight/literal/expected.png index 5891e360d5c..b17f23ae78f 100644 Binary files a/test/integration/render-tests/heatmap-weight/literal/expected.png and b/test/integration/render-tests/heatmap-weight/literal/expected.png differ diff --git a/test/integration/render-tests/icon-offset/literal/style.json b/test/integration/render-tests/icon-offset/literal/style.json index 5a1064be79e..79fae23766b 100644 --- a/test/integration/render-tests/icon-offset/literal/style.json +++ b/test/integration/render-tests/icon-offset/literal/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.0014 } }, "center": [ diff --git a/test/integration/render-tests/icon-offset/property-function/style.json b/test/integration/render-tests/icon-offset/property-function/style.json index 7c79a062fd8..f194adc3ab3 100644 --- a/test/integration/render-tests/icon-offset/property-function/style.json +++ b/test/integration/render-tests/icon-offset/property-function/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.0013 } }, "center": [ diff --git a/test/integration/render-tests/icon-offset/zoom-and-property-function/style.json b/test/integration/render-tests/icon-offset/zoom-and-property-function/style.json index 0f0c0c7fc70..67756a4c996 100644 --- a/test/integration/render-tests/icon-offset/zoom-and-property-function/style.json +++ b/test/integration/render-tests/icon-offset/zoom-and-property-function/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.0013 } }, "center": [ diff --git a/test/integration/render-tests/icon-text-fit/stretch-nine-part-just-height/style.json b/test/integration/render-tests/icon-text-fit/stretch-nine-part-just-height/style.json index d83664271be..1eec4488b4f 100644 --- a/test/integration/render-tests/icon-text-fit/stretch-nine-part-just-height/style.json +++ b/test/integration/render-tests/icon-text-fit/stretch-nine-part-just-height/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 200, - "height": 150 + "height": 150, + "allowed": 0.003 } }, "sources": { diff --git a/test/integration/render-tests/image/power-of-two/style.json b/test/integration/render-tests/image/power-of-two/style.json index ad01da2b03a..c9ae7128faf 100644 --- a/test/integration/render-tests/image/power-of-two/style.json +++ b/test/integration/render-tests/image/power-of-two/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 256, - "height": 256 + "height": 256, + "allowed": 0.0004 } }, "center": [34.5, 54.5], diff --git a/test/integration/render-tests/line-dasharray/zoom-history-line-metrics/style.json b/test/integration/render-tests/line-dasharray/zoom-history-line-metrics/style.json index 2af7c8f9967..8cd6f7b3d96 100644 --- a/test/integration/render-tests/line-dasharray/zoom-history-line-metrics/style.json +++ b/test/integration/render-tests/line-dasharray/zoom-history-line-metrics/style.json @@ -4,6 +4,7 @@ "test": { "width": 64, "height": 64, + "allowed": 0.0003, "operations": [ [ "wait" diff --git a/test/integration/render-tests/line-opacity/property-function/style.json b/test/integration/render-tests/line-opacity/property-function/style.json index 3c3893015f0..e27fa7059bf 100644 --- a/test/integration/render-tests/line-opacity/property-function/style.json +++ b/test/integration/render-tests/line-opacity/property-function/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.0002 } }, "center": [ diff --git a/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-mercator/style.json b/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-mercator/style.json index 49b358607a5..bd266f4be5b 100644 --- a/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-mercator/style.json +++ b/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-mercator/style.json @@ -4,7 +4,7 @@ "test": { "width": 512, "height": 512, - "allowed": 0.0003, + "allowed": 0.0006, "operations": [ ["setStyle", { "version": 8, diff --git a/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-winkel-tripel/style.json b/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-winkel-tripel/style.json index a387f4cc44d..b577e69dd57 100644 --- a/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-winkel-tripel/style.json +++ b/test/integration/render-tests/map-projections/integrations/set-projection/globe-to-winkel-tripel/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.0005, "operations": [ ["setStyle", { "version": 8, diff --git a/test/integration/render-tests/map-projections/integrations/set-projection/null-fallback-albers/style.json b/test/integration/render-tests/map-projections/integrations/set-projection/null-fallback-albers/style.json index f150ecfb5c4..4ec75936ce3 100644 --- a/test/integration/render-tests/map-projections/integrations/set-projection/null-fallback-albers/style.json +++ b/test/integration/render-tests/map-projections/integrations/set-projection/null-fallback-albers/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.0004, "operations": [ ["setStyle", { "version": 8, diff --git a/test/integration/render-tests/map-projections/integrations/set-projection/set-projection-precedence-fallback/style.json b/test/integration/render-tests/map-projections/integrations/set-projection/set-projection-precedence-fallback/style.json index b1403c203e7..096ee6d860b 100644 --- a/test/integration/render-tests/map-projections/integrations/set-projection/set-projection-precedence-fallback/style.json +++ b/test/integration/render-tests/map-projections/integrations/set-projection/set-projection-precedence-fallback/style.json @@ -4,6 +4,7 @@ "test": { "width": 512, "height": 512, + "allowed": 0.0002, "operations": [ ["setProjection", "mercator"], ["wait"], diff --git a/test/integration/render-tests/raster-masking/overlapping-vector/style.json b/test/integration/render-tests/raster-masking/overlapping-vector/style.json index d5253448cf3..1e5c0187771 100644 --- a/test/integration/render-tests/raster-masking/overlapping-vector/style.json +++ b/test/integration/render-tests/raster-masking/overlapping-vector/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.003 } }, "center": [ diff --git a/test/integration/render-tests/regressions/mapbox-gl-js#3612/style.json b/test/integration/render-tests/regressions/mapbox-gl-js#3612/style.json index dd100894e50..0fd9cf6a375 100644 --- a/test/integration/render-tests/regressions/mapbox-gl-js#3612/style.json +++ b/test/integration/render-tests/regressions/mapbox-gl-js#3612/style.json @@ -2,7 +2,8 @@ "version": 8, "metadata": { "test": { - "height": 256 + "height": 256, + "allowed": 0.00016 } }, "center": [ diff --git a/test/integration/render-tests/regressions/mapbox-gl-js#5740/style.json b/test/integration/render-tests/regressions/mapbox-gl-js#5740/style.json index 2ab02ada1c6..e9acfbdd9ef 100644 --- a/test/integration/render-tests/regressions/mapbox-gl-js#5740/style.json +++ b/test/integration/render-tests/regressions/mapbox-gl-js#5740/style.json @@ -5,6 +5,7 @@ "fadeDuration": 1000, "width": 64, "height": 64, + "allowed": 0.0008, "description": "Tests that fill-pattern cross-fading completes, by checking the rendering after the fade duration has elapsed.", "operations": [ [ diff --git a/test/integration/render-tests/regressions/mapbox-gl-js#6806/style.json b/test/integration/render-tests/regressions/mapbox-gl-js#6806/style.json index d67837a09b1..91b63b0ad03 100644 --- a/test/integration/render-tests/regressions/mapbox-gl-js#6806/style.json +++ b/test/integration/render-tests/regressions/mapbox-gl-js#6806/style.json @@ -4,6 +4,7 @@ "test": { "height": 64, "width": 64, + "allowed": 0.0008, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/runtime-styling/set-style-layer-change-source-layer/style.json b/test/integration/render-tests/runtime-styling/set-style-layer-change-source-layer/style.json index 697dfb540f4..6a788226d8e 100644 --- a/test/integration/render-tests/runtime-styling/set-style-layer-change-source-layer/style.json +++ b/test/integration/render-tests/runtime-styling/set-style-layer-change-source-layer/style.json @@ -4,6 +4,7 @@ "test": { "width": 64, "height": 64, + "allowed": 0.002, "operations": [ [ "setStyle", diff --git a/test/integration/render-tests/skybox/fill-extrusion-light/horizon-north-east/style.json b/test/integration/render-tests/skybox/fill-extrusion-light/horizon-north-east/style.json index 72d8eca4427..6683a4baeac 100644 --- a/test/integration/render-tests/skybox/fill-extrusion-light/horizon-north-east/style.json +++ b/test/integration/render-tests/skybox/fill-extrusion-light/horizon-north-east/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0002 } }, "light": { diff --git a/test/integration/render-tests/skybox/fill-extrusion-light/horizon-south-west/style.json b/test/integration/render-tests/skybox/fill-extrusion-light/horizon-south-west/style.json index e4a99e768aa..5062c284f3a 100644 --- a/test/integration/render-tests/skybox/fill-extrusion-light/horizon-south-west/style.json +++ b/test/integration/render-tests/skybox/fill-extrusion-light/horizon-south-west/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0002 } }, "light": { diff --git a/test/integration/render-tests/skybox/fill-extrusion-light/north-east/style.json b/test/integration/render-tests/skybox/fill-extrusion-light/north-east/style.json index 5d07530b4ee..6187336263c 100644 --- a/test/integration/render-tests/skybox/fill-extrusion-light/north-east/style.json +++ b/test/integration/render-tests/skybox/fill-extrusion-light/north-east/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0002 } }, "light": { diff --git a/test/integration/render-tests/skybox/fill-extrusion-light/north-west/style.json b/test/integration/render-tests/skybox/fill-extrusion-light/north-west/style.json index a0648eb32e1..91d2908709f 100644 --- a/test/integration/render-tests/skybox/fill-extrusion-light/north-west/style.json +++ b/test/integration/render-tests/skybox/fill-extrusion-light/north-west/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0002 } }, "light": { diff --git a/test/integration/render-tests/skybox/fill-extrusion-light/south-east/style.json b/test/integration/render-tests/skybox/fill-extrusion-light/south-east/style.json index 1aed34c43a0..5f477a9b5e1 100644 --- a/test/integration/render-tests/skybox/fill-extrusion-light/south-east/style.json +++ b/test/integration/render-tests/skybox/fill-extrusion-light/south-east/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0002 } }, "light": { diff --git a/test/integration/render-tests/skybox/fill-extrusion-light/south-west/style.json b/test/integration/render-tests/skybox/fill-extrusion-light/south-west/style.json index 53431e678ae..db7337b9b7d 100644 --- a/test/integration/render-tests/skybox/fill-extrusion-light/south-west/style.json +++ b/test/integration/render-tests/skybox/fill-extrusion-light/south-west/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0002 } }, "light": { diff --git a/test/integration/render-tests/symbol-sort-key/debug-mode-text-placement/style.json b/test/integration/render-tests/symbol-sort-key/debug-mode-text-placement/style.json index fad7aabe322..5d22601faea 100644 --- a/test/integration/render-tests/symbol-sort-key/debug-mode-text-placement/style.json +++ b/test/integration/render-tests/symbol-sort-key/debug-mode-text-placement/style.json @@ -5,7 +5,7 @@ "collisionDebug" : true, "height": 150, "width": 150, - "allowed": 0.0003 + "allowed": 0.008 } }, "center": [0, 0], diff --git a/test/integration/render-tests/terrain/hillshade--raster-interpolate-exaggeration/style.json b/test/integration/render-tests/terrain/hillshade--raster-interpolate-exaggeration/style.json index 0c2bb7ce114..453934f1a7f 100644 --- a/test/integration/render-tests/terrain/hillshade--raster-interpolate-exaggeration/style.json +++ b/test/integration/render-tests/terrain/hillshade--raster-interpolate-exaggeration/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.0005 } }, "center": [-113.26903, 35.9954], diff --git a/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/expected.png b/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/expected.png index 591f7922ffa..a4bb14c710f 100644 Binary files a/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/expected.png and b/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/expected.png differ diff --git a/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/style.json b/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/style.json index 5cec21940cb..795bca0dcef 100644 --- a/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/style.json +++ b/test/integration/render-tests/text-pitch-alignment/viewport-text-depthtest/style.json @@ -2,8 +2,7 @@ "version": 8, "metadata": { "test": { - "height": 256, - "allowed": 0.00025 + "height": 256 } }, "center": [ diff --git a/test/integration/render-tests/text-variable-anchor/icon-text-fit-collision-box/style.json b/test/integration/render-tests/text-variable-anchor/icon-text-fit-collision-box/style.json index 7b9b16d09c8..21908e9bc95 100644 --- a/test/integration/render-tests/text-variable-anchor/icon-text-fit-collision-box/style.json +++ b/test/integration/render-tests/text-variable-anchor/icon-text-fit-collision-box/style.json @@ -4,7 +4,7 @@ "test": { "height": 128, "width": 128, - "allowed": 0.0075, + "allowed": 0.0076, "collisionDebug": true } }, diff --git a/test/integration/render-tests/text-writing-mode/point_label/cjk-variable-anchors-vertical-horizontal-mode-icon-text-fit-terrain/style.json b/test/integration/render-tests/text-writing-mode/point_label/cjk-variable-anchors-vertical-horizontal-mode-icon-text-fit-terrain/style.json index 8a0d2a56cf3..acf36dfd359 100644 --- a/test/integration/render-tests/text-writing-mode/point_label/cjk-variable-anchors-vertical-horizontal-mode-icon-text-fit-terrain/style.json +++ b/test/integration/render-tests/text-writing-mode/point_label/cjk-variable-anchors-vertical-horizontal-mode-icon-text-fit-terrain/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 128, - "width": 256 + "width": 256, + "allowed": 0.003 } }, "center": [ 0, 0 ], diff --git a/test/integration/render-tests/within/paint-circle/style.json b/test/integration/render-tests/within/paint-circle/style.json index c7f27d00766..ed96a02f700 100644 --- a/test/integration/render-tests/within/paint-circle/style.json +++ b/test/integration/render-tests/within/paint-circle/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0003 } }, "zoom": 2, diff --git a/test/integration/render-tests/within/paint-icon/style.json b/test/integration/render-tests/within/paint-icon/style.json index 6bdeaa21840..f7d6a568b5a 100644 --- a/test/integration/render-tests/within/paint-icon/style.json +++ b/test/integration/render-tests/within/paint-icon/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0003 } }, "zoom": 2, diff --git a/test/integration/render-tests/within/paint-text/style.json b/test/integration/render-tests/within/paint-text/style.json index 09a9571d34f..25a1db2b1ef 100644 --- a/test/integration/render-tests/within/paint-text/style.json +++ b/test/integration/render-tests/within/paint-text/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0003 } }, "zoom": 2, diff --git a/test/integration/render-tests/within/within-feature-collection-geojson/style.json b/test/integration/render-tests/within/within-feature-collection-geojson/style.json index 55aef92253f..30f9fd11900 100644 --- a/test/integration/render-tests/within/within-feature-collection-geojson/style.json +++ b/test/integration/render-tests/within/within-feature-collection-geojson/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "width": 125, - "height": 125 + "height": 125, + "allowed": 0.0006 } }, "zoom": 2, diff --git a/test/integration/render-tests/within/within-feature-geojson/style.json b/test/integration/render-tests/within/within-feature-geojson/style.json index 76db109ccad..6623bd71a1b 100644 --- a/test/integration/render-tests/within/within-feature-geojson/style.json +++ b/test/integration/render-tests/within/within-feature-geojson/style.json @@ -3,7 +3,9 @@ "metadata": { "test": { "width": 64, - "height": 64 + "height": 64, + "allowed": 0.0003 + } }, "zoom": 2, diff --git a/test/integration/render-tests/zoomed-fill/negative-zoom/style.json b/test/integration/render-tests/zoomed-fill/negative-zoom/style.json index e197e225241..91dcf19062c 100644 --- a/test/integration/render-tests/zoomed-fill/negative-zoom/style.json +++ b/test/integration/render-tests/zoomed-fill/negative-zoom/style.json @@ -2,7 +2,7 @@ "version": 8, "metadata": { "test": { - "allowed": 0.0002, + "allowed": 0.0003, "width": 256, "height": 256 } diff --git a/test/integration/testem/testem.js b/test/integration/testem/testem.js index aa624ba0152..d25cede8562 100644 --- a/test/integration/testem/testem.js +++ b/test/integration/testem/testem.js @@ -4,6 +4,7 @@ const runAll = require('npm-run-all'); const chokidar = require('chokidar'); const rollup = require('rollup'); const notifier = require('node-notifier'); +const fs = require('fs'); // hack to be able to import ES modules inside a CommonJS one let generateFixtureJson, getAllFixtureGlobs, createServer, buildTape, rollupDevConfig, rollupTestConfig; @@ -24,6 +25,25 @@ const suiteName = process.env.SUITE_NAME; const suitePath = `${suiteName}-tests`; const ciOutputFile = `${rootFixturePath}${suitePath}/test-results.xml`; const fixtureBuildInterval = 2000; +const browser = process.env.BROWSER || "Chrome"; +const ci = process.env.npm_lifecycle_script.includes('testem ci'); + +let testFiles; +const testsToRunFile = "tests-to-run.txt"; + +if (fs.existsSync(testsToRunFile)) { + try { + let file = fs.readFileSync(testsToRunFile, 'utf8'); + // Remove BOM header which is written on Windows + file = file.replace(/^\uFEFF/, '').trim(); + // Convert windows to linux paths. Even on windows, we use path.posix for consisten path syntax. + file = file.replace(/^\uFEFF/, '').replace(/\\/g, '/'); + testFiles = file.split(/\r?\n/); + } catch (err) { + console.log("Failed to read file ", testsToRunFile); + console.error(err); + } +} const testPage = `test/integration/testem_page_${ process.env.BUILD === "production" ? "prod" : @@ -53,7 +73,7 @@ function getQueryParams() { return queryParams; } -const defaultTestemConfig = { +const testemConfig = { "test_page": testPage, "query_params": getQueryParams(), "proxies": { @@ -92,7 +112,7 @@ const defaultTestemConfig = { if (!beforeHookInvoked) { loadModules().then(() => { server = createServer(); - const buildPromise = config.appMode === 'ci' ? buildArtifactsCi() : buildArtifactsDev(); + const buildPromise = ci ? buildArtifactsCi() : buildArtifactsDev(); buildPromise.then(() => { server.listen(callback); }).catch((e) => { @@ -110,12 +130,14 @@ const defaultTestemConfig = { } }; -const renderTestemConfig = { - "launch_in_ci": [ "Chrome" ], +// Configuration for tests running in CI mode (i.e. test-... not watch-...) +const ciTestemConfig = { + "launch_in_ci": [ browser ], "reporter": "xunit", "report_file": ciOutputFile, "xunit_intermediate_output": true, - "tap_quiet_logs": true + "tap_quiet_logs": true, + "browser_disconnect_timeout": 90 // A longer disconnect time out prevents crashes on Windows Virtual Machines. }; function setChromeFlags(flags) { @@ -128,20 +150,27 @@ function setChromeFlags(flags) { }; } -const testemConfig = Object.assign({}, defaultTestemConfig); +if (ci) Object.assign(testemConfig, ciTestemConfig); -if (process.env.RENDER) Object.assign(testemConfig, renderTestemConfig); - -if (process.env.CI) { - // Set chrome flags for CircleCI to use llvmpipe driver (see https://github.com/mapbox/mapbox-gl-js/pull/10389). - const ciTestemConfig = setChromeFlags([ "--disable-backgrounding-occluded-windows", "--ignore-gpu-blocklist", "--use-gl=desktop" ]); - Object.assign(testemConfig, ciTestemConfig); - -} else if (process.env.RENDER && process.env.USE_ANGLE && ['metal', 'gl', 'vulkan', 'swiftshader', 'gles'].includes(process.env.USE_ANGLE)) { - // Allow setting chrome flag `--use-angle` for local development on render/query tests only. - // Search accepted values for `--use-angle` here: https://source.chromium.org/search?q=%22--use-angle%3D%22 - const angleTestemConfig = setChromeFlags([ `--use-angle=${process.env.USE_ANGLE}` ]); - Object.assign(testemConfig, angleTestemConfig); +if (browser === "Chrome") { + Object.assign(testemConfig, setChromeFlags([ "--disable-backgrounding-occluded-windows"])); + if (process.platform === "linux") { + // On Linux, set chrome flags for CircleCI to use llvmpipe driver instead of swiftshader + // This allows for more consistent behavior with MacOS development machines. + // (see https://github.com/mapbox/mapbox-gl-js/pull/10389). + const useOpenGL = setChromeFlags([ "--ignore-gpu-blocklist", "--use-gl=desktop" ]); + Object.assign(testemConfig, useOpenGL); + } if (process.env.USE_ANGLE) { + // Allow setting chrome flag `--use-angle` for local development on render/query tests only. + // Some devices (e.g. M1 Macs) seem to run test with significantly less failures when forcing the ANGLE backend to use Metal or OpenGL. + // Search accepted values for `--use-angle` here: https://source.chromium.org/search?q=%22--use-angle%3D%22 + if (!(['metal', 'gl', 'vulkan', 'swiftshader', 'gles'].includes(process.env.USE_ANGLE))) { + throw new Error(`Unrecognized value for 'use-angle': '${process.env.USE_ANGLE}'. Should be 'metal', 'gl', 'vulkan', 'swiftshader', or 'gles.'`); + } + console.log(`Chrome webgl using '${process.env.USE_ANGLE}'`); + const angleTestemConfig = setChromeFlags([ `--use-angle=${process.env.USE_ANGLE}` ]); + Object.assign(testemConfig, angleTestemConfig); + } } module.exports = testemConfig; @@ -150,7 +179,7 @@ module.exports = testemConfig; // Retuns a promise that resolves when all artifacts are built function buildArtifactsCi() { //1. Compile fixture data into a json file, so it can be bundled - generateFixtureJson(rootFixturePath, suitePath, outputPath, suitePath === 'render-tests'); + generateFixtureJson(rootFixturePath, suitePath, outputPath, suitePath === 'render-tests', testFiles); //2. Build tape const tapePromise = buildTape(); //3. Build test artifacts in parallel diff --git a/test/util/html_generator.js b/test/util/html_generator.js index 4ef9ec2fcc0..41d28325e5b 100644 --- a/test/util/html_generator.js +++ b/test/util/html_generator.js @@ -13,18 +13,21 @@ const generateResultHTML = template(`
<% if (r.status !== 'errored') { %> - + <% } %> <% if (r.expected) { %> - + <% } %> <% if (r.imgDiff) { %> - + <% } %> <% if (r.error) { %>

Error: <%- r.error.message %>

<% } %> <% if (r.jsonDiff) { %> 
<%- r.jsonDiff.trim() %>
<% } %> + <% if (r.status === 'failed') { %> + 
<%- JSON.stringify(r.style, null, 2) %>
+ <% } %>
`); @@ -42,7 +45,7 @@ input { position: absolute; opacity: 0; z-index: -1;} .tab-label { display: flex; color: white; border-radius: 5px; justify-content: space-between; padding: 1em; font-weight: bold; cursor: pointer; } .tab-label:hover { filter: brightness(85%); } .tab-label::after { content: "\\276F"; width: 1em; height: 1em; text-align: center; transition: all .35s; } -.tab-content { max-height: 0; padding: 0 1em; background: white; transition: all .35s; } +.tab-content { max-height: 0; padding: 0 1em; background: white; overflow: scroll; transition: all .35s; } .tab-content pre { font-size: 14px; margin: 0 0 10px; } input:checked + .tab-label { filter: brightness(90%); }; input:checked + .tab-label::after { transform: rotate(90deg); } diff --git a/yarn.lock b/yarn.lock index 6f85d8a1c1a..354e34dbe83 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2081,6 +2081,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: safe-buffer "^5.0.1" sha.js "^2.4.8" +cross-env@^7.0.3: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf" + integrity sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw== + dependencies: + cross-spawn "^7.0.1" + cross-fetch@3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" @@ -2099,7 +2106,7 @@ cross-spawn@^6.0.0, cross-spawn@^6.0.5: shebang-command "^1.2.0" which "^1.2.9" -cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: +cross-spawn@^7.0.0, cross-spawn@^7.0.1, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==