Web test reorganization

.ci.yaml

@@ -276,20 +276,11 @@ targets:
       release_build: "true"
       config_name: linux_android_debug_engine
 
-  - name: Linux Web Engine
-    recipe: engine/web_engine
+  - name: Linux Web Engine (V2)
+    recipe: engine_v2/engine_v2
+    bringup: true
     properties:
-      add_recipes_cq: "true"
-      cores: "32"
-      gcs_goldens_bucket: flutter_logs
-      gclient_variables: >-
-        {"download_emsdk": true}
-      dependencies: >-
-        [
-          {"dependency": "chrome_and_driver", "version": "version:111.0"},
-          {"dependency": "firefox", "version": "version:106.0"},
-          {"dependency": "goldctl", "version": "git_revision:3a77d0b12c697a840ca0c7705208e8622dc94603"}
-        ]
+      config_name: web_engine
     timeout: 60
     runIf:
       - DEPS
@@ -423,27 +414,6 @@ targets:
       ios_debug: "true"
     timeout: 60
 
-  - name: Mac Web Engine
-    recipe: engine/web_engine
-    properties:
-      add_recipes_cq: "true"
-      gcs_goldens_bucket: flutter_logs
-      gclient_variables: >-
-        {"download_emsdk": true}
-      dependencies: >-
-        [
-          {"dependency": "goldctl", "version": "git_revision:3a77d0b12c697a840ca0c7705208e8622dc94603"}
-        ]
-    timeout: 60
-    runIf:
-      - DEPS
-      - .ci.yaml
-      - lib/web_ui/**
-      - web_sdk/**
-      - tools/**
-      - ci/**
-      - flutter_frontend_server/**
-
   - name: Mac mac_ios_engine
     recipe: engine_v2/engine_v2
     timeout: 60
@@ -501,23 +471,6 @@ targets:
       add_recipes_cq: "true"
     timeout: 75
 
-  - name: Windows Web Engine
-    recipe: engine/web_engine
-    properties:
-      gclient_variables: >-
-        {"download_emsdk": true}
-      gcs_goldens_bucket: flutter_logs
-      dependencies: >-
-        [
-          {"dependency": "chrome_and_driver", "version": "version:111.0"}
-        ]
-    timeout: 60
-    runIf:
-      - DEPS
-      - .ci.yaml
-      - lib/web_ui/**
-      - web_sdk/**
-
   - name: Mac iOS Engine Profile
     recipe: engine/engine
     properties:

ci/builders/web_engine.json

@@ -1,41 +1,82 @@
 {
-    "generators": {
-        "pub_dirs": [
-            "flutter/lib/web_ui/",
-            "flutter/web_sdk/web_engine_tester/"
-        ],
-        "tasks": [
-            {
-                "name": "compile web_tests",
-                "parameters": [
-                    "run",
-                    "compile_tests"
-                ],
-                "scripts": [
-                    "out/host_debug_unopt/dart-sdk/bin/dart",
-                    "flutter/lib/web_ui/dev/felt.dart"
-                ]
+    "builds": [
+        {
+            "archives": [],
+            "drone_dimensions": [
+                "device_type=none",
+                "os=Linux"
+            ],
+            "gclient_variables": {
+                "download_android_deps": false,
+                "download_emsdk": true
             },
-            {
-                "name": "check licenses",
-                "parameters": [
-                    "check-licenses"
-                ],
-                "scripts": [
-                    "out/host_debug_unopt/dart-sdk/bin/dart",
-                    "flutter/lib/web_ui/dev/felt.dart"
+            "gn": [
+                "--web",
+                "--runtime-mode=release"
+            ],
+            "name": "wasm_release",
+            "ninja": {
+                "config": "wasm_release",
+                "targets": [
+                    "flutter/web_sdk:flutter_web_sdk_archive"
                 ]
             },
-            {
-                "name": "web engine analysis",
-                "parameters": [
-                    "analyze"
-                ],
-                "scripts": [
-                    "out/host_debug_unopt/dart-sdk/bin/dart",
-                    "flutter/lib/web_ui/dev/felt.dart"
+            "generators": {
+                "tasks": [
+                    {
+                        "name": "analyze web_ui",
+                        "parameters": [
+                            "analyze"
+                        ],
+                        "scripts": [
+                            "flutter/lib/web_ui/dev/felt"
+                        ]
+                    },
+                    {
+                        "name": "check licenses",
+                        "parameters": [
+                            "check-licenses"
+                        ],
+                        "scripts": [
+                            "flutter/lib/web_ui/dev/felt"
+                        ]
+                    },
+                    {
+                        "name": "compile web_tests",
+                        "parameters": [
+                            "test",
+                            "--compile"
+                        ],
+                        "scripts": [
+                            "flutter/lib/web_ui/dev/felt"
+                        ]
+                    }
                 ]
-            }
-        ]
-    }
+            },
+            "tests": []
+        }
+    ],
+    "tests": [
+        {
+            "name": "linux chrome dart2js web tests",
+            "drone_dimensions": [
+                "device_type=none",
+                "os=Linux"
+            ],
+            "parameters": [
+                "test",
+                "--run",
+                "--suite=chrome-dart2js-html-engine",
+                "--suite=chrome-dart2js-html-html",
+                "--suite=chrome-dart2js-html-ui",
+                "--suite=chrome-dart2js-canvaskit-canvaskit",
+                "--suite=chrome-dart2js-canvaskit-ui",
+                "--suite=chrome-dart2js-skwasm-skwasm_stub",
+                "--suite=chrome-full-dart2js-canvaskit-canvaskit",
+                "--suite=chrome-full-dart2js-canvaskit-ui"
+            ],
+            "language": "bash",
+            "script": "flutter/lib/web_ui/dev/felt"
+        }
+    ]
 }

lib/web_ui/README.md

@@ -22,75 +22,92 @@ To tell `felt` to do anything you call `felt SUBCOMMAND`, where `SUBCOMMAND` is
 one of the available subcommands, which can be listed by running `felt help`. To
 get help for a specific subcommand, run `felt help SUBCOMMAND`.
 
-The most useful subcommands are:
-
-- `felt build` - builds a local Flutter Web engine ready to be used by the
-  Flutter framework. To use a locally built web sdk, build with `felt build`, 
-  then pass `--local-web-sdk=wasm_release` to the `flutter` command, or to 
-  `dev/bots/test.dart` when running a web shard, such as `web_tests`.
-- `felt test` - runs web engine tests. By default, this runs all tests using
-  Chromium. Passing one or more paths to specific tests would run just the
-  specified tests. Run `felt help test` for more options.
-
-`build` and `test` take the `--watch` option, which automatically reruns the
-subcommand when a source file changes. This is handy when you are iterating
-quickly.
-
-#### Examples
-
-Builds the web engine, the runs a Flutter app using it:
-
+#### `felt build`
+The `build` subcommand builds web engine gn/ninja targets. Targets can be
+individually specified in the command line invocation, or if none are specified,
+all web engine targets are built. Common targets are as follows:
+  * `sdk` - The flutter_web_sdk itself.
+  * `canvaskit` - Flutter's version of canvakit.
+  * `canvaskit_chromium` - A version of canvaskit optimized for use with
+    chromium-based browsers.
+  * `skwasm` - Builds experimental skia wasm module renderer.
+The output of these steps is used in unit tests, and can be used with the flutter
+command via the `--local-web-sdk=wasm_release` command.
+
+##### Examples
+Builds all web engine targets, then runs a Flutter app using it:
 ```
 felt build
 cd path/to/some/app
-flutter --local-web-sdk=wasm_release run -d chrome
+flutter --local-web_sdk=wasm_release run -d chrome
 ```
 
-Runs all tests in Chromium:
-
+Builds only the `sdk` and the `canvaskit` targets:
 ```
-felt test
+felt build sdk canvaskit
 ```
 
-Runs a specific test:
-
+#### `felt test`
+The `test` subcommand will compile and/or run web engine unit test suites. For
+information on how test suites are structured, see the test configuration
+[readme][2].
+
+By default, `felt test` compiles and runs all suites that are compatible with the
+host system. Some useful flags supported by this command:
+  * `--compile` will only perform compilation of these suites without running them. 
+  * `--run` will only run the tests and not compile them, and assume they have been 
+    compiled in a previous run of the tool.
+  * `--list` will list all the test suites and test bundles and exit without
+    compiling or running anything.
+  * `--verbose` will output some extra information that may be useful for debugging.
+  * `--debug` will open a browser window and pause the tests before starting so that
+    breakpoints can be set before starting the test suites.
+
+Several other flags can be passed that filter which test suites should be run:
+  * `--browser` runs only the test suites that test on the browsers passed. Valid
+    values for this are `chrome`, `firefox`, `safari`, or `edge`.
+  * `--compiler` runs only the test suites that use a particular compiler. Valid
+    values for this are `dart2js` or `dart2wasm`
+  * `--renderer` runs only the test suites that use a particular renderer. Valid
+    values for this are `html`, `canvakit`, or `skwasm`
+  * `--suite` runs a suite by name.
+  * `--bundle` runs suites that target a particular test bundle.
+
+Filters of different types are logically ANDed together, but multiple filter flags
+of the same type are logically ORed together.
+
+The `test` command will also accept a list of paths to specific test files to be
+compiled and run. If none of these paths are specified, all tests are run, otherwise
+only the tests that are specified will run.
+
+##### Examples
+Runs all test suites in all compatible browsers:
 ```
-felt test test/engine/util_test.dart
+felt test
 ```
-
-Runs multiple specific tests:
-
+Runs a specific test on all compatible browsers:
 ```
-felt test test/engine/util_test.dart test/alarm_clock_test.dart
+felt test test/engine/util_test.dart
 ```
-
-Enable watch mode so that the test re-runs every time a source file changes:
-
+Runs multiple specific tests on all compatible browsers:
 ```
-felt test --watch test/engine/util_test.dart
+felt test test/engine/util_test.dart test/engine/alarm_clock_test.dart
 ```
-
-Runs tests in Firefox (requires a Linux computer):
-
+Runs only test suites that compile via dart2wasm:
 ```
-felt test --browser=firefox
+felt test --compiler dart2wasm
 ```
-
-Chromium and Firefox support debugging tests using the browser's developer
-tools. To run tests in debug mode add `--debug` to the `test` command, e.g.:
-
+Runs only test suites that run in Chrome and Safari:
 ```
-felt test --debug --browser=firefox test/alarm_clock_test.dart
+felt test --browser chrome --browser safari
 ```
 
 ### Optimizing local builds
 
 Concurrency of various build steps can be configured via environment variables:
 
-- `FELT_DART2JS_CONCURRENCY` specifies the number of concurrent `dart2js`
+- `FELT_COMPILE_CONCURRENCY` specifies the number of concurrent compiler
   processes used to compile tests. Default value is 8.
-- `FELT_TEST_CONCURRENCY` specifies the number of tests run concurrently.
-  Default value is 10.
 
 If you are a Google employee, you can use an internal instance of Goma (go/ma)
 to parallelize your ninja builds. Because Goma compiles code on remote servers,
@@ -99,7 +116,7 @@ this option is particularly effective for building on low-powered laptops.
 ### Test browsers
 
 Chromium, Firefox, and Safari for iOS are version-locked using the
-[browser_lock.yaml][2] configuration file. Safari for macOS is supplied by the
+[browser_lock.yaml][3] configuration file. Safari for macOS is supplied by the
 computer's operating system. Tests can be run in Edge locally, but Edge is not
 enabled on LUCI. Chromium is used as a proxy for Chrome, Edge, and other
 Chromium-based browsers.
@@ -301,7 +318,8 @@ Once you know the version for the Emscripten SDK, change the line in
 
 
 [1]: https://github.com/flutter/flutter/wiki/Setting-up-the-Engine-development-environment
-[2]: https://github.com/flutter/engine/blob/main/lib/web_ui/dev/browser_lock.yaml
+[2]: https://github.com/flutter/flutter/blob/main/lib/web_ui/test/README
+[3]: https://github.com/flutter/engine/blob/main/lib/web_ui/dev/browser_lock.yaml
 [4]: https://chrome-infra-packages.appspot.com/p/flutter_internal
 [5]: https://cs.opensource.google/flutter/recipes/+/master:recipes/engine/web_engine.py
 [6]: https://chromium.googlesource.com/chromium/src.git/+/main/docs/cipd_and_3pp.md#What-is-CIPD

lib/web_ui/dev/common.dart

@@ -11,6 +11,7 @@ import 'browser_lock.dart';
 import 'chrome.dart';
 import 'edge.dart';
 import 'environment.dart';
+import 'felt_config.dart';
 import 'firefox.dart';
 import 'safari_macos.dart';
 
@@ -261,16 +262,15 @@ const List<String> kAllBrowserNames = <String>[
 /// Creates an environment for a browser.
 ///
 /// The [browserName] matches the browser name passed as the `--browser` option.
-BrowserEnvironment getBrowserEnvironment(String browserName, { required bool enableWasmGC }) {
+BrowserEnvironment getBrowserEnvironment(BrowserName browserName, { required bool enableWasmGC }) {
   switch (browserName) {
-    case kChrome:
+    case BrowserName.chrome:
       return ChromeEnvironment(enableWasmGC);
-    case kEdge:
+    case BrowserName.edge:
       return EdgeEnvironment();
-    case kFirefox:
+    case BrowserName.firefox:
       return FirefoxEnvironment();
-    case kSafari:
+    case BrowserName.safari:
       return SafariMacOsEnvironment();
   }
-  throw UnsupportedError('Browser $browserName is not supported.');
 }

lib/web_ui/dev/environment.dart

@@ -143,8 +143,8 @@ class Environment {
   ///
   /// This is where compiled output goes.
   io.Directory get webUiBuildDir => io.Directory(pathlib.join(
-        webUiRootDir.path,
-        'build',
+        outDir.path,
+        'web_tests',
       ));
 
   /// Path to the ".dart_tool" directory, generated by various Dart tools.