[WIP] use frontend server for VM test platform

pkgs/test/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 1.17.0-dev
+
+* Change the default way VM tests are launched and ran to greatly speed up
+  loading performance.
+  * You can force the old strategy with `--use-data-isolate-strategy` flag if
+    you run into issues, but please also file a bug.
+
 ## 1.16.9
 
 * Disable stack trace chaining by default. It can be re-enabled by explicitly

pkgs/test/pubspec.yaml

@@ -1,5 +1,5 @@
 name: test
-version: 1.16.9
+version: 1.17.0-dev
 description: A full featured library for writing and running Dart tests.
 repository: https://github.com/dart-lang/test/blob/master/pkgs/test
 

pkgs/test/test/runner/data_isolate_strategy_test.dart

@@ -0,0 +1,66 @@
+// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn('vm')
+
+import 'dart:convert';
+import 'dart:isolate';
+
+import 'package:package_config/package_config.dart';
+import 'package:test_descriptor/test_descriptor.dart' as d;
+
+import 'package:test/test.dart';
+
+import '../io.dart';
+
+void main() {
+  late PackageConfig currentPackageConfig;
+
+  setUpAll(() async {
+    currentPackageConfig =
+        await loadPackageConfigUri((await Isolate.packageConfig)!);
+  });
+
+  setUp(() async {
+    await d
+        .file('package_config.json',
+            jsonEncode(PackageConfig.toJson(currentPackageConfig)))
+        .create();
+  });
+
+  group('The data isolate strategy', () {
+    test('can be enabled', () async {
+      // We confirm it is enabled by checking the error output for an invalid
+      // test, it looks a bit different.
+      await d.file('test.dart', 'invalid Dart file').create();
+      var test = await runTest(['--use-data-isolate-strategy', 'test.dart']);
+
+      expect(
+          test.stdout,
+          containsInOrder([
+            'Failed to load "test.dart":',
+            "Unable to spawn isolate: test.dart:1:9: Error: Expected ';' after this.",
+            'invalid Dart file'
+          ]));
+
+      await test.shouldExit(1);
+    });
+
+    test('can run tests', () async {
+      await d.file('test.dart', '''
+import 'package:test/test.dart';
+
+void main() {
+  test('true is true', () {
+    expect(true, isTrue);
+  });
+}
+      ''').create();
+      var test = await runTest(['--use-data-isolate-strategy', 'test.dart']);
+
+      expect(test.stdout, emitsThrough(contains('+1: All tests passed!')));
+      await test.shouldExit(0);
+    });
+  });
+}

pkgs/test/test/runner/runner_test.dart

@@ -93,6 +93,9 @@ Running Tests:
                                       to provide improved test performance but at the cost of
                                       debuggability.
     --no-retry                        Don't rerun tests that have retry set.
+    --use-data-isolate-strategy       Use `data:` uri isolates when spawning VM tests instead of the
+                                      default strategy. This may be faster when you only ever run a
+                                      single test suite at a time.
     --test-randomize-ordering-seed    Use the specified seed to randomize the execution order of test cases.
                                       Must be a 32bit unsigned integer or "random".
                                       If "random", pick a random seed to use.
@@ -167,8 +170,7 @@ $_usage''');
           test.stdout,
           containsInOrder([
             'Failed to load "test.dart":',
-            'Unable to spawn isolate: test.dart:1:9: Error: '
-                "Expected ';' after this.",
+            "test.dart:1:9: Error: Expected ';' after this.",
             'invalid Dart file'
           ]));
 
@@ -186,8 +188,7 @@ $_usage''');
           containsInOrder([
             '-1: loading test.dart [E]',
             'Failed to load "test.dart":',
-            'Unable to spawn isolate: test.dart:1:14: '
-                "Error: Expected ';' after this"
+            "test.dart:1:14: Error: Expected ';' after this"
           ]));
 
       await test.shouldExit(1);
@@ -204,8 +205,7 @@ $_usage''');
           containsInOrder([
             '-1: loading test.dart [E]',
             'Failed to load "test.dart":',
-            'Unable to spawn isolate: test.dart:1:8: Error: '
-                "Expected a declaration, but got ')'",
+            "test.dart:1:8: Error: Expected a declaration, but got ')'",
             '@TestOn)',
           ]));
 

pkgs/test/test/runner/test_on_test.dart

@@ -5,7 +5,10 @@
 @TestOn('vm')
 
 import 'dart:async';
+import 'dart:convert';
+import 'dart:isolate';
 
+import 'package:package_config/package_config.dart';
 import 'package:test_descriptor/test_descriptor.dart' as d;
 
 import 'package:test_core/src/util/io.dart';
@@ -14,6 +17,20 @@ import 'package:test/test.dart';
 import '../io.dart';
 
 void main() {
+  late PackageConfig currentPackageConfig;
+
+  setUpAll(() async {
+    currentPackageConfig =
+        await loadPackageConfigUri((await Isolate.packageConfig)!);
+  });
+
+  setUp(() async {
+    await d
+        .file('package_config.json',
+            jsonEncode(PackageConfig.toJson(currentPackageConfig)))
+        .create();
+  });
+
   group('for suite', () {
     test('runs a test suite on a matching platform', () async {
       await _writeTestFile('vm_test.dart', suiteTestOn: 'vm');

pkgs/test_core/CHANGELOG.md

@@ -1,7 +1,12 @@
-## 0.3.20
+## 0.3.20-dev
 
 * Disable stack trace chaining by default.
 
+* Change the default way VM tests are launched and ran to greatly speed up
+  loading performance.
+  * You can force the old strategy with `--use-data-isolate-strategy` flag if
+    you run into issues, but please also file a bug.
+
 ## 0.3.19
 
 * ~~Disable stack trace chaining by default.~~

pkgs/test_core/lib/src/runner/configuration.dart

@@ -174,6 +174,14 @@ class Configuration {
       });
   Set<String>? _knownPresets;
 
+  /// Whether to use the original `data:` URI isolate spawning strategy for VM
+  /// tests.
+  ///
+  /// This can make more sense than the default strategy in systems such as
+  /// `bazel` where only a single test suite is ran at a time.
+  bool get useDataIsolateStrategy => _useDataIsolateStrategy ?? false;
+  final bool? _useDataIsolateStrategy;
+
   /// Built-in runtimes whose settings are overridden by the user.
   final Map<String, RuntimeSettings> overrideRuntimes;
 
@@ -243,6 +251,7 @@ class Configuration {
       Map<String, RuntimeSettings>? overrideRuntimes,
       Map<String, CustomRuntime>? defineRuntimes,
       bool? noRetry,
+      bool? useDataIsolateStrategy,
 
       // Suite-level configuration
       bool? jsTrace,
@@ -292,6 +301,7 @@ class Configuration {
         overrideRuntimes: overrideRuntimes,
         defineRuntimes: defineRuntimes,
         noRetry: noRetry,
+        useDataIsolateStrategy: useDataIsolateStrategy,
         suiteDefaults: SuiteConfiguration(
             jsTrace: jsTrace,
             runSkipped: runSkipped,
@@ -354,6 +364,7 @@ class Configuration {
       Map<String, RuntimeSettings>? overrideRuntimes,
       Map<String, CustomRuntime>? defineRuntimes,
       bool? noRetry,
+      bool? useDataIsolateStrategy,
       SuiteConfiguration? suiteDefaults})
       : _help = help,
         customHtmlTemplatePath = customHtmlTemplatePath,
@@ -379,6 +390,7 @@ class Configuration {
         overrideRuntimes = _map(overrideRuntimes),
         defineRuntimes = _map(defineRuntimes),
         _noRetry = noRetry,
+        _useDataIsolateStrategy = useDataIsolateStrategy,
         suiteDefaults = pauseAfterLoad == true
             ? suiteDefaults?.change(timeout: Timeout.none) ??
                 SuiteConfiguration(timeout: Timeout.none)
@@ -513,6 +525,8 @@ class Configuration {
         defineRuntimes:
             mergeUnmodifiableMaps(defineRuntimes, other.defineRuntimes),
         noRetry: other._noRetry ?? _noRetry,
+        useDataIsolateStrategy:
+            other._useDataIsolateStrategy ?? _useDataIsolateStrategy,
         suiteDefaults: suiteDefaults.merge(other.suiteDefaults));
     result = result._resolvePresets();
 
@@ -549,6 +563,7 @@ class Configuration {
       Map<String, RuntimeSettings>? overrideRuntimes,
       Map<String, CustomRuntime>? defineRuntimes,
       bool? noRetry,
+      bool? useDataIsolateStrategy,
 
       // Suite-level configuration
       bool? jsTrace,
@@ -595,6 +610,8 @@ class Configuration {
         overrideRuntimes: overrideRuntimes ?? this.overrideRuntimes,
         defineRuntimes: defineRuntimes ?? this.defineRuntimes,
         noRetry: noRetry ?? _noRetry,
+        useDataIsolateStrategy:
+            useDataIsolateStrategy ?? _useDataIsolateStrategy,
         suiteDefaults: suiteDefaults.change(
             jsTrace: jsTrace,
             runSkipped: runSkipped,

pkgs/test_core/lib/src/runner/configuration/args.dart

@@ -107,6 +107,12 @@ final ArgParser _parser = (() {
       help: "Don't rerun tests that have retry set.",
       defaultsTo: false,
       negatable: false);
+  parser.addFlag('use-data-isolate-strategy',
+      help: 'Use `data:` uri isolates when spawning VM tests instead of the\n'
+          'default strategy. This may be faster when you only ever run a\n'
+          'single test suite at a time.',
+      defaultsTo: false,
+      negatable: false);
   parser.addOption('test-randomize-ordering-seed',
       help: 'Use the specified seed to randomize the execution order of test'
           ' cases.\n'
@@ -268,6 +274,7 @@ class _Parser {
         includeTags: includeTags,
         excludeTags: excludeTags,
         noRetry: _ifParsed('no-retry'),
+        useDataIsolateStrategy: _ifParsed('use-data-isolate-strategy'),
         testRandomizeOrderingSeed: testRandomizeOrderingSeed);
   }
 

pkgs/test_core/lib/src/runner/vm/platform.dart

@@ -7,13 +7,15 @@ import 'dart:developer';
 import 'dart:io';
 import 'dart:isolate';
 
+import 'package:async/async.dart';
 import 'package:coverage/coverage.dart';
 import 'package:path/path.dart' as p;
 import 'package:stream_channel/isolate_channel.dart';
 import 'package:stream_channel/stream_channel.dart';
 import 'package:test_api/backend.dart'; // ignore: deprecated_member_use
 import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
+import 'package:test_core/src/runner/vm/test_compiler.dart';
 import 'package:vm_service/vm_service.dart' hide Isolate;
 import 'package:vm_service/vm_service_io.dart';
 
@@ -25,13 +27,17 @@ import '../../runner/plugin/platform_helpers.dart';
 import '../../runner/runner_suite.dart';
 import '../../runner/suite.dart';
 import '../../util/dart.dart' as dart;
+import '../../util/package_config.dart';
 import '../package_version.dart';
 import 'environment.dart';
 
 /// A platform that loads tests in isolates spawned within this Dart process.
 class VMPlatform extends PlatformPlugin {
   /// The test runner configuration.
   final _config = Configuration.current;
+  final _compiler =
+      TestCompiler(p.join(p.current, '.dart_tool', 'pkg_test_kernel.bin'));
+  final _closeMemo = AsyncMemoizer<void>();
 
   VMPlatform();
 
@@ -40,15 +46,16 @@ class VMPlatform extends PlatformPlugin {
       throw UnimplementedError();
 
   @override
-  Future<RunnerSuite> load(String path, SuitePlatform platform,
+  Future<RunnerSuite?> load(String path, SuitePlatform platform,
       SuiteConfiguration suiteConfig, Object message) async {
     assert(platform.runtime == Runtime.vm);
 
     var receivePort = ReceivePort();
-    Isolate isolate;
+    Isolate? isolate;
     try {
       isolate =
           await _spawnIsolate(path, receivePort.sendPort, suiteConfig.metadata);
+      if (isolate == null) return null;
     } catch (error) {
       receivePort.close();
       rethrow;
@@ -58,7 +65,7 @@ class VMPlatform extends PlatformPlugin {
     StreamSubscription<Event>? eventSub;
     var channel = IsolateChannel.connectReceive(receivePort)
         .transformStream(StreamTransformer.fromHandlers(handleDone: (sink) {
-      isolate.kill();
+      isolate!.kill();
       eventSub?.cancel();
       client?.dispose();
       sink.close();
@@ -108,33 +115,54 @@ class VMPlatform extends PlatformPlugin {
     return await controller.suite;
   }
 
+  @override
+  Future close() => _closeMemo.runOnce(() => _compiler.dispose());
+
   /// Spawns an isolate and passes it [message].
   ///
   /// This isolate connects an [IsolateChannel] to [message] and sends the
   /// serialized tests over that channel.
-  Future<Isolate> _spawnIsolate(
+  Future<Isolate?> _spawnIsolate(
       String path, SendPort message, Metadata suiteMetadata) async {
-    var precompiledPath = _config.suiteDefaults.precompiledPath;
-    if (precompiledPath != null) {
-      return _spawnPrecompiledIsolate(path, message, precompiledPath);
-    } else if (_config.pubServeUrl != null) {
-      return _spawnPubServeIsolate(path, message, _config.pubServeUrl!);
-    } else {
-      return _spawnDataIsolate(path, message, suiteMetadata);
+    try {
+      var precompiledPath = _config.suiteDefaults.precompiledPath;
+      if (precompiledPath != null) {
+        return _spawnPrecompiledIsolate(path, message, precompiledPath);
+      } else if (_config.pubServeUrl != null) {
+        return _spawnPubServeIsolate(path, message, _config.pubServeUrl!);
+      } else if (_config.useDataIsolateStrategy) {
+        return _spawnDataIsolate(path, message, suiteMetadata);
+      } else {
+        return _spawnKernelIsolate(path, message, suiteMetadata);
+      }
+    } catch (_) {
+      if (_closeMemo.hasRun) return null;
+      rethrow;
     }
   }
+
+  /// Compiles [path] to kernel using [_compiler] and spawns that in an
+  /// isolate.
+  Future<Isolate> _spawnKernelIsolate(
+      String path, SendPort message, Metadata suiteMetadata) async {
+    final response =
+        await _compiler.compile(File(path).absolute.uri, suiteMetadata);
+    var compiledDill = response.kernelOutputUri?.toFilePath();
+    if (compiledDill == null || response.errorCount > 0) {
+      throw LoadException(path, response.compilerOutput ?? 'unknown error');
+    }
+    return await Isolate.spawnUri(p.toUri(compiledDill), [], message,
+        packageConfig: await packageConfigUri, checked: true);
+  }
 }
 
 Future<Isolate> _spawnDataIsolate(
     String path, SendPort message, Metadata suiteMetadata) async {
   return await dart.runInIsolate('''
     ${suiteMetadata.languageVersionComment ?? await rootPackageLanguageVersionComment}
     import "dart:isolate";
-
     import "package:test_core/src/bootstrap/vm.dart";
-
     import "${p.toUri(p.absolute(path))}" as test;
-
     void main(_, SendPort sendPort) {
       internalBootstrapVmTest(() => test.main, sendPort);
     }