From 77fdda20ba76934a93b83a2dfd8e7c3869e16c4d Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Tue, 7 Jul 2026 21:53:33 +0000 Subject: [PATCH 1/2] Support DART_TEST_REPORTER environment variable Allow configuring the test reporter using the `DART_TEST_REPORTER` environment variable. This variable is honored by: - Test suites run directly (when `dart:io` is available). - The test runner (when `--reporter` CLI flag is not used). The order of precedence for the test runner is: 1. `dart_test.yaml` 2. `DART_TEST_REPORTER` environment variable 3. `--reporter` CLI argument (highest priority) TAG=agy CONV=bc215ee6-37c5-4708-b470-be096f4accb4 --- pkgs/test/CHANGELOG.md | 7 +++ pkgs/test/pubspec.yaml | 4 +- .../runner/configuration/top_level_test.dart | 20 +++++++++ pkgs/test/test/runner/runner_test.dart | 45 +++++++++++++++++++ pkgs/test_core/CHANGELOG.md | 4 ++ pkgs/test_core/lib/src/executable.dart | 12 +++++ .../lib/src/runner/reporter/direct_io.dart | 43 +++++++++++------- pkgs/test_core/pubspec.yaml | 2 +- 8 files changed, 119 insertions(+), 18 deletions(-) diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index 39b012143..03e75d348 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -1,3 +1,10 @@ +## 1.32.0-wip + +* Add support for `DART_TEST_REPORTER` environment variable in test runner and + when tests are run directly on platforms which support `dart:io`. The + environment variable takes precedence over configuration in `dart_test.yaml` + but is overridden by the `--reporter` flag when passed to the test runner. + ## 1.31.2 * Add support for running tests as native CLI bundles (vm platform only). diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml index b4b4f8d85..3a7bba6fb 100644 --- a/pkgs/test/pubspec.yaml +++ b/pkgs/test/pubspec.yaml @@ -1,5 +1,5 @@ name: test -version: 1.31.2 +version: 1.32.0-wip description: >- A full featured library for writing and running Dart tests across platforms. repository: https://github.com/dart-lang/test/tree/master/pkgs/test @@ -36,7 +36,7 @@ dependencies: # Use an exact version until the test_api and test_core package are stable. test_api: 0.7.13 - test_core: 0.6.19 + test_core: 0.6.20-wip typed_data: ^1.3.0 web_socket_channel: '>=2.0.0 <4.0.0' diff --git a/pkgs/test/test/runner/configuration/top_level_test.dart b/pkgs/test/test/runner/configuration/top_level_test.dart index 833dd50e8..899952eba 100644 --- a/pkgs/test/test/runner/configuration/top_level_test.dart +++ b/pkgs/test/test/runner/configuration/top_level_test.dart @@ -406,6 +406,26 @@ void main() { await test.shouldExit(0); }); + test('DART_TEST_REPORTER overrides dart_test.yaml reporter', () async { + await d.file('dart_test.yaml', jsonEncode({'reporter': 'json'})).create(); + + await d.file('test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("success", () {}); + } + ''').create(); + + var test = await runTest( + ['test.dart'], + environment: {'DART_TEST_REPORTER': 'expanded'}, + ); + expect(test.stdout, neverEmits(contains('"testStart"'))); + expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); + await test.shouldExit(0); + }); + test('uses the specified concurrency', () async { await d.file('dart_test.yaml', jsonEncode({'concurrency': 2})).create(); diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index 97f83630e..3a945baa5 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart @@ -384,6 +384,51 @@ $_usage'''); await test.shouldExit(0); }); + test('directly with DART_TEST_REPORTER', () async { + await d.file('test.dart', _success).create(); + var test = await runDart( + ['test.dart'], + environment: {'DART_TEST_REPORTER': 'expanded'}, + ); + + expect( + test.stdout, + containsInOrder(['+0: success', '+1: All tests passed!']), + ); + await test.shouldExit(0); + }); + + test('with DART_TEST_REPORTER from runner', () async { + await d.file('test.dart', _success).create(); + var test = await runTest( + ['test.dart'], + environment: {'DART_TEST_REPORTER': 'expanded'}, + ); + + expect( + test.stdout, + containsInOrder(['+0: success', '+1: All tests passed!']), + ); + await test.shouldExit(0); + }); + + test('CLI flag overrides DART_TEST_REPORTER', () async { + await d.file('test.dart', _success).create(); + var test = await runTest( + ['test.dart', '--reporter', 'failures-only'], + environment: {'DART_TEST_REPORTER': 'expanded'}, + ); + + expect( + test.stdout, + allOf( + neverEmits(contains('success')), + emitsThrough(contains('All tests passed!')), + ), + ); + await test.shouldExit(0); + }); + // Regression test; this broke in 0.12.0-beta.9. test('on a file in a subdirectory', () async { await d.dir('dir', [d.file('test.dart', _success)]).create(); diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 63428a25c..226fb109b 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.6.20-wip + +* Add support for `DART_TEST_REPORTER` environment variable. + ## 0.6.19 * Add support for `-c cli` (the native CLI compiler) to the vm platform. diff --git a/pkgs/test_core/lib/src/executable.dart b/pkgs/test_core/lib/src/executable.dart index 750307d45..143b2ef67 100644 --- a/pkgs/test_core/lib/src/executable.dart +++ b/pkgs/test_core/lib/src/executable.dart @@ -14,6 +14,7 @@ import 'package:test_api/src/backend/util/pretty_print.dart'; // ignore: impleme import 'runner.dart'; import 'runner/application_exception.dart'; import 'runner/configuration.dart'; +import 'runner/configuration/reporters.dart'; import 'runner/no_tests_found_exception.dart'; import 'runner/version.dart'; import 'util/errors.dart'; @@ -110,6 +111,17 @@ Future _execute(List args) async { ); } + if (Platform.environment['DART_TEST_REPORTER'] case final envReporter?) { + if (!allReporters.containsKey(envReporter)) { + throw FormatException( + 'Unknown reporter "$envReporter" in DART_TEST_REPORTER.', + ); + } + fileConfiguration = fileConfiguration.merge( + Configuration.empty.change(reporter: envReporter), + ); + } + configuration = fileConfiguration.merge(configuration); } on SourceSpanFormatException catch (error) { stderr.writeln(error.toString(color: configuration.color)); diff --git a/pkgs/test_core/lib/src/runner/reporter/direct_io.dart b/pkgs/test_core/lib/src/runner/reporter/direct_io.dart index 461cc9bcd..0fa665afe 100644 --- a/pkgs/test_core/lib/src/runner/reporter/direct_io.dart +++ b/pkgs/test_core/lib/src/runner/reporter/direct_io.dart @@ -2,25 +2,38 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:io'; + import '../../util/io.dart'; import '../../util/print_sink.dart'; +import '../configuration.dart'; +import '../configuration/reporters.dart'; import '../engine.dart'; import '../reporter.dart'; import 'compact.dart'; import 'failures_only.dart'; -Reporter createDirectReporter(Engine engine) => canUseSpecialChars - ? CompactReporter.watch( - engine, - PrintSink(), - color: true, - printPath: false, - printPlatform: false, - ) - : FailuresOnlyReporter.watch( - engine, - PrintSink(), - color: false, - printPath: false, - printPlatform: false, - ); +Reporter createDirectReporter(Engine engine) { + if (Platform.environment['DART_TEST_REPORTER'] case final envReporter) { + if (allReporters[envReporter]?.factory case final factory?) { + return factory(Configuration.empty, engine, PrintSink()); + } + stderr.writeln('Unknown reporter "$envReporter" in DART_TEST_REPORTER.'); + } + + return canUseSpecialChars + ? CompactReporter.watch( + engine, + PrintSink(), + color: true, + printPath: false, + printPlatform: false, + ) + : FailuresOnlyReporter.watch( + engine, + PrintSink(), + color: false, + printPath: false, + printPlatform: false, + ); +} diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml index 1da8e56ec..e5f712533 100644 --- a/pkgs/test_core/pubspec.yaml +++ b/pkgs/test_core/pubspec.yaml @@ -1,5 +1,5 @@ name: test_core -version: 0.6.19 +version: 0.6.20-wip description: A basic library for writing tests and running them on the VM. repository: https://github.com/dart-lang/test/tree/master/pkgs/test_core issue_tracker: https://github.com/dart-lang/test/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Atest From 44707dd40ca3cb1b74121d7690cdb29dfeefdee0 Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Tue, 7 Jul 2026 22:16:33 +0000 Subject: [PATCH 2/2] Missed null check --- pkgs/test_core/lib/src/runner/reporter/direct_io.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/test_core/lib/src/runner/reporter/direct_io.dart b/pkgs/test_core/lib/src/runner/reporter/direct_io.dart index 0fa665afe..e5ccc7dae 100644 --- a/pkgs/test_core/lib/src/runner/reporter/direct_io.dart +++ b/pkgs/test_core/lib/src/runner/reporter/direct_io.dart @@ -14,7 +14,7 @@ import 'compact.dart'; import 'failures_only.dart'; Reporter createDirectReporter(Engine engine) { - if (Platform.environment['DART_TEST_REPORTER'] case final envReporter) { + if (Platform.environment['DART_TEST_REPORTER'] case final envReporter?) { if (allReporters[envReporter]?.factory case final factory?) { return factory(Configuration.empty, engine, PrintSink()); }