diff --git a/.ci.yaml b/.ci.yaml index f24589b43c5c..091cd3df2512 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -148,6 +148,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} + ] add_recipes_cq: "true" target_file: dart_unit_tests.yaml channel: master @@ -158,6 +162,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} + ] target_file: dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -167,6 +175,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} + ] target_file: dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -176,6 +188,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} + ] target_file: dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -1040,6 +1056,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} + ] target_file: windows_dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -1049,6 +1069,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} + ] target_file: windows_dart_unit_tests.yaml channel: master version_file: flutter_master.version diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index 16f02854cc3e..cf5420d8abed 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -6,13 +6,32 @@ library; import 'dart:async' show FutureOr; +import 'dart:io' as io show HttpClient, OSError, SocketException; import 'package:file/file.dart'; import 'package:file/local.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; import 'package:yaml/yaml.dart'; +import 'skia_client.dart'; +export 'skia_client.dart'; + +// If you are here trying to figure out how to use golden files for Flutter +// repos, consider reading this documentation: +// https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md + +// If you are trying to debug this package, you may like to use the golden test +// titled "Inconsequential golden test" in this file: packages/material_ui/test/goldens/goldens_test.dart + +// const String _kFlutterRootKey = 'FLUTTER_ROOT'; + +bool _isMainBranch(String? branch) { + return branch == 'main' || branch == 'master'; +} + /// Main method that can be used in a `flutter_test_config.dart` file to set /// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that /// works for the current test. _Which_ [FlutterGoldenFileComparator] is @@ -25,6 +44,12 @@ import 'package:yaml/yaml.dart'; /// tests using `flutter test`. This should not be called when running a test /// using `flutter run`, as in that environment, the [goldenFileComparator] is a /// [TrivialComparator]. +/// +/// An [HttpClient] is created when this method is called. That client is used +/// to communicate with the Skia Gold servers. Any [HttpOverrides] set in this +/// will affect whether this is effective or not. For example, if the current +/// override provides a mock client that always fails, then all calls to gold +/// comparison functions will fail. Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async { assert( goldenFileComparator is LocalFileComparator, @@ -35,44 +60,129 @@ Future testExecutable(FutureOr Function() testMain, {String? namePre '${goldenFileComparator.runtimeType}.\n' 'See also: https://flutter.dev/to/flutter-test-docs', ); + const Platform platform = LocalPlatform(); const FileSystem fs = LocalFileSystem(); + const ProcessManager process = LocalProcessManager(); + final httpClient = io.HttpClient(); namePrefix ??= FlutterGoldenFileComparator.getPackageName(fs); - goldenFileComparator = FlutterSkippingFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - 'Golden file testing is currently skipped.', - namePrefix: namePrefix, - fs: fs, - ); + if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { + goldenFileComparator = await FlutterPostSubmitFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + namePrefix: namePrefix, + log: (String _) {}, + fs: fs, + process: process, + httpClient: httpClient, + ); + } else if (FlutterPreSubmitFileComparator.isForEnvironment(platform)) { + goldenFileComparator = await FlutterPreSubmitFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + namePrefix: namePrefix, + log: (String _) {}, + fs: fs, + process: process, + httpClient: httpClient, + ); + } else if (FlutterSkippingFileComparator.isForEnvironment(platform)) { + goldenFileComparator = FlutterSkippingFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + 'Golden file testing is not executed on LUCI environments outside of ' + 'flutter, or in test shards that are not configured for using goldctl.', + platform: platform, + namePrefix: namePrefix, + log: (String _) {}, + fs: fs, + process: process, + httpClient: httpClient, + ); + } else { + goldenFileComparator = await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + log: (String _) {}, + fs: fs, + process: process, + httpClient: httpClient, + ); + } await testMain(); } /// Abstract base class golden file comparator specific to the `flutter/packages` /// repository. /// +/// Golden file testing for the `flutter/flutter` repository is handled by three +/// different [FlutterGoldenFileComparator]s, depending on the current testing +/// environment. +/// +/// * The [FlutterPostSubmitFileComparator] is utilized during post-submit +/// testing, after a pull request has landed on the master branch. This +/// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload +/// tests to the [Flutter Packages Gold dashboard](https://flutter-packages-gold.skia.org). +/// Flutter Gold manages the master golden files for the packages that use +/// matchesGoldenFile in the `flutter/packages` repository. +/// +/// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing, +/// before a pull request lands on the main branch. This +/// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing +/// contributors to view and check in visual differences before landing the +/// change. +/// +/// * The [FlutterLocalFileComparator] is used for local development testing. +/// This comparator will use the [SkiaGoldClient] to request baseline images +/// from [Flutter Packages Gold](https://flutter-packages-gold.skia.org) and +/// manually compare pixels. If a difference is detected, this comparator will +/// generate failure output illustrating the found difference. If a baseline +/// is not found for a given test image, it will consider it a new test and +/// output the new image for verification. +/// /// The [FlutterSkippingFileComparator] is utilized to skip tests outside -/// of the appropriate environments. Currently, some packages or environments -/// do not execute golden file testing, and as such do not require a -/// comparator. This comparator is also used when an internet connection is unavailable. +/// of the appropriate environments described above. Currently, some +/// packages or environments do not execute golden file testing, and as such do +/// not require a comparator. This comparator is also used when an internet +/// connection is unavailable. abstract class FlutterGoldenFileComparator extends GoldenFileComparator { /// Creates a [FlutterGoldenFileComparator] that will resolve golden file - /// URIs relative to the specified [basedir]. When testing locally, the - /// [basedir] will also contain any diffs from failed tests, or goldens - /// generated from newly introduced tests. + /// URIs relative to the specified [basedir], and retrieve golden baselines + /// using the [skiaClient]. The [basedir] is used for writing and accessing + /// information and files for interacting with the [skiaClient]. When testing + /// locally, the [basedir] will also contain any diffs from failed tests, or + /// goldens generated from newly introduced tests. @visibleForTesting - FlutterGoldenFileComparator(this.basedir, {required this.fs, this.namePrefix}); + FlutterGoldenFileComparator( + this.basedir, + this.skiaClient, { + required this.fs, + required this.platform, + this.namePrefix, + required this.log, + }); /// The directory to which golden file URIs will be resolved in [compare] and /// [update]. final Uri basedir; + /// A client for uploading image tests and making baseline requests to the + /// Flutter Gold Dashboard. + final SkiaGoldClient skiaClient; + /// The file system used to perform file access. final FileSystem fs; + /// The environment (current working directory, identity of the OS, + /// environment variables, etc). + final Platform platform; + /// The prefix that is added to all golden names. final String? namePrefix; + /// The logging function to use when reporting messages to the console. + final LogCallback log; + @override Future update(Uri golden, Uint8List imageBytes) async { final File goldenFile = getGoldenFile(golden); @@ -84,25 +194,32 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { Uri getTestUri(Uri key, int? version) => key; /// Calculate the appropriate basedir for the current test context. + /// + /// The optional [suffix] argument is used by the + /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator]. + /// These [FlutterGoldenFileComparator]s randomize their base directories to + /// maintain thread safety while using the `goldctl` tool. @protected @visibleForTesting static Directory getBaseDirectory( LocalFileComparator defaultComparator, { + required Platform platform, String? suffix, required FileSystem fs, }) { + // final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); final Directory comparisonRoot = switch (suffix) { - null => - fs.directory(fs.path.fromUri(defaultComparator.basedir)).childDirectory('skia_goldens'), + // null => flutterRoot.childDirectory(fs.path.join('bin', 'cache', 'pkg', 'skia_goldens')), + null => fs.directory(defaultComparator.basedir).childDirectory('skia_goldens'), _ => fs.systemTempDirectory.createTempSync(suffix), }; - return comparisonRoot; + return comparisonRoot; //.childDirectory(fs.path.relative(testPath, from: flutterRoot.path)); } /// Returns the golden [File] identified by the given [Uri]. @protected File getGoldenFile(Uri uri) { - final File goldenFile = fs.directory(fs.path.fromUri(basedir)).childFile(fs.path.fromUri(uri)); + final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); return goldenFile; } @@ -115,10 +232,8 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { final File pubspec = current.childFile('pubspec.yaml'); if (pubspec.existsSync()) { try { - final Object? yaml = loadYaml(pubspec.readAsStringSync()); - if (yaml is YamlMap) { - return yaml['name'] as String?; - } + final yaml = loadYaml(pubspec.readAsStringSync()) as YamlMap; + return yaml['name'] as String?; } catch (e) { // Ignore parsing errors and keep looking } @@ -127,14 +242,264 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { } return null; } + + /// Prepends the golden URL with the library name that encloses the current + /// test. + Uri _addPrefix(Uri golden) { + // Ensure the Uri ends in .png as the SkiaClient expects + assert( + golden.toString().split('.').last == 'png', + 'Golden files in the Flutter framework must end with the file extension ' + '.png.', + ); + + return Uri.parse([?namePrefix, golden.toString()].join('.')); + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in +/// post-submit. +/// +/// For testing across all platforms, the [SkiaGoldClient] is used to upload +/// images for framework-related golden tests and process results. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterPostSubmitFileComparator] that will test golden file + /// images against Skia Gold. + /// + /// The [fs] parameter is useful in tests, where the default + /// file system can be replaced by mock instances. + FlutterPostSubmitFileComparator( + super.basedir, + super.skiaClient, { + required super.fs, + required super.platform, + super.namePrefix, + required super.log, + }); + + /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative + /// path resolution of the provided `localFileComparator`. + /// + /// The [goldens] parameter is visible for testing purposes only. + static Future fromLocalFileComparator({ + SkiaGoldClient? goldens, + required LocalFileComparator localFileComparator, + required Platform platform, + String? namePrefix, + required LogCallback log, + required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, + }) async { + final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( + localFileComparator, + platform: platform, + suffix: 'flutter_goldens_postsubmit.', + fs: fs, + ); + + baseDirectory.createSync(recursive: true); + + goldens ??= SkiaGoldClient( + baseDirectory, + log: log, + platform: platform, + fs: fs, + process: process, + httpClient: httpClient, + ); + await goldens.auth(); + return FlutterPostSubmitFileComparator( + baseDirectory.uri, + goldens, + platform: platform, + namePrefix: namePrefix, + log: log, + fs: fs, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + await skiaClient.imgtestInit(); + golden = _addPrefix(golden); + + await update(golden, imageBytes); + final File goldenFile = getGoldenFile(golden); + try { + return await skiaClient.imgtestAdd(golden.path, goldenFile); + } on SkiaException catch (e) { + // Convert SkiaException -> TestFailure so that this class implements the + // contract of GoldenFileComparator, and matchesGoldenFile() converts the + // TestFailure into a standard reported test error (with a better stack + // trace, for example). + // + // https://github.com/flutter/flutter/issues/162621 + throw TestFailure('$e'); + } + } + + /// Decides based on the current environment if goldens tests should be + /// executed through Skia Gold. + static bool isForEnvironment(Platform platform) { + final bool luciPostSubmit = + platform.environment.containsKey('SWARMING_TASK_ID') && + platform.environment.containsKey('GOLDCTL') + // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. + && + !platform.environment.containsKey('GOLD_TRYJOB') + // Only run on main branch. + && + _isMainBranch(platform.environment['GIT_BRANCH']); + return luciPostSubmit; + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images before changes are +/// merged into the master branch. The comparator executes tryjobs using the +/// [SkiaGoldClient]. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPostSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold +/// dashboard in post-submit. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterPreSubmitFileComparator] that will test golden file + /// images against baselines requested from Flutter Gold. + /// + /// The [fs] parameter is useful in tests, where the default + /// file system can be replaced by mock instances. + FlutterPreSubmitFileComparator( + super.basedir, + super.skiaClient, { + required super.fs, + required super.platform, + super.namePrefix, + required super.log, + }); + + /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the + /// relative path resolution of the default [goldenFileComparator]. + /// + /// The [goldens] parameter is visible for testing purposes only. + static Future fromLocalFileComparator({ + SkiaGoldClient? goldens, + required LocalFileComparator localFileComparator, + required Platform platform, + Directory? testBasedir, + String? namePrefix, + required LogCallback log, + required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, + }) async { + final Directory baseDirectory = + testBasedir ?? + FlutterGoldenFileComparator.getBaseDirectory( + localFileComparator, + platform: platform, + suffix: 'flutter_goldens_presubmit.', + fs: fs, + ); + + if (!baseDirectory.existsSync()) { + baseDirectory.createSync(recursive: true); + } + + goldens ??= SkiaGoldClient( + baseDirectory, + platform: platform, + log: log, + fs: fs, + process: process, + httpClient: httpClient, + ); + + await goldens.auth(); + return FlutterPreSubmitFileComparator( + baseDirectory.uri, + goldens, + platform: platform, + namePrefix: namePrefix, + log: log, + fs: fs, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + await skiaClient.tryjobInit(); + golden = _addPrefix(golden); + + await update(golden, imageBytes); + final File goldenFile = getGoldenFile(golden); + + await skiaClient.tryjobAdd(golden.path, goldenFile); + + // This will always return true since golden file test failures are managed + // in pre-submit checks by the flutter-gold status check. + return true; + } + + /// Decides based on the current environment if goldens tests should be + /// executed as pre-submit tests with Skia Gold. + static bool isForEnvironment(Platform platform) { + final bool luciPreSubmit = + platform.environment.containsKey('SWARMING_TASK_ID') && + platform.environment.containsKey('GOLDCTL') && + platform.environment.containsKey('GOLD_TRYJOB') + // Only run on the main branch + && + _isMainBranch(platform.environment['GIT_BRANCH']); + return luciPreSubmit; + } } /// A [FlutterGoldenFileComparator] for testing conditions that do not execute /// golden file tests. +/// +/// Currently, this comparator is used on Luci environments when executing tests +/// outside of the flutter/flutter repository. +/// +/// See also: +/// +/// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator] +/// that tests golden images through Skia Gold. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { /// Creates a [FlutterSkippingFileComparator] that will skip tests that /// are not in the right environment for golden file testing. - FlutterSkippingFileComparator(super.basedir, this.reason, {super.namePrefix, required super.fs}); + FlutterSkippingFileComparator( + super.basedir, + super.skiaClient, + this.reason, { + super.namePrefix, + required super.platform, + required super.log, + required super.fs, + }); /// Describes the reason for using the [FlutterSkippingFileComparator]. final String reason; @@ -145,15 +510,200 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { String reason, { required LocalFileComparator localFileComparator, String? namePrefix, + required Platform platform, + required LogCallback log, required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, }) { final Uri basedir = localFileComparator.basedir; - return FlutterSkippingFileComparator(basedir, reason, namePrefix: namePrefix, fs: fs); + final skiaClient = SkiaGoldClient( + fs.directory(basedir), + platform: platform, + log: log, + fs: fs, + process: process, + httpClient: httpClient, + ); + return FlutterSkippingFileComparator( + basedir, + skiaClient, + reason, + namePrefix: namePrefix, + platform: platform, + log: log, + fs: fs, + ); } @override - Future compare(Uint8List imageBytes, Uri golden) async => true; + Future compare(Uint8List imageBytes, Uri golden) async { + log('Skipping "$golden" test: $reason'); + return true; + } @override Future update(Uri golden, Uint8List imageBytes) async {} + + /// Decides, based on the current environment, if this comparator should be + /// used. + /// + /// If we are in a CI environment, i.e. LUCI, but are not using the other + /// comparators, we skip. Otherwise we would fallback to the local comparator, + /// for which failures cannot be resolved in a CI environment. + static bool isForEnvironment(Platform platform) { + return platform.environment.containsKey('SWARMING_TASK_ID'); + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images locally on your +/// current machine. +/// +/// This comparator utilizes the [SkiaGoldClient] to request baseline images for +/// the given device under test for comparison. This comparator is initialized +/// when conditions for all other [FlutterGoldenFileComparator]s have not been +/// met, see the `isForEnvironment` method for each one listed below. +/// +/// The [FlutterLocalFileComparator] is intended to run on local machines and +/// serve as a smoke test during development. As such, it will not be able to +/// detect unintended changes on environments other than the currently executing +/// machine, until they are tested using the [FlutterPreSubmitFileComparator]. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPostSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold +/// dashboard. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterSkippingFileComparator], another +/// [FlutterGoldenFileComparator] that controls post-submit testing +/// conditions that do not execute golden file tests. +class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput { + /// Creates a [FlutterLocalFileComparator] that will test golden file + /// images against baselines requested from Flutter Gold. + /// + /// The [fs] parameter is useful in tests, where the default + /// file system can be replaced by mock instances. + FlutterLocalFileComparator( + super.basedir, + super.skiaClient, { + required super.fs, + required super.platform, + required super.log, + }); + + /// Creates a new [FlutterLocalFileComparator] that mirrors the + /// relative path resolution of the given [localFileComparator]. + /// + /// The [goldens] and [baseDirectory] parameters are + /// visible for testing purposes only. + static Future fromLocalFileComparator({ + SkiaGoldClient? goldens, + required LocalFileComparator localFileComparator, + required Platform platform, + Directory? baseDirectory, + required LogCallback log, + required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, + }) async { + baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( + localFileComparator, + platform: platform, + fs: fs, + ); + + if (!baseDirectory.existsSync()) { + baseDirectory.createSync(recursive: true); + } + + goldens ??= SkiaGoldClient( + baseDirectory, + platform: platform, + log: log, + fs: fs, + process: process, + httpClient: httpClient, + ); + try { + // Check if we can reach Gold. + await goldens.getExpectationForTest(''); + } on io.OSError catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'OSError occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + platform: platform, + log: log, + fs: fs, + ); + } on io.SocketException catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'SocketException occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + platform: platform, + log: log, + fs: fs, + ); + } on FormatException catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'FormatException occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + platform: platform, + log: log, + fs: fs, + ); + } + + return FlutterLocalFileComparator( + baseDirectory.uri, + goldens, + platform: platform, + log: log, + fs: fs, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + golden = _addPrefix(golden); + + final String testName = skiaClient.cleanTestName(golden.path); + late String? testExpectation; + testExpectation = await skiaClient.getExpectationForTest(testName); + + if (testExpectation == null || testExpectation.isEmpty) { + log( + 'No expectations provided by Skia Gold for test: $golden. ' + 'This may be a new test. If this is an unexpected result, check ' + 'https://flutter-packages-gold.skia.org.\n' + 'Validate image output found at $basedir', + ); + await update(golden, imageBytes); + return true; + } + + ComparisonResult result; + final List goldenBytes = await skiaClient.getImageBytes(testExpectation); + + result = await GoldenFileComparator.compareLists(imageBytes, goldenBytes); + + if (result.passed) { + result.dispose(); + return true; + } + + final String error = await generateFailureOutput(result, golden, basedir); + result.dispose(); + throw FlutterError(error); + } } diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart new file mode 100644 index 000000000000..3ca4fb90d37b --- /dev/null +++ b/script/flutter_goldens/lib/skia_client.dart @@ -0,0 +1,510 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// @docImport 'flutter_goldens.dart'; +library; + +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:crypto/crypto.dart'; +import 'package:file/file.dart'; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; + +// If you are here trying to figure out how to use golden files in relevant +// Flutter repos, consider reading this wiki page: +// https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md + +const String _kSDKKey = 'SDK_CHECKOUT_PATH'; +const String _kGoldctlKey = 'GOLDCTL'; + +/// Signature of callbacks used to inject [print] replacements. +typedef LogCallback = void Function(String); + +/// Exception thrown when an error is returned from the [SkiaGoldClient]. +class SkiaException implements Exception { + /// Creates a new `SkiaException` with a required error [message]. + const SkiaException(this.message); + + /// A message describing the error. + final String message; + + /// Returns a description of the Skia exception. + /// + /// The description always contains the [message]. + @override + String toString() => 'SkiaException: $message'; +} + +/// A client for uploading image tests and making baseline requests to the +/// Flutter Packages Gold Dashboard. +class SkiaGoldClient { + /// Creates a [SkiaGoldClient] with the given [workDirectory] and [Platform]. + /// + /// All other parameters are optional. They may be provided in tests to + /// override the defaults for [fs], [process], and [httpClient]. + SkiaGoldClient( + this.workDirectory, { + required this.fs, + required this.process, + required this.platform, + required this.httpClient, + required this.log, + }); + + /// The file system to use for storing the local clone of the repository. + /// + /// This is useful in tests, where a local file system (the default) can be + /// replaced by a memory file system. + final FileSystem fs; + + /// The environment (current working directory, identity of the OS, + /// environment variables, etc). + final Platform platform; + + /// A controller for launching sub-processes. + /// + /// This is useful in tests, where the real process manager (the default) can + /// be replaced by a mock process manager that doesn't really create + /// sub-processes. + final ProcessManager process; + + /// A client for making Http requests to the Flutter Packages Gold dashboard. + final io.HttpClient httpClient; + + /// The local [Directory] within the comparison root for the current test + /// context. In this directory, the client will create image and JSON files + /// for the goldctl tool to use. + /// + /// This is informed by [FlutterGoldenFileComparator.basedir]. It cannot be + /// null. + final Directory workDirectory; + + /// The logging function to use when reporting messages to the console. + final LogCallback log; + + /// The path to the local [Directory] where the goldctl tool is hosted. + /// + /// Uses the [platform] environment in this implementation. + String get _goldctl => platform.environment[_kGoldctlKey]!; + + /// Prepares the local work space for golden file testing and calls the + /// goldctl `auth` command. + /// + /// This ensures that the goldctl tool is authorized and ready for testing. + /// Used by the [FlutterPostSubmitFileComparator] and the + /// [FlutterPreSubmitFileComparator]. + Future auth() async { + if (await clientIsAuthorized()) { + return; + } + final authCommand = [ + _goldctl, + 'auth', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--luci', + ]; + + final io.ProcessResult result = await process.run(authCommand); + + if (result.exitCode != 0) { + final buf = StringBuffer() + ..writeln('Skia Gold authorization failed.') + ..writeln( + 'Luci environments authenticate using the file provided ' + 'by LUCI_CONTEXT. There may be an error with this file or Gold ' + 'authentication.', + ) + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}'); + throw SkiaException(buf.toString()); + } + } + + /// Signals if this client is initialized for uploading images to the Gold + /// service. + /// + /// Since Flutter framework tests are executed in parallel, and in random + /// order, this will signal is this instance of the Gold client has been + /// initialized. + bool _initialized = false; + + /// Executes the `imgtest init` command in the goldctl tool. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `init` argument initializes the current test. Used by the + /// [FlutterPostSubmitFileComparator]. + Future imgtestInit() async { + // This client has already been initialized + if (_initialized) { + return; + } + + final File keys = workDirectory.childFile('keys.json'); + final File failures = workDirectory.childFile('failures.json'); + + await keys.writeAsString(_getKeysJSON()); + await failures.create(); + final String commitHash = await _getCurrentCommit(); + + final imgtestInitCommand = [ + _goldctl, + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--commit', + commitHash, + '--keys-file', + keys.path, + '--failure-file', + failures.path, + '--passfail', + ]; + + if (imgtestInitCommand.contains(null)) { + final buf = StringBuffer() + ..writeln('A null argument was provided for Skia Gold imgtest init.') + ..writeln('Please confirm the settings of your golden file test.') + ..writeln('Arguments provided:'); + imgtestInitCommand.forEach(buf.writeln); + throw SkiaException(buf.toString()); + } + + final io.ProcessResult result = await process.run(imgtestInitCommand.cast()); + + if (result.exitCode != 0) { + _initialized = false; + final buf = StringBuffer() + ..writeln('Skia Gold imgtest init failed.') + ..writeln('An error occurred when initializing golden file test with ') + ..writeln('goldctl.') + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}'); + throw SkiaException(buf.toString()); + } + _initialized = true; + } + + /// Executes the `imgtest add` command in the goldctl tool. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `add` argument uploads the current image test. A response is + /// returned from the invocation of this command that indicates a pass or fail + /// result. + /// + /// The [testName] and [goldenFile] parameters reference the current + /// comparison being evaluated by the [FlutterPostSubmitFileComparator]. + Future imgtestAdd(String testName, File goldenFile) async { + final imgtestCommand = [ + _goldctl, + 'imgtest', + 'add', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--test-name', + cleanTestName(testName), + '--png-file', + goldenFile.path, + '--passfail', + ]; + + final io.ProcessResult result = await process.run(imgtestCommand); + + if (result.exitCode != 0) { + // If an unapproved image has made it to post-submit, throw to close the + // tree. + String? resultContents; + final File resultFile = workDirectory.childFile(fs.path.join('result-state.json')); + if (resultFile.existsSync()) { + resultContents = await resultFile.readAsString(); + } + + final buf = StringBuffer() + ..writeln('Skia Gold received an unapproved image in post-submit ') + ..writeln('testing. Golden file images in flutter/flutter are triaged ') + ..writeln('in pre-submit during code review for the given PR.') + ..writeln() + ..writeln('Visit https://flutter-gold.skia.org/ to view and approve ') + ..writeln('the image(s), or revert the associated change. For more ') + ..writeln('information, visit the wiki: ') + ..writeln( + 'https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md', + ) + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}') + ..writeln() + ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); + throw SkiaException(buf.toString()); + } + + return true; + } + + /// Signals if this client is initialized for uploading tryjobs to the Gold + /// service. + /// + /// Since Flutter framework tests are executed in parallel, and in random + /// order, this will signal is this instance of the Gold client has been + /// initialized for tryjobs. + bool _tryjobInitialized = false; + + /// Executes the `imgtest init` command in the goldctl tool for tryjobs. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `init` argument initializes the current tryjob. Used by the + /// [FlutterPreSubmitFileComparator]. + Future tryjobInit() async { + // This client has already been initialized + if (_tryjobInitialized) { + return; + } + + final File keys = workDirectory.childFile('keys.json'); + final File failures = workDirectory.childFile('failures.json'); + + await keys.writeAsString(_getKeysJSON()); + await failures.create(); + final String commitHash = await _getCurrentCommit(); + + final imgtestInitCommand = [ + _goldctl, + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--commit', + commitHash, + '--keys-file', + keys.path, + '--failure-file', + failures.path, + '--passfail', + '--crs', + 'github', + '--patchset_id', + commitHash, + ...getCIArguments(), + ]; + + if (imgtestInitCommand.contains(null)) { + final buf = StringBuffer() + ..writeln('A null argument was provided for Skia Gold tryjob init.') + ..writeln('Please confirm the settings of your golden file test.') + ..writeln('Arguments provided:'); + imgtestInitCommand.forEach(buf.writeln); + throw SkiaException(buf.toString()); + } + + final io.ProcessResult result = await process.run(imgtestInitCommand.cast()); + + if (result.exitCode != 0) { + _tryjobInitialized = false; + final buf = StringBuffer() + ..writeln('Skia Gold tryjobInit failure.') + ..writeln('An error occurred when initializing golden file tryjob with ') + ..writeln('goldctl.') + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}'); + throw SkiaException(buf.toString()); + } + _tryjobInitialized = true; + } + + /// Executes the `imgtest add` command in the goldctl tool for tryjobs. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `add` argument uploads the current image test. A response is + /// returned from the invocation of this command that indicates a pass or fail + /// result for the tryjob. + /// + /// The [testName] and [goldenFile] parameters reference the current + /// comparison being evaluated by the [FlutterPreSubmitFileComparator]. + /// + /// If the tryjob fails due to pixel differences, the method will succeed + /// as the failure will be triaged in the 'Flutter Gold' dashboard, and the + /// `stdout` will contain the failure message; otherwise will return `null`. + Future tryjobAdd(String testName, File goldenFile) async { + final imgtestCommand = [ + _goldctl, + 'imgtest', + 'add', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--test-name', + cleanTestName(testName), + '--png-file', + goldenFile.path, + ]; + + final io.ProcessResult result = await process.run(imgtestCommand); + + final resultStdout = result.stdout.toString(); + if (result.exitCode != 0 && + !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) { + String? resultContents; + final File resultFile = workDirectory.childFile(fs.path.join('result-state.json')); + if (resultFile.existsSync()) { + resultContents = await resultFile.readAsString(); + } + final buf = StringBuffer() + ..writeln('Unexpected Gold tryjobAdd failure.') + ..writeln('Tryjob execution for golden file test $testName failed for') + ..writeln('a reason unrelated to pixel comparison.') + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}') + ..writeln() + ..writeln() + ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); + throw SkiaException(buf.toString()); + } + return result.exitCode == 0 ? null : resultStdout; + } + + /// Returns the latest positive digest for the given test known to Flutter + /// Packages Gold at head. + Future getExpectationForTest(String testName) async { + late String? expectation; + final String traceID = getTraceID(testName); + final Uri requestForExpectations = Uri.parse( + 'https://flutter-packages-gold.skia.org/json/v2/latestpositivedigest/$traceID', + ); + late String rawResponse; + try { + final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations); + final io.HttpClientResponse response = await request.close(); + rawResponse = await utf8.decodeStream(response); + final dynamic jsonResponse = json.decode(rawResponse); + if (jsonResponse is! Map) { + throw const FormatException('Skia gold expectations do not match expected format.'); + } + expectation = jsonResponse['digest'] as String?; + } on FormatException catch (error) { + log( + 'Formatting error detected requesting expectations from Flutter Gold.\n' + 'error: $error\n' + 'url: $requestForExpectations\n' + 'response: $rawResponse', + ); + rethrow; + } + return expectation; + } + + /// Returns a list of bytes representing the golden image retrieved from the + /// Flutter Packages Gold dashboard. + /// + /// The provided image hash represents an expectation from Flutter Packages Gold. + Future> getImageBytes(String imageHash) async { + final imageBytes = []; + final Uri requestForImage = Uri.parse( + 'https://flutter-packages-gold.skia.org/img/images/$imageHash.png', + ); + final io.HttpClientRequest request = await httpClient.getUrl(requestForImage); + final io.HttpClientResponse response = await request.close(); + await response.forEach((List bytes) => imageBytes.addAll(bytes)); + return imageBytes; + } + + /// Returns the current commit hash of the packages repository. + Future _getCurrentCommit() async { + final String cleanPath = path.normalize(platform.environment[_kSDKKey]!); + + final io.ProcessResult revParse = await process.run([ + 'git', + 'rev-parse', + 'HEAD', + ], workingDirectory: cleanPath); + if (revParse.exitCode != 0) { + throw const SkiaException('Current commit of flutter/packages can not be found.'); + } + final String commit = (revParse.stdout as String).trim(); + + return commit; + } + + /// Returns a JSON String with keys value pairs used to uniquely identify the + /// configuration that generated the given golden file. + /// + /// Currently, the key value pairs being tracked are the platform the + /// image was rendered on and the Flutter channel the test was run on. + String _getKeysJSON() { + final keys = { + 'Platform': platform.operatingSystem, + 'CI': 'luci', + 'Channel': _channel, + }; + + return json.encode(keys); + } + + /// Removes the file extension from the [fileName] to represent the test name + /// properly. + String cleanTestName(String fileName) { + return fileName.split(path.extension(fileName))[0]; + } + + /// Returns a boolean value to prevent the client from re-authorizing itself + /// for multiple tests. + Future clientIsAuthorized() async { + final File authFile = workDirectory.childFile(fs.path.join('temp', 'auth_opt.json')); + + if (authFile.existsSync()) { + final String contents = await authFile.readAsString(); + final decoded = json.decode(contents) as Map; + return !(decoded['GSUtil'] as bool); + } + return false; + } + + /// Returns a list of arguments for initializing a tryjob based on the testing + /// environment. + List getCIArguments() { + final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last; + final List refs = platform.environment['GOLD_TRYJOB']!.split('/'); + final String pullRequest = refs[refs.length - 2]; + + return ['--changelist', pullRequest, '--cis', 'buildbucket', '--jobid', jobId]; + } + + String get _channel { + return platform.environment['CHANNEL'] ?? 'stable'; + } + + /// Returns a trace id based on the current testing environment to lookup + /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of + /// the image keys. + String getTraceID(String testName) { + final parameters = { + 'CI': 'luci', + 'Platform': platform.operatingSystem, + 'Channel': _channel, + 'name': testName, + 'source_type': 'flutter packages', + }; + final sorted = {}; + for (final String key in parameters.keys.toList()..sort()) { + sorted[key] = parameters[key]; + } + final String jsonTrace = json.encode(sorted); + final md5Sum = md5.convert(utf8.encode(jsonTrace)).toString(); + return md5Sum; + } +} diff --git a/script/flutter_goldens/test/comparator_selection_test.dart b/script/flutter_goldens/test/comparator_selection_test.dart new file mode 100644 index 000000000000..717ef092eb51 --- /dev/null +++ b/script/flutter_goldens/test/comparator_selection_test.dart @@ -0,0 +1,111 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_goldens/flutter_goldens.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:platform/platform.dart'; + +enum _Comparator { post, pre, skip, local } + +_Comparator _testRecommendations({ + bool hasLuci = false, + bool hasGold = false, + bool hasTryJob = false, + String branch = 'main', + String os = 'macos', +}) { + final Platform platform = FakePlatform( + environment: { + if (hasLuci) 'SWARMING_TASK_ID': '8675309', + if (hasGold) 'GOLDCTL': 'goldctl', + if (hasTryJob) 'GOLD_TRYJOB': 'git/ref/12345/head', + 'GIT_BRANCH': branch, + }, + operatingSystem: os, + ); + if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { + return _Comparator.post; + } + if (FlutterPreSubmitFileComparator.isForEnvironment(platform)) { + return _Comparator.pre; + } + if (FlutterSkippingFileComparator.isForEnvironment(platform)) { + return _Comparator.skip; + } + return _Comparator.local; +} + +void main() { + test('Comparator recommendations - main branch', () { + // If we're running locally (no CI), use a local comparator. + expect(_testRecommendations(), _Comparator.local); + expect(_testRecommendations(hasGold: true), _Comparator.local); + + // If we don't have gold but are on CI, we skip regardless. + expect(_testRecommendations(hasLuci: true), _Comparator.skip); + expect(_testRecommendations(hasLuci: true, hasTryJob: true), _Comparator.skip); + + // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. + expect(_testRecommendations(hasGold: true, hasLuci: true), _Comparator.post); + + // On Luci, with Gold, pre-submit. Flutter root and LUCI variables should have no effect. + expect(_testRecommendations(hasGold: true, hasLuci: true, hasTryJob: true), _Comparator.pre); + }); + + test('Comparator recommendations - release branch', () { + // If we're running locally (no CI), use a local comparator. + expect(_testRecommendations(branch: 'flutter-3.16-candidate.0'), _Comparator.local); + + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true), + _Comparator.local, + ); + + // If we don't have gold but are on CI, we skip regardless. + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true), + _Comparator.skip, + ); + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true, hasTryJob: true), + _Comparator.skip, + ); + + // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. Branch should make us skip. + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true), + _Comparator.skip, + ); + + // On Luci, with Gold, pre-submit. Flutter root and LUCI variables should have no effect. Branch should make us skip. + expect( + _testRecommendations( + branch: 'flutter-3.16-candidate.0', + hasGold: true, + hasLuci: true, + hasTryJob: true, + ), + _Comparator.skip, + ); + }); + + test('Comparator recommendations - Linux', () { + // If we're running locally (no CI), use a local comparator. + expect(_testRecommendations(os: 'linux'), _Comparator.local); + expect(_testRecommendations(os: 'linux', hasGold: true), _Comparator.local); + + // If we don't have gold but are on CI, we skip regardless. + expect(_testRecommendations(os: 'linux', hasLuci: true), _Comparator.skip); + expect(_testRecommendations(os: 'linux', hasLuci: true, hasTryJob: true), _Comparator.skip); + + // On Luci, with Gold, post-submit. Flutter root has no effect. + expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true), _Comparator.post); + + // On Luci, with Gold, pre-submit. Flutter root should have no effect. + expect( + _testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasTryJob: true), + _Comparator.pre, + ); + }); +} diff --git a/script/flutter_goldens/test/flutter_goldens_test.dart b/script/flutter_goldens/test/flutter_goldens_test.dart index 012c66e51217..05480da510db 100644 --- a/script/flutter_goldens/test/flutter_goldens_test.dart +++ b/script/flutter_goldens/test/flutter_goldens_test.dart @@ -4,30 +4,996 @@ // See also dev/automated_tests/flutter_test/flutter_gold_test.dart -import 'dart:typed_data'; +import 'dart:convert'; +import 'dart:io' hide Directory; import 'package:file/file.dart'; import 'package:file/memory.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter_goldens/flutter_goldens.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; + +import 'json_templates.dart'; const String _kFlutterRoot = '/flutter'; +// 1x1 transparent pixel +const List _kTestPngBytes = [ + 137, + 80, + 78, + 71, + 13, + 10, + 26, + 10, + 0, + 0, + 0, + 13, + 73, + 72, + 68, + 82, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 8, + 6, + 0, + 0, + 0, + 31, + 21, + 196, + 137, + 0, + 0, + 0, + 11, + 73, + 68, + 65, + 84, + 120, + 1, + 99, + 97, + 0, + 2, + 0, + 0, + 25, + 0, + 5, + 144, + 240, + 54, + 245, + 0, + 0, + 0, + 0, + 73, + 69, + 78, + 68, + 174, + 66, + 96, + 130, +]; + void main() { + group('SkiaGoldClient', () { + test('auth performs minimal work if already authorized', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos', environment: {}); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + final File authFile = fs.file('/workDirectory/temp/auth_opt.json') + ..createSync(recursive: true); + authFile.writeAsStringSync(authTemplate()); + process.fallbackProcessResult = ProcessResult(123, 0, '', ''); + await skiaClient.auth(); + + expect(process.workingDirectories, isEmpty); + }); + + test('gsutil is checked when authorization file is present', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos', environment: {}); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + final File authFile = fs.file('/workDirectory/temp/auth_opt.json') + ..createSync(recursive: true); + authFile.writeAsStringSync(authTemplate(gsutil: true)); + expect(await skiaClient.clientIsAuthorized(), isFalse); + }); + + test('throws for error state from auth', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLD_SERVICE_ACCOUNT': 'Service Account', + 'GOLDCTL': 'goldctl', + }, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect(skiaClient.auth(), throwsException); + }); + + test('throws for error state from init', () { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + environment: {'SDK_CHECKOUT_PATH': '/flutter', 'GOLDCTL': 'goldctl'}, + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + '/workDirectory/temp', + '--commit', + '12345678', + '--keys-file', + '/workDirectory/keys.json', + '--failure-file', + '/workDirectory/failures.json', + '--passfail', + ], null); + + process.processResults[gitInvocation] = ProcessResult(12345678, 0, '12345678', ''); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect(skiaClient.imgtestInit(), throwsException); + }); + + test('Only calls init once', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + environment: {'SDK_CHECKOUT_PATH': '/flutter', 'GOLDCTL': 'goldctl'}, + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + '/workDirectory/temp', + '--commit', + '1234', + '--keys-file', + '/workDirectory/keys.json', + '--failure-file', + '/workDirectory/failures.json', + '--passfail', + ], null); + + process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); + process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + // First call + await skiaClient.imgtestInit(); + + // Remove fake process result. + // If the init call is executed again, the fallback process will throw. + process.processResults.remove(goldctlInvocation); + + // Second call + await skiaClient.imgtestInit(); + }); + + test('Only calls tryjob init once', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'SWARMING_TASK_ID': '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB': 'refs/pull/49815/head', + 'SDK_CHECKOUT_PATH': '/flutter', + }, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + '/workDirectory/temp', + '--commit', + '1234', + '--keys-file', + '/workDirectory/keys.json', + '--failure-file', + '/workDirectory/failures.json', + '--passfail', + '--crs', + 'github', + '--patchset_id', + '1234', + '--changelist', + '49815', + '--cis', + 'buildbucket', + '--jobid', + '8885996262141582672', + ], null); + + process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); + process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + // First call + await skiaClient.tryjobInit(); + + // Remove fake process result. + // If the init call is executed again, the fallback process will throw. + process.processResults.remove(goldctlInvocation); + + // Second call + await skiaClient.tryjobInit(); + }); + + test('throws for error state from imgtestAdd', () { + final fs = MemoryFileSystem(); + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + final platform = FakePlatform( + environment: {'GOLDCTL': 'goldctl'}, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'add', + '--work-dir', + '/workDirectory/temp', + '--test-name', + 'golden_file_test', + '--png-file', + '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], null); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect(skiaClient.imgtestAdd('golden_file_test', goldenFile), throwsException); + }); + + test('correctly inits tryjob for luci', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'SWARMING_TASK_ID': '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB': 'refs/pull/49815/head', + }, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + final List ciArguments = skiaClient.getCIArguments(); + + expect( + ciArguments, + equals([ + '--changelist', + '49815', + '--cis', + 'buildbucket', + '--jobid', + '8885996262141582672', + ]), + ); + }); + + test('Creates traceID correctly', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'SWARMING_TASK_ID': '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB': 'refs/pull/49815/head', + }, + operatingSystem: 'linux', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + expect(skiaClient.getTraceID('flutter.golden.1'), equals('abe4ba07d57982f282adcd425aa8581f')); + }); + + test('Creates traceID correctly - locally - should defer to luci traceID', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos', environment: {}); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + expect(skiaClient.getTraceID('flutter.golden.1'), equals('405ca6a70c598037ab019d85f35f8357')); + }); + + test('throws for error state from imgtestAdd', () { + final fs = MemoryFileSystem(); + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + final platform = FakePlatform( + environment: {'GOLDCTL': 'goldctl'}, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'add', + '--work-dir', + '/workDirectory/temp', + '--test-name', + 'golden_file_test', + '--png-file', + '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], null); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect( + skiaClient.imgtestAdd('golden_file_test', goldenFile), + throwsA( + isA().having( + (SkiaException error) => error.message, + 'message', + contains('result-state.json'), + ), + ), + ); + }); + + test('throws for error state from tryjobAdd', () { + final fs = MemoryFileSystem(); + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + final platform = FakePlatform( + environment: {'GOLDCTL': 'goldctl'}, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'add', + '--work-dir', + '/workDirectory/temp', + '--test-name', + 'golden_file_test', + '--png-file', + '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], null); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + expect( + skiaClient.tryjobAdd('golden_file_test', goldenFile), + throwsA( + isA().having( + (SkiaException error) => error.message, + 'message', + contains('result-state.json'), + ), + ), + ); + }); + + group('Request Handling', () { + test('image bytes are processed properly', () async { + const expectation = '55109a4bed52acc780530f7a9aeff6c0'; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + final Uri imageUrl = Uri.parse( + 'https://flutter-packages-gold.skia.org/img/images/$expectation.png', + ); + final fakeImageRequest = FakeHttpClientRequest(); + final fakeImageResponse = FakeHttpImageResponse(imageResponseTemplate()); + + fakeHttpClient.request = fakeImageRequest; + fakeImageRequest.response = fakeImageResponse; + + final List masterBytes = await skiaClient.getImageBytes(expectation); + + expect(fakeHttpClient.lastUri, imageUrl); + expect(masterBytes, equals(_kTestPngBytes)); + }); + }); + }); + group('FlutterGoldenFileComparator', () { test('calculates the basedir correctly from defaultComparator for local testing', () async { final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final defaultComparator = FakeLocalFileComparator(); final Directory root = fs.directory('/')..createSync(recursive: true); defaultComparator.basedir = root.childDirectory('baz').uri; final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory( defaultComparator, + platform: platform, fs: fs, ); expect(basedir.uri, fs.directory('/baz/skia_goldens').uri); }); + test('ignores version number', () { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/')..createSync(recursive: true); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + FakeSkiaGoldClient(), + fs: fs, + platform: platform, + log: log.add, + ); + final Uri key = comparator.getTestUri(Uri.parse('foo.png'), 1); + expect(key, Uri.parse('foo.png')); + expect(log, isEmpty); + }); + + test('adds namePrefix', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + const packageName = 'sidedishes'; + const namePrefix = 'tomatosalad'; + const fileName = 'lettuce.png'; + final fakeSkiaClient = FakeSkiaGoldClient(); + final Directory basedir = fs.directory('$packageName/test/')..createSync(recursive: true); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + namePrefix: namePrefix, + log: log.add, + ); + await comparator.compare(Uint8List.fromList(_kTestPngBytes), Uri.parse(fileName)); + expect(fakeSkiaClient.testNames.single, '$namePrefix.$fileName'); + expect(log, isEmpty); + }); + + group('Post-Submit', () { + test('asserts .png format', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1'), + ); + }, + throwsA( + isA().having( + (AssertionError error) => error.toString(), + 'description', + contains( + 'Golden files in the Flutter framework must end with the file ' + 'extension .png.', + ), + ), + ), + ); + expect(log, isEmpty); + }); + + test('calls init during compare', () { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + expect(fakeSkiaClient.initCalls, 0); + comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + expect(fakeSkiaClient.initCalls, 1); + expect(log, isEmpty); + }); + + test('does not call init in during construction', () { + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + expect(fakeSkiaClient.initCalls, 0); + FlutterPostSubmitFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix), + platform: platform, + goldens: fakeSkiaClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(fakeSkiaClient.initCalls, 0); + }); + + test('reports a failure as a TestFailure', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + ThrowsOnImgTestAddSkiaClient( + message: 'Skia Gold received an unapproved image in post-submit', + ), + fs: fs, + platform: platform, + log: log.add, + ); + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + }, + throwsA( + isA().having( + (TestFailure error) => error.toString(), + 'description', + contains('Skia Gold received an unapproved image in post-submit'), + ), + ), + ); + expect(log, isEmpty); + }); + }); + + group('Pre-Submit', () { + test('asserts .png format', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1'), + ); + }, + throwsA( + isA().having( + (AssertionError error) => error.toString(), + 'description', + contains( + 'Golden files in the Flutter framework must end with the file ' + 'extension .png.', + ), + ), + ), + ); + expect(log, isEmpty); + }); + + test('calls init during compare', () { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + expect(fakeSkiaClient.tryInitCalls, 0); + comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + expect(fakeSkiaClient.tryInitCalls, 1); + expect(log, isEmpty); + }); + + test('does not call init in during construction', () { + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + expect(fakeSkiaClient.tryInitCalls, 0); + FlutterPostSubmitFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix), + platform: platform, + goldens: fakeSkiaClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(fakeSkiaClient.tryInitCalls, 0); + }); + }); + + group('Local', () { + test('asserts .png format', () async { + final log = []; + final fs = MemoryFileSystem(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterLocalFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: FakePlatform(operatingSystem: 'macos'), + log: log.add, + ); + const hash = '55109a4bed52acc780530f7a9aeff6c0'; + fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; + fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; + fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = 'flutter.golden_test.1'; + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1'), + ); + }, + throwsA( + isA().having( + (AssertionError error) => error.toString(), + 'description', + contains( + 'Golden files in the Flutter framework must end with the file ' + 'extension .png.', + ), + ), + ), + ); + expect(log, isEmpty); + }); + + test('passes when bytes match', () async { + final log = []; + final fs = MemoryFileSystem(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterLocalFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: FakePlatform(operatingSystem: 'macos'), + log: log.add, + ); + const hash = '55109a4bed52acc780530f7a9aeff6c0'; + fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; + fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; + fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = 'flutter.golden_test.1'; + expect( + await comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ), + isTrue, + ); + expect(log, isEmpty); + }); + + test( + 'returns FlutterSkippingGoldenFileComparator when network connection is unavailable', + () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + + const hash = '55109a4bed52acc780530f7a9aeff6c0'; + fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; + fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; + fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = 'flutter.golden_test.1'; + final fakeDirectory = FakeDirectory(); + fakeDirectory.existsSyncValue = true; + fakeDirectory.uri = Uri.parse('/flutter'); + + fakeSkiaClient.getExpectationForTestThrowable = const OSError("Can't reach Gold"); + final FlutterGoldenFileComparator comparator1 = + await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), + platform: platform, + goldens: fakeSkiaClient, + baseDirectory: fakeDirectory, + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(comparator1.runtimeType, FlutterSkippingFileComparator); + + fakeSkiaClient.getExpectationForTestThrowable = const SocketException("Can't reach Gold"); + final FlutterGoldenFileComparator comparator2 = + await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), + platform: platform, + goldens: fakeSkiaClient, + baseDirectory: fakeDirectory, + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(comparator2.runtimeType, FlutterSkippingFileComparator); + + fakeSkiaClient.getExpectationForTestThrowable = const FormatException("Can't reach Gold"); + final FlutterGoldenFileComparator comparator3 = + await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), + platform: platform, + goldens: fakeSkiaClient, + baseDirectory: fakeDirectory, + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(comparator3.runtimeType, FlutterSkippingFileComparator); + + // reset property or it will carry on to other tests + fakeSkiaClient.getExpectationForTestThrowable = null; + }, + ); + }); + group('_getPackageName', () { test('extracts name from pubspec.yaml', () { final fs = MemoryFileSystem(); @@ -66,21 +1032,175 @@ void main() { }); }); }); +} - group('FlutterSkippingFileComparator', () { - test('compare returns true', () async { - final fs = MemoryFileSystem(); - final comparator = FlutterSkippingFileComparator(Uri.parse('/basedir'), 'reason', fs: fs); - final bool result = await comparator.compare( - Uint8List.fromList([1, 2, 3]), - Uri.parse('golden.png'), +@immutable +class RunInvocation { + const RunInvocation(this.command, this.workingDirectory); + + final List command; + final String? workingDirectory; + + @override + int get hashCode => Object.hash(Object.hashAll(command), workingDirectory); + + bool _commandEquals(List other) { + if (other == command) { + return true; + } + if (other.length != command.length) { + return false; + } + for (var index = 0; index < other.length; index += 1) { + if (other[index] != command[index]) { + return false; + } + } + return true; + } + + @override + bool operator ==(Object other) { + if (other.runtimeType != runtimeType) { + return false; + } + return other is RunInvocation && + _commandEquals(other.command) && + other.workingDirectory == workingDirectory; + } + + @override + String toString() => '$command ($workingDirectory)'; +} + +class FakeProcessManager extends Fake implements ProcessManager { + Map processResults = {}; + + /// Used if [processResults] does not contain a matching invocation. + ProcessResult? fallbackProcessResult; + + final List workingDirectories = []; + + @override + Future run( + List command, { + String? workingDirectory, + Map? environment, + bool includeParentEnvironment = true, + bool runInShell = false, + Encoding? stdoutEncoding = systemEncoding, + Encoding? stderrEncoding = systemEncoding, + }) async { + workingDirectories.add(workingDirectory); + final ProcessResult? result = + processResults[RunInvocation(command.cast(), workingDirectory)]; + if (result == null && fallbackProcessResult == null) { + printOnFailure( + 'ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.', ); - expect(result, isTrue); - }); - }); + fail('See above.'); + } + return result ?? fallbackProcessResult!; + } +} + +// See also dev/automated_tests/flutter_test/flutter_gold_test.dart +class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { + Map expectationForTestValues = {}; + Exception? getExpectationForTestThrowable; + @override + Future getExpectationForTest(String testName) async { + if (getExpectationForTestThrowable != null) { + throw getExpectationForTestThrowable!; + } + return expectationForTestValues[testName] ?? ''; + } + + @override + Future auth() async {} + + final List testNames = []; + + int initCalls = 0; + @override + Future imgtestInit() async => initCalls += 1; + @override + Future imgtestAdd(String testName, File goldenFile) async { + testNames.add(testName); + return true; + } + + int tryInitCalls = 0; + @override + Future tryjobInit() async => tryInitCalls += 1; + @override + Future tryjobAdd(String testName, File goldenFile) async => null; + + Map> imageBytesValues = >{}; + @override + Future> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!; + + Map cleanTestNameValues = {}; + @override + String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? ''; +} + +class ThrowsOnImgTestAddSkiaClient extends Fake implements SkiaGoldClient { + ThrowsOnImgTestAddSkiaClient({required this.message}); + final String message; + + @override + Future imgtestInit() async { + // Assume this function works. + } + + @override + Future imgtestAdd(String testName, File goldenFile) { + throw SkiaException(message); + } } class FakeLocalFileComparator extends Fake implements LocalFileComparator { @override late Uri basedir; } + +class FakeDirectory extends Fake implements Directory { + late bool existsSyncValue; + @override + bool existsSync() => existsSyncValue; + + @override + late Uri uri; +} + +class FakeHttpClient extends Fake implements HttpClient { + late Uri lastUri; + late FakeHttpClientRequest request; + + @override + Future getUrl(Uri url) async { + lastUri = url; + return request; + } +} + +class FakeHttpClientRequest extends Fake implements HttpClientRequest { + late FakeHttpImageResponse response; + + @override + Future close() async { + return response; + } +} + +class FakeHttpImageResponse extends Fake implements HttpClientResponse { + FakeHttpImageResponse(this.response); + + final List> response; + + @override + Future forEach(void Function(List element) action) async { + response.forEach(action); + } +} diff --git a/script/flutter_goldens/test/json_templates.dart b/script/flutter_goldens/test/json_templates.dart new file mode 100644 index 000000000000..ee6c98a674ae --- /dev/null +++ b/script/flutter_goldens/test/json_templates.dart @@ -0,0 +1,94 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Json response template for the contents of the auth_opt.json file created by +/// goldctl. +String authTemplate({bool gsutil = false}) { + return ''' + { + "Luci":false, + "ServiceAccount":"${gsutil ? '' : '/packages/flutter/test/widgets/serviceAccount.json'}", + "GSUtil":$gsutil + } + '''; +} + +/// Json response template for Skia Gold image request: +/// https://flutter-gold.skia.org/img/images/{imageHash}.png +List> imageResponseTemplate() { + return >[ + [ + 137, + 80, + 78, + 71, + 13, + 10, + 26, + 10, + 0, + 0, + 0, + 13, + 73, + 72, + 68, + 82, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 8, + 6, + 0, + 0, + 0, + 31, + 21, + 196, + 137, + 0, + ], + [ + 0, + 0, + 11, + 73, + 68, + 65, + 84, + 120, + 1, + 99, + 97, + 0, + 2, + 0, + 0, + 25, + 0, + 5, + 144, + 240, + 54, + 245, + 0, + 0, + 0, + 0, + 73, + 69, + 78, + 68, + 174, + 66, + 96, + 130, + ], + ]; +}