diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 0c5cb3d452876..0158c116e1316 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1965,6 +1965,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/surface.dart + ../../../ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/surface_stats.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/transform.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html_image_codec.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/image_decoder.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/initialization.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/interval_tree.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/js_interop/js_loader.dart + ../../../flutter/LICENSE @@ -2018,6 +2019,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/shader_data.dart + ../../../f ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/shadow.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/canvas.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/filters.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/font_collection.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart + ../../../flutter/LICENSE @@ -2105,12 +2107,15 @@ ORIGIN: ../../../flutter/lib/web_ui/skwasm/filters.cpp + ../../../flutter/LICENS ORIGIN: ../../../flutter/lib/web_ui/skwasm/fonts.cpp + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/helpers.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/image.cpp + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/skwasm/library_skwasm_support.js + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/paint.cpp + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/path.cpp + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/picture.cpp + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/shaders.cpp + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/skwasm/skwasm_support.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/string.cpp + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/surface.cpp + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/skwasm/surface.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/text/line_metrics.cpp + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/text/paragraph.cpp + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/skwasm/text/paragraph_builder.cpp + ../../../flutter/LICENSE @@ -4613,6 +4618,7 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/surface.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/surface_stats.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/transform.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/html_image_codec.dart +FILE: ../../../flutter/lib/web_ui/lib/src/engine/image_decoder.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/initialization.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/interval_tree.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/js_interop/js_loader.dart @@ -4666,6 +4672,7 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/shader_data.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/shadow.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/canvas.dart +FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/filters.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/font_collection.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart @@ -4753,12 +4760,15 @@ FILE: ../../../flutter/lib/web_ui/skwasm/filters.cpp FILE: ../../../flutter/lib/web_ui/skwasm/fonts.cpp FILE: ../../../flutter/lib/web_ui/skwasm/helpers.h FILE: ../../../flutter/lib/web_ui/skwasm/image.cpp +FILE: ../../../flutter/lib/web_ui/skwasm/library_skwasm_support.js FILE: ../../../flutter/lib/web_ui/skwasm/paint.cpp FILE: ../../../flutter/lib/web_ui/skwasm/path.cpp FILE: ../../../flutter/lib/web_ui/skwasm/picture.cpp FILE: ../../../flutter/lib/web_ui/skwasm/shaders.cpp +FILE: ../../../flutter/lib/web_ui/skwasm/skwasm_support.h FILE: ../../../flutter/lib/web_ui/skwasm/string.cpp FILE: ../../../flutter/lib/web_ui/skwasm/surface.cpp +FILE: ../../../flutter/lib/web_ui/skwasm/surface.h FILE: ../../../flutter/lib/web_ui/skwasm/text/line_metrics.cpp FILE: ../../../flutter/lib/web_ui/skwasm/text/paragraph.cpp FILE: ../../../flutter/lib/web_ui/skwasm/text/paragraph_builder.cpp diff --git a/lib/web_ui/lib/src/engine.dart b/lib/web_ui/lib/src/engine.dart index a5ce06c5aaf98..6fb3113cd9b04 100644 --- a/lib/web_ui/lib/src/engine.dart +++ b/lib/web_ui/lib/src/engine.dart @@ -103,6 +103,7 @@ export 'engine/html/surface.dart'; export 'engine/html/surface_stats.dart'; export 'engine/html/transform.dart'; export 'engine/html_image_codec.dart'; +export 'engine/image_decoder.dart'; export 'engine/initialization.dart'; export 'engine/interval_tree.dart'; export 'engine/js_interop/js_loader.dart'; diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index 74220a502e40f..77fe97e73a3fe 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -381,17 +381,3 @@ class CkImage implements ui.Image, StackTraceDebugger { return '[$width\u00D7$height]'; } } - -/// Data for a single frame of an animated image. -class AnimatedImageFrameInfo implements ui.FrameInfo { - AnimatedImageFrameInfo(this._duration, this._image); - - final Duration _duration; - final CkImage _image; - - @override - Duration get duration => _duration; - - @override - ui.Image get image => _image; -} diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart index efe7c544d9085..337eac8679fdf 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart @@ -12,13 +12,9 @@ library image_wasm_codecs; import 'dart:async'; import 'dart:typed_data'; +import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; -import '../util.dart'; -import 'canvaskit_api.dart'; -import 'image.dart'; -import 'native_memory.dart'; - /// The CanvasKit implementation of [ui.Codec]. /// /// Wraps `SkAnimatedImage`. diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index e46c33d28e2be..05ef367e1b1f9 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -14,33 +14,15 @@ import 'dart:js_interop'; import 'dart:math' as math; import 'dart:typed_data'; -import 'package:meta/meta.dart'; import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; -Duration _kDefaultWebDecoderExpireDuration = const Duration(seconds: 3); -Duration _kWebDecoderExpireDuration = _kDefaultWebDecoderExpireDuration; - -/// Overrides the inactivity duration after which the web decoder is closed. -/// -/// This should only be used in tests. -void debugOverrideWebDecoderExpireDuration(Duration override) { - _kWebDecoderExpireDuration = override; -} - -/// Restores the web decoder inactivity expiry duration to its original value. -/// -/// This should only be used in tests. -void debugRestoreWebDecoderExpireDuration() { - _kWebDecoderExpireDuration = _kDefaultWebDecoderExpireDuration; -} - /// Image decoder backed by the browser's `ImageDecoder`. -class CkBrowserImageDecoder implements ui.Codec { +class CkBrowserImageDecoder extends BrowserImageDecoder { CkBrowserImageDecoder._({ - required this.contentType, - required this.data, - required this.debugSource, + required super.contentType, + required super.dataSource, + required super.debugSource, }); static Future create({ @@ -67,156 +49,17 @@ class CkBrowserImageDecoder implements ui.Codec { final CkBrowserImageDecoder decoder = CkBrowserImageDecoder._( contentType: contentType, - data: data, + dataSource: data.toJS, debugSource: debugSource, ); // Call once to initialize the decoder and populate late fields. - await decoder._getOrCreateWebDecoder(); + await decoder.initialize(); return decoder; } - final String contentType; - final Uint8List data; - final String debugSource; - - @override - late int frameCount; - - @override - late int repetitionCount; - - /// Whether this decoder has been disposed of. - /// - /// Once this turns true it stays true forever, and this decoder becomes - /// unusable. - bool _isDisposed = false; - @override - void dispose() { - _isDisposed = true; - - // This releases all resources, including any currently running decoding work. - _cachedWebDecoder?.close(); - _cachedWebDecoder = null; - } - - void _debugCheckNotDisposed() { - assert( - !_isDisposed, - 'Cannot use this image decoder. It has been disposed of.' - ); - } - - /// The index of the frame that will be decoded on the next call of [getNextFrame]; - int _nextFrameIndex = 0; - - /// Creating a new decoder is expensive, so we cache the decoder for reuse. - /// - /// This decoder is closed and the field is nulled out after some time of - /// inactivity. - ImageDecoder? _cachedWebDecoder; - - /// The underlying image decoder used to decode images. - /// - /// This value is volatile. It may be closed or become null any time. - /// - /// - /// This is only meant to be used in tests. - @visibleForTesting - ImageDecoder? get debugCachedWebDecoder => _cachedWebDecoder; - - final AlarmClock _cacheExpirationClock = AlarmClock(() => DateTime.now()); - - Future _getOrCreateWebDecoder() async { - if (_cachedWebDecoder != null) { - // Give the cached value some time for reuse, e.g. if the image is - // currently animating. - _cacheExpirationClock.datetime = DateTime.now().add(_kWebDecoderExpireDuration); - return _cachedWebDecoder!; - } - - // Null out the callback so the clock doesn't try to expire the decoder - // while it's initializing. There's no way to tell how long the - // initialization will take place. We just let it proceed at its own pace. - _cacheExpirationClock.callback = null; - try { - final ImageDecoder webDecoder = ImageDecoder(ImageDecoderOptions( - type: contentType.toJS, - data: data.toJS, - - // Flutter always uses premultiplied alpha when decoding. - premultiplyAlpha: 'premultiply'.toJS, - // "default" gives the browser the liberty to convert to display-appropriate - // color space, typically SRGB, which is what we want. - colorSpaceConversion: 'default'.toJS, - - // Flutter doesn't give the developer a way to customize this, so if this - // is an animated image we should prefer the animated track. - preferAnimation: true.toJS, - )); - - await promiseToFuture(webDecoder.tracks.ready); - - // Flutter doesn't have an API for progressive loading of images, so we - // wait until the image is fully decoded. - // package:js bindings don't work with getters that return a Promise, which - // is why js_util is used instead. - await promiseToFuture(getJsProperty(webDecoder, 'completed')); - frameCount = webDecoder.tracks.selectedTrack!.frameCount.toInt(); - - // We coerce the DOM's `repetitionCount` into an int by explicitly - // handling `infinity`. Note: This will still throw if the DOM returns a - // `NaN. - final double rawRepetitionCount = webDecoder.tracks.selectedTrack!.repetitionCount; - repetitionCount = rawRepetitionCount == double.infinity ? -1 : - rawRepetitionCount.toInt(); - _cachedWebDecoder = webDecoder; - - // Expire the decoder if it's not used for several seconds. If the image is - // not animated, it could mean that the framework has cached the frame and - // therefore doesn't need the decoder any more, or it could mean that the - // widget is gone and it's time to collect resources associated with it. - // If it's an animated image it means the animation has stopped, otherwise - // we'd see calls to [getNextFrame] which would update the expiry date on - // the decoder. If the animation is stopped for long enough, it's better - // to collect resources. If and when the animation resumes, a new decoder - // will be instantiated. - _cacheExpirationClock.callback = () { - _cachedWebDecoder?.close(); - _cachedWebDecoder = null; - _cacheExpirationClock.callback = null; - }; - _cacheExpirationClock.datetime = DateTime.now().add(_kWebDecoderExpireDuration); - - return webDecoder; - } catch (error) { - if (domInstanceOfString(error, 'DOMException')) { - if ((error as DomException).name == DomException.notSupported) { - throw ImageCodecException( - "Image file format ($contentType) is not supported by this browser's ImageDecoder API.\n" - 'Image source: $debugSource', - ); - } - } - throw ImageCodecException( - "Failed to decode image using the browser's ImageDecoder API.\n" - 'Image source: $debugSource\n' - 'Original browser error: $error' - ); - } - } - - @override - Future getNextFrame() async { - _debugCheckNotDisposed(); - final ImageDecoder webDecoder = await _getOrCreateWebDecoder(); - final DecodeResult result = await promiseToFuture( - webDecoder.decode(DecodeOptions(frameIndex: _nextFrameIndex.toJS)), - ); - final VideoFrame frame = result.image; - _nextFrameIndex = (_nextFrameIndex + 1) % frameCount; - + ui.Image generateImageFromVideoFrame(VideoFrame frame) { final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSource( frame, SkPartialImageInfo( @@ -227,154 +70,14 @@ class CkBrowserImageDecoder implements ui.Codec { height: frame.displayHeight, ), ); - - // Duration can be null if the image is not animated. However, Flutter - // requires a non-null value. 0 indicates that the frame is meant to be - // displayed indefinitely, which is fine for a static image. - final Duration duration = Duration(microseconds: frame.duration?.toInt() ?? 0); - if (skImage == null) { throw ImageCodecException( "Failed to create image from pixel data decoded using the browser's ImageDecoder.", ); } - final CkImage image = CkImage(skImage, videoFrame: frame); - return Future.value(AnimatedImageFrameInfo(duration, image)); - } -} - -/// Represents an image file format, such as PNG or JPEG. -class ImageFileFormat { - const ImageFileFormat(this.header, this.contentType); - - /// First few bytes in the file that uniquely identify the image file format. - /// - /// Null elements are treated as wildcard values and are not checked. This is - /// used to detect formats whose header is split up into multiple disjoint - /// parts, such that the first part is not unique enough to identify the - /// format. For example, without this, WebP may be confused with .ani - /// (animated cursor), .cda, and other formats that start with "RIFF". - final List header; - - /// The value that's passed as [_ImageDecoderOptions.type]. - /// - /// The server typically also uses this value as the "Content-Type" header, - /// but servers are not required to correctly detect the type. This value - /// is also known as MIME type. - final String contentType; - - /// All image file formats known to the Flutter Web engine. - /// - /// This list may need to be changed as browsers adopt new formats, and drop - /// support for obsolete ones. - /// - /// This list is checked linearly from top to bottom when detecting an image - /// type. It should therefore contain the most popular file formats at the - /// top, and less popular towards the bottom. - static const List values = [ - // ICO is not supported in Chrome. It is deemed too simple and too specific. See also: - // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/modules/webcodecs/image_decoder_external.cc;l=38;drc=fd8802b593110ea18a97ef044f8a40dd24a622ec - - // PNG - ImageFileFormat([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], 'image/png'), - - // GIF87a - ImageFileFormat([0x47, 0x49, 0x46, 0x38, 0x37, 0x61], 'image/gif'), - - // GIF89a - ImageFileFormat([0x47, 0x49, 0x46, 0x38, 0x39, 0x61], 'image/gif'), - - // JPEG - ImageFileFormat([0xFF, 0xD8, 0xFF], 'image/jpeg'), - - // WebP - ImageFileFormat([0x52, 0x49, 0x46, 0x46, null, null, null, null, 0x57, 0x45, 0x42, 0x50], 'image/webp'), - - // BMP - ImageFileFormat([0x42, 0x4D], 'image/bmp'), - ]; -} - -/// Function signature of [debugContentTypeDetector], which is the same as the -/// signature of [detectContentType]. -typedef DebugContentTypeDetector = String? Function(Uint8List); - -/// If not null, replaced the functionality of [detectContentType] with its own. -/// -/// This is useful in tests, for example, to test unsupported content types. -DebugContentTypeDetector? debugContentTypeDetector; - -/// Detects the image file format and returns the corresponding "Content-Type" -/// value (a.k.a. MIME type). -/// -/// The returned value can be passed to `ImageDecoder` when decoding an image. -/// -/// Returns null if [data] cannot be mapped to a known content type. -String? detectContentType(Uint8List data) { - if (debugContentTypeDetector != null) { - return debugContentTypeDetector!.call(data); - } - - formatLoop: for (final ImageFileFormat format in ImageFileFormat.values) { - if (data.length < format.header.length) { - continue; - } - - for (int i = 0; i < format.header.length; i++) { - final int? magicByte = format.header[i]; - if (magicByte == null) { - // Wildcard, accepts everything. - continue; - } - - final int headerByte = data[i]; - if (headerByte != magicByte) { - continue formatLoop; - } - } - - return format.contentType; - } - - if (isAvif(data)) { - return 'image/avif'; - } - - return null; -} - -/// A string of bytes that every AVIF image contains somehwere in its first 16 -/// bytes. -/// -/// This signature is necessary but not sufficient, which may lead to false -/// positives. For example, the file may be HEIC or a video. This is OK, -/// because in the worst case, the image decoder fails to decode the file. -/// This is something we must anticipate regardless of this detection logic. -/// The codec must already protect itself from downloaded files lying about -/// their contents. -/// -/// The alternative would be to implement a more precise detection, which would -/// add complexity and code size. This is how Chromium does it: -/// -/// https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/image-decoders/avif/avif_image_decoder.cc;l=504;drc=fd8802b593110ea18a97ef044f8a40dd24a622ec -final List _avifSignature = 'ftyp'.codeUnits; - -/// Optimistically detects whether [data] is an AVIF image file. -bool isAvif(Uint8List data) { - firstByteLoop: for (int i = 0; i < 16; i += 1) { - for (int j = 0; j < _avifSignature.length; j += 1) { - if (i + j >= data.length) { - // Reached EOF without finding the signature. - return false; - } - if (data[i + j] != _avifSignature[j]) { - continue firstByteLoop; - } - } - return true; + return CkImage(skImage, videoFrame: frame); } - return false; } Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFormat format) async { diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index 45f4df4b70f95..07fd58edfe841 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -202,7 +202,8 @@ class CanvasKitRenderer implements Renderer { }) async => skiaInstantiateImageCodec( list, targetWidth, - targetHeight); + targetHeight + ); @override Future instantiateImageCodecFromUrl( diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart index 82d5b00b5ef00..7e381fe5c476c 100644 --- a/lib/web_ui/lib/src/engine/dom.dart +++ b/lib/web_ui/lib/src/engine/dom.dart @@ -1394,8 +1394,8 @@ extension DomCanvasGradientExtension on DomCanvasGradient { @staticInterop class DomXMLHttpRequestEventTarget extends DomEventTarget {} -Future<_DomResponse> _rawHttpGet(String url) => - js_util.promiseToFuture<_DomResponse>(domWindow._fetch1(url.toJS)); +Future rawHttpGet(String url) => + js_util.promiseToFuture(domWindow._fetch1(url.toJS)); typedef MockHttpFetchResponseFactory = Future Function( String url); @@ -1423,15 +1423,15 @@ Future httpFetch(String url) async { } } try { - final _DomResponse domResponse = await _rawHttpGet(url); + final DomResponse domResponse = await rawHttpGet(url); return HttpFetchResponseImpl._(url, domResponse); } catch (requestError) { throw HttpFetchError(url, requestError: requestError); } } -Future<_DomResponse> _rawHttpPost(String url, String data) => - js_util.promiseToFuture<_DomResponse>(domWindow._fetch2( +Future _rawHttpPost(String url, String data) => + js_util.promiseToFuture(domWindow._fetch2( url.toJS, { 'method': 'POST', @@ -1449,7 +1449,7 @@ Future<_DomResponse> _rawHttpPost(String url, String data) => @visibleForTesting Future testOnlyHttpPost(String url, String data) async { try { - final _DomResponse domResponse = await _rawHttpPost(url, data); + final DomResponse domResponse = await _rawHttpPost(url, data); return HttpFetchResponseImpl._(url, domResponse); } catch (requestError) { throw HttpFetchError(url, requestError: requestError); @@ -1542,7 +1542,7 @@ class HttpFetchResponseImpl implements HttpFetchResponse { @override final String url; - final _DomResponse _domResponse; + final DomResponse _domResponse; @override int get status => _domResponse.status; @@ -1624,7 +1624,7 @@ abstract class HttpFetchPayload { class HttpFetchPayloadImpl implements HttpFetchPayload { HttpFetchPayloadImpl._(this._domResponse); - final _DomResponse _domResponse; + final DomResponse _domResponse; @override Future read(HttpFetchReader callback) async { @@ -1743,14 +1743,14 @@ class HttpFetchError implements Exception { @JS() @staticInterop -class _DomResponse {} +class DomResponse {} -extension _DomResponseExtension on _DomResponse { +extension DomResponseExtension on DomResponse { @JS('status') external JSNumber get _status; int get status => _status.toDart.toInt(); - external _DomHeaders get headers; + external DomHeaders get headers; external _DomReadableStream get body; @@ -1770,9 +1770,9 @@ extension _DomResponseExtension on _DomResponse { @JS() @staticInterop -class _DomHeaders {} +class DomHeaders {} -extension _DomHeadersExtension on _DomHeaders { +extension DomHeadersExtension on DomHeaders { @JS('get') external JSString? _get(JSString? headerName); String? get(String? headerName) => _get(headerName?.toJS)?.toDart; diff --git a/lib/web_ui/lib/src/engine/image_decoder.dart b/lib/web_ui/lib/src/engine/image_decoder.dart new file mode 100644 index 0000000000000..a6e3bbd524681 --- /dev/null +++ b/lib/web_ui/lib/src/engine/image_decoder.dart @@ -0,0 +1,442 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:js_interop'; +import 'dart:typed_data'; + +import 'package:meta/meta.dart'; +import 'package:ui/src/engine.dart'; +import 'package:ui/ui.dart' as ui; + +Duration _kDefaultWebDecoderExpireDuration = const Duration(seconds: 3); +Duration _kWebDecoderExpireDuration = _kDefaultWebDecoderExpireDuration; + +/// Overrides the inactivity duration after which the web decoder is closed. +/// +/// This should only be used in tests. +void debugOverrideWebDecoderExpireDuration(Duration override) { + _kWebDecoderExpireDuration = override; +} + +/// Restores the web decoder inactivity expiry duration to its original value. +/// +/// This should only be used in tests. +void debugRestoreWebDecoderExpireDuration() { + _kWebDecoderExpireDuration = _kDefaultWebDecoderExpireDuration; +} + +/// Image decoder backed by the browser's `ImageDecoder`. +abstract class BrowserImageDecoder implements ui.Codec { + BrowserImageDecoder({ + required this.contentType, + required this.dataSource, + required this.debugSource, + }); + + final String contentType; + final JSAny dataSource; + final String debugSource; + + @override + late int frameCount; + + @override + late int repetitionCount; + + /// Whether this decoder has been disposed of. + /// + /// Once this turns true it stays true forever, and this decoder becomes + /// unusable. + bool _isDisposed = false; + + @override + void dispose() { + _isDisposed = true; + + // This releases all resources, including any currently running decoding work. + _cachedWebDecoder?.close(); + _cachedWebDecoder = null; + } + + void _debugCheckNotDisposed() { + assert( + !_isDisposed, + 'Cannot use this image decoder. It has been disposed of.' + ); + } + + /// The index of the frame that will be decoded on the next call of [getNextFrame]; + int _nextFrameIndex = 0; + + /// Creating a new decoder is expensive, so we cache the decoder for reuse. + /// + /// This decoder is closed and the field is nulled out after some time of + /// inactivity. + /// + // TODO(jacksongardner): Evaluate whether this complexity is necessary. + // See https://github.com/flutter/flutter/issues/127548 + ImageDecoder? _cachedWebDecoder; + + /// The underlying image decoder used to decode images. + /// + /// This value is volatile. It may be closed or become null any time. + /// + /// + /// This is only meant to be used in tests. + @visibleForTesting + ImageDecoder? get debugCachedWebDecoder => _cachedWebDecoder; + + final AlarmClock _cacheExpirationClock = AlarmClock(() => DateTime.now()); + + Future initialize() => _getOrCreateWebDecoder(); + + Future _getOrCreateWebDecoder() async { + if (_cachedWebDecoder != null) { + // Give the cached value some time for reuse, e.g. if the image is + // currently animating. + _cacheExpirationClock.datetime = DateTime.now().add(_kWebDecoderExpireDuration); + return _cachedWebDecoder!; + } + + // Null out the callback so the clock doesn't try to expire the decoder + // while it's initializing. There's no way to tell how long the + // initialization will take place. We just let it proceed at its own pace. + _cacheExpirationClock.callback = null; + try { + final ImageDecoder webDecoder = ImageDecoder(ImageDecoderOptions( + type: contentType.toJS, + data: dataSource, + + // Flutter always uses premultiplied alpha when decoding. + premultiplyAlpha: 'premultiply'.toJS, + // "default" gives the browser the liberty to convert to display-appropriate + // color space, typically SRGB, which is what we want. + colorSpaceConversion: 'default'.toJS, + + // Flutter doesn't give the developer a way to customize this, so if this + // is an animated image we should prefer the animated track. + preferAnimation: true.toJS, + )); + + await promiseToFuture(webDecoder.tracks.ready); + + // Flutter doesn't have an API for progressive loading of images, so we + // wait until the image is fully decoded. + // package:js bindings don't work with getters that return a Promise, which + // is why js_util is used instead. + await promiseToFuture(getJsProperty(webDecoder, 'completed')); + frameCount = webDecoder.tracks.selectedTrack!.frameCount.toInt(); + + // We coerce the DOM's `repetitionCount` into an int by explicitly + // handling `infinity`. Note: This will still throw if the DOM returns a + // `NaN`. + final double rawRepetitionCount = webDecoder.tracks.selectedTrack!.repetitionCount; + repetitionCount = rawRepetitionCount == double.infinity ? -1 : + rawRepetitionCount.toInt(); + _cachedWebDecoder = webDecoder; + + // Expire the decoder if it's not used for several seconds. If the image is + // not animated, it could mean that the framework has cached the frame and + // therefore doesn't need the decoder any more, or it could mean that the + // widget is gone and it's time to collect resources associated with it. + // If it's an animated image it means the animation has stopped, otherwise + // we'd see calls to [getNextFrame] which would update the expiry date on + // the decoder. If the animation is stopped for long enough, it's better + // to collect resources. If and when the animation resumes, a new decoder + // will be instantiated. + _cacheExpirationClock.callback = () { + _cachedWebDecoder?.close(); + _cachedWebDecoder = null; + _cacheExpirationClock.callback = null; + }; + _cacheExpirationClock.datetime = DateTime.now().add(_kWebDecoderExpireDuration); + + return webDecoder; + } catch (error) { + if (domInstanceOfString(error, 'DOMException')) { + if ((error as DomException).name == DomException.notSupported) { + throw ImageCodecException( + "Image file format ($contentType) is not supported by this browser's ImageDecoder API.\n" + 'Image source: $debugSource', + ); + } + } + throw ImageCodecException( + "Failed to decode image using the browser's ImageDecoder API.\n" + 'Image source: $debugSource\n' + 'Original browser error: $error' + ); + } + } + + @override + Future getNextFrame() async { + _debugCheckNotDisposed(); + final ImageDecoder webDecoder = await _getOrCreateWebDecoder(); + final DecodeResult result = await promiseToFuture( + webDecoder.decode(DecodeOptions(frameIndex: _nextFrameIndex.toJS)), + ); + final VideoFrame frame = result.image; + _nextFrameIndex = (_nextFrameIndex + 1) % frameCount; + + // Duration can be null if the image is not animated. However, Flutter + // requires a non-null value. 0 indicates that the frame is meant to be + // displayed indefinitely, which is fine for a static image. + final Duration duration = Duration(microseconds: frame.duration?.toInt() ?? 0); + final ui.Image image = generateImageFromVideoFrame(frame); + return AnimatedImageFrameInfo(duration, image); + } + + ui.Image generateImageFromVideoFrame(VideoFrame frame); +} + +/// Data for a single frame of an animated image. +class AnimatedImageFrameInfo implements ui.FrameInfo { + AnimatedImageFrameInfo(this.duration, this.image); + + @override + final Duration duration; + + @override + final ui.Image image; +} + +/// Detects the image file format and returns the corresponding "Content-Type" +/// value (a.k.a. MIME type). +/// +/// The returned value can be passed to `ImageDecoder` when decoding an image. +/// +/// Returns null if [data] cannot be mapped to a known content type. +String? detectContentType(Uint8List data) { + if (debugContentTypeDetector != null) { + return debugContentTypeDetector!.call(data); + } + + formatLoop: for (final ImageFileFormat format in ImageFileFormat.values) { + if (data.length < format.header.length) { + continue; + } + + for (int i = 0; i < format.header.length; i++) { + final int? magicByte = format.header[i]; + if (magicByte == null) { + // Wildcard, accepts everything. + continue; + } + + final int headerByte = data[i]; + if (headerByte != magicByte) { + continue formatLoop; + } + } + + return format.contentType; + } + + if (isAvif(data)) { + return 'image/avif'; + } + + return null; +} + +/// Represents an image file format, such as PNG or JPEG. +class ImageFileFormat { + const ImageFileFormat(this.header, this.contentType); + + /// First few bytes in the file that uniquely identify the image file format. + /// + /// Null elements are treated as wildcard values and are not checked. This is + /// used to detect formats whose header is split up into multiple disjoint + /// parts, such that the first part is not unique enough to identify the + /// format. For example, without this, WebP may be confused with .ani + /// (animated cursor), .cda, and other formats that start with "RIFF". + final List header; + + /// The value that's passed as [_ImageDecoderOptions.type]. + /// + /// The server typically also uses this value as the "Content-Type" header, + /// but servers are not required to correctly detect the type. This value + /// is also known as MIME type. + final String contentType; + + /// All image file formats known to the Flutter Web engine. + /// + /// This list may need to be changed as browsers adopt new formats, and drop + /// support for obsolete ones. + /// + /// This list is checked linearly from top to bottom when detecting an image + /// type. It should therefore contain the most popular file formats at the + /// top, and less popular towards the bottom. + static const List values = [ + // ICO is not supported in Chrome. It is deemed too simple and too specific. See also: + // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/modules/webcodecs/image_decoder_external.cc;l=38;drc=fd8802b593110ea18a97ef044f8a40dd24a622ec + + // PNG + ImageFileFormat([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], 'image/png'), + + // GIF87a + ImageFileFormat([0x47, 0x49, 0x46, 0x38, 0x37, 0x61], 'image/gif'), + + // GIF89a + ImageFileFormat([0x47, 0x49, 0x46, 0x38, 0x39, 0x61], 'image/gif'), + + // JPEG + ImageFileFormat([0xFF, 0xD8, 0xFF], 'image/jpeg'), + + // WebP + ImageFileFormat([0x52, 0x49, 0x46, 0x46, null, null, null, null, 0x57, 0x45, 0x42, 0x50], 'image/webp'), + + // BMP + ImageFileFormat([0x42, 0x4D], 'image/bmp'), + ]; +} + +/// Function signature of [debugContentTypeDetector], which is the same as the +/// signature of [detectContentType]. +typedef DebugContentTypeDetector = String? Function(Uint8List); + +/// If not null, replaces the functionality of [detectContentType] with its own. +/// +/// This is useful in tests, for example, to test unsupported content types. +DebugContentTypeDetector? debugContentTypeDetector; + +/// A string of bytes that every AVIF image contains somehwere in its first 16 +/// bytes. +/// +/// This signature is necessary but not sufficient, which may lead to false +/// positives. For example, the file may be HEIC or a video. This is OK, +/// because in the worst case, the image decoder fails to decode the file. +/// This is something we must anticipate regardless of this detection logic. +/// The codec must already protect itself from downloaded files lying about +/// their contents. +/// +/// The alternative would be to implement a more precise detection, which would +/// add complexity and code size. This is how Chromium does it: +/// +/// https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/image-decoders/avif/avif_image_decoder.cc;l=504;drc=fd8802b593110ea18a97ef044f8a40dd24a622ec +final List _avifSignature = 'ftyp'.codeUnits; + +/// Optimistically detects whether [data] is an AVIF image file. +bool isAvif(Uint8List data) { + firstByteLoop: for (int i = 0; i < 16; i += 1) { + for (int j = 0; j < _avifSignature.length; j += 1) { + if (i + j >= data.length) { + // Reached EOF without finding the signature. + return false; + } + if (data[i + j] != _avifSignature[j]) { + continue firstByteLoop; + } + } + return true; + } + return false; +} + +// Wraps another codec and resizes each output image. +class ResizingCodec implements ui.Codec { + ResizingCodec( + this.delegate, { + this.targetWidth, + this.targetHeight, + this.allowUpscaling = true, + }); + + final ui.Codec delegate; + final int? targetWidth; + final int? targetHeight; + final bool allowUpscaling; + + @override + void dispose() => delegate.dispose(); + + @override + int get frameCount => delegate.frameCount; + + @override + Future getNextFrame() async { + final ui.FrameInfo frameInfo = await delegate.getNextFrame(); + return AnimatedImageFrameInfo( + frameInfo.duration, + scaleImageIfNeeded( + frameInfo.image, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling + ), + ); + } + + @override + int get repetitionCount => delegate.frameCount; +} + +ui.Size? _scaledSize( + int width, + int height, + int? targetWidth, + int? targetHeight, +) { + if (targetWidth == width && targetHeight == height) { + // Not scaled + return null; + } + if (targetWidth == null) { + if (targetHeight == null || targetHeight == height) { + // Not scaled. + return null; + } + targetWidth = (width * targetHeight / height).round(); + } else if (targetHeight == null) { + if (targetWidth == targetWidth) { + // Not scaled. + return null; + } + targetHeight = (height * targetWidth / width).round(); + } + return ui.Size(targetWidth.toDouble(), targetHeight.toDouble()); +} + +ui.Image scaleImageIfNeeded( + ui.Image image, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true, +}) { + final int width = image.width; + final int height = image.height; + final ui.Size? scaledSize = _scaledSize( + width, + height, + targetWidth, + targetHeight + ); + if (scaledSize == null) { + return image; + } + if (!allowUpscaling && + (scaledSize.width > width || scaledSize.height > height)) { + return image; + } + + final ui.Rect outputRect = ui.Rect.fromLTWH(0, 0, scaledSize.width, scaledSize.height); + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final ui.Canvas canvas = ui.Canvas(recorder, outputRect); + + canvas.drawImageRect( + image, + ui.Rect.fromLTWH(0, 0, width.toDouble(), height.toDouble()), + outputRect, + ui.Paint(), + ); + final ui.Picture picture = recorder.endRecording(); + final ui.Image finalImage = picture.toImageSync( + scaledSize.width.round(), + scaledSize.height.round() + ); + picture.dispose(); + image.dispose(); + return finalImage; +} diff --git a/lib/web_ui/lib/src/engine/safe_browser_api.dart b/lib/web_ui/lib/src/engine/safe_browser_api.dart index f6dde56c9e0c3..bc7b6bd3a4e7b 100644 --- a/lib/web_ui/lib/src/engine/safe_browser_api.dart +++ b/lib/web_ui/lib/src/engine/safe_browser_api.dart @@ -246,7 +246,7 @@ extension ImageDecoderExtension on ImageDecoder { class ImageDecoderOptions { external factory ImageDecoderOptions({ required JSString type, - required JSUint8Array data, + required JSAny data, required JSString premultiplyAlpha, JSNumber? desiredWidth, JSNumber? desiredHeight, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart index dfa5c19ea1003..9874f6a552a6c 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart @@ -10,6 +10,7 @@ library skwasm_impl; import 'dart:ffi'; export 'skwasm_impl/canvas.dart'; +export 'skwasm_impl/codecs.dart'; export 'skwasm_impl/filters.dart'; export 'skwasm_impl/font_collection.dart'; export 'skwasm_impl/image.dart'; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart new file mode 100644 index 0000000000000..91eedca9c8000 --- /dev/null +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart @@ -0,0 +1,33 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:js_interop'; + +import 'package:ui/src/engine.dart'; +import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; +import 'package:ui/ui.dart' as ui; + +class SkwasmImageDecoder extends BrowserImageDecoder { + SkwasmImageDecoder({ + required super.contentType, + required super.dataSource, + required super.debugSource, + }); + + @override + ui.Image generateImageFromVideoFrame(VideoFrame frame) { + final int width = frame.codedWidth.toInt(); + final int height = frame.codedHeight.toInt(); + final SkwasmSurface surface = (renderer as SkwasmRenderer).surface; + final int videoFrameId = surface.acquireObjectId(); + skwasmInstance.skwasmRegisterObject(videoFrameId.toJS, frame as JSAny); + skwasmInstance.skwasmTransferObjectToThread(videoFrameId.toJS, surface.threadId.toJS); + return SkwasmImage(imageCreateFromVideoFrame( + videoFrameId, + width, + height, + surface.handle, + )); + } +} diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart index 2045248d7730b..e0f3c76c8f487 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart @@ -65,10 +65,13 @@ class SkwasmImage implements ui.Image { bool get debugDisposed => _isDisposed; @override - SkwasmImage clone() => this; + SkwasmImage clone() { + imageRef(handle); + return SkwasmImage(handle); + } @override - bool isCloneOf(ui.Image other) => identical(this, other); + bool isCloneOf(ui.Image other) => other is SkwasmImage && handle == other.handle; @override List? debugGetOpenHandleStackTraces() => null; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart index 11db8650cdcd5..6f0eb810948cf 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart @@ -38,6 +38,22 @@ external ImageHandle imageCreateFromPixels( int rowByteCount, ); +@Native(symbol: 'image_createFromVideoFrame', isLeaf: true) +external ImageHandle imageCreateFromVideoFrame( + int videoFrameId, + int width, + int height, + SurfaceHandle handle, +); + +@Native(symbol:'image_ref', isLeaf: true) +external void imageRef(ImageHandle handle); + @Native(symbol: 'image_dispose', isLeaf: true) external void imageDispose(ImageHandle handle); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart index e9c58b1f21864..9aa48f69e01f0 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart @@ -21,6 +21,9 @@ external SurfaceHandle surfaceCreateFromCanvas( Pointer querySelector ); +@Native(symbol: 'surface_getThreadId', isLeaf: true) +external int surfaceGetThreadId(SurfaceHandle handle); + @Native( symbol: 'surface_setCallbackHandler', isLeaf: true) diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart index 401139622f5f4..76cec92901e79 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart @@ -19,6 +19,19 @@ class SkwasmInstance {} extension SkwasmInstanceExtension on SkwasmInstance { external JSNumber addFunction(JSFunction function, JSString signature); external void removeFunction(JSNumber functionPointer); + + @JS('skwasm_registerObject') + external void skwasmRegisterObject(JSNumber objectId, JSAny object); + + @JS('skwasm_unregisterObject') + external void skwasmUnregisterObject(JSNumber objectId); + + @JS('skwasm_getObject') + external JSAny skwasmGetObject(JSNumber objectId); + + @JS('skwasm_transferObjectToThread') + external void skwasmTransferObjectToThread(JSNumber objectId, JSNumber threadId); + external WebAssemblyMemory get wasmMemory; } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart index 99c18bacdd25b..d339fa6df9c0d 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'dart:async'; +import 'dart:js_interop'; import 'dart:math' as math; import 'dart:typed_data'; @@ -319,32 +320,6 @@ class SkwasmRenderer implements Renderer { indices: indices ); - ui.Size? _scaledSize( - int width, - int height, - int? targetWidth, - int? targetHeight, - ) { - if (targetWidth == width && targetHeight == height) { - // Not scaled - return null; - } - if (targetWidth == null) { - if (targetHeight == null || targetHeight == height) { - // Not scaled. - return null; - } - targetWidth = (width * targetHeight / height).round(); - } else if (targetHeight == null) { - if (targetWidth == targetWidth) { - // Not scaled. - return null; - } - targetHeight = (height * targetWidth / width).round(); - } - return ui.Size(targetWidth.toDouble(), targetHeight.toDouble()); - } - @override void decodeImageFromPixels( Uint8List pixels, @@ -357,43 +332,19 @@ class SkwasmRenderer implements Renderer { int? targetHeight, bool allowUpscaling = true }) { - ui.Size? scaledSize = _scaledSize( - width, - height, - targetWidth, - targetHeight - ); - if (!allowUpscaling && scaledSize != null && - (scaledSize.width > width || scaledSize.height > height)) { - scaledSize = null; - } final SkwasmImage pixelImage = SkwasmImage.fromPixels( pixels, width, height, format ); - if (scaledSize == null) { - callback(pixelImage); - return; - } - - final ui.Rect outputRect = ui.Rect.fromLTWH(0, 0, scaledSize.width, scaledSize.height); - final ui.PictureRecorder recorder = ui.PictureRecorder(); - final ui.Canvas canvas = ui.Canvas(recorder, outputRect); - - canvas.drawImageRect( + final ui.Image scaledImage = scaleImageIfNeeded( pixelImage, - ui.Rect.fromLTWH(0, 0, width.toDouble(), width.toDouble()), - outputRect, - ui.Paint(), - ); - final ui.Image finalImage = recorder.endRecording().toImageSync( - scaledSize.width.round(), - scaledSize.height.round() + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, ); - pixelImage.dispose(); - callback(finalImage); + callback(scaledImage); } @override @@ -409,13 +360,47 @@ class SkwasmRenderer implements Renderer { } @override - Future instantiateImageCodec(Uint8List list, {int? targetWidth, int? targetHeight, bool allowUpscaling = true}) { - throw UnimplementedError('instantiateImageCodec not yet implemented'); + Future instantiateImageCodec( + Uint8List list, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true + }) async { + final String? contentType = detectContentType(list); + if (contentType == null) { + throw Exception('Could not determine content type of image from data'); + } + final ui.Codec baseDecoder = SkwasmImageDecoder( + contentType: contentType, + dataSource: list.toJS, + debugSource: 'encoded image bytes', + ); + if (targetWidth == null && targetHeight == null) { + return baseDecoder; + } + return ResizingCodec( + baseDecoder, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling + ); } @override - Future instantiateImageCodecFromUrl(Uri uri, {WebOnlyImageCodecChunkCallback? chunkCallback}) { - throw UnimplementedError('instantiateImageCodecFromUrl not yet implemented'); + Future instantiateImageCodecFromUrl( + Uri uri, { + WebOnlyImageCodecChunkCallback? chunkCallback + }) async { + final DomResponse response = await rawHttpGet(uri.toString()); + final String? contentType = response.headers.get('Content-Type'); + if (contentType == null) { + throw Exception('Could not determine content type of image at url $uri'); + } + return SkwasmImageDecoder( + contentType: contentType, + dataSource: response.body as JSAny, + debugSource: uri.toString(), + ); } @override diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index 79fa506a9d099..049e770203b9f 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -21,11 +21,16 @@ class SkwasmSurface { return surface; } - SkwasmSurface._fromHandle(this._handle); - final SurfaceHandle _handle; + SkwasmSurface._fromHandle(this.handle) : threadId = surfaceGetThreadId(handle); + final SurfaceHandle handle; OnRenderCallbackHandle _callbackHandle = nullptr; final Map> _pendingCallbacks = >{}; + final int threadId; + + int _currentObjectId = 0; + int acquireObjectId() => ++_currentObjectId; + void _initialize() { _callbackHandle = OnRenderCallbackHandle.fromAddress( @@ -34,20 +39,20 @@ class SkwasmSurface { 'vii'.toJS ).toDart.toInt() ); - surfaceSetCallbackHandler(_handle, _callbackHandle); + surfaceSetCallbackHandler(handle, _callbackHandle); } void setSize(int width, int height) => - surfaceSetCanvasSize(_handle, width, height); + surfaceSetCanvasSize(handle, width, height); Future renderPicture(SkwasmPicture picture) { - final int callbackId = surfaceRenderPicture(_handle, picture.handle); + final int callbackId = surfaceRenderPicture(handle, picture.handle); return _registerCallback(callbackId); } Future rasterizeImage(SkwasmImage image, ui.ImageByteFormat format) async { final int callbackId = surfaceRasterizeImage( - _handle, + handle, image.handle, format.index, ); @@ -76,7 +81,7 @@ class SkwasmSurface { } void dispose() { - surfaceDestroy(_handle); + surfaceDestroy(handle); skwasmInstance.removeFunction(_callbackHandle.address.toJS); } } diff --git a/lib/web_ui/skwasm/BUILD.gn b/lib/web_ui/skwasm/BUILD.gn index f175ee8b73a34..96e38583b35e7 100644 --- a/lib/web_ui/skwasm/BUILD.gn +++ b/lib/web_ui/skwasm/BUILD.gn @@ -18,6 +18,7 @@ wasm_lib("skwasm") { "path.cpp", "picture.cpp", "shaders.cpp", + "skwasm_support.h", "string.cpp", "surface.cpp", "text/line_metrics.cpp", @@ -41,10 +42,14 @@ wasm_lib("skwasm") { "-sUSE_PTHREADS=1", "-lexports.js", "-sEXPORTED_FUNCTIONS=[stackAlloc]", - "-sEXPORTED_RUNTIME_METHODS=[addFunction,removeFunction]", + "-sEXPORTED_RUNTIME_METHODS=[addFunction,removeFunction,skwasm_registerObject,skwasm_unregisterObject,skwasm_getObject,skwasm_transferObjectToThread]", "-Wno-pthreads-mem-growth", + "--js-library", + rebase_path("library_skwasm_support.js"), ] + inputs = [ rebase_path("library_skwasm_support.js") ] + if (is_debug) { ldflags += [ "-sDEMANGLE_SUPPORT=1", diff --git a/lib/web_ui/skwasm/filters.cpp b/lib/web_ui/skwasm/filters.cpp index eadb28f13bcaf..d50763ead09fc 100644 --- a/lib/web_ui/skwasm/filters.cpp +++ b/lib/web_ui/skwasm/filters.cpp @@ -37,17 +37,14 @@ SKWASM_EXPORT SkImageFilter* imageFilter_createMatrix(SkScalar* matrix33, SKWASM_EXPORT SkImageFilter* imageFilter_createFromColorFilter( SkColorFilter* filter) { - filter->ref(); - return SkImageFilters::ColorFilter(sk_sp(filter), nullptr) + return SkImageFilters::ColorFilter(sk_ref_sp(filter), nullptr) .release(); } SKWASM_EXPORT SkImageFilter* imageFilter_compose(SkImageFilter* outer, SkImageFilter* inner) { - outer->ref(); - inner->ref(); - return SkImageFilters::Compose(sk_sp(outer), - sk_sp(inner)) + return SkImageFilters::Compose(sk_ref_sp(outer), + sk_ref_sp(inner)) .release(); } @@ -84,10 +81,8 @@ SKWASM_EXPORT SkColorFilter* colorFilter_createLinearToSRGBGamma() { SKWASM_EXPORT SkColorFilter* colorFilter_compose(SkColorFilter* outer, SkColorFilter* inner) { - outer->ref(); - inner->ref(); - return SkColorFilters::Compose(sk_sp(outer), - sk_sp(inner)) + return SkColorFilters::Compose(sk_ref_sp(outer), + sk_ref_sp(inner)) .release(); } diff --git a/lib/web_ui/skwasm/fonts.cpp b/lib/web_ui/skwasm/fonts.cpp index 4a21c8ed84463..6209730032ed8 100644 --- a/lib/web_ui/skwasm/fonts.cpp +++ b/lib/web_ui/skwasm/fonts.cpp @@ -29,9 +29,8 @@ SKWASM_EXPORT void fontCollection_dispose(FlutterFontCollection* collection) { } SKWASM_EXPORT SkTypeface* typeface_create(SkData* fontData) { - fontData->ref(); auto typeface = - SkFontMgr::RefDefault()->makeFromData(sk_sp(fontData)); + SkFontMgr::RefDefault()->makeFromData(sk_ref_sp(fontData)); return typeface.release(); } @@ -76,12 +75,12 @@ SKWASM_EXPORT void fontCollection_registerTypeface( FlutterFontCollection* collection, SkTypeface* typeface, SkString* fontName) { - typeface->ref(); if (fontName) { SkString alias = *fontName; - collection->provider->registerTypeface(sk_sp(typeface), alias); + collection->provider->registerTypeface(sk_ref_sp(typeface), + alias); } else { - collection->provider->registerTypeface(sk_sp(typeface)); + collection->provider->registerTypeface(sk_ref_sp(typeface)); } } diff --git a/lib/web_ui/skwasm/image.cpp b/lib/web_ui/skwasm/image.cpp index 31c219eb014e9..8216f4132090a 100644 --- a/lib/web_ui/skwasm/image.cpp +++ b/lib/web_ui/skwasm/image.cpp @@ -3,15 +3,31 @@ // found in the LICENSE file. #include "export.h" +#include "skwasm_support.h" +#include "surface.h" +#include "wrappers.h" #include "third_party/skia/include/core/SkColorSpace.h" #include "third_party/skia/include/core/SkData.h" #include "third_party/skia/include/core/SkImage.h" #include "third_party/skia/include/core/SkImageInfo.h" #include "third_party/skia/include/core/SkPicture.h" +#include "third_party/skia/include/gpu/GrBackendSurface.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/include/gpu/ganesh/GrExternalTextureGenerator.h" +#include "third_party/skia/include/gpu/ganesh/SkImageGanesh.h" +#include "third_party/skia/include/gpu/ganesh/SkSurfaceGanesh.h" +#include "third_party/skia/include/gpu/gl/GrGLInterface.h" +#include "third_party/skia/include/gpu/gl/GrGLTypes.h" + +#include +#include +#include using namespace SkImages; +namespace { + enum class PixelFormat { rgba8888, bgra8888, @@ -39,11 +55,66 @@ SkAlphaType alphaTypeForPixelFormat(PixelFormat format) { } } +class ExternalWebGLTexture : public GrExternalTexture { + public: + ExternalWebGLTexture(GrBackendTexture backendTexture, + GLuint textureId, + EMSCRIPTEN_WEBGL_CONTEXT_HANDLE context) + : _backendTexture(backendTexture), + _textureId(textureId), + _webGLContext(context) {} + + GrBackendTexture getBackendTexture() override { return _backendTexture; } + + void dispose() override { + Skwasm::makeCurrent(_webGLContext); + glDeleteTextures(1, &_textureId); + } + + private: + GrBackendTexture _backendTexture; + GLuint _textureId; + EMSCRIPTEN_WEBGL_CONTEXT_HANDLE _webGLContext; +}; +} // namespace + +class VideoFrameImageGenerator : public GrExternalTextureGenerator { + public: + VideoFrameImageGenerator(SkImageInfo ii, + SkwasmObjectId videoFrameId, + Skwasm::Surface* surface) + : GrExternalTextureGenerator(ii), + _videoFrameId(videoFrameId), + _surface(surface) {} + + ~VideoFrameImageGenerator() override { + _surface->disposeVideoFrame(_videoFrameId); + } + + std::unique_ptr generateExternalTexture( + GrRecordingContext* context, + GrMipMapped mipmapped) override { + GrGLTextureInfo glInfo; + glInfo.fID = skwasm_createGlTextureFromVideoFrame( + _videoFrameId, fInfo.width(), fInfo.height()); + glInfo.fFormat = GL_RGBA8_OES; + glInfo.fTarget = GL_TEXTURE_2D; + + GrBackendTexture backendTexture(fInfo.width(), fInfo.height(), mipmapped, + glInfo); + return std::make_unique( + backendTexture, glInfo.fID, emscripten_webgl_get_current_context()); + } + + private: + SkwasmObjectId _videoFrameId; + Skwasm::Surface* _surface; +}; + SKWASM_EXPORT SkImage* image_createFromPicture(SkPicture* picture, int32_t width, int32_t height) { - picture->ref(); - return DeferredFromPicture(sk_sp(picture), {width, height}, + return DeferredFromPicture(sk_ref_sp(picture), {width, height}, nullptr, nullptr, BitDepth::kU8, SkColorSpace::MakeSRGB()) .release(); @@ -54,16 +125,32 @@ SKWASM_EXPORT SkImage* image_createFromPixels(SkData* data, int height, PixelFormat pixelFormat, size_t rowByteCount) { - data->ref(); return SkImages::RasterFromData( SkImageInfo::Make(width, height, colorTypeForPixelFormat(pixelFormat), alphaTypeForPixelFormat(pixelFormat), SkColorSpace::MakeSRGB()), - sk_sp(data), rowByteCount) + sk_ref_sp(data), rowByteCount) .release(); } +SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObjectId videoFrameId, + int width, + int height, + Skwasm::Surface* surface) { + return SkImages::DeferredFromTextureGenerator( + std::make_unique( + SkImageInfo::Make(width, height, + SkColorType::kRGBA_8888_SkColorType, + SkAlphaType::kPremul_SkAlphaType), + videoFrameId, surface)) + .release(); +} + +SKWASM_EXPORT void image_ref(SkImage* image) { + image->ref(); +} + SKWASM_EXPORT void image_dispose(SkImage* image) { image->unref(); } diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js new file mode 100644 index 0000000000000..431b4e7da3581 --- /dev/null +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -0,0 +1,84 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file adds JavaScript APIs that are accessible to the C++ layer. +// See: https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html#implement-a-c-api-in-javascript + +mergeInto(LibraryManager.library, { + $skwasm_support_setup__postset: 'skwasm_support_setup();', + $skwasm_support_setup: function() { + const objectMap = new Map(); + skwasm_registerObject = function(id, object) { + objectMap.set(id, object); + }; + skwasm_unregisterObject = function(id) { + objectMap.delete(id); + } + skwasm_getObject = function(id) { + return objectMap.get(id); + } + + addEventListener('message', function (event) { + const transfers = event.data.skwasmObjectTransfers; + if (!transfers) { + return; + } + transfers.forEach(function(object, objectId) { + objectMap.set(objectId, object); + }); + }); + skwasm_transferObjectToMain = function(objectId) { + postMessage({ + skwasmObjectTransfers: new Map([ + [objectId, objectMap[objectId]] + ]) + }); + objectMap.delete(objectId); + } + skwasm_transferObjectToThread = function(objectId, threadId) { + PThread.pthreads[threadId].postMessage({ + skwasmObjectTransfers: new Map([ + [objectId, objectMap.get(objectId)] + ]) + }); + objectMap.delete(objectId); + } + _skwasm_createGlTextureFromVideoFrame = function(videoFrameId, width, height) { + const videoFrame = skwasm_getObject(videoFrameId); + const glCtx = GL.currentContext.GLctx; + const newTexture = glCtx.createTexture(); + glCtx.bindTexture(glCtx.TEXTURE_2D, newTexture); + glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); + + glCtx.texImage2D(glCtx.TEXTURE_2D, 0, glCtx.RGBA, width, height, 0, glCtx.RGBA, glCtx.UNSIGNED_BYTE, videoFrame); + + glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false); + glCtx.bindTexture(glCtx.TEXTURE_2D, null); + + const textureId = GL.getNewId(GL.textures); + GL.textures[textureId] = newTexture; + return textureId; + }, + _skwasm_disposeVideoFrame = function(videoFrameId) { + const videoFrame = skwasm_getObject(videoFrameId); + videoFrame.close(); + skwasm_unregisterObject(videoFrameId); + } + }, + $skwasm_registerObject: function() {}, + $skwasm_registerObject__deps: ['$skwasm_support_setup'], + $skwasm_unregisterObject: function() {}, + $skwasm_unregisterObject__deps: ['$skwasm_support_setup'], + $skwasm_getObject: function() {}, + $skwasm_getObject__deps: ['$skwasm_support_setup'], + $skwasm_transferObjectToMain: function() {}, + $skwasm_transferObjectToMain__deps: ['$skwasm_support_setup'], + $skwasm_transferObjectToThread: function() {}, + $skwasm_transferObjectToThread__deps: ['$skwasm_support_setup'], + skwasm_createGlTextureFromVideoFrame: function () {}, + skwasm_createGlTextureFromVideoFrame__deps: ['$skwasm_support_setup', '$skwasm_getObject'], + skwasm_disposeVideoFrame: function () {}, + skwasm_disposeVideoFrame__deps: ['$skwasm_support_setup', '$skwasm_getObject', '$skwasm_unregisterObject'], +}); + \ No newline at end of file diff --git a/lib/web_ui/skwasm/paint.cpp b/lib/web_ui/skwasm/paint.cpp index d0d9f7f58d515..62fc4d04f1b62 100644 --- a/lib/web_ui/skwasm/paint.cpp +++ b/lib/web_ui/skwasm/paint.cpp @@ -87,34 +87,17 @@ SKWASM_EXPORT SkScalar paint_getMiterLImit(SkPaint* paint) { } SKWASM_EXPORT void paint_setShader(SkPaint* paint, SkShader* shader) { - if (shader == nullptr) { - paint->setShader(nullptr); - return; - } - shader->ref(); - paint->setShader(sk_sp(shader)); + paint->setShader(sk_ref_sp(shader)); } SKWASM_EXPORT void paint_setImageFilter(SkPaint* paint, SkImageFilter* filter) { - if (filter == nullptr) { - paint->setImageFilter(nullptr); - } - filter->ref(); - paint->setImageFilter(sk_sp(filter)); + paint->setImageFilter(sk_ref_sp(filter)); } SKWASM_EXPORT void paint_setColorFilter(SkPaint* paint, SkColorFilter* filter) { - if (filter == nullptr) { - paint->setColorFilter(nullptr); - } - filter->ref(); - paint->setColorFilter(sk_sp(filter)); + paint->setColorFilter(sk_ref_sp(filter)); } SKWASM_EXPORT void paint_setMaskFilter(SkPaint* paint, SkMaskFilter* filter) { - if (filter == nullptr) { - paint->setMaskFilter(nullptr); - } - filter->ref(); - paint->setMaskFilter(sk_sp(filter)); + paint->setMaskFilter(sk_ref_sp(filter)); } diff --git a/lib/web_ui/skwasm/shaders.cpp b/lib/web_ui/skwasm/shaders.cpp index d1afdadaea64b..d7f7d36825e9d 100644 --- a/lib/web_ui/skwasm/shaders.cpp +++ b/lib/web_ui/skwasm/shaders.cpp @@ -128,9 +128,7 @@ SKWASM_EXPORT SkShader* shader_createRuntimeEffectShader( size_t childCount) { std::vector> childPointers; for (size_t i = 0; i < childCount; i++) { - auto child = children[i]; - child->ref(); - childPointers.emplace_back(child); + childPointers.emplace_back(sk_ref_sp(children[i])); } return runtimeEffect ->makeShader(SkData::MakeWithCopy(uniforms->data(), uniforms->size()), diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h new file mode 100644 index 0000000000000..990f0de0963a6 --- /dev/null +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -0,0 +1,19 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include + +using SkwasmObjectId = uint32_t; + +extern "C" { +extern void skwasm_transferObjectToMain(SkwasmObjectId objectId); +extern void skwasm_transferObjectToThread(SkwasmObjectId objectId, + pthread_t threadId); +extern unsigned int skwasm_createGlTextureFromVideoFrame( + SkwasmObjectId videoFrameId, + int width, + int height); +extern void skwasm_disposeVideoFrame(SkwasmObjectId videoFrameId); +} diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index 4027d37a5f0ad..10537ae963d52 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -2,330 +2,272 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include -#include -#include -#include -#include -#include -#include -#include "export.h" -#include "third_party/skia/include/core/SkCanvas.h" -#include "third_party/skia/include/core/SkColorSpace.h" -#include "third_party/skia/include/core/SkPicture.h" -#include "third_party/skia/include/core/SkSurface.h" -#include "third_party/skia/include/encode/SkPngEncoder.h" -#include "third_party/skia/include/gpu/GrDirectContext.h" -#include "third_party/skia/include/gpu/ganesh/SkSurfaceGanesh.h" -#include "third_party/skia/include/gpu/gl/GrGLInterface.h" -#include "third_party/skia/include/gpu/gl/GrGLTypes.h" -#include "wrappers.h" +#include "surface.h" using namespace Skwasm; -namespace { - -// This must be kept in sync with the `ImageByteFormat` enum in dart:ui. -enum class ImageByteFormat { - rawRgba, - rawStraightRgba, - rawUnmodified, - png, -}; - -class Surface; -void fDispose(Surface* surface); -void fSetCanvasSize(Surface* surface, int width, int height); -void fRenderPicture(Surface* surface, SkPicture* picture); -void fNotifyRenderComplete(Surface* surface, uint32_t callbackId); -void fOnRenderComplete(Surface* surface, uint32_t callbackId); -void fRasterizeImage(Surface* surface, - SkImage* image, - ImageByteFormat format, - uint32_t callbackId); -void fOnRasterizeComplete(Surface* surface, - SkData* imageData, - uint32_t callbackId); - -class Surface { - public: - using CallbackHandler = void(uint32_t, void*); - - // Main thread only - Surface(const char* canvasID) : _canvasID(canvasID) { - assert(emscripten_is_main_browser_thread()); - - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - emscripten_pthread_attr_settransferredcanvases(&attr, _canvasID.c_str()); - - pthread_create( - &_thread, &attr, - [](void* context) -> void* { - static_cast(context)->_runWorker(); - return nullptr; - }, - this); - } - - // Main thread only - void dispose() { - assert(emscripten_is_main_browser_thread()); - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VI, - reinterpret_cast(fDispose), nullptr, - this); - } - - // Main thread only - void setCanvasSize(int width, int height) { - assert(emscripten_is_main_browser_thread()); - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIII, - reinterpret_cast(fSetCanvasSize), - nullptr, this, width, height); - } +Surface::Surface(const char* canvasID) : _canvasID(canvasID) { + assert(emscripten_is_main_browser_thread()); + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + emscripten_pthread_attr_settransferredcanvases(&attr, _canvasID.c_str()); + + pthread_create( + &_thread, &attr, + [](void* context) -> void* { + static_cast(context)->_runWorker(); + return nullptr; + }, + this); +} - // Main thread only - uint32_t renderPicture(SkPicture* picture) { - assert(emscripten_is_main_browser_thread()); - uint32_t callbackId = ++_currentCallbackId; - picture->ref(); - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, - reinterpret_cast(fRenderPicture), - nullptr, this, picture); - - // After drawing to the surface, the browser implicitly flushed the drawing - // commands at the end of the event loop. As a result, in order to make - // sure we call back after the rendering has actually occurred, we issue - // the callback in a subsequent event, after the flushing has happened. - emscripten_dispatch_to_thread( - _thread, EM_FUNC_SIG_VII, - reinterpret_cast(fNotifyRenderComplete), nullptr, this, - callbackId); - return callbackId; - } +// Main thread only +void Surface::dispose() { + assert(emscripten_is_main_browser_thread()); + emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VI, + reinterpret_cast(fDispose), nullptr, + this); +} - uint32_t rasterizeImage(SkImage* image, ImageByteFormat format) { - uint32_t callbackId = ++_currentCallbackId; - image->ref(); +// Main thread only +void Surface::setCanvasSize(int width, int height) { + assert(emscripten_is_main_browser_thread()); + emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIII, + reinterpret_cast(fSetCanvasSize), + nullptr, this, width, height); +} - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIIII, - reinterpret_cast(fRasterizeImage), - nullptr, this, image, format, callbackId); - return callbackId; - } +// Main thread only +uint32_t Surface::renderPicture(SkPicture* picture) { + assert(emscripten_is_main_browser_thread()); + uint32_t callbackId = ++_currentCallbackId; + picture->ref(); + emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, + reinterpret_cast(fRenderPicture), + nullptr, this, picture); + + // After drawing to the surface, the browser implicitly flushes the drawing + // commands at the end of the event loop. As a result, in order to make + // sure we call back after the rendering has actually occurred, we issue + // the callback in a subsequent event, after the flushing has happened. + emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, + reinterpret_cast(fNotifyRenderComplete), + nullptr, this, callbackId); + return callbackId; +} - // Main thread only - void setCallbackHandler(CallbackHandler* callbackHandler) { - assert(emscripten_is_main_browser_thread()); - _callbackHandler = callbackHandler; - } +// Main thread only +uint32_t Surface::rasterizeImage(SkImage* image, ImageByteFormat format) { + assert(emscripten_is_main_browser_thread()); + uint32_t callbackId = ++_currentCallbackId; + image->ref(); - private: - // Worker thread only - void _runWorker() { - _init(); - emscripten_unwind_to_js_event_loop(); - } + emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIIII, + reinterpret_cast(fRasterizeImage), + nullptr, this, image, format, callbackId); + return callbackId; +} - // Worker thread only - void _init() { - EmscriptenWebGLContextAttributes attributes; - emscripten_webgl_init_context_attributes(&attributes); - - attributes.alpha = true; - attributes.depth = true; - attributes.stencil = true; - attributes.antialias = false; - attributes.premultipliedAlpha = true; - attributes.preserveDrawingBuffer = 0; - attributes.powerPreference = EM_WEBGL_POWER_PREFERENCE_DEFAULT; - attributes.failIfMajorPerformanceCaveat = false; - attributes.enableExtensionsByDefault = true; - attributes.explicitSwapControl = false; - attributes.renderViaOffscreenBackBuffer = true; - attributes.majorVersion = 2; - - _glContext = - emscripten_webgl_create_context(_canvasID.c_str(), &attributes); - if (!_glContext) { - printf("Failed to create context!\n"); - return; - } +void Surface::disposeVideoFrame(SkwasmObjectId videoFrameId) { + emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, + reinterpret_cast(fDisposeVideoFrame), + nullptr, this, videoFrameId); +} - makeCurrent(_glContext); +// Main thread only +void Surface::setCallbackHandler(CallbackHandler* callbackHandler) { + assert(emscripten_is_main_browser_thread()); + _callbackHandler = callbackHandler; +} - _grContext = GrDirectContext::MakeGL(GrGLMakeNativeInterface()); +// Worker thread only +void Surface::_runWorker() { + _init(); + emscripten_unwind_to_js_event_loop(); +} - // WebGL should already be clearing the color and stencil buffers, but do it - // again here to ensure Skia receives them in the expected state. - emscripten_glBindFramebuffer(GL_FRAMEBUFFER, 0); - emscripten_glClearColor(0, 0, 0, 0); - emscripten_glClearStencil(0); - emscripten_glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT); - _grContext->resetContext(kRenderTarget_GrGLBackendState | - kMisc_GrGLBackendState); +// Worker thread only +void Surface::_init() { + EmscriptenWebGLContextAttributes attributes; + emscripten_webgl_init_context_attributes(&attributes); + + attributes.alpha = true; + attributes.depth = true; + attributes.stencil = true; + attributes.antialias = false; + attributes.premultipliedAlpha = true; + attributes.preserveDrawingBuffer = 0; + attributes.powerPreference = EM_WEBGL_POWER_PREFERENCE_DEFAULT; + attributes.failIfMajorPerformanceCaveat = false; + attributes.enableExtensionsByDefault = true; + attributes.explicitSwapControl = false; + attributes.renderViaOffscreenBackBuffer = true; + attributes.majorVersion = 2; + + _glContext = emscripten_webgl_create_context(_canvasID.c_str(), &attributes); + if (!_glContext) { + printf("Failed to create context!\n"); + return; + } - // The on-screen canvas is FBO 0. Wrap it in a Skia render target so Skia - // can render to it. - _fbInfo.fFBOID = 0; - _fbInfo.fFormat = GL_RGBA8_OES; + makeCurrent(_glContext); - emscripten_glGetIntegerv(GL_SAMPLES, &_sampleCount); - emscripten_glGetIntegerv(GL_STENCIL_BITS, &_stencil); - } + _grContext = GrDirectContext::MakeGL(GrGLMakeNativeInterface()); - // Worker thread only - void _dispose() { delete this; } + // WebGL should already be clearing the color and stencil buffers, but do it + // again here to ensure Skia receives them in the expected state. + emscripten_glBindFramebuffer(GL_FRAMEBUFFER, 0); + emscripten_glClearColor(0, 0, 0, 0); + emscripten_glClearStencil(0); + emscripten_glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT); + _grContext->resetContext(kRenderTarget_GrGLBackendState | + kMisc_GrGLBackendState); - // Worker thread only - void _setCanvasSize(int width, int height) { - if (_canvasWidth != width || _canvasHeight != height) { - emscripten_set_canvas_element_size(_canvasID.c_str(), width, height); - _canvasWidth = width; - _canvasHeight = height; - _recreateSurface(); - } - } + // The on-screen canvas is FBO 0. Wrap it in a Skia render target so Skia + // can render to it. + _fbInfo.fFBOID = 0; + _fbInfo.fFormat = GL_RGBA8_OES; - // Worker thread only - void _recreateSurface() { - makeCurrent(_glContext); - GrBackendRenderTarget target(_canvasWidth, _canvasHeight, _sampleCount, - _stencil, _fbInfo); - _surface = SkSurfaces::WrapBackendRenderTarget( - _grContext.get(), target, kBottomLeft_GrSurfaceOrigin, - kRGBA_8888_SkColorType, SkColorSpace::MakeSRGB(), nullptr); - } + emscripten_glGetIntegerv(GL_SAMPLES, &_sampleCount); + emscripten_glGetIntegerv(GL_STENCIL_BITS, &_stencil); +} - // Worker thread only - void _renderPicture(const SkPicture* picture) { - if (!_surface) { - printf("Can't render picture with no surface.\n"); - return; - } +// Worker thread only +void Surface::_dispose() { + delete this; +} - makeCurrent(_glContext); - auto canvas = _surface->getCanvas(); - canvas->drawPicture(picture); - _surface->flush(); +// Worker thread only +void Surface::_setCanvasSize(int width, int height) { + if (_canvasWidth != width || _canvasHeight != height) { + emscripten_set_canvas_element_size(_canvasID.c_str(), width, height); + _canvasWidth = width; + _canvasHeight = height; + _recreateSurface(); } +} - void _rasterizeImage(SkImage* image, - ImageByteFormat format, - uint32_t callbackId) { - sk_sp data; - if (format == ImageByteFormat::png) { - data = SkPngEncoder::Encode(_grContext.get(), image, {}); - } else { - SkAlphaType alphaType = format == ImageByteFormat::rawStraightRgba - ? SkAlphaType::kUnpremul_SkAlphaType - : SkAlphaType::kPremul_SkAlphaType; - SkImageInfo info = SkImageInfo::Make(image->width(), image->height(), - SkColorType::kRGBA_8888_SkColorType, - alphaType, SkColorSpace::MakeSRGB()); - size_t bytesPerRow = 4 * image->width(); - size_t byteSize = info.computeByteSize(bytesPerRow); - data = SkData::MakeUninitialized(byteSize); - uint8_t* pixels = reinterpret_cast(data->writable_data()); - bool success = image->readPixels(_grContext.get(), image->imageInfo(), - pixels, bytesPerRow, 0, 0); - if (!success) { - printf("Failed to read pixels from image!\n"); - data = nullptr; - } - } - emscripten_sync_run_in_main_runtime_thread(EM_FUNC_SIG_VIII, - fOnRasterizeComplete, this, - data.release(), callbackId); - } +// Worker thread only +void Surface::_recreateSurface() { + makeCurrent(_glContext); + GrBackendRenderTarget target(_canvasWidth, _canvasHeight, _sampleCount, + _stencil, _fbInfo); + _surface = SkSurfaces::WrapBackendRenderTarget( + _grContext.get(), target, kBottomLeft_GrSurfaceOrigin, + kRGBA_8888_SkColorType, SkColorSpace::MakeSRGB(), nullptr); +} - void _onRasterizeComplete(SkData* data, uint32_t callbackId) { - _callbackHandler(callbackId, data); +// Worker thread only +void Surface::_renderPicture(const SkPicture* picture) { + if (!_surface) { + printf("Can't render picture with no surface.\n"); + return; } - // Worker thread only - void _notifyRenderComplete(uint32_t callbackId) { - emscripten_sync_run_in_main_runtime_thread( - EM_FUNC_SIG_VII, fOnRenderComplete, this, callbackId); - } + makeCurrent(_glContext); + auto canvas = _surface->getCanvas(); + canvas->drawPicture(picture); + _surface->flush(); +} - // Main thread only - void _onRenderComplete(uint32_t callbackId) { - assert(emscripten_is_main_browser_thread()); - _callbackHandler(callbackId, nullptr); +void Surface::_rasterizeImage(SkImage* image, + ImageByteFormat format, + uint32_t callbackId) { + sk_sp data; + if (format == ImageByteFormat::png) { + data = SkPngEncoder::Encode(_grContext.get(), image, {}); + } else { + SkAlphaType alphaType = format == ImageByteFormat::rawStraightRgba + ? SkAlphaType::kUnpremul_SkAlphaType + : SkAlphaType::kPremul_SkAlphaType; + SkImageInfo info = SkImageInfo::Make(image->width(), image->height(), + SkColorType::kRGBA_8888_SkColorType, + alphaType, SkColorSpace::MakeSRGB()); + size_t bytesPerRow = 4 * image->width(); + size_t byteSize = info.computeByteSize(bytesPerRow); + data = SkData::MakeUninitialized(byteSize); + uint8_t* pixels = reinterpret_cast(data->writable_data()); + bool success = image->readPixels(_grContext.get(), image->imageInfo(), + pixels, bytesPerRow, 0, 0); + if (!success) { + printf("Failed to read pixels from image!\n"); + data = nullptr; + } } + emscripten_sync_run_in_main_runtime_thread( + EM_FUNC_SIG_VIII, fOnRasterizeComplete, this, data.release(), callbackId); +} - std::string _canvasID; - CallbackHandler* _callbackHandler = nullptr; - uint32_t _currentCallbackId = 0; - - int _canvasWidth = 0; - int _canvasHeight = 0; +void Surface::_disposeVideoFrame(SkwasmObjectId objectId) { + skwasm_disposeVideoFrame(objectId); +} - EMSCRIPTEN_WEBGL_CONTEXT_HANDLE _glContext = 0; - sk_sp _grContext = nullptr; - sk_sp _surface = nullptr; - GrGLFramebufferInfo _fbInfo; - GrGLint _sampleCount; - GrGLint _stencil; +void Surface::_onRasterizeComplete(SkData* data, uint32_t callbackId) { + _callbackHandler(callbackId, data); +} - pthread_t _thread; +// Worker thread only +void Surface::_notifyRenderComplete(uint32_t callbackId) { + emscripten_sync_run_in_main_runtime_thread(EM_FUNC_SIG_VII, fOnRenderComplete, + this, callbackId); +} - friend void fDispose(Surface* surface); - friend void fSetCanvasSize(Surface* surface, int width, int height); - friend void fRenderPicture(Surface* surface, SkPicture* picture); - friend void fNotifyRenderComplete(Surface* surface, uint32_t callbackId); - friend void fOnRenderComplete(Surface* surface, uint32_t callbackId); - friend void fRasterizeImage(Surface* surface, - SkImage* image, - ImageByteFormat format, - uint32_t callbackId); - friend void fOnRasterizeComplete(Surface* surface, - SkData* imageData, - uint32_t callbackId); -}; +// Main thread only +void Surface::_onRenderComplete(uint32_t callbackId) { + assert(emscripten_is_main_browser_thread()); + _callbackHandler(callbackId, nullptr); +} -void fDispose(Surface* surface) { +void Surface::fDispose(Surface* surface) { surface->_dispose(); } -void fSetCanvasSize(Surface* surface, int width, int height) { +void Surface::fSetCanvasSize(Surface* surface, int width, int height) { surface->_setCanvasSize(width, height); } -void fRenderPicture(Surface* surface, SkPicture* picture) { +void Surface::fRenderPicture(Surface* surface, SkPicture* picture) { surface->_renderPicture(picture); picture->unref(); } -void fNotifyRenderComplete(Surface* surface, uint32_t callbackId) { +void Surface::fNotifyRenderComplete(Surface* surface, uint32_t callbackId) { surface->_notifyRenderComplete(callbackId); } -void fOnRenderComplete(Surface* surface, uint32_t callbackId) { +void Surface::fOnRenderComplete(Surface* surface, uint32_t callbackId) { surface->_onRenderComplete(callbackId); } -void fOnRasterizeComplete(Surface* surface, - SkData* imageData, - uint32_t callbackId) { +void Surface::fOnRasterizeComplete(Surface* surface, + SkData* imageData, + uint32_t callbackId) { surface->_onRasterizeComplete(imageData, callbackId); } -void fRasterizeImage(Surface* surface, - SkImage* image, - ImageByteFormat format, - uint32_t callbackId) { +void Surface::fRasterizeImage(Surface* surface, + SkImage* image, + ImageByteFormat format, + uint32_t callbackId) { surface->_rasterizeImage(image, format, callbackId); image->unref(); } -} // namespace + +void Surface::fDisposeVideoFrame(Surface* surface, + SkwasmObjectId videoFrameId) { + surface->_disposeVideoFrame(videoFrameId); +} SKWASM_EXPORT Surface* surface_createFromCanvas(const char* canvasID) { return new Surface(canvasID); } +SKWASM_EXPORT unsigned long surface_getThreadId(Surface* surface) { + return surface->getThreadId(); +} + SKWASM_EXPORT void surface_setCallbackHandler( Surface* surface, Surface::CallbackHandler* callbackHandler) { diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h new file mode 100644 index 0000000000000..3fc6d8edcf3c8 --- /dev/null +++ b/lib/web_ui/skwasm/surface.h @@ -0,0 +1,101 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "export.h" +#include "skwasm_support.h" +#include "third_party/skia/include/core/SkCanvas.h" +#include "third_party/skia/include/core/SkColorSpace.h" +#include "third_party/skia/include/core/SkPicture.h" +#include "third_party/skia/include/core/SkSurface.h" +#include "third_party/skia/include/encode/SkPngEncoder.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/include/gpu/ganesh/SkSurfaceGanesh.h" +#include "third_party/skia/include/gpu/gl/GrGLInterface.h" +#include "third_party/skia/include/gpu/gl/GrGLTypes.h" +#include "wrappers.h" + +namespace Skwasm { +// This must be kept in sync with the `ImageByteFormat` enum in dart:ui. +enum class ImageByteFormat { + rawRgba, + rawStraightRgba, + rawUnmodified, + png, +}; + +class Surface { + public: + public: + using CallbackHandler = void(uint32_t, void*); + + // Main thread only + Surface(const char* canvasID); + + unsigned long getThreadId() { return _thread; } + + // Main thread only + void dispose(); + void setCanvasSize(int width, int height); + uint32_t renderPicture(SkPicture* picture); + uint32_t rasterizeImage(SkImage* image, ImageByteFormat format); + void setCallbackHandler(CallbackHandler* callbackHandler); + + // Any thread + void disposeVideoFrame(SkwasmObjectId videoFrameId); + + private: + void _runWorker(); + void _init(); + void _dispose(); + void _setCanvasSize(int width, int height); + void _recreateSurface(); + void _renderPicture(const SkPicture* picture); + void _rasterizeImage(SkImage* image, + ImageByteFormat format, + uint32_t callbackId); + void _disposeVideoFrame(SkwasmObjectId objectId); + void _onRasterizeComplete(SkData* data, uint32_t callbackId); + void _notifyRenderComplete(uint32_t callbackId); + void _onRenderComplete(uint32_t callbackId); + + std::string _canvasID; + CallbackHandler* _callbackHandler = nullptr; + uint32_t _currentCallbackId = 0; + + int _canvasWidth = 0; + int _canvasHeight = 0; + + EMSCRIPTEN_WEBGL_CONTEXT_HANDLE _glContext = 0; + sk_sp _grContext = nullptr; + sk_sp _surface = nullptr; + GrGLFramebufferInfo _fbInfo; + GrGLint _sampleCount; + GrGLint _stencil; + + pthread_t _thread; + + static void fDispose(Surface* surface); + static void fSetCanvasSize(Surface* surface, int width, int height); + static void fRenderPicture(Surface* surface, SkPicture* picture); + static void fNotifyRenderComplete(Surface* surface, uint32_t callbackId); + static void fOnRenderComplete(Surface* surface, uint32_t callbackId); + static void fRasterizeImage(Surface* surface, + SkImage* image, + ImageByteFormat format, + uint32_t callbackId); + static void fOnRasterizeComplete(Surface* surface, + SkData* imageData, + uint32_t callbackId); + static void fDisposeVideoFrame(Surface* surface, SkwasmObjectId videoFrameId); +}; +} // namespace Skwasm diff --git a/lib/web_ui/test/ui/filters_test.dart b/lib/web_ui/test/ui/filters_test.dart index e2d10e91c1907..1b66d5bf434f5 100644 --- a/lib/web_ui/test/ui/filters_test.dart +++ b/lib/web_ui/test/ui/filters_test.dart @@ -4,7 +4,6 @@ import 'dart:async'; import 'dart:math' as math; -import 'dart:typed_data'; import 'package:test/bootstrap/browser.dart'; import 'package:test/test.dart'; @@ -19,37 +18,20 @@ void main() { internalBootstrapBrowserTest(() => testMain); } -// TODO(jacksongardner): Skwasm doesn't support image codecs yet. Once it does, -// we can just replace this roundabout loading mechanism with a normal image -// load using flutter APIs. -Future rgbaTestImageData() async { - final DomHTMLImageElement image = createDomHTMLImageElement(); - image.src = '/test_images/mandrill_128.png'; - await image.decode(); - final DomCanvasElement canvas = createDomCanvasElement(width: 128, height: 128); - final DomCanvasRenderingContext2D context = canvas.getContext('2d')! as DomCanvasRenderingContext2D; - context.drawImage(image, 0, 0); - return context.getImageData(0, 0, 128, 128).data; -} - Future testMain() async { setUpUnitTests( setUpTestViewDimensions: false, ); - final Uint8List testImageData = Uint8List.fromList(await rgbaTestImageData()); const ui.Rect region = ui.Rect.fromLTWH(0, 0, 128, 128); Future drawTestImageWithPaint(ui.Paint paint) async { - final Completer imageCompleter = Completer(); - ui.decodeImageFromPixels( - testImageData, - 128, - 128, - ui.PixelFormat.rgba8888, - (ui.Image image) => imageCompleter.complete(image), + final ui.Codec codec = await renderer.instantiateImageCodecFromUrl( + Uri(path: '/test_images/mandrill_128.png') ); - final ui.Image image = await imageCompleter.future; + + final ui.FrameInfo info = await codec.getNextFrame(); + final ui.Image image = info.image; expect(image.width, 128); expect(image.height, 128); final ui.PictureRecorder recorder = ui.PictureRecorder(); diff --git a/lib/web_ui/test/ui/image_golden_test.dart b/lib/web_ui/test/ui/image_golden_test.dart index 71884e6eebe6e..c1defa07ccfed 100644 --- a/lib/web_ui/test/ui/image_golden_test.dart +++ b/lib/web_ui/test/ui/image_golden_test.dart @@ -84,10 +84,17 @@ Future testMain() async { // `imageGenerator` should produce an image that is 150x150 pixels. void emitImageTests(String name, Future Function() imageGenerator) { group(name, () { + late ui.Image image; + setUp(() async { + image = await imageGenerator(); + }); + + tearDown(() { + image.dispose(); + }); + test('drawImage', () async { final ui.Image image = await imageGenerator(); - expect(image.width, 150); - expect(image.height, 150); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder, drawRegion); @@ -100,8 +107,6 @@ Future testMain() async { test('drawImageRect', () async { final ui.Image image = await imageGenerator(); - expect(image.width, 150); - expect(image.height, 150); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder, drawRegion); @@ -119,8 +124,6 @@ Future testMain() async { test('drawImageNine', () async { final ui.Image image = await imageGenerator(); - expect(image.width, 150); - expect(image.height, 150); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder, drawRegion); @@ -138,8 +141,6 @@ Future testMain() async { test('image_shader_cubic_rotated', () async { final ui.Image image = await imageGenerator(); - expect(image.width, 150); - expect(image.height, 150); final Float64List matrix = Matrix4.rotationZ(pi / 6).toFloat64(); final ui.ImageShader shader = ui.ImageShader( @@ -162,8 +163,6 @@ Future testMain() async { test('fragment_shader_sampler', () async { final ui.Image image = await imageGenerator(); - expect(image.width, 150); - expect(image.height, 150); final ui.FragmentProgram program = await renderer.createFragmentProgram('glitch_shader'); final ui.FragmentShader shader = program.fragmentShader(); @@ -189,18 +188,14 @@ Future testMain() async { test('toByteData_rgba', () async { final ui.Image image = await imageGenerator(); - expect(image.width, 150); - expect(image.height, 150); final ByteData? rgbaData = await image.toByteData(); expect(rgbaData, isNotNull); expect(rgbaData!.lengthInBytes, isNonZero); }); - test('toByteData_rgba', () async { + test('toByteData_png', () async { final ui.Image image = await imageGenerator(); - expect(image.width, 150); - expect(image.height, 150); final ByteData? pngData = await image.toByteData(format: ui.ImageByteFormat.png); expect(pngData, isNotNull); @@ -288,4 +283,25 @@ Future testMain() async { return completer.future; }); } + + emitImageTests('codec_uri', () async { + final ui.Codec codec = await renderer.instantiateImageCodecFromUrl( + Uri(path: '/test_images/mandrill_128.png') + ); + + final ui.FrameInfo info = await codec.getNextFrame(); + return info.image; + }); + + emitImageTests('codec_list_resized', () async { + final ByteBuffer data = await httpFetchByteBuffer('/test_images/mandrill_128.png'); + final ui.Codec codec = await renderer.instantiateImageCodec( + data.asUint8List(), + targetWidth: 150, + targetHeight: 150, + ); + + final ui.FrameInfo info = await codec.getNextFrame(); + return info.image; + }); }