Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Add a11y support for embedded iOS platform view #8156

Merged
cyanglaz merged 21 commits into from
Mar 26, 2019
Merged

Add a11y support for embedded iOS platform view #8156

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
3bcb1db
adding a11y support for ios platform view
Mar 13, 2019
8fcf6b0
comment fix
Mar 13, 2019
c036e80
formatting
Mar 13, 2019
05f77a3
make the node for platform view to be a container
Mar 13, 2019
b776a05
comments on ignoring other semantic configurations when platform view…
Mar 14, 2019
8ed39ee
review fixes
Mar 14, 2019
a9a0c15
cleanup a11y elements if platform view node changed to non platform v…
Mar 14, 2019
0e58ef8
remove unnecessary testing code
Mar 15, 2019
4e670db
nit fix
Mar 15, 2019
15c03ce
review fixes
Mar 15, 2019
6fb37fe
explain what are ignored
Mar 15, 2019
3e57265
renaming
Mar 15, 2019
4a06b68
formatting
Mar 15, 2019
4ea581f
adding another container under platform view semantic node to handle …
Mar 19, 2019
68251d8
bad revert fixes, added comments
Mar 19, 2019
c540565
rewording comments
Mar 19, 2019
4a29306
create a FlutterPlatformViewsController for platform view
Mar 19, 2019
fb72727
caching the _platformViewSemanticsContainer
Mar 21, 2019
5ff2ddc
create platform view container in updateSemantics
Mar 21, 2019
fae2cc0
comment fixes
Mar 21, 2019
0d3bb16
doc fixes
Mar 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,13 @@
composition_order_.push_back(view_id);
}

NSObject<FlutterPlatformView>* FlutterPlatformViewsController::GetPlatformViewByID(int view_id) {
if (views_.empty()) {
return nil;
}
return views_[view_id].get();
}

std::vector<SkCanvas*> FlutterPlatformViewsController::GetCurrentCanvases() {
std::vector<SkCanvas*> canvases;
for (size_t i = 0; i < composition_order_.size(); i++) {
Expand Down
2 changes: 2 additions & 0 deletions shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ class FlutterPlatformViewsController {

void PrerollCompositeEmbeddedView(int view_id);

NSObject<FlutterPlatformView>* GetPlatformViewByID(int view_id);

std::vector<SkCanvas*> GetCurrentCanvases();

SkCanvas* CompositeEmbeddedView(int view_id, const flow::EmbeddedViewParams& params);
Expand Down
9 changes: 8 additions & 1 deletion shell/platform/darwin/ios/framework/Source/accessibility_bridge.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,9 @@ class PlatformViewIOS;

class AccessibilityBridge final {
public:
AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view);
AccessibilityBridge(UIView* view,
PlatformViewIOS* platform_view,
FlutterPlatformViewsController* platform_views_controller);
~AccessibilityBridge();

void UpdateSemantics(blink::SemanticsNodeUpdates nodes,
Expand All @@ -129,6 +131,10 @@ class AccessibilityBridge final {

fml::WeakPtr<AccessibilityBridge> GetWeakPtr();

FlutterPlatformViewsController* flutter_platform_views_controller() const {
Copy link
Member

@cbracken cbracken Mar 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

... GetPlatformViewsController() const for consistency with the rest of the engine.

Copy link
Contributor Author

@cyanglaz cyanglaz Mar 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

return flutter_platform_views_controller_;
};

void clearState();

private:
Expand All @@ -139,6 +145,7 @@ class AccessibilityBridge final {

UIView* view_;
PlatformViewIOS* platform_view_;
FlutterPlatformViewsController* flutter_platform_views_controller_;
Copy link
Member

@cbracken cbracken Mar 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

platform_views_controller for consistency with elsewhere.

Copy link
Contributor Author

@cyanglaz cyanglaz Mar 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

fml::scoped_nsobject<NSMutableDictionary<NSNumber*, SemanticsObject*>> objects_;
fml::scoped_nsprotocol<FlutterBasicMessageChannel*> accessibility_channel_;
fml::WeakPtrFactory<AccessibilityBridge> weak_factory_;
Expand Down
20 changes: 17 additions & 3 deletions shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include "flutter/fml/logging.h"
#include "flutter/shell/platform/darwin/ios/platform_view_ios.h"
#include "flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h"

namespace {

Expand Down Expand Up @@ -436,8 +437,20 @@ - (nullable id)accessibilityElementAtIndex:(NSInteger)index {
return _semanticsObject;
}
SemanticsObject* child = [_semanticsObject children][index - 1];
if ([child hasChildren])
return [child accessibilityContainer];

// This 'if' block handles adding accessibility support for the embedded platform view.
// We first check if the child is a semantic node for a platform view.
// If so, we add the platform view as accessibilityElements of the child.
shell::FlutterPlatformViewsController *flutterPlatformViewsController = _bridge.get()->flutter_platform_views_controller();
if (child.node.platformViewId > -1 && flutterPlatformViewsController) {
Copy link
Member

@goderbauer goderbauer Mar 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if child at a later point no longer has a platfromViewId? How do you remove the native a11y elements?

Copy link
Contributor Author

@cyanglaz cyanglaz Mar 14, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can clean it up when it happens, but I guess it might require some ugly tracking logic or some ugly type check. Is it even possible that the child object's platformViewId got changed but the child still exists in the tree?

NSObject<FlutterPlatformView> *platformViewProtocolObject = flutterPlatformViewsController->GetPlatformViewByID(child.node.platformViewId);
UIView *platformView = [platformViewProtocolObject view];
child.accessibilityElements = @[platformView];
Copy link
Member

@goderbauer goderbauer Mar 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about accessibilityContainer on platfromView? Doesn't that have to link back?

Copy link
Contributor Author

@cyanglaz cyanglaz Mar 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don' think we need to set the accessibilityContainer on the platform view since it is already an UIView.
So our a11y tree finds this node for platform views, and we set its accessibilityElements as the platform view. Now the platform view will handle the a11y on however it wants, but in the end, because the child here is still part of our tree, it will return when the accessibilityElements is done traversing.

Copy link
Member

@goderbauer goderbauer Mar 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding is - and this may be incorrect - that in the iOS tree on iOS each node needs to know its parent (the accessibilityContainer) and it's children, if it has any. In this case, the platformView doesn't know what its a11y parent is. This may be a problem if the platform view is contained within a Flutter Scroll view: if you put a11y focus on a node within the platform view, iOS cannot walk up the hierarchy to find the scrolling a11y node and dispatch the scroll event there. At least, that how I think that would work....

Copy link
Contributor Author

@cyanglaz cyanglaz Mar 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you are correct if traverse the accessibility dynamically. (Using the a11y protocol methods etc.) My understanding is that If we directly set the accessibilityElements property, iOS would handle the traversal and we don't need to worry about the parents at all. I might be wrong, but when I test with google map view inside a list view it works fine (it goes into the google map then goes out). Is there something else you can think of that might fail if parent is not set? I can try them out.

Copy link
Contributor Author

@cyanglaz cyanglaz Mar 14, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So I tried to switch the code to make the accessibility dynamically traverse through the platform view. It indeed won't pick up the parent as you mentioned. And we'd have to access the platform view's code to actually do it. Now the platform view is completely controlled by the author of the individual plugin so I think we should not touch it at all since they might want to do something else within their accessibility. I guess the safer way to do it would just be statically assign the accessibilityElements, and let iOS work its magic.

return child;
}

if ([child hasChildren])
return [child accessibilityContainer];
return child;
}

Expand Down Expand Up @@ -485,9 +498,10 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {

namespace shell {

AccessibilityBridge::AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view)
AccessibilityBridge::AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view, FlutterPlatformViewsController *platform_views_controller)
: view_(view),
platform_view_(platform_view),
flutter_platform_views_controller_(platform_views_controller),
objects_([[NSMutableDictionary alloc] init]),
weak_factory_(this),
previous_route_id_(0),
Expand Down
4 changes: 2 additions & 2 deletions shell/platform/darwin/ios/platform_view_ios.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@

if (accessibility_bridge_) {
accessibility_bridge_.reset(
new AccessibilityBridge(static_cast<FlutterView*>(owner_controller_.get().view), this));
new AccessibilityBridge(static_cast<FlutterView*>(owner_controller_.get().view), this, [owner_controller.get() platformViewsController]));
}
// Do not call `NotifyCreated()` here - let FlutterViewController take care
// of that when its Viewport is sized. If `NotifyCreated()` is called here,
Expand Down Expand Up @@ -96,7 +96,7 @@
}
if (enabled && !accessibility_bridge_) {
accessibility_bridge_ = std::make_unique<AccessibilityBridge>(
static_cast<FlutterView*>(owner_controller_.get().view), this);
static_cast<FlutterView*>(owner_controller_.get().view), this, [owner_controller_.get() platformViewsController]);
} else if (!enabled && accessibility_bridge_) {
accessibility_bridge_.reset();
}
Expand Down