Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

ChromeOS/Android trackpad gestures #31595

Merged
fluttergithubbot merged 6 commits into from
Jun 14, 2022
Merged

ChromeOS/Android trackpad gestures #31595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6a21ca9
Android trackpad gestures
moffatman Feb 12, 2022
5120130
Add test
moffatman Mar 31, 2022
b41a92b
Use mock instead of shadow
moffatman Apr 5, 2022
eff543c
Address feedback
moffatman Apr 20, 2022
c7ee3db
Add comment
moffatman May 11, 2022
8df97aa
Feedback 2
moffatman Jun 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 70 additions & 31 deletions shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
import android.view.MotionEvent;
import androidx.annotation.IntDef;
import androidx.annotation.NonNull;
import androidx.annotation.VisibleForTesting;
import io.flutter.embedding.engine.renderer.FlutterRenderer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.HashMap;
import java.util.Map;

/** Sends touch information from Android to Flutter in a format that Flutter understands. */
public class AndroidTouchProcessor {
Expand All @@ -26,7 +29,7 @@ public class AndroidTouchProcessor {
PointerChange.PAN_ZOOM_UPDATE,
PointerChange.PAN_ZOOM_END
})
private @interface PointerChange {
public @interface PointerChange {
int CANCEL = 0;
int ADD = 1;
int REMOVE = 2;
Expand All @@ -48,7 +51,7 @@ public class AndroidTouchProcessor {
PointerDeviceKind.TRACKPAD,
PointerDeviceKind.UNKNOWN
})
private @interface PointerDeviceKind {
public @interface PointerDeviceKind {
int TOUCH = 0;
int MOUSE = 1;
int STYLUS = 2;
Expand All @@ -59,15 +62,15 @@ public class AndroidTouchProcessor {

// Must match the PointerSignalKind enum in pointer.dart.
@IntDef({PointerSignalKind.NONE, PointerSignalKind.SCROLL, PointerSignalKind.UNKNOWN})
private @interface PointerSignalKind {
public @interface PointerSignalKind {
int NONE = 0;
int SCROLL = 1;
int UNKNOWN = 2;
}

// Must match the unpacking code in hooks.dart.
private static final int POINTER_DATA_FIELD_COUNT = 35;
private static final int BYTES_PER_FIELD = 8;
@VisibleForTesting static final int BYTES_PER_FIELD = 8;

// This value must match the value in framework's platform_view.dart.
// This flag indicates whether the original Android pointer events were batched together.
Expand All @@ -76,12 +79,12 @@ public class AndroidTouchProcessor {
@NonNull private final FlutterRenderer renderer;
@NonNull private final MotionEventTracker motionEventTracker;

private static final int _POINTER_BUTTON_PRIMARY = 1;

private static final Matrix IDENTITY_TRANSFORM = new Matrix();

private final boolean trackMotionEvents;

private final Map<Integer, float[]> ongoingPans = new HashMap<>();

/**
* Constructs an {@code AndroidTouchProcessor} that will send touch event data to the Flutter
* execution context represented by the given {@link FlutterRenderer}.
Expand Down Expand Up @@ -220,6 +223,28 @@ private void addPointerForIndex(
}

int pointerKind = getPointerDeviceTypeForToolType(event.getToolType(pointerIndex));
// We use this in lieu of using event.getRawX and event.getRawY as we wish to support
// earlier versions than API level 29.
float viewToScreenCoords[] = {event.getX(pointerIndex), event.getY(pointerIndex)};
transformMatrix.mapPoints(viewToScreenCoords);
long buttons;
if (pointerKind == PointerDeviceKind.MOUSE) {
buttons = event.getButtonState() & 0x1F;
if (buttons == 0
&& event.getSource() == InputDevice.SOURCE_MOUSE
&& pointerChange == PointerChange.DOWN) {
// Some implementations translate trackpad scrolling into a mouse down-move-up event
// sequence with buttons: 0, such as ARC on a Chromebook. See #11420, a legacy
// implementation that uses the same condition but converts differently.
ongoingPans.put(event.getPointerId(pointerIndex), viewToScreenCoords);
}
Comment on lines +231 to +240
Copy link
Contributor

@dkwingsmt dkwingsmt May 12, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just tested on a real device and found out that ChromeOS now converts trackpad scrolling into events with tool type "finger" instead of "mouse". I've therefore fixed the logic as below. This also makes these events especially distinctive.

Suggested change
if (pointerKind == PointerDeviceKind.MOUSE) {
buttons = event.getButtonState() & 0x1F;
if (buttons == 0
&& event.getSource() == InputDevice.SOURCE_MOUSE
&& pointerChange == PointerChange.DOWN) {
// Some implementations translate trackpad scrolling into a mouse down-move-up event
// sequence with buttons: 0, such as ARC on a Chromebook. See #11420, a legacy
// implementation that uses the same condition but converts differently.
ongoingPans.put(event.getPointerId(pointerIndex), viewToScreenCoords);
}
if (pointerKind == PointerDeviceKind.MOUSE) {
buttons = event.getButtonState() & 0x1F;
} else if (pointerKind == PointerDeviceKind.TOUCH) {
buttons = 0;
// Chromebook with ARC translates trackpad scrolling into a touch down-move-up event
// sequence, which can be identified by a distinctive combination of input device "mouse",
// tool type "finger", and buttons 0.
if (event.getButtonState() == 0
&& event.getSource() == InputDevice.SOURCE_MOUSE
&& pointerChange == PointerChange.DOWN) {
ongoingPans.put(event.getPointerId(pointerIndex), viewToScreenCoords);
}

As a result, running the demo at https://docs.flutter.dev/cookbook/effects/drag-a-widget#interactive-example, moving the cursor onto the menu and scrolling the trackpad now correctly scrolls the list instead of dragging the items. :D

Copy link
Contributor Author

@moffatman moffatman May 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any concern that we have no idea which proportion of devices this will work on? I guess it's based on undocumented behaviour anyway, and Chrome OS is always up to date.

Copy link
Contributor

@dkwingsmt dkwingsmt May 12, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're required to make Flutter work on ChromeOS, so the portion of device shouldn't matter. Sure the behavior might change, and it's best we run an integration test, but I don't think we can achieve this in a short while. Maybe leave a TODO?

Copy link
Contributor

@dkwingsmt dkwingsmt May 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also maybe it's better to use switch instead.

Copy link
Contributor

@dkwingsmt dkwingsmt May 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Never mind, I'll handle this change in another PR. The current code passes all tests anyway.

} else if (pointerKind == PointerDeviceKind.STYLUS) {
buttons = (event.getButtonState() >> 4) & 0xF;
} else {
buttons = 0;
}

boolean isTrackpadPan = ongoingPans.containsKey(event.getPointerId(pointerIndex));

int signalKind =
event.getActionMasked() == MotionEvent.ACTION_SCROLL
Expand All @@ -230,39 +255,31 @@ private void addPointerForIndex(

packet.putLong(motionEventId); // motionEventId
packet.putLong(timeStamp); // time_stamp
packet.putLong(pointerChange); // change
packet.putLong(pointerKind); // kind
if (isTrackpadPan) {
packet.putLong(getPointerChangeForPanZoom(pointerChange)); // change
packet.putLong(PointerDeviceKind.TRACKPAD); // kind
} else {
packet.putLong(pointerChange); // change
packet.putLong(pointerKind); // kind
}
packet.putLong(signalKind); // signal_kind
packet.putLong(event.getPointerId(pointerIndex)); // device
packet.putLong(0); // pointer_identifier, will be generated in pointer_data_packet_converter.cc.

// We use this in lieu of using event.getRawX and event.getRawY as we wish to support
// earlier versions than API level 29.
float viewToScreenCoords[] = {event.getX(pointerIndex), event.getY(pointerIndex)};
transformMatrix.mapPoints(viewToScreenCoords);
packet.putDouble(viewToScreenCoords[0]); // physical_x
packet.putDouble(viewToScreenCoords[1]); // physical_y
if (isTrackpadPan) {
float[] panStart = ongoingPans.get(event.getPointerId(pointerIndex));
packet.putDouble(panStart[0]);
packet.putDouble(panStart[1]);
} else {
packet.putDouble(viewToScreenCoords[0]); // physical_x
packet.putDouble(viewToScreenCoords[1]); // physical_y
}

packet.putDouble(
0.0); // physical_delta_x, will be generated in pointer_data_packet_converter.cc.
packet.putDouble(
0.0); // physical_delta_y, will be generated in pointer_data_packet_converter.cc.

long buttons;
if (pointerKind == PointerDeviceKind.MOUSE) {
buttons = event.getButtonState() & 0x1F;
// TODO(dkwingsmt): Remove this fix after implementing touchpad gestures
// https://github.com/flutter/flutter/issues/23604#issuecomment-524471152
if (buttons == 0
&& event.getSource() == InputDevice.SOURCE_MOUSE
&& (pointerChange == PointerChange.DOWN || pointerChange == PointerChange.MOVE)) {
buttons = _POINTER_BUTTON_PRIMARY;
}
} else if (pointerKind == PointerDeviceKind.STYLUS) {
buttons = (event.getButtonState() >> 4) & 0xF;
} else {
buttons = 0;
}
packet.putLong(buttons); // buttons

packet.putLong(0); // obscured
Expand Down Expand Up @@ -317,12 +334,22 @@ private void addPointerForIndex(
packet.putDouble(0.0); // scroll_delta_x
}

packet.putDouble(0.0); // pan_x
packet.putDouble(0.0); // pan_y
if (isTrackpadPan) {
float[] panStart = ongoingPans.get(event.getPointerId(pointerIndex));
packet.putDouble(viewToScreenCoords[0] - panStart[0]);
packet.putDouble(viewToScreenCoords[1] - panStart[1]);
} else {
packet.putDouble(0.0); // pan_x
packet.putDouble(0.0); // pan_y
}
packet.putDouble(0.0); // pan_delta_x
packet.putDouble(0.0); // pan_delta_y
packet.putDouble(1.0); // scale
packet.putDouble(0.0); // rotation

if (isTrackpadPan && getPointerChangeForPanZoom(pointerChange) == PointerChange.PAN_ZOOM_END) {
ongoingPans.remove(event.getPointerId(pointerIndex));
}
}

@PointerChange
Expand Down Expand Up @@ -357,6 +384,18 @@ private int getPointerChangeForAction(int maskedAction) {
return -1;
}

@PointerChange
private int getPointerChangeForPanZoom(int pointerChange) {
if (pointerChange == PointerChange.DOWN) {
return PointerChange.PAN_ZOOM_START;
} else if (pointerChange == PointerChange.MOVE) {
return PointerChange.PAN_ZOOM_UPDATE;
} else if (pointerChange == PointerChange.UP || pointerChange == PointerChange.CANCEL) {
return PointerChange.PAN_ZOOM_END;
}
return -1;
}

@PointerDeviceKind
private int getPointerDeviceTypeForToolType(int toolType) {
switch (toolType) {
Expand Down
Loading