Add support for double tap action from Apple Pencil 2 (#39267)

* working commit

* some clean up

* white space fixes

* whitespace

* remove unused import

* Addressing PR comment, fix tests, update touch packet on Andriod

* formatting n whitespace again

* add new field to web ui pointer.dart

* update test

* whitespace

* fix test

* PR comments

* fix test

* whitespace

* fix test by factoring out logic into helper function

* whitespace

* fix malformed string

* pr comments

* fix type issue

* sigh whitespace

* revert test changes :)

* pr comments + separate out tests

* extra space

* change test name

Author: LouiseHsu

lib/ui/platform_dispatcher.dart

@@ -371,7 +371,7 @@ class PlatformDispatcher {
   //  * pointer_data.cc
   //  * pointer.dart
   //  * AndroidTouchProcessor.java
-  static const int _kPointerDataFieldCount = 35;
+  static const int _kPointerDataFieldCount = 36;
 
   static PointerDataPacket _unpackPointerDataPacket(ByteData packet) {
     const int kStride = Int64List.bytesPerElement;
@@ -417,6 +417,7 @@ class PlatformDispatcher {
         panDeltaY: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
         scale: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
         rotation: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        preferredStylusAuxiliaryAction: PointerPreferredStylusAuxiliaryAction.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)],
       ));
       assert(offset == (i + 1) * _kPointerDataFieldCount);
     }

lib/ui/pointer.dart

@@ -133,10 +133,31 @@ enum PointerSignalKind {
   /// A pointer-generated scale event (e.g. trackpad pinch).
   scale,
 
+  /// A stylus generated action (e.g. double tap on Apple Pencil 2)
+  stylusAuxiliaryAction,
+
   /// An unknown pointer signal kind.
   unknown
 }
 
+  /// The preferred action for stylus action
+enum PointerPreferredStylusAuxiliaryAction {
+  /// Ignore pointer input
+  ignore,
+
+  /// Show colour palette if available
+  showColorPalette,
+
+  /// Switch to eraser if available
+  switchEraser,
+
+  /// Switch to previous tool
+  switchPrevious,
+
+  /// unknown preferred action
+  unknown,
+}
+
 /// Information about the state of a pointer.
 class PointerData {
   /// Creates an object that represents the state of a pointer.
@@ -176,6 +197,7 @@ class PointerData {
     this.panDeltaY = 0.0,
     this.scale = 0.0,
     this.rotation = 0.0,
+    this.preferredStylusAuxiliaryAction = PointerPreferredStylusAuxiliaryAction.ignore,
   });
 
   /// Unique identifier that ties the [PointerEvent] to embedder event created it.
@@ -374,6 +396,11 @@ class PointerData {
   /// The current angle of the pan/zoom in radians, with 0.0 as the initial angle.
   final double rotation;
 
+  /// For events with signal kind of stylusAuxiliaryAction
+  ///
+  /// The current preferred action for stylusAuxiliaryAction, with ignore as the default.
+  final PointerPreferredStylusAuxiliaryAction preferredStylusAuxiliaryAction;
+
   @override
   String toString() => 'PointerData(x: $physicalX, y: $physicalY)';
 
@@ -413,7 +440,8 @@ class PointerData {
              'panDeltaX: $panDeltaX, '
              'panDeltaY: $panDeltaY, '
              'scale: $scale, '
-             'rotation: $rotation'
+             'rotation: $rotation, '
+             'preferredStylusAuxiliaryAction: $preferredStylusAuxiliaryAction'
            ')';
   }
 }

lib/ui/window/pointer_data.h

@@ -11,7 +11,7 @@ namespace flutter {
 
 // If this value changes, update the pointer data unpacking code in
 // platform_dispatcher.dart.
-static constexpr int kPointerDataFieldCount = 35;
+static constexpr int kPointerDataFieldCount = 36;
 static constexpr int kBytesPerField = sizeof(int64_t);
 // Must match the button constants in events.dart.
 enum PointerButtonMouse : int64_t {
@@ -63,6 +63,16 @@ struct alignas(8) PointerData {
     kScroll,
     kScrollInertiaCancel,
     kScale,
+    kStylusAuxiliaryAction,
+  };
+
+  // Must match the PreferredStylusAuxiliaryAction enum in pointer.dart.
+  enum class PreferredStylusAuxiliaryAction : int64_t {
+    kIgnore,
+    kShowColorPalette,
+    kSwitchEraser,
+    kSwitchPrevious,
+    kUnknown
   };
 
   int64_t embedder_id;
@@ -100,6 +110,7 @@ struct alignas(8) PointerData {
   double pan_delta_y;
   double scale;
   double rotation;
+  PreferredStylusAuxiliaryAction preferred_auxiliary_stylus_action;
 
   void Clear();
 };

lib/ui/window/pointer_data_packet_converter.cc

@@ -293,6 +293,7 @@ void PointerDataPacketConverter::ConvertPointerData(
     switch (pointer_data.signal_kind) {
       case PointerData::SignalKind::kScroll:
       case PointerData::SignalKind::kScrollInertiaCancel:
+      case PointerData::SignalKind::kStylusAuxiliaryAction:
       case PointerData::SignalKind::kScale: {
         // Makes sure we have an existing pointer
         auto iter = states_.find(pointer_data.device);

lib/ui/window/pointer_data_packet_converter_unittests.cc

@@ -45,6 +45,8 @@ void CreateSimulatedPointerData(PointerData& data,  // NOLINT
   data.platformData = 0;
   data.scroll_delta_x = 0.0;
   data.scroll_delta_y = 0.0;
+  data.preferred_auxiliary_stylus_action =
+      PointerData::PreferredStylusAuxiliaryAction::kIgnore;
 }
 
 void CreateSimulatedMousePointerData(PointerData& data,  // NOLINT
@@ -84,6 +86,8 @@ void CreateSimulatedMousePointerData(PointerData& data,  // NOLINT
   data.platformData = 0;
   data.scroll_delta_x = scroll_delta_x;
   data.scroll_delta_y = scroll_delta_y;
+  data.preferred_auxiliary_stylus_action =
+      PointerData::PreferredStylusAuxiliaryAction::kIgnore;
 }
 
 void CreateSimulatedTrackpadGestureData(PointerData& data,  // NOLINT
@@ -129,6 +133,8 @@ void CreateSimulatedTrackpadGestureData(PointerData& data,  // NOLINT
   data.pan_delta_y = 0.0;
   data.scale = scale;
   data.rotation = rotation;
+  data.preferred_auxiliary_stylus_action =
+      PointerData::PreferredStylusAuxiliaryAction::kIgnore;
 }
 
 void UnpackPointerPacket(std::vector<PointerData>& output,  // NOLINT

lib/web_ui/lib/pointer.dart

@@ -34,6 +34,24 @@ enum PointerSignalKind {
   unknown
 }
 
+  /// The preferred action for stylus action
+enum PointerPreferredStylusAuxiliaryAction {
+  /// Ignore pointer input
+  ignore,
+
+  /// Show colour palette if available
+  showColorPalette,
+
+  /// Switch to eraser if available
+  switchEraser,
+
+  /// Switch to previous tool
+  switchPrevious,
+
+  /// unknown preferred action
+  unknown,
+}
+
 class PointerData {
   const PointerData({
     this.embedderId = 0,
@@ -71,6 +89,7 @@ class PointerData {
     this.panDeltaY = 0.0,
     this.scale = 0.0,
     this.rotation = 0.0,
+    this.preferredStylusAuxiliaryAction = PointerPreferredStylusAuxiliaryAction.ignore,
   });
   final int embedderId;
   final Duration timeStamp;
@@ -107,6 +126,7 @@ class PointerData {
   final double panDeltaY;
   final double scale;
   final double rotation;
+  final PointerPreferredStylusAuxiliaryAction preferredStylusAuxiliaryAction;
 
   @override
   String toString() => 'PointerData(x: $physicalX, y: $physicalY)';
@@ -145,7 +165,8 @@ class PointerData {
            'panDeltaX: $panDeltaX, '
            'panDeltaY: $panDeltaY, '
            'scale: $scale, '
-           'rotation: $rotation'
+           'rotation: $rotation, '
+           'preferredStylusAuxiliaryAction: $preferredStylusAuxiliaryAction'
            ')';
   }
 }

shell/common/input_events_unittests.cc

@@ -176,6 +176,8 @@ void CreateSimulatedPointerData(PointerData& data,
   data.platformData = 0;
   data.scroll_delta_x = 0.0;
   data.scroll_delta_y = 0.0;
+  data.preferred_auxiliary_stylus_action =
+      PointerData::PreferredStylusAuxiliaryAction::kIgnore;
 }
 
 TEST_F(ShellTest, MissAtMostOneFrameForIrregularInputEvents) {

shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java

@@ -66,18 +66,36 @@ public class AndroidTouchProcessor {
     PointerSignalKind.SCROLL,
     PointerSignalKind.SCROLL_INERTIA_CANCEL,
     PointerSignalKind.SCALE,
+    PointerSignalKind.STYLUS_AUXILIARY_ACTION,
     PointerSignalKind.UNKNOWN
   })
   public @interface PointerSignalKind {
     int NONE = 0;
     int SCROLL = 1;
     int SCROLL_INERTIA_CANCEL = 2;
     int SCALE = 3;
+    int STYLUS_AUXILIARY_ACTION = 4;
+    int UNKNOWN = 5;
+  }
+
+  // Must match the PointerPreferredStylusAuxiliaryAction enum in pointer.dart.
+  @IntDef({
+    PointerPreferredStylusAuxiliaryAction.IGNORE,
+    PointerPreferredStylusAuxiliaryAction.SHOW_COLOR_PALETTE,
+    PointerPreferredStylusAuxiliaryAction.SWITCH_ERASER,
+    PointerPreferredStylusAuxiliaryAction.SWITCH_PREVIOUS,
+    PointerPreferredStylusAuxiliaryAction.UNKNOWN
+  })
+  public @interface PointerPreferredStylusAuxiliaryAction {
+    int IGNORE = 0;
+    int SHOW_COLOR_PALETTE = 1;
+    int SWITCH_ERASER = 2;
+    int SWITCH_PREVIOUS = 3;
     int UNKNOWN = 4;
   }
 
   // Must match the unpacking code in hooks.dart.
-  private static final int POINTER_DATA_FIELD_COUNT = 35;
+  private static final int POINTER_DATA_FIELD_COUNT = 36;
   @VisibleForTesting static final int BYTES_PER_FIELD = 8;
 
   // This value must match the value in framework's platform_view.dart.
@@ -355,6 +373,8 @@ private void addPointerForIndex(
     packet.putDouble(1.0); // scale
     packet.putDouble(0.0); // rotation
 
+    packet.putLong(PointerPreferredStylusAuxiliaryAction.IGNORE); // preferred stylus action
+
     if (isTrackpadPan && getPointerChangeForPanZoom(pointerChange) == PointerChange.PAN_ZOOM_END) {
       ongoingPans.remove(event.getPointerId(pointerIndex));
     }

shell/platform/darwin/ios/framework/Source/FlutterViewController.mm

@@ -56,7 +56,9 @@
 // This is left a FlutterBinaryMessenger privately for now to give people a chance to notice the
 // change. Unfortunately unless you have Werror turned on, incompatible pointers as arguments are
 // just a warning.
-@interface FlutterViewController () <FlutterBinaryMessenger, UIScrollViewDelegate>
+@interface FlutterViewController () <FlutterBinaryMessenger,
+                                     UIScrollViewDelegate,
+                                     UIPencilInteractionDelegate>
 @property(nonatomic, readwrite, getter=isDisplayingFlutterUI) BOOL displayingFlutterUI;
 @property(nonatomic, assign) BOOL isHomeIndicatorHidden;
 @property(nonatomic, assign) BOOL isPresentingViewControllerAnimating;
@@ -97,7 +99,7 @@ @interface FlutterViewController () <FlutterBinaryMessenger, UIScrollViewDelegat
 // Trackpad rotating
 @property(nonatomic, retain)
     UIRotationGestureRecognizer* rotationGestureRecognizer API_AVAILABLE(ios(13.4));
-
+@property(nonatomic, retain) UIPencilInteraction* pencilInteraction API_AVAILABLE(ios(13.4));
 /**
  * Creates and registers plugins used by this view controller.
  */
@@ -722,6 +724,10 @@ - (void)viewDidLoad {
   [self createTouchRateCorrectionVSyncClientIfNeeded];
 
   if (@available(iOS 13.4, *)) {
+    _pencilInteraction = [[UIPencilInteraction alloc] init];
+    _pencilInteraction.delegate = self;
+    [_flutterView addInteraction:_pencilInteraction];
+
     _hoverGestureRecognizer =
         [[UIHoverGestureRecognizer alloc] initWithTarget:self action:@selector(hoverEvent:)];
     _hoverGestureRecognizer.delegate = self;
@@ -895,6 +901,8 @@ - (void)dealloc {
   [_pinchGestureRecognizer release];
   _rotationGestureRecognizer.delegate = nil;
   [_rotationGestureRecognizer release];
+  _pencilInteraction.delegate = nil;
+  [_pencilInteraction release];
   [super dealloc];
 }
 
@@ -969,7 +977,7 @@ - (void)goToApplicationLifecycle:(nonnull NSString*)state {
     case UITouchTypeDirect:
     case UITouchTypeIndirect:
       return flutter::PointerData::DeviceKind::kTouch;
-    case UITouchTypeStylus:
+    case UITouchTypePencil:
       return flutter::PointerData::DeviceKind::kStylus;
     case UITouchTypeIndirectPointer:
       return flutter::PointerData::DeviceKind::kMouse;
@@ -1224,6 +1232,50 @@ - (void)invalidateTouchRateCorrectionVSyncClient {
   _touchRateCorrectionVSyncClient = nil;
 }
 
+#pragma mark - Stylus Events
+
+- (void)pencilInteractionDidTap:(UIPencilInteraction*)interaction API_AVAILABLE(ios(13.4)) {
+  flutter::PointerData pointer_data = [self createAuxillaryStylusActionData];
+
+  auto packet = std::make_unique<flutter::PointerDataPacket>(1);
+  packet->SetPointerData(/*index=*/0, pointer_data);
+  [_engine.get() dispatchPointerDataPacket:std::move(packet)];
+}
+
+- (flutter::PointerData)createAuxillaryStylusActionData API_AVAILABLE(ios(13.4)) {
+  flutter::PointerData pointer_data;
+  pointer_data.Clear();
+
+  switch (UIPencilInteraction.preferredTapAction) {
+    case UIPencilPreferredActionIgnore:
+      pointer_data.preferred_auxiliary_stylus_action =
+          flutter::PointerData::PreferredStylusAuxiliaryAction::kIgnore;
+      break;
+    case UIPencilPreferredActionShowColorPalette:
+      pointer_data.preferred_auxiliary_stylus_action =
+          flutter::PointerData::PreferredStylusAuxiliaryAction::kShowColorPalette;
+      break;
+    case UIPencilPreferredActionSwitchEraser:
+      pointer_data.preferred_auxiliary_stylus_action =
+          flutter::PointerData::PreferredStylusAuxiliaryAction::kSwitchEraser;
+      break;
+    case UIPencilPreferredActionSwitchPrevious:
+      pointer_data.preferred_auxiliary_stylus_action =
+          flutter::PointerData::PreferredStylusAuxiliaryAction::kSwitchPrevious;
+      break;
+    default:
+      pointer_data.preferred_auxiliary_stylus_action =
+          flutter::PointerData::PreferredStylusAuxiliaryAction::kUnknown;
+      break;
+  }
+
+  pointer_data.time_stamp = [[NSProcessInfo processInfo] systemUptime] * kMicrosecondsPerSecond;
+  pointer_data.kind = flutter::PointerData::DeviceKind::kStylus;
+  pointer_data.signal_kind = flutter::PointerData::SignalKind::kStylusAuxiliaryAction;
+
+  return pointer_data;
+}
+
 #pragma mark - Handle view resizing
 
 - (void)updateViewportMetrics {