From ccaa06367b03bc4203884139dac933cc851be7b7 Mon Sep 17 00:00:00 2001 From: Amir Hardon Date: Fri, 27 Jul 2018 10:58:56 -0700 Subject: [PATCH] AndroidView touch support. This PR adds 2 features to RenderAndroidView and AndroidView: 1. Hit testing behavior Adds a `PlatformViewHitTestBehavior` which is similar to `HitTestBehavior` without the `deferToChild` option (as platform views don't have child render objects) and with a `transparent` option which prevents it from forwarding any events to the Android view. 2. MotionEvent recomposing logic FlutterView and the framework `converter.dart` are working together to transform each Android MotionEvent object into one or more `PointerEvent` objects. This PR adds the reverse logic (in _MotionEventDispatcher which is used by RenderAndroidView) which turns a stream of PointerEvent objects into MotionEvent objects. The correctness of the recomposing logic is tested in an integration test which will land in a separate PR (the unit test PR is pretty big, trying to keep as many bite-size PRs for reviewer's convenience) --- .../lib/src/rendering/platform_view.dart | 191 +++++++++++++++++- .../lib/src/services/platform_views.dart | 15 ++ .../lib/src/widgets/platform_view.dart | 22 +- .../test/services/fake_platform_views.dart | 60 +++++- .../test/widgets/platform_view_test.dart | 191 +++++++++++++++++- 5 files changed, 471 insertions(+), 8 deletions(-) diff --git a/packages/flutter/lib/src/rendering/platform_view.dart b/packages/flutter/lib/src/rendering/platform_view.dart index 2b7f36facd..cb6279e3e8 100644 --- a/packages/flutter/lib/src/rendering/platform_view.dart +++ b/packages/flutter/lib/src/rendering/platform_view.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'dart:ui'; import 'package:flutter/foundation.dart'; +import 'package:flutter/gestures.dart'; import 'package:flutter/services.dart'; import 'box.dart'; @@ -13,6 +14,22 @@ import 'layer.dart'; import 'object.dart'; +/// How an embedded platform view behave during hit tests. +enum PlatformViewHitTestBehavior { + /// Opaque targets can be hit by hit tests, causing them to both receive + /// events within their bounds and prevent targets visually behind them from + /// also receiving events. + opaque, + + /// Translucent targets both receive events within their bounds and permit + /// targets visually behind them to also receive events. + translucent, + + /// Transparent targets don't receive events within their bounds and permit + /// targets visually behind them to receive events. + transparent, +} + enum _PlatformViewState { uninitialized, resizing, @@ -21,7 +38,8 @@ enum _PlatformViewState { /// A render object for an Android view. /// -/// [RenderAndroidView] is responsible for sizing and displaying an Android [View](https://developer.android.com/reference/android/view/View). +/// [RenderAndroidView] is responsible for sizing, displaying and passing touch events to an +/// Android [View](https://developer.android.com/reference/android/view/View). /// /// The render object's layout behavior is to fill all available space, the parent of this object must /// provide bounded layout constraints @@ -34,8 +52,12 @@ class RenderAndroidView extends RenderBox { /// Creates a render object for an Android view. RenderAndroidView({ @required AndroidViewController viewController, + @required this.hitTestBehavior, }) : assert(viewController != null), - _viewController = viewController; + assert(hitTestBehavior != null), + _viewController = viewController { + _motionEventsDispatcher = new _MotionEventsDispatcher(globalToLocal, viewController); + } _PlatformViewState _state = _PlatformViewState.uninitialized; @@ -47,10 +69,17 @@ class RenderAndroidView extends RenderBox { /// `viewController` must not be null. set viewController(AndroidViewController viewController) { assert(_viewController != null); + if (_viewController == viewController) + return; _viewController = viewController; _sizePlatformView(); } + /// How to behave during hit testing. + // The implicit setter is enough here as changing this value will just affect + // any newly arriving events there's nothing we need to invalidate. + PlatformViewHitTestBehavior hitTestBehavior; + @override bool get sizedByParent => true; @@ -60,6 +89,8 @@ class RenderAndroidView extends RenderBox { @override bool get isRepaintBoundary => true; + _MotionEventsDispatcher _motionEventsDispatcher; + @override void performResize() { size = constraints.biggest; @@ -96,4 +127,160 @@ class RenderAndroidView extends RenderBox { textureId: _viewController.textureId, )); } + + @override + bool hitTest(HitTestResult result, { Offset position }) { + if (hitTestBehavior == PlatformViewHitTestBehavior.transparent || !size.contains(position)) + return false; + result.add(new BoxHitTestEntry(this, position)); + return hitTestBehavior == PlatformViewHitTestBehavior.opaque; + } + + @override + bool hitTestSelf(Offset position) => hitTestBehavior != PlatformViewHitTestBehavior.transparent; + + @override + void handleEvent(PointerEvent event, HitTestEntry entry) { + _motionEventsDispatcher.handlePointerEvent(event); + } } + +typedef Offset _GlobalToLocal(Offset point); + +// Composes a stream of PointerEvent objects into AndroidMotionEvent objects +// and dispatches them to the associated embedded Android view. +class _MotionEventsDispatcher { + _MotionEventsDispatcher(this.globalToLocal, this.viewController); + + final Map pointerPositions = {}; + final Map pointerProperties = {}; + final _GlobalToLocal globalToLocal; + final AndroidViewController viewController; + + int nextPointerId = 0; + int downTimeMillis; + + void handlePointerEvent(PointerEvent event) { + if (event is PointerDownEvent) { + if (nextPointerId == 0) + downTimeMillis = event.timeStamp.inMilliseconds; + pointerProperties[event.pointer] = propertiesFor(event, nextPointerId++); + } + pointerPositions[event.pointer] = coordsFor(event); + + dispatchPointerEvent(event); + + if (event is PointerUpEvent) { + pointerPositions.remove(event.pointer); + pointerProperties.remove(event.pointer); + if (pointerProperties.isEmpty) { + nextPointerId = 0; + downTimeMillis = null; + } + } + if (event is PointerCancelEvent) { + pointerPositions.clear(); + pointerProperties.clear(); + nextPointerId = 0; + downTimeMillis = null; + } + } + + void dispatchPointerEvent(PointerEvent event) { + final List pointers = pointerPositions.keys.toList(); + final int pointerIdx = pointers.indexOf(event.pointer); + final int numPointers = pointers.length; + + // Android MotionEvent objects can batch information on multiple pointers. + // Flutter breaks these such batched events into multiple PointerEvent objects. + // When there are multiple active pointers we accumulate the information for all pointers + // as we get PointerEvents, and only send it to the embedded Android view when + // we see the last pointer. This way we achieve the same batching as Android. + if(isSinglePointerAction(event) && pointerIdx < numPointers - 1) + return; + + int action; + switch(event.runtimeType){ + case PointerDownEvent: + action = numPointers == 1 ? AndroidViewController.kActionDown + : AndroidViewController.pointerAction(pointerIdx, AndroidViewController.kActionPointerDown); + break; + case PointerUpEvent: + action = numPointers == 1 ? AndroidViewController.kActionUp + : AndroidViewController.pointerAction(pointerIdx, AndroidViewController.kActionPointerUp); + break; + case PointerMoveEvent: + action = AndroidViewController.kActionMove; + break; + case PointerCancelEvent: + action = AndroidViewController.kActionCancel; + break; + default: + return; + } + + final AndroidMotionEvent androidMotionEvent = new AndroidMotionEvent( + downTime: downTimeMillis, + eventTime: event.timeStamp.inMilliseconds, + action: action, + pointerCount: pointerPositions.length, + pointerProperties: pointers.map((int i) => pointerProperties[i]).toList(), + pointerCoords: pointers.map((int i) => pointerPositions[i]).toList(), + metaState: 0, + buttonState: 0, + xPrecision: 1.0, + yPrecision: 1.0, + deviceId: 0, + edgeFlags: 0, + source: 0, + flags: 0 + ); + viewController.sendMotionEvent(androidMotionEvent); + } + + AndroidPointerCoords coordsFor(PointerEvent event) { + final Offset position = globalToLocal(event.position); + return new AndroidPointerCoords( + orientation: event.orientation, + pressure: event.pressure, + // Currently the engine omits the pointer size, for now I'm fixing this to 0.33 which is roughly + // what I typically see on Android. + // + // TODO(amirh): Use the original pointer's size. + // https://github.com/flutter/flutter/issues/20300 + size: 0.333, + toolMajor: event.radiusMajor, + toolMinor: event.radiusMinor, + touchMajor: event.radiusMajor, + touchMinor: event.radiusMinor, + x: position.dx, + y: position.dy + ); + } + + AndroidPointerProperties propertiesFor(PointerEvent event, int pointerId) { + int toolType = AndroidPointerProperties.kToolTypeUnknown; + switch(event.kind) { + case PointerDeviceKind.touch: + toolType = AndroidPointerProperties.kToolTypeFinger; + break; + case PointerDeviceKind.mouse: + toolType = AndroidPointerProperties.kToolTypeMouse; + break; + case PointerDeviceKind.stylus: + toolType = AndroidPointerProperties.kToolTypeStylus; + break; + case PointerDeviceKind.invertedStylus: + toolType = AndroidPointerProperties.kToolTypeEraser; + break; + case PointerDeviceKind.unknown: + toolType = AndroidPointerProperties.kToolTypeUnknown; + break; + } + return new AndroidPointerProperties(id: pointerId, toolType: toolType); + } + + bool isSinglePointerAction(PointerEvent event) => + !(event is PointerDownEvent) && !(event is PointerUpEvent); +} + diff --git a/packages/flutter/lib/src/services/platform_views.dart b/packages/flutter/lib/src/services/platform_views.dart index 775134e406..78726aedc2 100644 --- a/packages/flutter/lib/src/services/platform_views.dart +++ b/packages/flutter/lib/src/services/platform_views.dart @@ -78,6 +78,21 @@ class PlatformViewsService { /// /// A Dart version of Android's [MotionEvent.PointerProperties](https://developer.android.com/reference/android/view/MotionEvent.PointerProperties). class AndroidPointerProperties { + /// Value for `toolType` when the tool type is unknown. + static const int kToolTypeUnknown = 0; + + /// Value for `toolType` when the tool type is a finger. + static const int kToolTypeFinger = 1; + + /// Value for `toolType` when the tool type is a stylus. + static const int kToolTypeStylus = 2; + + /// Value for `toolType` when the tool type is a mouse. + static const int kToolTypeMouse = 3; + + /// Value for `toolType` when the tool type is an eraser. + static const int kToolTypeEraser = 4; + /// Creates an AndroidPointerProperties. /// /// All parameters must not be null. diff --git a/packages/flutter/lib/src/widgets/platform_view.dart b/packages/flutter/lib/src/widgets/platform_view.dart index a784685a2b..7d8f467016 100644 --- a/packages/flutter/lib/src/widgets/platform_view.dart +++ b/packages/flutter/lib/src/widgets/platform_view.dart @@ -39,12 +39,14 @@ import 'framework.dart'; class AndroidView extends StatefulWidget { /// Creates a widget that embeds an Android view. /// - /// The `viewType` parameter must not be null. + /// The `viewType` and `hitTestBehavior` parameters must not be null. const AndroidView({ Key key, @required this.viewType, - this.onPlatformViewCreated + this.onPlatformViewCreated, + this.hitTestBehavior = PlatformViewHitTestBehavior.opaque, }) : assert(viewType != null), + assert(hitTestBehavior != null), super(key: key); /// The unique identifier for Android view type to be embedded by this widget. @@ -59,6 +61,11 @@ class AndroidView extends StatefulWidget { /// May be null. final PlatformViewCreatedCallback onPlatformViewCreated; + /// How this widget should behave during hit testing. + /// + /// This defaults to [PlatformViewHitTestBehavior.opaque]. + final PlatformViewHitTestBehavior hitTestBehavior; + @override State createState() => new _AndroidViewState(); } @@ -69,7 +76,10 @@ class _AndroidViewState extends State { @override Widget build(BuildContext context) { - return new _AndroidPlatformView(controller: _controller); + return new _AndroidPlatformView( + controller: _controller, + hitTestBehavior: widget.hitTestBehavior + ); } @override @@ -108,17 +118,21 @@ class _AndroidPlatformView extends LeafRenderObjectWidget { const _AndroidPlatformView({ Key key, @required this.controller, + @required this.hitTestBehavior, }) : assert(controller != null), + assert(hitTestBehavior != null), super(key: key); final AndroidViewController controller; + final PlatformViewHitTestBehavior hitTestBehavior; @override RenderObject createRenderObject(BuildContext context) => - new RenderAndroidView(viewController: controller); + new RenderAndroidView(viewController: controller, hitTestBehavior: hitTestBehavior); @override void updateRenderObject(BuildContext context, RenderAndroidView renderObject) { renderObject.viewController = controller; + renderObject.hitTestBehavior = hitTestBehavior; } } diff --git a/packages/flutter/test/services/fake_platform_views.dart b/packages/flutter/test/services/fake_platform_views.dart index e6c549621d..2bcbea33ef 100644 --- a/packages/flutter/test/services/fake_platform_views.dart +++ b/packages/flutter/test/services/fake_platform_views.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'dart:async'; +import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; @@ -14,7 +15,12 @@ class FakePlatformViewsController { } final TargetPlatform targetPlatform; + + Iterable get views => _views.values; final Map _views = {}; + + final Map> motionEvents = >{}; + final Set _registeredViewTypes = new Set(); int _textureCounter = 0; @@ -37,6 +43,8 @@ class FakePlatformViewsController { return _dispose(call); case 'resize': return _resize(call); + case 'touch': + return _touch(call); } return new Future.sync(() => null); } @@ -95,7 +103,28 @@ class FakePlatformViewsController { return new Future.sync(() => null); } - Iterable get views => _views.values; + Future _touch(MethodCall call) { + final List args = call.arguments; + final int id = args[0]; + final int action = args[3]; + final List> pointerProperties = args[5].cast>(); + final List> pointerCoords = args[6].cast>(); + final List pointerOffsets = []; + final List pointerIds = []; + for (int i = 0; i < pointerCoords.length; i++) { + pointerIds.add(pointerProperties[i][0]); + final double x = pointerCoords[i][7]; + final double y = pointerCoords[i][8]; + pointerOffsets.add(new Offset(x, y)); + } + + if (!motionEvents.containsKey(id)) + motionEvents[id] = []; + + motionEvents[id].add(new FakeMotionEvent(action, pointerIds, pointerOffsets)); + return new Future.sync(() => null); + } + } class FakePlatformView { @@ -124,3 +153,32 @@ class FakePlatformView { return 'FakePlatformView(id: $id, type: $type, size: $size)'; } } + +class FakeMotionEvent { + const FakeMotionEvent(this.action, this.pointerIds, this.pointers); + + final int action; + final List pointers; + final List pointerIds; + + + @override + bool operator ==(dynamic other) { + if (other is! FakeMotionEvent) + return false; + final FakeMotionEvent typedOther = other; + const ListEquality offsetsEq = ListEquality(); + const ListEquality pointersEq = ListEquality(); + return pointersEq.equals(pointerIds, typedOther.pointerIds) && + action == typedOther.action && + offsetsEq.equals(pointers, typedOther.pointers); + } + + @override + int get hashCode => hashValues(action, pointers, pointerIds); + + @override + String toString() { + return 'FakeMotionEvent(action: $action, pointerIds: $pointerIds, pointers: $pointers)'; + } +} diff --git a/packages/flutter/test/widgets/platform_view_test.dart b/packages/flutter/test/widgets/platform_view_test.dart index 09ec488572..6390bba13e 100644 --- a/packages/flutter/test/widgets/platform_view_test.dart +++ b/packages/flutter/test/widgets/platform_view_test.dart @@ -2,9 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'package:flutter/gestures.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; -import 'package:flutter_test/flutter_test.dart'; import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; import '../services/fake_platform_views.dart'; @@ -160,4 +162,191 @@ void main() { ]) ); }); + + testWidgets('Android view gets touch events', (WidgetTester tester) async { + final int currentViewId = platformViewsRegistry.getNextPlatformViewId(); + final FakePlatformViewsController viewsController = new FakePlatformViewsController(TargetPlatform.android); + viewsController.registerViewType('webview'); + await tester.pumpWidget( + const Align( + alignment: Alignment.topLeft, + child: SizedBox( + width: 200.0, + height: 100.0, + child: AndroidView(viewType: 'webview'), + ) + ) + ); + + final TestGesture gesture = await tester.startGesture(const Offset(50.0, 50.0)); + await gesture.up(); + + expect( + viewsController.motionEvents[currentViewId + 1], + orderedEquals( [ + const FakeMotionEvent(AndroidViewController.kActionDown, [0], [Offset(50.0, 50.0)]), + const FakeMotionEvent(AndroidViewController.kActionUp, [0], [Offset(50.0, 50.0)]), + ]) + ); + }); + + testWidgets('Android view transparent hit test behavior', (WidgetTester tester) async { + final int currentViewId = platformViewsRegistry.getNextPlatformViewId(); + final FakePlatformViewsController viewsController = new FakePlatformViewsController(TargetPlatform.android); + viewsController.registerViewType('webview'); + + int numPointerDownsOnParent = 0; + await tester.pumpWidget( + new Directionality( + textDirection: TextDirection.ltr, + child: new Stack( + children: [ + new Listener( + behavior: HitTestBehavior.opaque, + onPointerDown: (PointerDownEvent e) { numPointerDownsOnParent++; }, + ), + const Positioned( + child: SizedBox( + width: 200.0, + height: 100.0, + child: AndroidView( + viewType: 'webview', + hitTestBehavior: PlatformViewHitTestBehavior.transparent, + ), + ) + ), + ], + ), + ), + ); + + await tester.startGesture(const Offset(50.0, 50.0)); + + expect( + viewsController.motionEvents[currentViewId + 1], + isNull, + ); + expect( + numPointerDownsOnParent, + 1 + ); + }); + + testWidgets('Android view translucent hit test behavior', (WidgetTester tester) async { + final int currentViewId = platformViewsRegistry.getNextPlatformViewId(); + final FakePlatformViewsController viewsController = new FakePlatformViewsController(TargetPlatform.android); + viewsController.registerViewType('webview'); + + int numPointerDownsOnParent = 0; + await tester.pumpWidget( + new Directionality( + textDirection: TextDirection.ltr, + child: new Stack( + children: [ + new Listener( + behavior: HitTestBehavior.opaque, + onPointerDown: (PointerDownEvent e) { numPointerDownsOnParent++; }, + ), + const Positioned( + child: SizedBox( + width: 200.0, + height: 100.0, + child: AndroidView( + viewType: 'webview', + hitTestBehavior: PlatformViewHitTestBehavior.translucent, + ), + ) + ), + ], + ), + ), + ); + + await tester.startGesture(const Offset(50.0, 50.0)); + + expect( + viewsController.motionEvents[currentViewId + 1], + orderedEquals( [ + const FakeMotionEvent(AndroidViewController.kActionDown, [0], [Offset(50.0, 50.0)]), + ]) + ); + expect( + numPointerDownsOnParent, + 1 + ); + }); + + testWidgets('Android view opaque hit test behavior', (WidgetTester tester) async { + final int currentViewId = platformViewsRegistry.getNextPlatformViewId(); + final FakePlatformViewsController viewsController = new FakePlatformViewsController(TargetPlatform.android); + viewsController.registerViewType('webview'); + + int numPointerDownsOnParent = 0; + await tester.pumpWidget( + new Directionality( + textDirection: TextDirection.ltr, + child: new Stack( + children: [ + new Listener( + behavior: HitTestBehavior.opaque, + onPointerDown: (PointerDownEvent e) { numPointerDownsOnParent++; }, + ), + const Positioned( + child: SizedBox( + width: 200.0, + height: 100.0, + child: AndroidView( + viewType: 'webview', + hitTestBehavior: PlatformViewHitTestBehavior.opaque, + ), + ) + ), + ], + ), + ), + ); + + await tester.startGesture(const Offset(50.0, 50.0)); + + expect( + viewsController.motionEvents[currentViewId + 1], + orderedEquals( [ + const FakeMotionEvent(AndroidViewController.kActionDown, [0], [Offset(50.0, 50.0)]), + ]) + ); + expect( + numPointerDownsOnParent, + 0 + ); + }); + + testWidgets('Android view touch events are in virtual display\'s coordinate system', (WidgetTester tester) async { + final int currentViewId = platformViewsRegistry.getNextPlatformViewId(); + final FakePlatformViewsController viewsController = new FakePlatformViewsController(TargetPlatform.android); + viewsController.registerViewType('webview'); + await tester.pumpWidget( + new Align( + alignment: Alignment.topLeft, + child: new Container( + margin: const EdgeInsets.all(10.0), + child: const SizedBox( + width: 200.0, + height: 100.0, + child: AndroidView(viewType: 'webview'), + ), + ), + ) + ); + + final TestGesture gesture = await tester.startGesture(const Offset(50.0, 50.0)); + await gesture.up(); + + expect( + viewsController.motionEvents[currentViewId + 1], + orderedEquals( [ + const FakeMotionEvent(AndroidViewController.kActionDown, [0], [Offset(40.0, 40.0)]), + const FakeMotionEvent(AndroidViewController.kActionUp, [0], [Offset(40.0, 40.0)]), + ]) + ); + }); }