From 7a6af0cf2c7718fcd192c9143db6e5c6b7e60d2c Mon Sep 17 00:00:00 2001 From: David Reveman Date: Wed, 17 Feb 2021 13:09:19 -0500 Subject: [PATCH] Reland: Timer based pointer event resampling (#76195) This reverts #76179 and relands #73042 with an active timer check fix. Co-authored-by: David Reveman --- .../flutter/lib/src/gestures/binding.dart | 124 +++++++++--- ...binding_resample_event_on_widget_test.dart | 68 ++++--- .../gesture_binding_resample_event_test.dart | 186 ++++++++++++++++++ .../test/gestures/gesture_binding_test.dart | 14 -- 4 files changed, 331 insertions(+), 61 deletions(-) create mode 100644 packages/flutter/test/gestures/gesture_binding_resample_event_test.dart diff --git a/packages/flutter/lib/src/gestures/binding.dart b/packages/flutter/lib/src/gestures/binding.dart index 7aa5f08f6d..126cc15ff9 100644 --- a/packages/flutter/lib/src/gestures/binding.dart +++ b/packages/flutter/lib/src/gestures/binding.dart @@ -21,13 +21,24 @@ import 'resampler.dart'; typedef _HandleSampleTimeChangedCallback = void Function(); +/// Class that implements clock used for sampling. +class SamplingClock { + /// Returns current time. + DateTime now() => DateTime.now(); + + /// Returns a new stopwatch that uses the current time as reported by `this`. + Stopwatch stopwatch() => Stopwatch(); +} + // Class that handles resampling of touch events for multiple pointer // devices. // +// The `samplingInterval` is used to determine the approximate next +// time for resampling. // SchedulerBinding's `currentSystemFrameTimeStamp` is used to determine // sample time. class _Resampler { - _Resampler(this._handlePointerEvent, this._handleSampleTimeChanged); + _Resampler(this._handlePointerEvent, this._handleSampleTimeChanged, this._samplingInterval); // Resamplers used to filter incoming pointer events. final Map _resamplers = {}; @@ -35,9 +46,12 @@ class _Resampler { // Flag to track if a frame callback has been scheduled. bool _frameCallbackScheduled = false; - // Current frame time for resampling. + // Last frame time for resampling. Duration _frameTime = Duration.zero; + // Time since `_frameTime` was updated. + Stopwatch _frameTimeAge = Stopwatch(); + // Last sample time and time stamp of last event. // // Only used for debugPrint of resampling margin. @@ -50,12 +64,18 @@ class _Resampler { // Callback used to handle sample time changes. final _HandleSampleTimeChangedCallback _handleSampleTimeChanged; + // Interval used for sampling. + final Duration _samplingInterval; + + // Timer used to schedule resampling. + Timer? _timer; + // Add `event` for resampling or dispatch it directly if // not a touch event. void addOrDispatch(PointerEvent event) { final SchedulerBinding? scheduler = SchedulerBinding.instance; assert(scheduler != null); - // Add touch event to resampler or dispatch pointer event directly. + // Add touch event to resampler or dispatch pointer event directly. if (event.kind == PointerDeviceKind.touch) { // Save last event time for debugPrint of resampling margin. _lastEventTime = event.timeStamp; @@ -72,25 +92,43 @@ class _Resampler { // Sample and dispatch events. // - // `samplingOffset` is relative to the current frame time, which + // The `samplingOffset` is relative to the current frame time, which // can be in the past when we're not actively resampling. - // `samplingInterval` is used to determine the approximate next - // time for resampling. - // `currentSystemFrameTimeStamp` is used to determine the current - // frame time. - void sample(Duration samplingOffset, Duration samplingInterval) { + // The `samplingClock` is the clock used to determine frame time age. + void sample(Duration samplingOffset, SamplingClock clock) { final SchedulerBinding? scheduler = SchedulerBinding.instance; assert(scheduler != null); + // Initialize `_frameTime` if needed. This will be used for periodic + // sampling when frame callbacks are not received. + if (_frameTime == Duration.zero) { + _frameTime = Duration(milliseconds: clock.now().millisecondsSinceEpoch); + _frameTimeAge = clock.stopwatch()..start(); + } + + // Schedule periodic resampling if `_timer` is not already active. + if (_timer?.isActive != true) { + _timer = Timer.periodic(_samplingInterval, (_) => _onSampleTimeChanged()); + } + + // Calculate the effective frame time by taking the number + // of sampling intervals since last time `_frameTime` was + // updated into account. This allows us to advance sample + // time without having to receive frame callbacks. + final int samplingIntervalUs = _samplingInterval.inMicroseconds; + final int elapsedIntervals = _frameTimeAge.elapsedMicroseconds ~/ samplingIntervalUs; + final int elapsedUs = elapsedIntervals * samplingIntervalUs; + final Duration frameTime = _frameTime + Duration(microseconds: elapsedUs); + // Determine sample time by adding the offset to the current // frame time. This is expected to be in the past and not // result in any dispatched events unless we're actively // resampling events. - final Duration sampleTime = _frameTime + samplingOffset; + final Duration sampleTime = frameTime + samplingOffset; // Determine next sample time by adding the sampling interval // to the current sample time. - final Duration nextSampleTime = sampleTime + samplingInterval; + final Duration nextSampleTime = sampleTime + _samplingInterval; // Iterate over active resamplers and sample pointer events for // current sample time. @@ -106,23 +144,30 @@ class _Resampler { // Save last sample time for debugPrint of resampling margin. _lastSampleTime = sampleTime; + // Early out if another call to `sample` isn't needed. + if (_resamplers.isEmpty) { + _timer!.cancel(); + return; + } + // Schedule a frame callback if another call to `sample` is needed. - if (!_frameCallbackScheduled && _resamplers.isNotEmpty) { + if (!_frameCallbackScheduled) { _frameCallbackScheduled = true; - scheduler?.scheduleFrameCallback((_) { + // Add a post frame callback as this avoids producing unnecessary + // frames but ensures that sampling phase is adjusted to frame + // time when frames are produced. + scheduler?.addPostFrameCallback((_) { _frameCallbackScheduled = false; // We use `currentSystemFrameTimeStamp` here as it's critical that // sample time is in the same clock as the event time stamps, and // never adjusted or scaled like `currentFrameTimeStamp`. _frameTime = scheduler.currentSystemFrameTimeStamp; - assert(() { - if (debugPrintResamplingMargin) { - final Duration resamplingMargin = _lastEventTime - _lastSampleTime; - debugPrint('$resamplingMargin'); - } - return true; - }()); - _handleSampleTimeChanged(); + _frameTimeAge.reset(); + // Reset timer to match phase of latest frame callback. + _timer?.cancel(); + _timer = Timer.periodic(_samplingInterval, (_) => _onSampleTimeChanged()); + // Trigger an immediate sample time change. + _onSampleTimeChanged(); }); } } @@ -133,6 +178,18 @@ class _Resampler { resampler.stop(_handlePointerEvent); } _resamplers.clear(); + _frameTime = Duration.zero; + } + + void _onSampleTimeChanged() { + assert(() { + if (debugPrintResamplingMargin) { + final Duration resamplingMargin = _lastEventTime - _lastSampleTime; + debugPrint('$resamplingMargin'); + } + return true; + }()); + _handleSampleTimeChanged(); } } @@ -147,7 +204,8 @@ const Duration _defaultSamplingOffset = Duration(milliseconds: -38); // The sampling interval. // // Sampling interval is used to determine the approximate time for subsequent -// sampling. This is used to decide if early processing of up and removed events +// sampling. This is used to sample events when frame callbacks are not +// being received and decide if early processing of up and removed events // is appropriate. 16667 us for 60hz sampling interval. const Duration _samplingInterval = Duration(microseconds: 16667); @@ -270,7 +328,7 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H if (resamplingEnabled) { _resampler.addOrDispatch(event); - _resampler.sample(samplingOffset, _samplingInterval); + _resampler.sample(samplingOffset, _samplingClock); return; } @@ -398,10 +456,16 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H _hitTests.clear(); } + /// Overrides the sampling clock for debugging and testing. + /// + /// This value is ignored in non-debug builds. + @protected + SamplingClock? get debugSamplingClock => null; + void _handleSampleTimeChanged() { if (!locked) { if (resamplingEnabled) { - _resampler.sample(samplingOffset, _samplingInterval); + _resampler.sample(samplingOffset, _samplingClock); } else { _resampler.stop(); @@ -409,11 +473,23 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H } } + SamplingClock get _samplingClock { + SamplingClock value = SamplingClock(); + assert(() { + final SamplingClock? debugValue = debugSamplingClock; + if (debugValue != null) + value = debugValue; + return true; + }()); + return value; + } + // Resampler used to filter incoming pointer events when resampling // is enabled. late final _Resampler _resampler = _Resampler( _handlePointerEventImmediately, _handleSampleTimeChanged, + _samplingInterval, ); /// Enable pointer event resampling for touch devices by setting diff --git a/packages/flutter/test/gestures/gesture_binding_resample_event_on_widget_test.dart b/packages/flutter/test/gestures/gesture_binding_resample_event_on_widget_test.dart index bea0c48db2..3c3e397706 100644 --- a/packages/flutter/test/gestures/gesture_binding_resample_event_on_widget_test.dart +++ b/packages/flutter/test/gestures/gesture_binding_resample_event_on_widget_test.dart @@ -10,15 +10,35 @@ import 'dart:ui' as ui; +import 'package:clock/clock.dart'; import 'package:flutter/gestures.dart'; +import 'package:flutter/scheduler.dart'; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; +class TestResampleEventFlutterBinding extends AutomatedTestWidgetsFlutterBinding { + @override + SamplingClock? get debugSamplingClock => TestSamplingClock(this.clock); +} + +class TestSamplingClock implements SamplingClock { + TestSamplingClock(this._clock); + + @override + DateTime now() => _clock.now(); + + @override + Stopwatch stopwatch() => _clock.stopwatch(); + + final Clock _clock; +} + void main() { - final TestWidgetsFlutterBinding binding = AutomatedTestWidgetsFlutterBinding(); + final TestWidgetsFlutterBinding binding = TestResampleEventFlutterBinding(); testWidgets('PointerEvent resampling on a widget', (WidgetTester tester) async { assert(WidgetsBinding.instance == binding); Duration currentTestFrameTime() => Duration(milliseconds: binding.clock.now().millisecondsSinceEpoch); + void requestFrame() => SchedulerBinding.instance!.scheduleFrameCallback((_) {}); final Duration epoch = currentTestFrameTime(); final ui.PointerDataPacket packet = ui.PointerDataPacket( data: [ @@ -30,37 +50,37 @@ void main() { ui.PointerData( change: ui.PointerChange.down, physicalX: 0.0, + timeStamp: epoch + const Duration(milliseconds: 0), + ), + ui.PointerData( + change: ui.PointerChange.move, + physicalX: 15.0, timeStamp: epoch + const Duration(milliseconds: 10), ), ui.PointerData( change: ui.PointerChange.move, - physicalX: 10.0, + physicalX: 30.0, timeStamp: epoch + const Duration(milliseconds: 20), ), ui.PointerData( change: ui.PointerChange.move, - physicalX: 20.0, + physicalX: 45.0, timeStamp: epoch + const Duration(milliseconds: 30), ), ui.PointerData( change: ui.PointerChange.move, - physicalX: 30.0, + physicalX: 50.0, timeStamp: epoch + const Duration(milliseconds: 40), ), - ui.PointerData( - change: ui.PointerChange.move, - physicalX: 40.0, - timeStamp: epoch + const Duration(milliseconds: 50), - ), ui.PointerData( change: ui.PointerChange.up, - physicalX: 40.0, - timeStamp: epoch + const Duration(milliseconds: 60), + physicalX: 60.0, + timeStamp: epoch + const Duration(milliseconds: 40), ), ui.PointerData( change: ui.PointerChange.remove, - physicalX: 40.0, - timeStamp: epoch + const Duration(milliseconds: 70), + physicalX: 60.0, + timeStamp: epoch + const Duration(milliseconds: 40), ), ], ); @@ -84,29 +104,31 @@ void main() { ui.window.onPointerDataPacket!(packet); expect(events.length, 0); - await tester.pump(const Duration(milliseconds: 20)); + requestFrame(); + await tester.pump(const Duration(milliseconds: 10)); expect(events.length, 1); expect(events[0], isA()); expect(events[0].timeStamp, currentTestFrameTime() + kSamplingOffset); - expect(events[0].position, Offset(5.0 / ui.window.devicePixelRatio, 0.0)); + expect(events[0].position, Offset(7.5 / ui.window.devicePixelRatio, 0.0)); - // Now the system time is epoch + 40ms - await tester.pump(const Duration(milliseconds: 20)); + // Now the system time is epoch + 20ms + requestFrame(); + await tester.pump(const Duration(milliseconds: 10)); expect(events.length, 2); expect(events[1].timeStamp, currentTestFrameTime() + kSamplingOffset); expect(events[1], isA()); - expect(events[1].position, Offset(25.0 / ui.window.devicePixelRatio, 0.0)); - expect(events[1].delta, Offset(20.0 / ui.window.devicePixelRatio, 0.0)); + expect(events[1].position, Offset(22.5 / ui.window.devicePixelRatio, 0.0)); + expect(events[1].delta, Offset(15.0 / ui.window.devicePixelRatio, 0.0)); - // Now the system time is epoch + 60ms - await tester.pump(const Duration(milliseconds: 20)); + // Now the system time is epoch + 30ms + requestFrame(); + await tester.pump(const Duration(milliseconds: 10)); expect(events.length, 4); expect(events[2].timeStamp, currentTestFrameTime() + kSamplingOffset); expect(events[2], isA()); - expect(events[2].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0)); + expect(events[2].position, Offset(37.5 / ui.window.devicePixelRatio, 0.0)); expect(events[2].delta, Offset(15.0 / ui.window.devicePixelRatio, 0.0)); expect(events[3].timeStamp, currentTestFrameTime() + kSamplingOffset); expect(events[3], isA()); - expect(events[3].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0)); }); } diff --git a/packages/flutter/test/gestures/gesture_binding_resample_event_test.dart b/packages/flutter/test/gestures/gesture_binding_resample_event_test.dart new file mode 100644 index 0000000000..492af5fca1 --- /dev/null +++ b/packages/flutter/test/gestures/gesture_binding_resample_event_test.dart @@ -0,0 +1,186 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ui' as ui; + +import 'package:clock/clock.dart'; +import 'package:fake_async/fake_async.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/gestures.dart'; +import 'package:flutter/scheduler.dart'; + +import '../flutter_test_alternative.dart'; + +typedef HandleEventCallback = void Function(PointerEvent event); + +class TestResampleEventFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding { + HandleEventCallback? callback; + FrameCallback? postFrameCallback; + Duration? frameTime; + + @override + void handleEvent(PointerEvent event, HitTestEntry entry) { + super.handleEvent(event, entry); + if (callback != null) + callback?.call(event); + } + + @override + Duration get currentSystemFrameTimeStamp { + assert(frameTime != null); + return frameTime!; + } + + @override + int addPostFrameCallback(FrameCallback callback) { + postFrameCallback = callback; + return 0; + } + + @override + SamplingClock? get debugSamplingClock => TestSamplingClock(); +} + +class TestSamplingClock implements SamplingClock { + @override + DateTime now() => clock.now(); + + @override + Stopwatch stopwatch() => clock.stopwatch(); +} + +typedef ResampleEventTest = void Function(FakeAsync async); + +void testResampleEvent(String description, ResampleEventTest callback) { + test(description, () { + fakeAsync((FakeAsync async) { + callback(async); + }, initialTime: DateTime.utc(2015, 1, 1)); + }, skip: isBrowser); // Fake clock is not working with the web platform. +} + +void main() { + final TestResampleEventFlutterBinding binding = TestResampleEventFlutterBinding(); + testResampleEvent('Pointer event resampling', (FakeAsync async) { + Duration currentTime() => Duration(milliseconds: clock.now().millisecondsSinceEpoch); + final Duration epoch = currentTime(); + final ui.PointerDataPacket packet = ui.PointerDataPacket( + data: [ + ui.PointerData( + change: ui.PointerChange.add, + physicalX: 0.0, + timeStamp: epoch + const Duration(milliseconds: 0), + ), + ui.PointerData( + change: ui.PointerChange.down, + physicalX: 0.0, + timeStamp: epoch + const Duration(milliseconds: 10), + ), + ui.PointerData( + change: ui.PointerChange.move, + physicalX: 10.0, + timeStamp: epoch + const Duration(milliseconds: 20), + ), + ui.PointerData( + change: ui.PointerChange.move, + physicalX: 20.0, + timeStamp: epoch + const Duration(milliseconds: 30), + ), + ui.PointerData( + change: ui.PointerChange.move, + physicalX: 30.0, + timeStamp: epoch + const Duration(milliseconds: 40), + ), + ui.PointerData( + change: ui.PointerChange.move, + physicalX: 40.0, + timeStamp: epoch + const Duration(milliseconds: 50), + ), + ui.PointerData( + change: ui.PointerChange.move, + physicalX: 50.0, + timeStamp: epoch + const Duration(milliseconds: 60), + ), + ui.PointerData( + change: ui.PointerChange.up, + physicalX: 50.0, + timeStamp: epoch + const Duration(milliseconds: 70), + ), + ui.PointerData( + change: ui.PointerChange.remove, + physicalX: 50.0, + timeStamp: epoch + const Duration(milliseconds: 70), + ), + ], + ); + + const Duration samplingOffset = Duration(milliseconds: -5); + const Duration frameInterval = Duration(microseconds: 16667); + + GestureBinding.instance!.resamplingEnabled = true; + GestureBinding.instance!.samplingOffset = samplingOffset; + + final List events = []; + binding.callback = events.add; + + ui.window.onPointerDataPacket?.call(packet); + + // No pointer events should have been dispatched yet. + expect(events.length, 0); + + // Frame callback should have been requested. + FrameCallback? callback = binding.postFrameCallback; + binding.postFrameCallback = null; + expect(callback, isNotNull); + + binding.frameTime = epoch + const Duration(milliseconds: 15); + callback!(Duration.zero); + + // One pointer event should have been dispatched. + expect(events.length, 1); + expect(events[0], isA()); + expect(events[0].timeStamp, binding.frameTime! + samplingOffset); + expect(events[0].position, Offset(0.0 / ui.window.devicePixelRatio, 0.0)); + + // Second frame callback should have been requested. + callback = binding.postFrameCallback; + binding.postFrameCallback = null; + expect(callback, isNotNull); + + final Duration frameTime = epoch + const Duration(milliseconds: 25); + binding.frameTime = frameTime; + callback!(Duration.zero); + + // Second pointer event should have been dispatched. + expect(events.length, 2); + expect(events[1], isA()); + expect(events[1].timeStamp, binding.frameTime! + samplingOffset); + expect(events[1].position, Offset(10.0 / ui.window.devicePixelRatio, 0.0)); + expect(events[1].delta, Offset(10.0 / ui.window.devicePixelRatio, 0.0)); + + // Verify that resampling continues without a frame callback. + async.elapse(frameInterval * 1.5); + + // Third pointer event should have been dispatched. + expect(events.length, 3); + expect(events[2], isA()); + expect(events[2].timeStamp, frameTime + frameInterval + samplingOffset); + + async.elapse(frameInterval); + + // Remaining pointer events should have been dispatched. + expect(events.length, 5); + expect(events[3], isA()); + expect(events[3].timeStamp, frameTime + frameInterval * 2 + samplingOffset); + expect(events[4], isA()); + expect(events[4].timeStamp, frameTime + frameInterval * 2 + samplingOffset); + + async.elapse(frameInterval); + + // No more pointer events should have been dispatched. + expect(events.length, 5); + + GestureBinding.instance!.resamplingEnabled = false; + }); +} diff --git a/packages/flutter/test/gestures/gesture_binding_test.dart b/packages/flutter/test/gestures/gesture_binding_test.dart index b90bcb2b86..9a446f12f2 100644 --- a/packages/flutter/test/gestures/gesture_binding_test.dart +++ b/packages/flutter/test/gestures/gesture_binding_test.dart @@ -14,8 +14,6 @@ typedef HandleEventCallback = void Function(PointerEvent event); class TestGestureFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding { HandleEventCallback? callback; - FrameCallback? frameCallback; - Duration? frameTime; @override void handleEvent(PointerEvent event, HitTestEntry entry) { @@ -23,18 +21,6 @@ class TestGestureFlutterBinding extends BindingBase with GestureBinding, Schedul if (callback != null) callback?.call(event); } - - @override - Duration get currentSystemFrameTimeStamp { - assert(frameTime != null); - return frameTime!; - } - - @override - int scheduleFrameCallback(FrameCallback callback, {bool rescheduling = false}) { - frameCallback = callback; - return 0; - } } TestGestureFlutterBinding? _binding;