firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make gallery tests more robust (#15957)

Browse Source
This commit is contained in:
Hans Muller 2018-03-27 13:39:30 -07:00 committed by GitHub
parent 8ca99327f9
commit a0099a9016
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 202 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
examples/flutter_gallery/lib/demo/material/backdrop_demo.dart
View File

@ -14,6 +14,8 @@ class Category {
const Category({ this.title, this.assets });
final String title;
final List<String> assets;
@override
String toString() => '$runtimeType("$title")';
}
const List<Category> allCategories = const <Category>[
@ -178,7 +180,10 @@ class BackdropPanel extends StatelessWidget {
alignment: AlignmentDirectional.centerStart,
child: new DefaultTextStyle(
style: theme.textTheme.subhead,
child: title,
child: new Tooltip(
message: 'Tap to dismiss',
child: title,
),
),
),
),

2
examples/flutter_gallery/lib/gallery/item.dart
View File

@ -118,7 +118,7 @@ List<GalleryItem> _buildGalleryItems() {
),
new GalleryItem(
title: 'Data tables',
subtitle: 'Data tables',
subtitle: 'Rows and columns',
category: 'Material Components',
routeName: DataTableDemo.routeName,
buildRoute: (BuildContext context) => new DataTableDemo(),

111
examples/flutter_gallery/test/live_smoketest.dart
View File

@ -11,30 +11,55 @@ import 'package:flutter/scheduler.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter_gallery/gallery/app.dart';
import 'package:flutter_gallery/gallery/item.dart';
/// Reports success or failure to the native code.
// Reports success or failure to the native code.
const MethodChannel _kTestChannel = const MethodChannel('io.flutter.demo.gallery/TestLifecycleListener');
// The titles for all of the Gallery demos.
final List<String> _kAllDemos = kAllGalleryItems.map((GalleryItem item) => item.title).toList();
// We don't want to wait for animations to complete before tapping the
// back button in the demos with these titles.
const List<String> _kUnsynchronizedDemos = const <String>[
'Progress indicators',
'Activity Indicator',
'Video',
];
// These demos can't be backed out of by tapping a button whose
// tooltip is 'Back'.
const List<String> _kSkippedDemos = const <String>[
'Backdrop',
'Pull to refresh',
];
Future<Null> main() async {
try {
// Verify that _kUnsynchronizedDemos and _kSkippedDemos identify
// demos that actually exist.
if (!new Set<String>.from(_kAllDemos).containsAll(_kUnsynchronizedDemos))
fail('Unrecognized demo names in _kUnsynchronizedDemos: $_kUnsynchronizedDemos');
if (!new Set<String>.from(_kAllDemos).containsAll(_kSkippedDemos))
fail('Unrecognized demo names in _kSkippedDemos: $_kSkippedDemos');
runApp(const GalleryApp());
const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
final _LiveWidgetController controller = new _LiveWidgetController();
for (Demo demo in demos) {
print('Testing "${demo.title}" demo');
final Finder menuItem = find.text(demo.title);
for (String demo in _kAllDemos) {
print('Testing "$demo" demo');
final Finder menuItem = find.text(demo);
await controller.scrollIntoView(menuItem, alignment: 0.5);
await new Future<Null>.delayed(kWaitBetweenActions);
if (_kSkippedDemos.contains(demo)) {
print('> skipped $demo');
continue;
}
for (int i = 0; i < 2; i += 1) {
await controller.tap(menuItem); // Launch the demo
await new Future<Null>.delayed(kWaitBetweenActions);
controller.frameSync = demo.synchronized;
controller.frameSync = !_kUnsynchronizedDemos.contains(demo);
await controller.tap(find.byTooltip('Back'));
controller.frameSync = true;
await new Future<Null>.delayed(kWaitBetweenActions);
}
print('Success');
}
@ -45,70 +70,6 @@ Future<Null> main() async {
}
}
class Demo {
const Demo(this.title, {this.synchronized = true});
/// The title of the demo.
final String title;
/// True if frameSync should be enabled for this test.
final bool synchronized;
}
// Warning: this list must be kept in sync with the value of
// kAllGalleryItems.map((GalleryItem item) => item.title).toList();
const List<Demo> demos = const <Demo>[
// Demos
const Demo('Shrine'),
const Demo('Contact profile'),
const Demo('Animation'),
// Material Components
const Demo('Bottom navigation'),
const Demo('Buttons'),
const Demo('Cards'),
const Demo('Chips'),
const Demo('Date and time pickers'),
const Demo('Dialog'),
const Demo('Drawer'),
const Demo('Expand/collapse list control'),
const Demo('Expansion panels'),
const Demo('Floating action button'),
const Demo('Grid'),
const Demo('Icons'),
const Demo('Leave-behind list items'),
const Demo('List'),
const Demo('Menus'),
const Demo('Modal bottom sheet'),
const Demo('Page selector'),
const Demo('Persistent bottom sheet'),
const Demo('Progress indicators', synchronized: false),
const Demo('Pull to refresh'),
const Demo('Scrollable tabs'),
const Demo('Selection controls'),
const Demo('Sliders'),
const Demo('Snackbar'),
const Demo('Tabs'),
const Demo('Text fields'),
const Demo('Tooltips'),
// Cupertino Components
const Demo('Activity Indicator', synchronized: false),
const Demo('Buttons'),
const Demo('Dialogs'),
const Demo('Navigation'),
const Demo('Sliders'),
const Demo('Switches'),
// Media
const Demo('Animated images'),
// Style
const Demo('Colors'),
const Demo('Typography'),
];
class _LiveWidgetController {
final WidgetController _controller = new WidgetController(WidgetsBinding.instance);

10
examples/flutter_gallery/test/smoke_test.dart
View File

@ -102,11 +102,19 @@ Future<Null> smokeDemo(WidgetTester tester, String routeName) async {
await tester.pump();
await tester.pump(const Duration(milliseconds: 400));
// This demo's back button isn't initially visible.
if (routeName == '/material/backdrop') {
await tester.tap(find.byTooltip('Tap to dismiss'));
await tester.pumpAndSettle();
}
// Go back
await tester.pageBack();
await tester.pumpAndSettle();
await tester.pump(); // Start the pop "back" operation.
await tester.pump(); // Complete the willPop() Future.
await tester.pump(const Duration(milliseconds: 400)); // Wait until it has finished.
return null;
}
@ -126,8 +134,6 @@ Future<Null> runSmokeTest(WidgetTester tester) async {
final Finder finder = findGalleryItemByRouteName(tester, routeName);
Scrollable.ensureVisible(tester.element(finder), alignment: 0.5);
await tester.pumpAndSettle();
if (routeName == '/material/backdrop')
continue;
await smokeDemo(tester, routeName);
tester.binding.debugAssertNoTransientCallbacks('A transient callback was still active after leaving route $routeName');
}

4
examples/flutter_gallery/test_driver/memory_nav_test.dart
View File

@ -23,18 +23,14 @@ void main() {
final SerializableFinder menuItem = find.text('Text fields');
driver.waitFor(menuItem).then<Null>((Null value) async {
scroll = false;
await new Future<Null>.delayed(kWaitBetweenActions);
for (int i = 0; i < 15; i++) {
await driver.tap(menuItem);
await new Future<Null>.delayed(kWaitBetweenActions);
await driver.tap(find.byTooltip('Back'));
await new Future<Null>.delayed(kWaitBetweenActions);
}
completer.complete();
});
while (scroll) {
await driver.scroll(find.text('Flutter Gallery'), 0.0, -500.0, const Duration(milliseconds: 80));
await new Future<Null>.delayed(kWaitBetweenActions);
}
await completer.future;
}, timeout: const Timeout(const Duration(minutes: 1)));

13
examples/flutter_gallery/test_driver/transitions_perf.dart
View File

@ -2,10 +2,21 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show JsonEncoder;
import 'package:flutter_driver/driver_extension.dart';
import 'package:flutter_gallery/gallery/item.dart';
import 'package:flutter_gallery/main.dart' as app;
Future<String> _handleMessages(String message) async {
assert(message == 'demoNames');
return const JsonEncoder.withIndent(' ').convert(
kAllGalleryItems.map((GalleryItem item) => item.title).toList(),
);
}
void main() {
enableFlutterDriverExtension();
enableFlutterDriverExtension(handler: _handleMessages);
app.main();
}

174
examples/flutter_gallery/test_driver/transitions_perf_test.dart
View File

@ -3,7 +3,7 @@
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show JsonEncoder;
import 'dart:convert' show JsonEncoder, JsonDecoder;
import 'package:file/file.dart';
import 'package:file/local.dart';
@ -11,81 +11,46 @@ import 'package:flutter_driver/flutter_driver.dart';
import 'package:path/path.dart' as path;
import 'package:test/test.dart';
class Demo {
const Demo(this.title, {this.synchronized = true, this.profiled = false});
/// The title of the demo.
final String title;
/// True if frameSync should be enabled for this test.
final bool synchronized;
// True if timeline data should be collected for this test.
//
// Warning: The number of tests executed with timeline collection enabled
// significantly impacts heap size of the running app. When run with
// --trace-startup, as we do in this test, the VM stores trace events in an
// endless buffer instead of a ring buffer.
final bool profiled;
}
// Warning: this list must be kept in sync with the value of
// kAllGalleryItems.map((GalleryItem item) => item.title).toList();
const List<Demo> demos = const <Demo>[
// Demos
const Demo('Shrine', profiled: true),
const Demo('Contact profile', profiled: true),
const Demo('Animation', profiled: true),
// Material Components
const Demo('Bottom navigation', profiled: true),
const Demo('Buttons', profiled: true),
const Demo('Cards', profiled: true),
const Demo('Chips', profiled: true),
const Demo('Date and time pickers', profiled: true),
const Demo('Dialog', profiled: true),
const Demo('Drawer'),
const Demo('Expand/collapse list control'),
const Demo('Expansion panels'),
const Demo('Floating action button'),
const Demo('Grid'),
const Demo('Icons'),
const Demo('Leave-behind list items'),
const Demo('List'),
const Demo('Menus'),
const Demo('Modal bottom sheet'),
const Demo('Page selector'),
const Demo('Persistent bottom sheet'),
const Demo('Progress indicators', synchronized: false),
const Demo('Pull to refresh'),
const Demo('Scrollable tabs'),
const Demo('Selection controls'),
const Demo('Sliders'),
const Demo('Snackbar'),
const Demo('Tabs'),
const Demo('Text fields'),
const Demo('Tooltips'),
// Cupertino Components
const Demo('Activity Indicator', synchronized: false),
const Demo('Buttons'),
const Demo('Dialogs'),
const Demo('Navigation'),
const Demo('Pickers'),
const Demo('Sliders'),
const Demo('Switches'),
// Media
const Demo('Animated images'),
// Style
const Demo('Colors'),
const Demo('Typography'),
];
const FileSystem _fs = const LocalFileSystem();
const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
// Demos for which timeline data will be collected using
// FlutterDriver.traceAction().
//
// Warning: The number of tests executed with timeline collection enabled
// significantly impacts heap size of the running app. When run with
// --trace-startup, as we do in this test, the VM stores trace events in an
// endless buffer instead of a ring buffer.
//
// These names must match GalleryItem titles from kAllGalleryItems
// in examples/flutter_gallery/lib/gallery.item.dart
const List<String> kProfiledDemos = const <String>[
'Shrine',
'Contact profile',
'Animation',
'Bottom navigation',
'Buttons',
'Cards',
'Chips',
'Date and time pickers',
'Dialog',
];
// Demos that will be backed out of within FlutterDriver.runUnsynchronized();
//
// These names must match GalleryItem titles from kAllGalleryItems
// in examples/flutter_gallery/lib/gallery.item.dart
const List<String> kUnsynchronizedDemos = const <String>[
'Progress indicators',
'Activity Indicator',
'Video',
];
// All of the gallery demo titles in the order they appear on the
// gallery home page.
//
// These names are reported by the test app, see _handleMessages()
// in transitions_perf.dart.
List<String> _allDemos = <String>[];
/// Extracts event data from [events] recorded by timeline, validates it, turns
/// it into a histogram, and saves to a JSON file.
@ -155,25 +120,29 @@ Future<Null> saveDurationsHistogram(List<Map<String, dynamic>> events, String ou
/// Scrolls each demo menu item into view, launches it, then returns to the
/// home screen twice.
Future<Null> runDemos(Iterable<Demo> demos, FlutterDriver driver) async {
for (Demo demo in demos) {
print('Testing "${demo.title}" demo');
final SerializableFinder menuItem = find.text(demo.title);
await driver.scrollIntoView(menuItem, alignment: 0.5);
await new Future<Null>.delayed(kWaitBetweenActions);
Future<Null> runDemos(List<String> demos, FlutterDriver driver) async {
for (String demo in demos) {
print('Testing "$demo" demo');
final SerializableFinder menuItem = find.text(demo);
await driver.scrollUntilVisible(find.byType('CustomScrollView'), menuItem,
dyScroll: -48.0,
alignment: 0.5,
);
for (int i = 0; i < 2; i += 1) {
await driver.tap(menuItem); // Launch the demo
await new Future<Null>.delayed(kWaitBetweenActions);
if (demo.synchronized) {
await driver.tap(find.byTooltip('Back'));
} else {
// This demo's back button isn't initially visible.
if (demo == 'Backdrop')
await driver.tap(find.byTooltip('Tap to dismiss'));
if (kUnsynchronizedDemos.contains(demo)) {
await driver.runUnsynchronized<Future<Null>>(() async {
await new Future<Null>.delayed(kWaitBetweenActions);
await driver.tap(find.byTooltip('Back'));
});
} else {
await driver.tap(find.byTooltip('Back'));
}
await new Future<Null>.delayed(kWaitBetweenActions);
}
print('Success');
}
@ -184,10 +153,16 @@ void main([List<String> args = const <String>[]]) {
FlutterDriver driver;
setUpAll(() async {
driver = await FlutterDriver.connect();
if (args.contains('--with_semantics')) {
print('Enabeling semantics...');
await driver.setSemantics(true);
}
// See _handleMessages() in transitions_perf.dart.
_allDemos = const JsonDecoder().convert(await driver.requestData('demoNames'));
if (_allDemos.isEmpty)
throw 'no demo names found';
});
tearDownAll(() async {
@ -197,14 +172,15 @@ void main([List<String> args = const <String>[]]) {
test('all demos', () async {
// Collect timeline data for just a limited set of demos to avoid OOMs.
final Timeline timeline = await driver.traceAction(() async {
final Iterable<Demo> profiledDemos = demos.where((Demo demo) => demo.profiled);
await runDemos(profiledDemos, driver);
},
streams: const <TimelineStream>[
TimelineStream.dart,
TimelineStream.embedder,
]);
final Timeline timeline = await driver.traceAction(
() async {
await runDemos(kProfiledDemos, driver);
},
streams: const <TimelineStream>[
TimelineStream.dart,
TimelineStream.embedder,
],
);
// Save the duration (in microseconds) of the first timeline Frame event
// that follows a 'Start Transition' event. The Gallery app adds a
@ -214,9 +190,15 @@ void main([List<String> args = const <String>[]]) {
final String histogramPath = path.join(testOutputsDirectory, 'transition_durations.timeline.json');
await saveDurationsHistogram(timeline.json['traceEvents'], histogramPath);
// Scroll back to the top
await driver.scrollUntilVisible(find.byType('CustomScrollView'), find.text(_allDemos[0]),
dyScroll: 200.0,
alignment: 0.0
);
// Execute the remaining tests.
final Iterable<Demo> unprofiledDemos = demos.where((Demo demo) => !demo.profiled);
await runDemos(unprofiledDemos, driver);
final Set<String> unprofiledDemos = new Set<String>.from(_allDemos)..removeAll(kProfiledDemos);
await runDemos(unprofiledDemos.toList(), driver);
}, timeout: const Timeout(const Duration(minutes: 5)));
});

61
packages/flutter_driver/lib/src/driver/driver.dart
View File

@ -409,10 +409,71 @@ class FlutterDriver {
/// Scrolls the Scrollable ancestor of the widget located by [finder]
/// until the widget is completely visible.
///
/// If the widget located by [finder] is contained by a scrolling widget
/// that lazily creates its children, like [ListView] or [CustomScrollView],
/// then this method may fail because [finder] doesn't actually exist.
/// The [scrollUntilVisible] method can be used in this case.
Future<Null> scrollIntoView(SerializableFinder finder, { double alignment: 0.0, Duration timeout }) async {
return await _sendCommand(new ScrollIntoView(finder, alignment: alignment, timeout: timeout)).then((Map<String, dynamic> _) => null);
}
/// Repeatedly [scroll] the widget located by [scrollable] by [dxScroll] and
/// [dyScroll] until [item] is visible, and then use [scrollIntoView] to
/// ensure the item's final position matches [alignment].
///
/// The [scrollable] must locate the scrolling widget that contains [item].
/// Typically `find.byType('ListView') or `find.byType('CustomScrollView')`.
///
/// Atleast one of [dxScroll] and [dyScroll] must be non-zero.
///
/// If [item] is below the currently visible items, then specify a negative
/// value for [dyScroll] that's a small enough increment to expose [item]
/// without potentially scrolling it up and completely out of view. Similarly
/// if [item] is above, then specify a positve value for [dyScroll].
///
/// If [item] is to the right of the the currently visible items, then
/// specify a negative value for [dxScroll] that's a small enough increment to
/// expose [item] without potentially scrolling it up and completely out of
/// view. Similarly if [item] is to the left, then specify a positve value
/// for [dyScroll].
///
/// The [timeout] value should be long enough to accommodate as many scrolls
/// as needed to bring an item into view. The default is 10 seconds.
Future<Null> scrollUntilVisible(SerializableFinder scrollable, SerializableFinder item, {
double alignment: 0.0,
double dxScroll: 0.0,
double dyScroll: 0.0,
Duration timeout: const Duration(seconds: 10),
}) async {
assert(scrollable != null);
assert(item != null);
assert(alignment != null);
assert(dxScroll != null);
assert(dyScroll != null);
assert(dxScroll != 0.0 || dyScroll != 0.0);
assert(timeout != null);
// If the item is already visible then we're done.
bool isVisible = false;
try {
await waitFor(item, timeout: const Duration(milliseconds: 100));
isVisible = true;
} on DriverError {
// Assume that that waitFor timed out because the item isn't visible.
}
if (!isVisible) {
waitFor(item, timeout: timeout).then((Null _) { isVisible = true; });
while (!isVisible) {
await scroll(scrollable, dxScroll, dyScroll, const Duration(milliseconds: 100));
await new Future<Null>.delayed(const Duration(milliseconds: 500));
}
}
return scrollIntoView(item, alignment: alignment);
}
/// Returns the text in the `Text` widget located by [finder].
Future<String> getText(SerializableFinder finder, { Duration timeout }) async {
return GetTextResult.fromJson(await _sendCommand(new GetText(finder, timeout: timeout))).text;