firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reland "Add the refresh rate fields to perf_test #99710" (#99854)

Browse Source
This commit is contained in:
Chris Yang 2022-03-21 15:55:13 -07:00 committed by GitHub
parent c9312c784d
commit e6bd208196
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 200 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
dev/devicelab/lib/tasks/perf_tests.dart
View File

@ -791,6 +791,11 @@ class DevtoolsStartupTest {
}
}
/// A callback function to be used to mock the flutter drive command in PerfTests.
///
/// The `options` contains all the arguments in the `flutter drive` command in PerfTests.
typedef FlutterDriveCallback = void Function(List<String> options);
/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
@ -806,6 +811,8 @@ class PerfTest {
this.benchmarkScoreKeys,
this.dartDefine = '',
String? resultFilename,
this.device,
this.flutterDriveCallback,
}): _resultFilename = resultFilename;
const PerfTest.e2e(
@ -818,6 +825,8 @@ class PerfTest {
this.benchmarkScoreKeys = _kCommonScoreKeys,
this.dartDefine = '',
String resultFilename = 'e2e_perf_summary',
this.device,
this.flutterDriveCallback,
}) : saveTraceFile = false, timelineFileName = null, _resultFilename = resultFilename;
/// The directory where the app under test is defined.
@ -839,6 +848,15 @@ class PerfTest {
final bool needsFullTimeline;
/// Whether to save the trace timeline file `*.timeline.json`.
final bool saveTraceFile;
/// The device to test on.
///
/// If null, the device is selected depending on the current environment.
final Device? device;
/// The function called instead of the actually `flutter drive`.
///
/// If it is not `null`, `flutter drive` will not happen in the PerfTests.
final FlutterDriveCallback? flutterDriveCallback;
/// The keys of the values that need to be reported.
///
@ -876,13 +894,18 @@ class PerfTest {
String? writeSkslFileName,
}) {
return inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
late Device selectedDevice;
if (device != null) {
selectedDevice = device!;
} else {
selectedDevice = await devices.workingDevice;
}
await selectedDevice.unlock();
final String deviceId = selectedDevice.deviceId;
final String? localEngine = localEngineFromEnv;
final String? localEngineSrcPath = localEngineSrcPathFromEnv;
await flutter('drive', options: <String>[
final List<String> options = <String>[
if (localEngine != null)
...<String>['--local-engine', localEngine],
if (localEngineSrcPath != null)
@ -906,7 +929,12 @@ class PerfTest {
...<String>['--dart-define', dartDefine],
'-d',
deviceId,
]);
];
if (flutterDriveCallback != null) {
flutterDriveCallback!(options);
} else {
await flutter('drive', options:options);
}
final Map<String, dynamic> data = json.decode(
file('${_testOutputDirectory(testDirectory)}/$resultFilename.json').readAsStringSync(),
) as Map<String, dynamic>;
@ -943,6 +971,12 @@ class PerfTest {
if (data['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
if (data['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
],
if (data['30hz_frame_percentage'] != null) '30hz_frame_percentage',
if (data['60hz_frame_percentage'] != null) '60hz_frame_percentage',
if (data['80hz_frame_percentage'] != null) '80hz_frame_percentage',
if (data['90hz_frame_percentage'] != null) '90hz_frame_percentage',
if (data['120hz_frame_percentage'] != null) '120hz_frame_percentage',
if (data['illegal_refresh_rate_frame_count'] != null) 'illegal_refresh_rate_frame_count',
],
);
});

124
dev/devicelab/test/perf_tests_test.dart Normal file
View File

@ -0,0 +1,124 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert' show json;
import 'dart:io';
import 'package:flutter_devicelab/framework/devices.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'common.dart';
void main() {
late Directory testDirectory;
late File testTarget;
late Device device;
setUp(() async {
testDirectory = Directory.systemTemp.createTempSync('test_dir');
testTarget = File('${testDirectory.absolute.path}/test_file')..createSync();
device = const FakeDevice(deviceId: 'fakeDeviceId');
deviceOperatingSystem = DeviceOperatingSystem.fake;
});
// This tests when keys like `30hz_frame_percentage`, `60hz_frame_percentage` are not in the generated file.
test('runs perf tests, no crash if refresh rate percentage keys are not in the data', () async {
final Map<String, dynamic> fakeData = <String, dynamic>{
'frame_count': 5,
'average_frame_build_time_millis': 0.1,
'worst_frame_build_time_millis': 0.1,
'90th_percentile_frame_build_time_millis': 0.1,
'99th_percentile_frame_build_time_millis': 0.1,
'average_frame_rasterizer_time_millis': 0.1,
'worst_frame_rasterizer_time_millis': 0.1,
'90th_percentile_frame_rasterizer_time_millis': 0.1,
'99th_percentile_frame_rasterizer_time_millis': 0.1,
'average_layer_cache_count': 1,
'90th_percentile_layer_cache_count': 1,
'99th_percentile_layer_cache_count': 1,
'worst_layer_cache_count': 1,
'average_layer_cache_memory': 1,
'90th_percentile_layer_cache_memory': 1,
'99th_percentile_layer_cache_memory': 1,
'worst_layer_cache_memory': 1,
'average_picture_cache_count': 1,
'90th_percentile_picture_cache_count': 1,
'99th_percentile_picture_cache_count': 1,
'worst_picture_cache_count': 1,
'average_picture_cache_memory': 1,
'90th_percentile_picture_cache_memory': 1,
'99th_percentile_picture_cache_memory': 1,
'worst_picture_cache_memory': 1,
'new_gen_gc_count': 1,
'old_gen_gc_count': 1,
'average_vsync_transitions_missed': 1,
'90th_percentile_vsync_transitions_missed': 1,
'99th_percentile_vsync_transitions_missed': 1,
};
const String resultFileName = 'fake_result';
void driveCallback(List<String> arguments) {
final File resultFile = File('${testDirectory.absolute.path}/build/$resultFileName.json')..createSync(recursive: true);
resultFile.writeAsStringSync(json.encode(fakeData));
}
final PerfTest perfTest = PerfTest(testDirectory.absolute.path, testTarget.absolute.path, 'test_file', resultFilename: resultFileName, device: device, flutterDriveCallback: driveCallback);
final TaskResult result = await perfTest.run();
expect(result.data!['frame_count'], 5);
});
test('runs perf tests, successfully parse refresh rate percentage key-values from data`', () async {
final Map<String, dynamic> fakeData = <String, dynamic>{
'frame_count': 5,
'average_frame_build_time_millis': 0.1,
'worst_frame_build_time_millis': 0.1,
'90th_percentile_frame_build_time_millis': 0.1,
'99th_percentile_frame_build_time_millis': 0.1,
'average_frame_rasterizer_time_millis': 0.1,
'worst_frame_rasterizer_time_millis': 0.1,
'90th_percentile_frame_rasterizer_time_millis': 0.1,
'99th_percentile_frame_rasterizer_time_millis': 0.1,
'average_layer_cache_count': 1,
'90th_percentile_layer_cache_count': 1,
'99th_percentile_layer_cache_count': 1,
'worst_layer_cache_count': 1,
'average_layer_cache_memory': 1,
'90th_percentile_layer_cache_memory': 1,
'99th_percentile_layer_cache_memory': 1,
'worst_layer_cache_memory': 1,
'average_picture_cache_count': 1,
'90th_percentile_picture_cache_count': 1,
'99th_percentile_picture_cache_count': 1,
'worst_picture_cache_count': 1,
'average_picture_cache_memory': 1,
'90th_percentile_picture_cache_memory': 1,
'99th_percentile_picture_cache_memory': 1,
'worst_picture_cache_memory': 1,
'new_gen_gc_count': 1,
'old_gen_gc_count': 1,
'average_vsync_transitions_missed': 1,
'90th_percentile_vsync_transitions_missed': 1,
'99th_percentile_vsync_transitions_missed': 1,
'30hz_frame_percentage': 0.1,
'60hz_frame_percentage': 0.2,
'80hz_frame_percentage': 0.3,
'90hz_frame_percentage': 0.4,
'120hz_frame_percentage': 0.6,
'illegal_refresh_rate_frame_count': 10,
};
const String resultFileName = 'fake_result';
void driveCallback(List<String> arguments) {
final File resultFile = File('${testDirectory.absolute.path}/build/$resultFileName.json')..createSync(recursive: true);
resultFile.writeAsStringSync(json.encode(fakeData));
}
final PerfTest perfTest = PerfTest(testDirectory.absolute.path, testTarget.absolute.path, 'test_file', resultFilename: resultFileName, device: device, flutterDriveCallback: driveCallback);
final TaskResult result = await perfTest.run();
expect(result.data!['30hz_frame_percentage'], 0.1);
expect(result.data!['60hz_frame_percentage'], 0.2);
expect(result.data!['80hz_frame_percentage'], 0.3);
expect(result.data!['90hz_frame_percentage'], 0.4);
expect(result.data!['120hz_frame_percentage'], 0.6);
expect(result.data!['illegal_refresh_rate_frame_count'], 10);
});
}

35
packages/flutter_driver/lib/src/driver/refresh_rate_summarizer.dart
View File

@ -28,6 +28,10 @@ class RefreshRateSummary {
_numberOf60HzFrames++;
continue;
}
if ((refreshRate - 80).abs() < _kErrorMargin) {
_numberOf80HzFrames++;
continue;
}
if ((refreshRate - 90).abs() < _kErrorMargin) {
_numberOf90HzFrames++;
continue;
@ -41,25 +45,17 @@ class RefreshRateSummary {
assert(_numberOfTotalFrames ==
_numberOf30HzFrames +
_numberOf60HzFrames +
_numberOf80HzFrames +
_numberOf90HzFrames +
_numberOf120HzFrames +
_framesWithIllegalRefreshRate.length);
}
// The error margin to determine the frame refresh rate.
// For example, when we calculated a frame that has a refresh rate of 65, we consider the frame to be a 60Hz frame.
// Can be adjusted if necessary.
static const double _kErrorMargin = 6.0;
/// Number of frames with 30hz refresh rate
int get numberOf30HzFrames => _numberOf30HzFrames;
/// Number of frames with 60hz refresh rate
int get numberOf60HzFrames => _numberOf60HzFrames;
/// Number of frames with 90hz refresh rate
int get numberOf90HzFrames => _numberOf90HzFrames;
/// Number of frames with 120hz refresh rate
int get numberOf120HzFrames => _numberOf120HzFrames;
/// The percentage of 30hz frames.
///
/// For example, if this value is 20, it means there are 20 percent of total
@ -76,6 +72,14 @@ class RefreshRateSummary {
? _numberOf60HzFrames / _numberOfTotalFrames * 100
: 0;
/// The percentage of 80hz frames.
///
/// For example, if this value is 20, it means there are 20 percent of total
/// frames are 80hz. 0 means no frames are 80hz, 100 means all frames are 80hz.
double get percentageOf80HzFrames => _numberOfTotalFrames > 0
? _numberOf80HzFrames / _numberOfTotalFrames * 100
: 0;
/// The percentage of 90hz frames.
///
/// For example, if this value is 20, it means there are 20 percent of total
@ -84,7 +88,7 @@ class RefreshRateSummary {
? _numberOf90HzFrames / _numberOfTotalFrames * 100
: 0;
/// The percentage of 90hz frames.
/// The percentage of 120hz frames.
///
/// For example, if this value is 20, it means there are 20 percent of total
/// frames are 120hz. 0 means no frames are 120hz, 100 means all frames are 120hz.
@ -94,13 +98,14 @@ class RefreshRateSummary {
/// A list of all the frames with Illegal refresh rate.
///
/// A refresh rate is consider illegal if it does not belong to anyone below:
/// 30hz, 60hz, 90hz or 120hz.
/// A refresh rate is consider illegal if it does not belong to anyone of the refresh rate this class is
/// explicitly tracking.
List<double> get framesWithIllegalRefreshRate =>
_framesWithIllegalRefreshRate;
int _numberOf30HzFrames = 0;
int _numberOf60HzFrames = 0;
int _numberOf80HzFrames = 0;
int _numberOf90HzFrames = 0;
int _numberOf120HzFrames = 0;
int _numberOfTotalFrames = 0;

1
packages/flutter_driver/lib/src/driver/timeline_summary.dart
View File

@ -275,6 +275,7 @@ class TimelineSummary {
'total_ui_gc_time': gcSummarizer.totalGCTimeMillis,
'30hz_frame_percentage': refreshRateSummary.percentageOf30HzFrames,
'60hz_frame_percentage': refreshRateSummary.percentageOf60HzFrames,
'80hz_frame_percentage': refreshRateSummary.percentageOf80HzFrames,
'90hz_frame_percentage': refreshRateSummary.percentageOf90HzFrames,
'120hz_frame_percentage': refreshRateSummary.percentageOf120HzFrames,
'illegal_refresh_rate_frame_count': refreshRateSummary.framesWithIllegalRefreshRate.length,

20
packages/flutter_driver/test/src/real_tests/timeline_summary_test.dart
View File

@ -475,6 +475,7 @@ void main() {
'total_ui_gc_time': 0.4,
'30hz_frame_percentage': 0,
'60hz_frame_percentage': 0,
'80hz_frame_percentage': 0,
'90hz_frame_percentage': 0,
'120hz_frame_percentage': 0,
'illegal_refresh_rate_frame_count': 0,
@ -595,6 +596,7 @@ void main() {
'total_ui_gc_time': 0.4,
'30hz_frame_percentage': 0,
'60hz_frame_percentage': 100,
'80hz_frame_percentage': 0,
'90hz_frame_percentage': 0,
'120hz_frame_percentage': 0,
'illegal_refresh_rate_frame_count': 0,
@ -869,9 +871,11 @@ void main() {
final List<Map<String, dynamic>> events = <Map<String, dynamic>>[];
const int num30Hz = 10;
const int num60Hz = 20;
const int num80Hz = 20;
const int num90Hz = 20;
const int num120Hz = 40;
const int numIllegal = 10;
const int totalFrames = num30Hz + num60Hz + num80Hz + num90Hz + num120Hz + numIllegal;
// Add 30hz frames
events.addAll(_populateEvents(numberOfEvents: num30Hz,
@ -887,6 +891,12 @@ void main() {
margin: 0,
));
// Add 80hz frames
events.addAll(_populateEvents(numberOfEvents: num80Hz,
startTime: 0,
interval: 12000000,
margin: 0,
));
// Add 90hz frames
events.addAll(_populateEvents(numberOfEvents: num90Hz,
@ -910,10 +920,12 @@ void main() {
));
final RefreshRateSummary summary = _summarize(events);
expect(summary.percentageOf30HzFrames, closeTo(num30Hz, kCompareDelta));
expect(summary.percentageOf60HzFrames, closeTo(num60Hz, kCompareDelta));
expect(summary.percentageOf90HzFrames, closeTo(num90Hz, kCompareDelta));
expect(summary.percentageOf120HzFrames, closeTo(num120Hz, kCompareDelta));
expect(summary.percentageOf30HzFrames, closeTo(num30Hz/totalFrames*100, kCompareDelta));
expect(summary.percentageOf60HzFrames, closeTo(num60Hz/totalFrames*100, kCompareDelta));
expect(summary.percentageOf80HzFrames, closeTo(num80Hz/totalFrames*100, kCompareDelta));
expect(summary.percentageOf90HzFrames, closeTo(num90Hz/totalFrames*100, kCompareDelta));
expect(summary.percentageOf120HzFrames, closeTo(num120Hz/totalFrames*100, kCompareDelta));
expect(summary.framesWithIllegalRefreshRate, isNotEmpty);
expect(summary.framesWithIllegalRefreshRate.length, 10);
});