Parser machine logs (#118707)
* remove file reporter optional * decouple stack trace from metric collections * update tests; add collect metrics option * add failed tests property * add test for multiple stack failed * change path on result * create factory method * throw exception when test file failed to generate * remove catch of file exception * handle when no stacktrace on file reporter
This commit is contained in:
Jesús S Guerrero
GitHub
parent
d4b6898478
commit
d63987f71d
@ -1802,6 +1802,7 @@ Future<void> _runDartTest(String workingDirectory, {
|
||||
final List<String> args = <String>[
|
||||
'run',
|
||||
'test',
|
||||
'--file-reporter=json:${metricFile.path}',
|
||||
if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
|
||||
'-j$cpus',
|
||||
if (!hasColor)
|
||||
@ -1813,8 +1814,6 @@ Future<void> _runDartTest(String workingDirectory, {
|
||||
if (testPaths != null)
|
||||
for (final String testPath in testPaths)
|
||||
testPath,
|
||||
if (collectMetrics)
|
||||
'--file-reporter=json:${metricFile.path}',
|
||||
];
|
||||
final Map<String, String> environment = <String, String>{
|
||||
'FLUTTER_ROOT': flutterRoot,
|
||||
@ -1840,19 +1839,24 @@ Future<void> _runDartTest(String workingDirectory, {
|
||||
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
|
||||
);
|
||||
|
||||
final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
|
||||
final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
|
||||
info.writeAsStringSync(json.encode(test.errors));
|
||||
|
||||
if (collectMetrics) {
|
||||
try {
|
||||
final List<String> testList = <String>[];
|
||||
final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile);
|
||||
final Map<int, TestSpecs> allTestSpecs = test.allTestSpecs;
|
||||
for (final TestSpecs testSpecs in allTestSpecs.values) {
|
||||
testList.add(testSpecs.toJson());
|
||||
}
|
||||
if (testList.isNotEmpty) {
|
||||
final String testJson = json.encode(testList);
|
||||
final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json'));
|
||||
final File testResults = fileSystem.file(
|
||||
path.join(flutterRoot, 'test_results.json'));
|
||||
testResults.writeAsStringSync(testJson);
|
||||
}
|
||||
} on fs.FileSystemException catch (e){
|
||||
} on fs.FileSystemException catch (e) {
|
||||
print('Failed to generate metrics: $e');
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,8 +23,8 @@ void main() {
|
||||
{"missing": "entry"}
|
||||
{"other": true}''';
|
||||
file.writeAsStringSync(output);
|
||||
final Map<int, TestSpecs> result = generateMetrics(file);
|
||||
expect(result, isEmpty);
|
||||
final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
|
||||
expect(result.allTestSpecs, isEmpty);
|
||||
});
|
||||
|
||||
test('have metrics', () async {
|
||||
@ -47,7 +47,7 @@ void main() {
|
||||
{"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}
|
||||
{"success":true,"type":"done","time":4870}''';
|
||||
file.writeAsStringSync(output);
|
||||
final Map<int, TestSpecs> result = generateMetrics(file);
|
||||
final Map<int, TestSpecs> result = TestFileReporterResults.fromFile(file).allTestSpecs;
|
||||
expect(result, contains(0));
|
||||
expect(result, contains(1));
|
||||
expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart');
|
||||
@ -62,8 +62,37 @@ void main() {
|
||||
{"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
|
||||
{"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}''';
|
||||
file.writeAsStringSync(output);
|
||||
final Map<int, TestSpecs> result = generateMetrics(file);
|
||||
expect(result, isEmpty);
|
||||
final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
|
||||
expect(result.hasFailedTests, true);
|
||||
});
|
||||
|
||||
test('has failed stack traces', () async {
|
||||
final File file = fileSystem.file('success_file');
|
||||
const String output = '''
|
||||
{"protocolVersion":"0.1.1","runnerVersion":"1.22.1","pid":47372,"type":"start","time":0}
|
||||
{"suite":{"id":0,"platform":"vm","path":"test/tool_subsharding_test.dart"},"type":"suite","time":0}
|
||||
{"test":{"id":1,"name":"loading test/tool_subsharding_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":2}
|
||||
{"count":1,"time":11,"type":"allSuites"}
|
||||
{"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1021}
|
||||
{"group":{"id":2,"suiteID":0,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":null,"column":null,"url":null},"type":"group","time":1026}
|
||||
{"group":{"id":3,"suiteID":0,"parentID":2,"name":"generateMetrics","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":13,"column":3,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"group","time":1027}
|
||||
{"test":{"id":4,"name":"generateMetrics empty metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":20,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1027}
|
||||
{"testID":4,"error":"Expected: <true> Actual: <false>","stackTrace":"package:test_api expect test/tool_subsharding_test.dart 28:7 main.<fn>.<fn> ","isFailure":true,"type":"error","time":1095}
|
||||
{"testID":4,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1096}
|
||||
{"test":{"id":5,"name":"generateMetrics have metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":31,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1097}
|
||||
{"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":1103}
|
||||
{"test":{"id":6,"name":"generateMetrics missing success entry","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":60,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1103}
|
||||
{"testID":6,"error":"Expected: <false> Actual: <true>","stackTrace":"package:test_api expect test/tool_subsharding_test.dart 68:7 main.<fn>.<fn> ","isFailure":true,"type":"error","time":1107}
|
||||
{"testID":6,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1107}
|
||||
{"testID":6,"error":"my error","isFailure":true,"type":"error","time":1107}
|
||||
{"success":false,"type":"done","time":1120}''';
|
||||
file.writeAsStringSync(output);
|
||||
final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
|
||||
expect(result.hasFailedTests, true);
|
||||
expect(result.errors.length == 3, true);
|
||||
expect(result.errors[0].contains('Expected: <true> Actual: <false>'), true);
|
||||
expect(result.errors[1].contains('Expected: <false> Actual: <true>'), true);
|
||||
expect(result.errors[2].contains('my error'), true);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@ -38,42 +38,65 @@ class TestSpecs {
|
||||
}
|
||||
}
|
||||
|
||||
/// Intended to parse the output file of `dart test --file-reporter json:file_name
|
||||
Map<int, TestSpecs> generateMetrics(File metrics) {
|
||||
final Map<int, TestSpecs> allTestSpecs = <int, TestSpecs>{};
|
||||
if (!metrics.existsSync()) {
|
||||
return allTestSpecs;
|
||||
}
|
||||
class TestFileReporterResults {
|
||||
TestFileReporterResults._({
|
||||
required this.allTestSpecs,
|
||||
required this.hasFailedTests,
|
||||
required this.errors,
|
||||
});
|
||||
|
||||
bool success = false;
|
||||
for(final String metric in metrics.readAsLinesSync()) {
|
||||
final Map<String, dynamic> entry = json.decode(metric) as Map<String, dynamic>;
|
||||
if (entry.containsKey('suite')) {
|
||||
final Map<dynamic, dynamic> suite = entry['suite'] as Map<dynamic, dynamic>;
|
||||
allTestSpecs[suite['id'] as int] = TestSpecs(
|
||||
path: suite['path'] as String,
|
||||
startTime: entry['time'] as int,
|
||||
);
|
||||
} else if (_isMetricDone(entry, allTestSpecs)) {
|
||||
final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
|
||||
final int suiteID = group['suiteID'] as int;
|
||||
final TestSpecs testSpec = allTestSpecs[suiteID]!;
|
||||
testSpec.endTime = entry['time'] as int;
|
||||
} else if (entry.containsKey('success') && entry['success'] == true) {
|
||||
success = true;
|
||||
/// Intended to parse the output file of `dart test --file-reporter json:file_name
|
||||
factory TestFileReporterResults.fromFile(File metrics) {
|
||||
if (!metrics.existsSync()) {
|
||||
throw Exception('${metrics.path} does not exist');
|
||||
}
|
||||
|
||||
final Map<int, TestSpecs> testSpecs = <int, TestSpecs>{};
|
||||
bool hasFailedTests = true;
|
||||
final List<String> errors = <String>[];
|
||||
|
||||
for (final String metric in metrics.readAsLinesSync()) {
|
||||
final Map<String, Object?> entry = json.decode(metric) as Map<String, Object?>;
|
||||
if (entry.containsKey('suite')) {
|
||||
final Map<String, Object?> suite = entry['suite']! as Map<String, Object?>;
|
||||
addTestSpec(suite, entry['time']! as int, testSpecs);
|
||||
} else if (isMetricDone(entry, testSpecs)) {
|
||||
final Map<String, Object?> group = entry['group']! as Map<String, Object?>;
|
||||
final int suiteID = group['suiteID']! as int;
|
||||
addMetricDone(suiteID, entry['time']! as int, testSpecs);
|
||||
} else if (entry.containsKey('error')) {
|
||||
final String stackTrace = entry.containsKey('stackTrace') ? entry['stackTrace']! as String : '';
|
||||
errors.add('${entry['error']}\n $stackTrace');
|
||||
} else if (entry.containsKey('success') && entry['success'] == true) {
|
||||
hasFailedTests = false;
|
||||
}
|
||||
}
|
||||
|
||||
return TestFileReporterResults._(allTestSpecs: testSpecs, hasFailedTests: hasFailedTests, errors: errors);
|
||||
}
|
||||
|
||||
if (!success) { // means that not all tests succeeded therefore no metrics are stored
|
||||
return <int, TestSpecs>{};
|
||||
}
|
||||
return allTestSpecs;
|
||||
}
|
||||
final Map<int, TestSpecs> allTestSpecs;
|
||||
final bool hasFailedTests;
|
||||
final List<String> errors;
|
||||
|
||||
bool _isMetricDone(Map<String, dynamic> entry, Map<int, TestSpecs> allTestSpecs) {
|
||||
if (entry.containsKey('group') && entry['type'] as String == 'group') {
|
||||
final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
|
||||
return allTestSpecs.containsKey(group['suiteID'] as int);
|
||||
|
||||
static void addTestSpec(Map<String, Object?> suite, int time, Map<int, TestSpecs> allTestSpecs) {
|
||||
allTestSpecs[suite['id']! as int] = TestSpecs(
|
||||
path: suite['path']! as String,
|
||||
startTime: time,
|
||||
);
|
||||
}
|
||||
|
||||
static void addMetricDone(int suiteID, int time, Map<int, TestSpecs> allTestSpecs) {
|
||||
final TestSpecs testSpec = allTestSpecs[suiteID]!;
|
||||
testSpec.endTime = time;
|
||||
}
|
||||
|
||||
static bool isMetricDone(Map<String, Object?> entry, Map<int, TestSpecs> allTestSpecs) {
|
||||
if (entry.containsKey('group') && entry['type']! as String == 'group') {
|
||||
final Map<String, Object?> group = entry['group']! as Map<String, Object?>;
|
||||
return allTestSpecs.containsKey(group['suiteID']! as int);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user