diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 17cf7e2a80..388023da8e 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -1802,6 +1802,7 @@ Future _runDartTest(String workingDirectory, { final List args = [ 'run', 'test', + '--file-reporter=json:${metricFile.path}', if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed', '-j$cpus', if (!hasColor) @@ -1813,8 +1814,6 @@ Future _runDartTest(String workingDirectory, { if (testPaths != null) for (final String testPath in testPaths) testPath, - if (collectMetrics) - '--file-reporter=json:${metricFile.path}', ]; final Map environment = { 'FLUTTER_ROOT': flutterRoot, @@ -1840,19 +1839,24 @@ Future _runDartTest(String workingDirectory, { removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null, ); + final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name + final File info = fileSystem.file(path.join(flutterRoot, 'error.log')); + info.writeAsStringSync(json.encode(test.errors)); + if (collectMetrics) { try { final List testList = []; - final Map allTestSpecs = generateMetrics(metricFile); + final Map allTestSpecs = test.allTestSpecs; for (final TestSpecs testSpecs in allTestSpecs.values) { testList.add(testSpecs.toJson()); } if (testList.isNotEmpty) { final String testJson = json.encode(testList); - final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json')); + final File testResults = fileSystem.file( + path.join(flutterRoot, 'test_results.json')); testResults.writeAsStringSync(testJson); } - } on fs.FileSystemException catch (e){ + } on fs.FileSystemException catch (e) { print('Failed to generate metrics: $e'); } } diff --git a/dev/bots/test/tool_subsharding_test.dart b/dev/bots/test/tool_subsharding_test.dart index 19e9f8e4ff..1e409b0ac9 100644 --- a/dev/bots/test/tool_subsharding_test.dart +++ b/dev/bots/test/tool_subsharding_test.dart @@ -23,8 +23,8 @@ void main() { {"missing": "entry"} {"other": true}'''; file.writeAsStringSync(output); - final Map result = generateMetrics(file); - expect(result, isEmpty); + final TestFileReporterResults result = TestFileReporterResults.fromFile(file); + expect(result.allTestSpecs, isEmpty); }); test('have metrics', () async { @@ -47,7 +47,7 @@ void main() { {"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000} {"success":true,"type":"done","time":4870}'''; file.writeAsStringSync(output); - final Map result = generateMetrics(file); + final Map result = TestFileReporterResults.fromFile(file).allTestSpecs; expect(result, contains(0)); expect(result, contains(1)); expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart'); @@ -62,8 +62,37 @@ void main() { {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000} {"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}'''; file.writeAsStringSync(output); - final Map result = generateMetrics(file); - expect(result, isEmpty); + final TestFileReporterResults result = TestFileReporterResults.fromFile(file); + expect(result.hasFailedTests, true); + }); + + test('has failed stack traces', () async { + final File file = fileSystem.file('success_file'); + const String output = ''' + {"protocolVersion":"0.1.1","runnerVersion":"1.22.1","pid":47372,"type":"start","time":0} + {"suite":{"id":0,"platform":"vm","path":"test/tool_subsharding_test.dart"},"type":"suite","time":0} + {"test":{"id":1,"name":"loading test/tool_subsharding_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":2} + {"count":1,"time":11,"type":"allSuites"} + {"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1021} + {"group":{"id":2,"suiteID":0,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":null,"column":null,"url":null},"type":"group","time":1026} + {"group":{"id":3,"suiteID":0,"parentID":2,"name":"generateMetrics","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":13,"column":3,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"group","time":1027} + {"test":{"id":4,"name":"generateMetrics empty metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":20,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1027} + {"testID":4,"error":"Expected: Actual: ","stackTrace":"package:test_api expect test/tool_subsharding_test.dart 28:7 main.. ","isFailure":true,"type":"error","time":1095} + {"testID":4,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1096} + {"test":{"id":5,"name":"generateMetrics have metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":31,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1097} + {"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":1103} + {"test":{"id":6,"name":"generateMetrics missing success entry","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":60,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1103} + {"testID":6,"error":"Expected: Actual: ","stackTrace":"package:test_api expect test/tool_subsharding_test.dart 68:7 main.. ","isFailure":true,"type":"error","time":1107} + {"testID":6,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1107} + {"testID":6,"error":"my error","isFailure":true,"type":"error","time":1107} + {"success":false,"type":"done","time":1120}'''; + file.writeAsStringSync(output); + final TestFileReporterResults result = TestFileReporterResults.fromFile(file); + expect(result.hasFailedTests, true); + expect(result.errors.length == 3, true); + expect(result.errors[0].contains('Expected: Actual: '), true); + expect(result.errors[1].contains('Expected: Actual: '), true); + expect(result.errors[2].contains('my error'), true); }); }); } diff --git a/dev/bots/tool_subsharding.dart b/dev/bots/tool_subsharding.dart index 7d7d6766db..f9cbb3cfa1 100644 --- a/dev/bots/tool_subsharding.dart +++ b/dev/bots/tool_subsharding.dart @@ -38,42 +38,65 @@ class TestSpecs { } } -/// Intended to parse the output file of `dart test --file-reporter json:file_name -Map generateMetrics(File metrics) { - final Map allTestSpecs = {}; - if (!metrics.existsSync()) { - return allTestSpecs; - } +class TestFileReporterResults { + TestFileReporterResults._({ + required this.allTestSpecs, + required this.hasFailedTests, + required this.errors, + }); - bool success = false; - for(final String metric in metrics.readAsLinesSync()) { - final Map entry = json.decode(metric) as Map; - if (entry.containsKey('suite')) { - final Map suite = entry['suite'] as Map; - allTestSpecs[suite['id'] as int] = TestSpecs( - path: suite['path'] as String, - startTime: entry['time'] as int, - ); - } else if (_isMetricDone(entry, allTestSpecs)) { - final Map group = entry['group'] as Map; - final int suiteID = group['suiteID'] as int; - final TestSpecs testSpec = allTestSpecs[suiteID]!; - testSpec.endTime = entry['time'] as int; - } else if (entry.containsKey('success') && entry['success'] == true) { - success = true; + /// Intended to parse the output file of `dart test --file-reporter json:file_name + factory TestFileReporterResults.fromFile(File metrics) { + if (!metrics.existsSync()) { + throw Exception('${metrics.path} does not exist'); } + + final Map testSpecs = {}; + bool hasFailedTests = true; + final List errors = []; + + for (final String metric in metrics.readAsLinesSync()) { + final Map entry = json.decode(metric) as Map; + if (entry.containsKey('suite')) { + final Map suite = entry['suite']! as Map; + addTestSpec(suite, entry['time']! as int, testSpecs); + } else if (isMetricDone(entry, testSpecs)) { + final Map group = entry['group']! as Map; + final int suiteID = group['suiteID']! as int; + addMetricDone(suiteID, entry['time']! as int, testSpecs); + } else if (entry.containsKey('error')) { + final String stackTrace = entry.containsKey('stackTrace') ? entry['stackTrace']! as String : ''; + errors.add('${entry['error']}\n $stackTrace'); + } else if (entry.containsKey('success') && entry['success'] == true) { + hasFailedTests = false; + } + } + + return TestFileReporterResults._(allTestSpecs: testSpecs, hasFailedTests: hasFailedTests, errors: errors); } - if (!success) { // means that not all tests succeeded therefore no metrics are stored - return {}; - } - return allTestSpecs; -} + final Map allTestSpecs; + final bool hasFailedTests; + final List errors; -bool _isMetricDone(Map entry, Map allTestSpecs) { - if (entry.containsKey('group') && entry['type'] as String == 'group') { - final Map group = entry['group'] as Map; - return allTestSpecs.containsKey(group['suiteID'] as int); + + static void addTestSpec(Map suite, int time, Map allTestSpecs) { + allTestSpecs[suite['id']! as int] = TestSpecs( + path: suite['path']! as String, + startTime: time, + ); + } + + static void addMetricDone(int suiteID, int time, Map allTestSpecs) { + final TestSpecs testSpec = allTestSpecs[suiteID]!; + testSpec.endTime = time; + } + + static bool isMetricDone(Map entry, Map allTestSpecs) { + if (entry.containsKey('group') && entry['type']! as String == 'group') { + final Map group = entry['group']! as Map; + return allTestSpecs.containsKey(group['suiteID']! as int); + } + return false; } - return false; }