diff --git a/dev/customer_testing/lib/customer_test.dart b/dev/customer_testing/lib/customer_test.dart index b56b300444..3f661f1d89 100644 --- a/dev/customer_testing/lib/customer_test.dart +++ b/dev/customer_testing/lib/customer_test.dart @@ -14,6 +14,7 @@ class CustomerTest { final List fetch = []; final List update = []; final List test = []; + int? iterations; bool hasTests = false; for (final String line in testFile.readAsLinesSync().map((String line) => line.trim())) { if (line.isEmpty) { @@ -26,6 +27,14 @@ class CustomerTest { fetch.add(line.substring(6)); } else if (line.startsWith('update=')) { update.add(Directory(line.substring(7))); + } else if (line.startsWith('iterations=')) { + if (iterations != null) { + throw const FormatException('Cannot specify "iterations" directive multiple times.'); + } + iterations = int.parse(line.substring(11)); + if (iterations < 1) { + throw const FormatException('The "iterations" directive must have a positive integer value.'); + } } else if (line.startsWith('test=')) { hasTests = true; test.add(line.substring(5)); @@ -84,10 +93,11 @@ class CustomerTest { List.unmodifiable(fetch), List.unmodifiable(update), List.unmodifiable(test), + iterations, ); } - const CustomerTest._(this.contacts, this.fetch, this.update, this.tests); + const CustomerTest._(this.contacts, this.fetch, this.update, this.tests, this.iterations); // (e-mail regexp from HTML standard) static final RegExp _email = RegExp(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$"); @@ -98,4 +108,5 @@ class CustomerTest { final List fetch; final List update; final List tests; + final int? iterations; } diff --git a/dev/customer_testing/lib/runner.dart b/dev/customer_testing/lib/runner.dart index f453483d56..9a8b6d30bb 100644 --- a/dev/customer_testing/lib/runner.dart +++ b/dev/customer_testing/lib/runner.dart @@ -133,6 +133,14 @@ Future runTests({ if (verbose) { print('Running tests...'); } + if (instructions.iterations != null && instructions.iterations! < repeat) { + if (verbose) { + final String s = instructions.iterations == 1 ? '' : 's'; + print('Limiting to ${instructions.iterations} round$s rather than $repeat rounds because of "iterations" directive.'); + } + repeat = instructions.iterations!; + } + final Stopwatch stopwatch = Stopwatch()..start(); for (int iteration = 0; iteration < repeat; iteration += 1) { if (verbose && repeat > 1) { print('Round ${iteration + 1} of $repeat.'); @@ -146,8 +154,9 @@ Future runTests({ } } } + stopwatch.stop(); if (verbose && success) { - print('Tests finished.'); + print('Tests finished in ${(stopwatch.elapsed.inSeconds / repeat).toStringAsFixed(2)} seconds per iteration.'); } } } diff --git a/dev/customer_testing/run_tests.dart b/dev/customer_testing/run_tests.dart index c3c5f3482b..cb3f421418 100644 --- a/dev/customer_testing/run_tests.dart +++ b/dev/customer_testing/run_tests.dart @@ -24,7 +24,7 @@ Future run(List arguments) async { ..addOption( 'repeat', defaultsTo: '1', - help: 'How many times to run each test. Set to a high value to look for flakes.', + help: 'How many times to run each test. Set to a high value to look for flakes. If a test specifies a number of iterations, the lower of the two values is used.', valueHelp: 'count', ) ..addOption(