Move ios_content_validation_test to pre-submit tools test (#73577)

dev/devicelab/bin/tasks/ios_content_validation_test.dart

@@ -2,11 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:io';
-
 import 'package:flutter_devicelab/framework/apk_utils.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/framework/ios.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
 import 'package:flutter_devicelab/framework/utils.dart';
 import 'package:path/path.dart' as path;
@@ -15,173 +12,6 @@ Future<void> main() async {
   await task(() async {
     try {
       await runProjectTest((FlutterProject flutterProject) async {
-        section('Build app with with --obfuscate');
-        await inDirectory(flutterProject.rootPath, () async {
-          await flutter('build', options: <String>[
-            'ios',
-            '--release',
-            '--obfuscate',
-            '--split-debug-info=foo/',
-            '--no-codesign',
-          ]);
-        });
-        final String buildPath = path.join(
-          flutterProject.rootPath,
-          'build',
-          'ios',
-          'iphoneos',
-        );
-        final String outputAppPath = path.join(
-          buildPath,
-          'Runner.app',
-        );
-        final Directory outputAppFramework = Directory(path.join(
-          outputAppPath,
-          'Frameworks',
-          'App.framework',
-        ));
-
-        final File outputAppFrameworkBinary = File(path.join(
-          outputAppFramework.path,
-          'App',
-        ));
-
-        if (!outputAppFrameworkBinary.existsSync()) {
-          fail('Failed to produce expected output at ${outputAppFrameworkBinary.path}');
-        }
-
-        if (await dartObservatoryBonjourServiceFound(outputAppPath)) {
-          throw TaskResult.failure('Release bundle has unexpected NSBonjourServices');
-        }
-        if (await localNetworkUsageFound(outputAppPath)) {
-          throw TaskResult.failure('Release bundle has unexpected NSLocalNetworkUsageDescription');
-        }
-
-        section('Validate obfuscation');
-
-        // Verify that an identifier from the Dart project code is not present
-        // in the compiled binary.
-        await inDirectory(flutterProject.rootPath, () async {
-          final String response = await eval(
-            'grep',
-            <String>[flutterProject.name, outputAppFrameworkBinary.path],
-            canFail: true,
-          );
-          if (response.trim().contains('matches')) {
-            throw TaskResult.failure('Found project name in obfuscated dart library');
-          }
-        });
-
-        section('Validate release contents');
-
-        final Directory outputFlutterFramework = Directory(path.join(
-          flutterProject.rootPath,
-          outputAppPath,
-          'Frameworks',
-          'Flutter.framework',
-        ));
-
-        checkDirectoryNotExists(path.join(outputFlutterFramework.path, 'Headers'));
-        checkDirectoryNotExists(path.join(outputFlutterFramework.path, 'Modules'));
-        final File outputFlutterFrameworkBinary = File(path.join(
-          outputFlutterFramework.path,
-          'Flutter',
-        ));
-
-        if (!outputFlutterFrameworkBinary.existsSync()) {
-          fail('Failed to produce expected output at ${outputFlutterFrameworkBinary.path}');
-        }
-
-        // Archiving should contain a bitcode blob, but not building in release.
-        // This mimics Xcode behavior and present a developer from having to install a
-        // 300+MB app to test devices.
-        if (await containsBitcode(outputFlutterFrameworkBinary.path)) {
-          throw TaskResult.failure('Bitcode present in Flutter.framework');
-        }
-
-        section('Xcode backend script');
-
-        outputFlutterFramework.deleteSync(recursive: true);
-        outputAppFramework.deleteSync(recursive: true);
-        if (outputFlutterFramework.existsSync() || outputAppFramework.existsSync()) {
-          fail('Failed to delete embedded frameworks');
-        }
-
-        final String xcodeBackendPath = path.join(
-          flutterDirectory.path,
-          'packages',
-          'flutter_tools',
-          'bin',
-          'xcode_backend.sh'
-        );
-
-        // Simulate a common Xcode build setting misconfiguration
-        // where FLUTTER_APPLICATION_PATH is missing
-        final int result = await exec(
-          xcodeBackendPath,
-          <String>['embed_and_thin'],
-          environment: <String, String>{
-            'SOURCE_ROOT': flutterProject.iosPath,
-            'BUILT_PRODUCTS_DIR': path.join(
-              flutterProject.rootPath,
-              'build',
-              'ios',
-              'Release-iphoneos',
-            ),
-            'TARGET_BUILD_DIR': buildPath,
-            'FRAMEWORKS_FOLDER_PATH': 'Runner.app/Frameworks',
-            'VERBOSE_SCRIPT_LOGGING': '1',
-            'FLUTTER_BUILD_MODE': 'release',
-            'ACTION': 'install', // Skip bitcode stripping since we just checked that above.
-          },
-        );
-
-        if (result != 0) {
-          fail('xcode_backend embed_and_thin failed');
-        }
-
-        if (!outputFlutterFrameworkBinary.existsSync()) {
-          fail('Failed to re-embed ${outputFlutterFrameworkBinary.path}');
-        }
-
-        if (!outputAppFrameworkBinary.existsSync()) {
-          fail('Failed to re-embed ${outputAppFrameworkBinary.path}');
-        }
-
-        section('Clean build');
-
-        await inDirectory(flutterProject.rootPath, () async {
-          await flutter('clean');
-        });
-
-        section('Validate debug contents');
-
-        await inDirectory(flutterProject.rootPath, () async {
-          await flutter('build', options: <String>[
-            'ios',
-            '--debug',
-            '--no-codesign',
-          ]);
-        });
-
-        // Debug should also not contain bitcode.
-        if (await containsBitcode(outputFlutterFrameworkBinary.path)) {
-          throw TaskResult.failure('Bitcode present in Flutter.framework');
-        }
-
-        if (!await dartObservatoryBonjourServiceFound(outputAppPath)) {
-          throw TaskResult.failure('Debug bundle is missing NSBonjourServices');
-        }
-        if (!await localNetworkUsageFound(outputAppPath)) {
-          throw TaskResult.failure('Debug bundle is missing NSLocalNetworkUsageDescription');
-        }
-
-        section('Clean build');
-
-        await inDirectory(flutterProject.rootPath, () async {
-          await flutter('clean');
-        });
-
         section('Archive');
 
         await inDirectory(flutterProject.rootPath, () async {

dev/devicelab/lib/framework/ios.dart

@@ -4,8 +4,6 @@
 
 import 'dart:convert';
 
-import 'package:path/path.dart' as path;
-
 import 'utils.dart';
 
 typedef SimulatorFunction = Future<void> Function(String deviceId);
@@ -58,40 +56,6 @@ Future<bool> containsBitcode(String pathToBinary) async {
   return !emptyBitcodeMarkerFound;
 }
 
-Future<bool> dartObservatoryBonjourServiceFound(String appBundlePath) async =>
-  (await eval(
-    'plutil',
-    <String>[
-      '-extract',
-      'NSBonjourServices',
-      'xml1',
-      '-o',
-      '-',
-      path.join(
-        appBundlePath,
-        'Info.plist',
-      ),
-    ],
-    canFail: true,
-  )).contains('_dartobservatory._tcp');
-
-Future<bool> localNetworkUsageFound(String appBundlePath) async =>
-  await exec(
-    'plutil',
-    <String>[
-      '-extract',
-      'NSLocalNetworkUsageDescription',
-      'xml1',
-      '-o',
-      '-',
-      path.join(
-        appBundlePath,
-        'Info.plist',
-      ),
-    ],
-    canFail: true,
-  ) == 0;
-
 /// Creates and boots a new simulator, passes the new simulator's identifier to
 /// `testFunction`.
 ///

packages/flutter_tools/test/integration.shard/ios_content_validation_test.dart

@@ -0,0 +1,219 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:file_testing/file_testing.dart';
+import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/base/io.dart';
+import 'package:flutter_tools/src/build_info.dart';
+
+import '../src/common.dart';
+import '../src/darwin_common.dart';
+import 'test_utils.dart';
+
+void main() {
+  for (final BuildMode buildMode in <BuildMode>[BuildMode.debug, BuildMode.release]) {
+    group(buildMode.name, () {
+      String flutterRoot;
+      String projectRoot;
+      String flutterBin;
+      Directory tempDir;
+
+      Directory buildPath;
+      Directory outputApp;
+      Directory outputFlutterFramework;
+      File outputFlutterFrameworkBinary;
+      Directory outputAppFramework;
+      File outputAppFrameworkBinary;
+
+      setUpAll(() {
+        flutterRoot = getFlutterRoot();
+        tempDir = createResolvedTempDirectorySync('ios_content_validation.');
+        flutterBin = fileSystem.path.join(
+          flutterRoot,
+          'bin',
+          'flutter',
+        );
+
+        processManager.runSync(<String>[
+          flutterBin,
+          ...getLocalEngineArguments(),
+          'create',
+          '--platforms=ios',
+          '-i',
+          'objc',
+          'hello',
+        ], workingDirectory: tempDir.path);
+
+        projectRoot = tempDir.childDirectory('hello').path;
+
+        processManager.runSync(<String>[
+          flutterBin,
+          ...getLocalEngineArguments(),
+          'build',
+          'ios',
+          '--verbose',
+          '--no-codesign',
+          '--${buildMode.name}',
+          '--obfuscate',
+          '--split-debug-info=foo/',
+        ], workingDirectory: projectRoot);
+
+        buildPath = fileSystem.directory(fileSystem.path.join(
+          projectRoot,
+          'build',
+          'ios',
+          'iphoneos',
+        ));
+
+        outputApp = buildPath.childDirectory('Runner.app');
+
+        outputFlutterFramework = fileSystem.directory(
+          fileSystem.path.join(
+            outputApp.path,
+            'Frameworks',
+            'Flutter.framework',
+          ),
+        );
+
+        outputFlutterFrameworkBinary = outputFlutterFramework.childFile('Flutter');
+
+        outputAppFramework = fileSystem.directory(fileSystem.path.join(
+          outputApp.path,
+          'Frameworks',
+          'App.framework',
+        ));
+
+        outputAppFrameworkBinary = outputAppFramework.childFile('App');
+      });
+
+      tearDownAll(() {
+        tryToDelete(tempDir);
+      });
+
+      testWithoutContext('flutter build ios builds a valid app', () {
+        expect(outputAppFramework.childFile('App'), exists);
+
+        final File vmSnapshot = fileSystem.file(fileSystem.path.join(
+          outputAppFramework.path,
+          'flutter_assets',
+          'vm_snapshot_data',
+        ));
+
+        expect(vmSnapshot.existsSync(), buildMode == BuildMode.debug);
+
+        expect(outputFlutterFramework.childDirectory('Headers'), isNot(exists));
+        expect(outputFlutterFramework.childDirectory('Modules'), isNot(exists));
+
+        // Archiving should contain a bitcode blob, but not building.
+        // This mimics Xcode behavior and prevents a developer from having to install a
+        // 300+MB app.
+        expect(containsBitcode(outputFlutterFrameworkBinary.path, processManager), isFalse);
+      });
+
+      testWithoutContext('Info.plist dart observatory Bonjour service', () {
+        final String infoPlistPath = fileSystem.path.join(
+          outputApp.path,
+          'Info.plist',
+        );
+        final ProcessResult bonjourServices = processManager.runSync(
+          <String>[
+            'plutil',
+            '-extract',
+            'NSBonjourServices',
+            'xml1',
+            '-o',
+            '-',
+            infoPlistPath,
+          ],
+        );
+        final bool bonjourServicesFound = (bonjourServices.stdout as String).contains('_dartobservatory._tcp');
+        expect(bonjourServicesFound, buildMode == BuildMode.debug);
+
+        final ProcessResult localNetworkUsage = processManager.runSync(
+          <String>[
+            'plutil',
+            '-extract',
+            'NSLocalNetworkUsageDescription',
+            'xml1',
+            '-o',
+            '-',
+            infoPlistPath,
+          ],
+        );
+        final bool localNetworkUsageFound = localNetworkUsage.exitCode == 0;
+        expect(localNetworkUsageFound, buildMode == BuildMode.debug);
+      });
+
+      testWithoutContext('check symbols', () {
+        final ProcessResult symbols = processManager.runSync(
+          <String>[
+            'nm',
+            '-g',
+            outputAppFrameworkBinary.path,
+            '-arch',
+            'arm64',
+          ],
+        );
+        final bool aotSymbolsFound = (symbols.stdout as String).contains('_kDartVmSnapshot');
+        expect(aotSymbolsFound, buildMode != BuildMode.debug);
+      });
+
+      testWithoutContext('xcode_backend embed_and_thin', () {
+        outputFlutterFramework.deleteSync(recursive: true);
+        outputAppFramework.deleteSync(recursive: true);
+        expect(outputFlutterFrameworkBinary.existsSync(), isFalse);
+        expect(outputAppFrameworkBinary.existsSync(), isFalse);
+
+        final String xcodeBackendPath = fileSystem.path.join(
+          flutterRoot,
+          'packages',
+          'flutter_tools',
+          'bin',
+          'xcode_backend.sh',
+        );
+
+        // Simulate a common Xcode build setting misconfiguration
+        // where FLUTTER_APPLICATION_PATH is missing
+        final ProcessResult xcodeBackendResult = processManager.runSync(
+          <String>[
+            xcodeBackendPath,
+            'embed_and_thin',
+          ],
+          environment: <String, String>{
+            'SOURCE_ROOT': fileSystem.path.join(projectRoot, 'ios'),
+            'BUILT_PRODUCTS_DIR': fileSystem.path.join(
+              projectRoot,
+              'build',
+              'ios',
+              'Release-iphoneos',
+            ),
+            'TARGET_BUILD_DIR': buildPath.path,
+            'FRAMEWORKS_FOLDER_PATH': 'Runner.app/Frameworks',
+            'VERBOSE_SCRIPT_LOGGING': '1',
+            'FLUTTER_BUILD_MODE': 'release',
+            'ACTION': 'install',
+            // Skip bitcode stripping since we just checked that above.
+          },
+        );
+
+        expect(xcodeBackendResult.exitCode, 0);
+        expect(outputFlutterFrameworkBinary.existsSync(), isTrue);
+        expect(outputAppFrameworkBinary.existsSync(), isTrue);
+      }, skip: !platform.isMacOS || buildMode != BuildMode.release);
+
+      testWithoutContext('validate obfuscation', () {
+        final ProcessResult grepResult = processManager.runSync(<String>[
+          'grep',
+          '-i',
+          'hello',
+          outputAppFrameworkBinary.path,
+        ]);
+        expect(grepResult.stdout, isNot(contains('matches')));
+      });
+    },
+      skip: !platform.isMacOS,
+      timeout: const Timeout(Duration(minutes: 5)),
+    );
+  }
+}

packages/flutter_tools/test/integration.shard/macos_content_validation_test.dart

@@ -5,15 +5,15 @@
 import 'package:file_testing/file_testing.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/io.dart';
-import 'package:flutter_tools/src/convert.dart';
 
 import '../src/common.dart';
+import '../src/darwin_common.dart';
 import 'test_utils.dart';
 
 void main() {
   for (final String buildMode in <String>['Debug', 'Release']) {
     final String buildModeLower = buildMode.toLowerCase();
-    test('flutter build macos --$buildModeLower builds a valid app', () async {
+    test('flutter build macos --$buildModeLower builds a valid app', () {
       final String workingDirectory = fileSystem.path.join(
         getFlutterRoot(),
         'dev',
@@ -26,13 +26,13 @@ void main() {
         'flutter',
       );
 
-      await processManager.run(<String>[
+      processManager.runSync(<String>[
         flutterBin,
         ...getLocalEngineArguments(),
         'clean',
       ], workingDirectory: workingDirectory);
 
-      final ProcessResult result = await processManager.run(<String>[
+      final ProcessResult result = processManager.runSync(<String>[
         flutterBin,
         ...getLocalEngineArguments(),
         'build',
@@ -112,11 +112,11 @@ void main() {
           .childDirectory('A')
           .childFile('FlutterMacOS');
       expect(
-        await containsBitcode(outputFlutterFrameworkBinary.path),
+        containsBitcode(outputFlutterFrameworkBinary.path, processManager),
         isFalse,
       );
 
-      await processManager.run(<String>[
+      processManager.runSync(<String>[
         flutterBin,
         ...getLocalEngineArguments(),
         'clean',
@@ -126,47 +126,3 @@ void main() {
     );
   }
 }
-
-Future<bool> containsBitcode(String pathToBinary) async {
-  // See: https://stackoverflow.com/questions/32755775/how-to-check-a-static-library-is-built-contain-bitcode
-  final ProcessResult result = await processManager.run(<String>[
-    'otool',
-    '-l',
-    '-arch',
-    'arm64',
-    pathToBinary,
-  ]);
-  final String loadCommands = result.stdout as String;
-  if (!loadCommands.contains('__LLVM')) {
-    return false;
-  }
-  // Presence of the section may mean a bitcode marker was embedded (size=1), but there is no content.
-  if (!loadCommands.contains('size 0x0000000000000001')) {
-    return true;
-  }
-  // Check the false positives: size=1 wasn't referencing the __LLVM section.
-
-  bool emptyBitcodeMarkerFound = false;
-  //  Section
-  //  sectname __bundle
-  //  segname __LLVM
-  //  addr 0x003c4000
-  //  size 0x0042b633
-  //  offset 3932160
-  //  ...
-  final List<String> lines = LineSplitter.split(loadCommands).toList();
-  lines.asMap().forEach((int index, String line) {
-    if (line.contains('segname __LLVM') && lines.length - index - 1 > 3) {
-      final String emptyBitcodeMarker =
-          lines.skip(index - 1).take(3).firstWhere(
-                (String line) => line.contains(' size 0x0000000000000001'),
-                orElse: () => null,
-              );
-      if (emptyBitcodeMarker != null) {
-        emptyBitcodeMarkerFound = true;
-        return;
-      }
-    }
-  });
-  return !emptyBitcodeMarkerFound;
-}

packages/flutter_tools/test/src/darwin_common.dart

@@ -0,0 +1,52 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:process/process.dart';
+import 'package:flutter_tools/src/base/io.dart';
+
+bool containsBitcode(String pathToBinary, ProcessManager processManager) {
+  // See: https://stackoverflow.com/questions/32755775/how-to-check-a-static-library-is-built-contain-bitcode
+  final ProcessResult result = processManager.runSync(<String>[
+    'otool',
+    '-l',
+    '-arch',
+    'arm64',
+    pathToBinary,
+  ]);
+  final String loadCommands = result.stdout as String;
+  if (!loadCommands.contains('__LLVM')) {
+    return false;
+  }
+  // Presence of the section may mean a bitcode marker was embedded (size=1), but there is no content.
+  if (!loadCommands.contains('size 0x0000000000000001')) {
+    return true;
+  }
+  // Check the false positives: size=1 wasn't referencing the __LLVM section.
+
+  bool emptyBitcodeMarkerFound = false;
+  //  Section
+  //  sectname __bundle
+  //  segname __LLVM
+  //  addr 0x003c4000
+  //  size 0x0042b633
+  //  offset 3932160
+  //  ...
+  final List<String> lines = LineSplitter.split(loadCommands).toList();
+  lines.asMap().forEach((int index, String line) {
+    if (line.contains('segname __LLVM') && lines.length - index - 1 > 3) {
+      final String emptyBitcodeMarker =
+      lines.skip(index - 1).take(3).firstWhere(
+            (String line) => line.contains(' size 0x0000000000000001'),
+        orElse: () => null,
+      );
+      if (emptyBitcodeMarker != null) {
+        emptyBitcodeMarkerFound = true;
+        return;
+      }
+    }
+  });
+  return !emptyBitcodeMarkerFound;
+}