firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Dynamically add integration_tests and screenshots to native iOS test results (#95704)

Browse Source
This commit is contained in:
Jenn Magder 2022-01-14 12:05:15 -08:00 committed by GitHub
parent bbc68cd207
commit 88327e3b72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 307 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
packages/integration_test/example/integration_test/_extended_test_io.dart
View File

@ -25,6 +25,24 @@ void main() {
// Build our app.
app.main();
// Pump a frame.
await tester.pumpAndSettle();
// Verify that platform version is retrieved.
expect(
find.byWidgetPredicate(
(Widget widget) =>
widget is Text &&
widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
),
findsOneWidget,
);
});
testWidgets('verify screenshot', (WidgetTester tester) async {
// Build our app.
app.main();
// On Android, this is required prior to taking the screenshot.
await binding.convertFlutterSurfaceToImage();
@ -39,15 +57,5 @@ void main() {
expect(secondPng.isNotEmpty, isTrue);
expect(listEquals(firstPng, secondPng), isTrue);
// Verify that platform version is retrieved.
expect(
find.byWidgetPredicate(
(Widget widget) =>
widget is Text &&
widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
),
findsOneWidget,
);
});
}

14
packages/integration_test/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
View File

@ -20,6 +20,20 @@
ReferencedContainer = "container:Runner.xcodeproj">
</BuildableReference>
</BuildActionEntry>
<BuildActionEntry
buildForTesting = "YES"
buildForRunning = "NO"
buildForProfiling = "NO"
buildForArchiving = "NO"
buildForAnalyzing = "YES">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "769541C723A0351900E5C350"
BuildableName = "RunnerTests.xctest"
BlueprintName = "RunnerTests"
ReferencedContainer = "container:Runner.xcodeproj">
</BuildableReference>
</BuildActionEntry>
</BuildActionEntries>
</BuildAction>
<TestAction

85
packages/integration_test/example/ios/RunnerTests/RunnerTests.m
View File

@ -5,4 +5,89 @@
@import XCTest;
@import integration_test;
#pragma mark - Dynamic tests
INTEGRATION_TEST_IOS_RUNNER(RunnerTests)
@interface RunnerTests (DynamicTests)
@end
@implementation RunnerTests (DynamicTests)
- (void)setUp {
// Verify tests have been dynamically added from FLUTTER_TARGET=integration_test/extended_test.dart
XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyScreenshot")]);
XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyText")]);
XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"screenshotPlaceholder")]);
}
@end
#pragma mark - Fake test results
@interface IntegrationTestPlugin ()
- (instancetype)initForRegistration;
@end
@interface FLTIntegrationTestRunner ()
@property IntegrationTestPlugin *integrationTestPlugin;
@end
@interface FakeIntegrationTestPlugin : IntegrationTestPlugin
@property(nonatomic, nullable) NSDictionary<NSString *, NSString *> *testResults;
@end
@implementation FakeIntegrationTestPlugin
@synthesize testResults;
- (void)setupChannels:(id<FlutterBinaryMessenger>)binaryMessenger {
}
@end
#pragma mark - Behavior tests
@interface IntegrationTestTests : XCTestCase
@end
@implementation IntegrationTestTests
- (void)testDeprecatedIntegrationTest {
NSString *testResult;
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
BOOL testPass = [[IntegrationTestIosTest new] testIntegrationTest:&testResult];
#pragma clang diagnostic pop
XCTAssertTrue(testPass, @"%@", testResult);
}
- (void)testMethodNamesFromDartTests {
XCTAssertEqualObjects([FLTIntegrationTestRunner
testCaseNameFromDartTestName:@"this is a test"], @"testThisIsATest");
XCTAssertEqualObjects([FLTIntegrationTestRunner
testCaseNameFromDartTestName:@"VALIDATE multi-point 🚀 UNICODE123: 😁"], @"testValidateMultiPointUnicode123");
XCTAssertEqualObjects([FLTIntegrationTestRunner
testCaseNameFromDartTestName:@"!UPPERCASE:\\ lower_seperate?"], @"testUppercaseLowerSeperate");
}
- (void)testDuplicatedDartTests {
FakeIntegrationTestPlugin *fakePlugin = [[FakeIntegrationTestPlugin alloc] initForRegistration];
// These are unique test names in dart, but would result in duplicate
// XCTestCase names when the emojis are stripped.
fakePlugin.testResults = @{@"unique": @"dart test failure", @"emoji 🐢": @"success", @"emoji 🐇": @"failure"};
FLTIntegrationTestRunner *runner = [[FLTIntegrationTestRunner alloc] init];
runner.integrationTestPlugin = fakePlugin;
NSMutableDictionary<NSString *, NSString *> *failuresByTestName = [[NSMutableDictionary alloc] init];
[runner testIntegrationTestWithResults:^(SEL nativeTestSelector, BOOL success, NSString *failureMessage) {
NSString *testName = NSStringFromSelector(nativeTestSelector);
XCTAssertFalse([failuresByTestName.allKeys containsObject:testName]);
failuresByTestName[testName] = failureMessage;
}];
XCTAssertEqualObjects(failuresByTestName,
(@{@"testUnique": @"dart test failure",
@"testDuplicateTestNames": @"Cannot test \"emoji 🐇\", duplicate XCTestCase tests named testEmoji"}));
}
@end

36
packages/integration_test/ios/Classes/FLTIntegrationTestRunner.h Normal file
View File

@ -0,0 +1,36 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@import Foundation;
@class UIImage;
NS_ASSUME_NONNULL_BEGIN
typedef void (^FLTIntegrationTestResults)(SEL nativeTestSelector, BOOL success, NSString *_Nullable failureMessage);
@interface FLTIntegrationTestRunner : NSObject
/**
* Any screenshots captured by the plugin.
*/
@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
/**
* Starts dart tests and waits for results.
*
* @param testResult Will be called once per every completed dart test.
*/
- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult;
/**
* An appropriate XCTest method name based on the dart test name.
*
* Example: dart test "verify widget-ABC123" becomes "testVerifyWidgetABC123"
*/
+ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName;
@end
NS_ASSUME_NONNULL_END

77
packages/integration_test/ios/Classes/FLTIntegrationTestRunner.m Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import "FLTIntegrationTestRunner.h"
#import "IntegrationTestPlugin.h"
@import ObjectiveC.runtime;
@import UIKit;
@interface FLTIntegrationTestRunner ()
@property IntegrationTestPlugin *integrationTestPlugin;
@end
@implementation FLTIntegrationTestRunner
- (instancetype)init {
self = [super init];
_integrationTestPlugin = [IntegrationTestPlugin instance];
return self;
}
- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult {
IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
UIViewController *rootViewController = UIApplication.sharedApplication.delegate.window.rootViewController;
if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
testResult(NSSelectorFromString(@"testSetup"), NO, @"rootViewController was not expected FlutterViewController");
}
FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
[integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
// Spin the runloop.
while (!integrationTestPlugin.testResults) {
[NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]];
}
NSMutableSet<NSString *> *testCaseNames = [[NSMutableSet alloc] init];
[integrationTestPlugin.testResults enumerateKeysAndObjectsUsingBlock:^(NSString *test, NSString *result, BOOL *stop) {
NSString *testSelectorName = [[self class] testCaseNameFromDartTestName:test];
// Validate Objective-C test names are unique after sanitization.
if ([testCaseNames containsObject:testSelectorName]) {
NSString *reason = [NSString stringWithFormat:@"Cannot test \"%@\", duplicate XCTestCase tests named %@", test, testSelectorName];
testResult(NSSelectorFromString(@"testDuplicateTestNames"), NO, reason);
*stop = YES;
return;
}
[testCaseNames addObject:testSelectorName];
SEL testSelector = NSSelectorFromString(testSelectorName);
if ([result isEqualToString:@"success"]) {
testResult(testSelector, YES, nil);
} else {
testResult(testSelector, NO, result);
}
}];
}
- (NSDictionary<NSString *,UIImage *> *)capturedScreenshotsByName {
return self.integrationTestPlugin.capturedScreenshotsByName;
}
+ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName {
NSString *capitalizedString = dartTestName.localizedCapitalizedString;
// Objective-C method names must be alphanumeric.
NSCharacterSet *disallowedCharacters = NSCharacterSet.alphanumericCharacterSet.invertedSet;
// Remove disallowed characters.
NSString *upperCamelTestName = [[capitalizedString componentsSeparatedByCharactersInSet:disallowedCharacters] componentsJoinedByString:@""];
return [NSString stringWithFormat:@"test%@", upperCamelTestName];
}
@end

58
packages/integration_test/ios/Classes/IntegrationTestIosTest.h
View File

@ -2,16 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import <Foundation/Foundation.h>
@import Foundation;
@import ObjectiveC.runtime;
NS_ASSUME_NONNULL_BEGIN
@protocol FLTIntegrationTestScreenshotDelegate;
DEPRECATED_MSG_ATTRIBUTE("Use FLTIntegrationTestRunner instead.")
@interface IntegrationTestIosTest : NSObject
- (instancetype)initWithScreenshotDelegate:(nullable id<FLTIntegrationTestScreenshotDelegate>)delegate NS_DESIGNATED_INITIALIZER;
/**
* Initiate dart tests and wait for results. @c testResult will be set to a string describing the results.
*
@ -21,26 +19,48 @@ NS_ASSUME_NONNULL_BEGIN
@end
// For every Flutter dart test, dynamically generate an Objective-C method mirroring the test results
// so it is reported as a native XCTest run result.
// If the Flutter dart tests have captured screenshots, add them to the XCTest bundle.
#define INTEGRATION_TEST_IOS_RUNNER(__test_class) \
@interface __test_class : XCTestCase<FLTIntegrationTestScreenshotDelegate> \
@interface __test_class : XCTestCase \
@end \
\
@implementation __test_class \
\
- (void)testIntegrationTest { \
NSString *testResult; \
IntegrationTestIosTest *integrationTestIosTest = integrationTestIosTest = [[IntegrationTestIosTest alloc] initWithScreenshotDelegate:self]; \
BOOL testPass = [integrationTestIosTest testIntegrationTest:&testResult]; \
XCTAssertTrue(testPass, @"%@", testResult); \
} \
\
- (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(NSString *)name { \
XCTAttachment *attachment = [XCTAttachment attachmentWithImage:screenshot]; \
attachment.lifetime = XCTAttachmentLifetimeKeepAlways; \
if (name != nil) { \
attachment.name = name; \
+ (NSArray<NSInvocation *> *)testInvocations { \
FLTIntegrationTestRunner *integrationTestRunner = [[FLTIntegrationTestRunner alloc] init]; \
NSMutableArray<NSInvocation *> *testInvocations = [[NSMutableArray alloc] init]; \
[integrationTestRunner testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *failureMessage) { \
IMP assertImplementation = imp_implementationWithBlock(^(id _self) { \
XCTAssertTrue(success, @"%@", failureMessage); \
}); \
class_addMethod(self, testSelector, assertImplementation, "v@:"); \
NSMethodSignature *signature = [self instanceMethodSignatureForSelector:testSelector]; \
NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; \
invocation.selector = testSelector; \
[testInvocations addObject:invocation]; \
}]; \
NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName = integrationTestRunner.capturedScreenshotsByName; \
if (capturedScreenshotsByName.count > 0) { \
IMP screenshotImplementation = imp_implementationWithBlock(^(id _self) { \
[capturedScreenshotsByName enumerateKeysAndObjectsUsingBlock:^(NSString *name, UIImage *screenshot, BOOL *stop) { \
XCTAttachment *attachment = [XCTAttachment attachmentWithImage:screenshot]; \
attachment.lifetime = XCTAttachmentLifetimeKeepAlways; \
if (name != nil) { \
attachment.name = name; \
} \
[_self addAttachment:attachment]; \
}]; \
}); \
SEL attachmentSelector = NSSelectorFromString(@"screenshotPlaceholder"); \
class_addMethod(self, attachmentSelector, screenshotImplementation, "v@:"); \
NSMethodSignature *attachmentSignature = [self instanceMethodSignatureForSelector:attachmentSelector]; \
NSInvocation *attachmentInvocation = [NSInvocation invocationWithMethodSignature:attachmentSignature]; \
attachmentInvocation.selector = attachmentSelector; \
[testInvocations addObject:attachmentInvocation]; \
} \
[self addAttachment:attachment]; \
return testInvocations; \
} \
\
@end

61
packages/integration_test/ios/Classes/IntegrationTestIosTest.m
View File

@ -3,61 +3,40 @@
// found in the LICENSE file.
#import "IntegrationTestIosTest.h"
#import "IntegrationTestPlugin.h"
@interface IntegrationTestIosTest()
@property (nonatomic) IntegrationTestPlugin *integrationTestPlugin;
@end
#import "IntegrationTestPlugin.h"
#import "FLTIntegrationTestRunner.h"
#pragma mark - Deprecated
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-implementations"
@implementation IntegrationTestIosTest
- (instancetype)initWithScreenshotDelegate:(id<FLTIntegrationTestScreenshotDelegate>)delegate {
self = [super init];
_integrationTestPlugin = [IntegrationTestPlugin instance];
_integrationTestPlugin.screenshotDelegate = delegate;
return self;
}
- (instancetype)init {
return [self initWithScreenshotDelegate:nil];
}
- (BOOL)testIntegrationTest:(NSString **)testResult {
IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
UIViewController *rootViewController =
[[[[UIApplication sharedApplication] delegate] window] rootViewController];
if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
NSLog(@"expected FlutterViewController as rootViewController.");
return NO;
}
FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
[integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
while (!integrationTestPlugin.testResults) {
CFRunLoopRunInMode(kCFRunLoopDefaultMode, 1.f, NO);
}
NSDictionary<NSString *, NSString *> *testResults = integrationTestPlugin.testResults;
NSMutableArray<NSString *> *passedTests = [NSMutableArray array];
NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
NSLog(@"==================== Test Results =====================");
for (NSString *test in testResults.allKeys) {
NSString *result = testResults[test];
if ([result isEqualToString:@"success"]) {
NSLog(@"%@ passed.", test);
[passedTests addObject:test];
NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
NSMutableArray<NSString *> *testNames = [NSMutableArray array];
[[FLTIntegrationTestRunner new] testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *message) {
NSString *testName = NSStringFromSelector(testSelector);
[testNames addObject:testName];
if (success) {
NSLog(@"%@ passed.", testName);
} else {
NSLog(@"%@ failed: %@", test, result);
[failedTests addObject:test];
NSLog(@"%@ failed: %@", testName, message);
[failedTests addObject:testName];
}
}
}];
NSLog(@"================== Test Results End ====================");
BOOL testPass = failedTests.count == 0;
if (!testPass && testResult) {
if (!testPass && testResult != NULL) {
*testResult =
[NSString stringWithFormat:@"Detected failed integration test(s) %@ among %@",
failedTests.description, testResults.allKeys.description];
failedTests.description, testNames.description];
}
return testPass;
}
@end
#pragma clang diagnostic pop

14
packages/integration_test/ios/Classes/IntegrationTestPlugin.h
View File

@ -6,13 +6,6 @@
NS_ASSUME_NONNULL_BEGIN
@protocol FLTIntegrationTestScreenshotDelegate
/** This will be called when a dart integration test triggers a window screenshot with @c takeScreenshot. */
- (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(nullable NSString *)name;
@end
/** A Flutter plugin that's responsible for communicating the test results back
* to iOS XCTest. */
@interface IntegrationTestPlugin : NSObject <FlutterPlugin>
@ -23,6 +16,11 @@ NS_ASSUME_NONNULL_BEGIN
*/
@property(nonatomic, readonly, nullable) NSDictionary<NSString *, NSString *> *testResults;
/**
* Mapping of screenshot images by suggested names, captured by the dart tests.
*/
@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
/** Fetches the singleton instance of the plugin. */
+ (IntegrationTestPlugin *)instance;
@ -30,8 +28,6 @@ NS_ASSUME_NONNULL_BEGIN
- (instancetype)init NS_UNAVAILABLE;
@property(weak, nonatomic) id<FLTIntegrationTestScreenshotDelegate> screenshotDelegate;
@end
NS_ASSUME_NONNULL_END

17
packages/integration_test/ios/Classes/IntegrationTestPlugin.m
View File

@ -2,10 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@import UIKit;
#import "IntegrationTestPlugin.h"
@import UIKit;
static NSString *const kIntegrationTestPluginChannel = @"plugins.flutter.io/integration_test";
static NSString *const kMethodTestFinished = @"allTestsFinished";
static NSString *const kMethodScreenshot = @"captureScreenshot";
@ -16,10 +16,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
@property(nonatomic, readwrite) NSDictionary<NSString *, NSString *> *testResults;
- (instancetype)init NS_DESIGNATED_INITIALIZER;
@end
@implementation IntegrationTestPlugin {
NSDictionary<NSString *, NSString *> *_testResults;
NSMutableDictionary<NSString *, UIImage *> *_capturedScreenshotsByName;
}
+ (IntegrationTestPlugin *)instance {
@ -32,7 +35,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
}
- (instancetype)initForRegistration {
return [super init];
return [self init];
}
- (instancetype)init {
self = [super init];
_capturedScreenshotsByName = [NSMutableDictionary new];
return self;
}
+ (void)registerWithRegistrar:(NSObject<FlutterPluginRegistrar> *)registrar {
@ -59,7 +68,7 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
// If running as a native Xcode test, attach to test.
UIImage *screenshot = [self capturePngScreenshot];
NSString *name = call.arguments[@"name"];
[self.screenshotDelegate didTakeScreenshot:screenshot attachmentName:name];
_capturedScreenshotsByName[name] = screenshot;
// Also pass back along the channel for the driver to handle.
NSData *pngData = UIImagePNGRepresentation(screenshot);