From 3cb98a244811a050d0155303f674e7e916a49dec Mon Sep 17 00:00:00 2001 From: Jacob Relkin Date: Fri, 20 Jun 2014 10:43:31 -0700 Subject: [PATCH 1/5] Added a hook on SLTest for getting notified when an exception happens in a test run. Added a SLTestCaseExceptionInfo object which encapsulates the relevant exception data. --- Sources/Classes/SLTest.h | 4 ++ Sources/Classes/SLTest.m | 67 ++++++++++++++--------- Sources/Classes/SLTestCaseExceptionInfo.h | 19 +++++++ Sources/Classes/SLTestCaseExceptionInfo.m | 32 +++++++++++ Sources/Subliminal.h | 1 + Subliminal.xcodeproj/project.pbxproj | 8 +++ 6 files changed, 105 insertions(+), 26 deletions(-) create mode 100644 Sources/Classes/SLTestCaseExceptionInfo.h create mode 100644 Sources/Classes/SLTestCaseExceptionInfo.m diff --git a/Sources/Classes/SLTest.h b/Sources/Classes/SLTest.h index 10996e7..1917a94 100644 --- a/Sources/Classes/SLTest.h +++ b/Sources/Classes/SLTest.h @@ -25,6 +25,8 @@ #import "SLTestController+AppHooks.h" #import "SLStringUtilities.h" +@class SLTestCaseExceptionInfo; + /** `SLTest` is the abstract superclass of Subliminal integration tests. @@ -212,6 +214,8 @@ failed:(NSUInteger *)numCasesFailed failedUnexpectedly:(NSUInteger *)numCasesFailedUnexpectedly; +- (void)testRunDidCatchExceptionWithExceptionInfo:(SLTestCaseExceptionInfo *)exceptionInfo; + @end diff --git a/Sources/Classes/SLTest.m b/Sources/Classes/SLTest.m index 729c220..8a2df32 100644 --- a/Sources/Classes/SLTest.m +++ b/Sources/Classes/SLTest.m @@ -29,6 +29,7 @@ #import #import +#import "SLTestCaseExceptionInfo.h" // All exceptions thrown by SLTest must have names beginning with this prefix // so that `-[SLTest exceptionByAddingFileInfo:]` can determine whether to attach @@ -39,6 +40,11 @@ const NSTimeInterval SLWaitUntilTrueRetryDelay = 0.25; +@interface SLTest () + +@property (nonatomic, strong) SLTestCaseExceptionInfo *testCaseExceptionInfo; + +@end @implementation SLTest { NSString *_lastKnownFilename; @@ -232,20 +238,30 @@ + (BOOL)testCaseWithSelectorSupportsCurrentPlatform:(SEL)testCaseSelector { if ([testCaseName hasSuffix:@"_iPad"]) return (userInterfaceIdiom == UIUserInterfaceIdiomPad); if ([testCaseName hasSuffix:@"_iPhone"]) return (userInterfaceIdiom == UIUserInterfaceIdiomPhone); return YES; - } +} + +- (void)testRunDidCatchException:(NSException *)exception testCaseSelector:(SEL)testCaseSelector { + self.testCaseExceptionInfo = [SLTestCaseExceptionInfo exceptionInfoWithException:exception testCaseSelector:testCaseSelector]; + + NSException *exceptionToLog = [self exceptionByAddingFileInfo:self.testCaseExceptionInfo.exception]; + [[SLLogger sharedLogger] logException:exceptionToLog + expected:[self.testCaseExceptionInfo isExpected]]; + + [self testRunDidCatchExceptionWithExceptionInfo:self.testCaseExceptionInfo]; +} - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted failed:(NSUInteger *)numCasesFailed failedUnexpectedly:(NSUInteger *)numCasesFailedUnexpectedly { NSUInteger numberOfCasesExecuted = 0, numberOfCasesFailed = 0, numberOfCasesFailedUnexpectedly = 0; + BOOL testDidFailInSetUpOrTearDown = NO; @try { [self setUpTest]; } @catch (NSException *exception) { - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:[[self class] exceptionWasExpected:exception]]; + [self testRunDidCatchException:exception testCaseSelector:NULL]; testDidFailInSetUpOrTearDown = YES; } @@ -265,49 +281,46 @@ - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted // clear call site information, so at the least it won't be reused between test cases // (though we can't guarantee it won't be reused within a test case) [self clearLastKnownCallSite]; + [self clearTestCaseExceptionState]; - BOOL caseFailed = NO, failureWasExpected = NO; @try { [self setUpTestCaseWithSelector:unfocusedTestCaseSelector]; } @catch (NSException *exception) { - caseFailed = YES; - failureWasExpected = [[self class] exceptionWasExpected:exception]; - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:failureWasExpected]; + [self testRunDidCatchException:exception testCaseSelector:unfocusedTestCaseSelector]; } // Only execute the test case if set-up succeeded. - if (!caseFailed) { + if (!self.testCaseExceptionInfo) { @try { // We use objc_msgSend so that Clang won't complain about performSelector leaks // Make sure to send the actual test case selector ((void(*)(id, SEL))objc_msgSend)(self, NSSelectorFromString(testCaseName)); } @catch (NSException *exception) { - caseFailed = YES; - failureWasExpected = [[self class] exceptionWasExpected:exception]; - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:failureWasExpected]; + [self testRunDidCatchException:exception testCaseSelector:unfocusedTestCaseSelector]; } } + // If we didn't already fail, testCaseExceptionInfo will be nil + BOOL failureWasExpected = self.testCaseExceptionInfo.expected; + // Still perform tear-down even if set-up failed. // If the app is in an inconsistent state, then tear-down should fail. @try { [self tearDownTestCaseWithSelector:unfocusedTestCaseSelector]; } @catch (NSException *exception) { - BOOL caseHadFailed = caseFailed; - caseFailed = YES; - // don't override `failureWasExpected` if we had already failed - BOOL exceptionWasExpected = [[self class] exceptionWasExpected:exception]; - if (!caseHadFailed) failureWasExpected = exceptionWasExpected; - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:exceptionWasExpected]; + BOOL succeededUntilTeardown = self.testCaseExceptionInfo == nil; + + [self testRunDidCatchException:exception testCaseSelector:unfocusedTestCaseSelector]; + + if (succeededUntilTeardown) { + failureWasExpected = self.testCaseExceptionInfo.expected; + } } - if (caseFailed) { + if (self.testCaseExceptionInfo) { [[SLLogger sharedLogger] logTest:test caseFail:testCaseName expected:failureWasExpected]; numberOfCasesFailed++; if (!failureWasExpected) numberOfCasesFailedUnexpectedly++; @@ -324,8 +337,7 @@ - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted [self tearDownTest]; } @catch (NSException *exception) { - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:[[self class] exceptionWasExpected:exception]]; + [self testRunDidCatchException:exception testCaseSelector:NULL]; testDidFailInSetUpOrTearDown = YES; } @@ -340,6 +352,10 @@ - (void)wait:(NSTimeInterval)interval { [NSThread sleepForTimeInterval:interval]; } +- (void)clearTestCaseExceptionState { + self.testCaseExceptionInfo = nil; +} + - (void)recordLastKnownFile:(const char *)filename line:(int)lineNumber { _lastKnownFilename = [@(filename) lastPathComponent]; _lastKnownLineNumber = lineNumber; @@ -372,9 +388,8 @@ - (NSException *)exceptionByAddingFileInfo:(NSException *)exception { return exception; } -+ (BOOL)exceptionWasExpected:(NSException *)exception { - return [[exception name] isEqualToString:SLTestAssertionFailedException]; -} +// Abstract +- (void)testRunDidCatchExceptionWithExceptionInfo:(SLTestCaseExceptionInfo *)exceptionInfo {} @end diff --git a/Sources/Classes/SLTestCaseExceptionInfo.h b/Sources/Classes/SLTestCaseExceptionInfo.h new file mode 100644 index 0000000..c40aa93 --- /dev/null +++ b/Sources/Classes/SLTestCaseExceptionInfo.h @@ -0,0 +1,19 @@ +// +// SLTestCaseExceptionInfo.h +// Subliminal +// +// Created by Jacob Relkin on 6/20/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import + +@interface SLTestCaseExceptionInfo : NSObject + ++ (instancetype)exceptionInfoWithException:(NSException *)exception testCaseSelector:(SEL)testCaseSelector; + +@property (nonatomic, readonly, assign) SEL testCaseSelector; +@property (nonatomic, readonly, strong) NSException *exception; +@property (nonatomic, readonly, getter = isExpected) BOOL expected; + +@end diff --git a/Sources/Classes/SLTestCaseExceptionInfo.m b/Sources/Classes/SLTestCaseExceptionInfo.m new file mode 100644 index 0000000..13b2d66 --- /dev/null +++ b/Sources/Classes/SLTestCaseExceptionInfo.m @@ -0,0 +1,32 @@ +// +// SLTestCaseExceptionInfo.m +// Subliminal +// +// Created by Jacob Relkin on 6/20/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import "SLTestCaseExceptionInfo.h" +#import "SLTest.h" + +@interface SLTestCaseExceptionInfo () + +@property (nonatomic, readwrite, strong) NSException *exception; +@property (nonatomic, readwrite, assign) SEL testCaseSelector; + +@end + +@implementation SLTestCaseExceptionInfo + ++ (instancetype)exceptionInfoWithException:(NSException *)exception testCaseSelector:(SEL)testCaseSelector { + SLTestCaseExceptionInfo *info = [self new]; + info.exception = exception; + info.testCaseSelector = testCaseSelector; + return info; +} + +- (BOOL)isExpected { + return [self.exception.name isEqualToString:SLTestAssertionFailedException]; +} + +@end diff --git a/Sources/Subliminal.h b/Sources/Subliminal.h index c4b3f29..2e83439 100644 --- a/Sources/Subliminal.h +++ b/Sources/Subliminal.h @@ -23,6 +23,7 @@ #import "SLTestController.h" #import "SLTestController+AppHooks.h" #import "SLTest.h" +#import "SLTestCaseExceptionInfo.h" #import "SLDevice.h" #import "SLElement.h" diff --git a/Subliminal.xcodeproj/project.pbxproj b/Subliminal.xcodeproj/project.pbxproj index 2b2ca8e..8b1f41a 100644 --- a/Subliminal.xcodeproj/project.pbxproj +++ b/Subliminal.xcodeproj/project.pbxproj @@ -58,6 +58,8 @@ 627B5FD4194FACA50059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 627B5FD0194FA9D70059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard */; }; 62E7A633193EF84C00CB11AB /* SLStaticText.h in Headers */ = {isa = PBXBuildFile; fileRef = 62E7A631193EF84C00CB11AB /* SLStaticText.h */; settings = {ATTRIBUTES = (Public, ); }; }; 62E7A634193EF84C00CB11AB /* SLStaticText.m in Sources */ = {isa = PBXBuildFile; fileRef = 62E7A632193EF84C00CB11AB /* SLStaticText.m */; }; + C7FD55131954CAD700E99FB7 /* SLTestCaseExceptionInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = C7FD55111954CAD700E99FB7 /* SLTestCaseExceptionInfo.h */; settings = {ATTRIBUTES = (Public, ); }; }; + C7FD55141954CAD700E99FB7 /* SLTestCaseExceptionInfo.m in Sources */ = {isa = PBXBuildFile; fileRef = C7FD55121954CAD700E99FB7 /* SLTestCaseExceptionInfo.m */; }; CA75E78216697A1200D57E92 /* SLDevice.h in Headers */ = {isa = PBXBuildFile; fileRef = CA75E78016697A1200D57E92 /* SLDevice.h */; settings = {ATTRIBUTES = (Public, ); }; }; CA75E78516697C0000D57E92 /* SLDevice.m in Sources */ = {isa = PBXBuildFile; fileRef = CA75E78116697A1200D57E92 /* SLDevice.m */; }; CAC388051641CD7500F995F9 /* SLStringUtilities.h in Headers */ = {isa = PBXBuildFile; fileRef = CAC388031641CD7500F995F9 /* SLStringUtilities.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -308,6 +310,8 @@ 627B5FD0194FA9D70059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = SLTestMatchingElementsWithinTableHeaderView.storyboard; sourceTree = ""; }; 62E7A631193EF84C00CB11AB /* SLStaticText.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLStaticText.h; sourceTree = ""; }; 62E7A632193EF84C00CB11AB /* SLStaticText.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLStaticText.m; sourceTree = ""; }; + C7FD55111954CAD700E99FB7 /* SLTestCaseExceptionInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLTestCaseExceptionInfo.h; sourceTree = ""; }; + C7FD55121954CAD700E99FB7 /* SLTestCaseExceptionInfo.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLTestCaseExceptionInfo.m; sourceTree = ""; }; CA75E78016697A1200D57E92 /* SLDevice.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLDevice.h; sourceTree = ""; }; CA75E78116697A1200D57E92 /* SLDevice.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLDevice.m; sourceTree = ""; }; CAC388031641CD7500F995F9 /* SLStringUtilities.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLStringUtilities.h; sourceTree = ""; }; @@ -769,6 +773,8 @@ F0695DE3160138DF000B05D0 /* SLTestController.m */, F0271AFD162E0B950098F5F2 /* SLTestController+AppHooks.h */, F0271AFE162E0B950098F5F2 /* SLTestController+AppHooks.m */, + C7FD55111954CAD700E99FB7 /* SLTestCaseExceptionInfo.h */, + C7FD55121954CAD700E99FB7 /* SLTestCaseExceptionInfo.m */, F0695DE0160138DF000B05D0 /* SLTest.h */, F0695DE1160138DF000B05D0 /* SLTest.m */, F0695DD6160138DF000B05D0 /* UIAutomation */, @@ -1107,6 +1113,7 @@ F02DF30817EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.h in Headers */, F00800CE174C1C64001927AC /* SLPopover.h in Headers */, 50F3E18C1783A5CB00C6BD1B /* SLGeometry.h in Headers */, + C7FD55131954CAD700E99FB7 /* SLTestCaseExceptionInfo.h in Headers */, F043469F175ACE3A00D91F7F /* NSObject+SLAccessibilityDescription.h in Headers */, F04346AF175AD63E00D91F7F /* SLAccessibilityPath.h in Headers */, F04346A7175AD10200D91F7F /* NSObject+SLVisibility.h in Headers */, @@ -1379,6 +1386,7 @@ 622DA0BB194E2CB900EFFE05 /* SLDatePicker.m in Sources */, F0695DE9160138DF000B05D0 /* SLTestController.m in Sources */, F0271B00162E0B950098F5F2 /* SLTestController+AppHooks.m in Sources */, + C7FD55141954CAD700E99FB7 /* SLTestCaseExceptionInfo.m in Sources */, F02DF30917EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.m in Sources */, CAC388061641CD7500F995F9 /* SLStringUtilities.m in Sources */, 622DA090194AF1E200EFFE05 /* SLPickerView.m in Sources */, From c1196aac3a2d68a000f95284a454617844c13413 Mon Sep 17 00:00:00 2001 From: Jacob Relkin Date: Mon, 23 Jun 2014 15:16:45 -0700 Subject: [PATCH 2/5] Added documentation for `SLTestCaseExceptionInfo` and the corresponding hook on `SLTest`. --- Sources/Classes/SLTest.h | 8 ++++++++ Sources/Classes/SLTestCaseExceptionInfo.h | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/Sources/Classes/SLTest.h b/Sources/Classes/SLTest.h index 1917a94..6c41392 100644 --- a/Sources/Classes/SLTest.h +++ b/Sources/Classes/SLTest.h @@ -214,6 +214,14 @@ failed:(NSUInteger *)numCasesFailed failedUnexpectedly:(NSUInteger *)numCasesFailedUnexpectedly; +/** + If overridden, provides a hook for when an exception is caught within a test run. + + @param exceptionInfo An object which describes the exception that was thrown. Note: If this exception wasn't thrown within a test case, the `exceptionInfo`'s `testCaseSelector` will be NULL. + + @warning This method will be invoked for each exception that is handled by the test framework. + @see -[SLTestCaseExceptionInfo exceptionInfoWithException:testCaseSelector:] + */ - (void)testRunDidCatchExceptionWithExceptionInfo:(SLTestCaseExceptionInfo *)exceptionInfo; @end diff --git a/Sources/Classes/SLTestCaseExceptionInfo.h b/Sources/Classes/SLTestCaseExceptionInfo.h index c40aa93..e8b143b 100644 --- a/Sources/Classes/SLTestCaseExceptionInfo.h +++ b/Sources/Classes/SLTestCaseExceptionInfo.h @@ -8,12 +8,35 @@ #import +/** + `SLTestCaseExceptionInfo` is a class whose objects encapsulate the state of a test run when an exception was caught. + */ + @interface SLTestCaseExceptionInfo : NSObject +/** + Initializes a new `SLTestExceptionInfo` object. + @param exception The underlying `NSException` that was thrown. (Required) + @param testCaseSelector The test case selector at the time of throw, or NULL if thrown outside of a test case. + @see -[SLTest testRunDidCatchExceptionWithExceptionInfo:] + @return A new `SLTestExceptioninfo` object. + */ + (instancetype)exceptionInfoWithException:(NSException *)exception testCaseSelector:(SEL)testCaseSelector; +/** + The test case selector at the time of throw, or NULL if thrown outside of a test case. + The default value is NULL. + */ @property (nonatomic, readonly, assign) SEL testCaseSelector; + +/** + The underlying exception that was thrown. + */ @property (nonatomic, readonly, strong) NSException *exception; + +/** + If the thrown exception was expected. + */ @property (nonatomic, readonly, getter = isExpected) BOOL expected; @end From 70847f8afabbae4940061f61a3df690bc05b6977 Mon Sep 17 00:00:00 2001 From: Jacob Relkin Date: Fri, 22 Aug 2014 18:36:18 -0700 Subject: [PATCH 3/5] Pull request feedback on test failure hook. Added tests and updated documentation. --- Sources/Classes/Internal/SLTestState.h | 37 +++++++++++ Sources/Classes/Internal/SLTestState.m | 31 +++++++++ Sources/Classes/SLTest.h | 11 ++-- Sources/Classes/SLTest.m | 87 +++++++++++++------------- Sources/Classes/SLTestController.m | 20 +++--- Sources/Classes/SLTestFailure.h | 53 ++++++++++++++++ Sources/Classes/SLTestFailure.m | 47 ++++++++++++++ Sources/Subliminal.h | 2 +- Subliminal.xcodeproj/project.pbxproj | 24 ++++--- Unit Tests/SLTestTests.m | 43 ++++++++++++- 10 files changed, 286 insertions(+), 69 deletions(-) create mode 100644 Sources/Classes/Internal/SLTestState.h create mode 100644 Sources/Classes/Internal/SLTestState.m create mode 100644 Sources/Classes/SLTestFailure.h create mode 100644 Sources/Classes/SLTestFailure.m diff --git a/Sources/Classes/Internal/SLTestState.h b/Sources/Classes/Internal/SLTestState.h new file mode 100644 index 0000000..669f4bf --- /dev/null +++ b/Sources/Classes/Internal/SLTestState.h @@ -0,0 +1,37 @@ +// +// SLTestState.h +// Subliminal +// +// Created by Jacob Relkin on 8/22/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import + +@class SLTestFailure; + +/** + SLTestState objects define the state of a Subliminal test or test case. + */ + +@interface SLTestState : NSObject + +/** + Denotes whether the test or test case failed + */ +@property (nonatomic, readonly) BOOL failed; + +/** + If the test or test case failure was expected. + */ +@property (nonatomic, readonly) BOOL failureWasExpected; + +/** + Sets the properties above, from the failure's description. + The value of `failureWasExpected` is determined by the first failure that was encountered. + + @param failure The test failure to record. + */ +- (void)recordFailure:(SLTestFailure *)failure; + +@end diff --git a/Sources/Classes/Internal/SLTestState.m b/Sources/Classes/Internal/SLTestState.m new file mode 100644 index 0000000..7c6cc90 --- /dev/null +++ b/Sources/Classes/Internal/SLTestState.m @@ -0,0 +1,31 @@ +// +// SLTestState.m +// Subliminal +// +// Created by Jacob Relkin on 8/22/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import "SLTestState.h" +#import "SLTestFailure.h" + +@interface SLTestState () + +@property (nonatomic, readwrite) BOOL failed; +@property (nonatomic, readwrite) BOOL failureWasExpected; + +@end + +@implementation SLTestState + +- (void)recordFailure:(SLTestFailure *)failure { + NSParameterAssert(failure); + + if (!self.failed) { + self.failureWasExpected = failure.isExpected; + } + + self.failed = YES; +} + +@end diff --git a/Sources/Classes/SLTest.h b/Sources/Classes/SLTest.h index 8f08257..9a0b242 100644 --- a/Sources/Classes/SLTest.h +++ b/Sources/Classes/SLTest.h @@ -25,7 +25,7 @@ #import "SLTestController+AppHooks.h" #import "SLStringUtilities.h" -@class SLTestCaseExceptionInfo; +@class SLTestFailure; /** `SLTest` is the abstract superclass of Subliminal integration tests. @@ -215,14 +215,13 @@ failedUnexpectedly:(NSUInteger *)numCasesFailedUnexpectedly; /** - If overridden, provides a hook for when an exception is caught within a test run. - - @param exceptionInfo An object which describes the exception that was thrown. Note: If this exception wasn't thrown within a test case, the `exceptionInfo`'s `testCaseSelector` will be NULL. + If overridden, provides a hook into test and test case failures. + @param failure An object which describes the failure. @warning This method will be invoked for each exception that is handled by the test framework. - @see -[SLTestCaseExceptionInfo exceptionInfoWithException:testCaseSelector:] + @see -[SLTestFailure failureWithException:phase:testCaseSelector:] */ -- (void)testRunDidCatchExceptionWithExceptionInfo:(SLTestCaseExceptionInfo *)exceptionInfo; +- (void)testDidEncounterFailure:(SLTestFailure *)failure; @end diff --git a/Sources/Classes/SLTest.m b/Sources/Classes/SLTest.m index f6e9f37..f8b048f 100644 --- a/Sources/Classes/SLTest.m +++ b/Sources/Classes/SLTest.m @@ -29,7 +29,8 @@ #import #import -#import "SLTestCaseExceptionInfo.h" +#import "SLTestFailure.h" +#import "SLTestState.h" // All exceptions thrown by SLTest must have names beginning with this prefix // so that `-[SLTest exceptionByAddingFileInfo:]` can determine whether to attach @@ -40,12 +41,6 @@ const NSTimeInterval SLWaitUntilTrueRetryDelay = 0.25; -@interface SLTest () - -@property (nonatomic, strong) SLTestCaseExceptionInfo *testCaseExceptionInfo; - -@end - @implementation SLTest static NSString *__lastKnownFilename; @@ -240,36 +235,38 @@ + (BOOL)testCaseWithSelectorSupportsCurrentPlatform:(SEL)testCaseSelector { return YES; } -- (void)testRunDidCatchException:(NSException *)exception testCaseSelector:(SEL)testCaseSelector { - self.testCaseExceptionInfo = [SLTestCaseExceptionInfo exceptionInfoWithException:exception testCaseSelector:testCaseSelector]; - - NSException *exceptionToLog = [self exceptionByAddingFileInfo:self.testCaseExceptionInfo.exception]; +- (void)reportFailureInPhase:(SLTestFailurePhase)phase toState:(SLTestState *)state exception:(NSException *)exception testCaseSelector:(SEL)testCaseSelector { + SLTestFailure *failure = [SLTestFailure failureWithException:exception phase:phase testCaseSelector:testCaseSelector]; + NSException *exceptionToLog = [self exceptionByAddingFileInfo:exception]; [[SLLogger sharedLogger] logException:exceptionToLog - expected:[self.testCaseExceptionInfo isExpected]]; - - [self testRunDidCatchExceptionWithExceptionInfo:self.testCaseExceptionInfo]; + expected:[failure isExpected]]; + [state recordFailure:failure]; + [self testDidEncounterFailure:failure]; } - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted failed:(NSUInteger *)numCasesFailed failedUnexpectedly:(NSUInteger *)numCasesFailedUnexpectedly { NSUInteger numberOfCasesExecuted = 0, numberOfCasesFailed = 0, numberOfCasesFailedUnexpectedly = 0; + SLTestState *testState = [SLTestState new]; - - BOOL testDidFailInSetUpOrTearDown = NO; @try { [self setUpTest]; } @catch (NSException *exception) { - [self testRunDidCatchException:exception testCaseSelector:NULL]; - testDidFailInSetUpOrTearDown = YES; + [self reportFailureInPhase:SLTestFailurePhaseTestSetup + toState:testState + exception:exception + testCaseSelector:NULL]; } // if setUpTest failed, skip the test cases - if (!testDidFailInSetUpOrTearDown) { + if (!testState.failed) { NSString *test = NSStringFromClass([self class]); for (NSString *testCaseName in [[self class] testCasesToRun]) { @autoreleasepool { + SLTestState *testCaseState = [SLTestState new]; + // all logs below use the focused name, so that the logs are consistent // with what's actually running [[SLLogger sharedLogger] logTest:test caseStart:testCaseName]; @@ -281,52 +278,55 @@ - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted // clear call site information, so at the least it won't be reused between test cases // (though we can't guarantee it won't be reused within a test case) [SLTest clearLastKnownCallSite]; - [self clearTestCaseExceptionState]; @try { [self setUpTestCaseWithSelector:unfocusedTestCaseSelector]; } @catch (NSException *exception) { - [self testRunDidCatchException:exception testCaseSelector:unfocusedTestCaseSelector]; + [self reportFailureInPhase:SLTestFailurePhaseTestCaseSetup + toState:testCaseState + exception:exception + testCaseSelector:unfocusedTestCaseSelector]; } // Only execute the test case if set-up succeeded. - if (!self.testCaseExceptionInfo) { + if (!testCaseState.failed) { @try { // We use objc_msgSend so that Clang won't complain about performSelector leaks // Make sure to send the actual test case selector ((void(*)(id, SEL))objc_msgSend)(self, NSSelectorFromString(testCaseName)); } @catch (NSException *exception) { - [self testRunDidCatchException:exception testCaseSelector:unfocusedTestCaseSelector]; + [self reportFailureInPhase:SLTestFailurePhaseTestCaseExecution + toState:testCaseState + exception:exception + testCaseSelector:unfocusedTestCaseSelector]; + } } - // If we didn't already fail, testCaseExceptionInfo will be nil - BOOL failureWasExpected = self.testCaseExceptionInfo.expected; - // Still perform tear-down even if set-up failed. // If the app is in an inconsistent state, then tear-down should fail. @try { [self tearDownTestCaseWithSelector:unfocusedTestCaseSelector]; } @catch (NSException *exception) { - BOOL succeededUntilTeardown = self.testCaseExceptionInfo == nil; - - [self testRunDidCatchException:exception testCaseSelector:unfocusedTestCaseSelector]; - - if (succeededUntilTeardown) { - failureWasExpected = self.testCaseExceptionInfo.expected; - } + [self reportFailureInPhase:SLTestFailurePhaseTestCaseTeardown + toState:testCaseState + exception:exception + testCaseSelector:unfocusedTestCaseSelector]; } - if (self.testCaseExceptionInfo) { - [[SLLogger sharedLogger] logTest:test caseFail:testCaseName expected:failureWasExpected]; + if (testCaseState.failed) { + [[SLLogger sharedLogger] logTest:test caseFail:testCaseName expected:testCaseState.failureWasExpected]; numberOfCasesFailed++; - if (!failureWasExpected) numberOfCasesFailedUnexpectedly++; + if (!testCaseState.failureWasExpected) { + numberOfCasesFailedUnexpectedly++; + } } else { [[SLLogger sharedLogger] logTest:test casePass:testCaseName]; } + numberOfCasesExecuted++; } } @@ -337,25 +337,23 @@ - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted [self tearDownTest]; } @catch (NSException *exception) { - [self testRunDidCatchException:exception testCaseSelector:NULL]; - testDidFailInSetUpOrTearDown = YES; + [self reportFailureInPhase:SLTestFailurePhaseTestTeardown + toState:testState + exception:exception + testCaseSelector:NULL]; } if (numCasesExecuted) *numCasesExecuted = numberOfCasesExecuted; if (numCasesFailed) *numCasesFailed = numberOfCasesFailed; if (numCasesFailedUnexpectedly) *numCasesFailedUnexpectedly = numberOfCasesFailedUnexpectedly; - return !testDidFailInSetUpOrTearDown; + return !testState.failed; } - (void)wait:(NSTimeInterval)interval { [NSThread sleepForTimeInterval:interval]; } -- (void)clearTestCaseExceptionState { - self.testCaseExceptionInfo = nil; -} - + (void)recordLastKnownFile:(const char *)filename line:(int)lineNumber { __lastKnownFilename = [@(filename) lastPathComponent]; __lastKnownLineNumber = lineNumber; @@ -389,7 +387,8 @@ - (NSException *)exceptionByAddingFileInfo:(NSException *)exception { } // Abstract -- (void)testRunDidCatchExceptionWithExceptionInfo:(SLTestCaseExceptionInfo *)exceptionInfo {} +- (void)testDidEncounterFailure:(SLTestFailure *)failure {} + @end diff --git a/Sources/Classes/SLTestController.m b/Sources/Classes/SLTestController.m index af8c71e..723df90 100644 --- a/Sources/Classes/SLTestController.m +++ b/Sources/Classes/SLTestController.m @@ -367,16 +367,20 @@ - (void)runTests:(NSSet *)tests usingSeed:(unsigned int)seed withCompletionBlock BOOL testDidFinish = [test runAndReportNumExecuted:&numCasesExecuted failed:&numCasesFailed failedUnexpectedly:&numCasesFailedUnexpectedly]; - if (testDidFinish) { - [[SLLogger sharedLogger] logTestFinish:testName - withNumCasesExecuted:numCasesExecuted - numCasesFailed:numCasesFailed - numCasesFailedUnexpectedly:numCasesFailedUnexpectedly]; - if (numCasesFailed > 0) _numTestsFailed++; - } else { - [[SLLogger sharedLogger] logTestAbort:testName]; + + [[SLLogger sharedLogger] logTestFinish:testName + withNumCasesExecuted:numCasesExecuted + numCasesFailed:numCasesFailed + numCasesFailedUnexpectedly:numCasesFailedUnexpectedly]; + + if (!testDidFinish || numCasesFailed) { + if (!testDidFinish) { + [[SLLogger sharedLogger] logTestAbort:testName]; + } + _numTestsFailed++; } + _numTestsExecuted++; } } diff --git a/Sources/Classes/SLTestFailure.h b/Sources/Classes/SLTestFailure.h new file mode 100644 index 0000000..6ef10b1 --- /dev/null +++ b/Sources/Classes/SLTestFailure.h @@ -0,0 +1,53 @@ +// +// SLTestCaseFailure.h +// Subliminal +// +// Created by Jacob Relkin on 6/20/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import + +typedef NS_ENUM(NSUInteger, SLTestFailurePhase) { + SLTestFailurePhaseTestSetup, + SLTestFailurePhaseTestCaseSetup, + SLTestFailurePhaseTestCaseExecution, + SLTestFailurePhaseTestCaseTeardown, + SLTestFailurePhaseTestTeardown +}; + +/** + SLTestFailure objects hold failure information for Subliminal test and test cases. + */ + +@interface SLTestFailure : NSObject + +/** + @param exception The exception that was thrown to cause this failure. + @param phase The phase in the test lifecycle in which the failure happened. + @param testCaseSelector The failed test case's selector. (can be NULL) + @return A new SLTestFailure object. + */ ++ (instancetype)failureWithException:(NSException *)exception phase:(SLTestFailurePhase)phase testCaseSelector:(SEL)testCaseSelector; + +/** + The phase in the test lifecycle in which the failure happened. + */ +@property (nonatomic, readonly, assign) SLTestFailurePhase phase; + +/** + The failed test case's selector. (can be NULL) + */ +@property (nonatomic, readonly, assign) SEL testCaseSelector; + +/** + The exception that was thrown to cause this failure. + */ +@property (nonatomic, readonly, strong) NSException *exception; + +/** + If the failure was expected. + */ +@property (nonatomic, readonly, getter = isExpected) BOOL expected; + +@end diff --git a/Sources/Classes/SLTestFailure.m b/Sources/Classes/SLTestFailure.m new file mode 100644 index 0000000..26a6068 --- /dev/null +++ b/Sources/Classes/SLTestFailure.m @@ -0,0 +1,47 @@ +// +// SLTestCaseFailure.m +// Subliminal +// +// Created by Jacob Relkin on 6/20/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import "SLTestFailure.h" +#import "SLTest.h" + +@interface SLTestFailure () + +@property (nonatomic, readwrite, strong) NSException *exception; +@property (nonatomic, readwrite, assign) SEL testCaseSelector; +@property (nonatomic, readwrite, assign) SLTestFailurePhase phase; + +@end + +@implementation SLTestFailure + ++ (instancetype)failureWithException:(NSException *)exception phase:(SLTestFailurePhase)phase testCaseSelector:(SEL)testCaseSelector { + SLTestFailure *failure = [self new]; + failure.phase = phase; + failure.exception = exception; + failure.testCaseSelector = testCaseSelector; + return failure; +} + +- (BOOL)isEqual:(id)object { + if (![object isKindOfClass:[SLTestFailure class]]) { + return NO; + } + + SLTestFailure *otherObject = object; + return (otherObject.phase == self.phase && + otherObject.testCaseSelector == self.testCaseSelector && + otherObject.isExpected == self.isExpected && + [otherObject.exception.name isEqualToString:self.exception.name] && + [otherObject.exception.reason isEqualToString:self.exception.reason]); +} + +- (BOOL)isExpected { + return [self.exception.name isEqualToString:SLTestAssertionFailedException]; +} + +@end diff --git a/Sources/Subliminal.h b/Sources/Subliminal.h index 2e83439..3e6ec92 100644 --- a/Sources/Subliminal.h +++ b/Sources/Subliminal.h @@ -23,7 +23,7 @@ #import "SLTestController.h" #import "SLTestController+AppHooks.h" #import "SLTest.h" -#import "SLTestCaseExceptionInfo.h" +#import "SLTestFailure.h" #import "SLDevice.h" #import "SLElement.h" diff --git a/Subliminal.xcodeproj/project.pbxproj b/Subliminal.xcodeproj/project.pbxproj index 6aa353b..1a8535b 100644 --- a/Subliminal.xcodeproj/project.pbxproj +++ b/Subliminal.xcodeproj/project.pbxproj @@ -58,8 +58,10 @@ 627B5FD4194FACA50059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 627B5FD0194FA9D70059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard */; }; 62E7A633193EF84C00CB11AB /* SLStaticText.h in Headers */ = {isa = PBXBuildFile; fileRef = 62E7A631193EF84C00CB11AB /* SLStaticText.h */; settings = {ATTRIBUTES = (Public, ); }; }; 62E7A634193EF84C00CB11AB /* SLStaticText.m in Sources */ = {isa = PBXBuildFile; fileRef = 62E7A632193EF84C00CB11AB /* SLStaticText.m */; }; - C7FD55131954CAD700E99FB7 /* SLTestCaseExceptionInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = C7FD55111954CAD700E99FB7 /* SLTestCaseExceptionInfo.h */; settings = {ATTRIBUTES = (Public, ); }; }; - C7FD55141954CAD700E99FB7 /* SLTestCaseExceptionInfo.m in Sources */ = {isa = PBXBuildFile; fileRef = C7FD55121954CAD700E99FB7 /* SLTestCaseExceptionInfo.m */; }; + C7C0445019A7CE14001F82B9 /* SLTestState.h in Headers */ = {isa = PBXBuildFile; fileRef = C7C0444E19A7CE14001F82B9 /* SLTestState.h */; }; + C7C0445119A7CE14001F82B9 /* SLTestState.m in Sources */ = {isa = PBXBuildFile; fileRef = C7C0444F19A7CE14001F82B9 /* SLTestState.m */; }; + C7FD55131954CAD700E99FB7 /* SLTestFailure.h in Headers */ = {isa = PBXBuildFile; fileRef = C7FD55111954CAD700E99FB7 /* SLTestFailure.h */; settings = {ATTRIBUTES = (Public, ); }; }; + C7FD55141954CAD700E99FB7 /* SLTestFailure.m in Sources */ = {isa = PBXBuildFile; fileRef = C7FD55121954CAD700E99FB7 /* SLTestFailure.m */; }; CA75E78216697A1200D57E92 /* SLDevice.h in Headers */ = {isa = PBXBuildFile; fileRef = CA75E78016697A1200D57E92 /* SLDevice.h */; settings = {ATTRIBUTES = (Public, ); }; }; CA75E78516697C0000D57E92 /* SLDevice.m in Sources */ = {isa = PBXBuildFile; fileRef = CA75E78116697A1200D57E92 /* SLDevice.m */; }; CAC388051641CD7500F995F9 /* SLStringUtilities.h in Headers */ = {isa = PBXBuildFile; fileRef = CAC388031641CD7500F995F9 /* SLStringUtilities.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -311,8 +313,10 @@ 627B5FD0194FA9D70059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = SLTestMatchingElementsWithinTableHeaderView.storyboard; sourceTree = ""; }; 62E7A631193EF84C00CB11AB /* SLStaticText.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLStaticText.h; sourceTree = ""; }; 62E7A632193EF84C00CB11AB /* SLStaticText.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLStaticText.m; sourceTree = ""; }; - C7FD55111954CAD700E99FB7 /* SLTestCaseExceptionInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLTestCaseExceptionInfo.h; sourceTree = ""; }; - C7FD55121954CAD700E99FB7 /* SLTestCaseExceptionInfo.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLTestCaseExceptionInfo.m; sourceTree = ""; }; + C7C0444E19A7CE14001F82B9 /* SLTestState.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLTestState.h; sourceTree = ""; }; + C7C0444F19A7CE14001F82B9 /* SLTestState.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLTestState.m; sourceTree = ""; }; + C7FD55111954CAD700E99FB7 /* SLTestFailure.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SLTestFailure.h; path = ../SLTestFailure.h; sourceTree = ""; }; + C7FD55121954CAD700E99FB7 /* SLTestFailure.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = SLTestFailure.m; path = ../SLTestFailure.m; sourceTree = ""; }; CA75E78016697A1200D57E92 /* SLDevice.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLDevice.h; sourceTree = ""; }; CA75E78116697A1200D57E92 /* SLDevice.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLDevice.m; sourceTree = ""; }; CAC388031641CD7500F995F9 /* SLStringUtilities.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLStringUtilities.h; sourceTree = ""; }; @@ -602,6 +606,10 @@ F05C51E4171C8AE000A381BC /* SLMainThreadRef.m */, F02DF30617EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.h */, F02DF30717EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.m */, + C7FD55111954CAD700E99FB7 /* SLTestFailure.h */, + C7FD55121954CAD700E99FB7 /* SLTestFailure.m */, + C7C0444E19A7CE14001F82B9 /* SLTestState.h */, + C7C0444F19A7CE14001F82B9 /* SLTestState.m */, ); path = Internal; sourceTree = ""; @@ -775,8 +783,6 @@ F0695DE3160138DF000B05D0 /* SLTestController.m */, F0271AFD162E0B950098F5F2 /* SLTestController+AppHooks.h */, F0271AFE162E0B950098F5F2 /* SLTestController+AppHooks.m */, - C7FD55111954CAD700E99FB7 /* SLTestCaseExceptionInfo.h */, - C7FD55121954CAD700E99FB7 /* SLTestCaseExceptionInfo.m */, F0695DE0160138DF000B05D0 /* SLTest.h */, F0695DE1160138DF000B05D0 /* SLTest.m */, F0695DD6160138DF000B05D0 /* UIAutomation */, @@ -1109,6 +1115,7 @@ F0C07A531704011400C93F93 /* SLKeyboard.h in Headers */, F0C07A57170401E500C93F93 /* SLWebView.h in Headers */, F05C4F90171406EF00A381BC /* SLTerminal+ConvenienceFunctions.h in Headers */, + C7C0445019A7CE14001F82B9 /* SLTestState.h in Headers */, F05C51E5171C8AE000A381BC /* SLMainThreadRef.h in Headers */, F0A04E1D1749F70F002C7520 /* SLElement.h in Headers */, 2CE9AA4C17E3A747007EF0B5 /* SLSwitch.h in Headers */, @@ -1116,7 +1123,7 @@ F02DF30817EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.h in Headers */, F00800CE174C1C64001927AC /* SLPopover.h in Headers */, 50F3E18C1783A5CB00C6BD1B /* SLGeometry.h in Headers */, - C7FD55131954CAD700E99FB7 /* SLTestCaseExceptionInfo.h in Headers */, + C7FD55131954CAD700E99FB7 /* SLTestFailure.h in Headers */, F043469F175ACE3A00D91F7F /* NSObject+SLAccessibilityDescription.h in Headers */, F04346AF175AD63E00D91F7F /* SLAccessibilityPath.h in Headers */, F04346A7175AD10200D91F7F /* NSObject+SLVisibility.h in Headers */, @@ -1384,13 +1391,14 @@ 62E7A634193EF84C00CB11AB /* SLStaticText.m in Sources */, 50F3E18E1783A60100C6BD1B /* SLGeometry.m in Sources */, F0695DE4160138DF000B05D0 /* SLUIAElement.m in Sources */, + C7C0445119A7CE14001F82B9 /* SLTestState.m in Sources */, F0695DE5160138DF000B05D0 /* SLLogger.m in Sources */, F0695DE7160138DF000B05D0 /* SLTerminal.m in Sources */, F0695DE8160138DF000B05D0 /* SLTest.m in Sources */, 622DA0BB194E2CB900EFFE05 /* SLDatePicker.m in Sources */, F0695DE9160138DF000B05D0 /* SLTestController.m in Sources */, F0271B00162E0B950098F5F2 /* SLTestController+AppHooks.m in Sources */, - C7FD55141954CAD700E99FB7 /* SLTestCaseExceptionInfo.m in Sources */, + C7FD55141954CAD700E99FB7 /* SLTestFailure.m in Sources */, F02DF30917EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.m in Sources */, CAC388061641CD7500F995F9 /* SLStringUtilities.m in Sources */, 622DA090194AF1E200EFFE05 /* SLPickerView.m in Sources */, diff --git a/Unit Tests/SLTestTests.m b/Unit Tests/SLTestTests.m index 6e8643e..73de15d 100644 --- a/Unit Tests/SLTestTests.m +++ b/Unit Tests/SLTestTests.m @@ -407,7 +407,8 @@ - (void)testAllTestCasesRunByDefault { [[testMock expect] testOne]; [[testMock expect] testTwo]; [[testMock expect] testThree]; - + [[testMock reject] testDidEncounterFailure:OCMOCK_ANY]; + SLRunTestsAndWaitUntilFinished([NSSet setWithObject:testWithSomeTestCasesTest], nil); STAssertNoThrow([testMock verify], @"Test cases did not run as expected."); } @@ -427,6 +428,15 @@ - (void)testInvalidTestCasesAreNotRun { STAssertNoThrow([testMock verify], @"Invalid test cases were unexpectedly run."); } +- (void)testSuccesfulTestDoesntInvokeTestFailureHook { + Class testWithSomeTestCasesTest = [TestWithSomeTestCases class]; + id testMock = [OCMockObject partialMockForClass:testWithSomeTestCasesTest]; + [[testMock reject] testDidEncounterFailure:OCMOCK_ANY]; + + SLRunTestsAndWaitUntilFinished([NSSet setWithObject:testWithSomeTestCasesTest], nil); + SLAssertNoThrow([testMock verify], @"Test failure hook was unexpectedly invoked."); +} + // this test verifies the complete order in which testing normally executes, // but is mostly for illustration--it makes too many assertions // traditional "unit" tests follow @@ -543,22 +553,31 @@ - (void)runWithTestFailingInTestSetupOrTeardownToTestAnErrorAndTestAbortAreLogge // If either setup or teardown fails... NSException *exception; + SLTestFailure *failureToExpect = nil; + if (failInSetUp) { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test setup failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestSetup testCaseSelector:NULL]; + [[[failingTestMock expect] andThrow:exception] setUpTest]; } else { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test teardown failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestTeardown testCaseSelector:NULL]; + [[[failingTestMock expect] andThrow:exception] tearDownTest]; } // ...the test controller logs an error... [[_loggerMock expect] logError:[OCMArg any]]; - // ...and the test controller logs the test as aborted (rather than finishing)... + // ...the test encounters the expected failure... + [[failingTestMock expect] testDidEncounterFailure:failureToExpect]; + + // ...and the test controller logs the test as aborte/d (rather than finishing)... [[_loggerMock expect] logTestAbort:NSStringFromClass(failingTestClass)]; // ...and the test controller logs testing as finishing with one test executed, one test failing. @@ -660,27 +679,37 @@ - (void)runWithTestFailingInTestCaseSetupOrTeardownToTestAnErrorAndTestCaseFailA Class failingTestClass = [TestWithSomeTestCases class]; SEL failingTestCase = @selector(testOne); id failingTestMock = [OCMockObject partialMockForClass:failingTestClass]; + [failingTestMock setExpectationOrderMatters:YES]; OCMExpectationSequencer *failingTestSequencer = [OCMExpectationSequencer sequencerWithMocks:@[ failingTestMock, _loggerMock ]]; // *** Begin expected test run // If either test case setup or teardown fails... NSException *exception; + SLTestFailure *failureToExpect; + if (failInSetUp) { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test case setup failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestCaseSetup testCaseSelector:failingTestCase]; + [[[failingTestMock expect] andThrow:exception] setUpTestCaseWithSelector:failingTestCase]; } else { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test case teardown failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestCaseTeardown testCaseSelector:failingTestCase]; + [[[failingTestMock expect] andThrow:exception] tearDownTestCaseWithSelector:failingTestCase]; } // ...the test catches the exception and logs an error... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test encounters the expected failure... + [[failingTestMock expect] testDidEncounterFailure:failureToExpect]; + // ...and the test controller reports the test finishing with one test case having failed... // (and that failure was "expected" because it was due to an assertion failing) // (these values will need to be updated if the test class' definition changes) @@ -747,6 +776,9 @@ - (void)runWithTestFailingInTestCaseSetupWithExpectedFailure:(BOOL)expectedFailu // ...the test catches and logs the exception... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test calls -testDidEncounterFailure: with the expected failure. + [[failingTestMock expect] testDidEncounterFailure:[SLTestFailure failureWithException:setUpException phase:SLTestFailurePhaseTestCaseSetup testCaseSelector:failingTestCase]]; + // ...and then if test case teardown fails... NSException *tearDownException = exceptionWithReason(expectedFailureInTeardown, @"Test case teardown failed."); [[[failingTestMock expect] andThrow:tearDownException] tearDownTestCaseWithSelector:failingTestCase]; @@ -754,6 +786,9 @@ - (void)runWithTestFailingInTestCaseSetupWithExpectedFailure:(BOOL)expectedFailu // ...the test again catches and logs the exception... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test again calls -testDidEncounterFailure: with the expected failure. + [[failingTestMock expect] testDidEncounterFailure:[SLTestFailure failureWithException:tearDownException phase:SLTestFailurePhaseTestCaseTeardown testCaseSelector:failingTestCase]]; + // ...but when the test logs the test case as failing, // that failure is reported as "expected" or not depending on the first exception (setup) thrown... [[_loggerMock expect] logTest:NSStringFromClass(failingTestClass) @@ -894,11 +929,15 @@ - (void)runWithFailureInTestCaseDueToAssertionException:(BOOL)failWithAssertionE reason:@"Test case failed because element was not tappable." userInfo:nil]; } + [[[failingTestMock expect] andThrow:exception] testOne]; // ...the test catches the exception and logs an error... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test encounters a failure with the correct state... + [[failingTestMock expect] testDidEncounterFailure:[SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestCaseExecution testCaseSelector:failingTestCase]]; + // ...and logs the test case failing... [[_loggerMock expect] logTest:NSStringFromClass(failingTestClass) caseFail:NSStringFromSelector(failingTestCase) From d39a6a38db6741b3aa67533bbf5442ad0b1531c1 Mon Sep 17 00:00:00 2001 From: Jacob Relkin Date: Tue, 26 Aug 2014 11:09:23 -0700 Subject: [PATCH 4/5] Reverted changes to SLTestController. --- Sources/Classes/SLTestController.m | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Sources/Classes/SLTestController.m b/Sources/Classes/SLTestController.m index 0589c86..0ecd504 100644 --- a/Sources/Classes/SLTestController.m +++ b/Sources/Classes/SLTestController.m @@ -375,20 +375,16 @@ - (void)runTests:(NSSet *)tests usingSeed:(unsigned int)seed withCompletionBlock BOOL testDidFinish = [test runAndReportNumExecuted:&numCasesExecuted failed:&numCasesFailed failedUnexpectedly:&numCasesFailedUnexpectedly]; - - [[SLLogger sharedLogger] logTestFinish:testName - withNumCasesExecuted:numCasesExecuted - numCasesFailed:numCasesFailed - numCasesFailedUnexpectedly:numCasesFailedUnexpectedly]; - - if (!testDidFinish || numCasesFailed) { - if (!testDidFinish) { - [[SLLogger sharedLogger] logTestAbort:testName]; - } - + if (testDidFinish) { + [[SLLogger sharedLogger] logTestFinish:testName + withNumCasesExecuted:numCasesExecuted + numCasesFailed:numCasesFailed + numCasesFailedUnexpectedly:numCasesFailedUnexpectedly]; + if (numCasesFailed > 0) _numTestsFailed++; + } else { + [[SLLogger sharedLogger] logTestAbort:testName]; _numTestsFailed++; } - _numTestsExecuted++; } } From 625ed53b9e100ff8c6cc29a58970bdc709481862 Mon Sep 17 00:00:00 2001 From: Jacob Relkin Date: Tue, 26 Aug 2014 11:10:44 -0700 Subject: [PATCH 5/5] Deleted unused files. --- Sources/Classes/SLTestCaseExceptionInfo.h | 42 ----------------------- Sources/Classes/SLTestCaseExceptionInfo.m | 32 ----------------- 2 files changed, 74 deletions(-) delete mode 100644 Sources/Classes/SLTestCaseExceptionInfo.h delete mode 100644 Sources/Classes/SLTestCaseExceptionInfo.m diff --git a/Sources/Classes/SLTestCaseExceptionInfo.h b/Sources/Classes/SLTestCaseExceptionInfo.h deleted file mode 100644 index e8b143b..0000000 --- a/Sources/Classes/SLTestCaseExceptionInfo.h +++ /dev/null @@ -1,42 +0,0 @@ -// -// SLTestCaseExceptionInfo.h -// Subliminal -// -// Created by Jacob Relkin on 6/20/14. -// Copyright (c) 2014 Inkling. All rights reserved. -// - -#import - -/** - `SLTestCaseExceptionInfo` is a class whose objects encapsulate the state of a test run when an exception was caught. - */ - -@interface SLTestCaseExceptionInfo : NSObject - -/** - Initializes a new `SLTestExceptionInfo` object. - @param exception The underlying `NSException` that was thrown. (Required) - @param testCaseSelector The test case selector at the time of throw, or NULL if thrown outside of a test case. - @see -[SLTest testRunDidCatchExceptionWithExceptionInfo:] - @return A new `SLTestExceptioninfo` object. - */ -+ (instancetype)exceptionInfoWithException:(NSException *)exception testCaseSelector:(SEL)testCaseSelector; - -/** - The test case selector at the time of throw, or NULL if thrown outside of a test case. - The default value is NULL. - */ -@property (nonatomic, readonly, assign) SEL testCaseSelector; - -/** - The underlying exception that was thrown. - */ -@property (nonatomic, readonly, strong) NSException *exception; - -/** - If the thrown exception was expected. - */ -@property (nonatomic, readonly, getter = isExpected) BOOL expected; - -@end diff --git a/Sources/Classes/SLTestCaseExceptionInfo.m b/Sources/Classes/SLTestCaseExceptionInfo.m deleted file mode 100644 index 13b2d66..0000000 --- a/Sources/Classes/SLTestCaseExceptionInfo.m +++ /dev/null @@ -1,32 +0,0 @@ -// -// SLTestCaseExceptionInfo.m -// Subliminal -// -// Created by Jacob Relkin on 6/20/14. -// Copyright (c) 2014 Inkling. All rights reserved. -// - -#import "SLTestCaseExceptionInfo.h" -#import "SLTest.h" - -@interface SLTestCaseExceptionInfo () - -@property (nonatomic, readwrite, strong) NSException *exception; -@property (nonatomic, readwrite, assign) SEL testCaseSelector; - -@end - -@implementation SLTestCaseExceptionInfo - -+ (instancetype)exceptionInfoWithException:(NSException *)exception testCaseSelector:(SEL)testCaseSelector { - SLTestCaseExceptionInfo *info = [self new]; - info.exception = exception; - info.testCaseSelector = testCaseSelector; - return info; -} - -- (BOOL)isExpected { - return [self.exception.name isEqualToString:SLTestAssertionFailedException]; -} - -@end