From abf88844e6dc6bdd32b3e03377408d2b83cdf181 Mon Sep 17 00:00:00 2001 From: Sangwoo Im Date: Sat, 26 Dec 2020 12:11:31 -0500 Subject: [PATCH 1/2] An option added to keep only the latest test result for each test case are kept. So, retry attempts will end up with one failure or pass result. --- bluepill/src/BPReportCollector.h | 4 +++- bluepill/src/BPReportCollector.m | 32 ++++++++++++++++++++------------ bluepill/src/main.m | 3 ++- bp/src/BPConfiguration.h | 1 + bp/src/BPConfiguration.m | 2 ++ 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/bluepill/src/BPReportCollector.h b/bluepill/src/BPReportCollector.h index b28970af..428f4525 100644 --- a/bluepill/src/BPReportCollector.h +++ b/bluepill/src/BPReportCollector.h @@ -15,9 +15,11 @@ * @discussion collect xml reports from the reportsPath(recursive) and output a finalized report at finalReportPath * @param reportsPath parent path to the reports * @param finalReportsDir the directory where to save the final reports + * @param latestResultOnly If true, keeps only the latest result in the final report. */ + (void)collectReportsFromPath:(NSString *)reportsPath deleteCollected:(BOOL)deleteCollected - withOutputAtDir:(NSString *)finalReportsDir; + withOutputAtDir:(NSString *)finalReportsDir + keepingOnlyLatestResult:(BOOL)latestResultOnly; @end diff --git a/bluepill/src/BPReportCollector.m b/bluepill/src/BPReportCollector.m index 2fa3aa1d..d8616d06 100644 --- a/bluepill/src/BPReportCollector.m +++ b/bluepill/src/BPReportCollector.m @@ -33,7 +33,8 @@ @implementation BPReportCollector + (void)collectReportsFromPath:(NSString *)reportsPath deleteCollected:(BOOL)deleteCollected - withOutputAtDir:(NSString *)finalReportsDir { + withOutputAtDir:(NSString *)finalReportsDir + keepingOnlyLatestResult:(BOOL)latestResultOnly { NSFileManager *fileManager = [NSFileManager defaultManager]; NSString *finalReportPath = [finalReportsDir stringByAppendingPathComponent:@"TEST-FinalReport.xml"]; @@ -97,7 +98,8 @@ + (void)collectReportsFromPath:(NSString *)reportsPath } NSXMLDocument *jUnitReport = [self collateReports:reports andDeleteCollated:deleteCollected - withOutputAt:finalReportPath]; + withOutputAt:finalReportPath + keepingOnlyLatestResult:latestResultOnly]; // write a html report [[BPHTMLReportWriter new] writeHTMLReportWithJUnitReport:jUnitReport @@ -106,7 +108,8 @@ + (void)collectReportsFromPath:(NSString *)reportsPath + (NSXMLDocument *)collateReports:(NSMutableArray *)reports andDeleteCollated:(BOOL)deleteCollated - withOutputAt:(NSString *)finalReportPath { + withOutputAt:(NSString *)finalReportPath + keepingOnlyLatestResult: (BOOL)latestResultOnly { NSError *err; // sort them by modification date, newer reports trump old reports @@ -134,7 +137,7 @@ + (NSXMLDocument *)collateReports:(NSMutableArray *)reports [BPUtils printInfo:DEBUGINFO withString:@"TestSuite: %@", testSuiteName]; NSXMLElement *targetTestSuite = [[targetReport nodesForXPath:[NSString stringWithFormat:@"//testsuite[@name='%@']", testSuiteName] error:nil] firstObject]; if (targetTestSuite) { - [self collateTestSuite:testSuite into:targetTestSuite]; + [self collateTestSuite:testSuite into:targetTestSuite keepingOnlyLatestResult:latestResultOnly]; } else { NSXMLElement *testSuites = [[targetReport nodesForXPath:@"/testsuites" error:nil] firstObject]; [testSuites addChild:[testSuite copy]]; @@ -175,28 +178,28 @@ + (NSXMLDocument *)newEmptyXMLDocumentWithName:(NSString *)name { return doc; } -+ (void) collateTestSuite:(NSXMLElement *)testSuite into:(NSXMLElement *)targetTestSuite { ++ (void) collateTestSuite:(NSXMLElement *)testSuite into:(NSXMLElement *)targetTestSuite keepingOnlyLatestResult:(BOOL)latestResultOnly { [BPUtils printInfo:DEBUGINFO withString:@"Collating '%@' into '%@'", [[testSuite attributeForName:@"name"] stringValue], [[targetTestSuite attributeForName:@"name"] stringValue]]; // testsuite elements must have either all testsuite children or all testcase children NSString *firstChild = [[[testSuite children] firstObject] name]; if ([firstChild isEqualToString:@"testsuite"]) { - [self collateTestSuiteTestSuites:testSuite into:targetTestSuite]; + [self collateTestSuiteTestSuites:testSuite into:targetTestSuite keepingOnlyLatestResult:latestResultOnly]; } else if ([firstChild isEqualToString:@"testcase"]) { - [self collateTestSuiteTestCases:testSuite into:targetTestSuite]; + [self collateTestSuiteTestCases:testSuite into:targetTestSuite keepingOnlyLatestResult:latestResultOnly]; } else if (firstChild) { // empty [BPUtils printInfo:ERROR withString:@"Unknown child node in '%@': %@", [[testSuite attributeForName:@"name"] stringValue], firstChild]; assert(false); } } -+ (void)collateTestSuiteTestSuites:(NSXMLElement *)testSuite into:(NSXMLElement *)targetTestSuite { ++ (void)collateTestSuiteTestSuites:(NSXMLElement *)testSuite into:(NSXMLElement *)targetTestSuite keepingOnlyLatestResult:(BOOL)latestResultOnly { [BPUtils printInfo:DEBUGINFO withString:@"Collating TestSuites under: %@", [[testSuite attributeForName:@"name"]stringValue]]; for (NSXMLElement *ts in [testSuite nodesForXPath:@"testsuite" error:nil]) { NSXMLElement *tts = [[targetTestSuite nodesForXPath:[NSString stringWithFormat:@"testsuite[@name='%@']", [[ts attributeForName:@"name"] stringValue]] error:nil] firstObject]; if (tts) { [BPUtils printInfo:DEBUGINFO withString:@"match: %@", [[tts attributeForName:@"name"] stringValue]]; - [self collateTestSuiteTestCases:ts into:tts]; + [self collateTestSuiteTestCases:ts into:tts keepingOnlyLatestResult:latestResultOnly]; } else { [BPUtils printInfo:DEBUGINFO withString:@"inserting: %@", [[ts attributeForName:@"name"] stringValue]]; [targetTestSuite addChild:[ts copy]]; @@ -204,7 +207,7 @@ + (void)collateTestSuiteTestSuites:(NSXMLElement *)testSuite into:(NSXMLElement } } -+ (void)collateTestSuiteTestCases:(NSXMLElement *)testSuite into:(NSXMLElement *)targetTestSuite { ++ (void)collateTestSuiteTestCases:(NSXMLElement *)testSuite into:(NSXMLElement *)targetTestSuite keepingOnlyLatestResult:(BOOL)latestResultOnly { [BPUtils printInfo:DEBUGINFO withString:@"Collating TestCases under: %@", [[testSuite attributeForName:@"name"] stringValue]]; int testCaseCount = 0; for (NSXMLElement *testCase in [testSuite nodesForXPath:@"testcase" error:nil]) { @@ -212,10 +215,15 @@ + (void)collateTestSuiteTestCases:(NSXMLElement *)testSuite into:(NSXMLElement * NSString *name = [[testCase attributeForName:@"name"] stringValue]; NSXMLElement *targetTestCase = [[targetTestSuite nodesForXPath:[NSString stringWithFormat:@"testcase[@name='%@' and @classname='%@']", name, className] error:nil] firstObject]; - NSXMLElement *parent; + NSXMLElement *parent = (NSXMLElement *)[targetTestCase parent]; + + if (latestResultOnly) { + [parent removeChildAtIndex:[targetTestCase index]]; + testCaseCount--; + } + if (targetTestCase) { [BPUtils printInfo:DEBUGINFO withString:@"testcase match: %@", [[targetTestCase attributeForName:@"name"] stringValue]]; - parent = (NSXMLElement *)[targetTestCase parent]; // append the latest result at the end NSUInteger insertIndex; for (insertIndex = [targetTestCase index]; insertIndex < [[parent children] count]; insertIndex++) { diff --git a/bluepill/src/main.m b/bluepill/src/main.m index 2ba283de..35dde408 100644 --- a/bluepill/src/main.m +++ b/bluepill/src/main.m @@ -125,7 +125,8 @@ int main(int argc, char * argv[]) { // collect all the reports [BPReportCollector collectReportsFromPath:config.outputDirectory deleteCollected:(!config.keepIndividualTestReports) - withOutputAtDir:config.outputDirectory]; + withOutputAtDir:config.outputDirectory + keepingOnlyLatestResult:config.keepOnlyLatestTestResults]; } exit(rc); } diff --git a/bp/src/BPConfiguration.h b/bp/src/BPConfiguration.h index 29ebc422..aee75e73 100644 --- a/bp/src/BPConfiguration.h +++ b/bp/src/BPConfiguration.h @@ -105,6 +105,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic, strong) NSNumber *launchTimeout; @property (nonatomic, strong) NSNumber *deleteTimeout; @property (nonatomic) BOOL keepIndividualTestReports; +@property (nonatomic) BOOL keepOnlyLatestTestResults; @property (nonatomic, strong) NSArray *commandLineArguments; // command line arguments for the app @property (nonatomic, strong) NSDictionary *environmentVariables; diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index f6edddbd..1746eb11 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -110,6 +110,8 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { "Enable verbose logging"}, {'k', "keep-individual-test-reports", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepIndividualTestReports", "Keep individual test reports, in addition to the aggregated final report"}, + {'K', "keep-only-latest-test-results", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepOnlyLatestTestResults", + "Keeps only the latest test result for each test case in the final report. Hence, the passed tests with retry attempts won't show their failed attempts."}, // options without short-options {349, "additional-unit-xctests", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_LIST | BP_PATH, "additionalUnitTestBundles", From ca6d71f54a04a9f6dcf64949163f24fb35ce5008 Mon Sep 17 00:00:00 2001 From: Mikael Teklehaimanot Date: Tue, 16 Feb 2021 13:40:57 -0800 Subject: [PATCH 2/2] update configuration constants for requested feature --- bp/src/BPConfiguration.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index a2ee680b..3073d7d1 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -110,7 +110,7 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { "Enable verbose logging"}, {'k', "keep-individual-test-reports", BLUEPILL_BINARY | BP_BINARY, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepIndividualTestReports", "Keep individual test reports, in addition to the aggregated final report"}, - {'K', "keep-only-latest-test-results", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepOnlyLatestTestResults", + {'K', "keep-only-latest-test-results", BLUEPILL_BINARY | BP_BINARY, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepOnlyLatestTestResults", "Keeps only the latest test result for each test case in the final report. Hence, the passed tests with retry attempts won't show their failed attempts."}, // options without short-options