Preserve the order in which tests were run (#326)

In the JUnit report we were not preserving the order of the tests. Now we do.
MobileNativeFoundation · Apr 15, 2019 · 2d5e8c5 · 2d5e8c5
1 parent 6db475e
commit 2d5e8c5
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 12 deletions.
diff --git a/Bluepill-runner/Bluepill-runner/BPReportCollector.m b/Bluepill-runner/Bluepill-runner/BPReportCollector.m
@@ -209,8 +209,17 @@ + (void)collateTestSuiteTestCases:(NSXMLElement *)testSuite into:(NSXMLElement *
         if (targetTestCase) {
             [BPUtils printInfo:DEBUGINFO withString:@"testcase match: %@", [[targetTestCase attributeForName:@"name"] stringValue]];
             parent = (NSXMLElement *)[targetTestCase parent];
-            // append the latest result
-            [parent insertChild:[testCase copy] atIndex:[targetTestCase index] + 1];
+            // append the latest result at the end
+            NSUInteger insertIndex;
+            for (insertIndex = [targetTestCase index]; insertIndex < [[parent children] count]; insertIndex++) {
+                NSXMLElement *currentTestCase = (NSXMLElement *)[[parent children] objectAtIndex:insertIndex];
+                NSString *thisName = [[currentTestCase attributeForName:@"name"] stringValue];
+                NSString *thisClassName = [[currentTestCase attributeForName:@"classname"] stringValue];
+                if ([name isNotEqualTo:thisName] || [className isNotEqualTo:thisClassName]) {
+                    break;
+                }
+            }
+            [parent insertChild:[testCase copy] atIndex:insertIndex];
         } else {
             [BPUtils printInfo:DEBUGINFO withString:@"testcase insertion: %@", [[testCase attributeForName:@"name"] stringValue]];
             [targetTestSuite addChild:[testCase copy]];

diff --git a/Bluepill-runner/BluepillRunnerTests/BPReportCollectorTests.m b/Bluepill-runner/BluepillRunnerTests/BPReportCollectorTests.m
@@ -79,12 +79,26 @@ - (void)testCollectReportsFromPath {
     NSArray *testsuitesNodes =  [doc nodesForXPath:[NSString stringWithFormat:@".//%@", @"testsuites"] error:&error];
     NSXMLElement *root = testsuitesNodes[0];
     NSString *got = [[root attributeForName:@"tests"] stringValue];
-
-    XCTAssertTrue([got isEqualToString:@"25"], @"test count is wrong, wanted 25, got %@", got);
+    NSString *want = @"26";
+    XCTAssertTrue([got isEqualToString:want], @"test count is wrong, wanted %@, got %@", want, got);
     got = [[root attributeForName:@"errors"] stringValue];
-    XCTAssertTrue([got isEqualToString:@"2"], @"error count is wrong, wanted 2, got %@", got);
+    want = @"2";
+    XCTAssertTrue([got isEqualToString:want], @"error count is wrong, wanted %@, got %@", want, got);
     got = [[root attributeForName:@"failures"] stringValue];
-    XCTAssertTrue([got isEqualToString:@"2"], @"failure count is wrong, wanted 2, got %@", got);
+    want = @"3";
+    XCTAssertTrue([got isEqualToString:want], @"failure count is wrong, wanted %@, got %@", want, got);
+
+    // make sure the order is right
+    NSArray *retriedTests = [doc nodesForXPath:@"//testcase[@name='test2' and @classname='Class1']" error:nil];
+    XCTAssert(retriedTests.count == 3, @"Did not find three tries for test2");
+    XCTAssert([[retriedTests[0] nodesForXPath:@"failure" error:nil] count] == 1, @"First was not a failure");
+    XCTAssert([[retriedTests[1] nodesForXPath:@"failure" error:nil] count] == 1, @"Second was not a failure");
+    XCTAssert([[retriedTests[2] nodesForXPath:@"failure" error:nil] count] == 0, @"Third was not a success");
+
+
+    char * cmd = malloc(1024 * 10);
+    sprintf(cmd, "open -a '/Applications/Visual Studio Code.app' -- '%s'", [finalReport UTF8String]);
+    system(cmd);
 
     BOOL collatedReport = [[NSFileManager defaultManager] fileExistsAtPath:[path stringByAppendingPathComponent:@"1/report1.xml"]];
     XCTAssert(collatedReport == NO);

diff --git a/Bluepill-runner/BluepillRunnerTests/simulator/3/result3.xml b/Bluepill-runner/BluepillRunnerTests/simulator/3/result3.xml
@@ -1,8 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites name="Selected tests" tests="1" failures="0" errors="0">
-    <testsuite tests="1" failures="0" errors="0" name="Testsuite.xctest">
-        <testsuite tests="1" failures="0" errors="0" name="Testsuite">
+<testsuites name="Selected tests" tests="1" failures="1" errors="0">
+    <testsuite tests="1" failures="1" errors="0" name="Testsuite.xctest">
+        <testsuite tests="1" failures="1" errors="0" name="Testsuite">
             <testcase classname="Class1" name="test2">
+                <failure type="Failure" message="App Crashed">
+                    Unknown File:0
+                </failure>
                 <system-out>
                 </system-out>
             </testcase>

diff --git a/Bluepill-runner/BluepillRunnerTests/simulator/4/result4.xml b/Bluepill-runner/BluepillRunnerTests/simulator/4/result4.xml
@@ -1,7 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites name="Selected tests" tests="1" failures="0" errors="0">
-    <testsuite tests="1" failures="0" errors="0" name="Testsuite.xctest">
-        <testsuite tests="1" failures="0" errors="0" name="Testsuite">
+<testsuites name="Selected tests" tests="2" failures="0" errors="0">
+    <testsuite tests="2" failures="0" errors="0" name="Testsuite.xctest">
+        <testsuite tests="2" failures="0" errors="0" name="Testsuite">
+            <testcase classname="Class1" name="test2">
+                <system-out>
+                </system-out>
+            </testcase>
             <testcase classname="Class1" name="test3">
                 <system-out>
                 </system-out>