Exit code updates (#605)

* Improve exit code logging * Use matrix outcome to set exit code * Rely on FTL to set the exti status correctly
Flank · Oct 8, 2019 · 01d915b · 01d915b
1 parent 509cae7
commit 01d915b
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 26 deletions.
diff --git a/release_notes.md b/release_notes.md
@@ -1,6 +1,9 @@
 ## next (unreleased)
 
 - [#595](https://github.com/TestArmada/flank/pull/595) Rename `flaky-test-attempts` to `num-flaky-test-attempts`. Rename `repeat-tests` to `num-test-runs`. ([bootstraponline](https://github.com/bootstraponline))
+- [#605](https://github.com/TestArmada/flank/pull/605) Improve exit code logging. Use matrix outcome to set exit code. ([bootstraponline](https://github.com/bootstraponline))
+- [#597](https://github.com/TestArmada/flank/pull/597) Support parsing testLabExecutionId. ([yogurtearl](https://github.com/yogurtearl))
+- [#599](https://github.com/TestArmada/flank/pull/599) Disable FAIL_ON_UNKNOWN_PROPERTIES for forward compatibility. ([narenkmanoharan](https://github.com/narenkmanoharan))
 
 ## v7.0.2
 

diff --git a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt
@@ -36,10 +36,20 @@ class MatrixMap(
         var exitCode = 0
 
         map.values.forEach { matrix ->
-            if (matrix.state != MatrixState.FINISHED) return 2
-            if (matrix.failed()) exitCode = 1
+            if (matrix.state != MatrixState.FINISHED) {
+                matrix.logError("not finished")
+                return 2
+            }
+            if (matrix.failed()) {
+                matrix.logError("failed")
+                exitCode = 1
+            }
         }
 
         return exitCode
     }
+
+    private fun SavedMatrix.logError(message: String) {
+        println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.webLink}")
+    }
 }
diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
@@ -97,16 +97,14 @@ object ReportManager {
         val useFlakyTests = args.flakyTestAttempts > 0
         if (useFlakyTests) JUnitDedupe.modify(testSuite)
 
-        val testSuccessful = if (useFlakyTests) testSuite?.successful() ?: false else matrices.allSuccessful()
-
         listOf(
             CostReport,
             MatrixResultsReport
         ).map {
             it.run(matrices, testSuite, printToStdout = true, args = args)
         }
 
-        if (!testSuccessful) {
+        if (!matrices.allSuccessful()) {
             listOf(
                 HtmlErrorReport
             ).map { it.run(matrices, testSuite, printToStdout = false, args = args) }
@@ -115,14 +113,7 @@ object ReportManager {
         JUnitReport.run(matrices, testSuite, printToStdout = false, args = args)
         processJunitXml(testSuite, args, testShardChunks)
 
-        // FTL has a bug with matrix roll-up when using flakyTestAttempts
-        // as a work around, we calculate the success based on JUnit XML results.
-        val exitCode = if (useFlakyTests) {
-            if (testSuccessful) 0 else 1
-        } else {
-            matrices.exitCode()
-        }
-        return exitCode
+        return matrices.exitCode()
     }
 
     data class ShardEfficiency(

diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt
@@ -2,7 +2,6 @@ package ftl.reports.utils
 
 import com.google.common.truth.Truth.assertThat
 import ftl.args.AndroidArgs
-import ftl.args.IosArgs
 import ftl.reports.util.ReportManager
 import ftl.reports.xml.model.JUnitTestCase
 import ftl.reports.xml.model.JUnitTestResult
@@ -45,18 +44,6 @@ class ReportManagerTest {
         ReportManager.generate(matrix, mockArgs, emptyList())
     }
 
-    @Test
-    fun `generate correct exit code from multi-suite ios result`() {
-        // TODO: NPE on Windows
-        val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/ios_exit_code", IosArgs.default())
-        val mockArgs = mock(IosArgs::class.java)
-        `when`(mockArgs.smartFlankGcsPath).thenReturn("")
-        // Must set flaky test attempts > 0 for exit code to be based on JUnit XML results.
-        `when`(mockArgs.flakyTestAttempts).thenReturn(1)
-        val exitCode = ReportManager.generate(matrix, mockArgs, emptyList())
-        assertThat(exitCode).isEqualTo(0)
-    }
-
     @Test
     fun createShardEfficiencyListTest() {
         val oldRunTestCases = mutableListOf(