fix: Reflect gclouds outcome for robo tests

test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt

@@ -14,18 +14,20 @@ import ftl.util.StepOutcome.success
 import ftl.util.StepOutcome.unset
 
 internal fun Outcome?.getDetails(
-    testSuiteOverviewData: TestSuiteOverviewData?
+    testSuiteOverviewData: TestSuiteOverviewData?,
+    isRoboTest: Boolean = false
 ): String = when (this?.summary) {
-    success, flaky -> testSuiteOverviewData
-        ?.getSuccessOutcomeDetails(successDetail?.otherNativeCrash ?: false)
-        ?: "Unknown outcome"
+    success, flaky -> if (isRoboTest) "---" else getSuccessOutcomeDetails(testSuiteOverviewData)
     failure -> failureDetail.getFailureOutcomeDetails(testSuiteOverviewData)
     inconclusive -> inconclusiveDetail.formatOutcomeDetails()
     skipped -> skippedDetail.formatOutcomeDetails()
     unset -> "Unset outcome"
     else -> "Unknown outcome"
 }
 
+private fun Outcome?.getSuccessOutcomeDetails(data: TestSuiteOverviewData?) =
+    data?.getSuccessOutcomeDetails(this?.successDetail?.otherNativeCrash ?: false) ?: "Unknown outcome"
+
 private fun TestSuiteOverviewData.getSuccessOutcomeDetails(
     otherNativeCrash: Boolean
 ) = StringBuilder("$successCount test cases passed").apply {
@@ -43,6 +45,7 @@ internal fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: Test
     crashed == true -> "Application crashed"
     timedOut == true -> "Test timed out"
     notInstalled == true -> "App failed to install"
+    failedRoboscript == true -> "Test failed to run"
     else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary() ?: "Unknown failure"
 } + this?.takeIf { it.otherNativeCrash ?: false }?.let { NATIVE_CRASH_MESSAGE }.orEmpty()
 

test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt

@@ -2,15 +2,16 @@ package ftl.reports.outcome
 
 import com.google.api.services.toolresults.model.Environment
 
-fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair<BillableMinutes, List<TestOutcome>> =
-    steps.calculateAndroidBillableMinutes(projectId, testTimeout) to
-        if (environments.hasOutcome())
-            environments.createMatrixOutcomeSummaryUsingEnvironments()
-        else {
-            if (steps.isEmpty()) println("No test results found, something went wrong. Try re-running the tests.")
-            steps.createMatrixOutcomeSummaryUsingSteps()
-        }
+fun TestOutcomeContext.createMatrixOutcomeSummary() = billableMinutes() to outcomeSummary()
 
-private fun List<Environment>.hasOutcome() = isNotEmpty() && any { env ->
-    env.environmentResult?.outcome?.summary == null
-}.not()
+private fun TestOutcomeContext.billableMinutes() = steps.calculateAndroidBillableMinutes(projectId, testTimeout)
+
+private fun TestOutcomeContext.outcomeSummary() =
+    if (environments.hasOutcome())
+        createMatrixOutcomeSummaryUsingEnvironments()
+    else {
+        if (steps.isEmpty()) println("No test results found, something went wrong. Try re-running the tests.")
+        createMatrixOutcomeSummaryUsingSteps()
+    }
+
+private fun List<Environment>.hasOutcome() = isNotEmpty() && all { it.environmentResult?.outcome?.summary != null }

test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt

@@ -23,7 +23,7 @@ internal fun List<Step>.createTestSuiteOverviewData(): TestSuiteOverviewData = t
     .groupBy(Step::axisValue)
     .values
     .map { it.mapToTestSuiteOverviews().foldTestSuiteOverviewData() }
-    .fold(TestSuiteOverviewData()) { acc, data -> acc + data } // Fixme https://github.com/Flank/flank/issues/983
+    .fold(TestSuiteOverviewData()) { acc, data -> acc + data }
 
 private fun Step.isPrimaryStep() =
     multiStep?.primaryStep?.rollUp != null || multiStep == null

test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt

@@ -12,29 +12,33 @@ data class TestOutcome(
     val details: String = "",
 )
 
-fun List<Environment>.createMatrixOutcomeSummaryUsingEnvironments(): List<TestOutcome> =
-    map(Environment::getTestOutcome)
-
-private fun Environment.getTestOutcome(
-    outcome: Outcome? = environmentResult?.outcome
-) = TestOutcome(
-    device = axisValue(),
-    outcome = outcome?.summary ?: UNKNOWN_OUTCOME,
-    details = outcome.getDetails(createTestSuiteOverviewData()),
-)
-
-fun List<Step>.createMatrixOutcomeSummaryUsingSteps() = groupBy(Step::axisValue).map { (device, steps) ->
-    steps.getTestOutcome(device)
-}
-
-private fun List<Step>.getTestOutcome(
-    deviceModel: String,
-    outcome: Outcome? = getOutcomeFromSteps(),
-) = TestOutcome(
-    device = deviceModel,
-    outcome = outcome?.summary ?: UNKNOWN_OUTCOME,
-    details = outcome.getDetails(createTestSuiteOverviewData())
-)
+fun TestOutcomeContext.createMatrixOutcomeSummaryUsingEnvironments(): List<TestOutcome> = environments
+    .map { environment ->
+        TestOutcome(
+            device = environment.axisValue(),
+            outcome = environment.outcomeSummary,
+            details = environment.getOutcomeDetails(isRoboTest)
+        )
+    }
+
+private val Environment.outcomeSummary
+    get() = environmentResult?.outcome?.summary ?: UNKNOWN_OUTCOME
+
+private fun Environment.getOutcomeDetails(isRoboTest: Boolean) = environmentResult?.outcome.getDetails(createTestSuiteOverviewData(), isRoboTest)
+
+fun TestOutcomeContext.createMatrixOutcomeSummaryUsingSteps() = steps
+    .groupBy(Step::axisValue)
+    .map { (device, steps) ->
+        TestOutcome(
+            device = device,
+            outcome = steps.getOutcomeSummary(),
+            details = steps.getOutcomeDetails(isRoboTest)
+        )
+    }
+
+private fun List<Step>.getOutcomeSummary() = getOutcomeFromSteps()?.summary ?: UNKNOWN_OUTCOME
+
+private fun List<Step>.getOutcomeDetails(isRoboTest: Boolean) = getOutcomeFromSteps().getDetails(createTestSuiteOverviewData(), isRoboTest)
 
 private fun List<Step>.getOutcomeFromSteps(): Outcome? = maxByOrNull {
     StepOutcome.order.indexOf(it.outcome?.summary)

test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt

@@ -15,7 +15,8 @@ data class TestOutcomeContext(
     val projectId: String,
     val environments: List<Environment>,
     val steps: List<Step>,
-    val testTimeout: Long
+    val testTimeout: Long,
+    val isRoboTest: Boolean
 )
 
 fun TestMatrix.fetchTestOutcomeContext() = getToolResultsIds().let { ids ->
@@ -24,7 +25,8 @@ fun TestMatrix.fetchTestOutcomeContext() = getToolResultsIds().let { ids ->
         matrixId = testMatrixId,
         environments = GcToolResults.listAllEnvironments(ids),
         steps = GcToolResults.listAllSteps(ids),
-        testTimeout = testTimeout()
+        testTimeout = testTimeout(),
+        isRoboTest = isRoboTest()
     )
 }
 
@@ -44,3 +46,5 @@ private fun TestMatrix.testTimeout() = timeoutToSeconds(
         ?.testTimeout
         ?: "0s"
 )
+
+private fun TestMatrix.isRoboTest() = testExecutions.orEmpty().any { it?.testSpecification?.androidRoboTest != null }

test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt

@@ -11,7 +11,7 @@ internal fun beforeRunMessage(args: IArgs, testShardChunks: List<Chunk>): String
     val (classesCount, testsCount) = testShardChunks.partitionedTestCases.testAndClassesCount
 
     val result = StringBuilder()
-    val testString = if (testsCount > 0) "$testsCount test${s(testsCount)}" else ""
+    val testString = if (testsCount == 0 && classesCount != 0) "" else "$testsCount test${s(testsCount)}"
     val classString = if (classesCount > 0) "$classesCount parameterized class${es(classesCount)}" else ""
 
     result.appendLine(

test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt

@@ -6,11 +6,16 @@ import ftl.reports.api.data.TestSuiteOverviewData
 import ftl.util.StepOutcome
 import io.mockk.every
 import io.mockk.mockk
+import io.mockk.unmockkAll
+import org.junit.After
 import org.junit.Assert.assertEquals
 import org.junit.Test
 
 internal class OutcomeDetailsFormatterTest {
 
+    @After
+    fun tearDown() = unmockkAll()
+
     @Test
     fun `should return correct outcome details for success`() {
         // given
@@ -21,8 +26,8 @@ internal class OutcomeDetailsFormatterTest {
         val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
         val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
         val expectedMessage = "$successCount test cases passed, " +
-            "${testSuiteOverviewData.skipped} skipped, " +
-            "${testSuiteOverviewData.flakes} flaky"
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky"
 
         // when
         val result = mockedOutcome.getDetails(testSuiteOverviewData)
@@ -41,9 +46,9 @@ internal class OutcomeDetailsFormatterTest {
         val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
         val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
         val expectedMessage = "$successCount test cases passed, " +
-            "${testSuiteOverviewData.skipped} skipped, " +
-            "${testSuiteOverviewData.flakes} flaky" +
-            " (Native crash)"
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky" +
+                " (Native crash)"
 
         // when
         val result = mockedOutcome.getDetails(testSuiteOverviewData)
@@ -57,19 +62,14 @@ internal class OutcomeDetailsFormatterTest {
         // given
         val mockedOutcome = mockk<Outcome> {
             every { summary } returns StepOutcome.failure
-            every { failureDetail } returns mockk {
-                every { crashed } returns false
-                every { timedOut } returns false
-                every { notInstalled } returns false
-                every { otherNativeCrash } returns false
-            }
+            every { failureDetail } returns mockk(relaxed = true) {}
         }
         val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0)
         val expectedMessage = "${testSuiteOverviewData.failures} test cases failed, " +
-            "${testSuiteOverviewData.errors} errors, " +
-            "1 passed, " +
-            "${testSuiteOverviewData.skipped} skipped, " +
-            "${testSuiteOverviewData.flakes} flaky"
+                "${testSuiteOverviewData.errors} errors, " +
+                "1 passed, " +
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky"
 
         // when
         val result = mockedOutcome.getDetails(testSuiteOverviewData)
@@ -146,12 +146,7 @@ internal class OutcomeDetailsFormatterTest {
         // given
         val mockedOutcome = mockk<Outcome> {
             every { summary } returns StepOutcome.failure
-            every { failureDetail } returns mockk {
-                every { crashed } returns false
-                every { timedOut } returns false
-                every { notInstalled } returns false
-                every { otherNativeCrash } returns false
-            }
+            every { failureDetail } returns mockk(relaxed = true) {}
         }
         val expectedMessage = "Unknown failure"
 
@@ -354,4 +349,16 @@ internal class OutcomeDetailsFormatterTest {
             otherNativeCrash = null
         }.getFailureOutcomeDetails(null)
     }
+
+    @Test
+    fun `should print message for failed robo test`() {
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns FailureDetail().apply { failedRoboscript = true }
+        }
+
+        val result = mockedOutcome.getDetails(null, true)
+
+        assertEquals("Test failed to run", result)
+    }
 }

test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt

@@ -0,0 +1,126 @@
+package ftl.reports.outcome
+
+import com.google.api.services.toolresults.model.Environment
+import com.google.api.services.toolresults.model.Step
+import io.mockk.every
+import io.mockk.mockkStatic
+import io.mockk.unmockkAll
+import kotlin.reflect.full.createInstance
+import org.junit.After
+import org.junit.Assert.assertEquals
+import org.junit.Before
+import org.junit.Test
+
+class CreateMatrixOutcomeSummaryKtTest {
+
+    @Before
+    fun setUp() {
+        mockkStatic("ftl.reports.outcome.UtilKt")
+        mockkStatic("ftl.reports.outcome.BillableMinutesKt")
+    }
+
+    @After
+    fun tearDown() = unmockkAll()
+
+    @Test
+    fun `should create TestOutcome list for success robo test - from environments`() {
+        val env: Environment = make {
+            environmentResult = make {
+                outcome = make { summary = "success" }
+            }
+        }
+        every { env.axisValue() } returns "anyDevice"
+        val context = TestOutcomeContext(
+            matrixId = "anyMatrix",
+            projectId = "anyProject",
+            testTimeout = Long.MAX_VALUE,
+            steps = emptyList(),
+            isRoboTest = true,
+            environments = listOf(env)
+        )
+
+        val (_, result) = context.createMatrixOutcomeSummary()
+
+        assertEquals("---", result[0].details)
+        assertEquals("anyDevice", result[0].device)
+        assertEquals("success", result[0].outcome)
+    }
+
+    @Test
+    fun `should create TestOutcome list for failed robo test - from environments`() {
+        val env: Environment = make {
+            environmentResult = make {
+                outcome = make {
+                    summary = "failure"
+                    failureDetail = make { failedRoboscript = true }
+                }
+            }
+        }
+        every { env.axisValue() } returns "anyDevice"
+        val context = TestOutcomeContext(
+            matrixId = "anyMatrix",
+            projectId = "anyProject",
+            testTimeout = Long.MAX_VALUE,
+            steps = emptyList(),
+            isRoboTest = true,
+            environments = listOf(env)
+        )
+
+        val (_, result) = context.createMatrixOutcomeSummary()
+
+        assertEquals("Test failed to run", result[0].details)
+        assertEquals("anyDevice", result[0].device)
+        assertEquals("failure", result[0].outcome)
+    }
+
+    @Test
+    fun `should create TestOutcome list for success robo test - from steps`() {
+        val steps: List<Step> = listOf(make {
+            outcome = make { summary = "success" }
+        })
+        every { steps[0].axisValue() } returns "anyDevice"
+        every { steps.calculateAndroidBillableMinutes(any(), any()) } returns make()
+        val context = TestOutcomeContext(
+            matrixId = "anyMatrix",
+            projectId = "anyProject",
+            testTimeout = Long.MAX_VALUE,
+            steps = steps,
+            isRoboTest = true,
+            environments = emptyList()
+        )
+
+        val (_, result) = context.createMatrixOutcomeSummary()
+
+        assertEquals("---", result[0].details)
+        assertEquals("anyDevice", result[0].device)
+        assertEquals("success", result[0].outcome)
+    }
+
+    @Test
+    fun `should create TestOutcome list for failed robo test - from steps`() {
+        val steps: List<Step> = listOf(make {
+            outcome = make {
+                summary = "failure"
+                failureDetail = make { failedRoboscript = true }
+            }
+        })
+        every { steps[0].axisValue() } returns "anyDevice"
+        every { steps.calculateAndroidBillableMinutes(any(), any()) } returns make()
+        val context = TestOutcomeContext(
+            matrixId = "anyMatrix",
+            projectId = "anyProject",
+            testTimeout = Long.MAX_VALUE,
+            steps = steps,
+            isRoboTest = true,
+            environments = emptyList()
+        )
+
+        val (_, result) = context.createMatrixOutcomeSummary()
+
+        assertEquals("Test failed to run", result[0].details)
+        assertEquals("anyDevice", result[0].device)
+        assertEquals("failure", result[0].outcome)
+    }
+}
+
+private inline fun <reified T : Any> make(block: T.() -> Unit = {}): T = T::class.createInstance().apply(block)