feat(output): add test axis value column to summary table

docs(output): update summary table

docs/feature/summary_output.md

@@ -1,10 +1,11 @@
 # Formatted summary output 
 ```
-┌─────────┬──────────────────────┬─────────────────────┐
-│ OUTCOME │      MATRIX ID       │    TEST DETAILS     │
-├─────────┼──────────────────────┼─────────────────────┤
-│ success │ matrix-1z85qtvdnvb0l │ 4 test cases passed │
-└─────────┴──────────────────────┴─────────────────────┘
+┌─────────┬──────────────────────┬─────────────────┬─────────────────────────────────────────┐
+│ OUTCOME │      MATRIX ID       │ TEST AXIS VALUE │              TEST DETAILS               │
+├─────────┼──────────────────────┼─────────────────┼─────────────────────────────────────────┤
+│ success │ matrix-35czp85w4h3a7 │ greatqlte       │ 20 test cases passed                    │
+│ failure │ matrix-35czp85w4h3a7 │ Nexus6P         │ 1 test cases failed, 16 passed, 3 flaky │
+└─────────┴──────────────────────┴─────────────────┴─────────────────────────────────────────┘
 ```
 
 

test_runner/src/main/kotlin/ftl/json/MatrixMap.kt

@@ -39,9 +39,9 @@ class MatrixMap(
      */
     fun validateMatrices(shouldIgnore: Boolean = false) {
         map.values.run {
-            firstOrNull { it.canceledByUser() }?.let { throw MatrixCanceledError(it.outcomeDetails.orEmpty()) }
-            firstOrNull { it.infrastructureFail() }?.let { throw InfrastructureError(it.outcomeDetails.orEmpty()) }
-            firstOrNull { it.incompatibleFail() }?.let { throw IncompatibleTestDimensionError(it.outcomeDetails.orEmpty()) }
+            firstOrNull { it.canceledByUser() }?.run { throw MatrixCanceledError(outcomeDetails) }
+            firstOrNull { it.infrastructureFail() }?.run { throw InfrastructureError(outcomeDetails) }
+            firstOrNull { it.incompatibleFail() }?.run { throw IncompatibleTestDimensionError(outcomeDetails) }
             firstOrNull { it.state != MatrixState.FINISHED }?.let { throw FTLError(it) }
             filter { it.isFailed() }.let {
                 if (it.isNotEmpty()) throw FailedMatrixError(
@@ -53,6 +53,8 @@ class MatrixMap(
     }
 }
 
+private val SavedMatrix.outcomeDetails get() = testAxises.firstOrNull()?.details.orEmpty()
+
 fun Iterable<TestMatrix>.update(matrixMap: MatrixMap) = forEach { matrix ->
     matrixMap.map[matrix.testMatrixId]?.updateWithMatrix(matrix)?.let {
         matrixMap.map[matrix.testMatrixId] = it

test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt

@@ -13,7 +13,9 @@ import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
 import ftl.util.StepOutcome.unset
 
-internal fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?): String = when (summary) {
+internal fun Outcome?.getDetails(
+    testSuiteOverviewData: TestSuiteOverviewData?
+): String = when (this?.summary) {
     success, flaky -> testSuiteOverviewData
         ?.getSuccessOutcomeDetails(successDetail?.otherNativeCrash ?: false)
         ?: "Unknown outcome"

test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt

@@ -7,6 +7,7 @@ import ftl.reports.outcome.createMatrixOutcomeSummary
 import ftl.reports.outcome.fetchTestOutcomeContext
 import ftl.util.MatrixState.FINISHED
 import ftl.util.MatrixState.INVALID
+import ftl.util.StepOutcome
 import ftl.util.StepOutcome.failure
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
@@ -26,21 +27,24 @@ data class SavedMatrix(
     val downloaded: Boolean = false,
     val billableVirtualMinutes: Long = 0,
     val billablePhysicalMinutes: Long = 0,
-    val outcome: String = "",
-    val outcomeDetails: String = "",
     val clientDetails: Map<String, String>? = null,
     val gcsPathWithoutRootBucket: String = "",
     val gcsRootBucket: String = "",
     val webLinkWithoutExecutionDetails: String? = "",
-)
+    val testAxises: List<TestOutcome> = emptyList()
+) {
+    val outcome = testAxises.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty()
+}
 
 fun createSavedMatrix(testMatrix: TestMatrix) = SavedMatrix().updateWithMatrix(testMatrix)
 
-fun SavedMatrix.canceledByUser() = outcomeDetails == ABORTED_BY_USER_MESSAGE
+fun SavedMatrix.canceledByUser() = testAxises.any { it.details == ABORTED_BY_USER_MESSAGE }
+
+fun SavedMatrix.infrastructureFail() = testAxises.any { it.details == INFRASTRUCTURE_FAILURE_MESSAGE }
 
-fun SavedMatrix.infrastructureFail() = outcomeDetails == INFRASTRUCTURE_FAILURE_MESSAGE
+fun SavedMatrix.incompatibleFail() = testAxises.map { it.details }.intersect(incompatibleFails).isNotEmpty()
 
-fun SavedMatrix.incompatibleFail() = outcomeDetails in arrayOf(
+private val incompatibleFails = setOf(
     INCOMPATIBLE_APP_VERSION_MESSAGE,
     INCOMPATIBLE_ARCHITECTURE_MESSAGE,
     INCOMPATIBLE_DEVICE_MESSAGE
@@ -70,11 +74,11 @@ private fun SavedMatrix.updatedSavedMatrix(
 ): SavedMatrix = when (newMatrix.state) {
     state -> this
 
-    FINISHED -> newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary().let { (billableMinutes, outcome) ->
-        updateProperties(newMatrix).updateOutcome(outcome).updateBillableMinutes(billableMinutes)
+    FINISHED -> newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary().let { (billableMinutes, outcomes) ->
+        updateProperties(newMatrix).updateOutcome(outcomes).updateBillableMinutes(billableMinutes)
     }
 
-    INVALID -> updateProperties(newMatrix).updateOutcome(invalidTestOutcome())
+    INVALID -> updateProperties(newMatrix).updateOutcome(listOf(invalidTestOutcome()))
 
     else -> updateProperties(newMatrix)
 }
@@ -96,12 +100,11 @@ private fun SavedMatrix.updateBillableMinutes(billableMinutes: BillableMinutes)
     billableVirtualMinutes = billableMinutes.virtual,
 )
 
-private fun SavedMatrix.updateOutcome(testOutcome: TestOutcome) = copy(
-    outcome = testOutcome.outcome,
-    outcomeDetails = testOutcome.testDetails
+private fun SavedMatrix.updateOutcome(outcome: List<TestOutcome>) = copy(
+    testAxises = outcome
 )
 
 private fun invalidTestOutcome() = TestOutcome(
     outcome = "---",
-    testDetails = "Matrix is invalid"
+    details = "Matrix is invalid"
 )

test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt

@@ -0,0 +1,47 @@
+package ftl.json
+
+import ftl.reports.outcome.TestOutcome
+import ftl.util.StepOutcome.failure
+import ftl.util.StepOutcome.flaky
+import ftl.util.StepOutcome.success
+import ftl.util.SystemOutColor
+import ftl.util.TableColumn
+import ftl.util.buildTable
+
+fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
+
+fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
+    TableColumn(
+        header = OUTCOME_COLUMN_HEADER,
+        data = flatMapTestAxis { outcome },
+        dataColor = flatMapTestAxis { outcomeColor }
+    ),
+    TableColumn(
+        header = MATRIX_ID_COLUMN_HEADER,
+        data = flatMapTestAxis { matrix -> matrix.matrixId }
+    ),
+    TableColumn(
+        header = TEST_AXIS_VALUE_HEADER,
+        data = flatMapTestAxis { device }
+    ),
+    TableColumn(
+        header = OUTCOME_DETAILS_COLUMN_HEADER,
+        data = flatMapTestAxis { details }
+    )
+)
+
+private fun <T> List<SavedMatrix>.flatMapTestAxis(transform: TestOutcome.(SavedMatrix) -> T) =
+    flatMap { matrix -> matrix.testAxises.map { axis -> axis.transform(matrix) } }
+
+private val TestOutcome.outcomeColor
+    get() = when (outcome) {
+        failure -> SystemOutColor.RED
+        success -> SystemOutColor.GREEN
+        flaky -> SystemOutColor.BLUE
+        else -> SystemOutColor.DEFAULT
+    }
+
+private const val OUTCOME_COLUMN_HEADER = "OUTCOME"
+private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID"
+private const val TEST_AXIS_VALUE_HEADER = "TEST AXIS VALUE"
+private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS"

test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt

@@ -8,7 +8,7 @@ import ftl.json.SavedMatrix
 import ftl.json.isFailed
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
-import ftl.util.asPrintableTable
+import ftl.json.asPrintableTable
 import ftl.util.println
 import java.io.StringWriter
 import java.text.DecimalFormat

test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt

@@ -1,7 +1,6 @@
 package ftl.reports.outcome
 
 import com.google.api.services.toolresults.model.Step
-import com.google.api.services.toolresults.model.StepDimensionValueEntry
 import ftl.android.AndroidCatalog
 import ftl.util.billableMinutes
 import kotlin.math.min
@@ -41,6 +40,3 @@ private fun Step.getBillableSeconds(default: Long) =
     testExecutionStep?.testTiming?.testProcessDuration?.seconds?.let {
         min(it, default)
     }
-
-operator fun List<StepDimensionValueEntry>?.get(key: String) =
-    this?.firstOrNull { it.key == key }

test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt

@@ -2,7 +2,7 @@ package ftl.reports.outcome
 
 import com.google.api.services.toolresults.model.Environment
 
-fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair<BillableMinutes, TestOutcome> =
+fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair<BillableMinutes, List<TestOutcome>> =
     steps.calculateAndroidBillableMinutes(projectId, testTimeout) to
         if (environments.hasOutcome())
             environments.createMatrixOutcomeSummaryUsingEnvironments()

test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt

@@ -7,30 +7,37 @@ import ftl.json.getDetails
 import ftl.util.StepOutcome
 
 data class TestOutcome(
-    val outcome: String,
-    val testDetails: String
+    val device: String = "",
+    val outcome: String = "",
+    val details: String = "",
 )
 
-fun List<Environment>.createMatrixOutcomeSummaryUsingEnvironments(
-    outcome: Outcome? = getOutcomeFromEnvironments(),
-    testDetails: String? = outcome?.getDetails(map { it.createTestSuiteOverviewData() }.foldTestSuiteOverviewData())
+fun List<Environment>.createMatrixOutcomeSummaryUsingEnvironments(): List<TestOutcome> =
+    map(Environment::getTestOutcome)
+
+private fun Environment.getTestOutcome(
+    outcome: Outcome? = environmentResult?.outcome
 ) = TestOutcome(
-    outcome = outcome?.summary ?: "Unknown",
-    testDetails = testDetails ?: "Unknown outcome"
+    device = deviceModel(),
+    outcome = outcome?.summary ?: UNKNOWN_OUTCOME,
+    details = outcome.getDetails(createTestSuiteOverviewData()),
 )
 
-private fun List<Environment>.getOutcomeFromEnvironments(): Outcome? = maxByOrNull {
-    StepOutcome.order.indexOf(it.environmentResult?.outcome?.summary)
-}?.environmentResult?.outcome
+fun List<Step>.createMatrixOutcomeSummaryUsingSteps() = groupBy(Step::deviceModel).map { (device, steps) ->
+    steps.getTestOutcome(device)
+}
 
-fun List<Step>.createMatrixOutcomeSummaryUsingSteps(
+private fun List<Step>.getTestOutcome(
+    deviceModel: String,
     outcome: Outcome? = getOutcomeFromSteps(),
-    testDetails: String? = outcome?.getDetails(createTestSuiteOverviewData())
 ) = TestOutcome(
-    outcome = outcome?.summary ?: "Unknown",
-    testDetails = testDetails ?: "Unknown outcome"
+    device = deviceModel,
+    outcome = outcome?.summary ?: UNKNOWN_OUTCOME,
+    details = outcome.getDetails(createTestSuiteOverviewData())
 )
 
 private fun List<Step>.getOutcomeFromSteps(): Outcome? = maxByOrNull {
     StepOutcome.order.indexOf(it.outcome?.summary)
 }?.outcome
+
+private const val UNKNOWN_OUTCOME = "Unknown"

test_runner/src/main/kotlin/ftl/reports/outcome/Util.kt

@@ -1,7 +1,19 @@
 package ftl.reports.outcome
 
+import com.google.api.services.toolresults.model.Environment
+import com.google.api.services.toolresults.model.EnvironmentDimensionValueEntry
 import com.google.api.services.toolresults.model.Step
+import com.google.api.services.toolresults.model.StepDimensionValueEntry
 import ftl.environment.orUnknown
 
 internal fun Step.deviceModel() = dimensionValue["Model"]
     ?.value.orUnknown()
+
+internal fun Environment.deviceModel() = dimensionValue["Model"]
+    ?.value.orUnknown()
+
+operator fun List<StepDimensionValueEntry>?.get(key: String) =
+    this?.firstOrNull { it.key == key }
+
+operator fun List<EnvironmentDimensionValueEntry>?.get(key: String) =
+    this?.firstOrNull { it.key == key }

test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt

@@ -99,7 +99,7 @@ class SavedMatrixTest {
         assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1)
         assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName)
         assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket)
-        assertThat(savedMatrix.outcomeDetails).isNotEmpty()
+        assertThat(savedMatrix.testAxises.first().details).isNotEmpty()
     }
 
     @Test
@@ -133,7 +133,7 @@ class SavedMatrixTest {
         assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1)
         assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName)
         assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket)
-        assertThat(savedMatrix.outcomeDetails).isNotEmpty()
+        assertThat(savedMatrix.testAxises.first().details).isNotEmpty()
     }
 
     @Test
@@ -195,7 +195,7 @@ class SavedMatrixTest {
         testMatrix.state = INVALID
         savedMatrix = savedMatrix.updateWithMatrix(testMatrix)
         assertEquals(expectedOutcome, savedMatrix.outcome)
-        assertEquals(expectedOutcomeDetails, savedMatrix.outcomeDetails)
+        assertEquals(expectedOutcomeDetails, savedMatrix.testAxises.first().details)
         assertEquals(INVALID, savedMatrix.state)
     }
 

test_runner/src/test/kotlin/ftl/util/UtilsTest.kt

@@ -8,6 +8,7 @@ import ftl.json.SavedMatrixTest.Companion.createResultsStorage
 import ftl.json.SavedMatrixTest.Companion.createStepExecution
 import ftl.json.SavedMatrixTest.Companion.testMatrix
 import ftl.json.createSavedMatrix
+import ftl.reports.outcome.TestOutcome
 import ftl.run.cancelMatrices
 import ftl.test.util.FlankTestRunner
 import io.mockk.coEvery
@@ -372,12 +373,20 @@ class UtilsTest {
 private val testMatrix1 = mockk<SavedMatrix>(relaxed = true) {
     every { matrixId } returns "1"
     every { webLink } returns "www.flank.com/1"
-    every { outcome } returns "Failed"
-    every { outcomeDetails } returns "Test failed to run"
+    every { testAxises } returns listOf(
+        TestOutcome(
+            outcome = "Failed",
+            details = "Test failed to run"
+        )
+    )
 }
 private val testMatrix2 = mockk<SavedMatrix>(relaxed = true) {
     every { matrixId } returns "2"
     every { webLink } returns "www.flank.com/2"
-    every { outcome } returns "Failed"
-    every { outcomeDetails } returns "Test failed to run"
+    every { testAxises } returns listOf(
+        TestOutcome(
+            outcome = "Failed",
+            details = "Test failed to run"
+        )
+    )
 }