From 1aabb9d41117d088ca1faadaa4f4e39578aa111b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 21 Aug 2020 15:26:38 +0200 Subject: [PATCH] feat(output): add test axis value column to summary table docs(output): update summary table --- docs/feature/summary_output.md | 11 +++-- .../src/main/kotlin/ftl/json/MatrixMap.kt | 8 ++-- .../ftl/json/OutcomeDetailsFormatter.kt | 4 +- .../src/main/kotlin/ftl/json/SavedMatrix.kt | 29 +++++++----- .../kotlin/ftl/json/SavedMatrixTableUtil.kt | 47 +++++++++++++++++++ .../kotlin/ftl/reports/MatrixResultsReport.kt | 2 +- .../ftl/reports/outcome/BillableMinutes.kt | 4 -- .../outcome/CreateMatrixOutcomeSummary.kt | 2 +- .../kotlin/ftl/reports/outcome/TestOutcome.kt | 35 ++++++++------ .../main/kotlin/ftl/reports/outcome/Util.kt | 12 +++++ .../kotlin/ftl/util/SavedMatrixTableUtil.kt | 25 ---------- .../test/kotlin/ftl/json/SavedMatrixTest.kt | 6 +-- .../src/test/kotlin/ftl/util/UtilsTest.kt | 17 +++++-- 13 files changed, 128 insertions(+), 74 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt delete mode 100644 test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt diff --git a/docs/feature/summary_output.md b/docs/feature/summary_output.md index 213343cc65..ab7dfcaced 100644 --- a/docs/feature/summary_output.md +++ b/docs/feature/summary_output.md @@ -1,10 +1,11 @@ # Formatted summary output ``` -┌─────────┬──────────────────────┬─────────────────────┐ -│ OUTCOME │ MATRIX ID │ TEST DETAILS │ -├─────────┼──────────────────────┼─────────────────────┤ -│ success │ matrix-1z85qtvdnvb0l │ 4 test cases passed │ -└─────────┴──────────────────────┴─────────────────────┘ +┌─────────┬──────────────────────┬─────────────────┬─────────────────────────────────────────┐ +│ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ +├─────────┼──────────────────────┼─────────────────┼─────────────────────────────────────────┤ +│ success │ matrix-35czp85w4h3a7 │ greatqlte │ 20 test cases passed │ +│ failure │ matrix-35czp85w4h3a7 │ Nexus6P │ 1 test cases failed, 16 passed, 3 flaky │ +└─────────┴──────────────────────┴─────────────────┴─────────────────────────────────────────┘ ``` diff --git a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt index 52b868b16a..1bf57f86ff 100644 --- a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt +++ b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt @@ -39,9 +39,9 @@ class MatrixMap( */ fun validateMatrices(shouldIgnore: Boolean = false) { map.values.run { - firstOrNull { it.canceledByUser() }?.let { throw MatrixCanceledError(it.outcomeDetails.orEmpty()) } - firstOrNull { it.infrastructureFail() }?.let { throw InfrastructureError(it.outcomeDetails.orEmpty()) } - firstOrNull { it.incompatibleFail() }?.let { throw IncompatibleTestDimensionError(it.outcomeDetails.orEmpty()) } + firstOrNull { it.canceledByUser() }?.run { throw MatrixCanceledError(outcomeDetails) } + firstOrNull { it.infrastructureFail() }?.run { throw InfrastructureError(outcomeDetails) } + firstOrNull { it.incompatibleFail() }?.run { throw IncompatibleTestDimensionError(outcomeDetails) } firstOrNull { it.state != MatrixState.FINISHED }?.let { throw FTLError(it) } filter { it.isFailed() }.let { if (it.isNotEmpty()) throw FailedMatrixError( @@ -53,6 +53,8 @@ class MatrixMap( } } +private val SavedMatrix.outcomeDetails get() = testAxises.firstOrNull()?.details.orEmpty() + fun Iterable.update(matrixMap: MatrixMap) = forEach { matrix -> matrixMap.map[matrix.testMatrixId]?.updateWithMatrix(matrix)?.let { matrixMap.map[matrix.testMatrixId] = it diff --git a/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt index bea21cc2b2..d7a2a4ff9c 100644 --- a/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt +++ b/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt @@ -13,7 +13,9 @@ import ftl.util.StepOutcome.skipped import ftl.util.StepOutcome.success import ftl.util.StepOutcome.unset -internal fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?): String = when (summary) { +internal fun Outcome?.getDetails( + testSuiteOverviewData: TestSuiteOverviewData? +): String = when (this?.summary) { success, flaky -> testSuiteOverviewData ?.getSuccessOutcomeDetails(successDetail?.otherNativeCrash ?: false) ?: "Unknown outcome" diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt index d545df9a53..7c82a45c63 100644 --- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt @@ -7,6 +7,7 @@ import ftl.reports.outcome.createMatrixOutcomeSummary import ftl.reports.outcome.fetchTestOutcomeContext import ftl.util.MatrixState.FINISHED import ftl.util.MatrixState.INVALID +import ftl.util.StepOutcome import ftl.util.StepOutcome.failure import ftl.util.StepOutcome.inconclusive import ftl.util.StepOutcome.skipped @@ -26,21 +27,24 @@ data class SavedMatrix( val downloaded: Boolean = false, val billableVirtualMinutes: Long = 0, val billablePhysicalMinutes: Long = 0, - val outcome: String = "", - val outcomeDetails: String = "", val clientDetails: Map? = null, val gcsPathWithoutRootBucket: String = "", val gcsRootBucket: String = "", val webLinkWithoutExecutionDetails: String? = "", -) + val testAxises: List = emptyList() +) { + val outcome = testAxises.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty() +} fun createSavedMatrix(testMatrix: TestMatrix) = SavedMatrix().updateWithMatrix(testMatrix) -fun SavedMatrix.canceledByUser() = outcomeDetails == ABORTED_BY_USER_MESSAGE +fun SavedMatrix.canceledByUser() = testAxises.any { it.details == ABORTED_BY_USER_MESSAGE } + +fun SavedMatrix.infrastructureFail() = testAxises.any { it.details == INFRASTRUCTURE_FAILURE_MESSAGE } -fun SavedMatrix.infrastructureFail() = outcomeDetails == INFRASTRUCTURE_FAILURE_MESSAGE +fun SavedMatrix.incompatibleFail() = testAxises.map { it.details }.intersect(incompatibleFails).isNotEmpty() -fun SavedMatrix.incompatibleFail() = outcomeDetails in arrayOf( +private val incompatibleFails = setOf( INCOMPATIBLE_APP_VERSION_MESSAGE, INCOMPATIBLE_ARCHITECTURE_MESSAGE, INCOMPATIBLE_DEVICE_MESSAGE @@ -70,11 +74,11 @@ private fun SavedMatrix.updatedSavedMatrix( ): SavedMatrix = when (newMatrix.state) { state -> this - FINISHED -> newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary().let { (billableMinutes, outcome) -> - updateProperties(newMatrix).updateOutcome(outcome).updateBillableMinutes(billableMinutes) + FINISHED -> newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary().let { (billableMinutes, outcomes) -> + updateProperties(newMatrix).updateOutcome(outcomes).updateBillableMinutes(billableMinutes) } - INVALID -> updateProperties(newMatrix).updateOutcome(invalidTestOutcome()) + INVALID -> updateProperties(newMatrix).updateOutcome(listOf(invalidTestOutcome())) else -> updateProperties(newMatrix) } @@ -96,12 +100,11 @@ private fun SavedMatrix.updateBillableMinutes(billableMinutes: BillableMinutes) billableVirtualMinutes = billableMinutes.virtual, ) -private fun SavedMatrix.updateOutcome(testOutcome: TestOutcome) = copy( - outcome = testOutcome.outcome, - outcomeDetails = testOutcome.testDetails +private fun SavedMatrix.updateOutcome(outcome: List) = copy( + testAxises = outcome ) private fun invalidTestOutcome() = TestOutcome( outcome = "---", - testDetails = "Matrix is invalid" + details = "Matrix is invalid" ) diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt new file mode 100644 index 0000000000..8b98af9306 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt @@ -0,0 +1,47 @@ +package ftl.json + +import ftl.reports.outcome.TestOutcome +import ftl.util.StepOutcome.failure +import ftl.util.StepOutcome.flaky +import ftl.util.StepOutcome.success +import ftl.util.SystemOutColor +import ftl.util.TableColumn +import ftl.util.buildTable + +fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable() + +fun List.asPrintableTable(): String = buildTable( + TableColumn( + header = OUTCOME_COLUMN_HEADER, + data = flatMapTestAxis { outcome }, + dataColor = flatMapTestAxis { outcomeColor } + ), + TableColumn( + header = MATRIX_ID_COLUMN_HEADER, + data = flatMapTestAxis { matrix -> matrix.matrixId } + ), + TableColumn( + header = TEST_AXIS_VALUE_HEADER, + data = flatMapTestAxis { device } + ), + TableColumn( + header = OUTCOME_DETAILS_COLUMN_HEADER, + data = flatMapTestAxis { details } + ) +) + +private fun List.flatMapTestAxis(transform: TestOutcome.(SavedMatrix) -> T) = + flatMap { matrix -> matrix.testAxises.map { axis -> axis.transform(matrix) } } + +private val TestOutcome.outcomeColor + get() = when (outcome) { + failure -> SystemOutColor.RED + success -> SystemOutColor.GREEN + flaky -> SystemOutColor.BLUE + else -> SystemOutColor.DEFAULT + } + +private const val OUTCOME_COLUMN_HEADER = "OUTCOME" +private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID" +private const val TEST_AXIS_VALUE_HEADER = "TEST AXIS VALUE" +private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS" diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index f34601f399..5a34c6a81c 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -8,7 +8,7 @@ import ftl.json.SavedMatrix import ftl.json.isFailed import ftl.reports.util.IReport import ftl.reports.xml.model.JUnitTestResult -import ftl.util.asPrintableTable +import ftl.json.asPrintableTable import ftl.util.println import java.io.StringWriter import java.text.DecimalFormat diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt index 40bf2450e9..d5e1cde20c 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt @@ -1,7 +1,6 @@ package ftl.reports.outcome import com.google.api.services.toolresults.model.Step -import com.google.api.services.toolresults.model.StepDimensionValueEntry import ftl.android.AndroidCatalog import ftl.util.billableMinutes import kotlin.math.min @@ -41,6 +40,3 @@ private fun Step.getBillableSeconds(default: Long) = testExecutionStep?.testTiming?.testProcessDuration?.seconds?.let { min(it, default) } - -operator fun List?.get(key: String) = - this?.firstOrNull { it.key == key } diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt index 82b99c88dc..b05f4c17df 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt @@ -2,7 +2,7 @@ package ftl.reports.outcome import com.google.api.services.toolresults.model.Environment -fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair = +fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair> = steps.calculateAndroidBillableMinutes(projectId, testTimeout) to if (environments.hasOutcome()) environments.createMatrixOutcomeSummaryUsingEnvironments() diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt index 5f51c1965b..ab2e678c72 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt @@ -7,30 +7,37 @@ import ftl.json.getDetails import ftl.util.StepOutcome data class TestOutcome( - val outcome: String, - val testDetails: String + val device: String = "", + val outcome: String = "", + val details: String = "", ) -fun List.createMatrixOutcomeSummaryUsingEnvironments( - outcome: Outcome? = getOutcomeFromEnvironments(), - testDetails: String? = outcome?.getDetails(map { it.createTestSuiteOverviewData() }.foldTestSuiteOverviewData()) +fun List.createMatrixOutcomeSummaryUsingEnvironments(): List = + map(Environment::getTestOutcome) + +private fun Environment.getTestOutcome( + outcome: Outcome? = environmentResult?.outcome ) = TestOutcome( - outcome = outcome?.summary ?: "Unknown", - testDetails = testDetails ?: "Unknown outcome" + device = deviceModel(), + outcome = outcome?.summary ?: UNKNOWN_OUTCOME, + details = outcome.getDetails(createTestSuiteOverviewData()), ) -private fun List.getOutcomeFromEnvironments(): Outcome? = maxByOrNull { - StepOutcome.order.indexOf(it.environmentResult?.outcome?.summary) -}?.environmentResult?.outcome +fun List.createMatrixOutcomeSummaryUsingSteps() = groupBy(Step::deviceModel).map { (device, steps) -> + steps.getTestOutcome(device) +} -fun List.createMatrixOutcomeSummaryUsingSteps( +private fun List.getTestOutcome( + deviceModel: String, outcome: Outcome? = getOutcomeFromSteps(), - testDetails: String? = outcome?.getDetails(createTestSuiteOverviewData()) ) = TestOutcome( - outcome = outcome?.summary ?: "Unknown", - testDetails = testDetails ?: "Unknown outcome" + device = deviceModel, + outcome = outcome?.summary ?: UNKNOWN_OUTCOME, + details = outcome.getDetails(createTestSuiteOverviewData()) ) private fun List.getOutcomeFromSteps(): Outcome? = maxByOrNull { StepOutcome.order.indexOf(it.outcome?.summary) }?.outcome + +private const val UNKNOWN_OUTCOME = "Unknown" diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/Util.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/Util.kt index b775928564..c158201124 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/Util.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/Util.kt @@ -1,7 +1,19 @@ package ftl.reports.outcome +import com.google.api.services.toolresults.model.Environment +import com.google.api.services.toolresults.model.EnvironmentDimensionValueEntry import com.google.api.services.toolresults.model.Step +import com.google.api.services.toolresults.model.StepDimensionValueEntry import ftl.environment.orUnknown internal fun Step.deviceModel() = dimensionValue["Model"] ?.value.orUnknown() + +internal fun Environment.deviceModel() = dimensionValue["Model"] + ?.value.orUnknown() + +operator fun List?.get(key: String) = + this?.firstOrNull { it.key == key } + +operator fun List?.get(key: String) = + this?.firstOrNull { it.key == key } diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt deleted file mode 100644 index 19c461e8c9..0000000000 --- a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt +++ /dev/null @@ -1,25 +0,0 @@ -package ftl.util - -import ftl.json.SavedMatrix -import ftl.util.StepOutcome.failure -import ftl.util.StepOutcome.flaky -import ftl.util.StepOutcome.success - -fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable() - -fun List.asPrintableTable(): String = buildTable( - TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, dataColor = map { getOutcomeColor(it.outcome) }), - TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }), - TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, mapNotNull { it.outcomeDetails }) -) - -private fun getOutcomeColor(outcome: String) = when (outcome) { - failure -> SystemOutColor.RED - success -> SystemOutColor.GREEN - flaky -> SystemOutColor.BLUE - else -> SystemOutColor.DEFAULT -} - -private const val OUTCOME_COLUMN_HEADER = "OUTCOME" -private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID" -private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS" diff --git a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt b/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt index f2b66e2133..ec5230d1fe 100644 --- a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt @@ -99,7 +99,7 @@ class SavedMatrixTest { assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1) assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName) assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket) - assertThat(savedMatrix.outcomeDetails).isNotEmpty() + assertThat(savedMatrix.testAxises.first().details).isNotEmpty() } @Test @@ -133,7 +133,7 @@ class SavedMatrixTest { assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1) assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName) assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket) - assertThat(savedMatrix.outcomeDetails).isNotEmpty() + assertThat(savedMatrix.testAxises.first().details).isNotEmpty() } @Test @@ -195,7 +195,7 @@ class SavedMatrixTest { testMatrix.state = INVALID savedMatrix = savedMatrix.updateWithMatrix(testMatrix) assertEquals(expectedOutcome, savedMatrix.outcome) - assertEquals(expectedOutcomeDetails, savedMatrix.outcomeDetails) + assertEquals(expectedOutcomeDetails, savedMatrix.testAxises.first().details) assertEquals(INVALID, savedMatrix.state) } diff --git a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt index dcb76b7f6c..33de31e02b 100644 --- a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt @@ -8,6 +8,7 @@ import ftl.json.SavedMatrixTest.Companion.createResultsStorage import ftl.json.SavedMatrixTest.Companion.createStepExecution import ftl.json.SavedMatrixTest.Companion.testMatrix import ftl.json.createSavedMatrix +import ftl.reports.outcome.TestOutcome import ftl.run.cancelMatrices import ftl.test.util.FlankTestRunner import io.mockk.coEvery @@ -372,12 +373,20 @@ class UtilsTest { private val testMatrix1 = mockk(relaxed = true) { every { matrixId } returns "1" every { webLink } returns "www.flank.com/1" - every { outcome } returns "Failed" - every { outcomeDetails } returns "Test failed to run" + every { testAxises } returns listOf( + TestOutcome( + outcome = "Failed", + details = "Test failed to run" + ) + ) } private val testMatrix2 = mockk(relaxed = true) { every { matrixId } returns "2" every { webLink } returns "www.flank.com/2" - every { outcome } returns "Failed" - every { outcomeDetails } returns "Test failed to run" + every { testAxises } returns listOf( + TestOutcome( + outcome = "Failed", + details = "Test failed to run" + ) + ) }