feat: Dump shards and upload on every run (Flank#1171)

* Add dump shards with upload * Add dump shards before matrix validation * Move dumpShards before tests run * cr changes * Update test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt Co-authored-by: piotradamczyk5 <[email protected]> * Add tests to check is dump shards executed on newTestRun * rename tests * Removed redundant mock * added more tests Co-authored-by: piotradamczyk5 <[email protected]>
mercari · Oct 1, 2020 · 256147f · 256147f
1 parent 08d2970
commit 256147f
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 3 deletions.
diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt
@@ -5,6 +5,7 @@ import ftl.args.validate
 import ftl.cli.firebase.test.CommonRunCommand
 import ftl.config.FtlConstants
 import ftl.config.emptyAndroidConfig
+import ftl.gc.GcStorage
 import ftl.mock.MockServer
 import ftl.run.ANDROID_SHARD_FILE
 import ftl.run.dumpShards
@@ -47,7 +48,10 @@ class AndroidRunCommand : CommonRunCommand(), Runnable {
         val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate()
         runBlocking {
             if (dumpShards) dumpShards(args = config, obfuscatedOutput = obfuscate)
-            else newTestRun(config)
+            else {
+                config.dumpShardsWithGcloudUpload(obfuscate)
+                newTestRun(config)
+            }
         }
     }
 
@@ -57,3 +61,8 @@ class AndroidRunCommand : CommonRunCommand(), Runnable {
     )
     var dumpShards: Boolean = false
 }
+
+private suspend fun AndroidArgs.dumpShardsWithGcloudUpload(obfuscatedOutput: Boolean) {
+    dumpShards(args = this, obfuscatedOutput = obfuscatedOutput)
+    if (disableResultsUpload.not()) GcStorage.upload(ANDROID_SHARD_FILE, resultsBucket, resultsDir)
+}
diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt
@@ -5,6 +5,7 @@ import ftl.args.validate
 import ftl.cli.firebase.test.CommonRunCommand
 import ftl.config.FtlConstants
 import ftl.config.emptyIosConfig
+import ftl.gc.GcStorage
 import ftl.mock.MockServer
 import ftl.run.IOS_SHARD_FILE
 import ftl.run.dumpShards
@@ -49,6 +50,7 @@ class IosRunCommand : CommonRunCommand(), Runnable {
         if (dumpShards) {
             dumpShards(args = config, obfuscatedOutput = obfuscate)
         } else runBlocking {
+            config.dumpShardsWithGcloudUpload(obfuscate)
             newTestRun(config)
         }
     }
@@ -59,3 +61,8 @@ class IosRunCommand : CommonRunCommand(), Runnable {
     )
     var dumpShards: Boolean = false
 }
+
+private fun IosArgs.dumpShardsWithGcloudUpload(obfuscatedOutput: Boolean) {
+    dumpShards(args = this, obfuscatedOutput = obfuscatedOutput)
+    if (disableResultsUpload.not()) GcStorage.upload(IOS_SHARD_FILE, resultsBucket, resultsDir)
+}
diff --git a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt
@@ -30,7 +30,6 @@ suspend fun newTestRun(args: IArgs) = withTimeoutOrNull(args.parsedTimeout) {
 
         println()
         matrixMap.printMatricesWebLinks(args.project)
-
         matrixMap.validate(args.ignoreFailedTests)
     }
 }

diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt
@@ -1,11 +1,19 @@
 package ftl.cli.firebase.test.android
 
 import com.google.common.truth.Truth.assertThat
+import ftl.args.AndroidArgs
 import ftl.args.yml.AppTestPair
 import ftl.config.Device
 import ftl.config.FtlConstants
+import ftl.gc.GcStorage
+import ftl.run.ANDROID_SHARD_FILE
+import ftl.run.dumpShards
 import ftl.run.exception.FlankConfigurationError
 import ftl.test.util.FlankTestRunner
+import io.mockk.coVerify
+import io.mockk.mockkObject
+import io.mockk.mockkStatic
+import io.mockk.verify
 import org.junit.Assert.assertEquals
 import org.junit.Rule
 import org.junit.Test
@@ -523,4 +531,26 @@ class AndroidRunCommandTest {
         )
         cmd.run()
     }
+
+    @Test
+    fun `should dump shards on android test run`() {
+        mockkStatic("ftl.run.DumpShardsKt")
+        val runCmd = AndroidRunCommand()
+        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml"
+        runCmd.run()
+        coVerify { dumpShards(any<AndroidArgs>(), any(), any()) }
+    }
+
+    @Test
+    fun `should dump shards on android test run and not upload when disable-upload-results set`() {
+        mockkStatic("ftl.run.DumpShardsKt")
+        mockkObject(GcStorage) {
+            val runCmd = AndroidRunCommand()
+            runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml"
+            CommandLine(runCmd).parseArgs("--disable-results-upload")
+            runCmd.run()
+            coVerify { dumpShards(any<AndroidArgs>(), any(), any()) }
+            verify(inverse = true) { GcStorage.upload(ANDROID_SHARD_FILE, any(), any()) }
+        }
+    }
 }
diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt
@@ -1,10 +1,17 @@
 package ftl.cli.firebase.test.ios
 
 import com.google.common.truth.Truth.assertThat
+import ftl.args.IosArgs
 import ftl.config.Device
 import ftl.config.FtlConstants
 import ftl.config.FtlConstants.isWindows
+import ftl.gc.GcStorage
+import ftl.run.IOS_SHARD_FILE
+import ftl.run.dumpShards
 import ftl.test.util.FlankTestRunner
+import io.mockk.mockkObject
+import io.mockk.mockkStatic
+import io.mockk.verify
 import org.junit.Assume.assumeFalse
 import org.junit.Rule
 import org.junit.Test
@@ -337,4 +344,26 @@ class IosRunCommandTest {
 
         assertThat(cmd.config.common.flank.useAverageTestTimeForNewTests).isTrue()
     }
+
+    @Test
+    fun `should dump shards on ios test run`() {
+        mockkStatic("ftl.run.DumpShardsKt")
+        val runCmd = IosRunCommand()
+        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml"
+        runCmd.run()
+        verify { dumpShards(any<IosArgs>(), any(), any()) }
+    }
+
+    @Test
+    fun `should dump shards on ios test run and not upload when disable-upload-results set`() {
+        mockkStatic("ftl.run.DumpShardsKt")
+        mockkObject(GcStorage) {
+            val runCmd = IosRunCommand()
+            runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml"
+            CommandLine(runCmd).parseArgs("--disable-results-upload")
+            runCmd.run()
+            verify { dumpShards(any<IosArgs>(), any(), any()) }
+            verify(inverse = true) { GcStorage.upload(IOS_SHARD_FILE, any(), any()) }
+        }
+    }
 }
diff --git a/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt b/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt
@@ -53,7 +53,7 @@ class MatrixMapTest {
     }
 
     @Test
-    fun `invalid matrix should contains specyfic error message`() {
+    fun `invalid matrix should contain a specific error message`() {
         val testMatrix = testMatrix()
         testMatrix.testMatrixId = "123"
         testMatrix.state = MatrixState.INVALID
-Original file line number
+Diff line change
@@ Expand Up @@
             println()
             matrixMap.printMatricesWebLinks(args.project)
             matrixMap.validate(args.ignoreFailedTests)
         }
     }
@@ Expand Down @@