diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 2ae15f577c..4f33957582 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -15,9 +15,9 @@ flank: # test shards - the amount of groups to split the test suite into # set to -1 to use one shard per test. testShards: 1 - # test runs - the amount of times to run the tests. + # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x - testRuns: 1 + repeatTests: 1 # # always run - these tests are inserted at the beginning of every shard # # useful if you need to grant permissions or login before other tests run # test-targets-always-run: diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 50bb578a70..a1a5978493 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -27,9 +27,9 @@ flank: # test shards - the amount of groups to split the test suite into # set to -1 to use one shard per test. testShards: 1 - # test runs - the amount of times to run the tests. + # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x - testRuns: 1 + repeatTests: 1 # # always run - these tests are inserted at the beginning of every shard # # useful if you need to grant permissions or login before other tests run # test-targets-always-run: diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index dcb7de7ab0..f9ce2a7b1f 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -49,7 +49,7 @@ class AndroidArgs( private val flank = flankYml.flank val testShards = flank.testShards - override val testRuns = flank.testRuns + override val repeatTests = flank.repeatTests val testTargetsAlwaysRun = flank.testTargetsAlwaysRun // computed properties not specified in yaml @@ -124,7 +124,7 @@ AndroidArgs flank: testShards: $testShards - testRuns: $testRuns + repeatTests: $repeatTests testTargetsAlwaysRun: $testTargetsAlwaysRun """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 5017ea9b57..554f72e94d 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -4,6 +4,6 @@ interface IArgs { val async: Boolean val resultsBucket: String val projectId: String - val testRuns: Int + val repeatTests: Int val testShardChunks: List> } diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 4c023e2324..eec75f7db8 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -37,7 +37,7 @@ class IosArgs( private val flank = flankYml.flank val testShards = flank.testShards - override val testRuns = flank.testRuns + override val repeatTests = flank.repeatTests val testTargetsAlwaysRun = flank.testTargetsAlwaysRun private val iosFlank = iosFlankYml.flank @@ -89,7 +89,7 @@ IosArgs flank: testShards: $testShards - testRuns: $testRuns + repeatTests: $repeatTests testTargetsAlwaysRun: $testTargetsAlwaysRun # iOS flank testTargets: $testTargets diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index 213f18dcb1..9f2f79580c 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -8,18 +8,18 @@ import ftl.util.Utils.fatalError @JsonIgnoreProperties(ignoreUnknown = true) class FlankYmlParams( val testShards: Int = 1, - val testRuns: Int = 1, + val repeatTests: Int = 1, @field:JsonProperty("test-targets-always-run") val testTargetsAlwaysRun: List = emptyList() ) { companion object : IYmlKeys { - override val keys = listOf("testShards", "testRuns", "test-targets-always-run") + override val keys = listOf("testShards", "repeatTests", "test-targets-always-run") } init { if (testShards <= 0 && testShards != -1) fatalError("testShards must be >= 1 or -1") - if (testRuns < 1) fatalError("testRuns must be >= 1") + if (repeatTests < 1) fatalError("repeatTests must be >= 1") } } diff --git a/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt index 8e104a055d..cfd88f0af2 100644 --- a/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt @@ -22,7 +22,7 @@ object AndroidTestRunner { val apks = resolveApks(androidArgs, runGcsPath) val jobs = arrayListOf>() - val runCount = androidArgs.testRuns + val runCount = androidArgs.repeatTests val deviceCount = androidArgs.testShardChunks.size val shardCounter = ShardCounter() diff --git a/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt index c60d75aa81..145f2b9517 100644 --- a/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt @@ -53,7 +53,7 @@ object GenericTestRunner { } fun beforeRunMessage(args: IArgs): String { - val runCount = args.testRuns + val runCount = args.repeatTests val shardCount = args.testShardChunks.size val testsPerShard = args.testShardChunks.first().size val testsCount = args.testShardChunks.sumBy { it.size } diff --git a/test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt index 1f60faa1e2..962df8ce4c 100644 --- a/test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt @@ -31,7 +31,7 @@ object IosTestRunner { val xcTestParsed = Xctestrun.parse(yamlConfig.xctestrunFile) val jobs = arrayListOf>() - val runCount = yamlConfig.testRuns + val runCount = yamlConfig.repeatTests val deviceCount = yamlConfig.testShardChunks.size val shardCounter = ShardCounter() diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index b1a4909e2e..9b37677d33 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -80,7 +80,7 @@ class AndroidArgsFileTest { with(config) { assert(testShards, 1) - assert(testRuns, 1) + assert(repeatTests, 1) } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 507b70f582..8430c5594b 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -42,7 +42,7 @@ class AndroidArgsTest { flank: testShards: 7 - testRuns: 8 + repeatTests: 8 test-targets-always-run: - class example.Test#grantPermission """ @@ -124,7 +124,7 @@ class AndroidArgsTest { // FlankYml assert(testShards, 7) - assert(testRuns, 8) + assert(repeatTests, 8) assert(testTargetsAlwaysRun, listOf("class example.Test#grantPermission")) } } @@ -154,7 +154,7 @@ AndroidArgs flank: testShards: 7 - testRuns: 8 + repeatTests: 8 testTargetsAlwaysRun: [class example.Test#grantPermission] """.trimIndent() ) @@ -191,7 +191,7 @@ AndroidArgs // FlankYml assert(testShards, 1) - assert(testRuns, 1) + assert(repeatTests, 1) assert(testTargetsAlwaysRun, empty) } } diff --git a/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt b/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt index 39e7c9035d..f545e0b02f 100644 --- a/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt @@ -25,8 +25,8 @@ class FlankYmlTest { fun testValidArgs() { FlankYml() FlankYml(FlankYmlParams(testShards = -1)) - val yml = FlankYml(FlankYmlParams(testShards = 1, testRuns = 1)) - assertThat(yml.flank.testRuns).isEqualTo(1) + val yml = FlankYml(FlankYmlParams(testShards = 1, repeatTests = 1)) + assertThat(yml.flank.repeatTests).isEqualTo(1) assertThat(yml.flank.testShards).isEqualTo(1) assertThat(yml.flank.testTargetsAlwaysRun).isEqualTo(emptyList()) assertThat(FlankYml.map).isNotEmpty() @@ -39,8 +39,8 @@ class FlankYmlTest { } @Test - fun testInvalidTestRuns() { + fun testInvalidrepeatTests() { expectedExitRule.expectSystemExitWithStatus(-1) - FlankYml(FlankYmlParams(testRuns = 0)) + FlankYml(FlankYmlParams(repeatTests = 0)) } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt index 83f38190b2..d29b91f164 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt @@ -48,7 +48,7 @@ class IosArgsFileTest { ) ) assert(testShards, 1) - assert(testRuns, 1) + assert(repeatTests, 1) } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index c73c2e42c6..21c640ebe2 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -38,7 +38,7 @@ class IosArgsTest { flank: testShards: 7 - testRuns: 8 + repeatTests: 8 test-targets-always-run: - a/testGrantPermissions test-targets: @@ -84,7 +84,7 @@ class IosArgsTest { // FlankYml assert(testShards, 7) - assert(testRuns, 8) + assert(repeatTests, 8) assert(testTargetsAlwaysRun, listOf("a/testGrantPermissions")) // IosFlankYml @@ -111,7 +111,7 @@ IosArgs flank: testShards: 7 - testRuns: 8 + repeatTests: 8 testTargetsAlwaysRun: [a/testGrantPermissions] # iOS flank testTargets: [b/testBasicSelection] @@ -144,7 +144,7 @@ IosArgs // FlankYml assert(testShards, 1) - assert(testRuns, 1) + assert(repeatTests, 1) assert(testTargetsAlwaysRun, emptyList()) // IosFlankYml diff --git a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt index 03554fcd3e..4612c90112 100644 --- a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt +++ b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt @@ -50,7 +50,7 @@ gcloud: flank: testShards: 7 - testRuns: 8 + repeatTests: 8 test-targets-always-run: - . three: . @@ -98,7 +98,7 @@ gcloud: flank: testShards: . - testRuns: . + repeatTests: . test-targets-always-run: - . test-targets: diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.gcs.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.gcs.yml index 35220a005c..1f08a100a2 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.gcs.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.gcs.yml @@ -27,4 +27,4 @@ gcloud: flank: testShards: 1 - testRuns: 1 + repeatTests: 1 diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml index 81682d30d2..371634ec8d 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml @@ -17,4 +17,4 @@ flank: test-targets: - EarlGreyExampleMixedTests/testBasicSelection testShards: 1 - testRuns: 1 + repeatTests: 1 diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml index f91f601058..c532f8e726 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml @@ -15,4 +15,4 @@ flank: test-targets: - EarlGreyExampleMixedTests/testBasicSelection testShards: 1 - testRuns: 1 + repeatTests: 1 diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml index 8702bba843..d0530f6105 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml @@ -27,4 +27,4 @@ gcloud: flank: testShards: 1 - testRuns: 1 + repeatTests: 1 diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml index 85e5f60160..dbcb2ffa51 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml @@ -13,7 +13,7 @@ gcloud: flank: testShards: 2 - testRuns: 1 + repeatTests: 1 test-targets: - EarlGreyExampleMixedTests/testGrantCameraPermission - EarlGreyExampleMixedTests/testGrantMicrophonePermission diff --git a/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt index 5a6c8ad70c..cf2cc119ff 100644 --- a/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt @@ -9,9 +9,9 @@ import org.mockito.Mockito.mock class GenericTestRunnerTest { - private fun createMock(testRuns: Int, testShardChunks: List>): IArgs { + private fun createMock(repeatTests: Int, testShardChunks: List>): IArgs { val args = mock(IArgs::class.java) - `when`(args.testRuns).thenReturn(testRuns) + `when`(args.repeatTests).thenReturn(repeatTests) `when`(args.testShardChunks).thenReturn(testShardChunks) return args }