From 3fe1dad0020f0c72ced1f3ca919b255d7eb2f9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Wed, 13 Nov 2024 19:56:39 +0100 Subject: [PATCH 01/10] add support for JUnit 5 TestInstance.Lifecycle.PER_CLASS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- .../valueprovider/ValueProviderExtension.java | 38 +++-- .../tngtech/valueprovider/JUnit5Tests.java | 12 ++ ...clePerClassAllDisabledTestMethodsTest.java | 37 +++++ ...LifecyclePerClassDataProviderDemoTest.java | 132 ++++++++++++++++++ ...yclePerClassParameterizedTestDemoTest.java | 128 +++++++++++++++++ 5 files changed, 334 insertions(+), 13 deletions(-) create mode 100644 junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassAllDisabledTestMethodsTest.java create mode 100644 junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassDataProviderDemoTest.java create mode 100644 junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassParameterizedTestDemoTest.java diff --git a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java index ff1767c..9265c26 100644 --- a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java +++ b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java @@ -4,21 +4,15 @@ import java.lang.reflect.Method; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.AfterEachCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.BeforeEachCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.InvocationInterceptor; -import org.junit.jupiter.api.extension.ReflectiveInvocationContext; -import org.junit.jupiter.api.extension.TestExecutionExceptionHandler; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.api.extension.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.tngtech.valueprovider.ValueProviderExtension.TestMethodCycleState.BEFORE_FIRST_CYCLE; -import static com.tngtech.valueprovider.ValueProviderExtension.TestMethodCycleState.CYCLE_COMLETED; -import static com.tngtech.valueprovider.ValueProviderExtension.TestMethodCycleState.CYCLE_STARTED; +import static com.tngtech.valueprovider.ValueProviderExtension.TestMethodCycleState.*; import static java.lang.System.identityHashCode; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; public class ValueProviderExtension implements BeforeAllCallback, AfterAllCallback, @@ -47,7 +41,7 @@ enum TestMethodCycleState { public void beforeAll(ExtensionContext context) { logger.debug("{} beforeAll {}", identityHashCode(this), getTestClassName(context)); - startTestClassCycle(); + startTestClassCycleIf(context, PER_METHOD); } @Override @@ -55,6 +49,7 @@ public T interceptTestClassConstructor(Invocation invocation, ReflectiveI ExtensionContext extensionContext) throws Throwable { logger.debug("{} interceptTestClassConstructor {}", identityHashCode(this), buildQualifiedTestMethodName(extensionContext)); + startTestClassCycleIf(extensionContext, PER_CLASS); ensureStaticInitializationOfTestClass(extensionContext); startTestMethodCycle(); return invocation.proceed(); @@ -89,16 +84,23 @@ public void handleTestExecutionException(ExtensionContext context, Throwable thr public void afterEach(ExtensionContext context) { logger.debug("{} afterEach {}", identityHashCode(this), buildQualifiedTestMethodName(context)); - finishTestMethodCycle(); + finishTestMethodCycleIf(context, PER_METHOD); } @Override public void afterAll(ExtensionContext context) { logger.debug("{} afterAll {}", identityHashCode(this), getTestClassName(context)); + finishTestMethodCycleIf(context, PER_CLASS); finishTestClassCycle(); } + private void startTestClassCycleIf(ExtensionContext context, Lifecycle lifecycle) { + if (isLifecycle(context, lifecycle)) { + startTestClassCycle(); + } + } + private void startTestClassCycle() { ValueProviderFactory.startTestClassCycle(); resetTestMethodCycleState(); @@ -123,6 +125,12 @@ private void finishTestMethodCycleIfNecessary() { } } + private void finishTestMethodCycleIf(ExtensionContext context, Lifecycle lifecycle) { + if (isLifecycle(context, lifecycle)) { + finishTestMethodCycle(); + } + } + private void finishTestMethodCycle() { ValueProviderFactory.finishTestMethodCycle(); testMethodCycleState = CYCLE_COMLETED; @@ -150,4 +158,8 @@ private static String getTestMethodName(ExtensionContext context) { .map(Method::getName) .orElse(""); } + + private static boolean isLifecycle(ExtensionContext context, Lifecycle lifecycle) { + return lifecycle == context.getTestInstanceLifecycle().orElse(null); + } } diff --git a/junit5/src/test/java/com/tngtech/valueprovider/JUnit5Tests.java b/junit5/src/test/java/com/tngtech/valueprovider/JUnit5Tests.java index b33f1d2..f63baf7 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/JUnit5Tests.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/JUnit5Tests.java @@ -1,7 +1,12 @@ package com.tngtech.valueprovider; +import java.util.List; +import java.util.Objects; + import static com.tngtech.valueprovider.ValueProviderAsserter.reinitializeTestClassSeed; import static com.tngtech.valueprovider.ValueProviderAsserter.setSeedProperties; +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toList; public final class JUnit5Tests { /** @@ -16,4 +21,11 @@ public static void ensureDefinedFactoryState() { setSeedProperties(); reinitializeTestClassSeed(); } + + public static List asListWithoutNulls(ValueProvider... valueProviders) { + return stream(valueProviders) + .filter(Objects::nonNull) + .collect(toList()); + } + } diff --git a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassAllDisabledTestMethodsTest.java b/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassAllDisabledTestMethodsTest.java new file mode 100644 index 0000000..bc2e93f --- /dev/null +++ b/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassAllDisabledTestMethodsTest.java @@ -0,0 +1,37 @@ +package com.tngtech.valueprovider; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.api.extension.ExtendWith; + +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +/** + * No need for different sequences of enabled/disabled test methods + * like for {@link Lifecycle#PER_METHOD}, + * as test method cycle is started once in intercepted constructor, + * and only finished after last test method. + * + * @see DisabledEnabledDisabledTestMethodsTest + * @see EnabledDisabledEnabledTestMethodsTest + */ +@TestInstance(PER_CLASS) +@ExtendWith(ValueProviderExtension.class) +class LifecyclePerClassAllDisabledTestMethodsTest { + @Disabled + @Test + void a_test_disabled() { + } + + @Disabled + @Test + void b_test_disabled() { + } + + @Disabled + @Test + void c_test_disabled() { + } +} diff --git a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassDataProviderDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassDataProviderDemoTest.java new file mode 100644 index 0000000..98ce7a4 --- /dev/null +++ b/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassDataProviderDemoTest.java @@ -0,0 +1,132 @@ +package com.tngtech.valueprovider; + +import java.util.ArrayList; +import java.util.List; + +import com.tngtech.junit.dataprovider.DataProvider; +import com.tngtech.junit.dataprovider.UseDataProvider; +import com.tngtech.junit.dataprovider.UseDataProviderExtension; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.MethodOrderer.MethodName; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.tngtech.junit.dataprovider.DataProviders.$; +import static com.tngtech.junit.dataprovider.DataProviders.$$; +import static com.tngtech.valueprovider.JUnit5Tests.asListWithoutNulls; +import static com.tngtech.valueprovider.JUnit5Tests.ensureDefinedFactoryState; +import static com.tngtech.valueprovider.ValueProviderFactory.createRandomValueProvider; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +@TestMethodOrder(MethodName.class) +@ExtendWith({ValueProviderExtension.class, UseDataProviderExtension.class}) +class LifecyclePerClassDataProviderDemoTest { + private static final Logger logger = LoggerFactory.getLogger(LifecyclePerClassDataProviderDemoTest.class); + private static final ValueProvider classRandom1; + private static final ValueProvider classRandom2; + + static { + logger.debug("{}: static initialization", LifecyclePerClassDataProviderDemoTest.class.getSimpleName()); + ensureDefinedFactoryState(); + classRandom1 = createRandomValueProvider(); + classRandom2 = createRandomValueProvider(); + } + + private final ValueProvider instanceRandom = createRandomValueProvider(); + private ValueProvider beforeAllRandom; + private ValueProvider dataProviderRandom; + private ValueProvider beforeEachRandom; + private ValueProvider methodRandom; + + private final List randomsOfPreviousTestMethods = new ArrayList<>(); + + @BeforeAll + void beforeAll() { + beforeAllRandom = createRandomValueProvider(); + } + + @BeforeEach + void beforeEach() { + beforeEachRandom = createRandomValueProvider(); + } + + @AfterEach + void resetTestMethodRandoms() { + dataProviderRandom = null; + beforeEachRandom = null; + methodRandom = null; + } + + @DataProvider + Object[][] testValues1() { + logger.debug("create DataProvider 1"); + dataProviderRandom = createRandomValueProvider(); + return $$( + $(dataProviderRandom.fixedDecoratedString("1")), + $(dataProviderRandom.fixedDecoratedString("2")) + ); + } + + @TestTemplate + @UseDataProvider("testValues1") + void a_should_ensure_reproducible_ValueProvider_creation_for_DataProvider(String testValue) { + assertThat(testValue).isNotEmpty(); + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(dataProviderRandom, beforeEachRandom, methodRandom); + } + + @TestTemplate + @UseDataProvider("testValues1") + void b_should_ensure_reproducible_ValueProvider_creation_for_same_DataProvider(String testValue) { + assertThat(testValue).isNotEmpty(); + methodRandom = createRandomValueProvider(); + // @DataProvider is invoked ONCE BEFORE FIRST test method using it + verifyReproducibleValueProviderCreation(beforeEachRandom, methodRandom); + } + + @DataProvider + Object[][] testValues2() { + logger.debug("create DataProvider 2"); + dataProviderRandom = createRandomValueProvider(); + return $$( + $(dataProviderRandom.fixedDecoratedString("1")), + $(dataProviderRandom.fixedDecoratedString("2")) + ); + } + + @TestTemplate + @UseDataProvider("testValues2") + void c_should_ensure_proper_separation_of_test_class_and_test_method_cycles_for_DataProvider(String testValue) { + assertThat(testValue).isNotEmpty(); + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(dataProviderRandom, beforeEachRandom, methodRandom); + } + + @Test + void d_should_ensure_reproducible_ValueProvider_creation() { + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(beforeEachRandom, methodRandom); + } + + @Test + void e_should_ensure_proper_separation_of_test_class_and_test_method_cycles() { + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(beforeEachRandom, methodRandom); + } + + private void verifyReproducibleValueProviderCreation(ValueProvider... additionalMethodRandoms) { + List additionalMethodRandomList = asListWithoutNulls(additionalMethodRandoms); + new ValueProviderAsserter() + .addExpectedTestClassRandomValues(classRandom1, classRandom2) + .addExpectedTestMethodRandomValues(instanceRandom, beforeAllRandom) + .addExpectedTestMethodRandomValues(randomsOfPreviousTestMethods) + .addExpectedTestMethodRandomValues(additionalMethodRandomList) + .assertAllTestClassRandomValues() + .assertAllTestMethodRandomValues() + .assertAllSuffixes(); + randomsOfPreviousTestMethods.addAll(additionalMethodRandomList); + } +} diff --git a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassParameterizedTestDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassParameterizedTestDemoTest.java new file mode 100644 index 0000000..ae487ef --- /dev/null +++ b/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassParameterizedTestDemoTest.java @@ -0,0 +1,128 @@ +package com.tngtech.valueprovider; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.MethodOrderer.MethodName; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.tngtech.valueprovider.JUnit5Tests.asListWithoutNulls; +import static com.tngtech.valueprovider.JUnit5Tests.ensureDefinedFactoryState; +import static com.tngtech.valueprovider.ValueProviderFactory.createRandomValueProvider; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +@TestMethodOrder(MethodName.class) +@ExtendWith({ValueProviderExtension.class}) +class LifecyclePerClassParameterizedTestDemoTest { + private static final Logger logger = LoggerFactory.getLogger(LifecyclePerClassParameterizedTestDemoTest.class); + private static final ValueProvider classRandom1; + private static final ValueProvider classRandom2; + + static { + logger.debug("{}: static initialization", LifecyclePerClassParameterizedTestDemoTest.class.getSimpleName()); + ensureDefinedFactoryState(); + classRandom1 = createRandomValueProvider(); + classRandom2 = createRandomValueProvider(); + } + + private final ValueProvider instanceRandom = createRandomValueProvider(); + private ValueProvider beforeAllRandom; + private ValueProvider methodSourceRandom; + private ValueProvider beforeEachRandom; + private ValueProvider methodRandom; + + private final List randomsOfPreviousTestMethods = new ArrayList<>(); + + @BeforeAll + void beforeAll() { + beforeAllRandom = createRandomValueProvider(); + } + + @BeforeEach + void beforeEach() { + beforeEachRandom = createRandomValueProvider(); + } + + @AfterEach + void resetTestMethodRandoms() { + methodSourceRandom = null; + beforeEachRandom = null; + methodRandom = null; + } + + private Stream testValues1() { + logger.debug("create MethodSource 1"); + methodSourceRandom = createRandomValueProvider(); + return Stream.of( + methodSourceRandom.fixedDecoratedString("1"), + methodSourceRandom.fixedDecoratedString("2") + ); + } + + @ParameterizedTest + @MethodSource("testValues1") + void a_should_ensure_reproducible_ValueProvider_creation_for_MethodSource(String testValue) { + assertThat(testValue).isNotEmpty(); + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(methodSourceRandom, beforeEachRandom, methodRandom); + } + + @ParameterizedTest + @MethodSource("testValues1") + void b_should_ensure_reproducible_ValueProvider_creation_for_same_MethodSource(String testValue) { + assertThat(testValue).isNotEmpty(); + methodRandom = createRandomValueProvider(); + // @MethodSource is invoked for BEFORE EVERY test method using it + verifyReproducibleValueProviderCreation(methodSourceRandom, beforeEachRandom, methodRandom); + } + + private Stream testValues2() { + logger.debug("create MethodSource 2"); + methodSourceRandom = createRandomValueProvider(); + return Stream.of( + methodSourceRandom.fixedDecoratedString("1"), + methodSourceRandom.fixedDecoratedString("2") + ); + } + + @ParameterizedTest + @MethodSource("testValues2") + void c_should_ensure_proper_separation_of_test_class_and_test_method_cycles_for_MethodSource(String testValue) { + assertThat(testValue).isNotEmpty(); + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(methodSourceRandom, beforeEachRandom, methodRandom); + } + + @Test + void d_should_ensure_reproducible_ValueProvider_creation() { + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(beforeEachRandom, methodRandom); + } + + @Test + void e_should_ensure_proper_separation_of_test_class_and_test_method_cycles() { + methodRandom = createRandomValueProvider(); + verifyReproducibleValueProviderCreation(beforeEachRandom, methodRandom); + } + + private void verifyReproducibleValueProviderCreation(ValueProvider... additionalMethodRandoms) { + List additionalMethodRandomList = asListWithoutNulls(additionalMethodRandoms); + new ValueProviderAsserter() + .addExpectedTestClassRandomValues(classRandom1, classRandom2) + .addExpectedTestMethodRandomValues(instanceRandom, beforeAllRandom) + .addExpectedTestMethodRandomValues(randomsOfPreviousTestMethods) + .addExpectedTestMethodRandomValues(additionalMethodRandomList) + .assertAllTestClassRandomValues() + .assertAllTestMethodRandomValues() + .assertAllSuffixes(); + randomsOfPreviousTestMethods.addAll(additionalMethodRandomList); + } +} From dcb1383e7ea5e8199371e586fe474792a66ea03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Wed, 20 Nov 2024 13:38:12 +0100 Subject: [PATCH 02/10] provide all actual and expected RandomValues in failure message to ease error analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- .../valueprovider/ValueProviderAsserter.java | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/core/src/test/java/com/tngtech/valueprovider/ValueProviderAsserter.java b/core/src/test/java/com/tngtech/valueprovider/ValueProviderAsserter.java index 816bd85..474f55c 100644 --- a/core/src/test/java/com/tngtech/valueprovider/ValueProviderAsserter.java +++ b/core/src/test/java/com/tngtech/valueprovider/ValueProviderAsserter.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,6 +12,7 @@ import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_CLASS_SEED_PROPERTY; import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY; import static java.lang.System.setProperty; +import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; /** @@ -84,46 +86,47 @@ public ValueProviderAsserter assertAllSuffixes() { } private ValueProviderAsserter assertHasNextTestClassRandomValues(List providers) { - providers.forEach(this::assertHasNextTestClassRandomValues); - return this; - } - - /** - * Note that this method is coupled to the implementation of {@link InitializationCreator} wrt. {@link RandomValues} creation. - * It does currently NOT support the extra cycles that may be required for unique VP suffixes, - * as those can easily be avoided be appropriate initial seed values, at least for a small number of created VPs. - */ - private ValueProviderAsserter assertHasNextTestClassRandomValues(ValueProvider provider) { - assertThat(provider.getRandom()).isEqualTo(expectedNextTestClassRandomValues()); - return this; + return assertHasNextRandomValues(providers, "test-CLASS", this::expectedNextTestClassRandomValues); } private ValueProviderAsserter assertHasNextTestMethodRandomValues(List providers) { - providers.forEach(this::assertHasNextTestMethodRandomValues); - return this; + return assertHasNextRandomValues(providers, "test-METHOD", this::expectedNextTestMethodRandomValues); } /** - * @see #assertHasNextTestClassRandomValues(ValueProvider) + * Note that this method is coupled to the implementation of {@link InitializationCreator} wrt. {@link RandomValues} creation. + * It does currently NOT support the extra cycles that may be required for unique VP suffixes, + * as those can easily be avoided be appropriate initial seed values, at least for a small number of created VPs. */ - @SuppressWarnings("UnusedReturnValue") - private ValueProviderAsserter assertHasNextTestMethodRandomValues(ValueProvider provider) { - assertThat(provider.getRandom()).isEqualTo(expectedNextTestMethodRandomValues()); + private ValueProviderAsserter assertHasNextRandomValues( + List providers, + String testCycle, Supplier expectedRandomValuesSupplier) { + List expectedRandomValues = new ArrayList<>(); + List actualRandomValues = new ArrayList<>(); + providers.forEach(provider -> { + actualRandomValues.add(provider.getRandom()); + expectedRandomValues.add(expectedRandomValuesSupplier.get()); + }); + if (logger.isDebugEnabled()) { + List expectedSeeds = expectedRandomValues.stream() + .map(RandomValues::getSeed) + .collect(toList()); + logger.debug("assertHasNextRandomValues({}), expectedRandomValues seeds {}", testCycle, expectedSeeds); + } + assertThat(actualRandomValues).as("%s random values", testCycle).isEqualTo(expectedRandomValues); return this; } private RandomValues expectedNextTestClassRandomValues() { - return expectedNextRandomValues("test-CLASS", testClassSequence); + return expectedNextRandomValues(testClassSequence); } private RandomValues expectedNextTestMethodRandomValues() { - return expectedNextRandomValues("test-METHOD", testMethodSequence); + return expectedNextRandomValues(testMethodSequence); } - private RandomValues expectedNextRandomValues(String testCycle, RandomValuesSequence sequence) { - RandomValues random = sequence.nextRandomValues(); - logger.debug("expectedNextRandomValues({}), {}({})", testCycle, sequence.getSequenceCounter(), random.getSeed()); - return random; + private RandomValues expectedNextRandomValues(RandomValuesSequence sequence) { + return sequence.nextRandomValues(); } private ValueProviderAsserter assertSuffixes(Collection valueProviders) { From 7e17a154c7300427604c6f5fa9e5cbbbc42c0453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Wed, 20 Nov 2024 14:02:46 +0100 Subject: [PATCH 03/10] group related tests in respective subpackages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- .../{ => dataprovider}/DataProviderDemoTest.java | 5 ++++- .../LifecyclePerClassDataProviderDemoTest.java | 5 ++++- .../LifecyclePerClassParameterizedTestDemoTest.java | 5 ++++- .../{ => dataprovider}/ParameterizedTestDemoTest.java | 5 ++++- .../{ => dataprovider}/TestFactoryDemoTest.java | 5 ++++- .../{ => enableddisabled}/AllDisabledTestMethodsTest.java | 3 ++- .../DisabledEnabledDisabledTestMethodsTest.java | 3 ++- .../EnabledDisabledEnabledTestMethodsTest.java | 3 ++- .../LifecyclePerClassAllDisabledTestMethodsTest.java | 3 ++- 9 files changed, 28 insertions(+), 9 deletions(-) rename junit5/src/test/java/com/tngtech/valueprovider/{ => dataprovider}/DataProviderDemoTest.java (95%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => dataprovider}/LifecyclePerClassDataProviderDemoTest.java (96%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => dataprovider}/LifecyclePerClassParameterizedTestDemoTest.java (96%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => dataprovider}/ParameterizedTestDemoTest.java (94%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => dataprovider}/TestFactoryDemoTest.java (95%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => enableddisabled}/AllDisabledTestMethodsTest.java (83%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => enableddisabled}/DisabledEnabledDisabledTestMethodsTest.java (82%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => enableddisabled}/EnabledDisabledEnabledTestMethodsTest.java (82%) rename junit5/src/test/java/com/tngtech/valueprovider/{ => enableddisabled}/LifecyclePerClassAllDisabledTestMethodsTest.java (89%) diff --git a/junit5/src/test/java/com/tngtech/valueprovider/DataProviderDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/DataProviderDemoTest.java similarity index 95% rename from junit5/src/test/java/com/tngtech/valueprovider/DataProviderDemoTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/dataprovider/DataProviderDemoTest.java index ec36e39..baebd36 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/DataProviderDemoTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/DataProviderDemoTest.java @@ -1,4 +1,4 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.dataprovider; import java.util.ArrayList; import java.util.List; @@ -6,6 +6,9 @@ import com.tngtech.junit.dataprovider.DataProvider; import com.tngtech.junit.dataprovider.UseDataProvider; import com.tngtech.junit.dataprovider.UseDataProviderExtension; +import com.tngtech.valueprovider.ValueProvider; +import com.tngtech.valueprovider.ValueProviderAsserter; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassDataProviderDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/LifecyclePerClassDataProviderDemoTest.java similarity index 96% rename from junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassDataProviderDemoTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/dataprovider/LifecyclePerClassDataProviderDemoTest.java index 98ce7a4..60fb7ac 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassDataProviderDemoTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/LifecyclePerClassDataProviderDemoTest.java @@ -1,4 +1,4 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.dataprovider; import java.util.ArrayList; import java.util.List; @@ -6,6 +6,9 @@ import com.tngtech.junit.dataprovider.DataProvider; import com.tngtech.junit.dataprovider.UseDataProvider; import com.tngtech.junit.dataprovider.UseDataProviderExtension; +import com.tngtech.valueprovider.ValueProvider; +import com.tngtech.valueprovider.ValueProviderAsserter; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.*; import org.junit.jupiter.api.MethodOrderer.MethodName; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassParameterizedTestDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/LifecyclePerClassParameterizedTestDemoTest.java similarity index 96% rename from junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassParameterizedTestDemoTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/dataprovider/LifecyclePerClassParameterizedTestDemoTest.java index ae487ef..52e47bf 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassParameterizedTestDemoTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/LifecyclePerClassParameterizedTestDemoTest.java @@ -1,9 +1,12 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.dataprovider; import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; +import com.tngtech.valueprovider.ValueProvider; +import com.tngtech.valueprovider.ValueProviderAsserter; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.*; import org.junit.jupiter.api.MethodOrderer.MethodName; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/ParameterizedTestDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/ParameterizedTestDemoTest.java similarity index 94% rename from junit5/src/test/java/com/tngtech/valueprovider/ParameterizedTestDemoTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/dataprovider/ParameterizedTestDemoTest.java index a44c038..749e4a9 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/ParameterizedTestDemoTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/ParameterizedTestDemoTest.java @@ -1,9 +1,12 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.dataprovider; import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; +import com.tngtech.valueprovider.ValueProvider; +import com.tngtech.valueprovider.ValueProviderAsserter; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/TestFactoryDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/TestFactoryDemoTest.java similarity index 95% rename from junit5/src/test/java/com/tngtech/valueprovider/TestFactoryDemoTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/dataprovider/TestFactoryDemoTest.java index 98c6934..a941768 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/TestFactoryDemoTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/dataprovider/TestFactoryDemoTest.java @@ -1,8 +1,11 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.dataprovider; import java.util.List; import com.google.common.collect.ImmutableList; +import com.tngtech.valueprovider.ValueProvider; +import com.tngtech.valueprovider.ValueProviderAsserter; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DynamicTest; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/AllDisabledTestMethodsTest.java b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/AllDisabledTestMethodsTest.java similarity index 83% rename from junit5/src/test/java/com/tngtech/valueprovider/AllDisabledTestMethodsTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/AllDisabledTestMethodsTest.java index 962c4be..254c73e 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/AllDisabledTestMethodsTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/AllDisabledTestMethodsTest.java @@ -1,5 +1,6 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.enableddisabled; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.MethodOrderer.MethodName; import org.junit.jupiter.api.Test; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/DisabledEnabledDisabledTestMethodsTest.java b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/DisabledEnabledDisabledTestMethodsTest.java similarity index 82% rename from junit5/src/test/java/com/tngtech/valueprovider/DisabledEnabledDisabledTestMethodsTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/DisabledEnabledDisabledTestMethodsTest.java index 3558dfa..08d1969 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/DisabledEnabledDisabledTestMethodsTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/DisabledEnabledDisabledTestMethodsTest.java @@ -1,5 +1,6 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.enableddisabled; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.MethodOrderer.MethodName; import org.junit.jupiter.api.Test; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/EnabledDisabledEnabledTestMethodsTest.java b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/EnabledDisabledEnabledTestMethodsTest.java similarity index 82% rename from junit5/src/test/java/com/tngtech/valueprovider/EnabledDisabledEnabledTestMethodsTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/EnabledDisabledEnabledTestMethodsTest.java index e59d304..b53c17c 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/EnabledDisabledEnabledTestMethodsTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/EnabledDisabledEnabledTestMethodsTest.java @@ -1,5 +1,6 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.enableddisabled; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.MethodOrderer.MethodName; import org.junit.jupiter.api.Test; diff --git a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassAllDisabledTestMethodsTest.java b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/LifecyclePerClassAllDisabledTestMethodsTest.java similarity index 89% rename from junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassAllDisabledTestMethodsTest.java rename to junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/LifecyclePerClassAllDisabledTestMethodsTest.java index bc2e93f..9d467da 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/LifecyclePerClassAllDisabledTestMethodsTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/enableddisabled/LifecyclePerClassAllDisabledTestMethodsTest.java @@ -1,5 +1,6 @@ -package com.tngtech.valueprovider; +package com.tngtech.valueprovider.enableddisabled; +import com.tngtech.valueprovider.ValueProviderExtension; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; From 9183674bdd8bd8011a2834ea0a90922e40b80ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Wed, 20 Nov 2024 17:06:44 +0100 Subject: [PATCH 04/10] use slf4j-simple instead of log4j2 to ease debug logging for tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- build.gradle | 13 +++++-------- core/src/test/resources/log4j2.xml | 6 ------ core/src/test/resources/test-log4j2.xml | 20 -------------------- junit5/src/test/resources/log4j2.xml | 6 ------ junit5/src/test/resources/test-log4j2.xml | 20 -------------------- 5 files changed, 5 insertions(+), 60 deletions(-) delete mode 100644 core/src/test/resources/log4j2.xml delete mode 100644 core/src/test/resources/test-log4j2.xml delete mode 100644 junit5/src/test/resources/log4j2.xml delete mode 100644 junit5/src/test/resources/test-log4j2.xml diff --git a/build.gradle b/build.gradle index abe60ae..13cfc4c 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ ext { dependency = [ apache_commons : [group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'], guava : [group: 'com.google.guava', name: 'guava', version: '33.0.0-jre'], - slf4j : [group: 'org.slf4j', name: 'slf4j-api', version: '2.0.12'], + slf4j_api : [group: 'org.slf4j', name: 'slf4j-api', version: '2.0.16'], lombok : [group: 'org.projectlombok', name: 'lombok', version: '1.18.30'], junit4 : [group: 'junit', name: 'junit', version: '4.13.2'], @@ -48,9 +48,7 @@ ext { junit5_dataprovider : [group: 'com.tngtech.junit.dataprovider', name: 'junit-jupiter-dataprovider', version: '2.10'], assertj_core : [group: 'org.assertj', name: 'assertj-core', version: '3.25.3'], mockito : [group: 'org.mockito', name: 'mockito-core', version: '5.10.0'], - log4j_api : [group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.23.0'], - log4j_core : [group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.23.0'], - log4j_slf4j : [group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.23.0'], + slf4j_simple : [group: 'org.slf4j', name: 'slf4j-simple', version: '2.0.16'], junit4_engine : [group: 'org.junit.vintage', name: 'junit-vintage-engine', version: '5.10.2'], junit_jupiter_api : [group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.10.2'], @@ -110,15 +108,14 @@ subprojects { dependencies { implementation dependency.apache_commons implementation dependency.guava - implementation dependency.slf4j + implementation dependency.slf4j_api testImplementation dependency.assertj_core testImplementation dependency.junit4 testImplementation dependency.junit_jupiter_api - testRuntimeOnly dependency.log4j_slf4j - testRuntimeOnly dependency.log4j_api - testRuntimeOnly dependency.log4j_core + // hint: activate (debug) logging via system property -Dorg.slf4j.simpleLogger.defaultLogLevel=debug + testRuntimeOnly dependency.slf4j_simple } repositories { diff --git a/core/src/test/resources/log4j2.xml b/core/src/test/resources/log4j2.xml deleted file mode 100644 index 6cd4ca8..0000000 --- a/core/src/test/resources/log4j2.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/core/src/test/resources/test-log4j2.xml b/core/src/test/resources/test-log4j2.xml deleted file mode 100644 index 686a56a..0000000 --- a/core/src/test/resources/test-log4j2.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/junit5/src/test/resources/log4j2.xml b/junit5/src/test/resources/log4j2.xml deleted file mode 100644 index 6cd4ca8..0000000 --- a/junit5/src/test/resources/log4j2.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/junit5/src/test/resources/test-log4j2.xml b/junit5/src/test/resources/test-log4j2.xml deleted file mode 100644 index 2adfa35..0000000 --- a/junit5/src/test/resources/test-log4j2.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 3eeb5b2647c5233e42d9854a999457291c836f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Wed, 20 Nov 2024 13:38:24 +0100 Subject: [PATCH 05/10] add support for JUnit 5 @Nested test classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test with relevant combinations of Lifecycle PER_METHOD and PER_CLASS Signed-off-by: Jonas Höf --- .../valueprovider/ValueProviderExtension.java | 103 ++++++++++-- .../LifecycleMainPerClassNestedDemoTest.java | 127 +++++++++++++++ .../LifecycleMainPerMethodNestedDemoTest.java | 150 ++++++++++++++++++ 3 files changed, 366 insertions(+), 14 deletions(-) create mode 100644 junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerClassNestedDemoTest.java create mode 100644 junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerMethodNestedDemoTest.java diff --git a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java index 9265c26..7e9f4f5 100644 --- a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java +++ b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java @@ -2,6 +2,8 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Method; +import java.util.*; +import java.util.function.Consumer; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -45,13 +47,15 @@ public void beforeAll(ExtensionContext context) { } @Override - public T interceptTestClassConstructor(Invocation invocation, ReflectiveInvocationContext> invocationContext, + public T interceptTestClassConstructor( + Invocation invocation, + ReflectiveInvocationContext> invocationContext, ExtensionContext extensionContext) throws Throwable { logger.debug("{} interceptTestClassConstructor {}", identityHashCode(this), buildQualifiedTestMethodName(extensionContext)); startTestClassCycleIf(extensionContext, PER_CLASS); ensureStaticInitializationOfTestClass(extensionContext); - startTestMethodCycle(); + startTestMethodCycle(extensionContext); return invocation.proceed(); } @@ -84,19 +88,23 @@ public void handleTestExecutionException(ExtensionContext context, Throwable thr public void afterEach(ExtensionContext context) { logger.debug("{} afterEach {}", identityHashCode(this), buildQualifiedTestMethodName(context)); - finishTestMethodCycleIf(context, PER_METHOD); + if (testClassHierarchyHasOnlyLifecyclePerMethod(context)) { + finishTestMethodCycle(); + } } @Override public void afterAll(ExtensionContext context) { logger.debug("{} afterAll {}", identityHashCode(this), getTestClassName(context)); - finishTestMethodCycleIf(context, PER_CLASS); - finishTestClassCycle(); + if (isLastTestClassInHierarchyWithLifecyclePerClass(context)) { + finishTestMethodCycle(); + } + finishTestClassCycle(context); } private void startTestClassCycleIf(ExtensionContext context, Lifecycle lifecycle) { - if (isLifecycle(context, lifecycle)) { + if (isLifecycle(context, lifecycle) && !isNestedTestClass(context)) { startTestClassCycle(); } } @@ -106,13 +114,19 @@ private void startTestClassCycle() { resetTestMethodCycleState(); } - private void startTestMethodCycle() { + private void startTestMethodCycle(ExtensionContext context) { + if (isNestedTestClass(context)) { + return; + } finishTestMethodCycleIfNecessary(); ValueProviderFactory.startTestMethodCycle(); testMethodCycleState = CYCLE_STARTED; } - private void finishTestClassCycle() { + private void finishTestClassCycle(ExtensionContext context) { + if (isNestedTestClass(context)) { + return; + } finishTestMethodCycleIfNecessary(); ValueProviderFactory.finishTestClassCycle(); resetTestMethodCycleState(); @@ -125,12 +139,6 @@ private void finishTestMethodCycleIfNecessary() { } } - private void finishTestMethodCycleIf(ExtensionContext context, Lifecycle lifecycle) { - if (isLifecycle(context, lifecycle)) { - finishTestMethodCycle(); - } - } - private void finishTestMethodCycle() { ValueProviderFactory.finishTestMethodCycle(); testMethodCycleState = CYCLE_COMLETED; @@ -162,4 +170,71 @@ private static String getTestMethodName(ExtensionContext context) { private static boolean isLifecycle(ExtensionContext context, Lifecycle lifecycle) { return lifecycle == context.getTestInstanceLifecycle().orElse(null); } + + private static boolean isLastTestClassInHierarchyWithLifecyclePerClass(ExtensionContext context) { + if (!isLifecycle(context, PER_CLASS)) { + return false; + } + Set remainingLifecyclesInHierarchy = determineLifecyclesInTestClassHierarchy(context.getParent()); + return remainingLifecyclesInHierarchy.isEmpty() || containsOnlyLifecyclePerMethod(remainingLifecyclesInHierarchy); + } + + private static boolean testClassHierarchyHasOnlyLifecyclePerMethod(ExtensionContext context) { + Set lifecyclesInHierarchy = determineLifecyclesInTestClassHierarchy(Optional.of(context)); + return containsOnlyLifecyclePerMethod(lifecyclesInHierarchy); + } + + private static boolean containsOnlyLifecyclePerMethod(Set lifecyclesInHierarchy) { + return lifecyclesInHierarchy.size() == 1 && lifecyclesInHierarchy.contains(PER_METHOD); + } + + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private static Set determineLifecyclesInTestClassHierarchy(Optional optionalContext) { + Set lifecycles = new HashSet<>(); + traverseContextHierarchy(optionalContext, + context -> { + Optional lifecycle = context.getTestInstanceLifecycle(); + lifecycle.ifPresent(lifecycles::add); + } + ); + return lifecycles; + } + + private static boolean isNestedTestClass(ExtensionContext context) { + return determineNumTestClassesInHierarchy(context) > 1; + } + + /** + * Cannot check e.g. for context class being org.junit.jupiter.engine.descriptor.ClassExtensionContext, + * as this class is package private. Only ClassExtensionContext instances seem to have a non-empty + * {@link Lifecycle} (tested via debugger for JUnit 5.10.2), however, so this is used as criteria instead. + * This seems reasonable, as the {@link Lifecycle} is exactly what controls instantiation of test classes. + */ + private static int determineNumTestClassesInHierarchy(ExtensionContext startContext) { + List contextsWithLifecycle = new ArrayList<>(); + traverseContextHierarchy(startContext, + context -> { + boolean hasLifecycle = context.getTestInstanceLifecycle().isPresent(); + if (hasLifecycle) { + contextsWithLifecycle.add(context); + } + } + ); + return contextsWithLifecycle.size(); + } + + private static void traverseContextHierarchy(ExtensionContext startContext, + Consumer contextConsumer) { + traverseContextHierarchy(Optional.of(startContext), contextConsumer); + } + + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private static void traverseContextHierarchy(Optional optionalContext, + Consumer contextConsumer) { + while (optionalContext.isPresent()) { + ExtensionContext context = optionalContext.get(); + contextConsumer.accept(context); + optionalContext = context.getParent(); + } + } } diff --git a/junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerClassNestedDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerClassNestedDemoTest.java new file mode 100644 index 0000000..24a09d5 --- /dev/null +++ b/junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerClassNestedDemoTest.java @@ -0,0 +1,127 @@ +package com.tngtech.valueprovider.nested; + +import java.util.ArrayList; +import java.util.List; + +import com.tngtech.valueprovider.ValueProvider; +import com.tngtech.valueprovider.ValueProviderAsserter; +import com.tngtech.valueprovider.ValueProviderExtension; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.MethodOrderer.MethodName; +import org.junit.jupiter.api.extension.ExtendWith; + +import static com.tngtech.valueprovider.JUnit5Tests.ensureDefinedFactoryState; +import static com.tngtech.valueprovider.ValueProviderFactory.createRandomValueProvider; +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; + +@TestInstance(PER_CLASS) +@DisplayName("Main test class, Lifecycle PER_CLASS") +@ExtendWith(ValueProviderExtension.class) +class LifecycleMainPerClassNestedDemoTest { + private static final ValueProvider classRandom; + + static { + ensureDefinedFactoryState(); + classRandom = createRandomValueProvider(); + } + + private final List randomsOfPreviousTestMethods = new ArrayList<>(); + private final ValueProvider mainInstanceRandom = createRandomValueProvider(); + private ValueProvider mainBeforeAllRandom; + private ValueProvider mainBeforeEachRandom; + + @BeforeAll + void mainBeforeAll() { + mainBeforeAllRandom = createRandomValueProvider(); + } + + @BeforeEach + void mainBeforeEach() { + mainBeforeEachRandom = createRandomValueProvider(); + } + + @Test + void should_ensure_reproducible_ValueProvider_creation_in_main_class() { + verifyReproducibleValueProviderCreation(mainBeforeEachRandom, createRandomValueProvider()); + } + + @Test + void should_ensure_proper_separation_of_test_class_and_test_method_cycles_in_main_class() { + verifyReproducibleValueProviderCreation(mainBeforeEachRandom, createRandomValueProvider()); + } + + @Nested + @TestInstance(PER_METHOD) + @DisplayName("Nested test class, Lifecycle PER_METHOD") + class LifecycleNestedPerMethod { + private final ValueProvider nestedInstanceRandom = createRandomValueProvider(); + private ValueProvider nestedBeforeEachRandom; + + @BeforeEach + void nestedBeforeEach() { + nestedBeforeEachRandom = createRandomValueProvider(); + } + + @Test + void should_ensure_reproducible_ValueProvider_creation_in_nested_class() { + verifyReproducibleValueProviderCreation(nestedInstanceRandom, + mainBeforeEachRandom, nestedBeforeEachRandom, + createRandomValueProvider()); + } + + @Test + void should_ensure_proper_separation_of_test_class_and_test_method_cycles_in_nested_class() { + verifyReproducibleValueProviderCreation(nestedInstanceRandom, + mainBeforeEachRandom, nestedBeforeEachRandom, + createRandomValueProvider()); + } + } + + @Nested + @TestInstance(PER_CLASS) + @TestMethodOrder(MethodName.class) + @DisplayName("Nested test class, Lifecycle PER_CLASS") + class LifecycleNestedPerClass { + private final ValueProvider nestedInstanceRandom = createRandomValueProvider(); + private ValueProvider nestedBeforeAllRandom; + private ValueProvider nestedBeforeEachRandom; + + @BeforeAll + void nestedBeforeAll() { + nestedBeforeAllRandom = createRandomValueProvider(); + } + + @BeforeEach + void nestedBeforeEach() { + nestedBeforeEachRandom = createRandomValueProvider(); + } + + @Test + void a_should_ensure_reproducible_ValueProvider_creation_in_nested_class() { + verifyReproducibleValueProviderCreation(nestedInstanceRandom, nestedBeforeAllRandom, + mainBeforeEachRandom, nestedBeforeEachRandom, + createRandomValueProvider()); + } + + @Test + void b_should_ensure_proper_separation_of_test_class_and_test_method_cycles_in_nested_class() { + verifyReproducibleValueProviderCreation(mainBeforeEachRandom, nestedBeforeEachRandom, + createRandomValueProvider()); + } + } + + private void verifyReproducibleValueProviderCreation(ValueProvider... additionalTestMethodRandomValues) { + List additionalTestMethodRamdomValuesList = asList(additionalTestMethodRandomValues); + new ValueProviderAsserter() + .addExpectedTestClassRandomValues(classRandom) + .addExpectedTestMethodRandomValues(mainInstanceRandom, mainBeforeAllRandom) + .addExpectedTestMethodRandomValues(randomsOfPreviousTestMethods) + .addExpectedTestMethodRandomValues(additionalTestMethodRamdomValuesList) + .assertAllTestClassRandomValues() + .assertAllTestMethodRandomValues() + .assertAllSuffixes(); + randomsOfPreviousTestMethods.addAll(additionalTestMethodRamdomValuesList); + } +} diff --git a/junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerMethodNestedDemoTest.java b/junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerMethodNestedDemoTest.java new file mode 100644 index 0000000..9c05fe6 --- /dev/null +++ b/junit5/src/test/java/com/tngtech/valueprovider/nested/LifecycleMainPerMethodNestedDemoTest.java @@ -0,0 +1,150 @@ +package com.tngtech.valueprovider.nested; + +import java.util.ArrayList; +import java.util.List; + +import com.tngtech.valueprovider.RandomValues; +import com.tngtech.valueprovider.ValueProvider; +import com.tngtech.valueprovider.ValueProviderAsserter; +import com.tngtech.valueprovider.ValueProviderExtension; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.api.extension.ExtendWith; + +import static com.tngtech.valueprovider.JUnit5Tests.ensureDefinedFactoryState; +import static com.tngtech.valueprovider.ValueProviderFactory.createRandomValueProvider; +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@DisplayName("Main test class, default Lifecycle PER_METHOD") +@ExtendWith(ValueProviderExtension.class) +class LifecycleMainPerMethodNestedDemoTest { + private static final ValueProvider classRandom; + + static { + ensureDefinedFactoryState(); + classRandom = createRandomValueProvider(); + } + + private static ValueProvider mainBeforeAllRandom; + private final ValueProvider mainInstanceRandom = createRandomValueProvider(); + private ValueProvider mainBeforeEachRandom; + + @BeforeAll + static void mainBeforeAll() { + mainBeforeAllRandom = createRandomValueProvider(); + } + + @BeforeEach + void mainBeforeEach() { + mainBeforeEachRandom = createRandomValueProvider(); + } + + @Test + void should_ensure_reproducible_ValueProvider_creation_in_main_class() { + verifyReproducibleValueProviderCreation(mainBeforeEachRandom, createRandomValueProvider()); + } + + @Test + void should_ensure_proper_separation_of_test_class_and_test_method_cycles_in_main_class() { + verifyReproducibleValueProviderCreation(mainBeforeEachRandom, createRandomValueProvider()); + } + + @Nested + @DisplayName("Nested test class (level 1), default Lifecycle PER_METHOD") + class LifecycleNestedLevel1PerMethod { + private final ValueProvider nestedLevel1InstanceRandom = createRandomValueProvider(); + private ValueProvider nestedLevel1BeforeEachRandom; + + @BeforeEach + void nestedLevel1BeforeEach() { + nestedLevel1BeforeEachRandom = createRandomValueProvider(); + } + + @Nested + @DisplayName("Nested test class (level 2), default Lifecycle PER_METHOD") + class LifecycleNestedLevel2PerMethod { + private final ValueProvider nestedLevel2InstanceRandom = createRandomValueProvider(); + private ValueProvider nestedLevel2BeforeEachRandom; + + @BeforeEach + void nestedLevel2BeforeEach() { + nestedLevel2BeforeEachRandom = createRandomValueProvider(); + } + + @Test + void should_ensure_reproducible_ValueProvider_creation_with_more_than_one_nesting_level() { + verifyReproducibleValueProviderCreation(nestedLevel1InstanceRandom, nestedLevel2InstanceRandom, + mainBeforeEachRandom, nestedLevel1BeforeEachRandom, nestedLevel2BeforeEachRandom, + createRandomValueProvider()); + } + + @Test + void should_ensure_proper_separation_of_test_class_and_test_method_cycles_with_more_than_one_nesting_level() { + verifyReproducibleValueProviderCreation(nestedLevel1InstanceRandom, nestedLevel2InstanceRandom, + mainBeforeEachRandom, nestedLevel1BeforeEachRandom, nestedLevel2BeforeEachRandom, + createRandomValueProvider()); + } + } + } + + private void verifyReproducibleValueProviderCreation(ValueProvider... additionalTestMethodRandomValues) { + new ValueProviderAsserter() + .addExpectedTestClassRandomValues(classRandom, mainBeforeAllRandom) + .addExpectedTestMethodRandomValues(mainInstanceRandom) + .addExpectedTestMethodRandomValues(asList(additionalTestMethodRandomValues)) + .assertAllTestClassRandomValues() + .assertAllTestMethodRandomValues() + .assertAllSuffixes(); + } + + @Nested + @TestInstance(PER_CLASS) + @DisplayName("Nested test class, Lifecycle PER_CLASS") + class LifecycleNestedPerClass { + private final List randomsOfPreviousTestMethods = new ArrayList<>(); + private final ValueProvider nestedInstanceRandom = createRandomValueProvider(); + private ValueProvider nestedBeforeAllRandom; + private ValueProvider nestedBeforeEachRandom; + + @BeforeAll + void nestedBeforeAll() { + nestedBeforeAllRandom = createRandomValueProvider(); + } + + @BeforeEach + void nestedBeforeEach() { + nestedBeforeEachRandom = createRandomValueProvider(); + } + + @Test + void should_ensure_reproducible_ValueProvider_creation_with_more_than_one_nesting_level() { + verifyReproducibleValueProviderCreation(mainBeforeEachRandom, nestedBeforeEachRandom, + createRandomValueProvider()); + } + + @Test + void should_ensure_proper_separation_of_test_class_and_test_method_cycles_in_nested_class() { + verifyReproducibleValueProviderCreation(mainBeforeEachRandom, nestedBeforeEachRandom, + createRandomValueProvider()); + } + + /** + * Note: {@link Lifecycle#PER_CLASS} requires separate verification method + * that adds the just verified {@link RandomValues}, as the test method cycle must not be reset + * before completion of the last test method. + */ + private void verifyReproducibleValueProviderCreation(ValueProvider... additionalTestMethodRandomValues) { + List additionalTestMethodRamdomValuesList = asList(additionalTestMethodRandomValues); + new ValueProviderAsserter() + .addExpectedTestClassRandomValues(classRandom, mainBeforeAllRandom) + .addExpectedTestMethodRandomValues(mainInstanceRandom, nestedInstanceRandom, nestedBeforeAllRandom) + .addExpectedTestMethodRandomValues(randomsOfPreviousTestMethods) + .addExpectedTestMethodRandomValues(additionalTestMethodRamdomValuesList) + .assertAllTestClassRandomValues() + .assertAllTestMethodRandomValues() + .assertAllSuffixes(); + randomsOfPreviousTestMethods.addAll(additionalTestMethodRamdomValuesList); + } + } +} From 9b2d34b460d9d7177206185cc9e5cd46acdb5b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Mon, 2 Dec 2024 13:56:27 +0100 Subject: [PATCH 06/10] fix log message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * restrict to test class name, as test method is not set when interceptor method is invoked Signed-off-by: Jonas Höf --- .../java/com/tngtech/valueprovider/ValueProviderExtension.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java index 7e9f4f5..95a6c9c 100644 --- a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java +++ b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java @@ -52,7 +52,7 @@ public T interceptTestClassConstructor( ReflectiveInvocationContext> invocationContext, ExtensionContext extensionContext) throws Throwable { logger.debug("{} interceptTestClassConstructor {}", - identityHashCode(this), buildQualifiedTestMethodName(extensionContext)); + identityHashCode(this), getTestClassName(extensionContext)); startTestClassCycleIf(extensionContext, PER_CLASS); ensureStaticInitializationOfTestClass(extensionContext); startTestMethodCycle(extensionContext); From 0d090b4ea2f54be37d80c7a00d2e966f4bffe038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Thu, 28 Nov 2024 15:17:44 +0100 Subject: [PATCH 07/10] update README.md to reflect the differences for reproducing test failures depending on the JUnit 5 test lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- README.md | 206 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 145 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index f544acd..c4109d0 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ # value-provider -value-provider is a free library that facilitates writing realistic test data and in turn better tests for your Java application. -It works best in conjunction with reusable test data factories that encapsulate creating valid instances for your data objects. +value-provider is a free library that facilitates writing realistic test data and in turn better tests for your Java +application. +It works best in conjunction with reusable test data factories that encapsulate creating valid instances for your data +objects. value-provider consists of two major parts: -* the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) class which populates properties of test data objects with random values +* the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) class which populates properties + of test data objects with random values * infrastructure for reproducing said random data in case of test failures (JUnit5 extension, JUnit4 rules) ## Contributing @@ -14,7 +17,8 @@ Pull requests are welcome. For major changes, please open an issue first to disc Please make sure to update the tests as appropriate. -For further information, please refer to [CONTRIBUTING.md](CONTRIBUTING.md). For technical details, you may find the sequence diagrams in the `doc` directory helpful. +For further information, please refer to [CONTRIBUTING.md](CONTRIBUTING.md). For technical details, you may find the +sequence diagrams in the `doc` directory helpful. ## Installation @@ -28,13 +32,13 @@ value-provider has the following prerequisites: ```groovy // core library -testImplementation 'com.tngtech.valueprovider:value-provider-core:1.2.3' +testImplementation 'com.tngtech.valueprovider:value-provider-core:1.3.0' // infrastructure // for JUnit 5 -testImplementation 'com.tngtech.valueprovider:value-provider-junit5:1.2.3' +testImplementation 'com.tngtech.valueprovider:value-provider-junit5:1.3.0' // alternatively, for JUnit 4 -testImplementation 'com.tngtech.valueprovider:value-provider-junit4:1.2.3' +testImplementation 'com.tngtech.valueprovider:value-provider-junit4:1.3.0' ``` ### Maven @@ -47,25 +51,25 @@ testImplementation 'com.tngtech.valueprovider:value-provider-junit4:1.2.3' com.tngtech.valueprovider - value-provider-core - 1.2.3 - test + value-provider-core + 1.3.0 + test com.tngtech.valueprovider - value-provider-junit5 - 1.2.3 - test + value-provider-junit5 + 1.3.0 + test com.tngtech.valueprovider - value-provider-junit4 - 1.2.3 - test + value-provider-junit4 + 1.3.0 + test @@ -74,7 +78,8 @@ testImplementation 'com.tngtech.valueprovider:value-provider-junit4:1.2.3' ## Usage -We strongly recommend implementing reusable __test data factories__ that encapsulate creating valid instances for your test data objects. +We strongly recommend implementing reusable __test data factories__ that encapsulate creating valid instances for your +test data objects. #### A Simple test data factory @@ -96,7 +101,8 @@ public class Product { } ``` -... the test data factory would look like... (see also [ProductTestDataFactory](example/src/test/java/com/tngtech/valueprovider/example/ProductTestDataFactory.java)): +... the test data factory would look like... (see +also [ProductTestDataFactory](example/src/test/java/com/tngtech/valueprovider/example/ProductTestDataFactory.java)): ```java import com.tngtech.valueprovider.ValueProvider; @@ -126,7 +132,8 @@ The [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.j * select a [ProductCategory](example/src/main/java/com/tngtech/valueprovider/example/ProductCategory.java) at random * populate the `name` and `description` properties with a so called *decorated* string -What is a decorated string? Let's have a look at the following example output when invoking `ProductTestDataFactory.createProduct()` multiple times: +What is a decorated string? Let's have a look at the following example output when invoking +`ProductTestDataFactory.createProduct()` multiple times: ``` Product(category=CAR, name=nameaPr, description=descriptionaPr) @@ -134,16 +141,22 @@ Product(category=COMPUTER, name=nameyBp, description=descriptionyBp) Product(category=COMPUTER, name=namejeM, description=descriptionjeM) ``` -The *decoration* is simply a 3 letter suffix that is appended to the base string provided as parameter to the `fixedDecoratedString()` method. +The *decoration* is simply a 3 letter suffix that is appended to the base string provided as parameter to the +`fixedDecoratedString()` method. -Note that the suffix of all String properties of one [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java) instance stays the same, since the values are populated using the -same [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) instance. Conversely, different [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java) -instances contain properties with different suffixes. This is achieved by invoking `createRandomValueProvider()`, when calling +Note that the suffix of all String properties of +one [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java) instance stays the same, since the +values are populated using the +same [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) instance. Conversely, +different [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java) +instances contain properties with different suffixes. This is achieved by invoking `createRandomValueProvider()`, when +calling the [ProductTestDataFactory](example/src/test/java/com/tngtech/valueprovider/example/ProductTestDataFactory.java). #### Complex test data factories -Having seen the basics, let's move on to a more complex example. Consider an [Order](example/src/main/java/com/tngtech/valueprovider/example/Order.java): +Having seen the basics, let's move on to a more complex example. Consider +an [Order](example/src/main/java/com/tngtech/valueprovider/example/Order.java): ```java // ... @@ -197,18 +210,24 @@ shippingAddress=Address(zip=81571, city=S-citykwh, street=S-streetkwh, number=17 billingAddress=Address(zip=71331, city=B-citykwh, street=B-streetkwh, number=169)) ``` -Note that the 3 letter suffix that we already saw in the [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java) example is shared for the entire hierarchy of objects that +Note that the 3 letter suffix that we already saw in +the [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java) example is shared for the entire +hierarchy of objects that comprise an order. It therefore eases recognizing objects that belong together. -The output also demonstrates further aspects of randomization in test data factories. The 3 order objects all have a different number of order items, and e.g., the shipping and billing addresses have +The output also demonstrates further aspects of randomization in test data factories. The 3 order objects all have a +different number of order items, and e.g., the shipping and billing addresses have random zip codes or house numbers. -Last but not least, note the second aspect of string *decoration*. The products in the order items have an additional prefix in their string properties (e.g., 'A-', 'B-', ...). This is required to -differentiate multiple objects of the same kind. The same applies to the shipping and billing addresses. They have different prefixes, if they differ ('S-' vs. 'B-'), but no prefix, if they are the +Last but not least, note the second aspect of string *decoration*. The products in the order items have an additional +prefix in their string properties (e.g., 'A-', 'B-', ...). This is required to +differentiate multiple objects of the same kind. The same applies to the shipping and billing addresses. They have +different prefixes, if they differ ('S-' vs. 'B-'), but no prefix, if they are the same. Let's take a look at -[OrderTestDataFactory](example/src/test/java/com/tngtech/valueprovider/example/OrderTestDataFactory.java) - how to achieve all this: +[OrderTestDataFactory](example/src/test/java/com/tngtech/valueprovider/example/OrderTestDataFactory.java) - how to +achieve all this: ```java import com.tngtech.valueprovider.example.Order.OrderBuilder; @@ -268,7 +287,8 @@ public final class OrderTestDataFactory { } ``` -Sharing the same suffix for the entire object hierarchy is easy, just pass the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) instance to each invoked test data +Sharing the same suffix for the entire object hierarchy is easy, just pass +the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) instance to each invoked test data factory as in: ```java @@ -303,19 +323,29 @@ public final class OrderTestDataFactory { } ``` -The `copyWithChangedPrefix()` method takes the suffix of the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) for which it is called, and creates a new instance with the passed -prefix. Like the suffix, the prefix remains the same for the lifetime of the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java). +The `copyWithChangedPrefix()` method takes the suffix of +the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) for which it is called, and creates +a new instance with the passed +prefix. Like the suffix, the prefix remains the same for the lifetime of +the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java). A final aspect that is related to using lombok: -As opposed to a [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java), creating an [Order](example/src/main/java/com/tngtech/valueprovider/example/Order.java) is done via a -builder rather than via a factory method. The [OrderTestDataFactory](example/src/test/java/com/tngtech/valueprovider/example/OrderTestDataFactory.java) therefore has 4 methods, a pair +As opposed to a [Product](example/src/main/java/com/tngtech/valueprovider/example/Product.java), creating +an [Order](example/src/main/java/com/tngtech/valueprovider/example/Order.java) is done via a +builder rather than via a factory method. +The [OrderTestDataFactory](example/src/test/java/com/tngtech/valueprovider/example/OrderTestDataFactory.java) therefore +has 4 methods, a pair of `createOrder()` and a pair of `createOrderBuilder()` -methods. Again, one of each pair has the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) as parameter to allow passing it on to invoked test data factories. The other -one without parameter creates a new random [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java). +methods. Again, one of each pair has +the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) as parameter to allow passing it on +to invoked test data factories. The other +one without parameter creates a new +random [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java). ### Using factories in tests -Now comes the easy part: If you need a valid data object for your test, but don't care about its content, create one. If you need more than one data object with different but valid data, create +Now comes the easy part: If you need a valid data object for your test, but don't care about its content, create one. If +you need more than one data object with different but valid data, create another one: ```java @@ -338,7 +368,8 @@ class MyOrderTest { } ``` -If you want to control specific aspects of a data object that are important for your test, restrict your test code to only these aspects: +If you want to control specific aspects of a data object that are important for your test, restrict your test code to +only these aspects: ```java import static com.tngtech.valueprovider.example.OrderTestDataFactory.createOrderBuilder; @@ -356,13 +387,15 @@ class MyOrderTest { } ``` -Please refer to [OrderTest](example/src/test/java/com/tngtech/valueprovider/example/OrderTest.java) for more examples, and to +Please refer to [OrderTest](example/src/test/java/com/tngtech/valueprovider/example/OrderTest.java) for more examples, +and to the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) and its Javadoc to learn more about the methods it offers to populate the properties of your data objects. ### Reproducing test failures caused by random data -As you have learned by now, using randomness helps minimize the code for creating test data. However, this comes at a price: If you want to reproduce test failures that might be related to random +As you have learned by now, using randomness helps minimize the code for creating test data. However, this comes at a +price: If you want to reproduce test failures that might be related to random data, especially fom your CI suite of hundreds or even thousands of tests, it is vital to use the same data. value-provider supports this use case out of the box by providing infrastructure for reproducing test failures. @@ -384,9 +417,24 @@ class MyOrderTest { } ``` -If your test class is __derived__ from a base class, make sure to specify the [ValueProviderExtension](junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java) +If your test class is __derived__ from a base class, make sure to specify +the [ValueProviderExtension](junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java) in the __base class__ of the inheritance hierarchy. +###### JUnit 5 @TestInstance Lifecycle + +JUnit 5 allows to control the instantiation (lifecycle in JUnit terms) of the test class. +As an alternative to the default lifecycle `PER_METHOD`, i.e. new instantiation of the test class for each test method +execution, +it supports `PER_CLASS`, i.e. single instantiation of the test class for execution of all test method +(see [JUnit documentation](https://junit.org/junit5/docs/current/user-guide/#writing-tests-test-instance-lifecycle) +for details). + +The [ValueProviderExtension](junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java) supports both +lifecycles since version 1.3.0. There are some subtle differences +when it comes to reproducing test failures. Please refer to +the [respective part of this documentation](#reproducing-test-failures-and-test-lifecycle) for details + ##### JUnit 4 For __JUnit4__, use the @@ -403,7 +451,8 @@ public class MyOrderTest { } ``` -If your test uses __static__ test data created by using a [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java), use the +If your test uses __static__ test data created by using +a [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java), use the [ValueProviderClassRule](junit4/src/main/java/com/tngtech/valueprovider/ValueProviderClassRule.java) in addition: ```java @@ -430,7 +479,8 @@ in the __base class__ of the inheritance hierarchy. Otherwise, your test (or oth #### Reproducing test failures -If a test using the infrastructure fails, it provides information about the seed values used for generating the random data as shown in the following example: +If a test using the infrastructure fails, it provides information about the seed values used for generating the random +data as shown in the following example: ``` org.junit.ComparisonFailure: @@ -448,7 +498,8 @@ at com.tngtech.valueprovider.ValueProviderRule$1.evaluate(ValueProviderRule.java ... ``` -If the failure is related to random data, you can easily reproduce it. Just specify the above shown JVM system properties in the command line when you re-run the test, e.g., in your IDE: +If the failure is related to random data, you can easily reproduce it. Just specify the above shown JVM system +properties in the command line when you re-run the test, e.g., in your IDE: ``` -Dvalue.provider.factory.test.class.seed=0 @@ -456,53 +507,86 @@ If the failure is related to random data, you can easily reproduce it. Just spec -Dvalue.provider.factory.reference.date.time=2021-06-04T15:28:34.004 ``` -Note that seed values relate to individual test classes and methods, even if they have been run in a CI build together with other tests. Thus, it is sufficient to rerun the individual test in order -to reproduce the failure. +##### Reproducing test failures and test lifecycle + +For JUnit 4 and JUnit 5 with the default test lifecycle `PER_METHOD`, seed values relate to __individual test methods__ +within a test class, even if they have been run in a CI build together with other tests. Thus, it is sufficient to only +rerun __the individual test method__ to reproduce the failure. + +For the alternative JUnit 5 lifecycle `PER_CLASS`, seed values relate to __individual test classes__. Thus, you have to +rerun __all test methods__ of the test class (up to and including the failed one) to reproduce the failure. In addition, +we would highly recommend to ensure a defined execution sequence of the test methods via a respective `@TestMethodOrder` +annotation. + +JUnit 5 also supports inner `@Nested` test classes to ease structuring your tests (see +[JUnit documentation](https://junit.org/junit5/docs/current/user-guide/#writing-tests-nested) +for details). The nesting may be arbitrarily deep, i.e. `@Nested` classes may contain further `@Nested` classes. The +test lifecycle may be chosen individually for the main test class as well as for each `@Nested` test class. As long as +the main test class and all `@Nested` test classes use the default test lifecycle `PER_METHOD`, it is again sufficient +to rerun the individual test method, regardless if it is in the main class or any nested class. As soon as the lifecycle +`PER_CLASS` is used for one of the classes in the nesting hierarchy where the failure occured, you have to rerun __all +test methods of this hierarchy__ of test classes to reproduce the failure. ### Reproducible ValueProviders The above example code always used `ValueProviderFactory.createRandomValueProvider()` -to create a [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) that, in turn, generates random data. To be more precise, +to create a [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) that, in turn, generates +random data. To be more precise, the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) -is initialized with a random seed and will generate exactly the same data, if it is initialized with the same seed, and the same sequence of method invocations is executed. As you may have guessed +is initialized with a random seed and will generate exactly the same data, if it is initialized with the same seed, and +the same sequence of method invocations is executed. As you may have guessed already, reproducing test failures is based on this functionality. -So, if you need reproducible test data, e.g., to test a transformation of Java data to XML by verifying against a previously stored XML file, +So, if you need reproducible test data, e.g., to test a transformation of Java data to XML by verifying against a +previously stored XML file, use `ValueProviderFactory.createReproducibleValueProvider()` -and provide a seed value of your choice to create the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java). Your test data factories will accept this [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) just +and provide a seed value of your choice to create +the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java). Your test data factories will +accept this [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) just as any other one. ### Extending ValueProvider functionality -The [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) offers a considerable amount of common methods to fill properties of test data objects. Sooner or later however, +The [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) offers a considerable amount of +common methods to fill properties of test data objects. Sooner or later however, the need will arise to add project specific functionality. We advise the following approach: -* Create your own [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) class. Let it extend the [AbstractValueProvider](core/src/main/java/com/tngtech/valueprovider/AbstractValueProvider.java) provided by this library, and add the methods you need -* Create your own [ValueProviderFactory](core/src/main/java/com/tngtech/valueprovider/ValueProviderFactory.java) class and implement 2 static methods that create instances of +* Create your own [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) class. Let it extend + the [AbstractValueProvider](core/src/main/java/com/tngtech/valueprovider/AbstractValueProvider.java) provided by this + library, and add the methods you need +* Create your own [ValueProviderFactory](core/src/main/java/com/tngtech/valueprovider/ValueProviderFactory.java) class + and implement 2 static methods that create instances of your derived [AbstractValueProvider](core/src/main/java/com/tngtech/valueprovider/AbstractValueProvider.java) class * `createRandomValueProvider()` * `createReproducibleValueProvider()` * Use your own classes instead of the ones provided by this library in your test data factories and other test code. -Refer to [CustomValueProvider](example/src/main/java/com/tngtech/valueprovider/example/customprovider/CustomValueProvider.java) +Refer +to [CustomValueProvider](example/src/main/java/com/tngtech/valueprovider/example/customprovider/CustomValueProvider.java) , [CustomValueProviderFactory](example/src/main/java/com/tngtech/valueprovider/example/customprovider/CustomValueProviderFactory.java) -, and [CustomValueProviderFactoryTest](example/src/test/java/com/tngtech/valueprovider/example/customprovider/CustomValueProviderFactoryTest.java) +, +and [CustomValueProviderFactoryTest](example/src/test/java/com/tngtech/valueprovider/example/customprovider/CustomValueProviderFactoryTest.java) for a fully functional example. ### Limitations wrt. multithreaded tests -The infrastructure uses thread-local data to store the seed values and can therefore be used in parallel CI builds without any problems. +The infrastructure uses thread-local data to store the seed values and can therefore be used in parallel CI builds +without any problems. -However, reproducing test failures is __not possible for multithreaded test code__. Likewise, a [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) -initialized with a fixed seed value will not necessarily generate the same sequence of data if it is used by multiple threads, -as the sequence of method invocations from different threads is not reproducible, -and neither the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) nor the infrastructure provide any synchronisation. +However, reproducing test failures is __not possible for multithreaded test code__. Likewise, +a [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) +initialized with a fixed seed value will not necessarily generate the same sequence of data if it is used by multiple +threads, +as the sequence of method invocations from different threads is not reproducible, +and neither the [ValueProvider](core/src/main/java/com/tngtech/valueprovider/ValueProvider.java) nor the infrastructure +provide any synchronisation. ## License value-provider is published under the Apache License 2.0, see [license file](LICENSE) for details. -1Please note that we use [lombok](https://projectlombok.org/) and immutable data objects in our examples for convenience, but this is not a requirement for using value-provider. +1Please note that we use [lombok](https://projectlombok.org/) and immutable data objects in our examples for +convenience, but this is not a requirement for using value-provider. From f80020bf0dffb4af9b3efdbeaa1aecb8de10b172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Mon, 2 Dec 2024 14:43:50 +0100 Subject: [PATCH 08/10] remove TestConstructorDetector that has been made obsolete by JUnit 5.5 InvocationInterceptor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- ...roducible-test-class-and-method-cycle.puml | 83 ++++++------------- 1 file changed, 24 insertions(+), 59 deletions(-) diff --git a/doc/junit5-reproducible-test-class-and-method-cycle.puml b/doc/junit5-reproducible-test-class-and-method-cycle.puml index d1bb0e9..5f543b6 100644 --- a/doc/junit5-reproducible-test-class-and-method-cycle.puml +++ b/doc/junit5-reproducible-test-class-and-method-cycle.puml @@ -4,8 +4,6 @@ participant "Test A" as T_A participant "Test B" as T_B participant "VPExtension\n(JUnit5)" as VPF_X #lightblue -participant "TestConstructorDetector" as TCD -note over TCD: to detect test class instantiation\nand switch to test method cycle participant "ValueProviderFactory" as VPF note over VPF: Just shown for single thread,\nimplemented as ThreadLocal\nfor parallel test execution participant "DefaultInitializationCreator" as D_VPC @@ -34,26 +32,14 @@ VPF -> C_VPC : startTestCycle C_VPC -> C_VPC : initialize seed &\n reference date/time note right of C_VPC #lightgreen: using system properties VPF -> VPF : activateCreator(TestClassInitializationCreator) -VPF_X -> TCD : instantiation -activate TCD -VPF_X -> VPF : activateTestConstructorDetector T_A -> VPF : createRandomValueProvider note left #lightgreen: static initialization -VPF -> TCD : isCallFromTestConstructor -TCD --> VPF : false VPF -> C_VPC : createRandomValueProvider == Test A test method <1> == -JVM --> T_A : instantiation for test method <1> -activate T_A - -T_A -> VPF : createRandomValueProvider -note left #lightgreen: instance variables - -VPF -> TCD : isCallFromTestConstructor -TCD --> VPF : true -VPF -> VPF : startTestMethodCycle +JVM -> VPF_X : interceptTestClassConstructor +VPF_X -> VPF : startTestMethodCycle VPF -> VPF : activateCreator(TestMethodInitializationCreator) VPF -> M_VPC : startTestCycle(TestClassInitializationCreator) M_VPC -> M_VPC : initialize seed &\n reference date/time @@ -62,13 +48,13 @@ M_VPC -> M_VPC : copyValueProviderSuffixes(TestClassInitializationCreator) note right of M_VPC #lightgreen: required to ensure unique suffixes M_VPC -> M_VPC : copyReferenceDateTime(TestClassInitializationCreator) note right of M_VPC #lightgreen: required to ensure reproducible\ndate/time related test data -VPF -> VPF : deactivateTestConstructorDetector -VPF --> TCD -destroy TCD -VPF -> M_VPC : createRandomValueProvider +JVM <-- VPF_X : proceed +JVM --> T_A : instantiation for test method <1> +activate T_A -JVM -> VPF_X : postProcessTestInstance -note over VPF_X : nothing to do, if TestConstructorDetector\nswitched to test method cycle already +T_A -> VPF : createRandomValueProvider +note left #lightgreen: instance variables +VPF -> M_VPC : createRandomValueProvider T_A -> VPF : createRandomValueProvider note left #lightgreen: before methods @@ -85,24 +71,14 @@ VPF_X -> VPF : finishTestMethodCycle VPF -> M_VPC : finishTestCycle M_VPC -> M_VPC : resetValueProviderSuffixes VPF -> VPF : activateCreator(TestClassInitializationCreator) -VPF_X -> TCD : instantiation -activate TCD -VPF_X -> VPF : activateTestConstructorDetector JVM --> T_A : destruction for test method <1> destroy T_A newpage == Test A test method <2> == -JVM --> T_A : instantiation for test method <2> -activate T_A - -T_A -> VPF : createRandomValueProvider -note left #lightgreen: instance variables - -VPF -> TCD : isCallFromTestConstructor -TCD --> VPF : true -VPF -> VPF : startTestMethodCycle +JVM -> VPF_X : interceptTestClassConstructor +VPF_X -> VPF : startTestMethodCycle VPF -> VPF : activateCreator(TestMethodInitializationCreator) VPF -> M_VPC : startTestCycle(TestClassInitializationCreator) M_VPC -> M_VPC : initialize seed &\n reference date/time @@ -111,13 +87,13 @@ M_VPC -> M_VPC : copyValueProviderSuffixes(TestClassInitializationCreator) note right of M_VPC #lightgreen: required to ensure unique suffixes M_VPC -> M_VPC : copyReferenceDateTime(TestClassInitializationCreator) note right of M_VPC #lightgreen: required to ensure reproducible\ndate/time related test data -VPF -> VPF : deactivateTestConstructorDetector -VPF --> TCD -destroy TCD -VPF -> M_VPC : createRandomValueProvider +JVM <-- VPF_X : proceed +JVM --> T_A : instantiation for test method <2> +activate T_A -JVM -> VPF_X : postProcessTestInstance -note over VPF_X : nothing to do, if TestConstructorDetector\nswitched to test method cycle already +T_A -> VPF : createRandomValueProvider +note left #lightgreen: instance variables +VPF -> M_VPC : createRandomValueProvider T_A -> VPF : createRandomValueProvider note left #lightgreen: before methods @@ -134,8 +110,6 @@ VPF_X -> VPF : finishTestMethodCycle VPF -> M_VPC : finishTestCycle M_VPC -> M_VPC : resetValueProviderSuffixes VPF -> VPF : activateCreator(TestClassInitializationCreator) -VPF_X -> TCD : instantiation -activate TCD JVM --> T_A : destruction for test method <2> destroy T_A @@ -146,9 +120,6 @@ C_VPC -> C_VPC : resetValueProviderSuffixes VPF -> VPF : delete TestClassInitializationCreator destroy C_VPC VPF -> VPF : activateCreator(DefaultInitializationCreator) -VPF_X -> VPF : deactivateTestConstructorDetector -VPF --> TCD -destroy TCD JVM --> VPF_X destroy VPF_X newpage @@ -165,26 +136,13 @@ VPF -> C_VPC : startTestCycle C_VPC -> C_VPC : initialize seed &\n reference date/time note right of C_VPC #lightgreen: using system properties VPF -> VPF : activateCreator(TestClassInitializationCreator) -VPF_X -> TCD : instantiation -activate TCD -VPF_X -> VPF : activateTestConstructorDetector T_B -> VPF : createRandomValueProvider note left #lightgreen: static initialization -VPF -> TCD : isCallFromTestConstructor -TCD --> VPF : false VPF -> C_VPC : createRandomValueProvider == Test B test method <1> == -JVM --> T_B : instantiation for test method <1> -activate T_B - -JVM -> VPF_X : postProcessTestInstance -note over VPF_X : variant: if no VPs were created after test instatiation,\nmust switch to test method cycle here -VPF_X -> VPF : deactivateTestConstructorDetector -VPF --> TCD -destroy TCD - +JVM -> VPF_X : interceptTestClassConstructor VPF_X -> VPF : startTestMethodCycle VPF -> VPF : activateCreator(TestMethodInitializationCreator) VPF -> M_VPC : startTestCycle(TestClassInitializationCreator) @@ -194,6 +152,13 @@ M_VPC -> M_VPC : copyValueProviderSuffixes(TestClassInitializationCreator) note right of M_VPC #lightgreen: required to ensure unique suffixes M_VPC -> M_VPC : copyReferenceDateTime(TestClassInitializationCreator) note right of M_VPC #lightgreen: required to ensure reproducible\ndate/time related test data +JVM <-- VPF_X : proceed +JVM --> T_B : instantiation for test method <1> +activate T_B + +T_B -> VPF : createRandomValueProvider +note left #lightgreen: instance variables +VPF -> M_VPC : createRandomValueProvider T_B -> VPF : createRandomValueProvider note left #lightgreen: before methods From 99af692d490e73c932dd3ee2eea93f7274c334d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Mon, 2 Dec 2024 15:44:46 +0100 Subject: [PATCH 09/10] add sequence diagram for JUnit 5 Lifecycle PER_CLASS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- ...roducible-test-class-and-method-cycle.puml | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 doc/junit5-lifecycle-PER_CLASS-reproducible-test-class-and-method-cycle.puml diff --git a/doc/junit5-lifecycle-PER_CLASS-reproducible-test-class-and-method-cycle.puml b/doc/junit5-lifecycle-PER_CLASS-reproducible-test-class-and-method-cycle.puml new file mode 100644 index 0000000..0f1df1b --- /dev/null +++ b/doc/junit5-lifecycle-PER_CLASS-reproducible-test-class-and-method-cycle.puml @@ -0,0 +1,111 @@ +@startuml +participant "JVM\nJUnit\nRuntime" as JVM +participant "Test A,\nLifecycle\n'PER_CLASS'" as T_A + +participant "VPExtension\n(JUnit5)" as VPF_X #lightblue +participant "ValueProviderFactory" as VPF +note over VPF: Just shown for single thread,\nimplemented as ThreadLocal\nfor parallel test execution +participant "DefaultInitializationCreator" as D_VPC +note over D_VPC: 1. Uses arbitrary/random seed\n2. Does NOT ensure unique suffixes +participant "TestClassInitializationCreator" as C_VPC +participant "TestMethodInitializationCreator" as M_VPC +note over C_VPC, M_VPC: 1. Seed and reference date/time\n controllable via system properties\n2. Ensures unique suffixes\n (as far as possible) + +JVM --> VPF : class-loading & instantiation +activate VPF +VPF -> D_VPC : instantiation +activate D_VPC +VPF -> M_VPC : instantiation +activate M_VPC +VPF -> VPF : activateCreator(DefaultInitializationCreator) + +== Test A class-loading == +JVM --> VPF_X : instantiation +activate VPF_X #lightblue +JVM -> VPF_X : beforeAll +note right #lightgreen: nothing to do for Lifecycle PER_CLASS +JVM -> VPF_X : interceptTestClassConstructor +VPF_X -> VPF : startTestClassCycle +VPF -> C_VPC : instantiation +activate C_VPC +note right of C_VPC #lightgreen: different lifecycle required\nfor tests with JUnit4 DataProvider +VPF -> C_VPC : startTestCycle +C_VPC -> C_VPC : initialize seed &\n reference date/time +note right of C_VPC #lightgreen: using system properties +VPF -> VPF : activateCreator(TestClassInitializationCreator) +VPF_X -> JVM : trigger class-loading of Test A + +T_A -> VPF : createRandomValueProvider +note left #lightgreen: static initialization +VPF -> C_VPC : createRandomValueProvider + +VPF_X -> VPF : startTestMethodCycle +VPF -> VPF : activateCreator(TestMethodInitializationCreator) +VPF -> M_VPC : startTestCycle(TestClassInitializationCreator) +M_VPC -> M_VPC : initialize seed &\n reference date/time +note right of M_VPC #lightgreen: using system properties +M_VPC -> M_VPC : copyValueProviderSuffixes(TestClassInitializationCreator) +note right of M_VPC #lightgreen: required to ensure unique suffixes +M_VPC -> M_VPC : copyReferenceDateTime(TestClassInitializationCreator) +note right of M_VPC #lightgreen: required to ensure reproducible\ndate/time related test data +JVM <-- VPF_X : proceed +== Test A test method <1> == +JVM --> T_A : instantiation (once for all test methods) +activate T_A + +T_A -> VPF : createRandomValueProvider +note left #lightgreen: instance variables +VPF -> M_VPC : createRandomValueProvider + +T_A -> VPF : createRandomValueProvider +note left #lightgreen: before methods +VPF -> M_VPC : createRandomValueProvider + +JVM -> T_A : test method <1> +T_A -> VPF : createRandomValueProvider +note left #lightgreen: test method +VPF -> M_VPC : createRandomValueProvider + +T_A --> JVM : return +JVM -> VPF_X : afterEach +note right #lightgreen: nothing to do for Lifecycle PER_CLASS +newpage + +== Test A test method <2> == +T_A -> VPF : createRandomValueProvider +note left #lightgreen +(note: instance variables shared) +before methods +end note +VPF -> M_VPC : createRandomValueProvider + +JVM -> T_A : test method <2> +T_A -> VPF : createRandomValueProvider +note left #lightgreen: test method +VPF -> M_VPC : createRandomValueProvider + +T_A --> JVM : return +JVM -> VPF_X : afterEach +note right #lightgreen: nothing to do for Lifecycle PER_CLASS + +JVM --> T_A : destruction +destroy T_A + +JVM -> VPF_X : afterAll + +VPF_X -> VPF : finishTestMethodCycle +VPF -> M_VPC : finishTestCycle +M_VPC -> M_VPC : resetValueProviderSuffixes +VPF -> VPF : activateCreator(TestClassInitializationCreator) + +VPF_X -> VPF : finishTestClassCycle +VPF -> C_VPC : finishTestCycle +C_VPC -> C_VPC : resetValueProviderSuffixes +VPF -> VPF : delete TestClassInitializationCreator +destroy C_VPC +VPF -> VPF : activateCreator(DefaultInitializationCreator) + +JVM --> VPF_X +destroy VPF_X + +@enduml \ No newline at end of file From 465be2c5c9657095565b12c3370cd6e566c5a23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Mon, 9 Dec 2024 17:56:59 +0100 Subject: [PATCH 10/10] provide name of test class to re-run in failure reproduction info, if test class hierarchy of failed test contains Lifecycle PER_CLASS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- README.md | 15 +- .../valueprovider/ValueProviderException.java | 32 ++- .../ValueProviderExceptionTest.java | 20 +- .../valueprovider/ValueProviderExtension.java | 25 ++- .../ValueProviderExtensionFailureTest.java | 187 +++++++++++++----- 5 files changed, 215 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index c4109d0..7211b79 100644 --- a/README.md +++ b/README.md @@ -499,7 +499,7 @@ at com.tngtech.valueprovider.ValueProviderRule$1.evaluate(ValueProviderRule.java ``` If the failure is related to random data, you can easily reproduce it. Just specify the above shown JVM system -properties in the command line when you re-run the test, e.g., in your IDE: +properties in the command line when you re-run the failed test, e.g., in your IDE: ``` -Dvalue.provider.factory.test.class.seed=0 @@ -524,8 +524,17 @@ for details). The nesting may be arbitrarily deep, i.e. `@Nested` classes may co test lifecycle may be chosen individually for the main test class as well as for each `@Nested` test class. As long as the main test class and all `@Nested` test classes use the default test lifecycle `PER_METHOD`, it is again sufficient to rerun the individual test method, regardless if it is in the main class or any nested class. As soon as the lifecycle -`PER_CLASS` is used for one of the classes in the nesting hierarchy where the failure occured, you have to rerun __all -test methods of this hierarchy__ of test classes to reproduce the failure. +`PER_CLASS` is used for one or more classes in the nesting hierarchy where the failure occured, you have to re-run __all +test methods of this hierarchy__ of test classes to reproduce the failure. For convenience, the failure message +generated by the infrastructure provides the name of the root test class of this hierarchy in addition to the seed +values as shown in the following example: + +``` +"If the failure is related to random ValueProviders, re-run all tests of 'com.tngtech.valueprovider.ValueProviderExceptionTest' and specify the following system properties for the JVM to reproduce: +-Dvalue.provider.factory.test.class.seed=0 +-Dvalue.provider.factory.test.method.seed=-5385145878463633929 +-Dvalue.provider.factory.reference.date.time=2024-12-09T17:02:50.109" +``` ### Reproducible ValueProviders diff --git a/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java b/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java index 95c1a0f..083df54 100644 --- a/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java +++ b/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java @@ -1,29 +1,43 @@ package com.tngtech.valueprovider; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_CLASS_SEED_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY; -import static com.tngtech.valueprovider.ValueProviderFactory.getFormattedReferenceDateTime; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestClassSeed; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestMethodSeed; +import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; + +import static com.tngtech.valueprovider.InitializationCreator.*; +import static com.tngtech.valueprovider.ValueProviderFactory.*; import static java.lang.String.format; +import static java.util.Optional.empty; +@SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class ValueProviderException extends RuntimeException { ValueProviderException() { - super(provideFailureReproductionInfo()); + this(empty()); + } + + ValueProviderException(Optional> testClassToReRunForReproduction) { + super(provideFailureReproductionInfo(testClassToReRunForReproduction)); } - static String provideFailureReproductionInfo() { + @VisibleForTesting + static String provideFailureReproductionInfo(Optional> testClassToReRunForReproduction) { long testClassSeed = getTestClassSeed(); long testMethodSeed = getTestMethodSeed(); String referenceDateTime = getFormattedReferenceDateTime(); return format( - "If the failure is related to random ValueProviders, specify the following system properties for the JVM to reproduce:%n" + + "If the failure is related to random ValueProviders, %sspecify the following system properties for the JVM to reproduce:%n" + "-D%s=%d%n" + "-D%s=%d%n" + "-D%s=%s", + formatReRunMessageFor(testClassToReRunForReproduction), VALUE_PROVIDER_FACTORY_TEST_CLASS_SEED_PROPERTY, testClassSeed, VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY, testMethodSeed, VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY, referenceDateTime); } + + private static String formatReRunMessageFor(Optional> testClassToReRunForReproduction) { + return testClassToReRunForReproduction.map(testClass -> + format("re-run all tests of '%s' and ", testClass.getName())) + .orElse(""); + } } diff --git a/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java b/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java index d5334d1..a41474e 100644 --- a/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java +++ b/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java @@ -1,13 +1,11 @@ package com.tngtech.valueprovider; +import java.util.Optional; + import org.junit.jupiter.api.Test; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_CLASS_SEED_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY; -import static com.tngtech.valueprovider.ValueProviderFactory.getFormattedReferenceDateTime; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestClassSeed; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestMethodSeed; +import static com.tngtech.valueprovider.InitializationCreator.*; +import static com.tngtech.valueprovider.ValueProviderFactory.*; import static org.assertj.core.api.Assertions.assertThat; class ValueProviderExceptionTest { @@ -26,4 +24,14 @@ void should_show_seed_values_reference_date_time_and_respective_system_propertie VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY, VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY); } + + @Test + void should_show_test_class_to_re_run_for_failure_reproduction_if_provided() { + Class testClassToReRun = this.getClass(); + ValueProviderException exception = new ValueProviderException(Optional.of(testClassToReRun)); + + String message = exception.getMessage(); + + assertThat(message).contains(testClassToReRun.getName()); + } } \ No newline at end of file diff --git a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java index 95a6c9c..e5a0014 100644 --- a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java +++ b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java @@ -13,6 +13,7 @@ import static com.tngtech.valueprovider.ValueProviderExtension.TestMethodCycleState.*; import static java.lang.System.identityHashCode; +import static java.util.Optional.empty; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; @@ -78,9 +79,13 @@ public void beforeEach(ExtensionContext context) { public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable { logger.debug("{} handleTestExecutionException {}", identityHashCode(this), buildQualifiedTestMethodName(context)); + // If the test class hierarchy of the failed test method contains any class(es) with Lifecycle PER_CLASS, + // all test methods of this hierarchy must be re-run to reproduce the failure. + // The root test class of the hierarchy must therefore be shown in the failure reproduction info. + Optional> testClassToReRunForReproduction = getRootClassInHierarchyWithLifecyclePerClass(context); // Note: handleTestExecutionException() is invoked BEFORE afterEach, i.e. BEFORE seed is reset, // so that the correct seed values appear in the failure message - throwable.addSuppressed(new ValueProviderException()); + throwable.addSuppressed(new ValueProviderException(testClassToReRunForReproduction)); throw throwable; } @@ -179,6 +184,24 @@ private static boolean isLastTestClassInHierarchyWithLifecyclePerClass(Extension return remainingLifecyclesInHierarchy.isEmpty() || containsOnlyLifecyclePerMethod(remainingLifecyclesInHierarchy); } + private static Optional> getRootClassInHierarchyWithLifecyclePerClass(ExtensionContext startContext) { + List> testClassesInHierarchyWithLifecyclePerClass = new ArrayList<>(); + traverseContextHierarchy(startContext, context -> + addTestClassAtBeginningIfLifecyclePerClass(context, testClassesInHierarchyWithLifecyclePerClass)); + if (testClassesInHierarchyWithLifecyclePerClass.isEmpty()) { + return empty(); + } + return Optional.of(testClassesInHierarchyWithLifecyclePerClass.get(0)); + } + + private static void addTestClassAtBeginningIfLifecyclePerClass(ExtensionContext context, List> addTo) { + if (!isLifecycle(context, PER_CLASS)) { + return; + } + context.getTestClass().ifPresent(testClass -> + addTo.add(0, testClass)); + } + private static boolean testClassHierarchyHasOnlyLifecyclePerMethod(ExtensionContext context) { Set lifecyclesInHierarchy = determineLifecyclesInTestClassHierarchy(Optional.of(context)); return containsOnlyLifecyclePerMethod(lifecyclesInHierarchy); diff --git a/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java b/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java index 4ea6d7d..ea6ca99 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java @@ -2,41 +2,67 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; import org.assertj.core.api.Condition; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.junit.jupiter.api.condition.EnabledIf; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.platform.testkit.engine.EngineTestKit; import org.junit.platform.testkit.engine.Events; +import static com.tngtech.valueprovider.ValueProviderExtensionFailureTest.FailureInformation.expectRootClassOfFailingTestInReproductionInfo; +import static com.tngtech.valueprovider.ValueProviderExtensionFailureTest.FailureInformation.noTestClassInReproductionInfo; +import static java.util.Optional.empty; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.fail; +import static org.junit.jupiter.api.Named.named; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; +import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; -import static org.junit.platform.testkit.engine.EventConditions.event; -import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; -import static org.junit.platform.testkit.engine.EventConditions.test; -import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; -import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; -import static org.junit.platform.testkit.engine.TestExecutionResultConditions.suppressed; +import static org.junit.platform.testkit.engine.EventConditions.*; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.*; class ValueProviderExtensionFailureTest { private static final Map test2FailureInformation = new HashMap<>(); + /** + * To allow programmatic execution, but avoid running the failing tests as part of the gradle/CI-build. + */ + private static boolean enableFailureTest = false; + + private static boolean isFailureTestEnabled() { + return enableFailureTest; + } + + @BeforeEach + void enableFailureTest() { + enableFailureTest = true; + } @AfterEach void resetFailureInformation() { - FailureTest.enabled = false; + enableFailureTest = false; test2FailureInformation.clear(); } - @Test - void extension_should_provide_failure_reproduction_info() { - FailureTest.enabled = true; + private static Stream failureTests() { + return Stream.of( + arguments(named("simple failure tests, Lifecycle PER_METHOD", SimpleFailureTest.class)), + arguments(named("nested failure tests, Lifecycles PER_METHOD and PER_CLASS", MainOfNestedFailureTest.class)) + ); + } + @ParameterizedTest + @MethodSource("failureTests") + void extension_should_provide_failure_reproduction_info(Class failureTestClass) { Events testEvents = EngineTestKit.engine("junit-jupiter") - .selectors(selectClass(FailureTest.class)) + .selectors(selectClass(failureTestClass)) .execute() .testEvents(); @@ -57,60 +83,131 @@ private void assertExpectedTestFailure(Events testEvents, String testName) { suppressed(0, instanceOf(ValueProviderException.class)); Condition expectedFailureReproductionInfo = suppressed(0, message(x -> x.contains(failureInformation.expectedFailureReproductionInfo))); - testEvents.assertThatEvents() - .haveExactly(1, - event(test(testName), - finishedWithFailure( - expectedException, - expectedTestFailureMessage, - expectedSuppressedException, - expectedFailureReproductionInfo - ))); + Condition expectedRootClassOfFailingTestInReproductionInfo = + suppressed(0, message(x -> + { + boolean containsRootClassOfFailingTest = x.contains(failureInformation.classOfFailingTest.getName()); + boolean expectRootClassOfFailingTest = failureInformation.expectRootClassOfFailingTestInReproductionInfo; + return containsRootClassOfFailingTest == expectRootClassOfFailingTest; + })); + if (failureInformation.expectRootClassOfFailingTestInReproductionInfo) + testEvents.assertThatEvents() + .haveExactly(1, + event(test(testName), + finishedWithFailure( + expectedException, + expectedTestFailureMessage, + expectedSuppressedException, + expectedFailureReproductionInfo, + expectedRootClassOfFailingTestInReproductionInfo + ))); } + @SuppressWarnings("JUnitMalformedDeclaration") // false positive, class is executed programmatically @ExtendWith(ValueProviderExtension.class) - @EnabledIf("enabled") - static class FailureTest { - /** - * To control test execution and esp. avoid running the test as part of the gradle/CI-build. - */ - private static boolean enabled = false; - - @SuppressWarnings("unused") // used by JUnit @EnabledIf - static boolean enabled() { - return enabled; - } - + @EnabledIf("com.tngtech.valueprovider.ValueProviderExtensionFailureTest#isFailureTestEnabled") + static class SimpleFailureTest { @Test void simply_failing() { - addFailureInformation("simply_failing", "failing intentionally"); - //noinspection ResultOfMethodCallIgnored - fail("failing intentionally"); + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("simply_failing", failureInformation); + fail(failureMessage); } @Test void failing_with_assertion_error() { - addFailureInformation("failing_with_assertion_error", "My specific assertion description"); - //noinspection ConstantConditions - assertThat(true).as("My specific assertion description").isFalse(); + String failureMessage = "My specific assertion description"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_with_assertion_error", failureInformation); + assertThat(enableFailureTest).as(failureMessage).isFalse(); } } - private static void addFailureInformation(String testName, String expectedFailureMessage) { - test2FailureInformation.put(testName, new FailureInformation(expectedFailureMessage)); + @SuppressWarnings("JUnitMalformedDeclaration") // false positive, class is executed programmatically + @ExtendWith(ValueProviderExtension.class) + @EnabledIf("com.tngtech.valueprovider.ValueProviderExtensionFailureTest#isFailureTestEnabled") + static class MainOfNestedFailureTest { + @Test + void failing_in_main() { + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_in_main", failureInformation); + fail(failureMessage); + } + + @Nested + @TestInstance(PER_CLASS) + class NestedFailureTestLifecyclePerClass { + + @Test + void failing_in_nested_with_lifecycle_per_class() { + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = expectRootClassOfFailingTestInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_in_nested_with_lifecycle_per_class", failureInformation); + fail(failureMessage); + } + + @Nested + @TestInstance(PER_METHOD) + class NestedFailureTestChildOfLifecyclePerClass { + + @Test + void failing_in_child_with_lifecycle_per_method_of_parent_with_lifecycle_per_class() { + String failureMessage = "failing intentionally"; + Class parentWithLifecyclePerClass = NestedFailureTestLifecyclePerClass.class; + FailureInformation failureInformation = expectRootClassOfFailingTestInReproductionInfo(failureMessage, parentWithLifecyclePerClass); + addFailureInformation("failing_in_child_with_lifecycle_per_method_of_parent_with_lifecycle_per_class", + failureInformation); + fail(failureMessage); + } + } + } + + @Nested + class NestedFailureTestLifecyclePerMethod { + + @Test + void failing_in_nested_with_lifecycle_per_method() { + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_in_nested_with_lifecycle_per_method", failureInformation); + fail(failureMessage); + } + } + } + + private static void addFailureInformation(String testName, FailureInformation failureInformation) { + test2FailureInformation.put(testName, failureInformation); } private static FailureInformation getFailureInformation(String testName) { return test2FailureInformation.get(testName); } - private static class FailureInformation { + static class FailureInformation { private final String expectedTestFailureMessage; private final String expectedFailureReproductionInfo; + private final Class classOfFailingTest; + private final boolean expectRootClassOfFailingTestInReproductionInfo; + + static FailureInformation noTestClassInReproductionInfo(String expectedTestFailureMessage, Class rootClassOfFailingTest) { + return new FailureInformation(expectedTestFailureMessage, rootClassOfFailingTest, false); + } + + static FailureInformation expectRootClassOfFailingTestInReproductionInfo(String expectedTestFailureMessage, Class rootClassOfFailingTest) { + return new FailureInformation(expectedTestFailureMessage, rootClassOfFailingTest, true); + } - private FailureInformation(String expectedTestFailureMessage) { + private FailureInformation(String expectedTestFailureMessage, + Class rootClassOfFailingTest, boolean expectRootClassOfFailingTestInReproductionInfo) { this.expectedTestFailureMessage = expectedTestFailureMessage; - this.expectedFailureReproductionInfo = ValueProviderException.provideFailureReproductionInfo(); + Optional> testClassToReRunForReproduction = expectRootClassOfFailingTestInReproductionInfo + ? Optional.of(rootClassOfFailingTest) + : empty(); + this.expectedFailureReproductionInfo = ValueProviderException.provideFailureReproductionInfo(testClassToReRunForReproduction); + this.classOfFailingTest = rootClassOfFailingTest; + this.expectRootClassOfFailingTestInReproductionInfo = expectRootClassOfFailingTestInReproductionInfo; } } }