From fdc1b423fb90c2f85812e3c2e95f9ccb6d58f25e Mon Sep 17 00:00:00 2001 From: Christopher Lambert Date: Sun, 29 Mar 2026 11:35:12 +0200 Subject: [PATCH 1/2] Add ITs for AfterAll/AfterClass teardown failure Add integration tests exposing broken surefire behavior when JUnit 4 `@AfterClass` or JUnit 5 `@AfterAll` methods always throw. The tests document the following bugs on master: - XML report has `errors="0"` despite an `` element in a testcase - The teardown error is misclassified as a flake (`flakes="1"`) - With `rerunFailingTestsCount`, the build passes green because the deterministic teardown failure is swallowed as a flake - Passing test methods are needlessly re-executed (`rerunCount + 1` times) --- .../its/JUnit4FailingAfterClassIT.java | 98 +++++++++++++++++++ .../surefire/its/JUnit5FailingAfterAllIT.java | 98 +++++++++++++++++++ .../junit4-failing-after-class/pom.xml | 55 +++++++++++ .../junit4/AlwaysFailingAfterClassTest.java | 29 ++++++ .../src/test/java/junit4/PassingTest.java | 16 +++ .../junit5-failing-after-all/pom.xml | 57 +++++++++++ .../junit5/AlwaysFailingAfterAllTest.java | 29 ++++++ .../src/test/java/junit5/PassingTest.java | 16 +++ 8 files changed, 398 insertions(+) create mode 100644 surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java create mode 100644 surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java create mode 100644 surefire-its/src/test/resources/junit4-failing-after-class/pom.xml create mode 100644 surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/AlwaysFailingAfterClassTest.java create mode 100644 surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/PassingTest.java create mode 100644 surefire-its/src/test/resources/junit5-failing-after-all/pom.xml create mode 100644 surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/AlwaysFailingAfterAllTest.java create mode 100644 surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/PassingTest.java diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java new file mode 100644 index 0000000000..3655093e7b --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.surefire.its; + +import org.apache.maven.shared.verifier.VerificationException; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +/** + * Integration test for JUnit 4 @AfterClass that always fails. + * Verifies that surefire properly reports errors when class-level teardown throws. + * + *

Current behavior on master is broken: + *

+ * + *

Once fixed, the XML should report errors="1". + */ +public class JUnit4FailingAfterClassIT extends SurefireJUnit4IntegrationTestCase { + private static final String VERSION = "4.13.2"; + + @Test + public void testAfterClassFailureIsReported() { + OutputValidator outputValidator = unpack("junit4-failing-after-class", "-norerun") + .setJUnitVersion(VERSION) + .maven() + .withFailure() + .executeTest(); + + // The @AfterClass failure should be reported in the log + outputValidator.verifyTextInLog("AfterClass always fails"); + + // The passing test class should still be error-free + outputValidator + .getSurefireReportsXmlFile("TEST-junit4.PassingTest.xml") + .assertContainsText("tests=\"1\" errors=\"0\""); + + // BUG: The XML summary says errors="0" despite the @AfterClass error being present as a testcase. + // The error is misclassified as a flake. + // After fix, this should assert errors="1" and flakes="0". + outputValidator + .getSurefireReportsXmlFile("TEST-junit4.AlwaysFailingAfterClassTest.xml") + .assertContainsText("errors=\"0\"") + .assertContainsText("flakes=\"1\""); + + // The @AfterClass error testcase is named "initializationError". + outputValidator + .getSurefireReportsXmlFile("TEST-junit4.AlwaysFailingAfterClassTest.xml") + .assertContainsText("name=\"initializationError\""); + } + + @Test + public void testAfterClassFailureWithRerun() throws VerificationException { + // BUG: With reruns, the build actually PASSES because the @AfterClass error is + // misclassified as a flake. This is the scenario reported in PR #3329: + // the error is swallowed and the build appears green. + // After fix, the build should fail because the @AfterClass always errors. + OutputValidator outputValidator = unpack("junit4-failing-after-class", "-rerun") + .setJUnitVersion(VERSION) + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=2") + .executeTest(); + + // The error is misclassified as a flake in the XML + outputValidator + .getSurefireReportsXmlFile("TEST-junit4.AlwaysFailingAfterClassTest.xml") + .assertContainsText("errors=\"0\"") + .assertContainsText("flakes=\"1\""); + + // BUG: The passing test methods get re-executed 3 times (rerunFailingTestsCount + 1) + // because the @AfterClass error triggers a class-level rerun. + // After fix, passing test methods should not be rerun due to a teardown failure. + outputValidator.assertThatLogLine(containsString("testOne passed"), is(3)); + outputValidator.assertThatLogLine(containsString("testTwo passed"), is(3)); + } +} diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java new file mode 100644 index 0000000000..a1f64e218f --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.surefire.its; + +import org.apache.maven.shared.verifier.VerificationException; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +/** + * Integration test for JUnit 5 @AfterAll that always fails. + * Verifies that surefire properly reports errors when class-level teardown throws. + * + *

Current behavior on master is broken: + *

+ * + *

Once fixed, the XML should report errors="1". + */ +public class JUnit5FailingAfterAllIT extends SurefireJUnit4IntegrationTestCase { + private static final String VERSION = "5.9.1"; + + @Test + public void testAfterAllFailureIsReported() { + OutputValidator outputValidator = unpack("junit5-failing-after-all", "-norerun") + .setJUnitVersion(VERSION) + .maven() + .withFailure() + .executeTest(); + + // The @AfterAll failure should be reported in the log + outputValidator.verifyTextInLog("AfterAll always fails"); + + // The passing test class should still be error-free + outputValidator + .getSurefireReportsXmlFile("TEST-junit5.PassingTest.xml") + .assertContainsText("tests=\"1\" errors=\"0\""); + + // BUG: The XML summary says errors="0" despite the @AfterAll error being present as a testcase. + // The error is misclassified as a flake. + // After fix, this should assert errors="1" and flakes="0". + outputValidator + .getSurefireReportsXmlFile("TEST-junit5.AlwaysFailingAfterAllTest.xml") + .assertContainsText("errors=\"0\"") + .assertContainsText("flakes=\"1\""); + + // The @AfterAll error testcase is named "initializationError". + outputValidator + .getSurefireReportsXmlFile("TEST-junit5.AlwaysFailingAfterAllTest.xml") + .assertContainsText("name=\"initializationError\""); + } + + @Test + public void testAfterAllFailureWithRerun() throws VerificationException { + // BUG: With reruns, the build actually PASSES because the @AfterAll error is + // misclassified as a flake. This is the scenario reported in PR #3329: + // the error is swallowed and the build appears green. + // After fix, the build should fail because the @AfterAll always errors. + OutputValidator outputValidator = unpack("junit5-failing-after-all", "-rerun") + .setJUnitVersion(VERSION) + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=2") + .executeTest(); + + // The error is misclassified as a flake in the XML + outputValidator + .getSurefireReportsXmlFile("TEST-junit5.AlwaysFailingAfterAllTest.xml") + .assertContainsText("errors=\"0\"") + .assertContainsText("flakes=\"1\""); + + // BUG: The passing test methods get re-executed 3 times (rerunFailingTestsCount + 1) + // because the @AfterAll error triggers a class-level rerun. + // After fix, passing test methods should not be rerun due to a teardown failure. + outputValidator.assertThatLogLine(containsString("testOne passed"), is(3)); + outputValidator.assertThatLogLine(containsString("testTwo passed"), is(3)); + } +} diff --git a/surefire-its/src/test/resources/junit4-failing-after-class/pom.xml b/surefire-its/src/test/resources/junit4-failing-after-class/pom.xml new file mode 100644 index 0000000000..9e18e234c5 --- /dev/null +++ b/surefire-its/src/test/resources/junit4-failing-after-class/pom.xml @@ -0,0 +1,55 @@ + + + + + 4.0.0 + + org.apache.maven.plugins.surefire + junit4-failing-after-class + 1.0-SNAPSHOT + Test for failing @AfterClass in JUnit 4 + + + 1.8 + 1.8 + + + + + junit + junit + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + + diff --git a/surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/AlwaysFailingAfterClassTest.java b/surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/AlwaysFailingAfterClassTest.java new file mode 100644 index 0000000000..8e87f19675 --- /dev/null +++ b/surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/AlwaysFailingAfterClassTest.java @@ -0,0 +1,29 @@ +package junit4; + +import org.junit.AfterClass; +import org.junit.Test; + +/** + * Test class with @AfterClass that always fails. + * All test methods pass, but the class-level teardown always throws. + */ +public class AlwaysFailingAfterClassTest +{ + @AfterClass + public static void tearDown() + { + throw new IllegalStateException( "AfterClass always fails" ); + } + + @Test + public void testOne() + { + System.out.println( "testOne passed" ); + } + + @Test + public void testTwo() + { + System.out.println( "testTwo passed" ); + } +} diff --git a/surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/PassingTest.java b/surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/PassingTest.java new file mode 100644 index 0000000000..d60a7985c9 --- /dev/null +++ b/surefire-its/src/test/resources/junit4-failing-after-class/src/test/java/junit4/PassingTest.java @@ -0,0 +1,16 @@ +package junit4; + +import org.junit.Test; + +/** + * A simple passing test class to verify that other tests are unaffected + * by the failing @AfterClass in another test class. + */ +public class PassingTest +{ + @Test + public void testPassingOne() + { + System.out.println( "testPassingOne passed" ); + } +} diff --git a/surefire-its/src/test/resources/junit5-failing-after-all/pom.xml b/surefire-its/src/test/resources/junit5-failing-after-all/pom.xml new file mode 100644 index 0000000000..fc21ea7a59 --- /dev/null +++ b/surefire-its/src/test/resources/junit5-failing-after-all/pom.xml @@ -0,0 +1,57 @@ + + + + + 4.0.0 + + org.apache.maven.plugins.surefire + junit5-failing-after-all + 1.0-SNAPSHOT + Test for failing @AfterAll in JUnit 5 + + + 1.8 + 1.8 + + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + + + diff --git a/surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/AlwaysFailingAfterAllTest.java b/surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/AlwaysFailingAfterAllTest.java new file mode 100644 index 0000000000..9180a439c5 --- /dev/null +++ b/surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/AlwaysFailingAfterAllTest.java @@ -0,0 +1,29 @@ +package junit5; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +/** + * Test class with @AfterAll that always fails. + * All test methods pass, but the class-level teardown always throws. + */ +public class AlwaysFailingAfterAllTest +{ + @AfterAll + static void tearDown() + { + throw new IllegalStateException( "AfterAll always fails" ); + } + + @Test + public void testOne() + { + System.out.println( "testOne passed" ); + } + + @Test + public void testTwo() + { + System.out.println( "testTwo passed" ); + } +} diff --git a/surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/PassingTest.java b/surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/PassingTest.java new file mode 100644 index 0000000000..f5aa3381b3 --- /dev/null +++ b/surefire-its/src/test/resources/junit5-failing-after-all/src/test/java/junit5/PassingTest.java @@ -0,0 +1,16 @@ +package junit5; + +import org.junit.jupiter.api.Test; + +/** + * A simple passing test class to verify that other tests are unaffected + * by the failing @AfterAll in another test class. + */ +public class PassingTest +{ + @Test + public void testPassingOne() + { + System.out.println( "testPassingOne passed" ); + } +} From ca356d17f59d0daac289d3f9927ebbfa3f0a6fc5 Mon Sep 17 00:00:00 2001 From: Christopher Lambert Date: Mon, 6 Apr 2026 08:29:27 +0200 Subject: [PATCH 2/2] remove bug comments --- .../its/JUnit4FailingAfterClassIT.java | 28 ++----------------- .../surefire/its/JUnit5FailingAfterAllIT.java | 28 ++----------------- 2 files changed, 4 insertions(+), 52 deletions(-) diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java index 3655093e7b..8a93b45ed4 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit4FailingAfterClassIT.java @@ -29,14 +29,6 @@ /** * Integration test for JUnit 4 @AfterClass that always fails. * Verifies that surefire properly reports errors when class-level teardown throws. - * - *

Current behavior on master is broken: - *

- * - *

Once fixed, the XML should report errors="1". */ public class JUnit4FailingAfterClassIT extends SurefireJUnit4IntegrationTestCase { private static final String VERSION = "4.13.2"; @@ -49,49 +41,33 @@ public void testAfterClassFailureIsReported() { .withFailure() .executeTest(); - // The @AfterClass failure should be reported in the log outputValidator.verifyTextInLog("AfterClass always fails"); - // The passing test class should still be error-free outputValidator .getSurefireReportsXmlFile("TEST-junit4.PassingTest.xml") .assertContainsText("tests=\"1\" errors=\"0\""); - // BUG: The XML summary says errors="0" despite the @AfterClass error being present as a testcase. - // The error is misclassified as a flake. - // After fix, this should assert errors="1" and flakes="0". outputValidator .getSurefireReportsXmlFile("TEST-junit4.AlwaysFailingAfterClassTest.xml") + .assertContainsText("name=\"initializationError\"") .assertContainsText("errors=\"0\"") .assertContainsText("flakes=\"1\""); - - // The @AfterClass error testcase is named "initializationError". - outputValidator - .getSurefireReportsXmlFile("TEST-junit4.AlwaysFailingAfterClassTest.xml") - .assertContainsText("name=\"initializationError\""); } @Test public void testAfterClassFailureWithRerun() throws VerificationException { - // BUG: With reruns, the build actually PASSES because the @AfterClass error is - // misclassified as a flake. This is the scenario reported in PR #3329: - // the error is swallowed and the build appears green. - // After fix, the build should fail because the @AfterClass always errors. OutputValidator outputValidator = unpack("junit4-failing-after-class", "-rerun") .setJUnitVersion(VERSION) .maven() .addGoal("-Dsurefire.rerunFailingTestsCount=2") .executeTest(); - // The error is misclassified as a flake in the XML outputValidator .getSurefireReportsXmlFile("TEST-junit4.AlwaysFailingAfterClassTest.xml") + .assertContainsText("name=\"initializationError\"") .assertContainsText("errors=\"0\"") .assertContainsText("flakes=\"1\""); - // BUG: The passing test methods get re-executed 3 times (rerunFailingTestsCount + 1) - // because the @AfterClass error triggers a class-level rerun. - // After fix, passing test methods should not be rerun due to a teardown failure. outputValidator.assertThatLogLine(containsString("testOne passed"), is(3)); outputValidator.assertThatLogLine(containsString("testTwo passed"), is(3)); } diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java index a1f64e218f..4e079563b8 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5FailingAfterAllIT.java @@ -29,14 +29,6 @@ /** * Integration test for JUnit 5 @AfterAll that always fails. * Verifies that surefire properly reports errors when class-level teardown throws. - * - *

Current behavior on master is broken: - *

- * - *

Once fixed, the XML should report errors="1". */ public class JUnit5FailingAfterAllIT extends SurefireJUnit4IntegrationTestCase { private static final String VERSION = "5.9.1"; @@ -49,49 +41,33 @@ public void testAfterAllFailureIsReported() { .withFailure() .executeTest(); - // The @AfterAll failure should be reported in the log outputValidator.verifyTextInLog("AfterAll always fails"); - // The passing test class should still be error-free outputValidator .getSurefireReportsXmlFile("TEST-junit5.PassingTest.xml") .assertContainsText("tests=\"1\" errors=\"0\""); - // BUG: The XML summary says errors="0" despite the @AfterAll error being present as a testcase. - // The error is misclassified as a flake. - // After fix, this should assert errors="1" and flakes="0". outputValidator .getSurefireReportsXmlFile("TEST-junit5.AlwaysFailingAfterAllTest.xml") + .assertContainsText("name=\"initializationError\"") .assertContainsText("errors=\"0\"") .assertContainsText("flakes=\"1\""); - - // The @AfterAll error testcase is named "initializationError". - outputValidator - .getSurefireReportsXmlFile("TEST-junit5.AlwaysFailingAfterAllTest.xml") - .assertContainsText("name=\"initializationError\""); } @Test public void testAfterAllFailureWithRerun() throws VerificationException { - // BUG: With reruns, the build actually PASSES because the @AfterAll error is - // misclassified as a flake. This is the scenario reported in PR #3329: - // the error is swallowed and the build appears green. - // After fix, the build should fail because the @AfterAll always errors. OutputValidator outputValidator = unpack("junit5-failing-after-all", "-rerun") .setJUnitVersion(VERSION) .maven() .addGoal("-Dsurefire.rerunFailingTestsCount=2") .executeTest(); - // The error is misclassified as a flake in the XML outputValidator .getSurefireReportsXmlFile("TEST-junit5.AlwaysFailingAfterAllTest.xml") + .assertContainsText("name=\"initializationError\"") .assertContainsText("errors=\"0\"") .assertContainsText("flakes=\"1\""); - // BUG: The passing test methods get re-executed 3 times (rerunFailingTestsCount + 1) - // because the @AfterAll error triggers a class-level rerun. - // After fix, passing test methods should not be rerun due to a teardown failure. outputValidator.assertThatLogLine(containsString("testOne passed"), is(3)); outputValidator.assertThatLogLine(containsString("testTwo passed"), is(3)); }