[BP-2.1][FLINK-40284][tests] Introduce check to guard test names

flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/TestCodeArchitectureTestBase.java

@@ -19,6 +19,7 @@
 package org.apache.flink.architecture;
 
 import org.apache.flink.architecture.rules.ITCaseRules;
+import org.apache.flink.architecture.rules.TestNamingRules;
 
 import com.tngtech.archunit.junit.ArchTest;
 import com.tngtech.archunit.junit.ArchTests;
@@ -33,4 +34,6 @@
 public class TestCodeArchitectureTestBase {
 
     @ArchTest public static final ArchTests ITCASE = ArchTests.in(ITCaseRules.class);
+
+    @ArchTest public static final ArchTests TEST_NAMING = ArchTests.in(TestNamingRules.class);
 }

flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/rules/TestNamingRules.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.architecture.rules;
+
+import com.tngtech.archunit.base.DescribedPredicate;
+import com.tngtech.archunit.core.domain.JavaClass;
+import com.tngtech.archunit.junit.ArchTest;
+import com.tngtech.archunit.lang.ArchRule;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static com.tngtech.archunit.core.domain.JavaModifier.ABSTRACT;
+import static org.apache.flink.architecture.common.GivenJavaClasses.javaClassesThat;
+
+/**
+ * Rules ensuring executable test classes are named so the build actually runs them.
+ *
+ * <p>Surefire only runs the unit include pattern {@code **}{@code /*Test.*} in the {@code test}
+ * phase; integration tests follow the {@code *ITCase} convention. A concrete class that carries (or
+ * inherits) JUnit test methods but is named otherwise (e.g. {@code *Tests}) is silently skipped by
+ * the unit run. This rule flags such classes so they are renamed to {@code *Test} or {@code
+ * *ITCase}.
+ */
+public class TestNamingRules {
+
+    /** JUnit 5 and (for modules still mid-migration) JUnit 4 test method annotations. */
+    private static final List<String> TEST_METHOD_ANNOTATIONS =
+            Arrays.asList(
+                    "org.junit.jupiter.api.Test",
+                    "org.junit.jupiter.api.TestTemplate",
+                    "org.junit.jupiter.api.RepeatedTest",
+                    "org.junit.jupiter.api.TestFactory",
+                    "org.junit.jupiter.params.ParameterizedTest",
+                    "org.junit.Test");
+
+    /**
+     * A class JUnit would execute: it declares or inherits a test method. {@code getAllMethods()}
+     * covers inherited {@code @TestTemplate} methods, e.g. semantic-test suites that only extend a
+     * base and add no annotation themselves.
+     */
+    private static final DescribedPredicate<JavaClass> ARE_EXECUTABLE_TEST_CLASSES =
+            DescribedPredicate.describe(
+                    "are executable JUnit test classes",
+                    clazz ->
+                            clazz.getAllMethods().stream()
+                                    .anyMatch(
+                                            method ->
+                                                    TEST_METHOD_ANNOTATIONS.stream()
+                                                            .anyMatch(method::isAnnotatedWith)));
+
+    @ArchTest
+    public static final ArchRule TEST_CLASSES_SHOULD_BE_NAMED_TEST_OR_ITCASE =
+            javaClassesThat()
+                    .areTopLevelClasses()
+                    .and()
+                    .doNotHaveModifier(ABSTRACT)
+                    .and(ARE_EXECUTABLE_TEST_CLASSES)
+                    .should()
+                    .haveSimpleNameEndingWith("Test")
+                    .orShould()
+                    .haveSimpleNameEndingWith("Tests")
+                    .orShould()
+                    .haveSimpleNameEndingWith("ITCase")
+                    // not every module has such classes
+                    .allowEmptyShould(true)
+                    .as(
+                            "Executable test classes must be named *Test[s] or *ITCase so the surefire "
+                                    + "include pattern runs them");
+}

flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecks.java → flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecksTest.java

@@ -29,7 +29,7 @@
  * Verifies correct accesses with regards to endianness in {@link MemorySegment} (in both heap and
  * off-heap modes).
  */
-class EndiannessAccessChecks {
+class EndiannessAccessChecksTest {
 
     @Test
     void testOnHeapSegment() {

flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterIT.java → flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterITCase.java

@@ -27,10 +27,13 @@
 import org.apache.flink.connector.datagen.source.TestDataGenerators;
 import org.apache.flink.connector.file.sink.FileSink;
 import org.apache.flink.core.fs.FSDataOutputStream;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.junit5.MiniClusterExtension;
 
 import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 import org.junit.jupiter.api.io.TempDir;
 
 import java.io.File;
@@ -44,7 +47,14 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-public class CsvBulkWriterIT {
+class CsvBulkWriterITCase {
+
+    @RegisterExtension
+    static final MiniClusterExtension MINI_CLUSTER =
+            new MiniClusterExtension(
+                    new MiniClusterResourceConfiguration.Builder()
+                            .setNumberTaskManagers(2)
+                            .build());
 
     @TempDir File outDir;
 
@@ -53,7 +63,7 @@ public class CsvBulkWriterIT {
      * flush signal from Flink.
      */
     @Test
-    public void testNoDataIsWrittenBeforeFlinkFlush() throws Exception {
+    void testNoDataIsWrittenBeforeFlinkFlush() throws Exception {
 
         Configuration config = new Configuration();
         config.set(

flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java → flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompletenessTest.java

@@ -19,6 +19,13 @@
 package org.apache.flink.table.planner.plan.nodes.exec.testutils;
 
 import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
+import org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecHashAggregate;
+import org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecNestedLoopJoin;
+import org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecSortAggregate;
+import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecDeltaJoin;
+import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecGlobalWindowAggregate;
+import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLocalWindowAggregate;
+import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecProcessTableFunction;
 import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCalc;
 import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCorrelate;
 import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonGroupAggregate;
@@ -30,7 +37,6 @@
 
 import org.apache.flink.shaded.guava33.com.google.common.reflect.ClassPath;
 
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
@@ -42,21 +48,34 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.assertj.core.api.Assertions.fail;
+
 /** Validate restore tests exists for Exec Nodes. */
-public class RestoreTestCompleteness {
+class RestoreTestCompletenessTest {
 
     private static final Set<Class<? extends ExecNode<?>>> SKIP_EXEC_NODES =
-            new HashSet<Class<? extends ExecNode<?>>>() {
-                {
-                    /** Ignoring python based exec nodes temporarily. */
-                    add(StreamExecPythonCalc.class);
-                    add(StreamExecPythonCorrelate.class);
-                    add(StreamExecPythonOverAggregate.class);
-                    add(StreamExecPythonGroupAggregate.class);
-                    add(StreamExecPythonGroupTableAggregate.class);
-                    add(StreamExecPythonGroupWindowAggregate.class);
-                }
-            };
+            Set.of(
+                    /* Ignoring python based exec nodes temporarily. */
+                    StreamExecPythonCalc.class,
+                    StreamExecPythonCorrelate.class,
+                    StreamExecPythonOverAggregate.class,
+                    StreamExecPythonGroupAggregate.class,
+                    StreamExecPythonGroupTableAggregate.class,
+                    StreamExecPythonGroupWindowAggregate.class,
+
+                    // Covered by tests in WindowAggregateEventTimeRestoreTest
+                    StreamExecLocalWindowAggregate.class,
+                    StreamExecGlobalWindowAggregate.class,
+                    // restore tests for delta join and PTF were added in later releases
+                    StreamExecDeltaJoin.class,
+                    StreamExecProcessTableFunction.class,
+
+                    // There is jira for these 2 batch tests
+                    // https://issues.apache.org/jira/browse/FLINK-40306
+                    BatchExecHashAggregate.class,
+                    BatchExecNestedLoopJoin.class,
+                    // restore tests for batch sort aggregate was added in later releases
+                    BatchExecSortAggregate.class);
 
     private Class<? extends ExecNode<?>> getExecNode(Class<?> restoreTest)
             throws NoSuchMethodException,
@@ -85,7 +104,7 @@ private List<Class<? extends ExecNode<?>>> getChildExecNodes(Class<?> restoreTes
     }
 
     @Test
-    public void testMissingRestoreTest()
+    void testMissingRestoreTest()
             throws IOException,
                     NoSuchMethodException,
                     InstantiationException,
@@ -95,12 +114,14 @@ public void testMissingRestoreTest()
                 ExecNodeMetadataUtil.getVersionedExecNodes();
 
         Set<ClassPath.ClassInfo> classesInPackage =
-                ClassPath.from(this.getClass().getClassLoader())
-                        .getTopLevelClassesRecursive(
-                                "org.apache.flink.table.planner.plan.nodes.exec.stream")
-                        .stream()
-                        .filter(x -> RestoreTestBase.class.isAssignableFrom(x.load()))
-                        .collect(Collectors.toSet());
+                new HashSet<>(
+                        gatherClasses(
+                                RestoreTestBase.class,
+                                "org.apache.flink.table.planner.plan.nodes.exec.stream"));
+        classesInPackage.addAll(
+                gatherClasses(
+                        BatchRestoreTestBase.class,
+                        "org.apache.flink.table.planner.plan.nodes.exec.batch"));
 
         Set<Class<? extends ExecNode<?>>> execNodesWithRestoreTests = new HashSet<>();
 
@@ -116,18 +137,33 @@ public void testMissingRestoreTest()
             }
         }
 
+        Set<Class<? extends ExecNode<?>>> productionExecNodes = ExecNodeMetadataUtil.execNodes();
         for (Map.Entry<ExecNodeNameVersion, Class<? extends ExecNode<?>>> entry :
                 versionedExecNodes.entrySet()) {
             ExecNodeNameVersion execNodeNameVersion = entry.getKey();
             Class<? extends ExecNode<?>> execNode = entry.getValue();
-            if (!SKIP_EXEC_NODES.contains(execNode)) {
-                final String msg =
+            // Ignore test-only nodes that other tests leak into the shared LOOKUP_MAP via
+            // addTestNode().
+            if (!productionExecNodes.contains(execNode)) {
+                continue;
+            }
+            if (!SKIP_EXEC_NODES.contains(execNode)
+                    && !execNodesWithRestoreTests.contains(execNode)) {
+                fail(
                         "Missing restore test for "
                                 + execNodeNameVersion
                                 + "\nPlease add a restore test for "
-                                + execNode.toString();
-                Assertions.assertTrue(execNodesWithRestoreTests.contains(execNode), msg);
+                                + execNode.toString());
             }
         }
     }
+
+    private Set<ClassPath.ClassInfo> gatherClasses(Class<?> clazz, String packageName)
+            throws IOException {
+        return ClassPath.from(this.getClass().getClassLoader())
+                .getTopLevelClassesRecursive(packageName)
+                .stream()
+                .filter(x -> clazz.isAssignableFrom(x.load()))
+                .collect(Collectors.toSet());
+    }
 }

pom.xml

@@ -217,7 +217,7 @@ under the License.
 
 		<!-- Can be set to any value to reproduce a specific build. -->
 		<test.randomization.seed/>
-		<test.unit.pattern>**/*Test.*</test.unit.pattern>
+		<test.unit.pattern>**/*Test.*,**/Tests.*</test.unit.pattern>
 		<snappy.java.version>1.1.10.7</snappy.java.version>
         <commons-lang3.version>3.18.0</commons-lang3.version>
 	</properties>