Support Cartesian products of environments in MultiEnvTestEngine

compatibility/common/src/test/java/org/projectnessie/tools/compatibility/internal/TestNessieCompatibilityExtensions.java

@@ -27,6 +27,7 @@
 import org.assertj.core.api.SoftAssertions;
 import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
 import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.DisabledOnOs;
@@ -66,6 +67,12 @@ private void assertNoFailedTestEvents(EngineExecutionResults result) {
         .isEqualTo(0);
   }
 
+  @BeforeEach
+  public void beforeEach() {
+    MultiEnvTestEngine.clearRegistry();
+    MultiEnvTestFilter.clear();
+  }
+
   @Test
   void noVersions() {
     soft.assertThatThrownBy(
@@ -76,8 +83,7 @@ void noVersions() {
         .hasMessageContaining("TestEngine with ID 'nessie-multi-env' failed to discover tests")
         .cause()
         .hasMessageContaining(
-            "MultiEnvTestEngine was enabled, but test extensions did not discover any environment IDs")
-        .hasMessageContaining("OlderNessieClientsExtension");
+            "MultiEnvTestEngine was enabled, but test extensions [OlderNessieClientsExtension] did not discover any environment IDs.");
   }
 
   @Test

testing/multi-env-test-engine/src/main/java/org/projectnessie/junit/engine/MultiEnvDisplayNameGenerator.java

@@ -21,25 +21,28 @@
 public class MultiEnvDisplayNameGenerator implements DisplayNameGenerator {
 
   private final DisplayNameGenerator delegate;
-  private final String environment;
+  private final String environmentNames;
 
-  public MultiEnvDisplayNameGenerator(DisplayNameGenerator delegate, String environment) {
+  public MultiEnvDisplayNameGenerator(DisplayNameGenerator delegate, String environmentNames) {
     this.delegate = delegate;
-    this.environment = environment;
+    this.environmentNames = environmentNames;
   }
 
   @Override
   public String generateDisplayNameForClass(Class<?> testClass) {
-    return delegate.generateDisplayNameForClass(testClass) + " [" + environment + "]";
+    return delegate.generateDisplayNameForClass(testClass) + " [" + environmentNames + "]";
   }
 
   @Override
   public String generateDisplayNameForNestedClass(Class<?> nestedClass) {
-    return delegate.generateDisplayNameForNestedClass(nestedClass) + " [" + environment + "]";
+    return delegate.generateDisplayNameForNestedClass(nestedClass) + " [" + environmentNames + "]";
   }
 
   @Override
   public String generateDisplayNameForMethod(Class<?> testClass, Method testMethod) {
-    return delegate.generateDisplayNameForMethod(testClass, testMethod) + " [" + environment + "]";
+    return delegate.generateDisplayNameForMethod(testClass, testMethod)
+        + " ["
+        + environmentNames
+        + "]";
   }
 }

testing/multi-env-test-engine/src/main/java/org/projectnessie/junit/engine/MultiEnvExtensionRegistry.java

@@ -31,12 +31,10 @@
  * execution.
  */
 public class MultiEnvExtensionRegistry {
-  private final MutableExtensionRegistry registry;
+  private MutableExtensionRegistry registry;
 
   public MultiEnvExtensionRegistry() {
-    this.registry =
-        MutableExtensionRegistry.createRegistryWithDefaultExtensions(
-            new DefaultJupiterConfiguration(new EmptyConfigurationParameters()));
+    clear();
   }
 
   public void registerExtensions(Class<?> testClass) {
@@ -66,4 +64,10 @@ public Stream<? extends MultiEnvTestExtension> stream(Class<?> testClass) {
                 .flatMap(registry::stream);
     return r;
   }
+
+  public void clear() {
+    registry =
+        MutableExtensionRegistry.createRegistryWithDefaultExtensions(
+            new DefaultJupiterConfiguration(new EmptyConfigurationParameters()));
+  }
 }

testing/multi-env-test-engine/src/main/java/org/projectnessie/junit/engine/MultiEnvJupiterConfiguration.java

@@ -21,17 +21,17 @@
 
 public class MultiEnvJupiterConfiguration extends DefaultJupiterConfiguration {
 
-  private final String environment;
+  private final String environmentNames;
 
   public MultiEnvJupiterConfiguration(
-      ConfigurationParameters configurationParameters, String environment) {
+      ConfigurationParameters configurationParameters, String environmentNames) {
     super(configurationParameters);
-    this.environment = environment;
+    this.environmentNames = environmentNames;
   }
 
   @Override
   public DisplayNameGenerator getDefaultDisplayNameGenerator() {
     DisplayNameGenerator delegate = super.getDefaultDisplayNameGenerator();
-    return new MultiEnvDisplayNameGenerator(delegate, environment);
+    return new MultiEnvDisplayNameGenerator(delegate, environmentNames);
   }
 }

testing/multi-env-test-engine/src/main/java/org/projectnessie/junit/engine/MultiEnvTestDescriptorTree.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2023 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.junit.engine;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import org.junit.jupiter.engine.config.CachingJupiterConfiguration;
+import org.junit.jupiter.engine.config.JupiterConfiguration;
+import org.junit.jupiter.engine.descriptor.JupiterEngineDescriptor;
+import org.junit.jupiter.engine.discovery.DiscoverySelectorResolver;
+import org.junit.platform.engine.ConfigurationParameters;
+import org.junit.platform.engine.EngineDiscoveryRequest;
+import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.UniqueId;
+import org.junit.platform.engine.UniqueId.Segment;
+
+public class MultiEnvTestDescriptorTree {
+
+  private static final boolean FAIL_ON_MISSING_ENVIRONMENTS =
+      !Boolean.getBoolean("org.projectnessie.junit.engine.ignore-empty-environments");
+
+  private final TestDescriptor rootNode;
+  private final ConfigurationParameters configurationParameters;
+  private final AtomicBoolean foundAtLeastOneEnvironmentId = new AtomicBoolean();
+  private final List<String> processedExtensionNames = new ArrayList<>();
+  private List<TestDescriptor> latestLeafNodes;
+
+  public MultiEnvTestDescriptorTree(
+      TestDescriptor rootNode, ConfigurationParameters configurationParameters) {
+    this.rootNode = rootNode;
+    this.latestLeafNodes = Collections.singletonList(rootNode);
+    this.configurationParameters = configurationParameters;
+  }
+
+  public void appendDescriptorsForExtension(MultiEnvTestExtension multiEnvTestExtension) {
+    processedExtensionNames.add(multiEnvTestExtension.getClass().getSimpleName());
+
+    List<TestDescriptor> newLeafNodes = new ArrayList<>();
+    for (TestDescriptor parentNode : latestLeafNodes) {
+      for (String environmentId :
+          multiEnvTestExtension.allEnvironmentIds(configurationParameters)) {
+        foundAtLeastOneEnvironmentId.set(true);
+        UniqueId newNodeId =
+            parentNode.getUniqueId().append(multiEnvTestExtension.segmentType(), environmentId);
+        MultiEnvTestDescriptor newNode = new MultiEnvTestDescriptor(newNodeId, environmentId);
+        parentNode.addChild(newNode);
+        newLeafNodes.add(newNode);
+      }
+    }
+
+    latestLeafNodes = newLeafNodes;
+  }
+
+  public void addOriginalChildrenToFinishedTree(EngineDiscoveryRequest discoveryRequest) {
+    final String rootNodeSegmentValue = rootNode.getUniqueId().getSegments().get(0).getValue();
+
+    for (TestDescriptor leafNode : latestLeafNodes) {
+      String environmentNames =
+          leafNode.getUniqueId().getSegments().stream()
+              .map(Segment::getValue)
+              .filter(Predicate.not(rootNodeSegmentValue::equals))
+              .collect(Collectors.joining(","));
+      JupiterConfiguration nodeConfiguration =
+          new CachingJupiterConfiguration(
+              new MultiEnvJupiterConfiguration(configurationParameters, environmentNames));
+      JupiterEngineDescriptor discoverResult =
+          new JupiterEngineDescriptor(leafNode.getUniqueId(), nodeConfiguration);
+      new DiscoverySelectorResolver().resolveSelectors(discoveryRequest, discoverResult);
+      for (TestDescriptor childWithProperId : discoverResult.getChildren()) {
+        leafNode.addChild(childWithProperId);
+      }
+    }
+  }
+
+  public TestDescriptor getRootNode() {
+    if (FAIL_ON_MISSING_ENVIRONMENTS
+        && !processedExtensionNames.isEmpty()
+        && !foundAtLeastOneEnvironmentId.get()) {
+      throw new IllegalStateException(
+          String.format(
+              "%s was enabled, but test extensions %s did not discover any environment IDs.",
+              MultiEnvTestEngine.class.getSimpleName(),
+              Arrays.toString(processedExtensionNames.toArray())));
+    }
+
+    return rootNode;
+  }
+}

testing/multi-env-test-engine/src/main/java/org/projectnessie/junit/engine/MultiEnvTestEngine.java

@@ -16,16 +16,11 @@
 package org.projectnessie.junit.engine;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.atomic.AtomicBoolean;
 import org.junit.jupiter.engine.JupiterTestEngine;
-import org.junit.jupiter.engine.config.CachingJupiterConfiguration;
-import org.junit.jupiter.engine.config.DefaultJupiterConfiguration;
-import org.junit.jupiter.engine.config.JupiterConfiguration;
 import org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor;
-import org.junit.jupiter.engine.descriptor.JupiterEngineDescriptor;
-import org.junit.jupiter.engine.discovery.DiscoverySelectorResolver;
 import org.junit.platform.engine.EngineDiscoveryRequest;
 import org.junit.platform.engine.ExecutionRequest;
 import org.junit.platform.engine.TestDescriptor;
@@ -38,25 +33,41 @@
  * This is a JUnit5 Test Engine that delegates test discovery to {@link JupiterTestEngine} and
  * replicates the discovered tests for execution in multiple test environments.
  *
+ * <p>When multiple {@link MultiEnvTestExtension}s are applied to the same test, performs a
+ * cartesian product of available environments and their IDs.
+ *
+ * <p>For example:
+ *
+ * <ul>
+ *   <li>MultiEnvTestExtension segmentA with IDs 1, 2, 3
+ *   <li>MultiEnvTestExtension segmentB with IDs 1, 2
+ *   <li>MultiEnvTestExtension segmentC with ID 1
+ * </ul>
+ *
+ * will result in the following IDs:
+ *
+ * <ul>
+ *   <li>[engine:nessie-multi-env][segmentA:1][segmentB:1][segmentC:1]
+ *   <li>[engine:nessie-multi-env][segmentA:1][segmentB:2][segmentC:1]
+ *   <li>[engine:nessie-multi-env][segmentA:2][segmentB:1][segmentC:1]
+ *   <li>[engine:nessie-multi-env][segmentA:2][segmentB:2][segmentC:1]
+ *   <li>[engine:nessie-multi-env][segmentA:3][segmentB:1][segmentC:1]
+ *   <li>[engine:nessie-multi-env][segmentA:3][segmentB:2][segmentC:1]
+ * </ul>
+ *
  * <p>Actual test environments are expected to be managed by JUnit 5 extensions, implementing the
  * {@link MultiEnvTestExtension} interface.
  */
 public class MultiEnvTestEngine implements TestEngine {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(MultiEnvTestEngine.class);
-  private static final boolean FAIL_ON_MISSING_ENVIRONMENTS =
-      !Boolean.getBoolean("org.projectnessie.junit.engine.ignore-empty-environments");
 
   public static final String ENGINE_ID = "nessie-multi-env";
 
   private static final MultiEnvExtensionRegistry registry = new MultiEnvExtensionRegistry();
 
   private final JupiterTestEngine delegate = new JupiterTestEngine();
 
-  static MultiEnvExtensionRegistry registry() {
-    return registry;
-  }
-
   @Override
   public String getId() {
     return ENGINE_ID;
@@ -70,64 +81,36 @@ public void execute(ExecutionRequest request) {
   @Override
   public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) {
     try {
+      TestDescriptor originalRoot = delegate.discover(discoveryRequest, uniqueId);
+      List<TestDescriptor> originalChildren = new ArrayList<>(originalRoot.getChildren());
 
       // Scan for multi-env test extensions
-      TestDescriptor preliminaryResult = delegate.discover(discoveryRequest, uniqueId);
-      preliminaryResult.accept(
+      originalRoot.accept(
           descriptor -> {
             if (descriptor instanceof ClassBasedTestDescriptor) {
               Class<?> testClass = ((ClassBasedTestDescriptor) descriptor).getTestClass();
               registry.registerExtensions(testClass);
             }
           });
 
-      // JupiterEngineDescriptor must be the root, that's what the JUnit Jupiter engine
-      // implementation expects.
-      JupiterEngineDescriptor multiEnvDescriptor =
-          new JupiterEngineDescriptor(
-              uniqueId,
-              new DefaultJupiterConfiguration(discoveryRequest.getConfigurationParameters()));
+      // Note: this also removes the reference to parent from the child
+      originalChildren.forEach(originalRoot::removeChild);
 
-      List<String> extensions = new ArrayList<>();
-      AtomicBoolean envDiscovered = new AtomicBoolean();
+      // Append each extension's IDs in a new, nested, layer
+      MultiEnvTestDescriptorTree tree =
+          new MultiEnvTestDescriptorTree(
+              originalRoot, discoveryRequest.getConfigurationParameters());
       registry.stream()
-          .forEach(
-              ext -> {
-                extensions.add(ext.getClass().getSimpleName());
-                for (String envId :
-                    ext.allEnvironmentIds(discoveryRequest.getConfigurationParameters())) {
-                  envDiscovered.set(true);
-                  UniqueId segment = uniqueId.append(ext.segmentType(), envId);
-
-                  MultiEnvTestDescriptor envRoot = new MultiEnvTestDescriptor(segment, envId);
-                  multiEnvDescriptor.addChild(envRoot);
-
-                  JupiterConfiguration envRootConfiguration =
-                      new CachingJupiterConfiguration(
-                          new MultiEnvJupiterConfiguration(
-                              discoveryRequest.getConfigurationParameters(), envId));
-                  JupiterEngineDescriptor discoverResult =
-                      new JupiterEngineDescriptor(segment, envRootConfiguration);
-                  new DiscoverySelectorResolver()
-                      .resolveSelectors(discoveryRequest, discoverResult);
-
-                  List<? extends TestDescriptor> children =
-                      new ArrayList<>(discoverResult.getChildren());
-                  for (TestDescriptor child : children) {
-                    // Note: this also removes the reference to parent from the child
-                    discoverResult.removeChild(child);
-                    envRoot.addChild(child);
-                  }
-                }
-              });
-
-      if (!extensions.isEmpty() && !envDiscovered.get() && FAIL_ON_MISSING_ENVIRONMENTS) {
-        throw new IllegalStateException(
-            String.format(
-                "%s was enabled, but test extensions did not discover any environment IDs: %s",
-                getClass().getSimpleName(), extensions));
-      }
-      return multiEnvDescriptor;
+          .sorted(
+              Comparator.comparing(MultiEnvTestExtension::segmentPriority)
+                  .reversed()
+                  .thenComparing(MultiEnvTestExtension::segmentType))
+          .forEach(tree::appendDescriptorsForExtension);
+
+      // Migrate the actual tests from the root to each of the leaves of the new tree
+      tree.addOriginalChildrenToFinishedTree(discoveryRequest);
+
+      return tree.getRootNode();
     } catch (Exception e) {
       LOGGER.error("Failed to discover tests", e);
       throw new RuntimeException(e);
@@ -141,6 +124,10 @@ public Optional<String> getGroupId() {
 
   @Override
   public Optional<String> getArtifactId() {
-    return Optional.of("nessie-compatibility-tools");
+    return Optional.of("nessie-multi-env-test-engine");
+  }
+
+  public static void clearRegistry() {
+    registry.clear();
   }
 }

testing/multi-env-test-engine/src/main/java/org/projectnessie/junit/engine/MultiEnvTestExtension.java

@@ -37,4 +37,13 @@ public interface MultiEnvTestExtension extends Extension {
    * invocations of this method to ensure a stable test case creation order.
    */
   List<String> allEnvironmentIds(ConfigurationParameters configuration);
+
+  /**
+   * Allows {@link MultiEnvTestExtension}s to define their relative ordering within a JUnit
+   * UniqueID's segments. Higher numbers will appear earlier in the JUnit UniqueId. Extensions with
+   * the same segment priority will be sorted alphabetically by segmentType.
+   */
+  default int segmentPriority() {
+    return 0;
+  }
 }