[SUREFIRE-1963] Detecting multiple test-frameworks

.gitignore

@@ -13,3 +13,4 @@ build
 .surefire-*
 .DS_Store
 *.versionsBackup
+velocity.log

maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java

@@ -107,6 +107,7 @@
 import org.apache.maven.surefire.extensions.ForkNodeFactory;
 import org.apache.maven.surefire.providerapi.ConfigurableProviderInfo;
 import org.apache.maven.surefire.providerapi.ProviderDetector;
+import org.apache.maven.surefire.providerapi.ProviderDetectorRequest;
 import org.apache.maven.surefire.providerapi.ProviderInfo;
 import org.apache.maven.surefire.providerapi.ProviderRequirements;
 import org.apache.maven.surefire.shared.utils.io.FileUtils;
@@ -811,6 +812,16 @@ public abstract class AbstractSurefireMojo
     @Parameter
     private Map<String, String> jdkToolchain;
 
+    /**
+     * Option to specify the action when multiple tests frameworks are present on project classpath.
+     * <p>
+     * Can be {@code warn} - only warn, or {@code fail} to break execution.
+     *
+     * @since 3.0.0-M6
+     */
+    @Parameter
+    private String multipleFrameworks;
+
     /**
      *
      */
@@ -1194,13 +1205,31 @@ private void executeAfterPreconditionsChecked( @Nonnull DefaultScanResult scanRe
     protected List<ProviderInfo> createProviders( TestClassPath testClasspath )
         throws MojoExecutionException
     {
+        Artifact junitArtifact = getJunitArtifact();
         Artifact junitDepArtifact = getJunitDepArtifact();
-        return providerDetector.resolve( new DynamicProviderInfo( null ),
+
+        ProviderDetectorRequest request = new ProviderDetectorRequest();
+        request.setDynamicProvider( new DynamicProviderInfo( null ) );
+
+        // after change order of well known providers, please update:
+        // maven-surefire-plugin/src/site/apt/examples/providers.apt.vm
+        request.setWellKnownProviders(
             new JUnitPlatformProviderInfo( getJUnit5Artifact(), testClasspath ),
             new TestNgProviderInfo( getTestNgArtifact() ),
-            new JUnitCoreProviderInfo( getJunitArtifact(), junitDepArtifact ),
-            new JUnit4ProviderInfo( getJunitArtifact(), junitDepArtifact ),
-            new JUnit3ProviderInfo() );
+            new JUnitCoreProviderInfo( junitArtifact, junitDepArtifact ),
+            new JUnit4ProviderInfo( junitArtifact, junitDepArtifact )
+        );
+
+        request.setDefaultProvider( new JUnit3ProviderInfo() );
+        request.setMultipleFrameworks( multipleFrameworks );
+
+        List<ProviderInfo> providerInfoList = providerDetector.resolve( request );
+
+        if ( providerInfoList.isEmpty() )
+        {
+            throw new MojoExecutionException( "No test-framework providers for test execution was found." );
+        }
+        return providerInfoList;
     }
 
     private SurefireProperties setupProperties()

maven-surefire-common/src/main/java/org/apache/maven/surefire/providerapi/ProviderDetector.java

@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
@@ -46,35 +47,96 @@ public final class ProviderDetector
     private ServiceLoader serviceLoader;
 
     @Nonnull
-    public List<ProviderInfo> resolve( ConfigurableProviderInfo dynamicProvider, ProviderInfo... wellKnownProviders )
+    public List<ProviderInfo> resolve( ProviderDetectorRequest request )
     {
         List<ProviderInfo> providersToRun = new ArrayList<>();
         Set<String> manuallyConfiguredProviders = getManuallyConfiguredProviders();
         for ( String name : manuallyConfiguredProviders )
         {
-            ProviderInfo wellKnown = findByName( name, wellKnownProviders );
-            ProviderInfo providerToAdd = wellKnown != null ? wellKnown : dynamicProvider.instantiate( name );
+            ProviderInfo wellKnown = findByName( name, request.getWellKnownProviders() );
+            ProviderInfo providerToAdd =
+                wellKnown != null ? wellKnown : request.getDynamicProvider().instantiate( name );
             logger.info( "Using configured provider " + providerToAdd.getProviderName() );
             providersToRun.add( providerToAdd );
         }
-        return manuallyConfiguredProviders.isEmpty() ? autoDetectOneWellKnownProvider( wellKnownProviders )
-            : providersToRun;
+
+        return providersToRun.isEmpty() ? autoDetectOneWellKnownProvider( request ) : providersToRun;
     }
 
     @Nonnull
-    private List<ProviderInfo> autoDetectOneWellKnownProvider( ProviderInfo... wellKnownProviders )
+    private List<ProviderInfo> autoDetectOneWellKnownProvider( ProviderDetectorRequest request )
     {
-        List<ProviderInfo> providersToRun = new ArrayList<>();
-        for ( ProviderInfo wellKnownProvider : wellKnownProviders )
+        List<ProviderInfo> applicableProviders = new ArrayList<>();
+        for ( ProviderInfo wellKnownProvider : request.getWellKnownProviders() )
         {
             if ( wellKnownProvider.isApplicable() )
             {
-                logger.info( "Using auto detected provider " + wellKnownProvider.getProviderName() );
-                providersToRun.add( wellKnownProvider );
-                return providersToRun;
+                applicableProviders.add( wellKnownProvider );
+            }
+        }
+
+        if ( applicableProviders.isEmpty() )
+        {
+            return Collections.singletonList(  request.getDefaultProvider() );
+        }
+
+        ProviderInfo providerInfoToReturn = applicableProviders.get( 0 );
+        if ( applicableProviders.size() > 1 && !isJunit4SpecialCase( applicableProviders ) )
+        {
+            if ( request.isFailOnMultipleFrameworks() )
+            {
+                logger.error( "There are many providers automatically detected:" );
+                for ( ProviderInfo providerInfo : applicableProviders )
+                {
+                    logger.error( "   " + providerInfo.getProviderName() );
+                }
+                logger.error( "" );
+                logger.error( "Please select providers manually or check project dependency tree." );
+                logger.error( "" );
+                providerInfoToReturn = null;
+            }
+            else if ( request.isWarnOnMultipleFrameworks() )
+            {
+                logger.warn( "There are many providers automatically detected:" );
+                for ( ProviderInfo providerInfo : applicableProviders )
+                {
+                    logger.warn( "   " + providerInfo.getProviderName() );
+                }
+                logger.warn( "" );
+                logger.warn( "Only first will be used." );
+                logger.warn( "" );
             }
         }
-        return providersToRun;
+
+        if ( providerInfoToReturn != null )
+        {
+            logger.info( "Using auto detected provider " + providerInfoToReturn.getProviderName() );
+            return Collections.singletonList( providerInfoToReturn );
+        }
+
+        return Collections.<ProviderInfo>emptyList();
+    }
+
+    /**
+     * For JUnit, we have two providers and both can be applicable for junit &gt;= 4.7.
+     * In this special case we don't generate warning or break executions.
+     * <p>
+     * Should be removed after SUREFIRE-1494.
+     */
+    private boolean isJunit4SpecialCase( List<ProviderInfo> applicableProviders )
+    {
+
+        if ( applicableProviders.size() == 2 )
+        {
+            String providerName1 = applicableProviders.get( 0 ).getProviderName();
+            String providerName2 = applicableProviders.get( 1 ).getProviderName();
+
+            boolean p1 = providerName1.endsWith( "JUnitCoreProvider" ) || providerName1.endsWith( "JUnit4Provider" );
+            boolean p2 = providerName2.endsWith( "JUnitCoreProvider" ) || providerName2.endsWith( "JUnit4Provider" );
+            return p1 && p2;
+        }
+
+        return false;
     }
 
     private Set<String> getManuallyConfiguredProviders()

maven-surefire-common/src/main/java/org/apache/maven/surefire/providerapi/ProviderDetectorRequest.java

@@ -0,0 +1,90 @@
+package org.apache.maven.surefire.providerapi;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Locale;
+
+/**
+ * Request data for provider detector.
+ *
+ * @author Slawomir Jaranowski
+ */
+public class ProviderDetectorRequest
+{
+    private ConfigurableProviderInfo dynamicProvider;
+    private ProviderInfo defaultProvider;
+    private ProviderInfo[] wellKnownProviders;
+    private String multipleFrameworks;
+
+    public ConfigurableProviderInfo getDynamicProvider()
+    {
+        return dynamicProvider;
+    }
+
+    public void setDynamicProvider( ConfigurableProviderInfo dynamicProvider )
+    {
+        this.dynamicProvider = dynamicProvider;
+    }
+
+    public ProviderInfo getDefaultProvider()
+    {
+        return defaultProvider;
+    }
+
+    public void setDefaultProvider( ProviderInfo defaultProvider )
+    {
+        this.defaultProvider = defaultProvider;
+    }
+
+    public ProviderInfo[] getWellKnownProviders()
+    {
+        return wellKnownProviders;
+    }
+
+    public void setWellKnownProviders( ProviderInfo... wellKnownProviders )
+    {
+        this.wellKnownProviders = wellKnownProviders;
+    }
+
+    public boolean isWarnOnMultipleFrameworks()
+    {
+        return "warn".equals( multipleFrameworks );
+    }
+
+    public boolean isFailOnMultipleFrameworks()
+    {
+        return "fail".equals( multipleFrameworks );
+    }
+
+    public void setMultipleFrameworks( String multipleFrameworks )
+    {
+        if ( multipleFrameworks != null )
+        {
+            this.multipleFrameworks = multipleFrameworks.trim().toLowerCase( Locale.ROOT );
+            if ( !( "warn".equals( this.multipleFrameworks )
+                || "fail".equals( this.multipleFrameworks ) ) )
+            {
+                throw new IllegalArgumentException( "multipleFrameworks value:"
+                    + " \"" + multipleFrameworks + "\""
+                    + " must be one of \"warn\" or \"fail\"" );
+            }
+        }
+    }
+}

maven-surefire-common/src/test/java/org/apache/maven/surefire/providerapi/ProviderDetectorRequestTest.java

@@ -0,0 +1,88 @@
+package org.apache.maven.surefire.providerapi;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+/**
+ * Simple test for {@link ProviderDetectorRequest}
+ *
+ * @author Slawomir Jaranowski
+ */
+public class ProviderDetectorRequestTest
+{
+
+    @Rule
+    public final ExpectedException e = ExpectedException.none();
+
+    @Test
+    public void invalidAutoDetectMultipleFrameworks()
+    {
+        ProviderDetectorRequest request = new ProviderDetectorRequest();
+
+        e.expect( IllegalArgumentException.class );
+        request.setMultipleFrameworks( "invalidValue" );
+    }
+
+    @Test
+    public void nullMultipleFrameworks()
+    {
+        ProviderDetectorRequest request = new ProviderDetectorRequest();
+        request.setMultipleFrameworks( null );
+
+        assertThat( request.isWarnOnMultipleFrameworks() ).isFalse();
+        assertThat( request.isFailOnMultipleFrameworks() ).isFalse();
+    }
+
+    @Test
+    public void defaultMultipleFrameworks()
+    {
+        ProviderDetectorRequest request = new ProviderDetectorRequest();
+
+        assertThat( request.isWarnOnMultipleFrameworks() ).isFalse();
+        assertThat( request.isFailOnMultipleFrameworks() ).isFalse();
+    }
+
+    @Test
+    public void warnOnMultipleFrameworks()
+    {
+        ProviderDetectorRequest request = new ProviderDetectorRequest();
+
+        request.setMultipleFrameworks( "wArn" );
+
+        assertThat( request.isWarnOnMultipleFrameworks() ).isTrue();
+        assertThat( request.isFailOnMultipleFrameworks() ).isFalse();
+    }
+
+    @Test
+    public void failOnMultipleFrameworks()
+    {
+        ProviderDetectorRequest request = new ProviderDetectorRequest();
+
+        request.setMultipleFrameworks( "Fail" );
+
+        assertThat( request.isWarnOnMultipleFrameworks() ).isFalse();
+        assertThat( request.isFailOnMultipleFrameworks() ).isTrue();
+    }
+}