Fix #271 isolate test runner (#272)

* test: add test using ReflectiveTestRunner

* feat: set up the CL and isolate totally the runner

* feat: add options useReflection to use the ReflectiveTestRunner

* fix: use getMethodName instead of getDisplayName to build the Description

* pom: update version of DSpot

* docs: add description for useReflection option

* ci: update dspot version used in command line

* fix: update .class of TestListener in resources

.travis.yml

@@ -14,7 +14,7 @@ install:
 
 script:
   - mvn clean install
-  - cd dspot && java -jar target/dspot-1.0.2-SNAPSHOT-jar-with-dependencies.jar --example
+  - cd dspot && java -jar target/dspot-1.0.3-SNAPSHOT-jar-with-dependencies.jar --example
 
 after_success:
   - mvn clean test jacoco:report coveralls:report

README.md

@@ -15,7 +15,7 @@ The transformations result in new inputs and new explored paths. They also consi
 ### Command Line Usage
 ```
 Usage: java -jar target/dspot-1.0.0-jar-with-dependencies.jar
-                          [(-p|--path-to-properties) <./path/to/myproject.properties>] [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] [(-i|--iteration) <iteration>] [(-s|--test-criterion) <PitMutantScoreSelector | BranchCoverageTestSelector | JacocoCoverageSelector | TakeAllSelector | ChangeDetectorSelector>] [(-g|--max-test-amplified) <integer>] [-d|--descartes] [-k|--evosuite] [(-t|--test) my.package.MyClassTest1:my.package.MyClassTest2:...:my.package.MyClassTestN ] [(-c|--cases) testCases1:testCases2:...:testCasesN ] [(-o|--output-path) <output>] [(-m|--path-pit-result) <./path/to/mutations.csv>] [(-b|--automatic-builder) <MavenBuilder | GradleBuilder>] [(-j|--maven-home) <path to maven home>] [(-r|--randomSeed) <long integer>] [(-v|--timeOut) <long integer>] [--verbose] [-e|--example] [-h|--help]
+                          [(-p|--path-to-properties) <./path/to/myproject.properties>] [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] [(-i|--iteration) <iteration>] [(-s|--test-criterion) <PitMutantScoreSelector | BranchCoverageTestSelector | JacocoCoverageSelector | TakeAllSelector | ChangeDetectorSelector>] [(-g|--max-test-amplified) <integer>] [-d|--descartes] [-k|--evosuite] [(-t|--test) my.package.MyClassTest1:my.package.MyClassTest2:...:my.package.MyClassTestN ] [(-c|--cases) testCases1:testCases2:...:testCasesN ] [(-o|--output-path) <output>] [(-m|--path-pit-result) <./path/to/mutations.csv>] [(-b|--automatic-builder) <MavenBuilder | GradleBuilder>] [--useReflection] [(-j|--maven-home) <path to maven home>] [(-r|--randomSeed) <long integer>] [(-v|--timeOut) <long integer>] [--verbose] [-e|--example] [-h|--help]
 
   [(-p|--path-to-properties) <./path/to/myproject.properties>]
         [mandatory] specify the path to the configuration file (format Java
@@ -61,6 +61,10 @@ Usage: java -jar target/dspot-1.0.0-jar-with-dependencies.jar
         [optional] specify the automatic builder to build the project (default:
         MavenBuilder)
 
+  [--useReflection]
+        Use a totally isolate test runner. WARNING this test runner does not
+        support the usage of the JacocoCoverageSelector
+
   [(-j|--maven-home) <path to maven home>]
         specify the path to the maven home
 

dspot/pom.xml

@@ -7,7 +7,7 @@
     <parent>
         <groupId>fr.inria.stamp</groupId>
         <artifactId>dspot-parent</artifactId>
-        <version>1.0.2-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
 
     <artifactId>dspot</artifactId>

dspot/src/main/java/fr/inria/stamp/JSAPOptions.java

@@ -10,6 +10,7 @@
 import fr.inria.diversify.dspot.selector.TestSelector;
 import fr.inria.diversify.mutant.pit.GradlePitTaskAndOptions;
 import fr.inria.diversify.mutant.pit.MavenPitCommandAndOptions;
+import fr.inria.stamp.test.runner.TestRunnerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -113,6 +114,8 @@ public static Configuration parse(String[] args) {
 		GradlePitTaskAndOptions.descartesMode = jsapConfig.getBoolean("descartes");
 		GradlePitTaskAndOptions.evosuiteMode = jsapConfig.getBoolean("evosuite");
 
+		TestRunnerFactory.useReflectiveTestRunner = jsapConfig.getBoolean("useReflection");
+
 		return new Configuration(jsapConfig.getString("path"),
 				buildAmplifiersFromString(jsapConfig.getStringArray("amplifiers")),
 				jsapConfig.getInt("iteration"),
@@ -281,6 +284,11 @@ private static JSAP initJSAP() {
 		maxTestAmplified.setHelp("[optional] specify the maximum number of amplified test that dspot keep (before generating assertion)");
 		maxTestAmplified.setDefault("200");
 
+		Switch useReflection = new Switch("useReflection");
+		useReflection.setLongFlag("useReflection");
+		useReflection.setDefault("false");
+		useReflection.setHelp("Use a totally isolate test runner. WARNING this test runner does not support the usage of the JacocoCoverageSelector");
+
 		try {
 			jsap.registerParameter(pathToConfigFile);
 			jsap.registerParameter(amplifiers);
@@ -294,6 +302,7 @@ private static JSAP initJSAP() {
 			jsap.registerParameter(output);
 			jsap.registerParameter(mutantScore);
 			jsap.registerParameter(automaticBuilder);
+			jsap.registerParameter(useReflection);
 			jsap.registerParameter(mavenHome);
 			jsap.registerParameter(seed);
 			jsap.registerParameter(timeOut);

dspot/src/main/java/fr/inria/stamp/test/listener/TestListener.java

@@ -4,8 +4,6 @@
 import org.junit.runner.notification.Failure;
 import org.junit.runner.notification.RunListener;
 
-import java.lang.annotation.Annotation;
-import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -112,7 +110,7 @@ private static Description copyDescriptionFromObject(Object descriptionToBeCopie
                     .getMethod("getTestClass")
                     .invoke(descriptionToBeCopied);
             String displayname = (String) descriptionClass
-                    .getMethod("getDisplayName")
+                    .getMethod("getMethodName")
                     .invoke(descriptionToBeCopied);
             Collection<?> annotations = (Collection<?>) descriptionClass
                     .getMethod("getAnnotations")

dspot/src/main/java/fr/inria/stamp/test/runner/ReflectiveTestRunner.java

@@ -43,7 +43,7 @@ public class ReflectiveTestRunner extends AbstractTestRunner {
                     } catch (MalformedURLException e) {
                         throw new RuntimeException(e);
                     }
-                }).toArray(URL[]::new), ClassLoader.getSystemClassLoader().getParent());
+                }).toArray(URL[]::new), null);
     }
 
     private Object runUsingReflection(Class<?> classTest) {
@@ -79,9 +79,10 @@ public TestListener run(Class<?> testClass, Collection<String> testMethodNames,
             LOGGER.warn("Additional listeners is not supported for ReflectiveTestRunner");
         }
         ExecutorService executor = Executors.newSingleThreadExecutor();
-        final Future<?> submit = executor.submit(() ->
-            runUsingReflection(testClass)
-        );
+        final Future<?> submit = executor.submit(() -> {
+            Thread.currentThread().setContextClassLoader(classLoader);
+            return runUsingReflection(testClass);
+        });
         try {
             Object listener = submit.get(10000000 * (testMethodNames.size() + 1),
                     TimeUnit.MILLISECONDS);

dspot/src/main/java/fr/inria/stamp/test/runner/TestRunnerFactory.java

@@ -27,6 +27,8 @@ public class TestRunnerFactory {
         //STRING_ARRAY_LIST.add("org.mockito");
     }
 
+    public static boolean useReflectiveTestRunner = false;
+
     public static final Predicate<CtType<?>> containsSpecificAnnotation = testClass ->
             !(testClass.getElements(
                     new TypeFilter<CtTypeReference>(CtTypeReference.class) {
@@ -41,15 +43,15 @@ public boolean matches(CtTypeReference element) {
             ).isEmpty());
 
     public static TestRunner createRunner(CtType<?> testClass, String classpath) {
-        if (testClass != null && containsSpecificAnnotation.test(testClass)) {
+        if (useReflectiveTestRunner || testClass != null && containsSpecificAnnotation.test(testClass)) {
             return new ReflectiveTestRunner(classpath);
         } else {
             return new DefaultTestRunner(classpath);
         }
     }
 
     public static TestRunner createRunner(CtType<?> testClass, URLClassLoader classLoader) {
-        if (containsSpecificAnnotation.test(testClass)) {
+        if (useReflectiveTestRunner || containsSpecificAnnotation.test(testClass)) {
             return new ReflectiveTestRunner(classLoader);
         } else {
             return new DefaultTestRunner(classLoader);

dspot/src/test/java/fr/inria/stamp/test/launcher/TestLauncherTest.java

@@ -6,6 +6,7 @@
 import fr.inria.diversify.utils.sosiefier.InputConfiguration;
 import fr.inria.diversify.utils.sosiefier.InputProgram;
 import fr.inria.stamp.test.listener.TestListener;
+import fr.inria.stamp.test.runner.TestRunnerFactory;
 import org.apache.commons.io.FileUtils;
 import org.junit.Test;
 import spoon.Launcher;
@@ -359,4 +360,22 @@ System Properties must be described with the key systemProperties (i.e. systemPr
 		assertEquals(0, run.getFailingTests().size());
 		assertTrue(run.getFailingTests().isEmpty());
 	}
+
+	@Test
+	public void testLauncherOnTestUsingReflectiveTestRunnerOnTestThatUseSystemProperty() throws Exception {
+
+		/*
+			Using the ReflectiveTestRunner
+		 */
+		TestRunnerFactory.useReflectiveTestRunner = true;
+		Utils.init("src/test/resources/sample/sample.properties");
+		final CtClass aClass = Utils.findClass("fr.inria.systemproperties.SystemPropertiesTest");
+		final String classPath = AmplificationHelper.getClassPath(Utils.getCompiler(), Utils.getInputProgram());
+		final TestListener run = TestLauncher.run(Utils.getInputConfiguration(), classPath, aClass);
+		assertEquals(1, run.getPassingTests().size());
+		assertEquals(1, run.getRunningTests().size());
+		assertEquals(0, run.getFailingTests().size());
+		assertTrue(run.getFailingTests().isEmpty());
+		TestRunnerFactory.useReflectiveTestRunner = false;
+	}
 }

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>fr.inria.stamp</groupId>
     <artifactId>dspot-parent</artifactId>
-    <version>1.0.2-SNAPSHOT</version>
+    <version>1.0.3-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>