From f4a9cdcfeeb531045dedeee1b26a90af080d0a36 Mon Sep 17 00:00:00 2001 From: Benjamin DANGLOT Date: Mon, 11 Dec 2017 23:13:47 +0100 Subject: [PATCH] Fix #271 isolate test runner (#272) * test: add test using ReflectiveTestRunner * feat: set up the CL and isolate totally the runner * feat: add options useReflection to use the ReflectiveTestRunner * fix: use getMethodName instead of getDisplayName to build the Description * pom: update version of DSpot * docs: add description for useReflection option * ci: update dspot version used in command line * fix: update .class of TestListener in resources --- .travis.yml | 2 +- README.md | 6 +++++- dspot/pom.xml | 2 +- .../main/java/fr/inria/stamp/JSAPOptions.java | 9 +++++++++ .../stamp/test/listener/TestListener.java | 4 +--- .../test/runner/ReflectiveTestRunner.java | 9 +++++---- .../stamp/test/runner/TestRunnerFactory.java | 6 ++++-- .../resources/listener/TestListener.class | Bin 8571 -> 8554 bytes .../stamp/test/launcher/TestLauncherTest.java | 19 ++++++++++++++++++ pom.xml | 2 +- 10 files changed, 46 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index af20e9bf6..0b9662c98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ install: script: - mvn clean install - - cd dspot && java -jar target/dspot-1.0.2-SNAPSHOT-jar-with-dependencies.jar --example + - cd dspot && java -jar target/dspot-1.0.3-SNAPSHOT-jar-with-dependencies.jar --example after_success: - mvn clean test jacoco:report coveralls:report diff --git a/README.md b/README.md index eb7cc0c25..b18f5bbc2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The transformations result in new inputs and new explored paths. They also consi ### Command Line Usage ``` Usage: java -jar target/dspot-1.0.0-jar-with-dependencies.jar - [(-p|--path-to-properties) <./path/to/myproject.properties>] [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] [(-i|--iteration) ] [(-s|--test-criterion) ] [(-g|--max-test-amplified) ] [-d|--descartes] [-k|--evosuite] [(-t|--test) my.package.MyClassTest1:my.package.MyClassTest2:...:my.package.MyClassTestN ] [(-c|--cases) testCases1:testCases2:...:testCasesN ] [(-o|--output-path) ] [(-m|--path-pit-result) <./path/to/mutations.csv>] [(-b|--automatic-builder) ] [(-j|--maven-home) ] [(-r|--randomSeed) ] [(-v|--timeOut) ] [--verbose] [-e|--example] [-h|--help] + [(-p|--path-to-properties) <./path/to/myproject.properties>] [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] [(-i|--iteration) ] [(-s|--test-criterion) ] [(-g|--max-test-amplified) ] [-d|--descartes] [-k|--evosuite] [(-t|--test) my.package.MyClassTest1:my.package.MyClassTest2:...:my.package.MyClassTestN ] [(-c|--cases) testCases1:testCases2:...:testCasesN ] [(-o|--output-path) ] [(-m|--path-pit-result) <./path/to/mutations.csv>] [(-b|--automatic-builder) ] [--useReflection] [(-j|--maven-home) ] [(-r|--randomSeed) ] [(-v|--timeOut) ] [--verbose] [-e|--example] [-h|--help] [(-p|--path-to-properties) <./path/to/myproject.properties>] [mandatory] specify the path to the configuration file (format Java @@ -61,6 +61,10 @@ Usage: java -jar target/dspot-1.0.0-jar-with-dependencies.jar [optional] specify the automatic builder to build the project (default: MavenBuilder) + [--useReflection] + Use a totally isolate test runner. WARNING this test runner does not + support the usage of the JacocoCoverageSelector + [(-j|--maven-home) ] specify the path to the maven home diff --git a/dspot/pom.xml b/dspot/pom.xml index fe8cdb098..6ed5052f6 100644 --- a/dspot/pom.xml +++ b/dspot/pom.xml @@ -7,7 +7,7 @@ fr.inria.stamp dspot-parent - 1.0.2-SNAPSHOT + 1.0.3-SNAPSHOT dspot diff --git a/dspot/src/main/java/fr/inria/stamp/JSAPOptions.java b/dspot/src/main/java/fr/inria/stamp/JSAPOptions.java index f4c1ee08a..8b85e6163 100644 --- a/dspot/src/main/java/fr/inria/stamp/JSAPOptions.java +++ b/dspot/src/main/java/fr/inria/stamp/JSAPOptions.java @@ -10,6 +10,7 @@ import fr.inria.diversify.dspot.selector.TestSelector; import fr.inria.diversify.mutant.pit.GradlePitTaskAndOptions; import fr.inria.diversify.mutant.pit.MavenPitCommandAndOptions; +import fr.inria.stamp.test.runner.TestRunnerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,6 +114,8 @@ public static Configuration parse(String[] args) { GradlePitTaskAndOptions.descartesMode = jsapConfig.getBoolean("descartes"); GradlePitTaskAndOptions.evosuiteMode = jsapConfig.getBoolean("evosuite"); + TestRunnerFactory.useReflectiveTestRunner = jsapConfig.getBoolean("useReflection"); + return new Configuration(jsapConfig.getString("path"), buildAmplifiersFromString(jsapConfig.getStringArray("amplifiers")), jsapConfig.getInt("iteration"), @@ -281,6 +284,11 @@ private static JSAP initJSAP() { maxTestAmplified.setHelp("[optional] specify the maximum number of amplified test that dspot keep (before generating assertion)"); maxTestAmplified.setDefault("200"); + Switch useReflection = new Switch("useReflection"); + useReflection.setLongFlag("useReflection"); + useReflection.setDefault("false"); + useReflection.setHelp("Use a totally isolate test runner. WARNING this test runner does not support the usage of the JacocoCoverageSelector"); + try { jsap.registerParameter(pathToConfigFile); jsap.registerParameter(amplifiers); @@ -294,6 +302,7 @@ private static JSAP initJSAP() { jsap.registerParameter(output); jsap.registerParameter(mutantScore); jsap.registerParameter(automaticBuilder); + jsap.registerParameter(useReflection); jsap.registerParameter(mavenHome); jsap.registerParameter(seed); jsap.registerParameter(timeOut); diff --git a/dspot/src/main/java/fr/inria/stamp/test/listener/TestListener.java b/dspot/src/main/java/fr/inria/stamp/test/listener/TestListener.java index daf490322..e368983f5 100644 --- a/dspot/src/main/java/fr/inria/stamp/test/listener/TestListener.java +++ b/dspot/src/main/java/fr/inria/stamp/test/listener/TestListener.java @@ -4,8 +4,6 @@ import org.junit.runner.notification.Failure; import org.junit.runner.notification.RunListener; -import java.lang.annotation.Annotation; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -112,7 +110,7 @@ private static Description copyDescriptionFromObject(Object descriptionToBeCopie .getMethod("getTestClass") .invoke(descriptionToBeCopied); String displayname = (String) descriptionClass - .getMethod("getDisplayName") + .getMethod("getMethodName") .invoke(descriptionToBeCopied); Collection annotations = (Collection) descriptionClass .getMethod("getAnnotations") diff --git a/dspot/src/main/java/fr/inria/stamp/test/runner/ReflectiveTestRunner.java b/dspot/src/main/java/fr/inria/stamp/test/runner/ReflectiveTestRunner.java index d3992c77e..8b761f8ed 100644 --- a/dspot/src/main/java/fr/inria/stamp/test/runner/ReflectiveTestRunner.java +++ b/dspot/src/main/java/fr/inria/stamp/test/runner/ReflectiveTestRunner.java @@ -43,7 +43,7 @@ public class ReflectiveTestRunner extends AbstractTestRunner { } catch (MalformedURLException e) { throw new RuntimeException(e); } - }).toArray(URL[]::new), ClassLoader.getSystemClassLoader().getParent()); + }).toArray(URL[]::new), null); } private Object runUsingReflection(Class classTest) { @@ -79,9 +79,10 @@ public TestListener run(Class testClass, Collection testMethodNames, LOGGER.warn("Additional listeners is not supported for ReflectiveTestRunner"); } ExecutorService executor = Executors.newSingleThreadExecutor(); - final Future submit = executor.submit(() -> - runUsingReflection(testClass) - ); + final Future submit = executor.submit(() -> { + Thread.currentThread().setContextClassLoader(classLoader); + return runUsingReflection(testClass); + }); try { Object listener = submit.get(10000000 * (testMethodNames.size() + 1), TimeUnit.MILLISECONDS); diff --git a/dspot/src/main/java/fr/inria/stamp/test/runner/TestRunnerFactory.java b/dspot/src/main/java/fr/inria/stamp/test/runner/TestRunnerFactory.java index 6984cd6d0..93c49d7ae 100644 --- a/dspot/src/main/java/fr/inria/stamp/test/runner/TestRunnerFactory.java +++ b/dspot/src/main/java/fr/inria/stamp/test/runner/TestRunnerFactory.java @@ -27,6 +27,8 @@ public class TestRunnerFactory { //STRING_ARRAY_LIST.add("org.mockito"); } + public static boolean useReflectiveTestRunner = false; + public static final Predicate> containsSpecificAnnotation = testClass -> !(testClass.getElements( new TypeFilter(CtTypeReference.class) { @@ -41,7 +43,7 @@ public boolean matches(CtTypeReference element) { ).isEmpty()); public static TestRunner createRunner(CtType testClass, String classpath) { - if (testClass != null && containsSpecificAnnotation.test(testClass)) { + if (useReflectiveTestRunner || testClass != null && containsSpecificAnnotation.test(testClass)) { return new ReflectiveTestRunner(classpath); } else { return new DefaultTestRunner(classpath); @@ -49,7 +51,7 @@ public static TestRunner createRunner(CtType testClass, String classpath) { } public static TestRunner createRunner(CtType testClass, URLClassLoader classLoader) { - if (containsSpecificAnnotation.test(testClass)) { + if (useReflectiveTestRunner || containsSpecificAnnotation.test(testClass)) { return new ReflectiveTestRunner(classLoader); } else { return new DefaultTestRunner(classLoader); diff --git a/dspot/src/main/resources/listener/TestListener.class b/dspot/src/main/resources/listener/TestListener.class index 5b11baef2facf21e2b67473533301d0369bdefa5..3c8e5d6d12a8fc62ee71dbfe94fa9a2bf4afe6cb 100644 GIT binary patch delta 690 zcmW-fT}YF06vm%Zu4uilV{_kgdtX1MZcB4&mM&w>2&t@$l|g=#xzaFAvj_?#ioOPwQ{L4VaGJN6mnxuArECLG z;OE&4FK@EGHJg!!447br8#ef1=R8}r!HwncaHp-_po!8&Irx(;YRHw$zmQeg%MAg^ zf|3O|WzUxtg)T3DwI{=2@hm|)O2xSh8df0(*o_Dpu@-Hp;1S2T zIVuzDk%<_Bs1|MmH#j#^jkvn_g)>B378JQcv|UuQIWAZ4l)LNU5-$yL9&+^u8eu>a z6f{d`FC3yh*oSPii0giY(JEULvh@JR-5D8eh4Y(pJui_A{iP-@9 zz%qG5s{Img;XZET5$@m#?&2Bl;WY;F219r!z4sW#7a@Kih2I##Up%0t7$wDmhvdMp z{F5=#Fitui^DFNJjdG8#od)@X@Axv7S6xQ|nD=Pyl$2VCe`!Uui#5HDBHXPvP$kdk TCF%L!aA&h9zBHHAlD5$ncbOkpb_uO;tx#z<7z2t9+;qAW{uK?sz z@R>nXVohz4-m;FK?gQat98{)lh>Yb!Ikh*9uz8sySp$Ga!@FmmcrZlR?ljeEi#_t|1s@SyiOBy{ol4d@#bp z=1`1Jp-jr*_vR+b<6O%(U4W3rvlcfOB+SKIpo0~0uptq4q`|>fYpLFaSh%^#TCY!* zDn%+g&s&T2>4Ln=D29W!JiT8!S<>~gyKqirCtPJHuW!g^S+0Um5bDmEb- z1qfm@wxAGIC_)25XcE+pLLRkG#h1uLDO@N+Cbo*coGTsWR3WZT{^ii|Q-_zTmyw@p zrD~3^ll1jSzz#UEQ#RG{pmW&Q3_V()AS}#oxvT|A*nPy%M7Sqe^0jTi)N9C$3 t$Uu}UJRLM7JwJc(FO=i`Ga*{89@Vv*phkK;)-f%Hl<>^FQ<_qQL+F diff --git a/dspot/src/test/java/fr/inria/stamp/test/launcher/TestLauncherTest.java b/dspot/src/test/java/fr/inria/stamp/test/launcher/TestLauncherTest.java index ab683770f..71606f8ed 100644 --- a/dspot/src/test/java/fr/inria/stamp/test/launcher/TestLauncherTest.java +++ b/dspot/src/test/java/fr/inria/stamp/test/launcher/TestLauncherTest.java @@ -6,6 +6,7 @@ import fr.inria.diversify.utils.sosiefier.InputConfiguration; import fr.inria.diversify.utils.sosiefier.InputProgram; import fr.inria.stamp.test.listener.TestListener; +import fr.inria.stamp.test.runner.TestRunnerFactory; import org.apache.commons.io.FileUtils; import org.junit.Test; import spoon.Launcher; @@ -359,4 +360,22 @@ System Properties must be described with the key systemProperties (i.e. systemPr assertEquals(0, run.getFailingTests().size()); assertTrue(run.getFailingTests().isEmpty()); } + + @Test + public void testLauncherOnTestUsingReflectiveTestRunnerOnTestThatUseSystemProperty() throws Exception { + + /* + Using the ReflectiveTestRunner + */ + TestRunnerFactory.useReflectiveTestRunner = true; + Utils.init("src/test/resources/sample/sample.properties"); + final CtClass aClass = Utils.findClass("fr.inria.systemproperties.SystemPropertiesTest"); + final String classPath = AmplificationHelper.getClassPath(Utils.getCompiler(), Utils.getInputProgram()); + final TestListener run = TestLauncher.run(Utils.getInputConfiguration(), classPath, aClass); + assertEquals(1, run.getPassingTests().size()); + assertEquals(1, run.getRunningTests().size()); + assertEquals(0, run.getFailingTests().size()); + assertTrue(run.getFailingTests().isEmpty()); + TestRunnerFactory.useReflectiveTestRunner = false; + } } diff --git a/pom.xml b/pom.xml index 1bc58c475..f189d1e19 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ fr.inria.stamp dspot-parent - 1.0.2-SNAPSHOT + 1.0.3-SNAPSHOT pom