diff --git a/README.md b/README.md index 8041c8a42..eb7cc0c25 100644 --- a/README.md +++ b/README.md @@ -15,54 +15,70 @@ The transformations result in new inputs and new explored paths. They also consi ### Command Line Usage ``` Usage: java -jar target/dspot-1.0.0-jar-with-dependencies.jar - [(-p|--path) ] [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] [(-i|--iteration) ] [(-s|--selector) ] [(-t|--test) test1:test2:...:testN ] [(-o|--output) ] [(-m|--mutant) ] [-e|--example] [-h|--help] - - [(-p|--path-to-properties) <./path/to/myproject.properties>] - [mandatory] specify the path to the configuration file (format Java - properties) of the target project (e.g. ./foo.properties). - - [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] - [optional] specify the list of amplifiers to use. Default with all - available amplifiers. Possible values: - MethodAdd|MethodRemove|StatementAdderOnAssert|TestDataMutator - - [(-i|--iteration) ] - [optional] specify the number of amplification iteration. A larger - number may help to improve the test criterion (eg a larger number of - iterations mah help to kill more mutants). This has an impact on the - execution time: the more iterations, the longer DSpot runs. (default: 3) - - [(-s|--test-criterion) ] - [optional] specify the test adequacy criterion to be maximized with - amplification (default: PitMutantScoreSelector) - - [(-t|--test) my.package.MyClassTest1:my.package.MyClassTest2:...:my.package.MyClassTestN ] - [optional] fully qualified names of test classes to be amplified. If the - value is all, DSpot will amplify the whole test suite. You can also use - regex to describe a set of test classes. (default: all) - - [(-o|--output-path) ] - [optional] specify the output folder (default: dspot-report) - - [(-m|--path-pit-result) <./path/to/mutations.csv>] - [optional, expert mode] specify the path to the .csv of the original - result of Pit Test. If you use this option the selector will be forced - to PitMutantScoreSelector - - [(-r|--randomSeed) ] - specify a seed for the random object (used for all randomized operation) (default: 23) - - [(-v|--timeOut) ] - specify the timeout value of the degenerated tests in millisecond - (default: 10000) - - [--verbose] - - [-e|--example] - run the example of DSpot and leave - - [-h|--help] - shows this help + [(-p|--path-to-properties) <./path/to/myproject.properties>] [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] [(-i|--iteration) ] [(-s|--test-criterion) ] [(-g|--max-test-amplified) ] [-d|--descartes] [-k|--evosuite] [(-t|--test) my.package.MyClassTest1:my.package.MyClassTest2:...:my.package.MyClassTestN ] [(-c|--cases) testCases1:testCases2:...:testCasesN ] [(-o|--output-path) ] [(-m|--path-pit-result) <./path/to/mutations.csv>] [(-b|--automatic-builder) ] [(-j|--maven-home) ] [(-r|--randomSeed) ] [(-v|--timeOut) ] [--verbose] [-e|--example] [-h|--help] + + [(-p|--path-to-properties) <./path/to/myproject.properties>] + [mandatory] specify the path to the configuration file (format Java + properties) of the target project (e.g. ./foo.properties). + + [(-a|--amplifiers) Amplifier1:Amplifier2:...:AmplifierN ] + [optional] specify the list of amplifiers to use. Default with all + available amplifiers. Possible values: NumberLiteralAmplifier | + MethodAdd | MethodRemove | TestDataMutator | StatementAdd | None + (default: None) + + [(-i|--iteration) ] + [optional] specify the number of amplification iteration. A larger + number may help to improve the test criterion (eg a larger number of + iterations mah help to kill more mutants). This has an impact on the + execution time: the more iterations, the longer DSpot runs. (default: 3) + + [(-s|--test-criterion) ] + [optional] specify the test adequacy criterion to be maximized with + amplification (default: PitMutantScoreSelector) + + [(-g|--max-test-amplified) ] + [optional] specify the maximum number of amplified test that dspot keep + (before generating assertion) (default: 200) + + [(-t|--test) my.package.MyClassTest1:my.package.MyClassTest2:...:my.package.MyClassTestN ] + [optional] fully qualified names of test classes to be amplified. If the + value is all, DSpot will amplify the whole test suite. You can also use + regex to describe a set of test classes. (default: all) + + [(-c|--cases) testCases1:testCases2:...:testCasesN ] + specify the test cases to amplify + + [(-o|--output-path) ] + [optional] specify the output folder (default: dspot-report) + + [(-m|--path-pit-result) <./path/to/mutations.csv>] + [optional, expert mode] specify the path to the .csv of the original + result of Pit Test. If you use this option the selector will be forced + to PitMutantScoreSelector + + [(-b|--automatic-builder) ] + [optional] specify the automatic builder to build the project (default: + MavenBuilder) + + [(-j|--maven-home) ] + specify the path to the maven home + + [(-r|--randomSeed) ] + specify a seed for the random object (used for all randomized operation) + (default: 23) + + [(-v|--timeOut) ] + specify the timeout value of the degenerated tests in millisecond + (default: 10000) + + [--verbose] + + [-e|--example] + run the example of DSpot and leave + + [-h|--help] + shows this help ``` ### Output of DSpot @@ -90,13 +106,13 @@ myproject/ ``` Let's imagine you wish to run DSpot on `module1`. You'd need to create a properties file (e.g. `dspot.properties`), -that you can located under under `module1` and containing (for example): +that you can located under `module1` and containing (for example): ```properties # Relative path to the project root. project=.. # Path to the current module where we want to execute DSpot, relative to the project root -targetModule=mymodule/ +targetModule=module1/ # Relative path to the source project from this properties file src=src/main/java/ # Relative path to the test source project from this properties file @@ -219,6 +235,29 @@ There is 4 new unique path Print TestSuiteExampleAmpl with 6 amplified test cases in dspot-out/ ``` +###### Available Properties + +Here is the list of configuration properties of DSpot: + +* required properties: + * project: path to the project root directory. + * src: relative path (from project properties) to the source root directory. + * testSrc: relative path (from project properties) to the test source root directory. +* recommended properties: + * outputDirectory: path to the out of dspot. (default: output) + * javaVersion: version used of java (default: 5) + * maven.home: path to the executable maven. If no value is specified, it will try some defaults values + (for instance: `/usr/share/maven/`, `usr/local/Cellar/maven/3.3.9/libexec/` ...). +* optional properties: + * filter: string to filter on package or classes. + * maven.localRepository: path to the local repository of maven (.m2), if you need specific settings. + * excludedClasses: dspot will not amplify the excluded test classes. + * additionalClasspathElements: add elements to the classpath. (e.g. a jar file) + * excludedClasses: list of full qualified name of test classes to be excluded + by DSpot (see this [property file](https://github.com/STAMP-project/dspot/blob/master/dspot/src/test/resources/sample/sample.properties)) + * excludedTestCases: list of test name method to be excluded + by DSpot (see this [property file](https://github.com/STAMP-project/dspot/blob/master/dspot/src/test/resources/sample/sample.properties)) + ### API The whole procedure of amplification is done by the `fr.inria.diversify.dspot.DSpot` class. @@ -229,15 +268,7 @@ You can specify which amplifiers (as a list) you want to use. By default, DSpot * TestDataMutator: which transforms literals. * TestMethodCallAdder: which duplicatse an existing method call in the test case. * TestMethodCallRemover: which removes a method call in the test case. - * StatementAdderOnAssert: which adds calls to accessible methods on existing objects and creates new instances. - -The amplifier `StatementAdd` (unstable): which reuses existing objects and return values to add method calls to -accessible methods. Need the initialized InputProgram if DSpot to be well constructed. - -##### Amplifiers - -You can implement you own amplifier by implementing the `fr.inria.diversify.dspot.amplifier.Amplifier` interface and -giving it to DSpot. + * StatementAdd: which adds calls to accessible methods on existing objects and creates new instances. #### Test Selectors @@ -257,46 +288,6 @@ mutant killed, it gives: * the name of the method where the mutant is inserted. * the line where the mutant is inserted. -#### Snippets - -The simplest snippets is: -```java -InputConfiguration configuration = new InputConfiguration(); -DSpot dspot = new DSpot(configuration); -List amplifiedTestClass = dspot.amplifiyAllTests(); -``` -Which will run the default amplifiers, 3 times on the whole existing test suite, with the BranchCoverageTestSelector. - -To customize your DSpot, you can use several constructors: - -```java -new DSpot(configuration); -new DSpot(configuration, 1); //1 iteration -new DSpot(configuration, 1, Collections.singletonList(new TestDataMutator)); //1 iteration, one specific amplifier -DSpot dspot = new DSpot(configuration, 2, Arrays.asList(new TestDataMutator, new StatementAdderOnAssert())); //1 iteration, two specified amplifiers -dspot.addAmplifier(new StatementAdd(dspot.getInputProgram())); // or add an amplifiers after the construction. -new DSpot(configuration, new PitMutantScoreSelector());//3 iterations, default amplifier, PitMutantScoreSelector -``` - -###### Available Properties - -Here is the list of configuration properties of DSpot: - -* required properties: - * project: path to the project root directory. - * src: relative path (from project properties) to the source root directory. - * testSrc: relative path (from project properties) to the test source root directory. -* recommended properties: - * outputDirectory: path to the out of dspot. (default: output) - * javaVersion: version used of java (default: 5) - * maven.home: path to the executable maven. If no value is specified, it will try some defaults values - (for instance: `/usr/share/maven/`, `usr/local/Cellar/maven/3.3.9/libexec/` ...). -* optional properties: - * filter: string to filter on package or classes. - * maven.localRepository: path to the local repository of maven (.m2), if you need specific settings. - * excludedClasses: dspot will not amplify the excluded test classes. - * additionalClasspathElements: add elements to the classpath. (e.g. a jar file) - ### Licence DSpot is published under LGPL-3.0 (see [Licence.md](https://github.com/STAMP-project/dspot/blob/master/Licence.md) for diff --git a/dspot/src/main/java/fr/inria/diversify/dspot/Amplification.java b/dspot/src/main/java/fr/inria/diversify/dspot/Amplification.java index 568666f62..a8f1f89ab 100644 --- a/dspot/src/main/java/fr/inria/diversify/dspot/Amplification.java +++ b/dspot/src/main/java/fr/inria/diversify/dspot/Amplification.java @@ -181,10 +181,12 @@ private List> preAmplification(CtType classTest, List> t LOGGER.info("Try to add assertions before amplification"); final List> amplifiedTestToBeKept = assertGenerator.generateAsserts( classTest, testSelector.selectToAmplify(tests)); - result = compileAndRunTests(classTest, amplifiedTestToBeKept); - if (result == null) { - LOGGER.warn("Need a green test suite to run dspot"); - return Collections.emptyList(); + if (!amplifiedTestToBeKept.isEmpty()) { + result = compileAndRunTests(classTest, amplifiedTestToBeKept); + if (result == null) { + LOGGER.warn("Need a green test suite to run dspot"); + return Collections.emptyList(); + } } testSelector.selectToKeep(amplifiedTestToBeKept); return testSelector.getAmplifiedTestCases(); diff --git a/dspot/src/main/java/fr/inria/diversify/dspot/DSpot.java b/dspot/src/main/java/fr/inria/diversify/dspot/DSpot.java index 7980a29ea..e3c9e6d82 100644 --- a/dspot/src/main/java/fr/inria/diversify/dspot/DSpot.java +++ b/dspot/src/main/java/fr/inria/diversify/dspot/DSpot.java @@ -182,8 +182,9 @@ public CtType amplifyTest(CtType test, List> methods) { try { Counter.reset(); Amplification testAmplification = new Amplification(this.inputConfiguration, this.amplifiers, this.testSelector, this.compiler); + final List> filteredTestCases = this.filterTestCases(methods); long time = System.currentTimeMillis(); - testAmplification.amplification(test, methods, numberOfIterations); + testAmplification.amplification(test, filteredTestCases, numberOfIterations); final long elapsedTime = System.currentTimeMillis() - time; LOGGER.info("elapsedTime {}", elapsedTime); this.projectTimeJSON.add(new ClassTimeJSON(test.getQualifiedName(), elapsedTime)); @@ -203,6 +204,19 @@ public CtType amplifyTest(CtType test, List> methods) { } } + protected List> filterTestCases(List> testMethods) { + if (this.inputConfiguration.getProperty("excludedTestCases") == null) { + return testMethods; + } else { + final List excludedTestCases = Arrays.stream(this.inputConfiguration.getProperty("excludedTestCases").split(",")).collect(Collectors.toList()); + return testMethods.stream() + .filter(ctMethod -> + excludedTestCases.isEmpty() || + !excludedTestCases.contains(ctMethod.getSimpleName()) + ).collect(Collectors.toList()); + } + } + public InputProgram getInputProgram() { return inputProgram; } diff --git a/dspot/src/main/java/fr/inria/diversify/dspot/assertGenerator/AssertGenerator.java b/dspot/src/main/java/fr/inria/diversify/dspot/assertGenerator/AssertGenerator.java index a38486530..8e2b93a84 100644 --- a/dspot/src/main/java/fr/inria/diversify/dspot/assertGenerator/AssertGenerator.java +++ b/dspot/src/main/java/fr/inria/diversify/dspot/assertGenerator/AssertGenerator.java @@ -51,7 +51,11 @@ public List> generateAsserts(CtType testClass, List> MethodsAssertGenerator ags = new MethodsAssertGenerator(testClass, this.configuration, compiler); final List> amplifiedTestWithAssertion = ags.generateAsserts(cloneClass, testWithoutAssertions); - LOGGER.info("{} new tests with assertions generated", amplifiedTestWithAssertion.size()); + if (amplifiedTestWithAssertion.isEmpty()) { + LOGGER.info("Could not generate any test with assertions"); + } else { + LOGGER.info("{} new tests with assertions generated", amplifiedTestWithAssertion.size()); + } return amplifiedTestWithAssertion; } } diff --git a/dspot/src/test/java/fr/inria/diversify/dspot/DSpotTest.java b/dspot/src/test/java/fr/inria/diversify/dspot/DSpotTest.java index f063f3917..39f81bd17 100644 --- a/dspot/src/test/java/fr/inria/diversify/dspot/DSpotTest.java +++ b/dspot/src/test/java/fr/inria/diversify/dspot/DSpotTest.java @@ -2,9 +2,14 @@ import edu.emory.mathcs.backport.java.util.Collections; import fr.inria.diversify.Utils; +import fr.inria.diversify.dspot.amplifier.Amplifier; import fr.inria.diversify.dspot.amplifier.TestMethodCallAdder; import fr.inria.diversify.dspot.selector.JacocoCoverageSelector; +import fr.inria.diversify.dspot.selector.TestSelector; +import fr.inria.diversify.utils.DSpotUtils; +import fr.inria.diversify.utils.sosiefier.InputConfiguration; import org.junit.Test; +import spoon.reflect.declaration.CtMethod; import spoon.reflect.declaration.CtType; import java.util.List; @@ -20,12 +25,41 @@ public class DSpotTest extends AbstractTest { @Test public void testExcludedClassesInPropertyFile() throws Exception { - final DSpot dSpot = new DSpot(Utils.getInputConfiguration(), + + /* + Usage of properties: + - excludedClasses: list of full qualified name of test classes to be excluded (separated by comma ',') + - excludedTestCases: list of name of test cases (methods) to be excluded (separated by comma ',') + */ + + final MockDSpot dSpot = new MockDSpot(Utils.getInputConfiguration(), 1, Collections.singletonList(new TestMethodCallAdder()), new JacocoCoverageSelector() ); + // the test class fr.inria.filter.passing.PassingTest has 2 method, but only one is amplified + assertEquals(2, Utils.findClass("fr.inria.filter.passing.PassingTest").getMethods().size()); + // the test class fr.inria.filter.failing.FailingTest match the regex, but it is excluded in the properties final List ctTypes = dSpot.amplifyTest("fr.inria.filter.*"); assertEquals(1, ctTypes.size()); + // uses the mock to retrieve the number of method to be amplified + assertEquals(1, dSpot.numberOfMethod); + } + + private class MockDSpot extends DSpot { + + public int numberOfMethod = 0; + + public MockDSpot(InputConfiguration inputConfiguration, int numberOfIterations, List amplifiers, TestSelector testSelector) throws Exception { + super(inputConfiguration, numberOfIterations, amplifiers, testSelector); + } + + @Override + protected List> filterTestCases(List> testMethods) { + List> filteredMethods = super.filterTestCases(testMethods); + numberOfMethod = filteredMethods.size(); + return filteredMethods; + } } + } diff --git a/dspot/src/test/resources/sample/sample.properties b/dspot/src/test/resources/sample/sample.properties index 7f389af25..3ed7e44d1 100644 --- a/dspot/src/test/resources/sample/sample.properties +++ b/dspot/src/test/resources/sample/sample.properties @@ -5,4 +5,5 @@ testResources=src/test/resources outputDirectory=target/trash/ filter=fr.inria.sample.* systemProperties=admin=toto,passwd=tata -excludedClasses=fr.inria.filter.failing.* \ No newline at end of file +excludedClasses=fr.inria.filter.failing.* +excludedTestCases=failingTestCase \ No newline at end of file diff --git a/dspot/src/test/resources/sample/src/test/java/fr/inria/filter/passing/PassingTest.java b/dspot/src/test/resources/sample/src/test/java/fr/inria/filter/passing/PassingTest.java index 907cc1e74..1fcb6a498 100644 --- a/dspot/src/test/resources/sample/src/test/java/fr/inria/filter/passing/PassingTest.java +++ b/dspot/src/test/resources/sample/src/test/java/fr/inria/filter/passing/PassingTest.java @@ -15,4 +15,9 @@ public class PassingTest { public void testAssertion() throws Exception { assertFalse(false); } + + @Test + public void failingTestCase() throws Exception { + assertFalse(true); + } }