Merge pull request easy-software-ufal#12 from jpedroh/feature/gh_5

[Test Suite Generation] - Add deterministic generation for Evosuite.

docs/configuration.md

@@ -24,13 +24,16 @@ The following properties are related to the Test Suites Generation step.
 This property consists of an array with the name of the Generators which will be used in SMAT during Test Suite Generation step. If not set, all the implemented generators will be used. Valid values are: `randoop`, `randoop-modified`, `evosuite` and `evosuite-differential`.
 
 ### test_suite_generation_search_budget
-This property allows to customize the time **in seconds** provided for each generator during Test Suite Generation. Default value is 300 seconds. Please note that this option will be ignored when using deterministic Test Suites.
+This property allows to customize the time **in seconds** provided for each generator during Test Suite Generation. Default value is 300 seconds. Please note that this option will be ignored when using deterministic Test Suites. Be aware that when using deterministic test suites generation, the budget time is ignored and other stopping criterias are applied, since different computing power can influence in the time required to generate the same test suite.
 
 ### generate_deterministic_test_suites
 If set to true, SMAT will use deterministic versions of its generators, i.e., the generated suites will always be the same regardless of how many times the code is executed.
 
+### test_suite_generation_seed
+When `generate_deterministic_test_suites` is set to true, the user customizes the seed that is provided for the invoked test generation tools. Default value is `42`.
+
 ## Output Generation
 The following properties are related to the Output Generation step.
 
 ### output_generators
-This property consists of an array with the name of the Reports that are going to be written during the Output Generation step. If not set, all the implemented generators will be used. Valid values are: `behavior_changes`, `semantic_conflicts`, `test_suites`.
+This property consists of an array with the name of the Reports that are going to be written during the Output Generation step. If not set, all the implemented generators will be used. Valid values are: `behavior_changes`, `semantic_conflicts`, `test_suites`.

nimrod/test_suite_generation/generators/evosuite_differential_test_suite_generator.py

@@ -1,10 +1,8 @@
 import logging
-import os
-from typing import Dict, List
 from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis
 
 from nimrod.test_suite_generation.generators.evosuite_test_suite_generator import EvosuiteTestSuiteGenerator
-from nimrod.tools.bin import EVOSUITE, EVOSUITE_RUNTIME
+from nimrod.tools.bin import EVOSUITE
 
 
 class EvosuiteDifferentialTestSuiteGenerator(EvosuiteTestSuiteGenerator):
@@ -20,9 +18,16 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s
               '-projectCP', input_jar,
               f'-Dregressioncp={scenario.scenario_jars.base}',
               '-class', class_name,
-              f'-Dsearch_budget={self.SEARCH_BUDGET}',
               '-DOUTPUT_DIR=' + output_path,
+              '-Dminimize=false',
+              '-Djunit_check=false',
+              '-Dinline=false',
           ]
+
+          if use_determinism:
+            params += ["-Dstopping_condition=MaxStatements", f"-seed={self.SEED}"]
+          else:
+            params += [f'-Dsearch_budget={self.SEARCH_BUDGET}']
 
           if(len(methods) > 0):
             params.append(

nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from typing import Dict, List
+from typing import List
 from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis
 
 from nimrod.test_suite_generation.generators.test_suite_generator import \
@@ -10,8 +10,6 @@
 
 
 class EvosuiteTestSuiteGenerator(TestSuiteGenerator):
-    SEARCH_BUDGET = int(get_config().get('test_suite_generation_search_budget', 300))
-
     def get_generator_tool_name(self) -> str:
         return "EVOSUITE"
 
@@ -22,18 +20,20 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s
               '-jar', EVOSUITE,
               '-projectCP', input_jar,
               '-class', class_name,
-              '-Dtimeout', '5',
               '-Dassertion_strategy=all',
               '-Dp_reflection_on_private=0',
               '-Dreflection_start_percent=0',
               '-Dp_functional_mocking=0',
               '-Dfunctional_mocking_percent=0',
               '-Dminimize=false',
-              f'-Dsearch_budget={self.SEARCH_BUDGET}',
               '-Djunit_check=false',
               '-Dinline=false',
-              '-DOUTPUT_DIR=' + output_path,
           ]
+
+          if use_determinism:
+            params += ["-Dstopping_condition=MaxStatements", f"-seed={self.SEED}"]
+          else:
+            params += [f'-Dsearch_budget={self.SEARCH_BUDGET}']
 
           if(len(methods) > 0):
             params.append(

nimrod/test_suite_generation/generators/randoop_test_suite_generator.py

@@ -4,14 +4,11 @@
 
 from nimrod.test_suite_generation.generators.test_suite_generator import \
     TestSuiteGenerator
-from nimrod.tests.utils import get_config
 from nimrod.tools.java import Java
 from nimrod.utils import generate_classpath
 
 
 class RandoopTestSuiteGenerator(TestSuiteGenerator):
-    SEARCH_BUDGET = int(get_config().get('test_suite_generation_search_budget', 300))
-
     TARGET_METHODS_LIST_FILENAME = 'methods_to_test.txt'
     TARGET_CLASS_LIST_FILENAME = 'classes_to_test.txt'
 
@@ -34,7 +31,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s
         ]
 
         if use_determinism:
-            params += ["--randomseed=10",
+            params += [f"--randomseed={self.SEED}",
                        "--deterministic", "--time-limit=0", "--attempted-limit=4000"]
         else:
             params += [f"--time-limit={int(self.SEARCH_BUDGET)}"]

nimrod/test_suite_generation/generators/test_suite_generator.py

@@ -2,8 +2,9 @@
 import logging
 from os import makedirs, path
 from time import time
-from typing import Dict, List
+from typing import List
 
+from nimrod.tests.utils import get_config
 from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis
 from nimrod.tests.utils import get_base_output_path
 from nimrod.test_suite_generation.test_suite import TestSuite
@@ -14,6 +15,9 @@
 
 
 class TestSuiteGenerator(ABC):
+    SEARCH_BUDGET = int(get_config().get('test_suite_generation_search_budget', 300))
+    SEED = int(get_config().get('test_suite_generation_seed', 42))
+
     def __init__(self, java: Java) -> None:
         self._java = java