From 6815294d7c7c4e155c798632a35207a3af37b3cd Mon Sep 17 00:00:00 2001 From: Joao Pedro Henrique Date: Tue, 25 Oct 2022 15:56:42 -0300 Subject: [PATCH 1/5] feat: add deterministic test suite generation for Evosuite and Evosuite Diff --- .../evosuite_differential_test_suite_generator.py | 13 +++++++++---- .../generators/evosuite_test_suite_generator.py | 12 ++++++------ .../generators/randoop_test_suite_generator.py | 5 +---- .../generators/test_suite_generator.py | 5 ++++- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/nimrod/test_suite_generation/generators/evosuite_differential_test_suite_generator.py b/nimrod/test_suite_generation/generators/evosuite_differential_test_suite_generator.py index e59ac1c7..466aaa7c 100644 --- a/nimrod/test_suite_generation/generators/evosuite_differential_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/evosuite_differential_test_suite_generator.py @@ -1,10 +1,8 @@ import logging -import os -from typing import Dict, List from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.evosuite_test_suite_generator import EvosuiteTestSuiteGenerator -from nimrod.tools.bin import EVOSUITE, EVOSUITE_RUNTIME +from nimrod.tools.bin import EVOSUITE class EvosuiteDifferentialTestSuiteGenerator(EvosuiteTestSuiteGenerator): @@ -20,9 +18,16 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s '-projectCP', input_jar, f'-Dregressioncp={scenario.scenario_jars.base}', '-class', class_name, - f'-Dsearch_budget={self.SEARCH_BUDGET}', '-DOUTPUT_DIR=' + output_path, + '-Dminimize=false', + '-Djunit_check=false', + '-Dinline=false', ] + + if use_determinism: + params += ["-Dstopping_condition=MaxStatements", f"-seed={self.SEED}"] + else: + params += [f'-Dsearch_budget={self.SEARCH_BUDGET}'] if(len(methods) > 0): params.append( diff --git a/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py b/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py index 7b906df5..4a381682 100644 --- a/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py @@ -1,6 +1,6 @@ import logging import os -from typing import Dict, List +from typing import List from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import \ @@ -10,8 +10,6 @@ class EvosuiteTestSuiteGenerator(TestSuiteGenerator): - SEARCH_BUDGET = int(get_config().get('test_suite_generation_search_budget', 300)) - def get_generator_tool_name(self) -> str: return "EVOSUITE" @@ -22,18 +20,20 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s '-jar', EVOSUITE, '-projectCP', input_jar, '-class', class_name, - '-Dtimeout', '5', '-Dassertion_strategy=all', '-Dp_reflection_on_private=0', '-Dreflection_start_percent=0', '-Dp_functional_mocking=0', '-Dfunctional_mocking_percent=0', '-Dminimize=false', - f'-Dsearch_budget={self.SEARCH_BUDGET}', '-Djunit_check=false', '-Dinline=false', - '-DOUTPUT_DIR=' + output_path, ] + + if use_determinism: + params += ["-Dstopping_condition=MaxStatements", f"-seed={self.SEED}"] + else: + params += [f'-Dsearch_budget={self.SEARCH_BUDGET}'] if(len(methods) > 0): params.append( diff --git a/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py b/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py index bac1b2bd..2c4c2d02 100644 --- a/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py @@ -4,14 +4,11 @@ from nimrod.test_suite_generation.generators.test_suite_generator import \ TestSuiteGenerator -from nimrod.tests.utils import get_config from nimrod.tools.java import Java from nimrod.utils import generate_classpath class RandoopTestSuiteGenerator(TestSuiteGenerator): - SEARCH_BUDGET = int(get_config().get('test_suite_generation_search_budget', 300)) - TARGET_METHODS_LIST_FILENAME = 'methods_to_test.txt' TARGET_CLASS_LIST_FILENAME = 'classes_to_test.txt' @@ -34,7 +31,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s ] if use_determinism: - params += ["--randomseed=10", + params += [f"--randomseed={self.SEED}", "--deterministic", "--time-limit=0", "--attempted-limit=4000"] else: params += [f"--time-limit={int(self.SEARCH_BUDGET)}"] diff --git a/nimrod/test_suite_generation/generators/test_suite_generator.py b/nimrod/test_suite_generation/generators/test_suite_generator.py index 45f7ff8f..7222995f 100644 --- a/nimrod/test_suite_generation/generators/test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/test_suite_generator.py @@ -2,7 +2,7 @@ import logging from os import makedirs, path from time import time -from typing import Dict, List +from typing import List from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.tests.utils import get_base_output_path @@ -14,6 +14,9 @@ class TestSuiteGenerator(ABC): + SEARCH_BUDGET = int(get_config().get('test_suite_generation_search_budget', 300)) + SEED = int(get_config().get('test_suite_generation_seed', 42)) + def __init__(self, java: Java) -> None: self._java = java From fdd0e834eb2c1f4b13a1903821ab3ff62af92940 Mon Sep 17 00:00:00 2001 From: Joao Pedro Henrique Date: Tue, 25 Oct 2022 15:58:08 -0300 Subject: [PATCH 2/5] docs: add documentation for providing a seed --- docs/configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index a6d31e1e..a3df6d65 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -29,6 +29,9 @@ This property allows to customize the time **in seconds** provided for each gene ### generate_deterministic_test_suites If set to true, SMAT will use deterministic versions of its generators, i.e., the generated suites will always be the same regardless of how many times the code is executed. +### test_suite_generation_seed +When `generate_deterministic_test_suites` is set to true, this allows the user to customize the seed which will be provided for the generators implemented. Default value is `42`. + ## Output Generation The following properties are related to the Output Generation step. From 0e17a6ac7d7c77ae1f1acaf5a2514f9e06ddb5a6 Mon Sep 17 00:00:00 2001 From: Joao Pedro Henrique Date: Tue, 25 Oct 2022 16:03:11 -0300 Subject: [PATCH 3/5] fix: add missing import --- nimrod/test_suite_generation/generators/test_suite_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nimrod/test_suite_generation/generators/test_suite_generator.py b/nimrod/test_suite_generation/generators/test_suite_generator.py index 7222995f..6cbf898f 100644 --- a/nimrod/test_suite_generation/generators/test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/test_suite_generator.py @@ -4,6 +4,7 @@ from time import time from typing import List +from nimrod.tests.utils import get_config from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.tests.utils import get_base_output_path from nimrod.test_suite_generation.test_suite import TestSuite From 030a9254d2e4e4104687f237522be7834e41ad3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Henrique?= Date: Thu, 27 Oct 2022 00:40:41 -0300 Subject: [PATCH 4/5] docs: update description for test_suite_generation_seed --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a3df6d65..e31e5c4c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -30,10 +30,10 @@ This property allows to customize the time **in seconds** provided for each gene If set to true, SMAT will use deterministic versions of its generators, i.e., the generated suites will always be the same regardless of how many times the code is executed. ### test_suite_generation_seed -When `generate_deterministic_test_suites` is set to true, this allows the user to customize the seed which will be provided for the generators implemented. Default value is `42`. +When `generate_deterministic_test_suites` is set to true, the user customizes the seed that is provided for the invoked test generation tools. Default value is `42`. ## Output Generation The following properties are related to the Output Generation step. ### output_generators -This property consists of an array with the name of the Reports that are going to be written during the Output Generation step. If not set, all the implemented generators will be used. Valid values are: `behavior_changes`, `semantic_conflicts`, `test_suites`. \ No newline at end of file +This property consists of an array with the name of the Reports that are going to be written during the Output Generation step. If not set, all the implemented generators will be used. Valid values are: `behavior_changes`, `semantic_conflicts`, `test_suites`. From 86b0f24a3095f924261365f333f4f33f0d3531ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Henrique?= Date: Fri, 28 Oct 2022 22:01:48 -0300 Subject: [PATCH 5/5] docs: explain that budget is not used in deterministic test suite generation. --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index e31e5c4c..888f0bab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -24,7 +24,7 @@ The following properties are related to the Test Suites Generation step. This property consists of an array with the name of the Generators which will be used in SMAT during Test Suite Generation step. If not set, all the implemented generators will be used. Valid values are: `randoop`, `randoop-modified`, `evosuite` and `evosuite-differential`. ### test_suite_generation_search_budget -This property allows to customize the time **in seconds** provided for each generator during Test Suite Generation. Default value is 300 seconds. Please note that this option will be ignored when using deterministic Test Suites. +This property allows to customize the time **in seconds** provided for each generator during Test Suite Generation. Default value is 300 seconds. Please note that this option will be ignored when using deterministic Test Suites. Be aware that when using deterministic test suites generation, the budget time is ignored and other stopping criterias are applied, since different computing power can influence in the time required to generate the same test suite. ### generate_deterministic_test_suites If set to true, SMAT will use deterministic versions of its generators, i.e., the generated suites will always be the same regardless of how many times the code is executed.