From 909a591ccc717e73427685aa0febb43996c269cf Mon Sep 17 00:00:00 2001 From: MA Deldari Date: Thu, 9 May 2024 13:33:26 +1000 Subject: [PATCH 1/6] Update README.md (#556) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bc8e82d9a..02c853294 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ Steps to set up Defects4J 3. Add Defects4J's executables to your PATH: - `export PATH=$PATH:"path2defects4j"/framework/bin` + Example: "path2defects4j" like "/user/yourComputerUserName/desktop/defects4j/freamwork/bin"; 4. Check installation: - `defects4j info -p Lang` From 0afd8a2bebb81f453427ac78f17a547bae13d50b Mon Sep 17 00:00:00 2001 From: rjust Date: Wed, 8 May 2024 21:20:42 -0700 Subject: [PATCH 2/6] Clarify an example in the README. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 02c853294..f14e93fdf 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,8 @@ Steps to set up Defects4J 3. Add Defects4J's executables to your PATH: - `export PATH=$PATH:"path2defects4j"/framework/bin` - Example: "path2defects4j" like "/user/yourComputerUserName/desktop/defects4j/freamwork/bin"; + ("path2defects4j" points to the directory to which you cloned Defects4J; it + looks like "/user/yourComputerUserName/desktop/defects4j".) 4. Check installation: - `defects4j info -p Lang` From 3cda2c80aa195622c15da24ab38767d279f8462d Mon Sep 17 00:00:00 2001 From: rjust Date: Thu, 9 May 2024 11:31:14 -0700 Subject: [PATCH 3/6] Do not mutate during compilation unless an mml file is specified. --- init.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/init.sh b/init.sh index 9c65ef9fd..7f14b472a 100755 --- a/init.sh +++ b/init.sh @@ -136,6 +136,12 @@ cd "$DIR_REPOS" && ./get_repos.sh # # Download Major # +# Adapt Major's default wrapper scripts: +# - set headless to true to support Chart on machines without X. +# - do not mutate code unless an MML is specified (for historical reasons, +# major v1 was sometimes called without specifying an MML to simply act as +# javac; Major v2+'s default is to generate all mutants as opposed to none). +# echo echo "Setting up Major ... " MAJOR_VERSION="2.1.0" @@ -145,7 +151,9 @@ cd "$BASE" && rm -rf major \ && download_url_and_unzip "$MAJOR_URL/$MAJOR_ZIP" \ && rm "$MAJOR_ZIP" \ && perl -pi -e '$_ .= qq( -Djava.awt.headless=true \\\n) if /CodeCacheSize/' \ - major/bin/ant # Support Chart on machines without X. + major/bin/ant \ + && perl -pi -e '$_ .= qq(\nif [ -z "\$MML" ]; then javac \$*; exit \$?; fi\n) if /REFACTOR=/' \ + major/bin/major \ ################################################################################ # From 78d147176f0151c37da7f39dd4f291a219c05148 Mon Sep 17 00:00:00 2001 From: rjust Date: Thu, 9 May 2024 15:33:21 -0700 Subject: [PATCH 4/6] Update mutation template. --- framework/util/template.mml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/framework/util/template.mml b/framework/util/template.mml index 943816873..71809afdc 100644 --- a/framework/util/template.mml +++ b/framework/util/template.mml @@ -52,8 +52,3 @@ LIT(BOOLEAN); LIT(NUMBER); // Don't mutate String literals by default //LIT(STRING); - --LVR<"org.apache.commons.math3.util.FastMathLiteralArrays">; --ORU<"org.apache.commons.math3.util.FastMathLiteralArrays">; --LVR<"org.apache.commons.math.optimization.direct.BOBYQAOptimizer@bobyqb">; --STD<"org.apache.commons.math.optimization.direct.BOBYQAOptimizer@bobyqb">; From 76aca77a7adf93b88b6f2601170f484a3f5c89da Mon Sep 17 00:00:00 2001 From: rjust Date: Thu, 9 May 2024 17:12:13 -0700 Subject: [PATCH 5/6] Add test script to run defects4j mutation on all fixed versions. --- framework/test/jobs_mutation.pl | 77 +++++++++++++++++++++ framework/test/test_mutation_cmd.sh | 102 ++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100755 framework/test/jobs_mutation.pl create mode 100755 framework/test/test_mutation_cmd.sh diff --git a/framework/test/jobs_mutation.pl b/framework/test/jobs_mutation.pl new file mode 100755 index 000000000..42f079781 --- /dev/null +++ b/framework/test/jobs_mutation.pl @@ -0,0 +1,77 @@ +#!/usr/bin/env perl +# +#------------------------------------------------------------------------------- +# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +#------------------------------------------------------------------------------- + +=pod + +=head1 NAME + +jobs_mutation.pl -- output a list of invocations of the test_mutation_cmd.sh script, one line per bug. + +=head1 SYNOPSIS + + jobs_mutation.pl | shuf | parallel -j20 --progress + +=head1 DESCRIPTION + +Determines all active bugs and outputs a list of invocations of the +test_mutation_cmd.sh script -- one line per bug. The output list of +jobs can be processed in parallel, e.g., using GNU parallel. + +=cut +use warnings; +use strict; + +use FindBin; +use File::Basename; +use Cwd qw(abs_path); +use Getopt::Std; +use Pod::Usage; + +use lib abs_path("$FindBin::Bin/../core"); +use Constants; +use Project; + +# +# Process arguments and issue usage message if necessary. +# + +# Read all project modules +opendir(my $dir, "$CORE_DIR/Project") or die "Cannot open directory: $!"; +my @files = readdir($dir); +closedir($dir); + +# For each project module, instantiate project and determine its name and number +# of bugs. +my $total = 0; +my @projects = (); +for my $file (@files) { + $file =~ "^([^.]+)\.pm\$" or next; + my $pid=$1; + my $project = Project::create_project($pid); + my $name = $project->{prog_name}; + my @bug_ids = $project->get_bug_ids(); + for my $id (@bug_ids) { + print("./test_mutation_cmd.sh -p $pid -b $id \n"); + } +} diff --git a/framework/test/test_mutation_cmd.sh b/framework/test/test_mutation_cmd.sh new file mode 100755 index 000000000..df0e0c83c --- /dev/null +++ b/framework/test/test_mutation_cmd.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +################################################################################ +# +# This script runs the defects4j mutation command for all bugs or a subset of +# bugs for a given project. +# +# Examples for Lang: +# * All bugs: ./test_mutation_cmd.sh -pLang +# * Bugs 1-10: ./test_mutation_cmd.sh -pLang -b1..10 +# * Bugs 1 and 3: ./test_mutation_cmd.sh -pLang -b1 -b3 +# * Bugs 1-10 and 20: ./test_mutation_cmd.sh -pLang -b1..10 -b20 +# +################################################################################ +# Import helper subroutines and variables, and init Defects4J +source test.include + +# Print usage message and exit +usage() { + local known_pids=$(defects4j pids) + echo "usage: $0 -p [-b ... | -b ... ]" + echo "Project ids:" + for pid in $known_pids; do + echo " * $pid" + done + exit 1 +} + +# Check arguments +while getopts ":p:b:" opt; do + case $opt in + p) PID="$OPTARG" + ;; + b) if [[ "$OPTARG" =~ ^[0-9]*\.\.[0-9]*$ ]]; then + BUGS="$BUGS $(eval echo {$OPTARG})" + else + BUGS="$BUGS $OPTARG" + fi + ;; + \?) + echo "Unknown option: -$OPTARG" >&2 + usage + ;; + :) + echo "No argument provided: -$OPTARG." >&2 + usage + ;; + esac +done + +if [ "$PID" == "" ]; then + usage +fi + +if [ ! -e "$BASE_DIR/framework/core/Project/$PID.pm" ]; then + usage +fi + +init + +# Run all bugs, unless otherwise specified +if [ "$BUGS" == "" ]; then + BUGS="$(get_bug_ids $BASE_DIR/framework/projects/$PID/$BUGS_CSV_ACTIVE)" +fi + +# Create log file +script_name=$(echo $script | sed 's/\.sh$//') +LOG="$TEST_DIR/${script_name}$(printf '_%s_%s' $PID $$).log" +OUT_DIR="$TEST_DIR/${script_name}$(printf '_%s_%s' $PID $$).mutation_summary" +mkdir -p $OUT_DIR + +# Reproduce all bugs (and log all results), regardless of whether errors occur +HALT_ON_ERROR=0 + +test_dir=$(mktemp -d) +for bid in $(echo $BUGS); do + # Skip all bug ids that do not exist in the active-bugs csv + if ! grep -q "^$bid," "$BASE_DIR/framework/projects/$PID/$BUGS_CSV_ACTIVE"; then + warn "Skipping bug ID that is not listed in active-bugs csv: $PID-$bid" + continue + fi + + # Test mutation analysis for the fixed version only. + vid="${bid}f" + work_dir="$test_dir/$PID-$vid" + defects4j checkout -p $PID -v "$vid" -w "$work_dir" || die "checkout: $PID-$vid" + defects4j mutation -w "$work_dir" || die "mutation: $PID-$vid" + + cat "$work_dir/summary.csv" > "$OUT_DIR/$bid" +done +rm -rf $test_dir +HALT_ON_ERROR=1 + +# Print a summary of what went wrong +if [ $ERROR != 0 ]; then + printf '=%.s' $(seq 1 80) 1>&2 + echo 1>&2 + echo "The following errors occurred:" 1>&2 + cat $LOG 1>&2 +fi + +# Indicate whether an error occurred +exit $ERROR From 0c7bf6567254d1d1aa3d1a741af1abb748c4bc27 Mon Sep 17 00:00:00 2001 From: rjust Date: Thu, 9 May 2024 14:59:44 -0700 Subject: [PATCH 6/6] More details on testing. --- framework/test/README.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/framework/test/README.md b/framework/test/README.md index 07613b046..8d9c2930e 100644 --- a/framework/test/README.md +++ b/framework/test/README.md @@ -1,5 +1,7 @@ -Test and analysis scripts ----------------- +Testing and analysis +-------------------- + +## Scripts * `test.include`: Constants and helper functions for test scripts. @@ -29,3 +31,23 @@ Test and analysis scripts * `test_verify_bugs.sh`: Reproduces a bug, or a set of bugs, and verifies that the observed set of triggering tests matches the expected set of triggering tests. + +## CI testing +See [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) for the set of +tests that currently run in CI. + +## Local testing +Some tests take a long time to run and usually only need to be run for major +version updates (e.g., bumping the Java version or adding new defects). + +### Export command and exported properties +The `test_export_command.sh` test takes a long time as it checks out every +single defect and checks whether the exported paths, files, etc. make sense. + +### Reproducing all bugs (parallel) +Reproducing all bugs is the most time-consuming test. To speed it up, we use +GNU parallel (`-j` gives the number of parallel processes): +``` +./jobs_verify_bugs.pl | shuf > jobs.txt +parallel -j20 --progress < jobs.txt +```