diff --git a/move-mutation-test/README.md b/move-mutation-test/README.md index 07b87dbee7..54b22a4013 100644 --- a/move-mutation-test/README.md +++ b/move-mutation-test/README.md @@ -161,5 +161,41 @@ If the user wants to mutate only the `sum` function in the `Sum` module, the use RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --move-2 --mutate-functions sum --mutate-modules Sum ./target/release/move-mutation-test display-report coverage --path-to-report report.txt --modules Sum ``` +------------------------------------------------------------------------------------------------------------ +To optimize mutation testing by selecting operators based on their ability to [detect test coverage gaps](../move-mutator/doc/design.md#operator-effectiveness-analysis), use the `--mode` option. Operators that produce more surviving mutants are more effective at revealing gaps in test coverage, as surviving mutants indicate untested code paths. + +```bash +# Light mode - operators optimized for detecting test gaps with fewest mutants +RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode light + +# Medium mode - light + additional operators for broader test gap detection +RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode medium + +# Medium-only mode - only the operator added in medium (not including light) +RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode medium-only + +# Heavy mode - default, all operators for maximum test gap detection +RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode heavy + +# Heavy-only mode - only the operators added in heavy (not including light/medium) +RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode heavy-only +``` + +The modes include: +- **light**: `binary_operator_swap`, `break_continue_replacement`, `delete_statement` (3 operators) +- **medium**: light + `literal_replacement` (4 operators) +- **medium-only**: `literal_replacement` (1 operator - only what's added in medium) +- **heavy**: all 7 operators +- **heavy-only**: `unary_operator_replacement`, `binary_operator_replacement`, `if_else_replacement` (3 operators - only what's added in heavy) + +------------------------------------------------------------------------------------------------------------ +For fine-grained control over which operators to apply, use the `--operators` option with a comma-separated list: +```bash +RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --operators delete_statement,binary_operator_replacement,if_else_replacement +``` + +Available operators: `unary_operator_replacement`, `delete_statement`, `break_continue_replacement`, `binary_operator_replacement`, `if_else_replacement`, `literal_replacement`, `binary_operator_swap`. + +**Note:** The `--mode` and `--operators` options are mutually exclusive. [nextest]: https://github.com/nextest-rs/nextest diff --git a/move-mutation-test/src/cli.rs b/move-mutation-test/src/cli.rs index 29ef7754dc..f71045e4d7 100644 --- a/move-mutation-test/src/cli.rs +++ b/move-mutation-test/src/cli.rs @@ -6,7 +6,7 @@ use aptos::common::types::MovePackageOptions; use aptos_framework::extended_checks; use clap::Parser; use move_model::metadata::{CompilerVersion, LanguageVersion}; -use move_mutator::cli::{FunctionFilter, ModuleFilter}; +use move_mutator::cli::{FunctionFilter, ModuleFilter, OperatorModeArg}; use move_package::CompilerConfig; use std::path::PathBuf; @@ -42,6 +42,33 @@ pub struct CLIOptions { /// Remove averagely given percentage of mutants. See the doc for more details. #[clap(long, conflicts_with = "use_generated_mutants")] pub downsampling_ratio_percentage: Option, + + /// Mutation operator mode to balance speed and test gap detection. + /// + /// - light: binary_operator_swap, break_continue_replacement, delete_statement + /// - medium: light + literal_replacement + /// - medium-only: literal_replacement (only what's added in medium) + /// - heavy (default): all 7 operators + /// - heavy-only: unary_operator_replacement, binary_operator_replacement, if_else_replacement (only what's added in heavy) + #[clap( + long, + value_enum, + conflicts_with = "operators", + conflicts_with = "use_generated_mutants" + )] + pub mode: Option, + + /// Custom operator selection to run mutations on (comma-separated). + /// + /// Available operators: unary_operator_replacement, delete_statement, break_continue_replacement, binary_operator_replacement, if_else_replacement, literal_replacement, binary_operator_swap + #[clap( + long, + value_parser, + value_delimiter = ',', + conflicts_with = "mode", + conflicts_with = "use_generated_mutants" + )] + pub operators: Option>, } /// This function creates a mutator CLI options from the given mutation-test options. @@ -57,6 +84,8 @@ pub fn create_mutator_options( apply_coverage, // To run tests, compilation must succeed verify_mutants: true, + mode: options.mode, + operators: options.operators.clone(), ..Default::default() } } diff --git a/move-mutator/README.md b/move-mutator/README.md index d22cf7cdaf..b1834c1a1c 100644 --- a/move-mutator/README.md +++ b/move-mutator/README.md @@ -107,3 +107,42 @@ directory. They can be used to check the mutator tool as well. To check possible options, use the `--help` option. [nextest]: https://github.com/nextest-rs/nextest + +### Operator modes + +The mutator tool supports different operator modes to control which mutation operators are applied. Modes are designed to balance speed and the ability to detect test gaps. Operators that produce more surviving mutants are more effective at revealing gaps in test coverage, as surviving mutants indicate untested code paths. + +Use the `--mode` option to select a predefined operator set: +```bash +# Light mode: operators optimized for detecting test gaps with fewest mutants +./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode light + +# Medium mode: light + additional operators for broader test gap detection +./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode medium + +# Medium-only mode: only the operator added in medium (not including light) +./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode medium-only + +# Heavy mode (default): all available operators for maximum test gap detection +./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode heavy + +# Heavy-only mode: only the operators added in heavy (not including light/medium) +./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode heavy-only +``` + +The operator modes are based on [effectiveness analysis](doc/design.md#operator-effectiveness-analysis) where effectiveness measures the ability to detect test coverage gaps: +- **light**: `binary_operator_swap`, `break_continue_replacement`, `delete_statement` (3 operators) +- **medium**: light + `literal_replacement` (4 operators) +- **medium-only**: `literal_replacement` (1 operator - only what's added in medium) +- **heavy**: all 7 operators +- **heavy-only**: `unary_operator_replacement`, `binary_operator_replacement`, `if_else_replacement` (3 operators - only what's added in heavy) + +For fine-grained control, use the `--operators` option to specify exactly which operators to apply: +```bash +# Apply only specific operators +./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --operators delete_statement,binary_operator_replacement,if_else_replacement +``` + +Available operators: `unary_operator_replacement`, `delete_statement`, `break_continue_replacement`, `binary_operator_replacement`, `if_else_replacement`, `literal_replacement`, `binary_operator_swap`. + +**Note:** The `--mode` and `--operators` options are mutually exclusive. diff --git a/move-mutator/doc/design.md b/move-mutator/doc/design.md index 14e53cc467..88f05e7bad 100644 --- a/move-mutator/doc/design.md +++ b/move-mutator/doc/design.md @@ -205,6 +205,70 @@ inefficient. Once mutation places are identified, mutants are generated in reversed order (based on localization) to avoid that. Tools are ready to be extended to support the operator mixing, if needed. +### Operator filtering + +The Move mutator tool supports operator filtering to control which mutation +operators are applied during the mutation process. This feature allows users to +focus on specific operators or use predefined modes based on [operator +effectiveness](#operator-effectiveness-analysis). + +Modes are designed to balance speed and the ability to detect test gaps. Operators +that produce more surviving mutants are more effective at revealing gaps in test +coverage, as surviving mutants indicate untested code paths. + +Five predefined modes are available: +- **Light mode**: `binary_operator_swap`, `break_continue_replacement`, `delete_statement` (3 operators) +- **Medium mode**: Light + `literal_replacement` (4 operators) +- **Medium-only mode**: `literal_replacement` (1 operator - only what's added in medium) +- **Heavy mode** (default): All 7 available operators +- **Heavy-only mode**: `unary_operator_replacement`, `binary_operator_replacement`, `if_else_replacement` (3 operators - only what's added in heavy) + +Users can also specify custom operator sets using the `--operators` CLI option, +providing a comma-separated list of operator names. This allows for fine-grained +control over which operators are applied. + +Operator filtering is performed during AST traversal in the `mutate.rs` module. +When a potential mutation site is found, the tool checks if the corresponding +operator is enabled in the current mode before creating the mutant. This approach +prevents unnecessary mutant generation, making the process more efficient. + +#### Operator effectiveness analysis + +The data below was calculated by running the tool on the largest projects in +[Aptos' Move Framework](https://github.com/aptos-labs/aptos-core/tree/main/aptos-move/framework), +testing 22,597 mutants with an overall kill rate of 82.02%. + +The "Kill Rate" column shows the percentage of mutants killed by tests. While a +high kill rate indicates good test coverage, operators with **lower kill rates** +(more surviving mutants) are often **more effective at detecting test gaps**, as +surviving mutants reveal untested code paths. This is valuable for identifying +weaknesses in test suites. + +``` +╭──────┬─────────────────────────────┬────────┬────────┬───────────────┬───────────╮ +│ Rank │ Operator │ Tested │ Killed │ Kill Rate │ Survived │ +├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +│ #1 │ unary_operator_replacement │ 219 │ 219 │ 100.00% │ 0/219 │ +├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +│ #2 │ delete_statement │ 909 │ 895 │ 98.46% │ 14/909 │ +├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +│ #3 │ break_continue_replacement │ 26 │ 23 │ 88.46% │ 3/26 │ +├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +│ #4 │ binary_operator_replacement │ 7081 │ 6207 │ 87.66% │ 874/7081 │ +├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +│ #5 │ if_else_replacement │ 5310 │ 4579 │ 86.23% │ 731/5310 │ +├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +│ #6 │ literal_replacement │ 8781 │ 6498 │ 74.00% │ 2283/8781 │ +├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +│ #7 │ binary_operator_swap │ 271 │ 114 │ 42.07% │ 157/271 │ +╰──────┴─────────────────────────────┴────────┴────────┴───────────────┴───────────╯ +``` + +The predefined operator modes balance speed with test gap detection capability: +- **Light mode**: Operators with lower kill rates that efficiently reveal test gaps (3 operators) +- **Medium mode**: Light + operators that generate more comprehensive test coverage analysis (4 operators) +- **Heavy mode**: All operators for maximum test gap detection (7 operators) + The Move mutator tool implements the following mutation operators. ### Binary operator replacement diff --git a/move-mutator/src/cli.rs b/move-mutator/src/cli.rs index d30fef3fbb..4145aa5887 100644 --- a/move-mutator/src/cli.rs +++ b/move-mutator/src/cli.rs @@ -2,11 +2,21 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; +use clap::{Parser, ValueEnum}; use std::{path::PathBuf, str::FromStr}; pub const DEFAULT_OUTPUT_DIR: &str = "mutants_output"; +/// Mutation operator mode +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum OperatorModeArg { + Light, + Medium, + MediumOnly, + Heavy, + HeavyOnly, +} + /// Command line options for mutator #[derive(Parser, Debug, Clone)] pub struct CLIOptions { @@ -41,6 +51,22 @@ pub struct CLIOptions { /// Use the unit test coverage report to generate mutants for source code with unit test coverage. #[clap(long = "coverage", conflicts_with = "move_sources")] pub apply_coverage: bool, + + /// Mutation operator mode to balance speed and test gap detection. + /// + /// - light: binary_operator_swap, break_continue_replacement, delete_statement + /// - medium: light + literal_replacement + /// - medium-only: literal_replacement (only what's added in medium) + /// - heavy (default): all 7 operators + /// - heavy-only: unary_operator_replacement, binary_operator_replacement, if_else_replacement (only what's added in heavy) + #[clap(long, value_enum, conflicts_with = "operators")] + pub mode: Option, + + /// Custom operator selection to run mutations on (comma-separated). + /// + /// Available operators: unary_operator_replacement, delete_statement, break_continue_replacement, binary_operator_replacement, if_else_replacement, literal_replacement, binary_operator_swap + #[clap(long, value_parser, value_delimiter = ',', conflicts_with = "mode")] + pub operators: Option>, } /// Checker for conflicts with CLI arguments. @@ -82,6 +108,8 @@ impl Default for CLIOptions { no_overwrite: false, apply_coverage: false, downsampling_ratio_percentage: None, + mode: None, + operators: None, } } } diff --git a/move-mutator/src/configuration.rs b/move-mutator/src/configuration.rs index e29eac8f6b..092f4fa1b4 100644 --- a/move-mutator/src/configuration.rs +++ b/move-mutator/src/configuration.rs @@ -2,7 +2,11 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use crate::{cli::CLIOptions, coverage::Coverage}; +use crate::{ + cli::{CLIOptions, OperatorModeArg}, + coverage::Coverage, + operator_filter::OperatorMode, +}; use std::path::PathBuf; /// Mutator configuration for the Move project. @@ -14,17 +18,49 @@ pub struct Configuration { pub project_path: Option, /// Coverage report where the optional unit test coverage data is stored. pub(crate) coverage: Coverage, + /// Operator filter that determines which mutation operators are enabled. + pub operator_mode: OperatorMode, } impl Configuration { /// Creates a new configuration using command line options. - #[must_use] - pub fn new(project: CLIOptions, project_path: Option) -> Self { - Self { + pub fn new(project: CLIOptions, project_path: Option) -> anyhow::Result { + // Parse and validate the operator mode from CLI options + let operator_mode = Self::parse_operator_mode(&project)?; + + Ok(Self { project, project_path, // Coverage is disabled by default. coverage: Coverage::default(), + operator_mode, + }) + } + + fn parse_operator_mode(project: &CLIOptions) -> anyhow::Result { + match (&project.mode, &project.operators) { + // --operators specified + (None, Some(operators)) => { + let parsed_ops = OperatorMode::parse_operators(operators)?; + Ok(OperatorMode::Custom(parsed_ops)) + }, + // --mode specified + (Some(mode_arg), None) => { + let mode = match mode_arg { + OperatorModeArg::Light => OperatorMode::Light, + OperatorModeArg::Medium => OperatorMode::Medium, + OperatorModeArg::MediumOnly => OperatorMode::MediumOnly, + OperatorModeArg::Heavy => OperatorMode::Heavy, + OperatorModeArg::HeavyOnly => OperatorMode::HeavyOnly, + }; + Ok(mode) + }, + // neither specified + (None, None) => Ok(OperatorMode::default()), + // both specified - this should be prevented by clap conflicts + (Some(_), Some(_)) => { + unreachable!("Both --mode and --operators specified") + }, } } } diff --git a/move-mutator/src/lib.rs b/move-mutator/src/lib.rs index fdd1683f29..319bbd0374 100644 --- a/move-mutator/src/lib.rs +++ b/move-mutator/src/lib.rs @@ -15,6 +15,7 @@ pub mod configuration; pub(crate) mod coverage; mod mutant; mod operator; +pub mod operator_filter; mod operators; mod output; pub mod report; @@ -74,10 +75,19 @@ pub fn run_move_mutator( }; let mut mutator_configuration = - Configuration::new(options, Some(original_package_path.to_owned())); + Configuration::new(options, Some(original_package_path.to_owned()))?; trace!("Mutator configuration: {mutator_configuration:?}"); + let enabled_operators = mutator_configuration.operator_mode.get_operators(); + println!( + "Operator types being mutated ({}):", + enabled_operators.len() + ); + for (i, op) in enabled_operators.iter().enumerate() { + println!(" {}. {}", i + 1, op); + } + let package_path = mutator_configuration .project_path .clone() diff --git a/move-mutator/src/mutate.rs b/move-mutator/src/mutate.rs index 07157bda7d..dcf49b4d9d 100644 --- a/move-mutator/src/mutate.rs +++ b/move-mutator/src/mutate.rs @@ -7,6 +7,7 @@ use crate::{ configuration::Configuration, mutant::Mutant, operator::MutationOp, + operator_filter, operators::{ binary::Binary, binary_swap::BinarySwap, break_continue::BreakContinue, delete_stmt::DeleteStmt, ifelse::IfElse, literal::Literal, unary::Unary, ExpLoc, @@ -168,7 +169,7 @@ fn traverse_function( return true; } - result.extend(parse_expression_and_find_mutants(function, exp_data)); + result.extend(parse_expression_and_find_mutants(function, exp_data, conf)); true }); }; @@ -184,7 +185,11 @@ fn traverse_function( /// can be applied to it. /// When Move language is extended with new expressions, this function needs to be updated to support them. #[allow(clippy::too_many_lines)] -fn parse_expression_and_find_mutants(function: &FunctionEnv<'_>, exp: &ExpData) -> Vec { +fn parse_expression_and_find_mutants( + function: &FunctionEnv<'_>, + exp: &ExpData, + conf: &Configuration, +) -> Vec { let convert_exps_to_explocs = |exps: &[Exp]| -> Vec { exps.iter() .map(|e| { @@ -218,6 +223,12 @@ fn parse_expression_and_find_mutants(function: &FunctionEnv<'_>, exp: &ExpData) match exp { ExpData::Call(node_id, op, exps) => match op { Operation::MoveTo | Operation::Abort => { + if !conf + .operator_mode + .should_apply(operator_filter::Operator::DeleteStatement) + { + return vec![]; + } vec![Mutant::new(MutationOp::new(Box::new(DeleteStmt::new( exp.clone().into_exp(), function.module_env.env.get_node_loc(*node_id), @@ -242,21 +253,39 @@ fn parse_expression_and_find_mutants(function: &FunctionEnv<'_>, exp: &ExpData) | Operation::Shr | Operation::Xor => { let exps_loc = convert_exps_to_explocs(exps); - let mut result = vec![Mutant::new(MutationOp::new(Box::new(Binary::new( - op.clone(), - function.module_env.env.get_node_loc(*node_id), - exps_loc.clone(), - ))))]; + let mut result = Vec::new(); + + if conf + .operator_mode + .should_apply(operator_filter::Operator::BinaryOperatorReplacement) + { + result.push(Mutant::new(MutationOp::new(Box::new(Binary::new( + op.clone(), + function.module_env.env.get_node_loc(*node_id), + exps_loc.clone(), + ))))); + } - result.push(Mutant::new(MutationOp::new(Box::new(BinarySwap::new( - op.clone(), - function.module_env.env.get_node_loc(*node_id), - exps_loc, - ))))); + if conf + .operator_mode + .should_apply(operator_filter::Operator::BinaryOperatorSwap) + { + result.push(Mutant::new(MutationOp::new(Box::new(BinarySwap::new( + op.clone(), + function.module_env.env.get_node_loc(*node_id), + exps_loc, + ))))); + } result }, Operation::Not => { + if !conf + .operator_mode + .should_apply(operator_filter::Operator::UnaryOperatorReplacement) + { + return vec![]; + } let exps_loc = convert_exps_to_explocs(exps); vec![Mutant::new(MutationOp::new(Box::new(Unary::new( op.clone(), @@ -267,6 +296,12 @@ fn parse_expression_and_find_mutants(function: &FunctionEnv<'_>, exp: &ExpData) _ => vec![], }, ExpData::IfElse(_, cond, if_exp, else_exp) => { + if !conf + .operator_mode + .should_apply(operator_filter::Operator::IfElseReplacement) + { + return vec![]; + } let cond_loc = ExpLoc { exp: cond.clone(), loc: function.module_env.env.get_node_loc(cond.node_id()), @@ -286,6 +321,12 @@ fn parse_expression_and_find_mutants(function: &FunctionEnv<'_>, exp: &ExpData) ))))] }, ExpData::Value(node_id, value) => { + if !conf + .operator_mode + .should_apply(operator_filter::Operator::LiteralReplacement) + { + return vec![]; + } let mutants = vec![Mutant::new(MutationOp::new(Box::new(Literal::new( value.clone(), function.module_env.env.get_node_type(*node_id), @@ -293,9 +334,17 @@ fn parse_expression_and_find_mutants(function: &FunctionEnv<'_>, exp: &ExpData) ))))]; mutants }, - ExpData::LoopCont(node_id, ..) => vec![Mutant::new(MutationOp::new(Box::new( - BreakContinue::new(function.module_env.env.get_node_loc(*node_id)), - )))], + ExpData::LoopCont(node_id, ..) => { + if !conf + .operator_mode + .should_apply(operator_filter::Operator::BreakContinueReplacement) + { + return vec![]; + } + vec![Mutant::new(MutationOp::new(Box::new(BreakContinue::new( + function.module_env.env.get_node_loc(*node_id), + ))))] + }, ExpData::Return(..) | ExpData::Mutate(..) diff --git a/move-mutator/src/operator_filter.rs b/move-mutator/src/operator_filter.rs new file mode 100644 index 0000000000..b41b219980 --- /dev/null +++ b/move-mutator/src/operator_filter.rs @@ -0,0 +1,438 @@ +// Copyright © Eiger +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +//! Operator filtering module for mutation testing. +//! +//! This module provides functionality to filter mutation operators based on their effectiveness. +//! +//! It supports predefined modes (Light, Medium, Heavy) and custom operator selection, +//! where Light is the most effective (most killed mutants) and Heavy is mutating all operators. +//! +//! The way that the effectiveness was calculated is by running the tool on the biggest projects +//! in [Aptos' Move Framework](https://github.com/aptos-labs/aptos-core/tree/main/aptos-move/framework). +//! +//! These were the results: +//! Total mutants tested: 22597 +//! Total mutants killed: 18535 +//! Average effectiveness: 82.02% +//! +//! ╭──────┬─────────────────────────────┬────────┬────────┬───────────────┬───────────╮ +//! │ Rank │ Operator │ Tested │ Killed │ Effectiveness │ Kill Rate │ +//! ├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +//! │ #1 │ unary_operator_replacement │ 219 │ 219 │ 100.00% │ 219/219 │ +//! ├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +//! │ #2 │ delete_statement │ 909 │ 895 │ 98.46% │ 895/909 │ +//! ├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +//! │ #3 │ break_continue_replacement │ 26 │ 23 │ 88.46% │ 23/26 │ +//! ├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +//! │ #4 │ binary_operator_replacement │ 7081 │ 6207 │ 87.66% │ 6207/7081 │ +//! ├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +//! │ #5 │ if_else_replacement │ 5310 │ 4579 │ 86.23% │ 4579/5310 │ +//! ├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +//! │ #6 │ literal_replacement │ 8781 │ 6498 │ 74.00% │ 6498/8781 │ +//! ├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤ +//! │ #7 │ binary_operator_swap │ 271 │ 114 │ 42.07% │ 114/271 │ +//! ╰──────┴─────────────────────────────┴────────┴────────┴───────────────┴───────────╯ + +use crate::operators::binary::OPERATOR_NAME as BINARY_OPERATOR_NAME; +use crate::operators::binary_swap::OPERATOR_NAME as BINARY_SWAP_NAME; +use crate::operators::break_continue::OPERATOR_NAME as BREAK_CONTINUE_NAME; +use crate::operators::delete_stmt::OPERATOR_NAME as DELETE_STATEMENT_NAME; +use crate::operators::ifelse::OPERATOR_NAME as IF_ELSE_NAME; +use crate::operators::literal::OPERATOR_NAME as LITERAL_NAME; +use crate::operators::unary::OPERATOR_NAME as UNARY_OPERATOR_NAME; +use std::str::FromStr; + +/// Enum representing all available mutation operators. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Operator { + UnaryOperatorReplacement, + DeleteStatement, + BreakContinueReplacement, + BinaryOperatorReplacement, + IfElseReplacement, + LiteralReplacement, + BinaryOperatorSwap, +} + +impl Operator { + const fn as_str(self) -> &'static str { + match self { + Self::UnaryOperatorReplacement => UNARY_OPERATOR_NAME, + Self::DeleteStatement => DELETE_STATEMENT_NAME, + Self::BreakContinueReplacement => BREAK_CONTINUE_NAME, + Self::BinaryOperatorReplacement => BINARY_OPERATOR_NAME, + Self::IfElseReplacement => IF_ELSE_NAME, + Self::LiteralReplacement => LITERAL_NAME, + Self::BinaryOperatorSwap => BINARY_SWAP_NAME, + } + } + + const fn all() -> [Operator; 7] { + [ + Operator::UnaryOperatorReplacement, + Operator::DeleteStatement, + Operator::BreakContinueReplacement, + Operator::BinaryOperatorReplacement, + Operator::IfElseReplacement, + Operator::LiteralReplacement, + Operator::BinaryOperatorSwap, + ] + } +} + +impl FromStr for Operator { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + UNARY_OPERATOR_NAME => Ok(Self::UnaryOperatorReplacement), + DELETE_STATEMENT_NAME => Ok(Self::DeleteStatement), + BREAK_CONTINUE_NAME => Ok(Self::BreakContinueReplacement), + BINARY_OPERATOR_NAME => Ok(Self::BinaryOperatorReplacement), + IF_ELSE_NAME => Ok(Self::IfElseReplacement), + LITERAL_NAME => Ok(Self::LiteralReplacement), + BINARY_SWAP_NAME => Ok(Self::BinaryOperatorSwap), + _ => anyhow::bail!("Unknown operator: {}", s), + } + } +} + +/// Mutation operator mode that determines which operators are enabled. +/// +/// Modes are designed to balance speed and ability to detect test gaps. +/// Operators that produce more surviving mutants are more effective at revealing +/// gaps in test coverage, as surviving mutants indicate untested code paths. +/// +/// Mode breakdown: +/// - Light: binary_operator_swap, break_continue_replacement, delete_statement (3 operators) +/// - Medium: Light + literal_replacement (4 operators) +/// - Medium-only: literal_replacement (1 operator - only what's added in medium) +/// - Heavy: All 7 operators +/// - Heavy-only: unary_operator_replacement, binary_operator_replacement, if_else_replacement (3 operators - only what's added in heavy) +#[derive(Debug, Clone, PartialEq, Default)] +pub enum OperatorMode { + /// Light mode: Operators optimized for detecting test gaps with fewest mutants. + /// Includes: binary_operator_swap, break_continue_replacement, delete_statement + Light, + + /// Medium mode: Light operators + literal_replacement for broader test gap detection. + /// Includes: binary_operator_swap, break_continue_replacement, delete_statement, literal_replacement + Medium, + + /// Medium-only mode: Only the operator added in medium (not including light operators). + /// Includes: literal_replacement + MediumOnly, + + /// Heavy mode: All available operators for maximum test gap detection. + /// Includes all 7 operators, default mode. + #[default] + Heavy, + + /// Heavy-only mode: Only the operators added in heavy (not including light/medium operators). + /// Includes: unary_operator_replacement, binary_operator_replacement, if_else_replacement + HeavyOnly, + + /// Custom mode: User-specified set of operators. + /// The vector contains validated operators. + Custom(Vec), +} + +impl OperatorMode { + /// Returns the list of enabled operator names for this mode. + pub fn get_operators(&self) -> Vec<&str> { + self.operators_enum().iter().map(|op| op.as_str()).collect() + } + + /// Returns the list of enabled operators as an enum (internal use). + fn operators_enum(&self) -> Vec { + match self { + OperatorMode::Light => Self::light_operators(), + OperatorMode::Medium => Self::medium_operators(), + OperatorMode::MediumOnly => Self::medium_only_operators(), + OperatorMode::Heavy => Self::heavy_operators(), + OperatorMode::HeavyOnly => Self::heavy_only_operators(), + OperatorMode::Custom(ops) => ops.clone(), + } + } + + /// Returns operators for Light mode. + /// Operators with lower kill rates that are effective at detecting test gaps. + fn light_operators() -> Vec { + vec![ + Operator::BinaryOperatorSwap, + Operator::BreakContinueReplacement, + Operator::DeleteStatement, + ] + } + + /// Returns operators for Medium mode. + /// Light operators + literal_replacement. + fn medium_operators() -> Vec { + let mut ops = Self::light_operators(); + ops.push(Operator::LiteralReplacement); + ops + } + + /// Returns operators for Medium-only mode. + /// Only the operator added in medium (not including light). + fn medium_only_operators() -> Vec { + vec![Operator::LiteralReplacement] + } + + /// Returns operators for Heavy mode. + /// All available operators. + fn heavy_operators() -> Vec { + Operator::all().to_vec() + } + + /// Returns operators for Heavy-only mode. + /// Only the operators added in heavy (not including light/medium). + fn heavy_only_operators() -> Vec { + vec![ + Operator::UnaryOperatorReplacement, + Operator::BinaryOperatorReplacement, + Operator::IfElseReplacement, + ] + } + + /// Checks if the specified operator should be applied in this mode. + /// + /// # Arguments + /// + /// * `operator` - The operator to check. + /// + /// # Returns + /// + /// `true` if the operator is enabled in this mode, `false` otherwise. + pub fn should_apply(&self, operator: Operator) -> bool { + self.operators_enum().contains(&operator) + } + + /// Validates a list of operator names and returns an error if any are invalid. + /// + /// # Arguments + /// + /// * `operators` - Vector of operator names to validate. + /// + /// # Returns + /// + /// `Ok(())` if all operators are valid, `Err` with details of invalid operators. + pub fn validate_operators(operators: &[String]) -> anyhow::Result<()> { + let mut invalid_ops = Vec::new(); + + for op in operators { + if Operator::from_str(op).is_err() { + invalid_ops.push(op.clone()); + } + } + + if !invalid_ops.is_empty() { + anyhow::bail!( + "Invalid operator name(s): {}. Valid operators are: {}", + invalid_ops.join(", "), + Self::list_all_operators() + ); + } + + Ok(()) + } + + /// Parses a list of operator name strings into a vector of Operator enums. + /// + /// # Arguments + /// + /// * `operator_names` - Vector of operator name strings. + /// + /// # Returns + /// + /// `Ok(Vec)` if all operators are valid, `Err` otherwise. + pub fn parse_operators(operator_names: &[String]) -> anyhow::Result> { + Self::validate_operators(operator_names)?; + Ok(operator_names + .iter() + .map(|s| Operator::from_str(s).expect("already validated")) + .collect()) + } + + /// Returns a formatted string listing all available operators with their effectiveness. + pub fn list_all_operators() -> String { + Operator::all() + .iter() + .map(|op| format!(" - {}", op.as_str())) + .collect::>() + .join("\n") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_operator_as_str() { + assert_eq!( + Operator::UnaryOperatorReplacement.as_str(), + "unary_operator_replacement" + ); + assert_eq!(Operator::DeleteStatement.as_str(), "delete_statement"); + assert_eq!( + Operator::BinaryOperatorSwap.as_str(), + "binary_operator_swap" + ); + } + + #[test] + fn test_operator_from_str() { + assert_eq!( + Operator::from_str("unary_operator_replacement").unwrap(), + Operator::UnaryOperatorReplacement + ); + assert_eq!( + Operator::from_str("delete_statement").unwrap(), + Operator::DeleteStatement + ); + assert!(Operator::from_str("invalid").is_err()); + } + + #[test] + fn test_operator_all() { + let all = Operator::all(); + assert_eq!(all.len(), 7); + } + + #[test] + fn test_light_mode_operators() { + let mode = OperatorMode::Light; + let ops = mode.get_operators(); + assert_eq!(ops.len(), 3); + assert!(ops.contains(&Operator::BinaryOperatorSwap.as_str())); + assert!(ops.contains(&Operator::BreakContinueReplacement.as_str())); + assert!(ops.contains(&Operator::DeleteStatement.as_str())); + } + + #[test] + fn test_medium_mode_operators() { + let mode = OperatorMode::Medium; + let ops = mode.get_operators(); + assert_eq!(ops.len(), 4); + assert!(ops.contains(&Operator::BinaryOperatorSwap.as_str())); + assert!(ops.contains(&Operator::BreakContinueReplacement.as_str())); + assert!(ops.contains(&Operator::DeleteStatement.as_str())); + assert!(ops.contains(&Operator::LiteralReplacement.as_str())); + } + + #[test] + fn test_medium_only_mode_operators() { + let mode = OperatorMode::MediumOnly; + let ops = mode.get_operators(); + assert_eq!(ops.len(), 1); + assert!(ops.contains(&Operator::LiteralReplacement.as_str())); + } + + #[test] + fn test_heavy_only_mode_operators() { + let mode = OperatorMode::HeavyOnly; + let ops = mode.get_operators(); + assert_eq!(ops.len(), 3); + assert!(ops.contains(&Operator::UnaryOperatorReplacement.as_str())); + assert!(ops.contains(&Operator::BinaryOperatorReplacement.as_str())); + assert!(ops.contains(&Operator::IfElseReplacement.as_str())); + } + + #[test] + fn test_heavy_mode_operators() { + let mode = OperatorMode::Heavy; + let ops = mode.get_operators(); + assert_eq!(ops.len(), 7); + // All operators should be present + assert!(ops.contains(&Operator::UnaryOperatorReplacement.as_str())); + assert!(ops.contains(&Operator::DeleteStatement.as_str())); + assert!(ops.contains(&Operator::BreakContinueReplacement.as_str())); + assert!(ops.contains(&Operator::BinaryOperatorReplacement.as_str())); + assert!(ops.contains(&Operator::IfElseReplacement.as_str())); + assert!(ops.contains(&Operator::LiteralReplacement.as_str())); + assert!(ops.contains(&Operator::BinaryOperatorSwap.as_str())); + } + + #[test] + fn test_custom_mode() { + let mode = OperatorMode::Custom(vec![ + Operator::DeleteStatement, + Operator::BinaryOperatorReplacement, + ]); + let ops = mode.get_operators(); + assert_eq!(ops.len(), 2); + assert!(ops.contains(&Operator::DeleteStatement.as_str())); + assert!(ops.contains(&Operator::BinaryOperatorReplacement.as_str())); + } + + #[test] + fn test_should_apply() { + let mode = OperatorMode::Light; + assert!(mode.should_apply(Operator::BinaryOperatorSwap)); + assert!(mode.should_apply(Operator::BreakContinueReplacement)); + assert!(mode.should_apply(Operator::DeleteStatement)); + assert!(!mode.should_apply(Operator::LiteralReplacement)); + assert!(!mode.should_apply(Operator::UnaryOperatorReplacement)); + assert!(!mode.should_apply(Operator::BinaryOperatorReplacement)); + } + + #[test] + fn test_validate_operators_valid() { + let operators = vec![ + Operator::DeleteStatement.as_str().to_string(), + Operator::BinaryOperatorReplacement.as_str().to_string(), + ]; + assert!(OperatorMode::validate_operators(&operators).is_ok()); + } + + #[test] + fn test_validate_operators_invalid() { + let operators = vec![ + "invalid_operator".to_string(), + "another_invalid".to_string(), + ]; + let result = OperatorMode::validate_operators(&operators); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!(err_msg.contains("invalid_operator")); + assert!(err_msg.contains("another_invalid")); + } + + #[test] + fn test_validate_operators_mixed() { + let operators = vec![ + Operator::DeleteStatement.as_str().to_string(), + "invalid_operator".to_string(), + ]; + let result = OperatorMode::validate_operators(&operators); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + // Error message should mention the invalid operator + assert!(err_msg.contains("invalid_operator")); + // Error message will also list all valid operators (including delete_statement) + assert!(err_msg.contains("Invalid operator name")); + } + + #[test] + fn test_parse_operators() { + let operators = vec![ + Operator::DeleteStatement.as_str().to_string(), + Operator::BinaryOperatorReplacement.as_str().to_string(), + ]; + let result = OperatorMode::parse_operators(&operators); + assert!(result.is_ok()); + let parsed = result.unwrap(); + assert_eq!(parsed.len(), 2); + assert_eq!(parsed[0], Operator::DeleteStatement); + assert_eq!(parsed[1], Operator::BinaryOperatorReplacement); + } + + #[test] + fn test_default_mode() { + let mode = OperatorMode::default(); + assert_eq!(mode, OperatorMode::Heavy); + } +} diff --git a/move-mutator/src/operators/delete_stmt.rs b/move-mutator/src/operators/delete_stmt.rs index efb14d8359..d9cbd21275 100644 --- a/move-mutator/src/operators/delete_stmt.rs +++ b/move-mutator/src/operators/delete_stmt.rs @@ -38,6 +38,11 @@ impl MutationOperator for DeleteStmt { ); let cur_op = &source[start..end]; + // Skip mutation if this is an "assert!" macro. + if cur_op == "assert" { + return vec![]; + } + let ops: Vec<&str> = vec![MOVE_EMPTY_STMT]; ops.into_iter() diff --git a/move-mutator/src/operators/literal.rs b/move-mutator/src/operators/literal.rs index b2cd7bbd15..d6599b651b 100644 --- a/move-mutator/src/operators/literal.rs +++ b/move-mutator/src/operators/literal.rs @@ -48,6 +48,11 @@ impl MutationOperator for Literal { let end = self.loc.span().end().to_usize(); let cur_op = &source[start..end]; + // Skip mutation if this is an "assert!" macro. + if cur_op == "assert" { + return vec![]; + } + // Group of literal statements for possible Value types. // For each group use minimum and maximum values and some additional values: // - for u8, u16, u32, u64, u128 use minimum, maximum, value + 1, value - 1 diff --git a/move-mutator/src/output.rs b/move-mutator/src/output.rs index f7069c875e..3fe4f727a3 100644 --- a/move-mutator/src/output.rs +++ b/move-mutator/src/output.rs @@ -220,7 +220,7 @@ mod tests { no_overwrite: false, ..Default::default() }; - let config = Configuration::new(options, None); + let config = Configuration::new(options, None).unwrap(); assert!(setup_output_dir(&config).is_ok()); assert!(output_dir.exists()); } @@ -235,7 +235,7 @@ mod tests { no_overwrite: false, ..Default::default() }; - let config = Configuration::new(options, None); + let config = Configuration::new(options, None).unwrap(); assert!(setup_output_dir(&config).is_ok()); assert!(output_dir.exists()); } @@ -250,7 +250,7 @@ mod tests { no_overwrite: true, ..Default::default() }; - let config = Configuration::new(options, None); + let config = Configuration::new(options, None).unwrap(); assert!(setup_output_dir(&config).is_err()); } } diff --git a/move-mutator/tests/move-assets/simple_move_2_features/report.txt.spec-exp b/move-mutator/tests/move-assets/simple_move_2_features/report.txt.spec-exp index 81f7218189..fa1e8d0756 100644 --- a/move-mutator/tests/move-assets/simple_move_2_features/report.txt.spec-exp +++ b/move-mutator/tests/move-assets/simple_move_2_features/report.txt.spec-exp @@ -3,8 +3,8 @@ "sources/Enums.move": [ { "module_func": "Enums::rectangle_area", - "tested": 12, - "killed": 5, + "tested": 7, + "killed": 0, "mutants_alive_diffs": [ "--- original\n+++ modified\n@@ -6,7 +6,7 @@\n }\n\n fun rectangle_area(shape: Shape): u64 {\n-\t\tassert!(shape is Shape::Rectangle);\n+\t\tassert!(true);\n \t\tshape.width*shape.height\n }\n\n", "--- original\n+++ modified\n@@ -6,7 +6,7 @@\n }\n\n fun rectangle_area(shape: Shape): u64 {\n-\t\tassert!(shape is Shape::Rectangle);\n+\t\tassert!(false);\n \t\tshape.width*shape.height\n }\n\n", @@ -14,13 +14,7 @@ "--- original\n+++ modified\n@@ -7,7 +7,7 @@\n\n fun rectangle_area(shape: Shape): u64 {\n \t\tassert!(shape is Shape::Rectangle);\n-\t\tshape.width*shape.height\n+\t\tshape.width/shape.height\n }\n\n \tenum Colour { Red, Green, Blue }\n", "--- original\n+++ modified\n@@ -7,7 +7,7 @@\n\n fun rectangle_area(shape: Shape): u64 {\n \t\tassert!(shape is Shape::Rectangle);\n-\t\tshape.width*shape.height\n+\t\tshape.width%shape.height\n }\n\n \tenum Colour { Red, Green, Blue }\n" ], - "mutants_killed_diff": [ - "--- original\n+++ modified\n@@ -6,7 +6,7 @@\n }\n\n fun rectangle_area(shape: Shape): u64 {\n-\t\tassert!(shape is Shape::Rectangle);\n+\t\t{}!(shape is Shape::Rectangle);\n \t\tshape.width*shape.height\n }\n\n", - "--- original\n+++ modified\n@@ -6,7 +6,7 @@\n }\n\n fun rectangle_area(shape: Shape): u64 {\n-\t\tassert!(shape is Shape::Rectangle);\n+\t\t0!(shape is Shape::Rectangle);\n \t\tshape.width*shape.height\n }\n\n", - "--- original\n+++ modified\n@@ -6,7 +6,7 @@\n }\n\n fun rectangle_area(shape: Shape): u64 {\n-\t\tassert!(shape is Shape::Rectangle);\n+\t\t18446744073709551615!(shape is Shape::Rectangle);\n \t\tshape.width*shape.height\n }\n\n", - "--- original\n+++ modified\n@@ -6,7 +6,7 @@\n }\n\n fun rectangle_area(shape: Shape): u64 {\n-\t\tassert!(shape is Shape::Rectangle);\n+\t\t14566554180833181697!(shape is Shape::Rectangle);\n \t\tshape.width*shape.height\n }\n\n", - "--- original\n+++ modified\n@@ -6,7 +6,7 @@\n }\n\n fun rectangle_area(shape: Shape): u64 {\n-\t\tassert!(shape is Shape::Rectangle);\n+\t\t14566554180833181695!(shape is Shape::Rectangle);\n \t\tshape.width*shape.height\n }\n\n" - ] + "mutants_killed_diff": [] } ], "sources/FriendVisibility.move": [