Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions move-mutation-test/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,5 +161,41 @@ If the user wants to mutate only the `sum` function in the `Sum` module, the use
RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --move-2 --mutate-functions sum --mutate-modules Sum
./target/release/move-mutation-test display-report coverage --path-to-report report.txt --modules Sum
```
------------------------------------------------------------------------------------------------------------
To optimize mutation testing by selecting operators based on their ability to [detect test coverage gaps](../move-mutator/doc/design.md#operator-effectiveness-analysis), use the `--mode` option. Operators that produce more surviving mutants are more effective at revealing gaps in test coverage, as surviving mutants indicate untested code paths.

```bash
# Light mode - operators optimized for detecting test gaps with fewest mutants
RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode light

# Medium mode - light + additional operators for broader test gap detection
RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode medium

# Medium-only mode - only the operator added in medium (not including light)
RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode medium-only

# Heavy mode - default, all operators for maximum test gap detection
RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode heavy

# Heavy-only mode - only the operators added in heavy (not including light/medium)
RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --mode heavy-only
```

The modes include:
- **light**: `binary_operator_swap`, `break_continue_replacement`, `delete_statement` (3 operators)
- **medium**: light + `literal_replacement` (4 operators)
- **medium-only**: `literal_replacement` (1 operator - only what's added in medium)
- **heavy**: all 7 operators
- **heavy-only**: `unary_operator_replacement`, `binary_operator_replacement`, `if_else_replacement` (3 operators - only what's added in heavy)

------------------------------------------------------------------------------------------------------------
For fine-grained control over which operators to apply, use the `--operators` option with a comma-separated list:
```bash
RUST_LOG=info ./target/release/move-mutation-test run --package-dir move-mutator/tests/move-assets/simple --output report.txt --operators delete_statement,binary_operator_replacement,if_else_replacement
```

Available operators: `unary_operator_replacement`, `delete_statement`, `break_continue_replacement`, `binary_operator_replacement`, `if_else_replacement`, `literal_replacement`, `binary_operator_swap`.

**Note:** The `--mode` and `--operators` options are mutually exclusive.

[nextest]: https://github.com/nextest-rs/nextest
31 changes: 30 additions & 1 deletion move-mutation-test/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use aptos::common::types::MovePackageOptions;
use aptos_framework::extended_checks;
use clap::Parser;
use move_model::metadata::{CompilerVersion, LanguageVersion};
use move_mutator::cli::{FunctionFilter, ModuleFilter};
use move_mutator::cli::{FunctionFilter, ModuleFilter, OperatorModeArg};
use move_package::CompilerConfig;
use std::path::PathBuf;

Expand Down Expand Up @@ -42,6 +42,33 @@ pub struct CLIOptions {
/// Remove averagely given percentage of mutants. See the doc for more details.
#[clap(long, conflicts_with = "use_generated_mutants")]
pub downsampling_ratio_percentage: Option<usize>,

/// Mutation operator mode to balance speed and test gap detection.
///
/// - light: binary_operator_swap, break_continue_replacement, delete_statement
/// - medium: light + literal_replacement
/// - medium-only: literal_replacement (only what's added in medium)
/// - heavy (default): all 7 operators
/// - heavy-only: unary_operator_replacement, binary_operator_replacement, if_else_replacement (only what's added in heavy)
#[clap(
long,
value_enum,
conflicts_with = "operators",
conflicts_with = "use_generated_mutants"
)]
pub mode: Option<OperatorModeArg>,

/// Custom operator selection to run mutations on (comma-separated).
///
/// Available operators: unary_operator_replacement, delete_statement, break_continue_replacement, binary_operator_replacement, if_else_replacement, literal_replacement, binary_operator_swap
#[clap(
long,
value_parser,
value_delimiter = ',',
conflicts_with = "mode",
conflicts_with = "use_generated_mutants"
)]
pub operators: Option<Vec<String>>,
}

/// This function creates a mutator CLI options from the given mutation-test options.
Expand All @@ -57,6 +84,8 @@ pub fn create_mutator_options(
apply_coverage,
// To run tests, compilation must succeed
verify_mutants: true,
mode: options.mode,
operators: options.operators.clone(),
..Default::default()
}
}
Expand Down
39 changes: 39 additions & 0 deletions move-mutator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,42 @@ directory. They can be used to check the mutator tool as well.
To check possible options, use the `--help` option.

[nextest]: https://github.com/nextest-rs/nextest

### Operator modes

The mutator tool supports different operator modes to control which mutation operators are applied. Modes are designed to balance speed and the ability to detect test gaps. Operators that produce more surviving mutants are more effective at revealing gaps in test coverage, as surviving mutants indicate untested code paths.

Use the `--mode` option to select a predefined operator set:
```bash
# Light mode: operators optimized for detecting test gaps with fewest mutants
./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode light

# Medium mode: light + additional operators for broader test gap detection
./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode medium

# Medium-only mode: only the operator added in medium (not including light)
./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode medium-only

# Heavy mode (default): all available operators for maximum test gap detection
./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode heavy

# Heavy-only mode: only the operators added in heavy (not including light/medium)
./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --mode heavy-only
```

The operator modes are based on [effectiveness analysis](doc/design.md#operator-effectiveness-analysis) where effectiveness measures the ability to detect test coverage gaps:
- **light**: `binary_operator_swap`, `break_continue_replacement`, `delete_statement` (3 operators)
- **medium**: light + `literal_replacement` (4 operators)
- **medium-only**: `literal_replacement` (1 operator - only what's added in medium)
- **heavy**: all 7 operators
- **heavy-only**: `unary_operator_replacement`, `binary_operator_replacement`, `if_else_replacement` (3 operators - only what's added in heavy)

For fine-grained control, use the `--operators` option to specify exactly which operators to apply:
```bash
# Apply only specific operators
./target/release/move-mutator --package-dir move-mutator/tests/move-assets/simple/ --operators delete_statement,binary_operator_replacement,if_else_replacement
```

Available operators: `unary_operator_replacement`, `delete_statement`, `break_continue_replacement`, `binary_operator_replacement`, `if_else_replacement`, `literal_replacement`, `binary_operator_swap`.

**Note:** The `--mode` and `--operators` options are mutually exclusive.
64 changes: 64 additions & 0 deletions move-mutator/doc/design.md
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,70 @@ inefficient. Once mutation places are identified, mutants are generated in
reversed order (based on localization) to avoid that. Tools are ready to be
extended to support the operator mixing, if needed.

### Operator filtering

The Move mutator tool supports operator filtering to control which mutation
operators are applied during the mutation process. This feature allows users to
focus on specific operators or use predefined modes based on [operator
effectiveness](#operator-effectiveness-analysis).

Modes are designed to balance speed and the ability to detect test gaps. Operators
that produce more surviving mutants are more effective at revealing gaps in test
coverage, as surviving mutants indicate untested code paths.

Five predefined modes are available:
- **Light mode**: `binary_operator_swap`, `break_continue_replacement`, `delete_statement` (3 operators)
- **Medium mode**: Light + `literal_replacement` (4 operators)
- **Medium-only mode**: `literal_replacement` (1 operator - only what's added in medium)
- **Heavy mode** (default): All 7 available operators
- **Heavy-only mode**: `unary_operator_replacement`, `binary_operator_replacement`, `if_else_replacement` (3 operators - only what's added in heavy)

Users can also specify custom operator sets using the `--operators` CLI option,
providing a comma-separated list of operator names. This allows for fine-grained
control over which operators are applied.

Operator filtering is performed during AST traversal in the `mutate.rs` module.
When a potential mutation site is found, the tool checks if the corresponding
operator is enabled in the current mode before creating the mutant. This approach
prevents unnecessary mutant generation, making the process more efficient.

#### Operator effectiveness analysis

The data below was calculated by running the tool on the largest projects in
[Aptos' Move Framework](https://github.com/aptos-labs/aptos-core/tree/main/aptos-move/framework),
testing 22,597 mutants with an overall kill rate of 82.02%.

The "Kill Rate" column shows the percentage of mutants killed by tests. While a
high kill rate indicates good test coverage, operators with **lower kill rates**
(more surviving mutants) are often **more effective at detecting test gaps**, as
surviving mutants reveal untested code paths. This is valuable for identifying
weaknesses in test suites.

```
╭──────┬─────────────────────────────┬────────┬────────┬───────────────┬───────────╮
│ Rank │ Operator │ Tested │ Killed │ Kill Rate │ Survived │
├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤
│ #1 │ unary_operator_replacement │ 219 │ 219 │ 100.00% │ 0/219 │
├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤
│ #2 │ delete_statement │ 909 │ 895 │ 98.46% │ 14/909 │
├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤
│ #3 │ break_continue_replacement │ 26 │ 23 │ 88.46% │ 3/26 │
├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤
│ #4 │ binary_operator_replacement │ 7081 │ 6207 │ 87.66% │ 874/7081 │
├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤
│ #5 │ if_else_replacement │ 5310 │ 4579 │ 86.23% │ 731/5310 │
├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤
│ #6 │ literal_replacement │ 8781 │ 6498 │ 74.00% │ 2283/8781 │
├──────┼─────────────────────────────┼────────┼────────┼───────────────┼───────────┤
│ #7 │ binary_operator_swap │ 271 │ 114 │ 42.07% │ 157/271 │
╰──────┴─────────────────────────────┴────────┴────────┴───────────────┴───────────╯
```

The predefined operator modes balance speed with test gap detection capability:
- **Light mode**: Operators with lower kill rates that efficiently reveal test gaps (3 operators)
- **Medium mode**: Light + operators that generate more comprehensive test coverage analysis (4 operators)
- **Heavy mode**: All operators for maximum test gap detection (7 operators)

The Move mutator tool implements the following mutation operators.

### Binary operator replacement
Expand Down
30 changes: 29 additions & 1 deletion move-mutator/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,21 @@
// Copyright © Aptos Foundation
// SPDX-License-Identifier: Apache-2.0

use clap::Parser;
use clap::{Parser, ValueEnum};
use std::{path::PathBuf, str::FromStr};

pub const DEFAULT_OUTPUT_DIR: &str = "mutants_output";

/// Mutation operator mode
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum OperatorModeArg {
Light,
Medium,
MediumOnly,
Heavy,
HeavyOnly,
}

/// Command line options for mutator
#[derive(Parser, Debug, Clone)]
pub struct CLIOptions {
Expand Down Expand Up @@ -41,6 +51,22 @@ pub struct CLIOptions {
/// Use the unit test coverage report to generate mutants for source code with unit test coverage.
#[clap(long = "coverage", conflicts_with = "move_sources")]
pub apply_coverage: bool,

/// Mutation operator mode to balance speed and test gap detection.
///
/// - light: binary_operator_swap, break_continue_replacement, delete_statement
/// - medium: light + literal_replacement
/// - medium-only: literal_replacement (only what's added in medium)
/// - heavy (default): all 7 operators
/// - heavy-only: unary_operator_replacement, binary_operator_replacement, if_else_replacement (only what's added in heavy)
#[clap(long, value_enum, conflicts_with = "operators")]
pub mode: Option<OperatorModeArg>,

/// Custom operator selection to run mutations on (comma-separated).
///
/// Available operators: unary_operator_replacement, delete_statement, break_continue_replacement, binary_operator_replacement, if_else_replacement, literal_replacement, binary_operator_swap
#[clap(long, value_parser, value_delimiter = ',', conflicts_with = "mode")]
pub operators: Option<Vec<String>>,
}

/// Checker for conflicts with CLI arguments.
Expand Down Expand Up @@ -82,6 +108,8 @@ impl Default for CLIOptions {
no_overwrite: false,
apply_coverage: false,
downsampling_ratio_percentage: None,
mode: None,
operators: None,
}
}
}
Expand Down
44 changes: 40 additions & 4 deletions move-mutator/src/configuration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@
// Copyright © Aptos Foundation
// SPDX-License-Identifier: Apache-2.0

use crate::{cli::CLIOptions, coverage::Coverage};
use crate::{
cli::{CLIOptions, OperatorModeArg},
coverage::Coverage,
operator_filter::OperatorMode,
};
use std::path::PathBuf;

/// Mutator configuration for the Move project.
Expand All @@ -14,17 +18,49 @@ pub struct Configuration {
pub project_path: Option<PathBuf>,
/// Coverage report where the optional unit test coverage data is stored.
pub(crate) coverage: Coverage,
/// Operator filter that determines which mutation operators are enabled.
pub operator_mode: OperatorMode,
}

impl Configuration {
/// Creates a new configuration using command line options.
#[must_use]
pub fn new(project: CLIOptions, project_path: Option<PathBuf>) -> Self {
Self {
pub fn new(project: CLIOptions, project_path: Option<PathBuf>) -> anyhow::Result<Self> {
// Parse and validate the operator mode from CLI options
let operator_mode = Self::parse_operator_mode(&project)?;

Ok(Self {
project,
project_path,
// Coverage is disabled by default.
coverage: Coverage::default(),
operator_mode,
})
}

fn parse_operator_mode(project: &CLIOptions) -> anyhow::Result<OperatorMode> {
match (&project.mode, &project.operators) {
// --operators specified
(None, Some(operators)) => {
let parsed_ops = OperatorMode::parse_operators(operators)?;
Ok(OperatorMode::Custom(parsed_ops))
},
// --mode specified
(Some(mode_arg), None) => {
let mode = match mode_arg {
OperatorModeArg::Light => OperatorMode::Light,
OperatorModeArg::Medium => OperatorMode::Medium,
OperatorModeArg::MediumOnly => OperatorMode::MediumOnly,
OperatorModeArg::Heavy => OperatorMode::Heavy,
OperatorModeArg::HeavyOnly => OperatorMode::HeavyOnly,
};
Ok(mode)
},
// neither specified
(None, None) => Ok(OperatorMode::default()),
// both specified - this should be prevented by clap conflicts
(Some(_), Some(_)) => {
unreachable!("Both --mode and --operators specified")
},
}
}
}
12 changes: 11 additions & 1 deletion move-mutator/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ pub mod configuration;
pub(crate) mod coverage;
mod mutant;
mod operator;
pub mod operator_filter;
mod operators;
mod output;
pub mod report;
Expand Down Expand Up @@ -74,10 +75,19 @@ pub fn run_move_mutator(
};

let mut mutator_configuration =
Configuration::new(options, Some(original_package_path.to_owned()));
Configuration::new(options, Some(original_package_path.to_owned()))?;

trace!("Mutator configuration: {mutator_configuration:?}");

let enabled_operators = mutator_configuration.operator_mode.get_operators();
println!(
"Operator types being mutated ({}):",
enabled_operators.len()
);
for (i, op) in enabled_operators.iter().enumerate() {
println!(" {}. {}", i + 1, op);
}

let package_path = mutator_configuration
.project_path
.clone()
Expand Down
Loading