From 446344f15568a90b2c289a59275d08690b25a31a Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Wed, 6 Sep 2023 15:54:35 -0700 Subject: [PATCH 01/15] Some readme fixes --- README.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 571063b..ad6fd57 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Gambit is a state-of-the-art mutation system for Solidity. By applying predefined syntax transformations called _mutation operators_ (for - example, `a + b` -> `a - b`) to a Solidity program's source code, Gambit + example, convert `a + b` to `a - b`) to a Solidity program's source code, Gambit generates variants of the program called _mutants_. Mutants can be used to evaluate test suites or specs used for formal verification: each mutant represents a potential bug in the program, and @@ -12,8 +12,8 @@ Mutants can be used to evaluate test suites or specs used for formal 1. Gambit is written in Rust. You'll need to [install Rust and Cargo](https://www.rust-lang.org/tools/install) to build Gambit. -2. Gambit uses the solc, the Solidity compiler, to generate mutants. You'll need - to have solc binary that is compatable with the project you are mutating (see +2. Gambit uses `solc`, the Solidity compiler, to generate mutants. You'll need + to have a `solc` binary that is compatible with the project you are mutating (see the `--solc` option in `gambit mutate --help`) ## Installation @@ -21,7 +21,7 @@ Mutants can be used to evaluate test suites or specs used for formal You can download prebuilt Gambit binaries for Mac and Linux from our [releases](https://github.com/Certora/gambit/releases) page. -To build Gambit from source, clone this repository and run +To build Gambit from source, clone [the Gambit repository](https://github.com/Certora/gambit) and run ``` cargo install --path . @@ -41,10 +41,10 @@ Gambit has two main commands: `mutate` and `summary`. `gambit mutate` is responsible for mutating code, and `gambit summary` is a convenience command for summarizing generated mutants in a human-readable way. -Running `gambit mutate` will invoke the solidity compiler via `solc`, so make +Running `gambit mutate` will invoke `solc`, so make sure it is visible on your `PATH`. Alternatively, you can specify where Gambit can find the Solidity compiler with the option `--solc path/to/solc`, or specify a -version of solc (e.g., solc8.12) with the option `--solc solc8.12`. +`solc` binary (e.g., `solc8.12`) with the option `--solc solc8.12`. _**Note:** All tests (`cargo test`) are currently run using solc8.13. Your tests may fail if your `solc` points at a different version of the compiler._ @@ -65,7 +65,7 @@ instead: ```bash gambit mutate --json gambit_conf.json -``` +``` Run `gambit --help` for more information. @@ -78,25 +78,27 @@ In the following section we provide examples of how to run Gambit using both ## Examples -Unless otherwise noted, examples use code from `benchmarks/` and are run from -the root of this repository. +Unless otherwise noted, examples use code from [benchmarks/](https://github.com/Certora/gambit/tree/master/benchmarks) +and are run from the root of the [Gambit repository](https://github.com/Certora/gambit). -### Example 1: Mutating a Single File +### Example 1: Mutating a single file To mutate a single file, use the `--filename` option (or `-f`), followed by the file to mutate. ```bash -gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol +gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol ``` 
 Generated 34 mutants in 0.69 seconds
-_**Note:** The mutated file must located within your current working directory or +_**Note:** +The mutated file must located within your current working directory or one of its subdirectories. If you want to mutate code in an arbitrary directory, -use the `--sourceroot` option._ +use the `--sourceroot` option. +_ ### Example 2: Mutating and Downsampling @@ -417,8 +419,8 @@ passed directly to solc. All pass-through arguments are prefixed with `solc-`: | :-------------------- | :---------------------------------------------------------------------------- | | `--solc_base_path` | passes a value to solc's `--base-path` argument | | `--solc_allow_paths` | passes a value to solc's `--allow-paths` argument | -| `--solc_include_path` | passes a value to solc's `--include-path` argument | -| `--solc_remappings` | passes a value to directly to solc: this should be of the form `prefix=path`. | +| `--solc_include_path` | passes a value to solc's `--include-path` argument | +| `--solc_remappings` | passes a value to directly to solc: this should be of the form `prefix=path`. | ## Mutation Operators Gambit implements the following mutation operators From b2b0fa61eeb5791a0aa5847e0b77de276773a6fa Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Wed, 6 Sep 2023 16:48:00 -0700 Subject: [PATCH 02/15] Updated README --- README.md | 116 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index ad6fd57..0ce381b 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,10 @@ gambit mutate --json gambit_conf.json Run `gambit --help` for more information. -_**Note:** all relative paths specified in a JSON configuration file are interpreted -to be relative to the config file's parent directory._ +_**Note:** +All relative paths specified in a JSON configuration file are interpreted +to be relative to the configuration file's parent directory. +_ In the following section we provide examples of how to run Gambit using both `--filename` and `--json`. We provide more complete documentation in the @@ -95,12 +97,12 @@ Generated 34 mutants in 0.69 seconds _**Note:** -The mutated file must located within your current working directory or +The mutated file must be located within your current working directory or one of its subdirectories. If you want to mutate code in an arbitrary directory, use the `--sourceroot` option. _ -### Example 2: Mutating and Downsampling +### Example 2: Mutating and downsampling The above command produced 34 mutants which may be more than you need. Gambit provides a way to randomly downsample the number of mutants with the @@ -113,7 +115,7 @@ gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol -n 3 Generated 3 mutants in 0.15 seconds -### Example 3: Viewing Gambit Results +### Example 3: Viewing Gambit results _**Note:** this example assumes you've just completed Example 2_ Gambit outputs all of its results in `gambit_out`: @@ -147,45 +149,45 @@ For instance, `gambit summary --mids 3 4 5` will only print info for mutant ids 3 through 5. -### Example 4: Specifying solc Pass-through Arguments -Solc may need some extra information to successfully run on a file or a project. -Gambit enables this with _pass-through arguments_ that, as the name suggests, -are passed directly through to the solc compiler. +### Example 4: Specifying solc pass-through arguments +The Solidity compiler (`solc`) may need some extra information to successfully +run on a file or a project. Gambit enables this with _pass-through arguments_ +that, as the name suggests, are passed directly through to the `solc` compiler. For projects that have complex dependencies and imports, you may need to: -* **Specify base-paths**: To specify the Solidity [--base-path][basepath] +* **Specify base paths**: To specify the Solidity [`--base-path`][basepath] argument, use `--solc_base_path`: ```bash - gambit mutate --filename path/to/file.sol --solc_base_path base/path/dir/. + gambit mutate --filename path/to/file.sol --solc_base_path base/path/dir ``` * **Specify remappings:** To indicate where Solidity should find libraries, - use solc's [import remapping][remapping] syntax with `--solc_remappings`: + use `solc`'s [import remapping][remapping] syntax with `--solc_remappings`: ```bash gambit mutate --filename path/to/file.sol \ --solc_remappings @openzepplin=node_modules/@openzeppelin @foo=node_modules/@foo ``` -* **Specify allow-paths:** To include additional allowed paths via solc's - [--allow-paths][allowed] argument, use `--solc_allow_paths`: +* **Specify allow paths:** To include additional allowed paths via `solc`'s + [`--allow-paths`][allowed] argument, use `--solc_allow_paths`: ```bash gambit mutate --filename path/to/file.sol \ - --solc_allow_paths PATH1 --solc_allow_paths PATH2 ... + --solc_allow_paths PATH1 --solc_allow_paths PATH2 ... ``` * **Specify include-path:** To make an additional source directory available - to the default import callback via solc's [--include-path][included] argument, use - `--solc_include_path`: + to the default import callback via `solc`'s [--include-path][included] argument, + use `--solc_include_path`: ```bash gambit mutate --filename path/to/file.sol --solc_include_path PATH ``` -* **Use optimization:** To run the solidity compiler with optimizations (solc's - `--optimize` argument), use `--solc_optimize`: +* **Use optimization:** To run the solidity compiler with optimizations + (`solc`'s `--optimize` argument), use `--solc_optimize`: ```bash gambit mutate --filename path/to/file.sol --solc_optimize @@ -195,22 +197,26 @@ For projects that have complex dependencies and imports, you may need to: [basepath]: https://docs.soliditylang.org/en/v0.8.17/path-resolution.html#base-path-and-include-paths [allowed]: https://docs.soliditylang.org/en/v0.8.17/path-resolution.html#allowed-paths -### Example 5: The `--sourceroot` Option -Gambit needs to track the location of sourcefiles that it mutates within a + +### Example 5: The `--sourceroot` option + +Gambit needs to track the location of source files that it mutates within a project: for instance, imagine there are files `foo/Foo.sol` and `bar/Foo.sol`. These are separate files, and their path prefixes are needed to determine this. -Gambit addresses this with the `--sourceroot` option: the sourceroot indicates +Gambit addresses this with the `--sourceroot` option: the source root indicates to Gambit the root of the files that are being mutated, and all source file -paths (both original and mutated) are reported relative to this sourceroot. +paths (both original and mutated) are reported relative to this source root. -_If Gambit encounters a source file that does not belong to the sourceroot it -will print an error message and exit._ +_**Note:** +If Gambit encounters a source file that does not belong to the source root it +will print an error message and exit. +_ _When running `gambit mutate` with the `--filename` option, -sourceroot defaults to the current working directory. +source root defaults to the current working directory. When running `gambit mutate` with the `--json` option, -sourceroot defaults to the directory containing the configuration JSON._ +source root defaults to the directory containing the configuration JSON._ Here are some examples of using the `--sourceroot` option. @@ -218,7 +224,7 @@ Here are some examples of using the `--sourceroot` option. ```bash gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol -n 1 - cat gambit_out/mutants.log + cat gambit_out/mutants.log find gambit_out/mutants -name "*.sol" ``` @@ -230,17 +236,17 @@ Here are some examples of using the `--sourceroot` option. gambit_out/mutants/1/benchmarks/BinaryOpMutation/BinaryOpMutation.sol - The first command generates a single mutant, and its sourcepath is relative to `.`, - the default sourceroot. We can see that the reported paths in `mutants.log`, + The first command generates a single mutant, and its source path is relative to `.`, + the default source root. We can see that the reported paths in `mutants.log`, and the mutant file path in `gambit_out/mutants/1`, are the relative to this - sourceroot: `benchmarks/BinaryOpMutation/BinaryOpMutation.sol` - + source root: `benchmarks/BinaryOpMutation/BinaryOpMutation.sol` + 2. Suppose we want our paths to be reported relative to `benchmarks/BinaryOpMutation`. We can run ```bash gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol -n 1 --sourceroot benchmarks/BinaryOpMutation - cat gambit_out/mutants.log + cat gambit_out/mutants.log find gambit_out/mutants -name "*.sol" ``` @@ -254,10 +260,10 @@ Here are some examples of using the `--sourceroot` option. The reported filenames, and the offset path inside of - `gambit_out/mutants/1/`, are now relative to the sourceroot that we + `gambit_out/mutants/1/`, are now relative to the source root that we specified. -3. Finally, suppose we use a sourceroot that doesn't contain the source file: +3. Finally, suppose we use a source root that doesn't contain the source file: ```bash gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol -n 1 --sourceroot scripts @@ -267,12 +273,12 @@ Here are some examples of using the `--sourceroot` option. 
-   [ERROR gambit] [!!] Illegal Configuration: Resolved filename `/Users/USER/Gambit/benchmarks/BinaryOpMutation/BinaryOpMutation.sol` is not prefixed by the derived sourceroot /Users/USER/Gambit/scripts
+   [ERROR gambit] [!!] Illegal Configuration: Resolved filename `/Users/USER/Gambit/benchmarks/BinaryOpMutation/BinaryOpMutation.sol` is not prefixed by the derived source root /Users/USER/Gambit/scripts
Gambit prints an error and exits. -### Example 6: Running Gambit Through a Configuration File +### Example 6: Running Gambit using a configuration file To run gambit with a configuration file, use the `--json` argument: ```bash @@ -297,16 +303,19 @@ mutants that you want to apply, the specific functions you wish to mutate, and more. See the [`benchmark/config-jsons` directory][config-examples] for examples. -_**Note:** Any paths provided by the configuration file are resolved relative to -the configuration file's parent directory._ +_**Note:** +Any paths provided by the configuration file are resolved relative to the +configuration file's parent directory. +_ + ## Configuration Files Configuration files allow you to save complex configurations and perform multiple mutations at once. Gambit uses a simple JSON object format to store mutation options, where each `--option VALUE` specified on the CLI is represented as a `"option": VALUE` key/value pair in the JSON object. Boolean `--flag`s are enabled by storing them as true: `"flag": true`. For instance, -`--no_overwrite` would be written as `"no_overwrite": true"`. +`--no_overwrite` would be written as `"no_overwrite": true`. As an example, consider the command from Example 1: @@ -361,6 +370,8 @@ directory of the configuration file_. So if the JSON file listed above was moved to the `benchmarks/` directory the `"filename"` would need to be updated to `BinaryOpMutation/BinaryOpMutation.sol`. + + ## Results Directory `gambit mutate` produces all results in an output directory (default: @@ -395,6 +406,7 @@ This has the following structure: + `mutants.log`: a log file with all mutant information. This is similar to `results.json` but in a different format and with different information + ## CLI Options `gambit mutate` supports the following options; for a comprehensive list, run @@ -406,21 +418,23 @@ This has the following structure: | `-o`, `--outdir` | specify Gambit's output directory (defaults to `gambit_out`) | | `--no_overwrite` | do not overwrite an output directory; if the output directory exists, print an error and exit | | `-n`, `--num_mutants` | randomly downsample to a given number of mutants. | -| `-s`, `--seed` | specify a random seed. For reproducability, Gambit defaults to using the seed `0`. To randomize the seed use `--random-seed` | -| `--random_seed` | use a random seed. Note this overrides any value specified by `--seed` | +| `-s`, `--seed` | specify a random seed. For reproducibility, Gambit defaults to using the seed `0`. To randomize the seed use `--random_seed` | +| `--random_seed` | use a random seed. Note that this overrides any value specified by `--seed` | | `--contract` | specify a specific contract name to mutate; by default mutate all contracts | | `--functions` | specify one or more functions to mutate; by default mutate all functions | | `--mutations` | specify one or more mutation operators to use; only generates mutants that are created using the specified operators | +| `--skip_validate` | only generate mutants without validating them by compilation | Gambit also supports _pass-through arguments_, which are arguments that are -passed directly to solc. All pass-through arguments are prefixed with `solc-`: +passed directly to the solidity compiler. All pass-through arguments are +prefixed with `solc_`: -| Option | Description | -| :-------------------- | :---------------------------------------------------------------------------- | -| `--solc_base_path` | passes a value to solc's `--base-path` argument | -| `--solc_allow_paths` | passes a value to solc's `--allow-paths` argument | -| `--solc_include_path` | passes a value to solc's `--include-path` argument | -| `--solc_remappings` | passes a value to directly to solc: this should be of the form `prefix=path`. | +| Option | Description | +| :-------------------- | :------------------------------------------------------------------------------ | +| `--solc_base_path` | passes a value to `solc`'s `--base-path` argument | +| `--solc_include_path` | passes a value to `solc`'s `--include-path` argument | +| `--solc_remappings` | passes a value to directly to `solc`: this should be of the form `prefix=path`. | +| `--solc_allow_paths` | passes a value to `solc`'s `--allow-paths` argument | ## Mutation Operators Gambit implements the following mutation operators @@ -430,8 +444,8 @@ Gambit implements the following mutation operators | **binary-op-mutation** | Replace a binary operator with another | `a+b` -> `a-b` | | **unary-operator-mutation** | Replace a unary operator with another | `~a` -> `-a` | | **require-mutation** | Alter the condition of a `require` statement | `require(some_condition())` -> `require(true)` | -| **assignment-mutation** | Replaces the rhs of a mutation | `x = foo();` -> `x = -1;` | -| **delete-expression-mutation** | Replace an expression statement with `assert(true)` | `foo();` -> `assert(true);` | +| **assignment-mutation** | Replaces the right hand side of an assignment | `x = foo();` -> `x = -1;` | +| **delete-expression-mutation** | Replaces an expression with a no-op (`assert(true)`) | `foo();` -> `assert(true);` | | **if-cond-mutation** | Mutate the conditional of an `if` statement | `if (C) {...}` -> `if (true) {...}` | | **swap-arguments-operator-mutation** | Swap the order of non-commutative operators | `a - b` -> `b - a` | | **elim-delegate-mutation** | Change a `delegatecall()` to a `call()` | `_c.delegatecall(...)` -> `_c.call(...)` | From 9acc90e804845cf4bebab65dad3ed832c9a98f70 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Wed, 6 Sep 2023 18:20:28 -0700 Subject: [PATCH 03/15] Updated readme --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0ce381b..6c42983 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,10 @@ sure it is visible on your `PATH`. Alternatively, you can specify where Gambit c find the Solidity compiler with the option `--solc path/to/solc`, or specify a `solc` binary (e.g., `solc8.12`) with the option `--solc solc8.12`. -_**Note:** All tests (`cargo test`) are currently run using solc8.13. Your tests may fail if your `solc` points at - a different version of the compiler._ +_**Note:** +All tests (`cargo test`) are currently run using solc8.13. Your tests may fail if your `solc` points at + a different version of the compiler. + _ ### Running `gambit mutate` @@ -116,7 +118,9 @@ Generated 3 mutants in 0.15 seconds ### Example 3: Viewing Gambit results -_**Note:** this example assumes you've just completed Example 2_ +_**Note:** +This example assumes you've just completed Example 2. +_ Gambit outputs all of its results in `gambit_out`: @@ -371,7 +375,6 @@ to the `benchmarks/` directory the `"filename"` would need to be updated to `BinaryOpMutation/BinaryOpMutation.sol`. - ## Results Directory `gambit mutate` produces all results in an output directory (default: From 06638e571c7a175d433237033375469e37abfdd8 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Wed, 6 Sep 2023 18:24:41 -0700 Subject: [PATCH 04/15] Added a script to generate rtd docs --- scripts/generate_rtd_markdown.py | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 scripts/generate_rtd_markdown.py diff --git a/scripts/generate_rtd_markdown.py b/scripts/generate_rtd_markdown.py new file mode 100644 index 0000000..ef8b2ab --- /dev/null +++ b/scripts/generate_rtd_markdown.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +""" +Generate RTD version of the Gambit README +""" + +from argparse import ArgumentParser +from typing import Optional +import re +import hashlib + + +def line_is_anchor(line: str) -> bool: + return line.startswith("") + '(test-anchor)=' + >>> get_anchor("# README") + """ + + if not line.startswith("")].strip() + return anchor + + +def translate(readme_file_path: str) -> str: + with open(readme_file_path) as f: + original = f.read() + lines = original.split("\n") + lines2 = [] + + note_start = -1 # Track if we've started a note + for i, line in enumerate(lines): + anchor = get_anchor(line) + if anchor is not None: + lines2.append(anchor) + elif "_**note:**" == line.strip().lower(): + if note_start > -1: + raise RuntimeError( + f"Already in note from line {note_start + 1}, cannot start new note on line {i+1}" + ) + note_start = i + lines2.append("```{note}") + elif "_**note:**" in line.strip().lower(): + raise RuntimeError( + f"Illegal note start on line {i+1}: new note tags '_**Note:**' and their closing '_' must be on their own lines" + ) + + elif note_start > -1 and line.strip() == "_": + note_start = -1 + lines2.append("```") + else: + # replace internal links + l = replace_internal_references(line) + lines2.append(l.strip("\n")) + signature = hashlib.md5(original.encode()).hexdigest() + lines2.append(f"") + return "\n".join(lines2) + "\n" + + +def main(): + parser = ArgumentParser() + parser.add_argument("readme_file", help="README.md file to translate to RTD") + parser.add_argument("--output", "-o", default="gambit.md", help="output file") + + args = parser.parse_args() + rtd = translate(args.readme_file) + with open(args.output, "w+") as f: + print("Writing to", args.output) + f.write(rtd) + + +if __name__ == "__main__": + main() From 53f162717af02fbb8b0cc9ca9f628ed6f3f3065a Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Thu, 7 Sep 2023 15:38:48 -0700 Subject: [PATCH 05/15] Updated CI --- .github/workflows/gambit.yml | 24 +++++++++++++++++++++++- .gitignore | 3 +++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 5d05820..ad4833d 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -130,4 +130,26 @@ jobs: else gh release upload $TAG gambit-linux-$TAG/gambit-linux-$TAG gh release upload $TAG gambit-macos-$TAG/gambit-macos-$TAG - fi \ No newline at end of file + fi + +name: Check RTD Docs + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Check that RTD Docs are Up To Date + run: python3 scripts/check_rtd_docs_up_to_date.py + + - name: Check Exit Code + run: | + if [[ $? -ne 0 ]]; then + echo "Error: documentation is not synced" + exit 1 + fi + diff --git a/.gitignore b/.gitignore index 0bfcc3a..5463df2 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ Cargo.lock # vscode .vscode/ + +# python +**/__pycache__/ From 5d40c544f3f2ab15a7b6934b14d58eceeced3486 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Thu, 7 Sep 2023 15:41:16 -0700 Subject: [PATCH 06/15] Updated README and added scripts for synching w/ RTD --- README.md | 154 ++++++++++++++++++++++++++- scripts/check_rtd_docs_up_to_date.py | 83 +++++++++++++++ scripts/generate_rtd_markdown.py | 59 +++++++++- 3 files changed, 290 insertions(+), 6 deletions(-) create mode 100644 scripts/check_rtd_docs_up_to_date.py diff --git a/README.md b/README.md index 6c42983..7e82fe1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,149 @@ # Gambit: Mutant Generation for Solidity + + + + + + + + Gambit is a state-of-the-art mutation system for Solidity. By applying predefined syntax transformations called _mutation operators_ (for example, convert `a + b` to `a - b`) to a Solidity program's source code, Gambit @@ -47,9 +191,9 @@ find the Solidity compiler with the option `--solc path/to/solc`, or specify a `solc` binary (e.g., `solc8.12`) with the option `--solc solc8.12`. _**Note:** -All tests (`cargo test`) are currently run using solc8.13. Your tests may fail if your `solc` points at - a different version of the compiler. - _ +All tests (`cargo test`) are currently run using `solc8.13`. Your tests may fail + if your `solc` points at a different version of the compiler. +_ ### Running `gambit mutate` @@ -153,7 +297,7 @@ For instance, `gambit summary --mids 3 4 5` will only print info for mutant ids 3 through 5. -### Example 4: Specifying solc pass-through arguments +### Example 4: Specifying `solc` pass-through arguments The Solidity compiler (`solc`) may need some extra information to successfully run on a file or a project. Gambit enables this with _pass-through arguments_ that, as the name suggests, are passed directly through to the `solc` compiler. @@ -457,6 +601,7 @@ Gambit implements the following mutation operators For more details on each mutation type, refer to the [full documentation](https://docs.certora.com/en/latest/docs/gambit/gambit.html#mutation-types). + ## Contact If you have ideas for interesting mutations or other features, we encourage you to make a PR or [email](mailto:chandra@certora.com) us. @@ -467,6 +612,7 @@ We thank [Vishal Canumalla](https://homes.cs.washington.edu/~vishalc/) for their excellent contributions to an earlier prototype of Gambit. + [config-examples]: https://github.com/Certora/gambit/blob/master/benchmarks/config-jsons/ [test6]: https://github.com/Certora/gambit/blob/master/benchmarks/config-jsons/test6.json diff --git a/scripts/check_rtd_docs_up_to_date.py b/scripts/check_rtd_docs_up_to_date.py new file mode 100644 index 0000000..0f15236 --- /dev/null +++ b/scripts/check_rtd_docs_up_to_date.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +from argparse import ArgumentParser +from urllib.request import urlopen +import difflib +import hashlib +import os +import sys +import re +from generate_rtd_markdown import translate_readme_to_rtd + +OUR_README_PATH = os.path.join(os.path.dirname(__file__), "..", "README.md") + +THEIR_README_URL_NO_BRANCH = ( + "https://raw.githubusercontent.com/Certora/Documentation/{}/docs/gambit/gambit.md" +) + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--branch", default="master", help="Branch to check README from" + ) + args = parser.parse_args() + + exit_code = check_rtd_docs_up_to_date(branch=args.branch) + + sys.exit(exit_code) + + +def find_signature(contents: str) -> str: + pattern = r"" + m = re.search(pattern, contents) + if m is None: + return None + return m.group(1) + + +def check_rtd_docs_up_to_date(branch="master") -> int: + url = THEIR_README_URL_NO_BRANCH.format(branch) + + with open(OUR_README_PATH) as f: + our_readme_contents = f.read() + + our_md5 = hashlib.md5(our_readme_contents.encode()).hexdigest() + + try: + their_readme_contents = urlopen(url).read() + their_md5 = find_signature(their_readme_contents.decode("utf-8")) + + except RuntimeError as e: + print(f"Could not read `gambit.md` from {url}") + print(f"Error: {e}") + return 127 + + print("local md5: ", our_md5) + print("remote md5:", their_md5) + print() + if our_md5 == their_md5: + print(f"MD5 Hashes Match: Documentation is synced") + return 0 + else: + print(f"MD5 Hashes Do Not Match!") + print() + our_translated_readme_contents = translate_readme_to_rtd(OUR_README_PATH) + print("Unified diff: Local vs Remote") + print("=============================") + print() + print( + "".join( + difflib.unified_diff( + our_translated_readme_contents.splitlines(keepends=True), + str(their_readme_contents.decode("utf-8")).splitlines( + keepends=True + ), + ) + ) + ) + return 1 + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_rtd_markdown.py b/scripts/generate_rtd_markdown.py index ef8b2ab..4cf069f 100644 --- a/scripts/generate_rtd_markdown.py +++ b/scripts/generate_rtd_markdown.py @@ -56,14 +56,42 @@ def get_anchor(line: str) -> Optional[str]: return anchor -def translate(readme_file_path: str) -> str: +def is_suppress(line: str) -> bool: + return "" == line.upper().replace(" ", "").strip() + + +def is_end_suppress(line: str) -> bool: + return "" == line.upper().replace(" ", "").strip() + + +def is_emit(line: str) -> bool: + return "" and emit_start > -1: + emit_start = -1 + + # Handle escaped comments from inside of an emit + elif is_escaped_open_comment(line) and emit_start > -1: + lines2.append("") + + elif is_suppress(line): + if suppress_start > 0: + raise RuntimeError( + f"Cannot start a new suppression on line {i+1}: already in a suppression tag started at line {suppress_start+1}" + ) + suppress_start = i + elif is_end_suppress(line): + raise RuntimeError( + f"Illegal end suppress on line {i+1}: not currently in a suppress" + ) else: # replace internal links l = replace_internal_references(line) @@ -97,7 +152,7 @@ def main(): parser.add_argument("--output", "-o", default="gambit.md", help="output file") args = parser.parse_args() - rtd = translate(args.readme_file) + rtd = translate_readme_to_rtd(args.readme_file) with open(args.output, "w+") as f: print("Writing to", args.output) f.write(rtd) From dec736d3842e554631e10b326668a203f543a2f4 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Thu, 7 Sep 2023 15:44:32 -0700 Subject: [PATCH 07/15] Fix ci --- .github/workflows/gambit.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index ad4833d..8dc69c4 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -132,12 +132,7 @@ jobs: gh release upload $TAG gambit-macos-$TAG/gambit-macos-$TAG fi -name: Check RTD Docs - -on: [push] - -jobs: - test: + check-docs: runs-on: ubuntu-latest steps: - name: Checkout code From fffccee5044b1e0caf5f0b5e0519a1fbf567f59c Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Thu, 7 Sep 2023 15:46:36 -0700 Subject: [PATCH 08/15] Testing c --- .github/workflows/gambit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 8dc69c4..f508e6c 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -139,7 +139,7 @@ jobs: uses: actions/checkout@v2 - name: Check that RTD Docs are Up To Date - run: python3 scripts/check_rtd_docs_up_to_date.py + run: python3 scripts/check_rtd_docs_up_to_date.py --branch bkushigian/fix_gambit_readme - name: Check Exit Code run: | From a17fe2751c971d6f382057b656bc41a2b1dbd4c9 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Thu, 7 Sep 2023 16:30:03 -0700 Subject: [PATCH 09/15] Updated CI to point documentation checking to master --- .github/workflows/gambit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index f508e6c..8dc69c4 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -139,7 +139,7 @@ jobs: uses: actions/checkout@v2 - name: Check that RTD Docs are Up To Date - run: python3 scripts/check_rtd_docs_up_to_date.py --branch bkushigian/fix_gambit_readme + run: python3 scripts/check_rtd_docs_up_to_date.py - name: Check Exit Code run: | From 852a6b9290874ba4060602060de2fe9cb4040aa6 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Sat, 9 Sep 2023 14:53:35 -0700 Subject: [PATCH 10/15] Improved doc checks --- scripts/check_rtd_docs_up_to_date.py | 61 ++++++++++++++++------------ scripts/generate_rtd_markdown.py | 16 ++++++-- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/scripts/check_rtd_docs_up_to_date.py b/scripts/check_rtd_docs_up_to_date.py index 0f15236..da0158b 100644 --- a/scripts/check_rtd_docs_up_to_date.py +++ b/scripts/check_rtd_docs_up_to_date.py @@ -3,7 +3,6 @@ from argparse import ArgumentParser from urllib.request import urlopen import difflib -import hashlib import os import sys import re @@ -21,60 +20,68 @@ def main(): parser.add_argument( "--branch", default="master", help="Branch to check README from" ) + parser.add_argument( + "--no_colors", action="store_true", help="Do not use ansi color on outputs" + ) args = parser.parse_args() - exit_code = check_rtd_docs_up_to_date(branch=args.branch) + exit_code = check_rtd_docs_up_to_date(branch=args.branch, colors=not args.no_colors) sys.exit(exit_code) -def find_signature(contents: str) -> str: - pattern = r"" - m = re.search(pattern, contents) - if m is None: - return None - return m.group(1) +def print_unified_diff(diff, colors=True): + color_fn = {} + if colors: + try: + from ansi.color import fg + + color_fn = {"+": fg.green, "-": fg.red, "@": fg.blue} + except ImportError: + colors = False + for line in diff: + l: str = line + if colors: + if l.startswith("+++") or l.startswith("---"): + l = fg.yellow(l) + else: + f = color_fn.get(l[0], str) + l = f(l) + print(l, end="") -def check_rtd_docs_up_to_date(branch="master") -> int: + +def check_rtd_docs_up_to_date(branch="master", colors=True) -> int: url = THEIR_README_URL_NO_BRANCH.format(branch) with open(OUR_README_PATH) as f: our_readme_contents = f.read() - our_md5 = hashlib.md5(our_readme_contents.encode()).hexdigest() - try: - their_readme_contents = urlopen(url).read() - their_md5 = find_signature(their_readme_contents.decode("utf-8")) + their_readme_contents = urlopen(url).read().decode("utf-8") except RuntimeError as e: print(f"Could not read `gambit.md` from {url}") print(f"Error: {e}") return 127 - print("local md5: ", our_md5) - print("remote md5:", their_md5) print() - if our_md5 == their_md5: - print(f"MD5 Hashes Match: Documentation is synced") + if our_readme_contents == their_readme_contents: + print(f"Docs are in sync!") return 0 else: - print(f"MD5 Hashes Do Not Match!") + print(f"Docs are out of sync!") print() our_translated_readme_contents = translate_readme_to_rtd(OUR_README_PATH) print("Unified diff: Local vs Remote") print("=============================") print() - print( - "".join( - difflib.unified_diff( - our_translated_readme_contents.splitlines(keepends=True), - str(their_readme_contents.decode("utf-8")).splitlines( - keepends=True - ), - ) - ) + print_unified_diff( + difflib.unified_diff( + our_translated_readme_contents.splitlines(keepends=True), + their_readme_contents.splitlines(keepends=True), + ), + colors=colors, ) return 1 diff --git a/scripts/generate_rtd_markdown.py b/scripts/generate_rtd_markdown.py index 4cf069f..5f0747f 100644 --- a/scripts/generate_rtd_markdown.py +++ b/scripts/generate_rtd_markdown.py @@ -7,7 +7,6 @@ from argparse import ArgumentParser from typing import Optional import re -import hashlib def line_is_anchor(line: str) -> bool: @@ -72,6 +71,16 @@ def is_escaped_open_comment(line: str) -> bool: return line.strip() == r"<\!--" +def is_note_end(line: str) -> bool: + """ + A note ends when a line is ended by an underscore. We double check to ensure + that the line doesn't end with two underscores. + """ + l = line.strip() + if l.endswith("_"): + return len(l) == 1 or l[-2] != "_" + + def is_escaped_closed_comment(line: str) -> bool: return line.strip() == r"--\>" @@ -107,8 +116,9 @@ def translate_readme_to_rtd(readme_file_path: str) -> str: f"Illegal note start on line {i+1}: new note tags '_**Note:**' and their closing '_' must be on their own lines" ) - elif note_start > -1 and line.strip() == "_": + elif is_note_end(line): note_start = -1 + lines2.append(line.rstrip("\n").rstrip("_")) lines2.append("```") elif is_emit(line): @@ -141,8 +151,6 @@ def translate_readme_to_rtd(readme_file_path: str) -> str: # replace internal links l = replace_internal_references(line) lines2.append(l.strip("\n")) - signature = hashlib.md5(original.encode()).hexdigest() - lines2.append(f"") return "\n".join(lines2) + "\n" From e4b8b7380f73a2478079a03733f7560fc4612509 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Sat, 9 Sep 2023 14:56:06 -0700 Subject: [PATCH 11/15] workflow --- .github/workflows/gambit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 8dc69c4..346092d 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -138,6 +138,9 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + - name: PIP install + run: pip install ansi + - name: Check that RTD Docs are Up To Date run: python3 scripts/check_rtd_docs_up_to_date.py From 51623b6bafa02050726b958c44657af5ab211c9d Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Sat, 9 Sep 2023 15:11:30 -0700 Subject: [PATCH 12/15] Fixes --- README.md | 53 ++++++++++++++------------------ scripts/generate_rtd_markdown.py | 4 +-- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5b82e3a..7e1d597 100644 --- a/README.md +++ b/README.md @@ -31,28 +31,26 @@ 3. Create a new PR in https://github.com/Certora/Documentation with the new Gambit docs + 4. Create a new PR in Gambit repo. Note that CI will check that the RTD + documentation is up to date (see section "Checking RTDs are Up To Date" + below). If this fails, CI will also fail, and you will be unable to merge + into `master` until changes to the Gambit README are propagated to the RTD + docs. + + 5. Once the PR from (3) is merged and CI is passing in this repository, merge + the PR from (4) into master. ## Checking RTDs are Up To Date - - In addition to translating this document to the RTD format, - `generate_rtd_markdown.py` also adds the md5 checksum of the original - `README.md` contents to an HTML comment in the translated `gambit.md`: - - ``` - <\!-- signature: CHECKSUM --\> - ``` - - You can check to ensure that the current version of `docs/gambit/gambit.md` in - the Certora Documentation repo is up to date with the version of the - `README.md` in your working tree by running - - To check that this file and RTD Gambit docs are in sync, run: + To check that the RTD Gambit docs are in sync with Gambit's README, run ``` python scripts/check_rtd_docs_upt_to_date.py ``` + + This will translate the Gambit README to a string, pull the RTD docs from the + Github Repo, and do a equality check on the two strings. You can optionally specify a `--branch` argument to choose another branch in the Certora Documentation repo (default is `'master'`) @@ -109,17 +107,17 @@ Some note goes here ``` - We don't have access tho this here, so I've implemented a simple system, + We don't have access to this here, so I've implemented a simple system, where all notes begin with a line containing: ```markdown _**Note:** ``` - and end with a line containing only: + and end with a line ending with `_`: ```markdown - _ + and this is the last line of my note._ ``` So, a full note would look like: @@ -127,7 +125,8 @@ ```markdown _**Note:** This is a note. The opening tag is on its own line, and the closing italic - is on its own line. This is to make parsing easy, and to keep diffs minimal! + is at the end of the final line. This is to make parsing easy, and to keep + diffs minimal!_ ``` --> @@ -192,8 +191,7 @@ find the Solidity compiler with the option `--solc path/to/solc`, or specify a _**Note:** All tests (`cargo test`) are currently run using `solc8.13`. Your tests may fail - if your `solc` points at a different version of the compiler. -_ + if your `solc` points at a different version of the compiler._ ### Running `gambit mutate` @@ -217,8 +215,7 @@ Run `gambit --help` for more information. _**Note:** All relative paths specified in a JSON configuration file are interpreted -to be relative to the configuration file's parent directory. -_ +to be relative to the configuration file's parent directory._ In the following section we provide examples of how to run Gambit using both `--filename` and `--json`. We provide more complete documentation in the @@ -245,8 +242,7 @@ Generated 34 mutants in 0.69 seconds _**Note:** The mutated file must be located within your current working directory or one of its subdirectories. If you want to mutate code in an arbitrary directory, -use the `--sourceroot` option. -_ +use the `--sourceroot` option._ ### Example 2: Mutating and downsampling @@ -263,8 +259,7 @@ Generated 3 mutants in 0.15 seconds ### Example 3: Viewing Gambit results _**Note:** -This example assumes you've just completed Example 2. -_ +This example assumes you've just completed Example 2._ Gambit outputs all of its results in `gambit_out`: @@ -358,8 +353,7 @@ paths (both original and mutated) are reported relative to this source root. _**Note:** If Gambit encounters a source file that does not belong to the source root it -will print an error message and exit. -_ +will print an error message and exit._ _When running `gambit mutate` with the `--filename` option, source root defaults to the current working directory. @@ -453,8 +447,7 @@ examples. _**Note:** Any paths provided by the configuration file are resolved relative to the -configuration file's parent directory. -_ +configuration file's parent directory._ ## Configuration Files diff --git a/scripts/generate_rtd_markdown.py b/scripts/generate_rtd_markdown.py index 5f0747f..366e1a0 100644 --- a/scripts/generate_rtd_markdown.py +++ b/scripts/generate_rtd_markdown.py @@ -116,9 +116,9 @@ def translate_readme_to_rtd(readme_file_path: str) -> str: f"Illegal note start on line {i+1}: new note tags '_**Note:**' and their closing '_' must be on their own lines" ) - elif is_note_end(line): + elif note_start > -1 and is_note_end(line): note_start = -1 - lines2.append(line.rstrip("\n").rstrip("_")) + lines2.append(line.rstrip().rstrip("_")) lines2.append("```") elif is_emit(line): From 44831a93f2032704dbebe3b13862636ecbb4d924 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Sat, 9 Sep 2023 15:19:23 -0700 Subject: [PATCH 13/15] Fix bug --- scripts/check_rtd_docs_up_to_date.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check_rtd_docs_up_to_date.py b/scripts/check_rtd_docs_up_to_date.py index da0158b..1ea5545 100644 --- a/scripts/check_rtd_docs_up_to_date.py +++ b/scripts/check_rtd_docs_up_to_date.py @@ -66,13 +66,13 @@ def check_rtd_docs_up_to_date(branch="master", colors=True) -> int: return 127 print() - if our_readme_contents == their_readme_contents: + our_translated_readme_contents = translate_readme_to_rtd(OUR_README_PATH) + if our_translated_readme_contents == their_readme_contents: print(f"Docs are in sync!") return 0 else: print(f"Docs are out of sync!") print() - our_translated_readme_contents = translate_readme_to_rtd(OUR_README_PATH) print("Unified diff: Local vs Remote") print("=============================") print() From 05247e5ebe3b33e64deccdfab8927ea066982689 Mon Sep 17 00:00:00 2001 From: Ben Kushigian Date: Sun, 10 Sep 2023 13:34:51 -0700 Subject: [PATCH 14/15] Updated README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7e1d597..6487d95 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,7 @@ For projects that have complex dependencies and imports, you may need to: gambit mutate --filename path/to/file.sol --solc_include_path PATH ``` -* **Use optimization:** To run the solidity compiler with optimizations +* **Use optimization:** To run the Solidity compiler with optimizations (`solc`'s `--optimize` argument), use `--solc_optimize`: ```bash @@ -442,7 +442,7 @@ The configuration file is a JSON file containing the command line arguments for In addition to specifying the command line arguments, you can list the specific mutants that you want to apply, the specific functions you wish to mutate, and -more. See the [`benchmark/config-jsons` directory][config-examples] for +more. See the [`benchmark/config-jsons` directory][config-examples] for examples. _**Note:** @@ -566,15 +566,15 @@ This has the following structure: | `--skip_validate` | only generate mutants without validating them by compilation | Gambit also supports _pass-through arguments_, which are arguments that are -passed directly to the solidity compiler. All pass-through arguments are +passed directly to the Solidity compiler. All pass-through arguments are prefixed with `solc_`: | Option | Description | | :-------------------- | :------------------------------------------------------------------------------ | +| `--solc_allow_paths` | passes a value to `solc`'s `--allow-paths` argument | | `--solc_base_path` | passes a value to `solc`'s `--base-path` argument | | `--solc_include_path` | passes a value to `solc`'s `--include-path` argument | | `--solc_remappings` | passes a value to directly to `solc`: this should be of the form `prefix=path`. | -| `--solc_allow_paths` | passes a value to `solc`'s `--allow-paths` argument | ## Mutation Operators Gambit implements the following mutation operators From 73a15580f20e1ad22b37106b5f2d85b3178ae1df Mon Sep 17 00:00:00 2001 From: Chandrakana Nandi Date: Thu, 21 Dec 2023 14:16:42 -0800 Subject: [PATCH 15/15] update readme and fix generate_rtd --- README.md | 17 ++++++---- scripts/generate_rtd_markdown.py | 56 +++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6487d95..7f48ed3 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,8 @@ file to mutate. ```bash gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol ``` - + +This will generate: 
 Generated 34 mutants in 0.69 seconds
@@ -253,6 +254,8 @@ provides a way to randomly downsample the number of mutants with the ```bash gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol -n 3 ``` + +which will generate: 
 Generated 3 mutants in 0.15 seconds
@@ -266,7 +269,8 @@ Gambit outputs all of its results in `gambit_out`: ```bash tree -L 2 gambit_out ``` - + +This produces: 
 gambit_out
 ├── gambit_results.json
@@ -371,7 +375,7 @@ Here are some examples of using the `--sourceroot` option.
    ```
 
    This should output the following:
-   
+
    
    Generated 1 mutants in 0.13 seconds
    1,BinaryOpMutation,benchmarks/BinaryOpMutation/BinaryOpMutation.sol,23:10, % ,*
@@ -394,7 +398,6 @@ Here are some examples of using the `--sourceroot` option.
 
    which will output:
 
-   
    
    Generated 1 mutants in 0.13 seconds
    1,BinaryOpMutation,BinaryOpMutation.sol,23:10, % ,*
@@ -410,10 +413,10 @@ Here are some examples of using the `--sourceroot` option.
    ```bash
    gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol -n 1 --sourceroot scripts
    ```
+
    This will try to find the specified file inside of `scripts`, and since it
    doesn't exist Gambit reports the error:
 
-   
    
    [ERROR gambit] [!!] Illegal Configuration: Resolved filename `/Users/USER/Gambit/benchmarks/BinaryOpMutation/BinaryOpMutation.sol` is not prefixed by the derived source root /Users/USER/Gambit/scripts
    

@@ -521,7 +524,9 @@ to the `benchmarks/` directory the `"filename"` would need to be updated to
 gambit mutate -f benchmarks/BinaryOpMutation/BinaryOpMutation.sol -n 5
 tree gambit_out -L 2
 ```
-
+
+which produces:
+
 
 Generated 5 mutants in 0.15 seconds
 
diff --git a/scripts/generate_rtd_markdown.py b/scripts/generate_rtd_markdown.py
index 366e1a0..b52d864 100644
--- a/scripts/generate_rtd_markdown.py
+++ b/scripts/generate_rtd_markdown.py
@@ -81,6 +81,21 @@ def is_note_end(line: str) -> bool:
         return len(l) == 1 or l[-2] != "_"
 
 
+def is_tag(tag: str, line: str):
+    """
+    Check if a line consists
+    """
+    return line.strip() in (f"<{tag}>", f"")
+
+
+def is_warning_end(line: str) -> bool:
+    """
+    A warning ends when a line is ended by an underscore. We double check to
+    ensure that the line doesn't end with two underscores.
+    """
+    return is_note_end(line)
+
+
 def is_escaped_closed_comment(line: str) -> bool:
     return line.strip() == r"--\>"
 
@@ -92,7 +107,8 @@ def translate_readme_to_rtd(readme_file_path: str) -> str:
     lines2 = []
 
     suppress_start = -1  # Track if we are suppressing
-    note_start = -1  # Track if we've started a note
+    admonition_start = -1
+    admonition_start = -1  # Track if we've started a note
     emit_start = -1
     for i, line in enumerate(lines):
         # First, check if we are suppressing
@@ -105,19 +121,36 @@ def translate_readme_to_rtd(readme_file_path: str) -> str:
         if anchor is not None:
             lines2.append(anchor)
         elif "_**note:**" == line.strip().lower():
-            if note_start > -1:
+            if admonition_start > -1:
                 raise RuntimeError(
-                    f"Already in note from line {note_start + 1}, cannot start new note on line {i+1}"
+                    f"Already in note from line {admonition_start + 1}, cannot start new note on line {i+1}"
                 )
-            note_start = i
+            admonition_start = i
             lines2.append("```{note}")
         elif "_**note:**" in line.strip().lower():
             raise RuntimeError(
                 f"Illegal note start on line {i+1}: new note tags '_**Note:**' and their closing '_' must be on their own lines"
             )
 
-        elif note_start > -1 and is_note_end(line):
-            note_start = -1
+        elif admonition_start > -1 and is_note_end(line):
+            admonition_start = -1
+            lines2.append(line.rstrip().rstrip("_"))
+            lines2.append("```")
+
+        elif "_**warning:**" == line.strip().lower():
+            if admonition_start > -1:
+                raise RuntimeError(
+                    f"Already in warning from line {admonition_start + 1}, cannot start new warning on line {i+1}"
+                )
+            admonition_start = i
+            lines2.append("```{warning}")
+        elif "_**warning:**" in line.strip().lower():
+            raise RuntimeError(
+                f"Illegal warning start on line {i+1}: new warning tags '_**Warning:**' and their closing '_' must be on their own lines"
+            )
+
+        elif admonition_start > -1 and is_warning_end(line):
+            admonition_start = -1
             lines2.append(line.rstrip().rstrip("_"))
             lines2.append("```")
 
@@ -128,6 +161,10 @@ def translate_readme_to_rtd(readme_file_path: str) -> str:
                 )
             emit_start = i
 
+        elif is_tag("pre", line):
+            num_spaces = len(line) - len(line.lstrip(' '))
+            lines2.append(f"{num_spaces * ' '}```")
+
         elif line.strip() == "-->" and emit_start > -1:
             emit_start = -1
 
@@ -149,9 +186,10 @@ def translate_readme_to_rtd(readme_file_path: str) -> str:
             )
         else:
             # replace internal links
-            l = replace_internal_references(line)
-            lines2.append(l.strip("\n"))
-    return "\n".join(lines2) + "\n"
+            lines2.append(line.strip("\n"))
+    combined = "\n".join(lines2) + "\n"
+    combined = replace_internal_references(combined)
+    return combined
 
 
 def main():