Merge pull request #5 from tum-i4/develop

Merge develop into main

.github/workflows/rust.yml → .github/workflows/cargo-test.yml

@@ -1,4 +1,4 @@
-name: Build + Test
+name: rustyrts - Test
 
 on:
   push:
@@ -25,12 +25,16 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         profile: minimal
-        toolchain: nightly-2023-01-20
+        toolchain: nightly-2023-12-28
         override: true
     - name: Install Rust components
       run: rustup component add rustc-dev llvm-tools-preview
     - name: Check Rust version
       run: cargo --version && rustc --version
+
+    - name: Rust Cache
+      uses: Swatinem/rust-cache@v2.7.0
+
     - name: Build
       run: cargo build --verbose
     - name: Run tests

.helix/config.toml

@@ -0,0 +1 @@
+editor.workspace-lsp-roots = ["rustyrts-dynamic-rlib", "rustyrts-dynamic-runner"]

Cargo.toml

@@ -6,6 +6,9 @@ edition = "2021"
 [package.metadata.rust-analyzer]
 rustc_private=true
 
+[profile.release]
+debug = true
+
 [[bin]]
 name = "cargo-rustyrts"
 
@@ -15,12 +18,6 @@ name = "rustyrts-static"
 [[bin]]
 name = "rustyrts-dynamic"
 
-[features]
-default = ["print_paths"]
-
-print_paths = []
-monomorphize = []
-
 [dependencies]
 log = "0.4"
 serde_json = "1.0.61"
@@ -31,4 +28,9 @@ env_logger = "0.10.0"
 regex = "1.7.1"
 lazy_static = "1.4.0"
 once_cell = "1.17.1"
-threadpool = "1.8.1"
+threadpool = "1.8.1"
+bimap = "0.6.3"
+
+[dev-dependencies]
+tempdir = "0.3.7"
+test-case = "3.1.0"

README.md

@@ -1,16 +1,27 @@
-# `rustyRTS`
+# RustyRTS
 
-`rustyRTS` is a regression test selection tool for Rust projects.
+RustyRTS is a regression test selection tool for Rust projects.
+It provides two ways of selecting tests:
 
-# Prerequisites
-- developed on Rust nightly-2023-01-20 - in other versions the API of rustc_driver may differ slightly
+- `cargo rustyrts dynamic` instruments all binaries to trace which functions are executed during the tests
+  - ${\color{lightgreen}+++}$ extremely precise
+  - ${\color{lightgreen}+}$ can trace child processes (linux only)
+  - ${\color{red}-}$ tampers with binaries (not always desired)
+  - ${\color{red}-}$ needs to isolate tests in separate processes if tests are executed in parallel (not always feasible)
+  - ${\color{red}-}$ needs to execute test sequentially/single-threaded on Windows
+  - ${\color{orange}/}$ small compilation overhead, moderate runtime overhead
 
-# Setup
-To build `rustyRTS` simply run:
-```
-$ cargo install --path .
-```
-This will build the required executables and install them to your local cargo directory.
+- `cargo rustyrts static` creates a directed dependency graph via static analysis
+  - ${\color{lightgreen}+}$ quite precise
+  - ${\color{lightgreen}+}$ does not tamper with binaries at all
+  - ${\color{lightgreen}+}$ no runtime overhead
+  - ${\color{orange}/}$ moderate compilation overhead
+
+Whenever it detects that some test depends on a function that has changed, this test is selected.
+
+# Rust version
+RustyRTS depends on the internals of the `rustc` compiler, which are quite unstable.
+It has been developed for *v1.77.0-nightly* and can currently only be used with this specific toolchain version. 
 
 ## Setup Rust toolchain
 The correct toolchain should be installed automatically when building `rustyRTS`.
@@ -20,6 +31,13 @@ $ rustup default nightly-2023-01-20 # (recommended: to use this toolchain by def
 $ rustup override set nightly-2023-01-20 # (to use this toolchain in current directory only)
 ```
 
+# How to install
+To install RustyRTS simply run:
+```
+$ cargo install --path .
+```
+This will first install the required toolchain, if it is not present, and then build the required executables and install them to your local cargo directory.
+
 # Usage
 | Command  | Explanation |
 | -------- | ----------- |
@@ -29,70 +47,8 @@ $ rustup override set nightly-2023-01-20 # (to use this toolchain in current dir
 
 ## Custom arguments to `rustc`, `cargo build` or `cargo test`
 `cargo rustyrts [dynamic|static] <args rustyrts> -- <args cargo build> -- <args rustc> -- <args cargo test (may contain --)>`
+(We are planning to make this more ergonomic soon...)
 
 For example:
 `cargo rustyrts dynamic -- -- --emit=mir` - to generate a human-readable representation of the MIR
 `cargo rustyrts dynamic -- -- -- -- --test-threads 1` - to execute test single-threaded without forking
-
-# How tests are selected
-
-## Checksums
-RustyRTS (both static and dynamic) keeps track of modifications to the code by calculating and comparing checksums of MIR [`Body`s](https://doc.rust-lang.org/stable/nightly-rustc/rustc_middle/mir/struct.Body.html), which correspond to functions. When the checksum of a `Body` differs between old and new revision, it is considered changed.
-
-Furthermore, the checksums of [`ConstAllocation`s](https://doc.rust-lang.org/stable/nightly-rustc/rustc_middle/mir/interpret/allocation/struct.ConstAllocation.html) corresponding to `static var` or `static mut var` are contributing to the checksum of every `Body` that accesses the respective variable. This enables dynamic RustyRTS to recognize changes in compile-time evaluation, where instrumentation for tracing is not possible. In static RustyRTS this allows to restrict the analysis to functions that are relevant at runtime (i.e. not only used in compile-time evaluation).
-
-Lastly, the checksums of [`VtblEntry`s](https://doc.rust-lang.org/stable/nightly-rustc/rustc_middle/ty/vtable/enum.VtblEntry.html) (vtable entries) are contributing to the checksum of the function that they are pointing to.
-Assume a vtable entry that was pointing to a function a) in the old revision is now pointing to a different function b) in the new revision.
-Because static RustyRTS is working entirely on the graph data of the new revision, it is sufficient to consider function b) changed, as long as there is a continuous path from a corresponding test to function b).
-Dynamic RustyRTS is comparing traces originating from the old revision, which is why function a) would be considered changed.
-Because static RustyRTS can distinguish whether a function is called via dynamic or static dispatch, these additional checksums of vtable entries only contribute in the case of dynamic dispatch.
-
-## Dynamic
-Dynamic RustyRTS collects traces containing the names of all functions that are called during the execution of a test. Some helper functions and global variables are used to obtain those traces:
-- a `static HashSet<(&'static str, ..)>` for collecting names of traced functions
-- `trace(input: &'static str, ..)` is used to append `input` to the hash set 
-- `pre_test()` which initializes the hash set
-- `post_test(test_name: &str)` which writes the content of the hash set, i.e. the traces to a file identified by the name of the test, where the traces can be inspected in the subsequent run
-
-Further, on unix-like systems only:
-- `pre_main()` which initializes the hash set, in case this has not already been done
-- `post_main()` which appends the content of the hash set to a file identified by the `ppid` of the currently running process
-- in both `post_test(test_name: &str)` and `post_main()` traces in files identified by the `pid` of the process (i.e. the `ppid` of any forked process), are appended to the hash set before exporting the traces
-
-During compilation, the MIR is modified, automatically generating MIR code that does not reflect in source code. Dynamic RustyRTS injects function calls into certain MIR `Body`s:
-- a call to `trace(<fn_name>)` at the beginning of every MIR `Body`
-- a call to `pre_test()` at the beginning of every test function
-- a call to `post_test(<test_name>)` at the end of every test function
-
-On unix-like systems only:
-- a call to `pre_main()` at the beginning of every main function
-- a call to `post_main()` at the end of every main function
-
-Calls to `post_test(test_name: &str)` and `post_main()` are injected in such a way, that as long as the process terminates gracefully (i.e. either by exiting or by unwinding) the traces are written to the filesystem. A process crashing will result in the traces not being written!
-
-Warning: `trace(<>)` internally uses allocations and locks, such that using custom allocators or signals may lead to a deadlock because of non-reentrant locks. (Using reentrant locks would lead to a stack overflow, which is equally bad.)
-
-On unix-like systems, a special test runner is used to fork for every test case, thus isolating the tests in their own process.
-Forking ensures that traces do not intermix, when executing tests in parallel. When executing tests sequentially, forking is not necessary and can be omitted.
-
-During the subsequent run, the traces are compared to the set of changed `Body`s. If these two sets overlap, the corresponding test is considered affected.
-
-## Static
-Static RustyRTS analyzes the MIR during compilation, without modifying it, to build a (directed) dependency graph. Edges are created according to the following criteria:
-1. `EdgeType::Closure`:         function -> contained Closure
-2. `EdgeType::Generator`:       function -> contained Generator
-3. 1. `EdgeType::FnDefTrait`:   caller function -> callee `fn` (for assoc `fn`s in `trait {..}`)
-3. 3. `EdgeType::FnDefImpl`:    caller function -> callee `fn` (for assoc `fn`s in `impl .. {..}`)
-3. 3. `EdgeType::FnDef`:        caller function -> callee `fn` (for non-assoc `fn`s, i.e. not inside `impl .. {..}`)
-3. 4. `EdgeType::FnDefDyn`:     caller function -> callee `fn` + !dyn (for functions in `trait {..} called by dynamic dispatch)
-4. `EdgeType::TraitImpl`:       function in `trait` definition + !dyn -> function in trait impl (`impl <trait> for ..`)  + !dyn 
-5. `EdgeType::DynFn`:           (only for associated functions) function + !dyn -> function
-6. `EdgeType::Drop`:            function -> destructor (`drop()` function) of referenced abstract datatype
-
-The suffix "!dyn" is used to distinguish static and dynamic dispatch. Checksums from vtable entries only contribute to the function they are pointing to with suffix !dyn.
-
-All these functions are not yet monomorphized. Using names of fully monomorphized functions may increase precision, but turns out to be impractical. On larger projects, it would bloat up the graph, such that reading the graph takes a long time. Moreover, RustyRTS compares checksums of non-monomorphized functions.
-It is nevertheless possible to use fully monomorphized function names using the `monomorphize` feature.
-
-
-When there is a path from a test to a changed `Body`, the test is considered affected.

build.rs

@@ -8,11 +8,13 @@ fn cargo() -> Command {
 }
 
 fn main() {
-    build_library("rustyrts-dynamic-rlib");
-    build_library("rustyrts-dynamic-runner");
+    if std::env::var("RUSTYRTS_SKIP_BUILD").is_err() {
+        build_library("rustyrts-dynamic-rlib");
+        build_library("rustyrts-dynamic-runner");
 
-    install_rlib("rustyrts_dynamic_rlib", "rustyrts-dynamic-rlib");
-    install_staticlib("rustyrts_dynamic_runner", "rustyrts-dynamic-runner");
+        install_rlib("rustyrts_dynamic_rlib", "rustyrts-dynamic-rlib");
+        install_staticlib("rustyrts_dynamic_runner", "rustyrts-dynamic-runner");
+    }
 }
 
 fn build_library(dir_name: &str) {
@@ -42,6 +44,7 @@ fn build_library(dir_name: &str) {
     path.push(dir_name);
     cmd.current_dir(path);
     cmd.arg("build");
+    cmd.arg("--release");
 
     match cmd.status() {
         Ok(exit) => {
@@ -64,7 +67,7 @@ fn install_rlib(name: &str, dir_name: &str) {
     path.push(dir);
     path.push(dir_name);
     path.push("target");
-    path.push("debug");
+    path.push("release");
     //path.push("deps");
 
     let files: Vec<DirEntry> = read_dir(path)
@@ -77,19 +80,7 @@ fn install_rlib(name: &str, dir_name: &str) {
     //let rmeta_file = find_file(&format!("lib{}", name), ".rmeta", &files);
     let d_file = find_file(name, ".d", &files);
 
-    let mut cargo_home = {
-        let maybe_cargo_home = std::env::var("CARGO_HOME");
-        if let Ok(cargo_home) = maybe_cargo_home {
-            PathBuf::from(cargo_home)
-        } else {
-            let home = std::env::var("HOME").expect("Unable to find HOME environment variable");
-            let mut path = PathBuf::new();
-            path.push(home);
-            path.push(".cargo");
-            path
-        }
-    };
-    cargo_home.push("bin");
+    let cargo_home = get_cargo_home();
 
     if let Some(entry) = rlib_file {
         let src = entry.path();
@@ -119,7 +110,7 @@ fn install_staticlib(name: &str, dir_name: &str) {
     let mut dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
     dir.push(dir_name);
     dir.push("target");
-    dir.push("debug");
+    dir.push("release");
     //dir.push("deps");
 
     let files: Vec<DirEntry> = read_dir(dir)
@@ -131,19 +122,7 @@ fn install_staticlib(name: &str, dir_name: &str) {
     let a_file = find_file(&format!("lib{}", name), ".a", &files);
     let d_file = find_file(name, ".d", &files);
 
-    let mut cargo_home = {
-        let maybe_cargo_home = std::env::var("CARGO_HOME");
-        if let Ok(cargo_home) = maybe_cargo_home {
-            PathBuf::from(cargo_home)
-        } else {
-            let home = std::env::var("HOME").expect("Unable to find HOME environment variable");
-            let mut path = PathBuf::new();
-            path.push(home);
-            path.push(".cargo");
-            path
-        }
-    };
-    cargo_home.push("bin");
+    let cargo_home = get_cargo_home();
 
     if let Some(entry) = a_file {
         let src = entry.path();
@@ -160,6 +139,23 @@ fn install_staticlib(name: &str, dir_name: &str) {
     }
 }
 
+fn get_cargo_home() -> PathBuf {
+    let mut cargo_home = {
+        let maybe_cargo_home = std::env::var("CARGO_HOME");
+        if let Ok(cargo_home) = maybe_cargo_home {
+            PathBuf::from(cargo_home)
+        } else {
+            let home = std::env::var("HOME").expect("Unable to find HOME environment variable");
+            let mut path = PathBuf::new();
+            path.push(home);
+            path.push(".cargo");
+            path
+        }
+    };
+    cargo_home.push("bin");
+    cargo_home
+}
+
 fn find_file<'a>(
     starts_with: &str,
     ends_with: &str,

rust-toolchain.toml

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-01-20"
-components = [ "rustc-dev", "llvm-tools-preview" ]
+channel = "nightly-2023-12-28"
+components = [ "rustc-dev", "rust-src", "llvm-tools"]

rustyrts-dynamic-rlib/rust-toolchain.toml

@@ -0,0 +1 @@
+../rust-toolchain.toml