From 90aa555189a48988d2a22e5e33aed5e4b6d51c1b Mon Sep 17 00:00:00 2001 From: Alex Kuzmin <6849426+alxkzmn@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:25:48 +0800 Subject: [PATCH] Speed up compilation (#7) * Speed up compilation * Fix typo * fix: ensure circuit output directory is created only if it doesn't exist --- Cargo.lock | 30 ++++++++++- Cargo.toml | 4 +- src/transpile.rs | 128 +++++++++++++++++++++++++++++++++++++---------- 3 files changed, 132 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f62d386..3a1eb2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,14 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "cc" -version = "1.0.99" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +dependencies = [ + "jobserver", + "libc", + "shlex", +] [[package]] name = "fnv" @@ -20,6 +25,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "libc" +version = "0.2.168" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" + [[package]] name = "num-bigint" version = "0.4.5" @@ -75,6 +95,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index ed95d02..7e426ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ homepage = "https://github.com/chancehudson/rust-witness" repository = "https://github.com/chancehudson/rust-witness.git" [dependencies] -cc = "1.0" +cc = {version ="1.2.3", features = ["parallel"]} fnv = "1.0.7" num-bigint = "0.4.0" num-traits = "0.2.0" @@ -16,7 +16,7 @@ paste = "1.0.0" walkdir = "2.5.0" [build-dependencies] -cc = "1.0" +cc = {version ="1.2.3", features = ["parallel"]} walkdir = "2.5.0" [lib] diff --git a/src/transpile.rs b/src/transpile.rs index 1df1cc2..e5e4e27 100644 --- a/src/transpile.rs +++ b/src/transpile.rs @@ -84,6 +84,7 @@ pub fn transpile_wasm(wasmdir: String) { .flag("-Wno-null-character") .flag("-Wno-c2x-extensions"); + let mut last_modified_file = std::time::SystemTime::UNIX_EPOCH; for entry in WalkDir::new(wasmdir) { let e = entry.unwrap(); let path = e.path(); @@ -96,7 +97,7 @@ pub fn transpile_wasm(wasmdir: String) { if ext != "wasm" { continue; } - // with make source files with the same name as the wasm binary file + // make source files with the same name as the wasm binary file let circuit_name = path.file_stem().unwrap(); let circuit_name_compressed = circuit_name .to_str() @@ -117,40 +118,115 @@ pub fn transpile_wasm(wasmdir: String) { ) .as_str(), ); + // w2c2 is using a fixed naming convention when splitting source by the number of functions ("-f n" flag). + // The output files are named s00..01.c, s00..02.c, s00..03.c, etc., and a main file named after the wasm file. + // As there may be multiple wasm files, we need to transpile each wasm file into a separate directory to prevent + // w2c2 from overwriting the s..x.c files. + + let circuit_out_dir = + Path::new(&circuit_out_dir).join(Path::new(circuit_name.to_str().unwrap())); + + if !circuit_out_dir.exists() { + fs::create_dir(&circuit_out_dir).expect("Failed to create circuit output directory"); + } + let out = Path::new(&circuit_out_dir) .join(Path::new(path.file_name().unwrap())) .with_extension("c"); - if out.exists() { + // Check if the source file needs to be regenerated + if needs_regeneration(path, &out) { + // first generate the c source + w2c2() + .arg("-p") + .arg("-m") + .arg("-f 1") + .arg(path) + .arg(out.clone()) + .spawn() + .expect("Failed to spawn w2c2") + .wait() + .expect("w2c2 command errored"); + + let contents = fs::read_to_string(out.clone()).unwrap(); + // make the data constants static to prevent duplicate symbol errors + fs::write( + out.clone(), + contents.replace("const U8 d", "static const U8 d"), + ) + .expect("Error modifying data symbols"); + } else { println!( - "Source file already exists, overwriting: {}", - &out.display() + "C source files are up to date, skipping transpilation: {}", + path.display() + ); + last_modified_file = std::cmp::max( + last_modified_file, + fs::metadata(&out) + .expect("Failed to read metadata") + .modified() + .expect("Failed to read modified time"), ); } - // first generate the c source - w2c2() - .arg("-p") - .arg("-m") - .arg(path) - .arg(out.clone()) - .spawn() - .expect("Failed to spawn w2c2") - .wait() - .expect("w2c2 command errored"); - - let contents = fs::read_to_string(out.clone()).unwrap(); - // make the data constants static to prevent duplicate symbol errors - fs::write( - out.clone(), - contents.replace("const U8 d", "static const U8 d"), - ) - .expect("Error modifying data symbols"); builder.file(out.clone()); + // Add all the files to the builder that start with "s0..." and end with ".c" (the results of w2c2 `-f` flag) + for entry in WalkDir::new(circuit_out_dir.clone()) { + let e = entry.unwrap(); + let path = e.path(); + if path.is_dir() { + continue; + } + let ext = path.extension().and_then(OsStr::to_str).unwrap_or(""); + if ext != "c" { + continue; + } + if path + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("s0") + && path.file_name().unwrap().to_str().unwrap().ends_with(".c") + { + builder.file(path); + } + } + } + + // Detect if "libcircuit.a" is up to date and skip compilation in that case + let libcircuit = Path::new(&circuit_out_dir).join("libcircuit.a"); + let mut libcircuit_modified_time = std::time::SystemTime::UNIX_EPOCH; + if libcircuit.exists() { + libcircuit_modified_time = fs::metadata(&libcircuit) + .expect("Failed to read metadata") + .modified() + .expect("Failed to read modified time"); + } + + if libcircuit_modified_time < last_modified_file || !libcircuit.exists() { + // write filename prefixed handler functions + let handlers = Path::new(circuit_out_dir.as_str()).join("handlers.c"); + fs::write(handlers, handler).expect("Error writing handler source"); + + builder.compile("circuit"); + } else { + println!("Compiled circuit is up to date, skipping build"); + } +} + +fn needs_regeneration(source: &Path, generated: &Path) -> bool { + if !generated.exists() { + return true; } + let source_metadata = fs::metadata(source).expect("Failed to read source metadata"); + let generated_metadata = fs::metadata(generated).expect("Failed to read generated metadata"); - // write filename prefixed handler functions - let handlers = Path::new(circuit_out_dir.as_str()).join("handlers.c"); - fs::write(handlers, handler).expect("Error writing handler source"); + let source_modified = source_metadata + .modified() + .expect("Failed to read source modification time"); + let generated_modified = generated_metadata + .modified() + .expect("Failed to read generated modification time"); - builder.compile("circuit"); + source_modified > generated_modified }