diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7acd4dc489f..2580836edae 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -51,7 +51,6 @@ jobs: - uses: actions/checkout@v6 - uses: jdx/mise-action@v3 - uses: taiki-e/install-action@cargo-llvm-cov - - uses: taiki-e/install-action@nextest - name: Fetch proof params and RPC test snapshots run: | cargo run --bin forest-dev --no-default-features --profile quick -- fetch-test-snapshots --actor-bundle $FOREST_ACTOR_BUNDLE_PATH diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 1bb03fc1469..78e45d25b9a 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -37,6 +37,7 @@ env: CXX: "sccache clang++" FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters RUST_LOG: error + FOREST_ACTOR_BUNDLE_PATH: /var/tmp/forest_actor_bundle.car.zst jobs: tests-release: @@ -59,13 +60,6 @@ jobs: continue-on-error: true - name: Checkout Sources uses: actions/checkout@v6 - - uses: actions/cache/restore@v5 - with: - path: | - /home/runner/.cache/forest/test/rpc-snapshots/rpc_test - # Broad key + prefix-based restore so the latest rpcsnap- is picked up. - key: rpcsnap- - enableCrossOsArchive: true - name: Setup sccache uses: mozilla-actions/sccache-action@v0.0.9 timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} @@ -77,7 +71,7 @@ jobs: uses: taiki-e/install-action@nextest - name: Fetch proof params and RPC test snapshots run: | - cargo run --bin forest-dev --no-default-features --profile quick -- fetch-test-snapshots + cargo run --bin forest-dev --no-default-features --profile quick -- fetch-test-snapshots --actor-bundle $FOREST_ACTOR_BUNDLE_PATH ls -ahl $FIL_PROOFS_PARAMETER_CACHE - uses: jdx/mise-action@v3 - run: | @@ -86,14 +80,3 @@ jobs: # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" FOREST_TEST_SKIP_PROOF_PARAM_CHECK: 1 - - id: get-cache-hash - run: | - ls -lhR ~/.cache/forest/test/rpc-snapshots/rpc_test/* - echo "hash=$(openssl md5 ~/.cache/forest/test/rpc-snapshots/rpc_test/* | sort | openssl md5 | cut -d ' ' -f 2)" >> $GITHUB_OUTPUT - shell: bash - - uses: actions/cache/save@v5 - with: - path: | - /home/runner/.cache/forest/test/rpc-snapshots/rpc_test - key: rpcsnap-${{ steps.get-cache-hash.outputs.hash }} - enableCrossOsArchive: true diff --git a/Cargo.lock b/Cargo.lock index cc43c101605..8501526fd9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2639,6 +2639,18 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "faststr" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] + [[package]] name = "fdlimit" version = "0.3.0" @@ -3312,7 +3324,6 @@ dependencies = [ "itertools 0.14.0", "jsonrpsee", "jsonwebtoken", - "justjson", "keccak-hash", "kubert-prometheus-process", "lazy-regex", @@ -3376,6 +3387,7 @@ dependencies = [ "slotmap", "smallvec", "smart-default", + "sonic-rs", "spire_enum", "sqlx", "stacker", @@ -5227,12 +5239,6 @@ dependencies = [ "simple_asn1", ] -[[package]] -name = "justjson" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7e253b574775d0ebd7975c471fc18f72f0775a4d42b563b5fbc3c4068aa1075" - [[package]] name = "k256" version = "0.13.4" @@ -6275,6 +6281,26 @@ dependencies = [ "unsigned-varint 0.7.2", ] +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "nalgebra" version = "0.33.2" @@ -7346,6 +7372,26 @@ dependencies = [ "cc", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "pulley-interpreter" version = "36.0.5" @@ -7578,6 +7624,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + [[package]] name = "rand" version = "0.8.5" @@ -7787,6 +7842,12 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" + [[package]] name = "replace_with" version = "0.1.8" @@ -7910,6 +7971,35 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "rkyv" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360b333c61ae24e5af3ae7c8660bd6b21ccd8200dbbc5d33c2454421e85b9c69" +dependencies = [ + "bytes", + "hashbrown 0.16.1", + "indexmap 2.13.0", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02f8cdd12b307ab69fe0acf4cd2249c7460eb89dce64a0febadf934ebb6a9e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "rlimit" version = "0.10.2" @@ -8721,6 +8811,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" @@ -8849,6 +8945,45 @@ dependencies = [ "sha1", ] +[[package]] +name = "sonic-number" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a74044c092f4f43ca7a6cfd62854cf9fb5ac8502b131347c990bf22bef1dfe" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4425ea8d66ec950e0a8f2ef52c766cc3d68d661d9a0845c353c40833179fd866" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "ryu", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror 2.0.18", +] + +[[package]] +name = "sonic-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5707edbfb34a40c9f2a55fa09a49101d9fec4e0cc171ce386086bd9616f34257" +dependencies = [ + "cfg-if", +] + [[package]] name = "spin" version = "0.9.8" diff --git a/Cargo.toml b/Cargo.toml index b376667a997..12faeff5653 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,7 +115,6 @@ is-terminal = "0.4" itertools = "0.14" jsonrpsee = { version = "0.26", features = ["server", "ws-client", "http-client", "macros"] } jsonwebtoken = { version = "10", features = ["aws_lc_rs"] } -justjson = "0.3" keccak-hash = "0.12" kubert-prometheus-process = "0.2" lazy-regex = "3" @@ -196,6 +195,7 @@ similar = "2" slotmap = "1" smallvec = "1" smart-default = "0.7" +sonic-rs = "0.5" spire_enum = "1" stacker = "0.1" static_assertions = "1" diff --git a/src/dev/subcommands/mod.rs b/src/dev/subcommands/mod.rs index 2d9719872e7..d32a1199e63 100644 --- a/src/dev/subcommands/mod.rs +++ b/src/dev/subcommands/mod.rs @@ -6,6 +6,9 @@ mod state_cmd; use crate::cli_shared::cli::HELP_MESSAGE; use crate::networks::generate_actor_bundle; use crate::rpc::Client; +use crate::state_manager::utils::state_compute::{ + get_state_snapshot_file, list_state_snapshot_files, +}; use crate::utils::net::{DownloadFileOption, download_file_with_cache}; use crate::utils::proofs_api::ensure_proof_params_downloaded; use crate::utils::version::FOREST_VERSION_STRING; @@ -63,8 +66,27 @@ async fn fetch_test_snapshots(actor_bundle: Option) -> anyhow::Result<( println!("Wrote the actors bundle to {}", actor_bundle.display()); } + // Prepare state computation and validation snapshots + fetch_state_tests().await?; + // Prepare RPC test snapshots - fetch_rpc_tests().await + fetch_rpc_tests().await?; + + Ok(()) +} + +pub async fn fetch_state_tests() -> anyhow::Result<()> { + let files = list_state_snapshot_files().await?; + let mut joinset = JoinSet::new(); + for file in files { + joinset.spawn(async move { get_state_snapshot_file(&file).await }); + } + for result in joinset.join_all().await { + if let Err(e) = result { + tracing::warn!("{e}"); + } + } + Ok(()) } async fn fetch_rpc_tests() -> anyhow::Result<()> { diff --git a/src/rpc/json_validator.rs b/src/rpc/json_validator.rs index cfb5e1e31b6..f8dc71b4256 100644 --- a/src/rpc/json_validator.rs +++ b/src/rpc/json_validator.rs @@ -7,7 +7,6 @@ //! This means JSON like `{"/":"cid1", "/":"cid2"}` will keep only the last value, which can lead to unexpected behavior in RPC calls. use ahash::HashSet; -use justjson::Value; pub const STRICT_JSON_ENV: &str = "FOREST_STRICT_JSON"; @@ -36,28 +35,22 @@ pub fn validate_json_for_duplicates(json_str: &str) -> Result<(), String> { return Ok(()); } - fn check_value(value: &Value) -> Result<(), String> { - match value { - Value::Object(obj) => { + fn check_value(value: &sonic_rs::Value) -> Result<(), String> { + match value.as_ref() { + sonic_rs::ValueRef::Object(obj) => { let mut seen = HashSet::default(); - for entry in obj.iter() { - let key = entry - .key - .as_str() - .ok_or_else(|| "Invalid JSON key".to_string())?; - + for (key, value) in obj.iter() { if !seen.insert(key) { return Err(format!( - "duplicate key '{}' in JSON object - this likely indicates malformed input. \ - Set {}=0 to disable this check", - key, STRICT_JSON_ENV + "duplicate key '{key}' in JSON object - this likely indicates malformed input. \ + Set {STRICT_JSON_ENV}=0 to disable this check" )); } - check_value(&entry.value)?; + check_value(value)?; } Ok(()) } - Value::Array(arr) => { + sonic_rs::ValueRef::Array(arr) => { for item in arr.iter() { check_value(item)?; } @@ -67,7 +60,7 @@ pub fn validate_json_for_duplicates(json_str: &str) -> Result<(), String> { } } // defer to serde_json for invalid JSON - let value = match Value::from_json(json_str) { + let value: sonic_rs::Value = match sonic_rs::from_str(json_str) { Ok(v) => v, Err(_) => return Ok(()), }; diff --git a/src/state_manager/utils.rs b/src/state_manager/utils.rs index a4b2d59c833..5c02fa27e5b 100644 --- a/src/state_manager/utils.rs +++ b/src/state_manager/utils.rs @@ -194,13 +194,17 @@ pub mod state_compute { utils::net::{DownloadFileOption, download_file_with_cache}, }; use directories::ProjectDirs; + use sonic_rs::JsonValueTrait; use std::{ path::{Path, PathBuf}, sync::{Arc, LazyLock}, time::{Duration, Instant}, }; + use tokio::io::AsyncReadExt; use url::Url; + const DO_SPACE_ROOT: &str = "https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/"; + #[allow(dead_code)] pub async fn get_state_compute_snapshot( chain: &NetworkChain, @@ -209,12 +213,25 @@ pub mod state_compute { get_state_snapshot(chain, "state_compute", epoch).await } + #[allow(dead_code)] + async fn get_state_validate_snapshot( + chain: &NetworkChain, + epoch: i64, + ) -> anyhow::Result { + get_state_snapshot(chain, "state_validate", epoch).await + } + #[allow(dead_code)] pub async fn get_state_snapshot( chain: &NetworkChain, bucket: &str, epoch: i64, ) -> anyhow::Result { + let file = format!("{bucket}/{chain}_{epoch}.forest.car.zst"); + get_state_snapshot_file(&file).await + } + + pub async fn get_state_snapshot_file(file: &str) -> anyhow::Result { static SNAPSHOT_CACHE_DIR: LazyLock = LazyLock::new(|| { let project_dir = ProjectDirs::from("com", "ChainSafe", "Forest"); project_dir @@ -223,9 +240,7 @@ pub mod state_compute { .join("state_compute_snapshots") }); - let url = Url::parse(&format!( - "https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/{bucket}/{chain}_{epoch}.forest.car.zst" - ))?; + let url = Url::parse(&format!("{DO_SPACE_ROOT}{file}"))?; Ok(crate::utils::retry( crate::utils::RetryArgs { timeout: Some(Duration::from_secs(30)), @@ -307,6 +322,32 @@ pub mod state_compute { Ok(()) } + pub async fn list_state_snapshot_files() -> anyhow::Result> { + let url = Url::parse(&format!("{DO_SPACE_ROOT}?format=json&prefix=state_"))?; + let mut json_str = String::new(); + crate::utils::net::reader(url.as_str(), DownloadFileOption::NonResumable, None) + .await? + .read_to_string(&mut json_str) + .await?; + let obj: sonic_rs::Object = sonic_rs::from_str(&json_str)?; + let files = obj + .iter() + .filter_map(|(k, v)| { + if k == "Contents" + && let sonic_rs::ValueRef::Array(arr) = v.as_ref() + && let Some(first) = arr.first() + && let Some(file) = first.as_str() + && file.ends_with(".car.zst") + { + Some(file.to_string()) + } else { + None + } + }) + .collect(); + Ok(files) + } + #[cfg(test)] mod tests { //! @@ -316,11 +357,12 @@ pub mod state_compute { use super::*; use crate::chain_sync::tipset_syncer::validate_tipset; - async fn get_state_validate_snapshot( - chain: &NetworkChain, - epoch: i64, - ) -> anyhow::Result { - get_state_snapshot(chain, "state_validate", epoch).await + #[tokio::test(flavor = "multi_thread")] + async fn test_list_state_snapshot_files() { + let files = list_state_snapshot_files().await.unwrap(); + println!("{files:?}"); + assert!(files.len() > 1); + get_state_snapshot_file(&files[0]).await.unwrap(); } // FVM@4