diff --git a/testool/Config.toml b/testool/Config.toml index 761bfeb93b..9352ea0589 100644 --- a/testool/Config.toml +++ b/testool/Config.toml @@ -1,177 +1,14 @@ [[suite]] id="default" -path="tests/src/GeneralStateTestsFiller/**/*" -max_gas = 500000 -max_steps = 1000 -ignore_tests = [] +path="tests/src/GeneralStateTestsFiller/VMTests/vmArithmeticTest/addFiller.yml" +max_gas = 0 +max_steps = 1000000 [[suite]] id="nightly" path="tests/src/GeneralStateTestsFiller/**/*" max_gas = 0 max_steps = 100000 -ignore_tests=[] - -[[suite]] -id = "light" -path="tests/src/GeneralStateTestsFiller/**/*" -max_gas = 500000 -max_steps = 1000 -allow_tests=[ - "add_d0(add_neg1_neg1)_g0_v0", - "mul_d0(mul_2_3)_g0_v0", - "sub_d0(sub_23_1)_g0_v0", - "div_d0(div_2_big)_g0_v0", - "sdiv_d0(sdiv_1_neg1)_g0_v0", - "mod_d0(mod_2_3)_g0_v0", - "smod_d0(smod_2_3)_g0_v0", - "addmod_d0(addmod_1_2_2)_g0_v0", - "mulmod_d0(mm_1_2_2)_g0_v0", - "exp_d7(exp_2_257)_g0_v0", - "signextend_d0(invalid_byte)_g0_v0", - "lt_d0(lt_neg2_0)_g0_v0", - "gt_d0(gt_neg2_0)_g0_v0", - "slt_d0(slt_neg2_0)_g0_v0", - "sgt_d0(sgt_neg2_0)_g0_v0", - "eq_d0(eq_neg5_neg3)_g0_v0", - "iszero_d0(iszero_neg1)_g0_v0", - "and_d0(and_2_2)_g0_v0", - "or_d0(or_2_2)_g0_v0", - "xor_d0(xor_2_2)_g0_v0", - "not_d0_g0_v0", - "byte_d0(byte_31_big)_g0_v0", - "shl_-1_0_d0_g0_v0", - "shr_-1_0_d0_g0_v0", - # SAR - # KECCAK - # ADDRESS - # BALANCE - "envInfo_d8(origin)_g0_v0", - "envInfo_d4(caller)_g0_v0", - "envInfo_d5(callvalue)_g0_v0", - "calldataload_d0(two_bytes)_g0_v0", - "envInfo_d9(calldatasize)_g0_v0", - "calldatacopy_d7(sec)_g0_v0", - "envInfo_d6(codesize)_g0_v0", - "envInfo_d1(codecopy)_g0_v0", - "envInfo_d7(gasprice)_g0_v0", - # EXTCODESIZE - # EXTCODECOPY - # RETURNCODESIZE - # RETURNCODECOPY - # EXTCODEHASH (need to find a test for it) - "blockInfo_d0(coinbase)_g0_v0", - "blockInfo_d4(timestamp)_g0_v0", - "blockInfo_d3(number)_g0_v0", - "blockInfo_d1(difficulty)_g0_v0", - "blockInfo_d2(gaslimit)_g0_v0", - "chainId_d0_g0_v0", - "selfBalance_d0_g0_v0", - # no tests for BASEFEE ? - "refundMax_d0_g0_v0", # POP - "mload16bitBound_d0_g0_v0", - "gasCostMemory_d44_g0_v0", # MSTORE - "randomStatetest148_d0_g0_v0", # MSTORE8 - "SLOAD_Bounds_d0_g0_v0", - "sstoreGas_d0_g0_v0", - "JUMP_Bounds2_d0_g0_v0", - "JUMPI_Bounds_d0_g0_v0", - "callcode_checkPC_d0_g0_v0", # PC - "randomStatetest310_d0_g0_v0", # MSIZE - "RawCallGas_d0_g0_v0", - # JUMPDEST - "push_d0(push1)_g0_v0", - "push_d1(push2)_g0_v0", - "push_d2(push3)_g0_v0", - "push_d3(push4)_g0_v0", - "push_d4(push5)_g0_v0", - "push_d5(push6)_g0_v0", - "push_d6(push7)_g0_v0", - "push_d7(push8)_g0_v0", - "push_d8(push9)_g0_v0", - "push_d9(push10)_g0_v0", - "push_d10(push11)_g0_v0", - "push_d11(push12)_g0_v0", - "push_d12(push13)_g0_v0", - "push_d13(push14)_g0_v0", - "push_d14(push15)_g0_v0", - "push_d15(push16)_g0_v0", - "push_d16(push17)_g0_v0", - "push_d17(push18)_g0_v0", - "push_d18(push19)_g0_v0", - "push_d19(push20)_g0_v0", - "push_d20(push21)_g0_v0", - "push_d21(push22)_g0_v0", - "push_d22(push23)_g0_v0", - "push_d23(push24)_g0_v0", - "push_d24(push25)_g0_v0", - "push_d25(push26)_g0_v0", - "push_d26(push27)_g0_v0", - "push_d27(push28)_g0_v0", - "push_d28(push29)_g0_v0", - "push_d29(push30)_g0_v0", - "push_d30(push31)_g0_v0", - "push_d31(push32)_g0_v0", - "dup_d0(dup1)_g0_v0", - "dup_d1(dup2)_g0_v0", - "dup_d2(dup3)_g0_v0", - "dup_d3(dup4)_g0_v0", - "dup_d4(dup5)_g0_v0", - "dup_d5(dup6)_g0_v0", - "dup_d6(dup7)_g0_v0", - "dup_d7(dup8)_g0_v0", - "dup_d8(dup9)_g0_v0", - "dup_d9(dup10)_g0_v0", - "dup_d10(dup11)_g0_v0", - "dup_d11(dup12)_g0_v0", - "dup_d12(dup13)_g0_v0", - "dup_d13(dup14)_g0_v0", - "dup_d14(dup15)_g0_v0", - "dup_d15(dup16)_g0_v0", - "swap_d0(swap1)_g0_v0", - "swap_d1(swap2)_g0_v0", - "swap_d2(swap3)_g0_v0", - "swap_d3(swap4)_g0_v0", - "swap_d4(swap5)_g0_v0", - "swap_d5(swap6)_g0_v0", - "swap_d6(swap7)_g0_v0", - "swap_d7(swap8)_g0_v0", - "swap_d8(swap9)_g0_v0", - "swap_d9(swap10)_g0_v0", - "swap_d10(swap11)_g0_v0", - "swap_d11(swap12)_g0_v0", - "swap_d12(swap13)_g0_v0", - "swap_d13(swap14)_g0_v0", - "swap_d14(swap15)_g0_v0", - "swap_d15(swap16)_g0_v0", - "log0_emptyMem_d0_g0_v0", - "log1_emptyMem_d0_g0_v0", - "log2_emptyMem_d0_g0_v0", - "log3_emptyMem_d0_g0_v0", - "log4_emptyMem_d0_g0_v0", - # CREATE - "callcode_checkPC_d0_g0_v0", # for CALL - # CALLCODE - # RETURN - # DELEGATECALL - # CREATE2 - # STATICCALL - # REVERT - -] - -[[set]] -id = "sigkill" -desc = "tests that sigkill" -tests = [] - -# skipped tests, do not need to be fixed -------------------------------------------------- - -[[skip_tests]] -desc = "" -tests = [] - -# ignored paths ------------------------------------------------------------------------- [[skip_paths]] desc = "unimplemented" @@ -179,7 +16,7 @@ paths = [ "EIP1559", "EIP2930", "stPreCompiledContracts", - "stZeroKnowledge" + "stZeroKnowledge" ] [[skip_paths]] @@ -221,6 +58,8 @@ paths = [ "create2callPrecompilesFiller.json", "callToNonExistentFiller.json", "tackDepthLimitSECFiller.json", + "callDataCopyOffsetFiller.json", + "calldatacopy_dejavu2Filler.json", "ValueOverflowFiller" # weird 0x:biginteger 0x... ] @@ -235,3 +74,19 @@ paths = [ "RevertRemoteSubCallStorageOOGFiller.yml", "solidityExampleFiller.yml" ] + +[[skip_paths]] +desc = "todo!" +paths = [ + "CreateAddressWarmAfterFailFiller.yml", + "result_cancunEnvConvertionFiller.json", + "result_mergeEnvConvertionFiller.json", + "eoaEmptyFiller.yml" +] + + +[[skip_paths]] +desc = "unused" +paths = [ + "doubleSelfdestructTestFiller.yml" +] diff --git a/testool/codehash.txt b/testool/codehash.txt index 233a3ad2d3..0b31f3a03a 100644 --- a/testool/codehash.txt +++ b/testool/codehash.txt @@ -2896,3 +2896,46 @@ c4bf5ad3a96e11d4a794acd75d486639b5cf225f871db8d20d56123b23f526d4=600060006000600 d8a56c290ee5169eb894f0fa07066821d23f5f3d40b071d6e0d6c7afb282fe37=5a6000556000600060006000600073c94f5374fce5edbc8e2a8697c15331677e6ebf0b61ea60f2600155600160645500 abd7afc4a0b5580e7dc271b92f328b8e6996a971d6cf6612cb392fa447f2709f=5a6000556000600060006000600073c94f5374fce5edbc8e2a8697c15331677e6ebf0b61ea60f1600155600160645500 9555f80f23ad28f27a451dcf714ba3ca9508c507d16c2f5c74f9e9d991b51ec2=5a600055600060006000600073c94f5374fce5edbc8e2a8697c15331677e6ebf0b61ea60f4600155600160645500 +681a49d6032393d3f7b5f66cad1e2c029ac72708b3c84e3ce94174d99d7fbc06=600435606481610100811460295761020081146033576103008114603d576104008114604757604e565b601b82019150604e565b601b82019150604e565b601882019150604e565b6018820191505b5060008060008060008686f16000555050 +e5c6a102b383b20ce844126abb7d0bf51fea53914c23ff516cd9e6cb2188e0cb=416000806000806000856000f15050 +c65e9a13daad46d0fb23c588c7ecb5ee3a807bf026711736ca0638314fc2c49c=416000806000806000856000f25050 +304e4e2c9002c9663b1f664e3501f04a6b8aab9fb8e0d251afe05408c897daab=41600080600080846000f45050 +bdbc1d342b1d78c86d9db1b2908f7f01f274e5e44bb72405e510bf331ca42215=41600080600080846000fa5050 +23baac6d5c80a3460b14c20dcdace0680ab6ee414b3fb2cbfbda963d208a8170=416000806000806004356000811461004e57600181146100615760028114610077576003811461008a576004811461009d57600581146100bb57600681146100d957600781146100f557600080fd5b600f94505a9350853b91505a925061010d565b601394505a93506000806000883c5a925061010d565b600f94505a9350853f91505a925061010d565b600f94505a9350853191505a925061010d565b602194505a935060008060008060008a612710f191505a925061010d565b602194505a935060008060008060008a612710f291505a925061010d565b601e94505a935060008060008089612710f491505a925061010d565b601e94505a935060008060008089612710fa91505a92505b5083828403036000558060005260206000f3 +c9be2eac47d065bde64cf971fbce63007dd79301665543a3e3dabe3cceb32650=6000806000806000803560601c620186a0f160005560018055 +38a0e59befa52d28044471ca5eb4f9930bb6e4e2ff33b993fc62e60202869fff=600080600080610600620186a0fa6000556001805560016000601f3e600051600255 +a616d9465e4127a877fb2d1fc36427f0fa72fc388d02b10ef21cbd7f09091766=6000806000806000803560601c620186a0f160005560018055 +625c7b72397db0ffa1e8435286765d34a367dfaf0c486d0ad3cd57e59f19e20f=6000356000526000803660008073c94f5374fce5edbc8e2a8697c15331677e6ebf0b62989680f1806000556001805550 +f0c8600835d1911fdce3f30d919a430be6d79f9cf8e0b58dbc1aeeaec0240ca4=7f600a80600080396000f3000000000000000000000000000000000000000000006000526000355a63deadbeef82600080f55a8203600a5580600055505050 +52bb67412ac0148ba8454b8d14eecb2df80f6d9d9c2185fbfb353244389a6531=6000356000526000803660008061c0de62989680f1806000556001805550 +2c2982f5372d0bb97d234f16210ecdf8e6ff9275eb34d60af05dab73a6193ba8=7f600a80600080396000f3000000000000000000000000000000000000000000006000526000355a81600080f05a8203600a5580600055505050 +c802c45b226f9562759b05a08789691010b15e1ac224b7c313180df28adac6f4=6000806000806000620e49716000f150 +4570dfe181646d399548df1310a95afe2ee9958e7d53455310a1a275cfa0c524=60016000556000806000806000620c0dea5af15060016000f3 +73b716163efca33a23f793875569df9e759d4097ad21a389b3119db4291e1f2b=600160005560018055600080600080620c0dea5af45060016000f3 +81051233c31b2337d5dbc235f028cfaf2370153cb7a7dba659a46498002efbd7=6001600055600180556000806000806000620c0dea5af25060016000f3 +e3d73815b5fc103b09264ed463c8bc5565d05d85c94cc023599d787f9eed66c2=60016000556000806000806000620c0ded5af15060016000f3 +72811360dafe6138c6488fd76cace07214dda6d9a45f6ffbfdcf98865e89cce5=60016000556000806000806000620c0de05af15060016000f3 +210e3aadfd37ac7de088bcac902b74cf48a8ff68a7103d84c6d189a1437ae473=6001600055600180556000600155620c0de1803b80600080843c80600080f0506001808201f3 +409f61826f7356f59fcbdf1f13a6679279e9920f8dadcff570f0a32c0cce2fd5=6001600055600180556000600155620c0de1803b80600080843c600081600080f5506001808201f3 +0c1f1c2b53e6b6a0c7732ff85e9be54e3f4421012fe9364c43268a3aefa5e714=6000600060006000600073c94f5374fce5edbc8e2a8697c15331677e6ebf0b61ea60f1506d64600c6000556000526005601bf360005230ff00 +fa8a9f80bc520300a3798d8c68e181adab921285010b614549869de5ccd27d1c=7f60016000f30000000000000000000000000000000000000000000000000000006000526005600080f060005560018055 +aafbf83c0f3710c167ebc33f27dceb2eb9d28abb1fb05c21e34b1f9e2233c6d8=6001600055600180556000600155620c0de1600f7f6001600055600060005560016000f30000000000000000000000000000000000600052600081600080f5506001808201f3 +451c55c3eba6157da3ed475ff5c85c21b0bbed3ff08121e3e5e2d815dba5714d=6001600055600180556000600155620c0de1600f7f6001600055600060005560016000f30000000000000000000000000000000000600052600081600080f5506001808201f3 +d2ade8c7069746243223af9eff36a334c9efc8fdd8d168858e7f987bd9074a05=60016000f3 +2801af36164dbee3c029113259d657d1c2175f1fe62f432b894000772c16f845=5a60015560043560243560443561c0de3b8060008061c0de3c826001820353816101005260008460f0811460385760f581146045576052565b610120600080f091506052565b615a17610120600080f591505b50806000555a60015403600155803f6002555050505050 +7b6065bf0a39a0029168a47a75e335f270df1f36d3db2736e7d3d6b53ff44a36=5a600035602060008060008061010086f150815a10600055602060008060008061010086f250815a10600155602060006020600061010085f450815a10600255602060006020600061010085fa50815a106003555050 +3f5c413fc75d017c094476ccfe60240068d906fad96ec1552be536191129fd68=600160010160005500 +e5c786ec740075936fbc7e1ee84d9ab75a38278663c27f7416374f5ded41a231=30ff00 +5b29c457a7d88022ec5b28f39e9b15622da6d884643a771edd4c9e97dd0e331a=670123456789abcdef600052600080600f60008073eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee61fffff150 +c970ee84da99f7186034b44f7bbc9d521569320a1c643f41efe8bf9f52c7dc72=600a68010000000000000001601f3960005160008103601a57005b50 +748350f391f4b406b2e18729bc7632c714cdde93ba664241521fb3b54096cb2b=61100060002050 +f35ce58ac83c4fa7d9b1d8eb8a75494a87a4df27f06bb4d2d7198546bcf0a634=6110005150 +8b92e3499ee6364588d2152a82bf2e4c8e22e35b6b3c65f5e53fb172b598046a=6000806000558060015580600255806003558060045580600555806006558060075559905050 +40450c645189cc3193cfbc4f731f3b3789af2ab8a04f348ed17fb9ed68164da4=6061600053601060015360006002536060600353600060045360f36005536006600080f050 +68992affd32f78e54656d7b1853b46c9031fedfc5f6769d8336d80596650c68c=60008035126000600135128160005581600155816002558160035581600455816005558160065581600755816008558160095581600a5581600b5581600c5581600d5581600e5581600f5560008115605757617eea90505b60008203606457617ee090505b600080600080600061c0de86f150505050 +9fbaa879e71f83700a2ef45abdc3e546888dbb59c374cb1f683d6d58e4b96b35=32ff +0f27e05ea187e05f8d1414fea9d1cef952045895fa0ea1b9afcb067663c27238=32806000558031603155803b603b55803f603f55600181013f61013f5561bad13f61bad15561bad23f61bad25561bad33f61bad35561bad43f61bad4555a600080600080600435865af1505a810360f1555a9050600080600080600061dead5af1505a810360ff555050 +e8a7a68124e629036058991fac713b358337fe7412d37c3d9b89a331520d18b9=348060011c6000806000808461dead62011170f18183036000806000808461dead62011170f15050505050 +7ebea5551401edbb59e83be5648bb70b08dec05208045272c020d206acf1a6c1=600054600181016000558062e4970101ff +5a96de1b93928214fa164ae3487c73e2ac678feb29cd5bdbf107342152d30f1f=60003560f81c61ffff60003560e81c1660008203601857005b60ff820360225780ff5b600080600080600086865af103603757600080fd5b5050 +c34ba972d8ae90026ee747e3e40f100666a67dbecaf1ea05faeb474b619bcb2e=600060ff60005360106001536000600253600080600360008061dead5af19050806000556110003160015561dead3160025560003560f81c806001811460605760028114607357600381146087576004811460a0576005811460bd57600080fd5b600080600380600261dead5af1925060d8565b60008060036000600261dead5af1925060d8565b600160025360008060036000600261dead5af1925060d8565b60016000536001600253600080600360008061dead5af1925060d8565b6001600053600160025360008060036000600261dead5af192505b50816010556110003160115561dead31601255611001316013555050 diff --git a/testool/docker/solc/Dockerfile b/testool/docker/solc/Dockerfile index 16b58ac0ca..c7e9f95a23 100644 --- a/testool/docker/solc/Dockerfile +++ b/testool/docker/solc/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /solidity RUN apk update && apk add boost-dev boost-static build-base cmake git RUN git clone https://github.com/ethereum/solidity . -RUN git checkout v0.8.13 +RUN git checkout v0.8.17 WORKDIR /solidity/build RUN cmake .. -DCMAKE_BUILD_TYPE=Release diff --git a/testool/src/abi.rs b/testool/src/abi.rs index 604463702a..e3e9287bbb 100644 --- a/testool/src/abi.rs +++ b/testool/src/abi.rs @@ -21,7 +21,7 @@ pub fn encode_funccall(spec: &str) -> Result { "uint" => ParamType::Uint(256), "uint256" => ParamType::Uint(256), "bool" => ParamType::Bool, - _ => panic!("unimplemented abi type {:?}", t), + _ => panic!("unimplemented abi type {t:?}"), }; let encode_type = |t, v: &str| match t { @@ -35,7 +35,7 @@ pub fn encode_funccall(spec: &str) -> Result { ParamType::Bool => match v.to_lowercase().as_str() { "true" | "0x01" => Ok(Token::Bool(true)), "false" | "0x00" => Ok(Token::Bool(false)), - _ => panic!("unexpected boolean '{}'", v), + _ => panic!("unexpected boolean '{v}'"), }, _ => unimplemented!(), }; @@ -44,7 +44,7 @@ pub fn encode_funccall(spec: &str) -> Result { .iter() .enumerate() .map(|(n, t)| Param { - name: format!("p{}", n), + name: format!("p{n}"), kind: map_type(t), internal_type: None, }) diff --git a/testool/src/compiler.rs b/testool/src/compiler.rs index e6e4912280..53296cab3c 100644 --- a/testool/src/compiler.rs +++ b/testool/src/compiler.rs @@ -9,6 +9,7 @@ use std::{ path::PathBuf, process::{Command, Stdio}, str::FromStr, + sync::Mutex, }; struct Cache { @@ -70,7 +71,7 @@ struct CompilerInput { sources: HashMap, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] enum Language { Solidity, @@ -81,14 +82,14 @@ enum Language { #[serde(rename_all = "camelCase")] struct CompilerSettings { optimizer: Optimizer, + evm_version: String, output_selection: HashMap>>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] struct Optimizer { enabled: bool, - details: HashMap, } #[derive(Debug, Serialize, Deserialize)] @@ -98,7 +99,7 @@ struct Source { } impl CompilerInput { - pub fn new_default(language: Language, src: &str) -> Self { + pub fn new_default(language: Language, src: &str, evm_version: Option<&str>) -> Self { let mut sources = HashMap::new(); sources.insert( "stdin".to_string(), @@ -108,38 +109,26 @@ impl CompilerInput { ); CompilerInput { language, - settings: Default::default(), + settings: CompilerSettings::new_default(evm_version), sources, } } } -impl Default for CompilerSettings { - fn default() -> Self { +impl CompilerSettings { + fn new_default(evm_version: Option<&str>) -> Self { let mut output_selection = HashMap::new(); let mut selection = HashMap::new(); selection.insert("*".to_string(), vec!["evm.bytecode".to_string()]); output_selection.insert("*".to_string(), selection); CompilerSettings { + evm_version: evm_version.unwrap_or("berlin").to_string(), optimizer: Default::default(), output_selection, } } } -impl Default for Optimizer { - fn default() -> Self { - let mut details = HashMap::new(); - details.insert("peephole".to_string(), false); - details.insert("inliner".to_string(), false); - details.insert("jumpdestRemover".to_string(), false); - Optimizer { - enabled: false, - details, - } - } -} - #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] struct CompilationResult { @@ -198,16 +187,17 @@ struct SourceLocation { #[derive(Default)] pub struct Compiler { - cache: Option, + cache: Option>, compile: bool, } impl Compiler { pub fn new(compile: bool, cache_path: Option) -> Result { - let cache = cache_path.map(Cache::new).transpose()?; + let cache = cache_path.map(Cache::new).transpose()?.map(Mutex::new); Ok(Compiler { compile, cache }) } + /// the concurrency level of the exec is controlled by rayon parallelism fn exec(args: &[&str], stdin: &str) -> Result { let mut child = Command::new("docker") .args(args) @@ -239,7 +229,7 @@ impl Compiler { } /// compiles ASM code - pub fn asm(&mut self, src: &str) -> Result { + pub fn asm(&self, src: &str) -> Result { let mut bytecode = Bytecode::default(); for op in src.split(';') { let op = match bytecode::OpcodeWithData::from_str(op.trim()) { @@ -253,9 +243,13 @@ impl Compiler { } /// compiles LLL code - pub fn lll(&mut self, src: &str) -> Result { - if let Some(bytecode) = self.cache.as_mut().and_then(|c| c.get(src)) { - return Ok(bytecode.clone()); + pub fn lll(&self, src: &str) -> Result { + if let Some(bytecode) = self + .cache + .as_ref() + .and_then(|c| c.lock().unwrap().get(src).cloned()) + { + return Ok(bytecode); } if !self.compile { bail!("No way to compile LLLC for '{}'", src) @@ -264,37 +258,45 @@ impl Compiler { let stdout = Self::exec(&["run", "-i", "--rm", "lllc"], src)?; let bytecode = Bytes::from(hex::decode(stdout.trim())?); - if let Some(cache) = &mut self.cache { - cache.insert(src, bytecode.clone())?; + if let Some(ref cache) = self.cache { + cache.lock().unwrap().insert(src, bytecode.clone())?; } Ok(bytecode) } /// compiles YUL code - pub fn yul(&mut self, src: &str) -> Result { - self.solc(Language::Yul, src) + pub fn yul(&self, src: &str, evm_version: Option<&str>) -> Result { + self.solc(Language::Yul, src, evm_version) } /// compiles Solidity code - pub fn solidity(&mut self, src: &str) -> Result { - self.solc(Language::Solidity, src) + pub fn solidity(&self, src: &str, evm_version: Option<&str>) -> Result { + self.solc(Language::Solidity, src, evm_version) } - fn solc(&mut self, language: Language, src: &str) -> Result { - if let Some(bytecode) = self.cache.as_mut().and_then(|c| c.get(src)) { - return Ok(bytecode.clone()); + fn solc(&self, language: Language, src: &str, evm_version: Option<&str>) -> Result { + if let Some(bytecode) = self + .cache + .as_ref() + .and_then(|c| c.lock().unwrap().get(src).cloned()) + { + return Ok(bytecode); } if !self.compile { bail!("No way to compile {:?} for '{}'", language, src) } - let compiler_input = CompilerInput::new_default(language, src); + let compiler_input = CompilerInput::new_default(language, src, evm_version); let stdout = Self::exec( &["run", "-i", "--rm", "solc", "--standard-json", "-"], serde_json::to_string(&compiler_input).unwrap().as_str(), )?; - let mut compilation_result: CompilationResult = serde_json::from_str(&stdout)?; + let mut compilation_result: CompilationResult = serde_json::from_str(&stdout) + .map_err(|e| { + println!("---\n{language:?}\n{src}\n{evm_version:?}\n{e:?}\n{stdout}\n-----") + }) + .unwrap(); let bytecode = compilation_result .contracts .remove("stdin") @@ -309,8 +311,8 @@ impl Compiler { let bytecode = Bytes::from(hex::decode(bytecode)?); - if let Some(cache) = &mut self.cache { - cache.insert(src, bytecode.clone())?; + if let Some(ref cache) = self.cache { + cache.lock().unwrap().insert(src, bytecode.clone())?; } Ok(bytecode) @@ -347,6 +349,7 @@ mod test { } } "#, + None, )?; assert_eq!( hex::encode(out), @@ -357,7 +360,7 @@ mod test { #[test] #[ignore] fn test_docker_solidity() -> anyhow::Result<()> { - let out = super::Compiler::new(true, None)?.solidity("contract A{}")?; + let out = super::Compiler::new(true, None)?.solidity("contract A{}", None)?; assert_eq!( hex::encode(out), "6080604052348015600f57600080fd5b50603c80601d6000396000f3fe6080604052600080fdfea164736f6c637828302e382e31332d646576656c6f702e323032322e352e31312b636f6d6d69742e61626161356330650030" diff --git a/testool/src/config.rs b/testool/src/config.rs index 6e89598cb3..6295f475ac 100644 --- a/testool/src/config.rs +++ b/testool/src/config.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, ensure, Context, Result}; +use anyhow::{anyhow, Context, Result}; use serde::Deserialize; const CONFIG_FILE: &str = "Config.toml"; @@ -6,8 +6,9 @@ const CONFIG_FILE: &str = "Config.toml"; #[derive(Debug, Clone, Deserialize)] pub struct Config { pub suite: Vec, - pub set: Vec, + #[serde(default)] pub skip_paths: Vec, + #[serde(default)] pub skip_tests: Vec, } @@ -17,9 +18,6 @@ pub struct TestSuite { pub path: String, pub max_gas: u64, pub max_steps: u64, - - ignore_tests: Option>, - allow_tests: Option>, } impl Default for TestSuite { @@ -29,24 +27,6 @@ impl Default for TestSuite { path: String::default(), max_gas: u64::MAX, max_steps: u64::MAX, - ignore_tests: Some(Vec::new()), - allow_tests: None, - } - } -} - -impl TestSuite { - pub fn allowed(&self, test_id: &str) -> bool { - if let Some(ignore_tests) = &self.ignore_tests { - ignore_tests - .binary_search_by(|id| id.as_str().cmp(test_id)) - .is_err() - } else if let Some(allow_tests) = &self.allow_tests { - allow_tests - .binary_search_by(|id| id.as_str().cmp(test_id)) - .is_ok() - } else { - unreachable!() } } } @@ -54,40 +34,8 @@ impl TestSuite { impl Config { pub fn load() -> Result { let content = std::fs::read_to_string(CONFIG_FILE) - .context(format!("Unable to open {}", CONFIG_FILE))?; - let mut config: Config = toml::from_str(&content).context("parsing toml")?; - - // Append all tests defined in sets into the tests - config.suite = config - .suite - .clone() - .into_iter() - .map(|mut suite| { - let (allow, defined) = match (&suite.allow_tests, &suite.ignore_tests) { - (Some(allow), None) => (true, allow), - (None, Some(ignore)) => (false, ignore), - _ => bail!("ignore_tests or allow_tests should be specified"), - }; - let mut all = Vec::new(); - for test_name in defined { - if let Some(setname) = test_name.strip_prefix('&') { - let set: Vec<_> = config.set.iter().filter(|ts| ts.id == setname).collect(); - ensure!(!set.is_empty(), "no tests sets found for id '{}'", setname); - set.iter().for_each(|ts| all.append(&mut ts.tests.clone())); - } else { - all.push(test_name.clone()); - } - } - all.sort(); - if allow { - suite.allow_tests = Some(all); - } else { - suite.ignore_tests = Some(all); - } - Ok(suite) - }) - .collect::>()?; - Ok(config) + .context(format!("Unable to open {CONFIG_FILE}"))?; + toml::from_str(&content).context("parsing toml") } pub fn suite(&self, name: &str) -> Result<&TestSuite> { self.suite diff --git a/testool/src/main.rs b/testool/src/main.rs index caf7df3f49..e521420ec6 100644 --- a/testool/src/main.rs +++ b/testool/src/main.rs @@ -47,9 +47,13 @@ struct Args { #[clap(long)] ls: bool, - /// Cache execution results + /// Cache execution results, default to be latest result file #[clap(long)] - cache: Option, + cache: Option, + + /// do not use cache + #[clap(long)] + use_cache: bool, /// whitelist level from cache result #[clap(short, long, value_parser, value_delimiter = ',')] @@ -73,10 +77,10 @@ struct Args { } fn run_single_test(test: StateTest, circuits_config: CircuitsConfig) -> Result<()> { - println!("{}", &test); + log::info!("{}", &test); let trace = geth_trace(test.clone())?; crate::utils::print_trace(trace)?; - println!( + log::info!( "result={:?}", run_test(test, TestSuite::default(), circuits_config) ); @@ -155,30 +159,54 @@ fn go() -> Result<()> { REPORT_FOLDER, args.suite, timestamp, git_hash ); + let cache_file_name = if !args.use_cache { + None + } else { + let mut history_reports = + glob::glob(format!("{REPORT_FOLDER}/{}.*.*.csv", args.suite).as_str())? + .collect::, glob::GlobError>>()? + .into_iter() + .map(|path| { + path.metadata() + .and_then(|meta| meta.created()) + .map(|created| (path, created)) + }) + .collect::, std::io::Error>>()?; + // sort by timestamp + history_reports.sort_by_key(|(_, created)| *created); + // use latest cache if exists + args.cache + .or_else(|| history_reports.pop().map(|(path, _)| path)) + }; + // when running a report, the tests result of the containing cache file // are used, but by default removing all Ignored tests // Another way is to skip the test which level not in whitelist_levels - let mut previous_results = if let Some(cache_filename) = args.cache { + let mut previous_results = if let Some(cache_filename) = cache_file_name { let whitelist_levels = HashSet::::from_iter(args.levels); - let mut previous_results = Results::from_file(PathBuf::from(cache_filename))?; + let mut previous_results = Results::from_file(cache_filename).unwrap(); + + info!("loaded {} test results", previous_results.tests.len()); if !whitelist_levels.is_empty() { // if whitelist is provided, test not in whitelist will be skip previous_results .tests .retain(|_, test| !whitelist_levels.contains(&test.level)); } else { - // by default only skip ignore - previous_results - .tests - .retain(|_, test| test.level != ResultLevel::Ignored); + // by default skip ignore and success tests + previous_results.tests.retain(|_, test| { + test.level == ResultLevel::Ignored || test.level == ResultLevel::Success + }); } previous_results } else { Results::default() }; + previous_results.set_cache(PathBuf::from(csv_filename)); + previous_results.write_cache()?; run_statetests_suite(state_tests, &circuits_config, &suite, &mut previous_results)?; // filter non-csv files and files from the same commit @@ -188,7 +216,7 @@ fn go() -> Result<()> { let filename = f.unwrap().file_name().to_str().unwrap().to_string(); (filename.starts_with(&format!("{}.", args.suite)) && filename.ends_with(".csv") - && !filename.contains(&format!(".{}.", git_hash))) + && !filename.contains(&format!(".{git_hash}."))) .then_some(filename) }) .collect(); @@ -196,8 +224,8 @@ fn go() -> Result<()> { files.sort_by(|f, s| s.cmp(f)); let previous = if !files.is_empty() { let file = files.remove(0); - let path = format!("{}/{}", REPORT_FOLDER, file); - info!("Comparing with previous results in {}", path); + let path = format!("{REPORT_FOLDER}/{file}"); + info!("Comparing with previous results in {path}"); Some((file, Results::from_file(PathBuf::from(path))?)) } else { None @@ -209,7 +237,7 @@ fn go() -> Result<()> { info!("{}", html_filename); } else { let mut results = if let Some(cache_filename) = args.cache { - Results::with_cache(PathBuf::from(cache_filename))? + Results::with_cache(cache_filename)? } else { Results::default() }; @@ -231,6 +259,6 @@ fn go() -> Result<()> { fn main() { if let Err(err) = go() { - eprintln!("Error found {}", err); + eprintln!("Error found {err}"); } } diff --git a/testool/src/statetest/executor.rs b/testool/src/statetest/executor.rs index e46dbdcfeb..524f0280d2 100644 --- a/testool/src/statetest/executor.rs +++ b/testool/src/statetest/executor.rs @@ -4,7 +4,7 @@ use bus_mapping::{ circuit_input_builder::{CircuitInputBuilder, FixedCParams}, mock::BlockData, }; -use eth_types::{geth_types, Address, Bytes, GethExecTrace, U256, U64}; +use eth_types::{geth_types, Address, Bytes, GethExecTrace, ToBigEndian, U256, U64}; use ethers_core::{ k256::ecdsa::SigningKey, types::{transaction::eip2718::TypedTransaction, TransactionRequest}, @@ -36,21 +36,37 @@ pub enum StateTestError { SkipTestMaxGasLimit(u64), #[error("SkipTestMaxSteps({0})")] SkipTestMaxSteps(usize), + #[error("SkipTestSelfDestruct")] + SkipTestSelfDestruct, + #[error("SkipTestDifficulty")] + // scroll evm always returns 0 for "difficulty" opcode + SkipTestDifficulty, + #[error("SkipTestBalanceOverflow")] + SkipTestBalanceOverflow, #[error("Exception(expected:{expected:?}, found:{found:?})")] Exception { expected: bool, found: String }, } impl StateTestError { pub fn is_skip(&self) -> bool { + // Avoid lint `variant is never constructed` if no feature skip-self-destruct. + let _ = StateTestError::SkipTestSelfDestruct; + let _ = StateTestError::SkipTestDifficulty; + matches!( self, - StateTestError::SkipTestMaxSteps(_) | StateTestError::SkipTestMaxGasLimit(_) + StateTestError::SkipTestMaxSteps(_) + | StateTestError::SkipTestMaxGasLimit(_) + | StateTestError::SkipTestSelfDestruct + | StateTestError::SkipTestBalanceOverflow + | StateTestError::SkipTestDifficulty ) } } #[derive(Default, Debug, Clone)] pub struct CircuitsConfig { + pub verbose: bool, pub super_circuit: bool, } @@ -58,11 +74,13 @@ fn check_post( builder: &CircuitInputBuilder, post: &HashMap, ) -> Result<(), StateTestError> { + log::trace!("check post"); // check if the generated account data is the expected one for (address, expected) in post { let (_, actual) = builder.sdb.get_account(address); if expected.balance.map(|v| v == actual.balance) == Some(false) { + log::error!("balance mismatch, expected {expected:?} actual {actual:?}"); return Err(StateTestError::BalanceMismatch { expected: expected.balance.unwrap(), found: actual.balance, @@ -70,6 +88,7 @@ fn check_post( } if expected.nonce.map(|v| v == actual.nonce) == Some(false) { + log::error!("nonce mismatch, expected {expected:?} actual {actual:?}"); return Err(StateTestError::NonceMismatch { expected: expected.nonce.unwrap(), found: actual.nonce, @@ -96,6 +115,9 @@ fn check_post( for (slot, expected_value) in &expected.storage { let actual_value = actual.storage.get(slot).cloned().unwrap_or_else(U256::zero); if expected_value != &actual_value { + log::error!( + "StorageMismatch address {address:?}, expected {expected:?} actual {actual:?}" + ); return Err(StateTestError::StorageMismatch { slot: *slot, expected: *expected_value, @@ -104,6 +126,7 @@ fn check_post( } } } + log::trace!("check post done"); Ok(()) } @@ -137,7 +160,7 @@ fn into_traceconfig(st: StateTest) -> (String, TraceConfig, StateTestResult) { number: U64::from(st.env.current_number), difficulty: st.env.current_difficulty, gas_limit: U256::from(st.env.current_gas_limit), - base_fee: U256::one(), + base_fee: st.env.current_base_fee, }, transactions: vec![geth_types::Transaction { @@ -155,7 +178,7 @@ fn into_traceconfig(st: StateTest) -> (String, TraceConfig, StateTestResult) { r: sig.r, s: sig.s, }], - accounts: st.pre, + accounts: st.pre.into_iter().collect(), ..Default::default() }, st.result, @@ -171,6 +194,45 @@ pub fn geth_trace(st: StateTest) -> Result { Ok(geth_traces.remove(0)) } +fn check_geth_traces( + geth_traces: &[GethExecTrace], + suite: &TestSuite, + verbose: bool, +) -> Result<(), StateTestError> { + if geth_traces.iter().any(|gt| { + gt.struct_logs.iter().any(|sl| { + sl.op == eth_types::evm_types::OpcodeId::SELFDESTRUCT + || sl.op == eth_types::evm_types::OpcodeId::INVALID(0xff) + }) + }) { + return Err(StateTestError::SkipTestSelfDestruct); + } + + if geth_traces.iter().any(|gt| { + gt.struct_logs + .iter() + .any(|sl| sl.op == eth_types::evm_types::OpcodeId::DIFFICULTY) + }) { + return Err(StateTestError::SkipTestDifficulty); + } + + if geth_traces[0].struct_logs.len() as u64 > suite.max_steps { + return Err(StateTestError::SkipTestMaxSteps( + geth_traces[0].struct_logs.len(), + )); + } + + if suite.max_gas > 0 && geth_traces[0].gas > suite.max_gas { + return Err(StateTestError::SkipTestMaxGasLimit(geth_traces[0].gas)); + } + if verbose { + if let Err(e) = crate::utils::print_trace(geth_traces[0].clone()) { + log::error!("fail to pretty print trace {e:?}"); + } + } + Ok(()) +} + pub fn run_test( st: StateTest, suite: TestSuite, @@ -180,6 +242,11 @@ pub fn run_test( let (_, trace_config, post) = into_traceconfig(st.clone()); + for acc in trace_config.accounts.values() { + if acc.balance.to_be_bytes()[0] != 0u8 { + return Err(StateTestError::SkipTestBalanceOverflow); + } + } let geth_traces = external_tracer::trace(&trace_config); let geth_traces = match (geth_traces, st.exception) { @@ -199,15 +266,7 @@ pub fn run_test( } }; - if geth_traces[0].struct_logs.len() as u64 > suite.max_steps { - return Err(StateTestError::SkipTestMaxSteps( - geth_traces[0].struct_logs.len(), - )); - } - - if suite.max_gas > 0 && geth_traces[0].gas > suite.max_gas { - return Err(StateTestError::SkipTestMaxGasLimit(geth_traces[0].gas)); - } + check_geth_traces(&geth_traces, &suite, circuits_config.verbose)?; let transactions = trace_config .transactions @@ -293,7 +352,6 @@ pub fn run_test( let prover = MockProver::run(k, &circuit, instance).unwrap(); prover.assert_satisfied_par(); }; - check_post(&builder, &post)?; Ok(()) diff --git a/testool/src/statetest/json.rs b/testool/src/statetest/json.rs index c136152cda..b60c5157f1 100644 --- a/testool/src/statetest/json.rs +++ b/testool/src/statetest/json.rs @@ -1,17 +1,25 @@ use super::{ parse, - spec::{AccountMatch, Env, StateTest}, + spec::{AccountMatch, Env, StateTest, DEFAULT_BASE_FEE}, }; use crate::{compiler::Compiler, utils::MainnetFork}; use anyhow::{bail, Result}; use eth_types::{geth_types::Account, Address, U256}; use ethers_core::{k256::ecdsa::SigningKey, utils::secret_key_to_address}; use serde::Deserialize; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; + +use serde_json::value::Value; + +fn default_block_base_fee() -> String { + DEFAULT_BASE_FEE.to_string() +} #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] struct TestEnv { + #[serde(default = "default_block_base_fee")] + current_base_fee: String, current_coinbase: String, current_difficulty: String, current_gas_limit: String, @@ -33,6 +41,8 @@ struct AccountPost { code: Option, nonce: Option, storage: Option>, + #[allow(dead_code)] + shouldnotexist: Option, } #[derive(Debug, Clone, Deserialize)] @@ -45,7 +55,7 @@ struct AccountPre { #[derive(Debug, Clone, Deserialize)] struct Expect { - indexes: Indexes, + indexes: Option, network: Vec, result: HashMap, } @@ -88,18 +98,19 @@ impl Refs { } pub struct JsonStateTestBuilder<'a> { - compiler: &'a mut Compiler, + compiler: &'a Compiler, } impl<'a> JsonStateTestBuilder<'a> { - pub fn new(compiler: &'a mut Compiler) -> Self { + pub fn new(compiler: &'a Compiler) -> Self { Self { compiler } } /// generates `StateTest` vectors from a ethereum josn test specification pub fn load_json(&mut self, path: &str, source: &str) -> Result> { let mut state_tests = Vec::new(); - let tests: HashMap = serde_json::from_str(source)?; + let tests: HashMap = + serde_json::from_str(&strip_json_comments(source))?; for (test_name, test) in tests { let env = Self::parse_env(&test.env)?; @@ -134,9 +145,21 @@ impl<'a> JsonStateTestBuilder<'a> { let mut expects = Vec::new(); for expect in test.expect { - let data_refs = Self::parse_refs(&expect.indexes.data)?; - let gas_refs = Self::parse_refs(&expect.indexes.gas)?; - let value_refs = Self::parse_refs(&expect.indexes.value)?; + // Considered as Anys if missing `indexes`. + let (data_refs, gas_refs, value_refs) = if let Some(indexes) = expect.indexes { + ( + Self::parse_refs(&indexes.data)?, + Self::parse_refs(&indexes.gas)?, + Self::parse_refs(&indexes.value)?, + ) + } else { + ( + Refs(vec![Ref::Any]), + Refs(vec![Ref::Any]), + Refs(vec![Ref::Any]), + ) + }; + let result = self.parse_accounts_post(&expect.result)?; if MainnetFork::in_network_range(&expect.network)? { @@ -162,10 +185,7 @@ impl<'a> JsonStateTestBuilder<'a> { state_tests.push(StateTest { path: path.to_string(), - id: format!( - "{}_d{}_g{}_v{}", - test_name, idx_data, idx_gas, idx_value - ), + id: format!("{test_name}_d{idx_data}_g{idx_gas}_v{idx_value}"), env: env.clone(), pre: pre.clone(), result: result.clone(), @@ -191,6 +211,8 @@ impl<'a> JsonStateTestBuilder<'a> { /// parse env section fn parse_env(env: &TestEnv) -> Result { Ok(Env { + current_base_fee: parse::parse_u256(&env.current_base_fee) + .unwrap_or_else(|_| U256::from(DEFAULT_BASE_FEE)), current_coinbase: parse::parse_address(&env.current_coinbase)?, current_difficulty: parse::parse_u256(&env.current_difficulty)?, current_gas_limit: parse::parse_u64(&env.current_gas_limit)?, @@ -204,8 +226,8 @@ impl<'a> JsonStateTestBuilder<'a> { fn parse_accounts_pre( &mut self, accounts_pre: &HashMap, - ) -> Result> { - let mut accounts = HashMap::new(); + ) -> Result> { + let mut accounts = BTreeMap::new(); for (address, acc) in accounts_pre { let address = parse::parse_address(address)?; let mut storage = HashMap::new(); @@ -291,6 +313,25 @@ impl<'a> JsonStateTestBuilder<'a> { } } +fn strip_json_comments(json: &str) -> String { + fn strip(value: Value) -> Value { + use Value::*; + match value { + Array(vec) => Array(vec.into_iter().map(strip).collect()), + Object(map) => Object( + map.into_iter() + .filter(|(k, _)| !k.starts_with("//")) + .map(|(k, v)| (k, strip(v))) + .collect(), + ), + _ => value, + } + } + + let value: Value = serde_json::from_str(json).unwrap(); + strip(value).to_string() +} + #[cfg(test)] mod test { use super::*; @@ -360,8 +401,8 @@ mod test { "#; #[test] fn test_json_parse() -> Result<()> { - let mut compiler = Compiler::new(true, None)?; - let mut builder = JsonStateTestBuilder::new(&mut compiler); + let compiler = Compiler::new(true, None)?; + let mut builder = JsonStateTestBuilder::new(&compiler); let test = builder.load_json("test_path", JSON)?.remove(0); let acc095e = Address::from_str("0x095e7baea6a6c7c4c2dfeb977efac326af552d87")?; @@ -370,6 +411,7 @@ mod test { path: "test_path".to_string(), id: "add11_d0_g0_v0".to_string(), env: Env { + current_base_fee: U256::from(DEFAULT_BASE_FEE), current_coinbase: Address::from_str("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba")?, current_difficulty: U256::from(131072u64), current_gas_limit: 0xFF112233445566, @@ -391,13 +433,14 @@ mod test { nonce: 0, value: U256::from(100000u64), data: Bytes::from(hex::decode("6001")?), - pre: HashMap::from([( + pre: BTreeMap::from([( acc095e, Account { address: acc095e, + nonce: 0.into(), balance: U256::from(1000000000000000000u64), code: Bytes::from(hex::decode("600160010160005500")?), - ..Default::default() + storage: HashMap::new(), }, )]), result: HashMap::from([( @@ -417,4 +460,13 @@ mod test { Ok(()) } + + #[test] + fn test_strip() { + let original = r#"{"//a":"a1","b":[{"c":"c1","//d":"d1"}]}"#; + let expected = r#"{"b":[{"c":"c1"}]}"#; + + let stripped = strip_json_comments(original); + assert_eq!(expected, stripped); + } } diff --git a/testool/src/statetest/parse.rs b/testool/src/statetest/parse.rs index 2cd2d32a4e..0576cc8d35 100644 --- a/testool/src/statetest/parse.rs +++ b/testool/src/statetest/parse.rs @@ -5,9 +5,14 @@ use crate::{abi, Compiler}; use anyhow::{bail, Context, Result}; use eth_types::{Address, Bytes, H256, U256}; use log::debug; +use once_cell::sync::Lazy; +use regex::Regex; type Label = String; +static YUL_FRAGMENT_PARSER: Lazy = + Lazy::new(|| Regex::new(r#"\s*(?P\w+)?\s*(?P\{[\S\s]*)"#).unwrap()); + /// returns the element as an address pub fn parse_address(as_str: &str) -> Result
{ let hex = as_str.strip_prefix("0x").unwrap_or(as_str); @@ -61,7 +66,7 @@ fn decompose_tags(expr: &str) -> HashMap { /// returns the element as calldata bytes, supports 0x, :raw, :abi, :yul and /// { LLL } -pub fn parse_calldata(compiler: &mut Compiler, as_str: &str) -> Result<(Bytes, Option