Update test submodule to post-Istanbul tests (#590)

* tests/ethereum-tests: update to newest develop

* kore-json: do not specify version of python

* driver: add postStateHash key

* Makefile: rename vm_tests => all_vm_tests, bchain_tests => all_bchain_tests

* Makefile: add test-slow-conformance and test-failing-conformance

* tests/{failing,slow}.{ocaml,llvm}: clear failing/slow lists

* tests/{failing,slow}.{ocaml,llvm}: add slow/failing tests

-   A test is slow if it takes >120s or >10GB RAM.
-   A test is failing if it fails before 120s is up.

* tests/failing.web3: remove now passing test

* kore-json: use sys.stdout.write instead of print

* kore-json.py: add back in newline

Makefile

@@ -44,7 +44,7 @@ OPAM ?= opam
         build build-all build-ocaml build-java build-node build-haskell build-llvm build-web3                                                          \
         defn java-defn ocaml-defn node-defn web3-defn haskell-defn llvm-defn                                                                           \
         split-tests                                                                                                                                    \
-        test test-all test-conformance test-rest-conformance test-all-conformance                                                                      \
+        test test-all test-conformance test-rest-conformance test-all-conformance test-slow-conformance test-failing-conformance                       \
         test-vm test-rest-vm test-all-vm test-bchain test-rest-bchain test-all-bchain                                                                  \
         test-web3 test-all-web3 test-failing-web3                                                                                                      \
         test-prove test-prove-benchmarks test-prove-functional test-prove-opcodes test-prove-erc20 test-prove-bihu test-prove-examples test-klab-prove \
@@ -453,21 +453,23 @@ failing_conformance_tests = $(shell cat tests/failing.$(TEST_CONCRETE_BACKEND))
 
 test-all-conformance: test-all-vm test-all-bchain
 test-rest-conformance: test-rest-vm test-rest-bchain
+test-slow-conformance: $(slow_conformance_tests:=.run)
+test-failing-conformance: $(failing_conformance_tests:=.run)
 test-conformance: test-vm test-bchain
 
-vm_tests         = $(wildcard tests/ethereum-tests/VMTests/*/*.json)
-quick_vm_tests   = $(filter-out $(slow_conformance_tests), $(vm_tests))
+all_vm_tests     = $(wildcard tests/ethereum-tests/VMTests/*/*.json)
+quick_vm_tests   = $(filter-out $(slow_conformance_tests), $(all_vm_tests))
 passing_vm_tests = $(filter-out $(failing_conformance_tests), $(quick_vm_tests))
-rest_vm_tests    = $(filter-out $(passing_vm_tests), $(vm_tests))
+rest_vm_tests    = $(filter-out $(passing_vm_tests), $(all_vm_tests))
 
-test-all-vm: $(vm_tests:=.run)
+test-all-vm: $(all_vm_tests:=.run)
 test-rest-vm: $(rest_vm_tests:=.run)
 test-vm: $(passing_vm_tests:=.run)
 
-bchain_tests         = $(wildcard tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/*/*.json)
-quick_bchain_tests   = $(filter-out $(slow_conformance_tests), $(bchain_tests))
+all_bchain_tests     = $(wildcard tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/*/*.json)
+quick_bchain_tests   = $(filter-out $(slow_conformance_tests), $(all_bchain_tests))
 passing_bchain_tests = $(filter-out $(failing_conformance_tests), $(quick_bchain_tests))
-rest_bchain_tests    = $(filter-out $(passing_bchain_tests), $(bchain_tests))
+rest_bchain_tests    = $(filter-out $(passing_bchain_tests), $(all_bchain_tests))
 
 test-all-bchain: $(all_bchain_tests:=.run)
 test-rest-bchain: $(rest_bchain_tests:=.run)

driver.md

@@ -294,7 +294,7 @@ Note that `TEST` is sorted here so that key `"network"` comes before key `"pre"`
 ```k
     syntax Set ::= "#postKeys" [function] | "#allPostKeys" [function] | "#checkKeys" [function]
  // -------------------------------------------------------------------------------------------
-    rule #postKeys    => ( SetItem("post") SetItem("postState") )
+    rule #postKeys    => ( SetItem("post") SetItem("postState") SetItem("postStateHash") )
     rule #allPostKeys => ( #postKeys SetItem("expect") SetItem("export") SetItem("expet") )
     rule #checkKeys   => ( #allPostKeys SetItem("logs") SetItem("out") SetItem("gas")
                            SetItem("blockHeader") SetItem("transactions") SetItem("uncleHeaders") SetItem("genesisBlockHeader")

kore-json.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 import json
 import sys
@@ -48,13 +48,14 @@ def print_kast(data, sort="SortJSON"):
     raise AssertionError
 
 def print_klabel(s):
-  print("Lbl" + s.replace("_", "'Unds'").replace("`", "").replace("(.KList)", "{}"), end=' ')
+  sys.stdout.write("Lbl" + s.replace("_", "'Unds'").replace("`", "").replace("(.KList)", "{}") + " ")
 
-print("LblinitGeneratedTopCell{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Stop'Map{}(),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$PGM\")),inj{SortJSON{}, SortKItem{}}(", end=' ')
+sys.stdout.write("LblinitGeneratedTopCell{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Stop'Map{}(),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$PGM\")),inj{SortJSON{}, SortKItem{}}( ")
 print_kast(data)
-print("))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$SCHEDULE\")),inj{SortSchedule{}, SortKItem{}}(", end=' ')
+sys.stdout.write("))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$SCHEDULE\")),inj{SortSchedule{}, SortKItem{}}( ")
 print_klabel(sys.argv[2])
-print("()))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$MODE\")),inj{SortMode{}, SortKItem{}}(", end=' ')
+sys.stdout.write("()))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$MODE\")),inj{SortMode{}, SortKItem{}}( ")
 print_klabel(sys.argv[3])
-print("()))))")
-print()
+sys.stdout.write("()))))\n")
+sys.stdout.write("\n")
+sys.stdout.flush()

tests/failing.llvm

@@ -1,5 +1,8 @@
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g1v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRecursiveCreate/recursiveCreateReturnValue_d0g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d1g0v0.json

tests/failing.ocaml

@@ -1,4 +1,8 @@
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g1v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRecursiveCreate/recursiveCreateReturnValue_d0g0v0.json

tests/failing.web3

@@ -1,7 +1,6 @@
 tests/web3/erc20/erc20capped.in.json
 tests/web3/erc20/erc20capped-partial.in.json
 tests/web3/erc20/erc20detailed.in.json
-tests/web3/erc20/erc20detailed-partial.in.json
 tests/web3/erc20/tokentimelock.in.json
 tests/web3/erc20/tokentimelock-partial.in.json
 tests/web3/eth_getBlockByNumber1Tx-2.in.json

tests/slow.llvm

@@ -1,19 +1,15 @@
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call1MB1024Calldepth_d0g1v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCodeSizeLimit/codesizeOOGInvalidSize_d0g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCodeSizeLimit/codesizeValid_d0g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_3_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_3_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_ecrec_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity2_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_rip160_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_sha256_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Callcode50000_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_2_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_d0g1v0.json
@@ -22,7 +18,6 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSpecialTest/JUMPDEST_AttackwithJump_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth_d1g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d0g0v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d1g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2_d1g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_3_d0g0v0.json
@@ -43,7 +38,3 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
 tests/ethereum-tests/VMTests/vmPerformance/loop-add-10M.json
 tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-10M.json
 tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-unr100-10M.json
-tests/ethereum-tests/VMTests/vmPerformance/loop-exp-1b-1M.json
-tests/ethereum-tests/VMTests/vmPerformance/loop-exp-nop-1M.json
-tests/ethereum-tests/VMTests/vmPerformance/loop-mul.json
-tests/ethereum-tests/VMTests/vmPerformance/loop-mulmod-2M.json

tests/slow.ocaml

@@ -1,6 +1,5 @@
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call1MB1024Calldepth_d0g1v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d0g0v0.json
+tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g1v0.json
@@ -10,16 +9,11 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_ecrec_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity2_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_rip160_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_sha256_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Callcode50000_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_2_d0g1v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_d0g1v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSpecialTest/JUMPDEST_Attack_d0g0v0.json
-tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSpecialTest/JUMPDEST_AttackwithJump_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth_d1g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d0g0v0.json
 tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d1g0v0.json
@@ -43,7 +37,5 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
 tests/ethereum-tests/VMTests/vmPerformance/loop-add-10M.json
 tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-10M.json
 tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-unr100-10M.json
-tests/ethereum-tests/VMTests/vmPerformance/loop-exp-1b-1M.json
-tests/ethereum-tests/VMTests/vmPerformance/loop-exp-nop-1M.json
 tests/ethereum-tests/VMTests/vmPerformance/loop-mul.json
 tests/ethereum-tests/VMTests/vmPerformance/loop-mulmod-2M.json