From ee332bd51fff9ffa505c4b40816c8ca39ddb69a6 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Thu, 28 Sep 2023 13:14:45 -0300 Subject: [PATCH 01/38] feat: fix scripts to use different paths if mina is a submodule of o1js --- src/lib/crypto/kimchi_backend/common/gen_version.sh | 13 +++++++++++-- src/lib/mina_version/normal/gen.sh | 7 ++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/lib/crypto/kimchi_backend/common/gen_version.sh b/src/lib/crypto/kimchi_backend/common/gen_version.sh index 6f3c97a1971..4260551bbbf 100755 --- a/src/lib/crypto/kimchi_backend/common/gen_version.sh +++ b/src/lib/crypto/kimchi_backend/common/gen_version.sh @@ -1,8 +1,17 @@ #!/usr/bin/env bash set -e -o pipefail if [ -z ${MARLIN_REPO_SHA+x} ]; then - marlin_submodule_dir=$(git submodule status | grep proof-systems | sed 's/^[-\ ]//g' | cut -d ' ' -f 2) - marlin_repo_sha=$(cd $marlin_submodule_dir && git rev-parse --short=8 --verify HEAD) + # Check for the existence of the 'mina' submodule + git_root=$(git rev-parse --show-toplevel) + mina_submodule=$(git submodule status | grep "mina" || true) + + if [[ -n "$mina_submodule" ]]; then + marlin_submodule_dir=$(git -C "$git_root/src/mina" submodule status | grep proof-systems | sed 's/^[-\ ]//g' | cut -d ' ' -f 2) + marlin_repo_sha=$(git -C "$git_root/src/mina/$marlin_submodule_dir" rev-parse --short=8 --verify HEAD) + else + marlin_submodule_dir=$(git submodule status | grep proof-systems | sed 's/^[-\ ]//g' | cut -d ' ' -f 2) + marlin_repo_sha=$(git -C "$marlin_submodule_dir" rev-parse --short=8 --verify HEAD) + fi else marlin_repo_sha=$(cut -b -8 <<< "$MARLIN_REPO_SHA") fi diff --git a/src/lib/mina_version/normal/gen.sh b/src/lib/mina_version/normal/gen.sh index ee45e3683ec..ad34bfa5b6c 100755 --- a/src/lib/mina_version/normal/gen.sh +++ b/src/lib/mina_version/normal/gen.sh @@ -12,7 +12,12 @@ pushd "$root" > /dev/null if [[ -e .git ]] && ! git diff --quiet; then id="[DIRTY]$id"; fi commit_date="${MINA_COMMIT_DATE-$(git show HEAD -s --format="%cI" || echo "")}" - pushd src/lib/crypto/proof-systems > /dev/null + mina_submodule=$(git submodule status | grep "mina" || true) + if [[ -n "$mina_submodule" ]]; then + pushd src/mina/src/lib/crypto/proof-systems > /dev/null + else + pushd src/lib/crypto/proof-systems > /dev/null + fi marlin_commit_id="${MARLIN_COMMIT_ID-$(git rev-parse --verify HEAD || echo "")}" marlin_commit_id_short="$(printf '%s' "$marlin_commit_id" | cut -c1-8)" if [[ -e .git ]] && ! git diff --quiet; then marlin_commit_id="[DIRTY]$marlin_commit_id"; fi From 4f20eb65c1f0f6c0b7e8c55a07ff592fbc22f974 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 11 Oct 2023 10:37:37 -0700 Subject: [PATCH 02/38] fix(libp2p_ipc/dune): update compiler flags to suppress warnings -53-55 --- src/libp2p_ipc/dune | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libp2p_ipc/dune b/src/libp2p_ipc/dune index 9819684e44c..a1e76f17f01 100644 --- a/src/libp2p_ipc/dune +++ b/src/libp2p_ipc/dune @@ -1,7 +1,7 @@ (library (name libp2p_ipc) (public_name libp2p_ipc) - (flags -w -53) + (flags -w -53-55) (inline_tests (flags -verbose -show-counts)) (libraries ;; opam libraries From 3b186d6598982eb998e6c15b6e174de35870c5d6 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Fri, 20 Oct 2023 00:59:06 +0100 Subject: [PATCH 03/38] Reset feature flags before assertion --- src/lib/pickles/step.ml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/pickles/step.ml b/src/lib/pickles/step.ml index f2476f8360b..ee196b1597c 100644 --- a/src/lib/pickles/step.ml +++ b/src/lib/pickles/step.ml @@ -346,8 +346,7 @@ struct Option.map ~f:(Scalar_challenge.map ~f:Challenge.Constant.of_tock_field) (O.joint_combiner_chal o) - ; feature_flags = - t.statement.proof_state.deferred_values.plonk.feature_flags + ; feature_flags = Plonk_types.Features.none_bool } in let xi = scalar_chal O.v in From 641b361509e1dba229cafb26273019f9a12749ee Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 29 Nov 2023 18:02:24 +0100 Subject: [PATCH 04/38] Apply txns: move the stop check to the beginning of the function. --- src/lib/staged_ledger/staged_ledger.ml | 49 +++++++++++++------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index a714941e2b2..e750b04dfe9 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -2093,31 +2093,30 @@ module T = struct Sequence.fold_until transactions_by_fee ~init:(Sequence.empty, [], 0) ~f:(fun (valid_seq, invalid_txns, count) txn -> - match - O1trace.sync_thread - "validate_transaction_against_staged_ledger" (fun () -> - Transaction_validator.apply_transaction_first_pass - ~constraint_constants ~global_slot validating_ledger - ~txn_state_view:current_state_view - (Command (User_command.forget_check txn)) ) - with - | Error e -> - [%log error] - ~metadata: - [ ("user_command", User_command.Valid.to_yojson txn) - ; ("error", Error_json.error_to_yojson e) - ] - "Staged_ledger_diff creation: Skipping user command: \ - $user_command due to error: $error" ; - Continue (valid_seq, (txn, e) :: invalid_txns, count) - | Ok _txn_partially_applied -> - let valid_seq' = - Sequence.append (Sequence.singleton txn) valid_seq - in - let count' = count + 1 in - if count' >= Scan_state.free_space t.scan_state then - Stop (valid_seq', invalid_txns) - else Continue (valid_seq', invalid_txns, count') ) + if count < Scan_state.free_space t.scan_state then + match + O1trace.sync_thread + "validate_transaction_against_staged_ledger" (fun () -> + Transaction_validator.apply_transaction_first_pass + ~constraint_constants ~global_slot validating_ledger + ~txn_state_view:current_state_view + (Command (User_command.forget_check txn)) ) + with + | Error e -> + [%log error] + ~metadata: + [ ("user_command", User_command.Valid.to_yojson txn) + ; ("error", Error_json.error_to_yojson e) + ] + "Staged_ledger_diff creation: Skipping user command: \ + $user_command due to error: $error" ; + Continue (valid_seq, (txn, e) :: invalid_txns, count) + | Ok _txn_partially_applied -> + let valid_seq' = + Sequence.append (Sequence.singleton txn) valid_seq + in + Continue (valid_seq', invalid_txns, count + 1) + else Stop (valid_seq, invalid_txns) ) ~finish:(fun (valid, invalid, _) -> (valid, invalid)) in [%log internal] "Generate_staged_ledger_diff" ; From 041c426c56c6ec3ae6810d1c3689972f6bfa28a0 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 30 Nov 2023 14:31:15 +0100 Subject: [PATCH 05/38] Add the zkapp command per block limit for block production. --- src/lib/block_producer/block_producer.ml | 8 +- src/lib/mina_lib/config.ml | 1 + src/lib/mina_lib/mina_lib.ml | 1 + src/lib/staged_ledger/staged_ledger.ml | 87 ++++++++++++------- src/lib/staged_ledger/staged_ledger.mli | 1 + .../frontier_base/breadcrumb.ml | 3 +- 6 files changed, 66 insertions(+), 35 deletions(-) diff --git a/src/lib/block_producer/block_producer.ml b/src/lib/block_producer/block_producer.ml index 2bc82d7e0fb..b6108860a00 100644 --- a/src/lib/block_producer/block_producer.ml +++ b/src/lib/block_producer/block_producer.ml @@ -163,7 +163,7 @@ let report_transaction_inclusion_failures ~logger failed_txns = let generate_next_state ~constraint_constants ~previous_protocol_state ~time_controller ~staged_ledger ~transactions ~get_completed_work ~logger ~(block_data : Consensus.Data.Block_data.t) ~winner_pk ~scheduled_time - ~log_block_creation ~block_reward_threshold = + ~log_block_creation ~block_reward_threshold ?zkapp_cmd_limit = let open Interruptible.Let_syntax in let previous_protocol_state_body_hash = Protocol_state.body previous_protocol_state |> Protocol_state.Body.hash @@ -200,7 +200,7 @@ let generate_next_state ~constraint_constants ~previous_protocol_state staged_ledger ~coinbase_receiver ~logger ~current_state_view:previous_state_view ~transactions_by_fee:transactions ~get_completed_work - ~log_block_creation ~supercharge_coinbase + ~log_block_creation ~supercharge_coinbase ~zkapp_cmd_limit |> Result.map ~f:(fun (diff, failed_txns) -> if not (List.is_empty failed_txns) then don't_wait_for @@ -611,7 +611,8 @@ let run ~context:(module Context : CONTEXT) ~vrf_evaluator ~prover ~verifier ~trust_system ~get_completed_work ~transaction_resource_pool ~time_controller ~consensus_local_state ~coinbase_receiver ~frontier_reader ~transition_writer ~set_next_producer_timing ~log_block_creation - ~block_reward_threshold ~block_produced_bvar ~vrf_evaluation_state ~net = + ~zkapp_cmd_limit ~block_reward_threshold ~block_produced_bvar + ~vrf_evaluation_state ~net = let open Context in O1trace.sync_thread "produce_blocks" (fun () -> let genesis_breadcrumb = @@ -752,6 +753,7 @@ let run ~context:(module Context : CONTEXT) ~vrf_evaluator ~prover ~verifier ~staged_ledger:(Breadcrumb.staged_ledger crumb) ~transactions ~get_completed_work ~logger ~log_block_creation ~winner_pk:winner_pubkey ~block_reward_threshold + ?zkapp_cmd_limit in [%log internal] "Generate_next_state_done" ; match next_state_opt with diff --git a/src/lib/mina_lib/config.ml b/src/lib/mina_lib/config.ml index 330d8c2a0bb..3378d9b2130 100644 --- a/src/lib/mina_lib/config.ml +++ b/src/lib/mina_lib/config.ml @@ -59,6 +59,7 @@ type t = ; uptime_submitter_keypair : Keypair.t option [@default None] ; uptime_send_node_commit : bool [@default false] ; stop_time : int + ; zkapp_cmd_limit : int option [@default None] ; graphql_control_port : int option [@default None] } [@@deriving make] diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index 415a2e4c2a6..e0fc2603540 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -1380,6 +1380,7 @@ let start t = ~log_block_creation:t.config.log_block_creation ~block_reward_threshold:t.config.block_reward_threshold ~block_produced_bvar:t.components.block_produced_bvar + ~zkapp_cmd_limit:t.config.zkapp_cmd_limit ~vrf_evaluation_state:t.vrf_evaluation_state ~net:t.components.net ; perform_compaction t ; let () = diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index e750b04dfe9..bbfce8df10c 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -1983,11 +1983,51 @@ module T = struct : Ledger.unattached_mask ) ; r + let try_applying_txn (t : t) ?zkapp_cmd_limit ~logger ~validate + (valid_seq, invalid_txns, zkapp_count, total_count) txn = + let open Continue_or_stop in + if total_count < Scan_state.free_space t.scan_state then + match txn with + | User_command.Zkapp_command _ + when Option.value_map ~default:false ~f:(( >= ) zkapp_count) + zkapp_cmd_limit -> + Continue (valid_seq, invalid_txns, zkapp_count, total_count) + | _ -> ( + match + O1trace.sync_thread "validate_transaction_against_staged_ledger" + (fun () -> + validate (Transaction.Command (User_command.forget_check txn)) ) + with + | Error e -> + [%log error] + ~metadata: + [ ("user_command", User_command.Valid.to_yojson txn) + ; ("error", Error_json.error_to_yojson e) + ] + "Staged_ledger_diff creation: Skipping user command: \ + $user_command due to error: $error" ; + Continue + (valid_seq, (txn, e) :: invalid_txns, zkapp_count, total_count) + | Ok _txn_partially_applied -> + let valid_seq' = + Sequence.append (Sequence.singleton txn) valid_seq + in + let zkapp_count' = + match txn with + | Zkapp_command _ -> + zkapp_count + 1 + | Signed_command _ -> + zkapp_count + in + Continue (valid_seq', invalid_txns, zkapp_count', total_count + 1) + ) + else Stop (valid_seq, invalid_txns) + let create_diff ~(constraint_constants : Genesis_constants.Constraint_constants.t) ~(global_slot : Mina_numbers.Global_slot_since_genesis.t) ?(log_block_creation = false) t ~coinbase_receiver ~logger - ~current_state_view + ~current_state_view ~zkapp_cmd_limit ~(transactions_by_fee : User_command.Valid.t Sequence.t) ~(get_completed_work : Transaction_snark_work.Statement.t @@ -2088,36 +2128,17 @@ module T = struct ; ("proof_count", `Int proof_count) ] ; [%log internal] "Validate_and_apply_transactions" ; + let validate = + Transaction_validator.apply_transaction_first_pass + ~constraint_constants ~global_slot validating_ledger + ~txn_state_view:current_state_view + in (*Transactions in reverse order for faster removal if there is no space when creating the diff*) let valid_on_this_ledger, invalid_on_this_ledger = Sequence.fold_until transactions_by_fee - ~init:(Sequence.empty, [], 0) - ~f:(fun (valid_seq, invalid_txns, count) txn -> - if count < Scan_state.free_space t.scan_state then - match - O1trace.sync_thread - "validate_transaction_against_staged_ledger" (fun () -> - Transaction_validator.apply_transaction_first_pass - ~constraint_constants ~global_slot validating_ledger - ~txn_state_view:current_state_view - (Command (User_command.forget_check txn)) ) - with - | Error e -> - [%log error] - ~metadata: - [ ("user_command", User_command.Valid.to_yojson txn) - ; ("error", Error_json.error_to_yojson e) - ] - "Staged_ledger_diff creation: Skipping user command: \ - $user_command due to error: $error" ; - Continue (valid_seq, (txn, e) :: invalid_txns, count) - | Ok _txn_partially_applied -> - let valid_seq' = - Sequence.append (Sequence.singleton txn) valid_seq - in - Continue (valid_seq', invalid_txns, count + 1) - else Stop (valid_seq, invalid_txns) ) - ~finish:(fun (valid, invalid, _) -> (valid, invalid)) + ~init:(Sequence.empty, [], 0, 0) + ~f:(try_applying_txn t ?zkapp_cmd_limit ~validate ~logger) + ~finish:(fun (valid, invalid, _, _) -> (valid, invalid)) in [%log internal] "Generate_staged_ledger_diff" ; let diff, log = @@ -2246,7 +2267,7 @@ let%test_module "staged ledger tests" = Sl.can_apply_supercharged_coinbase_exn ~winner ~global_slot ~epoch_ledger (* Functor for testing with different instantiated staged ledger modules. *) - let create_and_apply_with_state_body_hash + let create_and_apply_with_state_body_hash ?zkapp_cmd_limit ?(coinbase_receiver = coinbase_receiver) ?(winner = self_pk) ~(current_state_view : Zkapp_precondition.Protocol_state.View.t) ~global_slot ~state_and_body_hash sl txns stmt_to_work = @@ -2258,7 +2279,7 @@ let%test_module "staged ledger tests" = Sl.create_diff ~constraint_constants ~global_slot !sl ~logger ~current_state_view ~transactions_by_fee:txns ~get_completed_work:stmt_to_work ~supercharge_coinbase - ~coinbase_receiver + ~coinbase_receiver ~zkapp_cmd_limit in let diff, _invalid_txns = match diff with @@ -3335,7 +3356,7 @@ let%test_module "staged ledger tests" = ~logger ~current_state_view ~transactions_by_fee:cmds_this_iter ~get_completed_work:stmt_to_work ~coinbase_receiver - ~supercharge_coinbase:true + ~supercharge_coinbase:true ~zkapp_cmd_limit:None in match diff_result with | Ok (diff, _invalid_txns) -> @@ -4164,6 +4185,7 @@ let%test_module "staged ledger tests" = ~transactions_by_fee:(Sequence.of_list [ invalid_command ]) ~get_completed_work:(stmt_to_work_zero_fee ~prover:self_pk) ~coinbase_receiver ~supercharge_coinbase:false + ~zkapp_cmd_limit:None in ( match diff_result with | Ok (diff, _invalid_txns) -> @@ -4253,6 +4275,7 @@ let%test_module "staged ledger tests" = ~transactions_by_fee:(Sequence.of_list [ valid_command ]) ~get_completed_work:(stmt_to_work_zero_fee ~prover:self_pk) ~coinbase_receiver ~supercharge_coinbase:false + ~zkapp_cmd_limit:None in match diff_result with | Error e -> @@ -4477,6 +4500,7 @@ let%test_module "staged ledger tests" = ] ) ~get_completed_work:(stmt_to_work_zero_fee ~prover:self_pk) ~coinbase_receiver ~supercharge_coinbase:false + ~zkapp_cmd_limit:None with | Error e -> Error.raise (Pre_diff_info.Error.to_error e) @@ -4802,6 +4826,7 @@ let%test_module "staged ledger tests" = ~get_completed_work: (stmt_to_work_zero_fee ~prover:self_pk) ~coinbase_receiver ~supercharge_coinbase:false + ~zkapp_cmd_limit:None with | Error e -> Error.raise (Pre_diff_info.Error.to_error e) diff --git a/src/lib/staged_ledger/staged_ledger.mli b/src/lib/staged_ledger/staged_ledger.mli index fce202716e6..d9d2dd817be 100644 --- a/src/lib/staged_ledger/staged_ledger.mli +++ b/src/lib/staged_ledger/staged_ledger.mli @@ -242,6 +242,7 @@ val create_diff : -> coinbase_receiver:Public_key.Compressed.t -> logger:Logger.t -> current_state_view:Zkapp_precondition.Protocol_state.View.t + -> zkapp_cmd_limit:int option -> transactions_by_fee:User_command.Valid.t Sequence.t -> get_completed_work: ( Transaction_snark_work.Statement.t diff --git a/src/lib/transition_frontier/frontier_base/breadcrumb.ml b/src/lib/transition_frontier/frontier_base/breadcrumb.ml index 335056ed51b..566e4ade2c3 100644 --- a/src/lib/transition_frontier/frontier_base/breadcrumb.ml +++ b/src/lib/transition_frontier/frontier_base/breadcrumb.ml @@ -336,6 +336,7 @@ module For_tests = struct precomputed_values.Precomputed_values.constraint_constants ~constants:precomputed_values.consensus_constants in + let zkapp_cmd_limit = None in let%map supercharge_coinbase = Quickcheck.Generator.bool in fun parent_breadcrumb -> let open Deferred.Let_syntax in @@ -384,7 +385,7 @@ module For_tests = struct ~global_slot:current_state_view.global_slot_since_genesis ~constraint_constants:precomputed_values.constraint_constants ~coinbase_receiver ~current_state_view ~supercharge_coinbase - ~transactions_by_fee:transactions ~get_completed_work + ~transactions_by_fee:transactions ~get_completed_work ~zkapp_cmd_limit |> Result.map_error ~f:Staged_ledger.Pre_diff_info.Error.to_error |> Or_error.ok_exn in From 09bcee6e556d41aa1d7affd286cbfc07edf55d02 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 4 Dec 2023 17:03:45 +0100 Subject: [PATCH 06/38] Skip transactions whose dependencies have been skipped. --- src/lib/merkle_ledger/database.ml | 4 +- src/lib/staged_ledger/staged_ledger.ml | 86 +++++++++++++++++++------- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/src/lib/merkle_ledger/database.ml b/src/lib/merkle_ledger/database.ml index ec1e2c1e856..a0a173e98b7 100644 --- a/src/lib/merkle_ledger/database.ml +++ b/src/lib/merkle_ledger/database.ml @@ -713,7 +713,9 @@ module Make (Inputs : Inputs_intf) : List.map locations ~f:Location.merkle_path_dependencies_exn in let all_locs = - List.map list_of_dependencies ~f:(fun deps -> List.map ~f:fst deps |> expand_query) |> List.concat + List.map list_of_dependencies ~f:(fun deps -> + List.map ~f:fst deps |> expand_query ) + |> List.concat in let hashes = get_hash_batch_exn mdb all_locs in snd @@ List.fold_map ~init:hashes ~f:compute_path list_of_dependencies diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index bbfce8df10c..479fad34f37 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -1983,15 +1983,54 @@ module T = struct : Ledger.unattached_mask ) ; r - let try_applying_txn (t : t) ?zkapp_cmd_limit ~logger ~validate - (valid_seq, invalid_txns, zkapp_count, total_count) txn = - let open Continue_or_stop in - if total_count < Scan_state.free_space t.scan_state then + module Application_state = struct + type txn = + ( Signed_command.With_valid_signature.t + , Zkapp_command.Valid.t ) + User_command.t_ + + type t = + { valid_seq : txn Sequence.t + ; invalid : (txn * Error.t) list + ; skipped_by_fee_payer : txn list Account_id.Map.t + ; zkapp_space_remaining : int option + ; total_space_remaining : int + } + + let init ~total_limit ?zkapp_limit = + { valid_seq = Sequence.empty + ; invalid = [] + ; skipped_by_fee_payer = Account_id.Map.empty + ; zkapp_space_remaining = zkapp_limit + ; total_space_remaining = total_limit + } + + let txn_key = function + | User_command.Zkapp_command cmd -> + Zkapp_command.(Valid.forget cmd |> fee_payer) + | User_command.Signed_command cmd -> + Signed_command.(forget_check cmd |> fee_payer) + + let add_skipped_txn t (txn : txn) = + Account_id.Map.update t.skipped_by_fee_payer (txn_key txn) + ~f:(Option.value_map ~default:[ txn ] ~f:(List.cons txn)) + + let dependency_skipped txn t = + Account_id.Map.find t.skipped_by_fee_payer (txn_key txn) |> Option.is_some + + let try_applying_txn ~logger ~validate (state : t) (txn : txn) = + let open Continue_or_stop in match txn with + | _ when state.total_space_remaining < 1 -> + Stop (state.valid_seq, state.invalid) | User_command.Zkapp_command _ - when Option.value_map ~default:false ~f:(( >= ) zkapp_count) - zkapp_cmd_limit -> - Continue (valid_seq, invalid_txns, zkapp_count, total_count) + when Option.value_map ~default:false ~f:(( > ) 1) + state.zkapp_space_remaining -> + Continue + { state with skipped_by_fee_payer = add_skipped_txn state txn } + | _ when dependency_skipped txn state -> + Continue + { state with skipped_by_fee_payer = add_skipped_txn state txn } | _ -> ( match O1trace.sync_thread "validate_transaction_against_staged_ledger" @@ -2006,22 +2045,21 @@ module T = struct ] "Staged_ledger_diff creation: Skipping user command: \ $user_command due to error: $error" ; - Continue - (valid_seq, (txn, e) :: invalid_txns, zkapp_count, total_count) + Continue { state with invalid = (txn, e) :: state.invalid } | Ok _txn_partially_applied -> - let valid_seq' = - Sequence.append (Sequence.singleton txn) valid_seq + let valid_seq = + Sequence.append (Sequence.singleton txn) state.valid_seq in - let zkapp_count' = - match txn with - | Zkapp_command _ -> - zkapp_count + 1 - | Signed_command _ -> - zkapp_count + let zkapp_space_remaining = + Option.map state.zkapp_space_remaining ~f:(fun limit -> + match txn with + | Zkapp_command _ -> + limit - 1 + | Signed_command _ -> + limit ) in - Continue (valid_seq', invalid_txns, zkapp_count', total_count + 1) - ) - else Stop (valid_seq, invalid_txns) + Continue { state with valid_seq; zkapp_space_remaining } ) + end let create_diff ~(constraint_constants : Genesis_constants.Constraint_constants.t) @@ -2136,9 +2174,11 @@ module T = struct (*Transactions in reverse order for faster removal if there is no space when creating the diff*) let valid_on_this_ledger, invalid_on_this_ledger = Sequence.fold_until transactions_by_fee - ~init:(Sequence.empty, [], 0, 0) - ~f:(try_applying_txn t ?zkapp_cmd_limit ~validate ~logger) - ~finish:(fun (valid, invalid, _, _) -> (valid, invalid)) + ~init: + (Application_state.init ?zkapp_limit:zkapp_cmd_limit + ~total_limit:(Scan_state.free_space t.scan_state) ) + ~f:(Application_state.try_applying_txn ~validate ~logger) + ~finish:(fun state -> (state.valid_seq, state.invalid)) in [%log internal] "Generate_staged_ledger_diff" ; let diff, log = From 83b0b1dd093038d9a7bcb62bddcccf13a4613361 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 5 Dec 2023 16:20:30 +0100 Subject: [PATCH 07/38] Add a compile-time configuration of zkapp command limit. --- src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml | 11 +++++++++++ src/config/debug.mlh | 1 + src/config/dev.mlh | 1 + src/config/dev_medium_curves.mlh | 1 + src/config/dev_snark.mlh | 1 + src/config/devnet.mlh | 1 + src/config/fake_hash.mlh | 1 + src/config/fork.mlh | 1 + src/config/fork_at_3757.mlh | 2 +- src/config/fuzz_medium.mlh | 1 + src/config/fuzz_small.mlh | 1 + src/config/integration_tests.mlh | 1 + src/config/lightnet.mlh | 1 + src/config/mainnet.mlh | 1 + src/config/nonconsensus_mainnet.mlh | 1 + src/config/nonconsensus_medium_curves.mlh | 1 + src/config/print_versioned_types.mlh | 1 + src/config/test_archive_processor.mlh | 1 + src/config/test_postake.mlh | 1 + src/config/test_postake_catchup.mlh | 1 + src/config/test_postake_five_even_txns.mlh | 1 + src/config/test_postake_full_epoch.mlh | 1 + src/config/test_postake_holy_grail.mlh | 1 + src/config/test_postake_medium_curves.mlh | 1 + src/config/test_postake_snarkless.mlh | 1 + src/config/test_postake_snarkless_medium_curves.mlh | 1 + src/config/test_postake_split.mlh | 1 + src/config/test_postake_split_medium_curves.mlh | 1 + src/config/test_postake_three_producers.mlh | 1 + src/config/testnet_postake.mlh | 1 + src/config/testnet_postake_many_producers.mlh | 1 + .../testnet_postake_many_producers_medium_curves.mlh | 1 + src/config/testnet_postake_medium_curves.mlh | 1 + src/config/testnet_postake_snarkless.mlh | 1 + src/config/testnet_postake_snarkless_fake_hash.mlh | 1 + src/config/testnet_public.mlh | 1 + src/config/txpool_size.mlh | 1 + 37 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index be21243b53e..e23d64fb7c9 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -14,6 +14,16 @@ let () = Async.Scheduler.set_record_backtraces true [%%endif] +[%%ifndef zkapp_cmd_limit] + +let zkapp_cmd_limit = None + +[%%else] + +[%%inject "zkapp_cmd_limit", zkapp_cmd_limit] + +[%%endif] + type mina_initialization = { mina : Mina_lib.t ; client_trustlist : Unix.Cidr.t list option @@ -1358,6 +1368,7 @@ Pass one of -peer, -peer-list-file, -seed, -peer-list-url.|} ; ; shutdown_on_disconnect = true ; num_threads = snark_worker_parallelism_flag } + ~zkapp_cmd_limit ~snark_coordinator_key:run_snark_coordinator_flag ~snark_pool_disk_location:(conf_dir ^/ "snark_pool") ~wallets_disk_location:(conf_dir ^/ "wallets") diff --git a/src/config/debug.mlh b/src/config/debug.mlh index 12decae561b..7320bf29dc3 100644 --- a/src/config/debug.mlh +++ b/src/config/debug.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/dev.mlh b/src/config/dev.mlh index 648592bda69..8a3fd03d5a7 100644 --- a/src/config/dev.mlh +++ b/src/config/dev.mlh @@ -35,3 +35,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/dev_medium_curves.mlh b/src/config/dev_medium_curves.mlh index 357ad451c3a..ce9adca2723 100644 --- a/src/config/dev_medium_curves.mlh +++ b/src/config/dev_medium_curves.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/dev_snark.mlh b/src/config/dev_snark.mlh index 2ebdcf40877..d6a99590415 100644 --- a/src/config/dev_snark.mlh +++ b/src/config/dev_snark.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/devnet.mlh b/src/config/devnet.mlh index 4ead97ed46b..5e0db186f23 100644 --- a/src/config/devnet.mlh +++ b/src/config/devnet.mlh @@ -46,3 +46,4 @@ (* 2*block_window_duration *) [%%define compaction_interval 360000] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/fake_hash.mlh b/src/config/fake_hash.mlh index 7bc6edfc50e..4d93a244f90 100644 --- a/src/config/fake_hash.mlh +++ b/src/config/fake_hash.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/fork.mlh b/src/config/fork.mlh index 65e7457ec82..e6504910de5 100644 --- a/src/config/fork.mlh +++ b/src/config/fork.mlh @@ -1,3 +1,4 @@ [%%undef fork_previous_length] [%%undef fork_previous_state_hash] [%%undef fork_genesis_slot] +[%%undef zkapp_cmd_limit] diff --git a/src/config/fork_at_3757.mlh b/src/config/fork_at_3757.mlh index 0c27577faf0..f4b4cd6fc6d 100644 --- a/src/config/fork_at_3757.mlh +++ b/src/config/fork_at_3757.mlh @@ -1,3 +1,3 @@ [%%define fork_previous_length 3757] [%%define fork_previous_state_hash "3NKR3QYJ7qwxiGgX39umahgdT8BH5yXBQwQtpYZdvodCXcsndK7f"] -[%%define fork_genesis_slot 12796] \ No newline at end of file +[%%define fork_genesis_slot 12796][%%undef zkapp_cmd_limit] diff --git a/src/config/fuzz_medium.mlh b/src/config/fuzz_medium.mlh index 180e221c963..8d577a8a421 100644 --- a/src/config/fuzz_medium.mlh +++ b/src/config/fuzz_medium.mlh @@ -32,3 +32,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/fuzz_small.mlh b/src/config/fuzz_small.mlh index 6bd5e3af1b1..d08978539de 100644 --- a/src/config/fuzz_small.mlh +++ b/src/config/fuzz_small.mlh @@ -32,3 +32,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/integration_tests.mlh b/src/config/integration_tests.mlh index a4d92492e3c..afd5ab13d1b 100644 --- a/src/config/integration_tests.mlh +++ b/src/config/integration_tests.mlh @@ -5,3 +5,4 @@ (* This profile is only used for the test executive binary, so we don't need snark keys, a valid genesis proof, etc. *) [%%import "/src/config/proof_level/none.mlh"] +[%%undef zkapp_cmd_limit] diff --git a/src/config/lightnet.mlh b/src/config/lightnet.mlh index d18125aa34f..131339d32a3 100644 --- a/src/config/lightnet.mlh +++ b/src/config/lightnet.mlh @@ -46,3 +46,4 @@ (* 2*block_window_duration *) [%%define compaction_interval 360000] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/mainnet.mlh b/src/config/mainnet.mlh index f7f5d015a7e..986ca0c42eb 100644 --- a/src/config/mainnet.mlh +++ b/src/config/mainnet.mlh @@ -46,3 +46,4 @@ (* 2*block_window_duration *) [%%define compaction_interval 360000] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/nonconsensus_mainnet.mlh b/src/config/nonconsensus_mainnet.mlh index 8210abd05e1..4cbc2c3edd6 100644 --- a/src/config/nonconsensus_mainnet.mlh +++ b/src/config/nonconsensus_mainnet.mlh @@ -2,3 +2,4 @@ [%%undef consensus_mechanism] [%%undef compaction_interval] +[%%undef zkapp_cmd_limit] diff --git a/src/config/nonconsensus_medium_curves.mlh b/src/config/nonconsensus_medium_curves.mlh index a23808ff664..6d719774d4c 100644 --- a/src/config/nonconsensus_medium_curves.mlh +++ b/src/config/nonconsensus_medium_curves.mlh @@ -32,3 +32,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/print_versioned_types.mlh b/src/config/print_versioned_types.mlh index 597d2df3bd2..0718dc6e2f4 100644 --- a/src/config/print_versioned_types.mlh +++ b/src/config/print_versioned_types.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_archive_processor.mlh b/src/config/test_archive_processor.mlh index fb70411bda6..83f70ac422e 100644 --- a/src/config/test_archive_processor.mlh +++ b/src/config/test_archive_processor.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake.mlh b/src/config/test_postake.mlh index 8b91e0c0a10..0740e45d322 100644 --- a/src/config/test_postake.mlh +++ b/src/config/test_postake.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_catchup.mlh b/src/config/test_postake_catchup.mlh index 32b331cbb72..92faf7ca6b6 100644 --- a/src/config/test_postake_catchup.mlh +++ b/src/config/test_postake_catchup.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_five_even_txns.mlh b/src/config/test_postake_five_even_txns.mlh index fa8e6b0c607..b926937e7c7 100644 --- a/src/config/test_postake_five_even_txns.mlh +++ b/src/config/test_postake_five_even_txns.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_full_epoch.mlh b/src/config/test_postake_full_epoch.mlh index e349c690f35..2c2dcd750c4 100644 --- a/src/config/test_postake_full_epoch.mlh +++ b/src/config/test_postake_full_epoch.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_holy_grail.mlh b/src/config/test_postake_holy_grail.mlh index 5fa916779e4..c87f5880289 100644 --- a/src/config/test_postake_holy_grail.mlh +++ b/src/config/test_postake_holy_grail.mlh @@ -32,3 +32,4 @@ [%%import "/src/config/fork.mlh"] [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_medium_curves.mlh b/src/config/test_postake_medium_curves.mlh index e480e1a5b46..01ddc77ce9d 100644 --- a/src/config/test_postake_medium_curves.mlh +++ b/src/config/test_postake_medium_curves.mlh @@ -32,3 +32,4 @@ [%%import "/src/config/fork.mlh"] [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_snarkless.mlh b/src/config/test_postake_snarkless.mlh index d99cf409b67..6f7a712de7f 100644 --- a/src/config/test_postake_snarkless.mlh +++ b/src/config/test_postake_snarkless.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%define compaction_interval 360000] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_snarkless_medium_curves.mlh b/src/config/test_postake_snarkless_medium_curves.mlh index 7131df31f2f..b89ddd77d5e 100644 --- a/src/config/test_postake_snarkless_medium_curves.mlh +++ b/src/config/test_postake_snarkless_medium_curves.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_split.mlh b/src/config/test_postake_split.mlh index d1d310832f1..8082d95451e 100644 --- a/src/config/test_postake_split.mlh +++ b/src/config/test_postake_split.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_split_medium_curves.mlh b/src/config/test_postake_split_medium_curves.mlh index 393c4ea6818..30694446149 100644 --- a/src/config/test_postake_split_medium_curves.mlh +++ b/src/config/test_postake_split_medium_curves.mlh @@ -33,3 +33,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/test_postake_three_producers.mlh b/src/config/test_postake_three_producers.mlh index 8699b4827bb..57823d30bfb 100644 --- a/src/config/test_postake_three_producers.mlh +++ b/src/config/test_postake_three_producers.mlh @@ -34,3 +34,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/testnet_postake.mlh b/src/config/testnet_postake.mlh index 7b238dfdfd7..ff93f597eb4 100644 --- a/src/config/testnet_postake.mlh +++ b/src/config/testnet_postake.mlh @@ -34,3 +34,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/testnet_postake_many_producers.mlh b/src/config/testnet_postake_many_producers.mlh index d766e7217ce..827ff8f4b74 100644 --- a/src/config/testnet_postake_many_producers.mlh +++ b/src/config/testnet_postake_many_producers.mlh @@ -32,3 +32,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/testnet_postake_many_producers_medium_curves.mlh b/src/config/testnet_postake_many_producers_medium_curves.mlh index 30a7e64bcb8..d8256c49784 100644 --- a/src/config/testnet_postake_many_producers_medium_curves.mlh +++ b/src/config/testnet_postake_many_producers_medium_curves.mlh @@ -34,3 +34,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/testnet_postake_medium_curves.mlh b/src/config/testnet_postake_medium_curves.mlh index 9b69a2d5ec1..206355d8b50 100644 --- a/src/config/testnet_postake_medium_curves.mlh +++ b/src/config/testnet_postake_medium_curves.mlh @@ -46,3 +46,4 @@ (* 2*block_window_duration *) [%%define compaction_interval 360000] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/testnet_postake_snarkless.mlh b/src/config/testnet_postake_snarkless.mlh index 3f20bf5f0c4..cd17d561a6d 100644 --- a/src/config/testnet_postake_snarkless.mlh +++ b/src/config/testnet_postake_snarkless.mlh @@ -34,3 +34,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/testnet_postake_snarkless_fake_hash.mlh b/src/config/testnet_postake_snarkless_fake_hash.mlh index e051017ead0..f7b56e9eb09 100644 --- a/src/config/testnet_postake_snarkless_fake_hash.mlh +++ b/src/config/testnet_postake_snarkless_fake_hash.mlh @@ -35,3 +35,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 0] +[%%undef zkapp_cmd_limit] diff --git a/src/config/testnet_public.mlh b/src/config/testnet_public.mlh index d5a1e3b0803..abbdad8638f 100644 --- a/src/config/testnet_public.mlh +++ b/src/config/testnet_public.mlh @@ -34,3 +34,4 @@ [%%import "/src/config/features/dev.mlh"] [%%undef compaction_interval] [%%define vrf_poll_interval 5000] +[%%undef zkapp_cmd_limit] diff --git a/src/config/txpool_size.mlh b/src/config/txpool_size.mlh index ab13a419f6a..5a6f6a5b408 100644 --- a/src/config/txpool_size.mlh +++ b/src/config/txpool_size.mlh @@ -1,3 +1,4 @@ (* Note this value needs to be consistent across nodes to prevent spurious bans. see comment in transaction_pool.ml for more details. *) [%%define pool_max_size 3000] +[%%undef zkapp_cmd_limit] From 333c5180e18f08dace2d45e4baacf937e48198f2 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 6 Dec 2023 09:29:41 +0100 Subject: [PATCH 08/38] Make an optional argument mandatory. --- src/lib/block_producer/block_producer.ml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/block_producer/block_producer.ml b/src/lib/block_producer/block_producer.ml index b6108860a00..c86e5e85c60 100644 --- a/src/lib/block_producer/block_producer.ml +++ b/src/lib/block_producer/block_producer.ml @@ -160,10 +160,10 @@ let report_transaction_inclusion_failures ~logger failed_txns = let leftover_bytes = available_bytes - base_error_size - 2 in wrap_error (`List (generate_errors failed_txns leftover_bytes)) ) -let generate_next_state ~constraint_constants ~previous_protocol_state - ~time_controller ~staged_ledger ~transactions ~get_completed_work ~logger - ~(block_data : Consensus.Data.Block_data.t) ~winner_pk ~scheduled_time - ~log_block_creation ~block_reward_threshold ?zkapp_cmd_limit = +let generate_next_state ~zkapp_cmd_limit ~constraint_constants + ~previous_protocol_state ~time_controller ~staged_ledger ~transactions + ~get_completed_work ~logger ~(block_data : Consensus.Data.Block_data.t) + ~winner_pk ~scheduled_time ~log_block_creation ~block_reward_threshold = let open Interruptible.Let_syntax in let previous_protocol_state_body_hash = Protocol_state.body previous_protocol_state |> Protocol_state.Body.hash @@ -753,7 +753,7 @@ let run ~context:(module Context : CONTEXT) ~vrf_evaluator ~prover ~verifier ~staged_ledger:(Breadcrumb.staged_ledger crumb) ~transactions ~get_completed_work ~logger ~log_block_creation ~winner_pk:winner_pubkey ~block_reward_threshold - ?zkapp_cmd_limit + ~zkapp_cmd_limit in [%log internal] "Generate_next_state_done" ; match next_state_opt with From 9f2d840a879f23778a84f5c7bae7f8821b33ebb1 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 6 Dec 2023 09:36:00 +0100 Subject: [PATCH 09/38] Insert missing new line. --- src/config/fork_at_3757.mlh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/config/fork_at_3757.mlh b/src/config/fork_at_3757.mlh index f4b4cd6fc6d..b21ba72d7ba 100644 --- a/src/config/fork_at_3757.mlh +++ b/src/config/fork_at_3757.mlh @@ -1,3 +1,4 @@ [%%define fork_previous_length 3757] [%%define fork_previous_state_hash "3NKR3QYJ7qwxiGgX39umahgdT8BH5yXBQwQtpYZdvodCXcsndK7f"] -[%%define fork_genesis_slot 12796][%%undef zkapp_cmd_limit] +[%%define fork_genesis_slot 12796] +[%%undef zkapp_cmd_limit] From 824bd288f4e9a6a3f776a7686e01e47f9a5bc576 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 6 Dec 2023 09:57:51 +0100 Subject: [PATCH 10/38] Make the limit mutable. Style improvements. --- src/lib/mina_lib/config.ml | 2 +- src/lib/staged_ledger/staged_ledger.ml | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/lib/mina_lib/config.ml b/src/lib/mina_lib/config.ml index 3378d9b2130..3aa64d9207d 100644 --- a/src/lib/mina_lib/config.ml +++ b/src/lib/mina_lib/config.ml @@ -59,7 +59,7 @@ type t = ; uptime_submitter_keypair : Keypair.t option [@default None] ; uptime_send_node_commit : bool [@default false] ; stop_time : int - ; zkapp_cmd_limit : int option [@default None] + ; mutable zkapp_cmd_limit : int option [@default None] ; graphql_control_port : int option [@default None] } [@@deriving make] diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index 479fad34f37..f7fc92dbc48 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -2016,16 +2016,15 @@ module T = struct ~f:(Option.value_map ~default:[ txn ] ~f:(List.cons txn)) let dependency_skipped txn t = - Account_id.Map.find t.skipped_by_fee_payer (txn_key txn) |> Option.is_some + Account_id.Map.mem t.skipped_by_fee_payer (txn_key txn) - let try_applying_txn ~logger ~validate (state : t) (txn : txn) = + let try_applying_txn ~logger ~apply (state : t) (txn : txn) = let open Continue_or_stop in - match txn with + match (state.zkapp_space_remaining, txn) with | _ when state.total_space_remaining < 1 -> Stop (state.valid_seq, state.invalid) - | User_command.Zkapp_command _ - when Option.value_map ~default:false ~f:(( > ) 1) - state.zkapp_space_remaining -> + | (Some zkapp_limit, User_command.Zkapp_command _) + when zkapp_limit > 1 -> Continue { state with skipped_by_fee_payer = add_skipped_txn state txn } | _ when dependency_skipped txn state -> @@ -2035,7 +2034,7 @@ module T = struct match O1trace.sync_thread "validate_transaction_against_staged_ledger" (fun () -> - validate (Transaction.Command (User_command.forget_check txn)) ) + apply (Transaction.Command (User_command.forget_check txn)) ) with | Error e -> [%log error] @@ -2166,7 +2165,7 @@ module T = struct ; ("proof_count", `Int proof_count) ] ; [%log internal] "Validate_and_apply_transactions" ; - let validate = + let apply = Transaction_validator.apply_transaction_first_pass ~constraint_constants ~global_slot validating_ledger ~txn_state_view:current_state_view @@ -2177,7 +2176,7 @@ module T = struct ~init: (Application_state.init ?zkapp_limit:zkapp_cmd_limit ~total_limit:(Scan_state.free_space t.scan_state) ) - ~f:(Application_state.try_applying_txn ~validate ~logger) + ~f:(Application_state.try_applying_txn ~apply ~logger) ~finish:(fun state -> (state.valid_seq, state.invalid)) in [%log internal] "Generate_staged_ledger_diff" ; From d89299e38e4a7f64fbd29a36fb093c97b8ada9ce Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 7 Dec 2023 09:07:36 +0100 Subject: [PATCH 11/38] Implement a GQL mutation to alter the limit. --- src/lib/mina_graphql/mina_graphql.ml | 14 +++++++++++++- src/lib/staged_ledger/staged_ledger.ml | 15 ++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/lib/mina_graphql/mina_graphql.ml b/src/lib/mina_graphql/mina_graphql.ml index c24e3e30d45..13aeca287e9 100644 --- a/src/lib/mina_graphql/mina_graphql.ml +++ b/src/lib/mina_graphql/mina_graphql.ml @@ -2561,7 +2561,19 @@ module Queries = struct if not with_seq_no then Error "Missing sequence information" else Ok (Itn_logger.get_logs start_log_id) ) - let commands = [ auth; slots_won; internal_logs ] + let zkapp_cmd_limit = + field "zkAppCommandLimit" + ~args: + Arg.[ arg "limit" ~doc:"ZkApp commands per block limit." ~typ:int ] + ~typ:int + ~doc:"Set zkApp commands per block limit for the block producer." + ~resolve:(fun { ctx = _with_seq_no, mina; _ } () limit -> + let open Mina_lib.Config in + let config = Mina_lib.config mina in + config.zkapp_cmd_limit <- limit ; + limit ) + + let commands = [ auth; slots_won; internal_logs; zkapp_cmd_limit ] end end diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index f7fc92dbc48..df2b23cd302 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -2023,13 +2023,18 @@ module T = struct match (state.zkapp_space_remaining, txn) with | _ when state.total_space_remaining < 1 -> Stop (state.valid_seq, state.invalid) - | (Some zkapp_limit, User_command.Zkapp_command _) - when zkapp_limit > 1 -> + | Some zkapp_limit, User_command.Zkapp_command _ when zkapp_limit > 1 -> Continue - { state with skipped_by_fee_payer = add_skipped_txn state txn } - | _ when dependency_skipped txn state -> + { state with + skipped_by_fee_payer = add_skipped_txn state txn + ; zkapp_space_remaining = Some (zkapp_limit - 1) + } + | Some zkapp_limit, _ when dependency_skipped txn state -> Continue - { state with skipped_by_fee_payer = add_skipped_txn state txn } + { state with + skipped_by_fee_payer = add_skipped_txn state txn + ; zkapp_space_remaining = Some (zkapp_limit - 1) + } | _ -> ( match O1trace.sync_thread "validate_transaction_against_staged_ledger" From 6bfdbb4584b78ca0066b992ec56e7dfa8e29bc9c Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 8 Dec 2023 11:06:24 +0100 Subject: [PATCH 12/38] Move test helpers from transaction_logic tests to appropriate libs. --- src/lib/mina_base/test/helpers/dune | 25 +++++++++++++ .../test/helpers}/test_account.ml | 0 .../test/helpers}/zkapp_cmd_builder.ml | 0 src/lib/mina_ledger/test/helpers/dune | 33 +++++++++++++++++ .../test/helpers/ledger_helpers.ml | 37 +++++++++++++++++++ src/lib/transaction_logic/test/dune | 4 +- src/lib/transaction_logic/test/helpers.ml | 35 ------------------ src/lib/transaction_logic/test/predicates.ml | 1 + .../test/transaction_logic/dune | 2 + .../transaction_logic/transaction_logic.ml | 6 ++- src/lib/transaction_logic/test/zkapp_logic.ml | 6 ++- 11 files changed, 109 insertions(+), 40 deletions(-) create mode 100644 src/lib/mina_base/test/helpers/dune rename src/lib/{transaction_logic/test => mina_base/test/helpers}/test_account.ml (100%) rename src/lib/{transaction_logic/test => mina_base/test/helpers}/zkapp_cmd_builder.ml (100%) create mode 100644 src/lib/mina_ledger/test/helpers/dune create mode 100644 src/lib/mina_ledger/test/helpers/ledger_helpers.ml diff --git a/src/lib/mina_base/test/helpers/dune b/src/lib/mina_base/test/helpers/dune new file mode 100644 index 00000000000..f7734988c12 --- /dev/null +++ b/src/lib/mina_base/test/helpers/dune @@ -0,0 +1,25 @@ +(library + (name mina_base_test_helpers) + (libraries + ;; opam libraries + base + base.caml + core_kernel + integers + sexplib0 + yojson + ;; local libraries + currency + mina_base + mina_base.import + mina_numbers + monad_lib + signature_lib) + (preprocess + (pps + ppx_base + ppx_let + ppx_assert)) + (instrumentation + (backend bisect_ppx))) + diff --git a/src/lib/transaction_logic/test/test_account.ml b/src/lib/mina_base/test/helpers/test_account.ml similarity index 100% rename from src/lib/transaction_logic/test/test_account.ml rename to src/lib/mina_base/test/helpers/test_account.ml diff --git a/src/lib/transaction_logic/test/zkapp_cmd_builder.ml b/src/lib/mina_base/test/helpers/zkapp_cmd_builder.ml similarity index 100% rename from src/lib/transaction_logic/test/zkapp_cmd_builder.ml rename to src/lib/mina_base/test/helpers/zkapp_cmd_builder.ml diff --git a/src/lib/mina_ledger/test/helpers/dune b/src/lib/mina_ledger/test/helpers/dune new file mode 100644 index 00000000000..257b0774791 --- /dev/null +++ b/src/lib/mina_ledger/test/helpers/dune @@ -0,0 +1,33 @@ +(library + (name mina_ledger_test_helpers) + (libraries + ;; opam libraries + base + base.caml + core_kernel + core + integers + sexplib0 + yojson + ;; local libraries + currency + kimchi_backend_common + kimchi_backend.pasta + kimchi_backend.pasta.basic + mina_base + mina_base_test_helpers + mina_ledger + mina_numbers + monad_lib + pickles + pickles.backend + pickles_types + signature_lib) + (preprocess + (pps + ppx_base + ppx_let + ppx_assert)) + (instrumentation + (backend bisect_ppx))) + diff --git a/src/lib/mina_ledger/test/helpers/ledger_helpers.ml b/src/lib/mina_ledger/test/helpers/ledger_helpers.ml new file mode 100644 index 00000000000..f384c0ba90f --- /dev/null +++ b/src/lib/mina_ledger/test/helpers/ledger_helpers.ml @@ -0,0 +1,37 @@ +open Core +open Mina_base +open Mina_base_test_helpers +open Mina_numbers +open Signature_lib +module Ledger = Mina_ledger.Ledger.Ledger_inner + +let noncemap (accounts : Test_account.t list) : + Account_nonce.t Public_key.Compressed.Map.t = + Public_key.Compressed.Map.of_alist_exn + @@ List.map ~f:(fun a -> (a.pk, a.nonce)) accounts + +let bin_log n = + let rec find candidate = + if Int.pow 2 candidate >= n then candidate else find (Int.succ candidate) + in + find 0 + +let ledger_of_accounts accounts = + let open Result.Let_syntax in + let open Test_account in + let module R = Monad_lib.Make_ext2 (Result) in + let depth = bin_log @@ List.length accounts in + let ledger = Ledger.empty ~depth () in + let%map () = + R.iter_m accounts ~f:(fun a -> + let acc_id = account_id a in + let account : Account.t = Account.initialize acc_id in + Ledger.create_new_account ledger acc_id + { account with + balance = a.balance + ; nonce = a.nonce + ; zkapp = a.zkapp + ; token_id = a.token_id + } ) + in + ledger diff --git a/src/lib/transaction_logic/test/dune b/src/lib/transaction_logic/test/dune index ee03e9fd538..9cdfe032e77 100644 --- a/src/lib/transaction_logic/test/dune +++ b/src/lib/transaction_logic/test/dune @@ -21,10 +21,12 @@ kimchi_backend.pasta.basic mina_base mina_base_import + mina_base_test_helpers + mina_ledger + mina_ledger_test_helpers mina_transaction mina_transaction_logic mina_numbers - mina_ledger mina_wire_types monad_lib pasta_bindings diff --git a/src/lib/transaction_logic/test/helpers.ml b/src/lib/transaction_logic/test/helpers.ml index 142f3e56085..5299de0b91f 100644 --- a/src/lib/transaction_logic/test/helpers.ml +++ b/src/lib/transaction_logic/test/helpers.ml @@ -1,7 +1,3 @@ -open Core -open Mina_base -open Mina_numbers -open Signature_lib module Ledger = Mina_ledger.Ledger.Ledger_inner module Transaction_logic = Mina_transaction_logic.Make (Ledger) @@ -12,34 +8,3 @@ module Zk_cmd_result = struct let sexp_of_t (txn, _) = Transaction_logic.Transaction_applied.Zkapp_command_applied.sexp_of_t txn end - -let noncemap (accounts : Test_account.t list) : - Account_nonce.t Public_key.Compressed.Map.t = - Public_key.Compressed.Map.of_alist_exn - @@ List.map ~f:(fun a -> (a.pk, a.nonce)) accounts - -let bin_log n = - let rec find candidate = - if Int.pow 2 candidate >= n then candidate else find (Int.succ candidate) - in - find 0 - -let test_ledger accounts = - let open Result.Let_syntax in - let open Test_account in - let module R = Monad_lib.Make_ext2 (Result) in - let depth = bin_log @@ List.length accounts in - let ledger = Ledger.empty ~depth () in - let%map () = - R.iter_m accounts ~f:(fun a -> - let acc_id = account_id a in - let account : Account.t = Account.initialize acc_id in - Ledger.create_new_account ledger acc_id - { account with - balance = a.balance - ; nonce = a.nonce - ; zkapp = a.zkapp - ; token_id = a.token_id - } ) - in - ledger diff --git a/src/lib/transaction_logic/test/predicates.ml b/src/lib/transaction_logic/test/predicates.ml index cfb57d834a7..6e72f327379 100644 --- a/src/lib/transaction_logic/test/predicates.ml +++ b/src/lib/transaction_logic/test/predicates.ml @@ -1,6 +1,7 @@ open Core open Currency open Mina_base +open Mina_base_test_helpers open Signature_lib open Zkapp_command diff --git a/src/lib/transaction_logic/test/transaction_logic/dune b/src/lib/transaction_logic/test/transaction_logic/dune index 2c65f11f8dc..fd23f64bc05 100644 --- a/src/lib/transaction_logic/test/transaction_logic/dune +++ b/src/lib/transaction_logic/test/transaction_logic/dune @@ -22,10 +22,12 @@ kimchi_backend.pasta.basic mina_base mina_base_import + mina_base_test_helpers mina_transaction mina_transaction_logic mina_numbers mina_ledger + mina_ledger_test_helpers mina_wire_types monad_lib pasta_bindings diff --git a/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml b/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml index 6427a86f6a5..0a10f142e7a 100644 --- a/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml +++ b/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml @@ -1,6 +1,8 @@ open Core_kernel open Currency open Mina_base +open Mina_base_test_helpers +open Mina_ledger_test_helpers open Mina_transaction_logic open Signature_lib open Transaction_logic_tests @@ -86,7 +88,7 @@ let simple_payment () = let txn = signed_command ~fee ~sender ~receiver amount in let txn_state_view = protocol_state in let ledger = - match test_ledger accounts with Ok l -> l | Error _ -> assert false + match Ledger_helpers.ledger_of_accounts accounts with Ok l -> l | Error _ -> assert false in [%test_pred: Transaction_applied.t list Or_error.t] Or_error.is_ok (Transaction_logic.apply_transactions ~constraint_constants ~global_slot @@ -98,7 +100,7 @@ let simple_payment_signer_different_from_fee_payer () = let txn = signed_command ~signer:receiver ~fee ~sender ~receiver amount in let txn_state_view = protocol_state in let ledger = - match test_ledger accounts with Ok l -> l | Error _ -> assert false + match Ledger_helpers.ledger_of_accounts accounts with Ok l -> l | Error _ -> assert false in expect_failure ~error: diff --git a/src/lib/transaction_logic/test/zkapp_logic.ml b/src/lib/transaction_logic/test/zkapp_logic.ml index 295a17c8db8..4e5010ef7fe 100644 --- a/src/lib/transaction_logic/test/zkapp_logic.ml +++ b/src/lib/transaction_logic/test/zkapp_logic.ml @@ -1,6 +1,8 @@ open Core_kernel open Currency open Mina_base +open Mina_base_test_helpers +open Mina_ledger_test_helpers open Mina_numbers open Signature_lib open Helpers @@ -30,9 +32,9 @@ let%test_module "Test transaction logic." = let run_zkapp_cmd ~fee_payer ~fee ~accounts txns = let open Result.Let_syntax in let cmd = - zkapp_cmd ~noncemap:(noncemap accounts) ~fee:(fee_payer, fee) txns + zkapp_cmd ~noncemap:(Ledger_helpers.noncemap accounts) ~fee:(fee_payer, fee) txns in - let%bind ledger = test_ledger accounts in + let%bind ledger = Ledger_helpers.ledger_of_accounts accounts in let%map txn, _ = Transaction_logic.apply_zkapp_command_unchecked ~constraint_constants ~global_slot:Global_slot_since_genesis.(of_int 120) From 89dd6c8d8e7e583d16692f32bd9526973d398aaf Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 8 Dec 2023 14:21:35 +0100 Subject: [PATCH 13/38] Tests for the soft zkapp command limit. --- src/lib/staged_ledger/staged_ledger.ml | 132 +++++++++--------- src/lib/staged_ledger/staged_ledger.mli | 39 ++++++ src/lib/staged_ledger/test/dune | 33 +++++ src/lib/staged_ledger/test/main.ml | 11 ++ .../test/txn_application_test.ml | 57 ++++++++ .../transaction_logic/transaction_logic.ml | 12 +- src/lib/transaction_logic/test/zkapp_logic.ml | 4 +- 7 files changed, 221 insertions(+), 67 deletions(-) create mode 100644 src/lib/staged_ledger/test/dune create mode 100644 src/lib/staged_ledger/test/main.ml create mode 100644 src/lib/staged_ledger/test/txn_application_test.ml diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index df2b23cd302..b9dfadb37e3 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -1997,7 +1997,7 @@ module T = struct ; total_space_remaining : int } - let init ~total_limit ?zkapp_limit = + let init ?zkapp_limit ~total_limit = { valid_seq = Sequence.empty ; invalid = [] ; skipped_by_fee_payer = Account_id.Map.empty @@ -2018,7 +2018,7 @@ module T = struct let dependency_skipped txn t = Account_id.Map.mem t.skipped_by_fee_payer (txn_key txn) - let try_applying_txn ~logger ~apply (state : t) (txn : txn) = + let try_applying_txn ?logger ~apply (state : t) (txn : txn) = let open Continue_or_stop in match (state.zkapp_space_remaining, txn) with | _ when state.total_space_remaining < 1 -> @@ -2042,13 +2042,14 @@ module T = struct apply (Transaction.Command (User_command.forget_check txn)) ) with | Error e -> - [%log error] - ~metadata: - [ ("user_command", User_command.Valid.to_yojson txn) - ; ("error", Error_json.error_to_yojson e) - ] - "Staged_ledger_diff creation: Skipping user command: \ - $user_command due to error: $error" ; + Option.iter logger ~f:(fun logger -> + [%log error] + ~metadata: + [ ("user_command", User_command.Valid.to_yojson txn) + ; ("error", Error_json.error_to_yojson e) + ] + "Staged_ledger_diff creation: Skipping user command: \ + $user_command due to error: $error" ) ; Continue { state with invalid = (txn, e) :: state.invalid } | Ok _txn_partially_applied -> let valid_seq = @@ -2268,9 +2269,67 @@ end include T +module Test_helpers = struct + let constraint_constants = + Genesis_constants.Constraint_constants.for_unit_tests + + let dummy_state_and_view ?global_slot () = + let state = + let consensus_constants = + let genesis_constants = Genesis_constants.for_unit_tests in + Consensus.Constants.create ~constraint_constants + ~protocol_constants:genesis_constants.protocol + in + let compile_time_genesis = + let open Staged_ledger_diff in + (*not using Precomputed_values.for_unit_test because of dependency cycle*) + Mina_state.Genesis_protocol_state.t + ~genesis_ledger:Genesis_ledger.(Packed.t for_unit_tests) + ~genesis_epoch_data:Consensus.Genesis_epoch_data.for_unit_tests + ~constraint_constants ~consensus_constants ~genesis_body_reference + in + compile_time_genesis.data + in + let state_with_global_slot = + match global_slot with + | None -> + state + | Some global_slot -> + (*Protocol state views are always from previous block*) + let prev_global_slot = + Option.value ~default:Mina_numbers.Global_slot_since_genesis.zero + (Mina_numbers.Global_slot_since_genesis.sub global_slot + Mina_numbers.Global_slot_span.one ) + in + let consensus_state = + Consensus.Proof_of_stake.Exported.Consensus_state.Unsafe + .dummy_advance + (Mina_state.Protocol_state.consensus_state state) + ~new_global_slot_since_genesis:prev_global_slot + ~increase_epoch_count:false + in + let body = + Mina_state.Protocol_state.Body.For_tests.with_consensus_state + (Mina_state.Protocol_state.body state) + consensus_state + in + Mina_state.Protocol_state.create + ~previous_state_hash: + (Mina_state.Protocol_state.previous_state_hash state) + ~body + in + ( state_with_global_slot + , Mina_state.Protocol_state.Body.view + (Mina_state.Protocol_state.body state_with_global_slot) ) + + let dummy_state_view ?global_slot () = + dummy_state_and_view ?global_slot () |> snd +end + let%test_module "staged ledger tests" = ( module struct module Sl = T + open Test_helpers let () = Backtrace.elide := false ; @@ -2286,9 +2345,6 @@ let%test_module "staged ledger tests" = let proof_level = Genesis_constants.Proof_level.for_unit_tests - let constraint_constants = - Genesis_constants.Constraint_constants.for_unit_tests - let logger = Logger.null () let `VK vk, `Prover zkapp_prover = @@ -2351,58 +2407,6 @@ let%test_module "staged ledger tests" = sl := sl' ; (ledger_proof, diff', is_new_stack, pc_update, supercharge_coinbase) - let dummy_state_and_view ?global_slot () = - let state = - let consensus_constants = - let genesis_constants = Genesis_constants.for_unit_tests in - Consensus.Constants.create ~constraint_constants - ~protocol_constants:genesis_constants.protocol - in - let compile_time_genesis = - let open Staged_ledger_diff in - (*not using Precomputed_values.for_unit_test because of dependency cycle*) - Mina_state.Genesis_protocol_state.t - ~genesis_ledger:Genesis_ledger.(Packed.t for_unit_tests) - ~genesis_epoch_data:Consensus.Genesis_epoch_data.for_unit_tests - ~constraint_constants ~consensus_constants ~genesis_body_reference - in - compile_time_genesis.data - in - let state_with_global_slot = - match global_slot with - | None -> - state - | Some global_slot -> - (*Protocol state views are always from previous block*) - let prev_global_slot = - Option.value ~default:Mina_numbers.Global_slot_since_genesis.zero - (Mina_numbers.Global_slot_since_genesis.sub global_slot - Mina_numbers.Global_slot_span.one ) - in - let consensus_state = - Consensus.Proof_of_stake.Exported.Consensus_state.Unsafe - .dummy_advance - (Mina_state.Protocol_state.consensus_state state) - ~new_global_slot_since_genesis:prev_global_slot - ~increase_epoch_count:false - in - let body = - Mina_state.Protocol_state.Body.For_tests.with_consensus_state - (Mina_state.Protocol_state.body state) - consensus_state - in - Mina_state.Protocol_state.create - ~previous_state_hash: - (Mina_state.Protocol_state.previous_state_hash state) - ~body - in - ( state_with_global_slot - , Mina_state.Protocol_state.Body.view - (Mina_state.Protocol_state.body state_with_global_slot) ) - - let dummy_state_view ?global_slot () = - dummy_state_and_view ?global_slot () |> snd - let create_and_apply ?(coinbase_receiver = coinbase_receiver) ?(winner = self_pk) ~global_slot ~protocol_state_view ~state_and_body_hash sl txns stmt_to_work = diff --git a/src/lib/staged_ledger/staged_ledger.mli b/src/lib/staged_ledger/staged_ledger.mli index d9d2dd817be..16d9e719a44 100644 --- a/src/lib/staged_ledger/staged_ledger.mli +++ b/src/lib/staged_ledger/staged_ledger.mli @@ -232,6 +232,32 @@ val apply_diff_unchecked : val current_ledger_proof : t -> Ledger_proof.t option +(* Internals of the txn application. This is only exposed to facilitate + writing unit tests. *) +module Application_state : sig + type txn = + ( Signed_command.With_valid_signature.t + , Zkapp_command.Valid.t ) + User_command.t_ + + type t = + { valid_seq : txn Sequence.t + ; invalid : (txn * Error.t) list + ; skipped_by_fee_payer : txn list Account_id.Map.t + ; zkapp_space_remaining : int option + ; total_space_remaining : int + } + + val init : ?zkapp_limit:int -> total_limit:int -> t + + val try_applying_txn : + ?logger:Logger.t + -> apply:(User_command.t Transaction.t_ -> ('a, Error.t) Result.t) + -> t + -> txn + -> (t, txn Sequence.t * (txn * Error.t) list) Continue_or_stop.t +end + (* This should memoize the snark verifications *) val create_diff : @@ -303,3 +329,16 @@ val check_commands : *) val latest_block_accounts_created : t -> previous_block_state_hash:State_hash.t -> Account_id.t list + +module Test_helpers : sig + val dummy_state_and_view : + ?global_slot:Mina_numbers.Global_slot_since_genesis.t + -> unit + -> Mina_state.Protocol_state.value + * Zkapp_precondition.Protocol_state.View.t + + val dummy_state_view : + ?global_slot:Mina_numbers.Global_slot_since_genesis.t + -> unit + -> Zkapp_precondition.Protocol_state.View.t +end diff --git a/src/lib/staged_ledger/test/dune b/src/lib/staged_ledger/test/dune new file mode 100644 index 00000000000..da69b4a0265 --- /dev/null +++ b/src/lib/staged_ledger/test/dune @@ -0,0 +1,33 @@ +(tests + (names main) + (libraries + ;; opam libraries + alcotest + async_kernel + async_unix + base + base.caml + core_kernel + integers + sexplib0 + yojson + ;; local libraries + genesis_constants + mina_base + mina_base_test_helpers + mina_generators + mina_ledger + mina_ledger_test_helpers + mina_numbers + mina_transaction + mina_wire_types + staged_ledger + transaction_snark) + (preprocess + (pps + ppx_base + ppx_let + ppx_assert)) + (instrumentation + (backend bisect_ppx))) + diff --git a/src/lib/staged_ledger/test/main.ml b/src/lib/staged_ledger/test/main.ml new file mode 100644 index 00000000000..1300057e20b --- /dev/null +++ b/src/lib/staged_ledger/test/main.ml @@ -0,0 +1,11 @@ +open Alcotest + +let () = + run "Test staged ledger." + [ Txn_application_test. + ( "txn application" + , [ test_case + "Any txn can be applied if there's space in the scan state." + `Quick apply_against_non_empty_scan_state + ] ) + ] diff --git a/src/lib/staged_ledger/test/txn_application_test.ml b/src/lib/staged_ledger/test/txn_application_test.ml new file mode 100644 index 00000000000..49949c833f0 --- /dev/null +++ b/src/lib/staged_ledger/test/txn_application_test.ml @@ -0,0 +1,57 @@ +open Core_kernel +open Mina_base +open Mina_generators +open Mina_numbers +open Mina_transaction +open Staged_ledger + +type apply = + User_command.t Transaction.t_ + -> Ledger.Transaction_partially_applied.t Or_error.t + +let gen_apply_and_txn : (apply * Application_state.txn) Quickcheck.Generator.t = + let open Quickcheck.Generator in + let open Let_syntax in + let constraint_constants = + Genesis_constants.Constraint_constants.for_unit_tests + in + let%bind (txn, _, _, validating_ledger) = User_command_generators.zkapp_command_with_ledger () in + let%map global_slot = Global_slot_since_genesis.gen in + let current_state_view = Test_helpers.dummy_state_view ~global_slot () in + let apply = + Transaction_snark.Transaction_validator.apply_transaction_first_pass + ~constraint_constants ~global_slot validating_ledger + ~txn_state_view:current_state_view + in + (apply, txn) + +let gen_application_state : Application_state.t Quickcheck.Generator.t = + let open Application_state in + let open Quickcheck.Generator in + let open Let_syntax in + let%bind valid_seq = return Sequence.empty in + let%bind invalid = return [] in + let%bind skipped_by_fee_payer = return Account_id.Map.empty in + let%bind zkapp_space = Int.gen_incl (-1) 100 in + let zkapp_space_remaining = Option.some_if (zkapp_space >= 0) zkapp_space in + let%map total_space_remaining = Int.gen_incl 0 100 in + { valid_seq + ; invalid + ; skipped_by_fee_payer + ; zkapp_space_remaining + ; total_space_remaining + } + +let apply_against_non_empty_scan_state () = + Quickcheck.test + (Quickcheck.Generator.tuple2 gen_apply_and_txn gen_application_state) + ~f:(fun ((apply, txn), state) -> + match Application_state.try_applying_txn ~apply state txn with + | Continue state' when state.total_space_remaining > 0 -> + [%test_pred: int] + (fun delta -> delta = 0 || delta = 1) + (state.total_space_remaining - state'.total_space_remaining) + | Stop _ when state.total_space_remaining = 0 -> + () + | Continue _ -> failwith "Application continues when it should have stopped." + | Stop _ -> failwith "Application stopped when it should have continued." ) diff --git a/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml b/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml index 0a10f142e7a..42825d66e8d 100644 --- a/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml +++ b/src/lib/transaction_logic/test/transaction_logic/transaction_logic.ml @@ -88,7 +88,11 @@ let simple_payment () = let txn = signed_command ~fee ~sender ~receiver amount in let txn_state_view = protocol_state in let ledger = - match Ledger_helpers.ledger_of_accounts accounts with Ok l -> l | Error _ -> assert false + match Ledger_helpers.ledger_of_accounts accounts with + | Ok l -> + l + | Error _ -> + assert false in [%test_pred: Transaction_applied.t list Or_error.t] Or_error.is_ok (Transaction_logic.apply_transactions ~constraint_constants ~global_slot @@ -100,7 +104,11 @@ let simple_payment_signer_different_from_fee_payer () = let txn = signed_command ~signer:receiver ~fee ~sender ~receiver amount in let txn_state_view = protocol_state in let ledger = - match Ledger_helpers.ledger_of_accounts accounts with Ok l -> l | Error _ -> assert false + match Ledger_helpers.ledger_of_accounts accounts with + | Ok l -> + l + | Error _ -> + assert false in expect_failure ~error: diff --git a/src/lib/transaction_logic/test/zkapp_logic.ml b/src/lib/transaction_logic/test/zkapp_logic.ml index 4e5010ef7fe..30891040418 100644 --- a/src/lib/transaction_logic/test/zkapp_logic.ml +++ b/src/lib/transaction_logic/test/zkapp_logic.ml @@ -32,7 +32,9 @@ let%test_module "Test transaction logic." = let run_zkapp_cmd ~fee_payer ~fee ~accounts txns = let open Result.Let_syntax in let cmd = - zkapp_cmd ~noncemap:(Ledger_helpers.noncemap accounts) ~fee:(fee_payer, fee) txns + zkapp_cmd + ~noncemap:(Ledger_helpers.noncemap accounts) + ~fee:(fee_payer, fee) txns in let%bind ledger = Ledger_helpers.ledger_of_accounts accounts in let%map txn, _ = From 304ccd97c537f448ed9bea967fe2b8dcbf61c547 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:39:48 +0000 Subject: [PATCH 14/38] Code review changes --- src/lib/block_producer/block_producer.ml | 7 ++++--- src/lib/staged_ledger/staged_ledger.ml | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/block_producer/block_producer.ml b/src/lib/block_producer/block_producer.ml index c86e5e85c60..9b3a1c17369 100644 --- a/src/lib/block_producer/block_producer.ml +++ b/src/lib/block_producer/block_producer.ml @@ -71,6 +71,8 @@ let lift_sync f = [%log' error (Logger.create ())] "Ivar.fill bug is here!" ; Ivar.fill ivar (f ()) ) ) +let zkapp_cmd_limit = ref None + module Singleton_scheduler : sig type t @@ -611,8 +613,7 @@ let run ~context:(module Context : CONTEXT) ~vrf_evaluator ~prover ~verifier ~trust_system ~get_completed_work ~transaction_resource_pool ~time_controller ~consensus_local_state ~coinbase_receiver ~frontier_reader ~transition_writer ~set_next_producer_timing ~log_block_creation - ~zkapp_cmd_limit ~block_reward_threshold ~block_produced_bvar - ~vrf_evaluation_state ~net = + ~block_reward_threshold ~block_produced_bvar ~vrf_evaluation_state ~net = let open Context in O1trace.sync_thread "produce_blocks" (fun () -> let genesis_breadcrumb = @@ -753,7 +754,7 @@ let run ~context:(module Context : CONTEXT) ~vrf_evaluator ~prover ~verifier ~staged_ledger:(Breadcrumb.staged_ledger crumb) ~transactions ~get_completed_work ~logger ~log_block_creation ~winner_pk:winner_pubkey ~block_reward_threshold - ~zkapp_cmd_limit + ~zkapp_cmd_limit:!zkapp_cmd_limit in [%log internal] "Generate_next_state_done" ; match next_state_opt with diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index b9dfadb37e3..f9192e25aaf 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -2028,12 +2028,14 @@ module T = struct { state with skipped_by_fee_payer = add_skipped_txn state txn ; zkapp_space_remaining = Some (zkapp_limit - 1) + ; total_space_remaining = state.total_space_remaining - 1 } | Some zkapp_limit, _ when dependency_skipped txn state -> Continue { state with skipped_by_fee_payer = add_skipped_txn state txn ; zkapp_space_remaining = Some (zkapp_limit - 1) + ; total_space_remaining = state.total_space_remaining - 1 } | _ -> ( match From 5f55e329847ed3c4b44513bcb903115b8e93d0b5 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:08:02 +0000 Subject: [PATCH 15/38] Code review changes for Staged Ledger --- src/lib/staged_ledger/staged_ledger.ml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index f9192e25aaf..60370e261c6 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -2023,20 +2023,12 @@ module T = struct match (state.zkapp_space_remaining, txn) with | _ when state.total_space_remaining < 1 -> Stop (state.valid_seq, state.invalid) - | Some zkapp_limit, User_command.Zkapp_command _ when zkapp_limit > 1 -> + | Some zkapp_limit, User_command.Zkapp_command _ when zkapp_limit < 1 -> Continue - { state with - skipped_by_fee_payer = add_skipped_txn state txn - ; zkapp_space_remaining = Some (zkapp_limit - 1) - ; total_space_remaining = state.total_space_remaining - 1 - } - | Some zkapp_limit, _ when dependency_skipped txn state -> + { state with skipped_by_fee_payer = add_skipped_txn state txn } + | Some _, _ when dependency_skipped txn state -> Continue - { state with - skipped_by_fee_payer = add_skipped_txn state txn - ; zkapp_space_remaining = Some (zkapp_limit - 1) - ; total_space_remaining = state.total_space_remaining - 1 - } + { state with skipped_by_fee_payer = add_skipped_txn state txn } | _ -> ( match O1trace.sync_thread "validate_transaction_against_staged_ledger" @@ -2065,7 +2057,12 @@ module T = struct | Signed_command _ -> limit ) in - Continue { state with valid_seq; zkapp_space_remaining } ) + Continue + { state with + valid_seq + ; zkapp_space_remaining + ; total_space_remaining = state.total_space_remaining - 1 + } ) end let create_diff From 0dd7c1ebbd739c1f0ccd8b7d614dd6dddb15f0f5 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:28:35 +0000 Subject: [PATCH 16/38] Code review - remove config location for zkapp_lim --- src/lib/mina_lib/config.ml | 1 - src/lib/mina_lib/mina_lib.ml | 1 - 2 files changed, 2 deletions(-) diff --git a/src/lib/mina_lib/config.ml b/src/lib/mina_lib/config.ml index 3aa64d9207d..330d8c2a0bb 100644 --- a/src/lib/mina_lib/config.ml +++ b/src/lib/mina_lib/config.ml @@ -59,7 +59,6 @@ type t = ; uptime_submitter_keypair : Keypair.t option [@default None] ; uptime_send_node_commit : bool [@default false] ; stop_time : int - ; mutable zkapp_cmd_limit : int option [@default None] ; graphql_control_port : int option [@default None] } [@@deriving make] diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index e0fc2603540..415a2e4c2a6 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -1380,7 +1380,6 @@ let start t = ~log_block_creation:t.config.log_block_creation ~block_reward_threshold:t.config.block_reward_threshold ~block_produced_bvar:t.components.block_produced_bvar - ~zkapp_cmd_limit:t.config.zkapp_cmd_limit ~vrf_evaluation_state:t.vrf_evaluation_state ~net:t.components.net ; perform_compaction t ; let () = From a61d60a7d72fcf3ffd27c24c4327005c8893ad37 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:30:20 +0000 Subject: [PATCH 17/38] Code review - ref in block-producer using and graphql changes --- src/lib/block_producer/block_producer.ml | 2 +- src/lib/mina_compile_config/mina_compile_config.ml | 2 ++ src/lib/mina_graphql/mina_graphql.ml | 6 ++---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/block_producer/block_producer.ml b/src/lib/block_producer/block_producer.ml index 9b3a1c17369..0a3c69b2021 100644 --- a/src/lib/block_producer/block_producer.ml +++ b/src/lib/block_producer/block_producer.ml @@ -71,7 +71,7 @@ let lift_sync f = [%log' error (Logger.create ())] "Ivar.fill bug is here!" ; Ivar.fill ivar (f ()) ) ) -let zkapp_cmd_limit = ref None +let zkapp_cmd_limit = ref Mina_compile_config.zkapp_cmd_limit module Singleton_scheduler : sig type t diff --git a/src/lib/mina_compile_config/mina_compile_config.ml b/src/lib/mina_compile_config/mina_compile_config.ml index 4255a5d6dfc..ea405d101b9 100644 --- a/src/lib/mina_compile_config/mina_compile_config.ml +++ b/src/lib/mina_compile_config/mina_compile_config.ml @@ -68,3 +68,5 @@ let max_event_elements = 100 let max_action_elements = 100 [%%inject "network_id", network] + +let zkapp_cmd_limit = None diff --git a/src/lib/mina_graphql/mina_graphql.ml b/src/lib/mina_graphql/mina_graphql.ml index 13aeca287e9..3e5c7f3bc4b 100644 --- a/src/lib/mina_graphql/mina_graphql.ml +++ b/src/lib/mina_graphql/mina_graphql.ml @@ -2567,10 +2567,8 @@ module Queries = struct Arg.[ arg "limit" ~doc:"ZkApp commands per block limit." ~typ:int ] ~typ:int ~doc:"Set zkApp commands per block limit for the block producer." - ~resolve:(fun { ctx = _with_seq_no, mina; _ } () limit -> - let open Mina_lib.Config in - let config = Mina_lib.config mina in - config.zkapp_cmd_limit <- limit ; + ~resolve:(fun { ctx = _with_seq_no, _mina; _ } () limit -> + Block_producer.zkapp_cmd_limit := limit ; limit ) let commands = [ auth; slots_won; internal_logs; zkapp_cmd_limit ] From f189927cae59c0fa1629ed2790a84eddb4813951 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Thu, 14 Dec 2023 10:21:21 +0000 Subject: [PATCH 18/38] Remove redundant argument --- src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index e23d64fb7c9..7abfc802c4f 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -1368,7 +1368,6 @@ Pass one of -peer, -peer-list-file, -seed, -peer-list-url.|} ; ; shutdown_on_disconnect = true ; num_threads = snark_worker_parallelism_flag } - ~zkapp_cmd_limit ~snark_coordinator_key:run_snark_coordinator_flag ~snark_pool_disk_location:(conf_dir ^/ "snark_pool") ~wallets_disk_location:(conf_dir ^/ "wallets") From 9b690c7b39e99b883ca38c6ed619af1f6194b7f1 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:23:48 +0000 Subject: [PATCH 19/38] Fix formatting in test so pipeline will pass --- src/lib/staged_ledger/test/txn_application_test.ml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/staged_ledger/test/txn_application_test.ml b/src/lib/staged_ledger/test/txn_application_test.ml index 49949c833f0..a03eb54c9d2 100644 --- a/src/lib/staged_ledger/test/txn_application_test.ml +++ b/src/lib/staged_ledger/test/txn_application_test.ml @@ -15,10 +15,12 @@ let gen_apply_and_txn : (apply * Application_state.txn) Quickcheck.Generator.t = let constraint_constants = Genesis_constants.Constraint_constants.for_unit_tests in - let%bind (txn, _, _, validating_ledger) = User_command_generators.zkapp_command_with_ledger () in + let%bind txn, _, _, validating_ledger = + User_command_generators.zkapp_command_with_ledger () + in let%map global_slot = Global_slot_since_genesis.gen in let current_state_view = Test_helpers.dummy_state_view ~global_slot () in - let apply = + let apply = Transaction_snark.Transaction_validator.apply_transaction_first_pass ~constraint_constants ~global_slot validating_ledger ~txn_state_view:current_state_view @@ -53,5 +55,7 @@ let apply_against_non_empty_scan_state () = (state.total_space_remaining - state'.total_space_remaining) | Stop _ when state.total_space_remaining = 0 -> () - | Continue _ -> failwith "Application continues when it should have stopped." - | Stop _ -> failwith "Application stopped when it should have continued." ) + | Continue _ -> + failwith "Application continues when it should have stopped." + | Stop _ -> + failwith "Application stopped when it should have continued." ) From 0703dff9a26350af9fb2e4d8a0c15340f137ae77 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:37:40 +0000 Subject: [PATCH 20/38] Add ppx_version to fix linting error --- src/lib/mina_base/test/helpers/dune | 6 +----- src/lib/mina_ledger/test/helpers/dune | 6 +----- src/lib/staged_ledger/test/dune | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/lib/mina_base/test/helpers/dune b/src/lib/mina_base/test/helpers/dune index f7734988c12..74d4677386d 100644 --- a/src/lib/mina_base/test/helpers/dune +++ b/src/lib/mina_base/test/helpers/dune @@ -16,10 +16,6 @@ monad_lib signature_lib) (preprocess - (pps - ppx_base - ppx_let - ppx_assert)) + (pps ppx_base ppx_let ppx_assert ppx_version)) (instrumentation (backend bisect_ppx))) - diff --git a/src/lib/mina_ledger/test/helpers/dune b/src/lib/mina_ledger/test/helpers/dune index 257b0774791..919aaae1265 100644 --- a/src/lib/mina_ledger/test/helpers/dune +++ b/src/lib/mina_ledger/test/helpers/dune @@ -24,10 +24,6 @@ pickles_types signature_lib) (preprocess - (pps - ppx_base - ppx_let - ppx_assert)) + (pps ppx_base ppx_let ppx_assert ppx_version)) (instrumentation (backend bisect_ppx))) - diff --git a/src/lib/staged_ledger/test/dune b/src/lib/staged_ledger/test/dune index da69b4a0265..74e76dc22b0 100644 --- a/src/lib/staged_ledger/test/dune +++ b/src/lib/staged_ledger/test/dune @@ -24,10 +24,6 @@ staged_ledger transaction_snark) (preprocess - (pps - ppx_base - ppx_let - ppx_assert)) + (pps ppx_base ppx_let ppx_assert ppx_version)) (instrumentation (backend bisect_ppx))) - From 7cc4eb3d12d58c97ad942ce2dd8994ae4e8952e0 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 15 Dec 2023 08:18:43 +0000 Subject: [PATCH 21/38] Code review changes - conditional compilation --- src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml | 10 ---------- src/lib/mina_compile_config/mina_compile_config.ml | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index 7abfc802c4f..be21243b53e 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -14,16 +14,6 @@ let () = Async.Scheduler.set_record_backtraces true [%%endif] -[%%ifndef zkapp_cmd_limit] - -let zkapp_cmd_limit = None - -[%%else] - -[%%inject "zkapp_cmd_limit", zkapp_cmd_limit] - -[%%endif] - type mina_initialization = { mina : Mina_lib.t ; client_trustlist : Unix.Cidr.t list option diff --git a/src/lib/mina_compile_config/mina_compile_config.ml b/src/lib/mina_compile_config/mina_compile_config.ml index ea405d101b9..7de63fc0396 100644 --- a/src/lib/mina_compile_config/mina_compile_config.ml +++ b/src/lib/mina_compile_config/mina_compile_config.ml @@ -69,4 +69,14 @@ let max_action_elements = 100 [%%inject "network_id", network] +[%%ifndef zkapp_cmd_limit] + let zkapp_cmd_limit = None + +[%%else] + +[%%inject "zkapp_cmd_limit", zkapp_cmd_limit] + +let zkapp_cmd_limit = Some zkapp_cmd_limit + +[%%endif] From ea442a0d4f1b8f0a99f9bab362728948dde4fd25 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:36:11 +0000 Subject: [PATCH 22/38] Remove unneeded variable --- src/lib/mina_graphql/mina_graphql.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mina_graphql/mina_graphql.ml b/src/lib/mina_graphql/mina_graphql.ml index 3e5c7f3bc4b..0a10c705550 100644 --- a/src/lib/mina_graphql/mina_graphql.ml +++ b/src/lib/mina_graphql/mina_graphql.ml @@ -2567,7 +2567,7 @@ module Queries = struct Arg.[ arg "limit" ~doc:"ZkApp commands per block limit." ~typ:int ] ~typ:int ~doc:"Set zkApp commands per block limit for the block producer." - ~resolve:(fun { ctx = _with_seq_no, _mina; _ } () limit -> + ~resolve:(fun { ctx = _; _ } () limit -> Block_producer.zkapp_cmd_limit := limit ; limit ) From 080acace858dadb74d3158ff2d473d701af4f0a3 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 4 Jan 2024 15:51:41 +0100 Subject: [PATCH 23/38] Fix duplicated rows in zkapp_account_precondition table --- .../add_constraint.sql | 2 + ...licated_zkapp_account_precondition_rows.sh | 56 +++++++++++++++++++ src/app/archive/zkapp_tables.sql | 1 + src/app/rosetta/download-missing-blocks.sh | 6 +- 4 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 scripts/archive/zkapp_duplicated_precondition/add_constraint.sql create mode 100755 scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh diff --git a/scripts/archive/zkapp_duplicated_precondition/add_constraint.sql b/scripts/archive/zkapp_duplicated_precondition/add_constraint.sql new file mode 100644 index 00000000000..13802417040 --- /dev/null +++ b/scripts/archive/zkapp_duplicated_precondition/add_constraint.sql @@ -0,0 +1,2 @@ +ALTER TABLE zkapp_account_precondition +ADD CONSTRAINT zkapp_account_precondition_unique UNIQUE(balance_id, receipt_chain_hash, delegate_id, state_id, action_state_id, proved_state, is_new, nonce_id); \ No newline at end of file diff --git a/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh b/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh new file mode 100755 index 00000000000..0f9db642226 --- /dev/null +++ b/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +if [ $# -gt 2 ]; then + echo "Usage $0 archive-db" + exit 0 +fi + +ARCHIVE_DB=$1 +BODY_DATA_BCK=zkapp_update_body_data_table.bck +PRECONDITION_DATA_BCK=zkapp_precondition_table.bck +PRECONDITION_DATA_TMP=zkapp_precondition_table.tmp +PRECONDITION_UPDATE_SCRIPT=zkapp_precondition_script.sql +BODY_UPDATE_SCRIPT=zkapp_update_body_script.sql + +echo "This script will produce two scripts which will remove duplicated zkppp account preconditions rows in archive db" "$ARCHIVE_DB" + +rm -f $BODY_DATA_BCK +rm -f $PRECONDITION_DATA_BCK +rm -f $PRECONDITION_DATA_TMP +rm -f $PRECONDITION_UPDATE_SCRIPT +rm -f $BODY_UPDATE_SCRIPT + +ALL_TABLES="balance_id,receipt_chain_hash,delegate_id,state_id,action_state_id,proved_state, is_new, nonce_id" + +echo "Creating backup of zkapp_account_preconditions table " "$PRECONDITION_DATA_BCK" + +echo "SELECT * FROM zkapp_account_precondition" | \ + psql -U postgres --csv -q -t "$ARCHIVE_DB" > $PRECONDITION_DATA_BCK + +echo "Creating backup of zkapp_update_body table " "$BODY_DATA_BCK" + +echo "SELECT * FROM zkapp_account_update_body" | \ + psql -U postgres --csv -q -t "$ARCHIVE_DB" > $BODY_DATA_BCK + + +echo "Creating temporary file of zkapp_account_preconditions table " "$PRECONDITION_DATA_TMP" + +echo "SELECT MIN(id) as id, $ALL_TABLES, MAX(id) as duplicated_id \ + FROM zkapp_account_precondition \ + GROUP BY $ALL_TABLES \ + HAVING COUNT(*) > 1" | \ + psql -U postgres --csv -q -t "$ARCHIVE_DB" > $PRECONDITION_DATA_TMP + +echo "Creating SQL scripts for removing duplications" "$PRECONDITION_UPDATE_SCRIPT" "and updating references" "$BODY_UPDATE_SCRIPT" + +cat $PRECONDITION_DATA_TMP | while IFS= read -r line +do + ID_TO_REPLACE=$(echo "$line" | awk -F , '{print $1}'); + ID_TO_REMOVE=$(echo "$line" | awk -F , '{print $10}'); + echo -n . + echo "$ID_TO_REPLACE $ID_TO_REMOVE" | awk '{print "UPDATE zkapp_account_update_body SET zkapp_account_precondition_id = " $1 " WHERE zkapp_account_precondition_id =" $2 ";"}' >> $BODY_UPDATE_SCRIPT + echo "$ID_TO_REMOVE" | awk '{print "DELETE FROM zkapp_account_precondition WHERE id=" $1 ";"}' >> $PRECONDITION_UPDATE_SCRIPT +done + +echo +echo "Scripts are ready $PRECONDITION_UPDATE_SCRIPT, $BODY_UPDATE_SCRIPT !" diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index 03445a146aa..5ddf525573f 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -162,6 +162,7 @@ CREATE TABLE zkapp_account_precondition , action_state_id int REFERENCES zkapp_field(id) , proved_state boolean , is_new boolean +, UNIQUE(balance_id, receipt_chain_hash, delegate_id, state_id, action_state_id, proved_state, is_new, nonce_id) ); CREATE TABLE zkapp_accounts diff --git a/src/app/rosetta/download-missing-blocks.sh b/src/app/rosetta/download-missing-blocks.sh index b8dfb8c45ab..eec7422c89c 100755 --- a/src/app/rosetta/download-missing-blocks.sh +++ b/src/app/rosetta/download-missing-blocks.sh @@ -1,8 +1,8 @@ #!/bin/bash -BLOCKS_BUCKET="${BLOCKS_BUCKET:=https://storage.googleapis.com/mina_network_block_data}" +BLOCKS_BUCKET="${BLOCKS_BUCKET:=gs://mina_network_block_data}" -set -u +set -ux MINA_NETWORK=${1} # Postgres database connection string and related variables @@ -25,7 +25,7 @@ function populate_db() { function download_block() { echo "Downloading $1 block" - curl -sO "${BLOCKS_BUCKET}/${1}" + gsutil cp "${BLOCKS_BUCKET}/${1}" . } HASH='map(select(.metadata.parent_hash != null and .metadata.parent_height != null)) | .[0].metadata.parent_hash' From e81a72c395877debfe695e2d9f4a135b18e20522 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 4 Jan 2024 16:08:48 +0100 Subject: [PATCH 24/38] Revert "Auxiliary commit to revert individual files from 080acace858dadb74d3158ff2d473d701af4f0a3" This reverts commit 4883c25838c8650d74fb51e97baad2c663a60390. --- src/app/rosetta/download-missing-blocks.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/rosetta/download-missing-blocks.sh b/src/app/rosetta/download-missing-blocks.sh index eec7422c89c..b8dfb8c45ab 100755 --- a/src/app/rosetta/download-missing-blocks.sh +++ b/src/app/rosetta/download-missing-blocks.sh @@ -1,8 +1,8 @@ #!/bin/bash -BLOCKS_BUCKET="${BLOCKS_BUCKET:=gs://mina_network_block_data}" +BLOCKS_BUCKET="${BLOCKS_BUCKET:=https://storage.googleapis.com/mina_network_block_data}" -set -ux +set -u MINA_NETWORK=${1} # Postgres database connection string and related variables @@ -25,7 +25,7 @@ function populate_db() { function download_block() { echo "Downloading $1 block" - gsutil cp "${BLOCKS_BUCKET}/${1}" . + curl -sO "${BLOCKS_BUCKET}/${1}" } HASH='map(select(.metadata.parent_hash != null and .metadata.parent_height != null)) | .[0].metadata.parent_hash' From 32464642e4ae796db89cf99828c2c0ef5c86657a Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Thu, 4 Jan 2024 20:08:22 +0100 Subject: [PATCH 25/38] Apply suggestions from Serhii in code review Co-authored-by: Serhii Shymkiv --- .../remove_duplicated_zkapp_account_precondition_rows.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh b/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh index 0f9db642226..04db7b2d7cc 100755 --- a/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh +++ b/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh @@ -12,7 +12,7 @@ PRECONDITION_DATA_TMP=zkapp_precondition_table.tmp PRECONDITION_UPDATE_SCRIPT=zkapp_precondition_script.sql BODY_UPDATE_SCRIPT=zkapp_update_body_script.sql -echo "This script will produce two scripts which will remove duplicated zkppp account preconditions rows in archive db" "$ARCHIVE_DB" +echo "This Shell script will produce two SQL scripts that will remove duplicated zkApp account preconditions rows in the archive DB" "$ARCHIVE_DB" rm -f $BODY_DATA_BCK rm -f $PRECONDITION_DATA_BCK @@ -53,4 +53,4 @@ do done echo -echo "Scripts are ready $PRECONDITION_UPDATE_SCRIPT, $BODY_UPDATE_SCRIPT !" +echo "SQL scripts are ready: $PRECONDITION_UPDATE_SCRIPT, $BODY_UPDATE_SCRIPT !" From a146c1230936828bae10a69fd51a6e8406782f4c Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Thu, 4 Jan 2024 21:19:23 +0000 Subject: [PATCH 26/38] End the suffering, turn back on `implicit_transitive_deps` --- src/app/archive/dune-project | 2 +- src/app/archive/lib/dune-project | 2 +- src/app/benchmarks/dune-project | 2 +- src/app/cli/src/dune-project | 2 +- src/app/dump_blocks/dune-project | 2 +- src/app/logproc/dune-project | 2 +- src/app/reformat/dune-project | 2 +- src/app/rosetta/dune-project | 2 +- src/app/runtime_genesis_ledger/dune-project | 2 +- src/app/zkapp_test_transaction/dune-project | 2 +- src/dune-project | 2 +- src/lib/transition_frontier/dune-project | 2 +- src/nonconsensus/dune-project | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/app/archive/dune-project b/src/app/archive/dune-project index 2b006b630f4..edce9fabb47 100644 --- a/src/app/archive/dune-project +++ b/src/app/archive/dune-project @@ -1,3 +1,3 @@ (lang dune 2.8) (name archive_lib) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/archive/lib/dune-project b/src/app/archive/lib/dune-project index 92eab6bf10f..cfef69d8b4f 100644 --- a/src/app/archive/lib/dune-project +++ b/src/app/archive/lib/dune-project @@ -1,3 +1,3 @@ (lang dune 2.8) (name graphql_query) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/benchmarks/dune-project b/src/app/benchmarks/dune-project index 30f53fc0038..ae55889bf2f 100644 --- a/src/app/benchmarks/dune-project +++ b/src/app/benchmarks/dune-project @@ -1,3 +1,3 @@ (lang dune 3.3) (name benchmarks) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/cli/src/dune-project b/src/app/cli/src/dune-project index 6f8123e33f7..9ad7e7d0bf5 100644 --- a/src/app/cli/src/dune-project +++ b/src/app/cli/src/dune-project @@ -1,3 +1,3 @@ (lang dune 3.3) (name graphql) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/dump_blocks/dune-project b/src/app/dump_blocks/dune-project index aa4c91fbd90..0fa63202f48 100644 --- a/src/app/dump_blocks/dune-project +++ b/src/app/dump_blocks/dune-project @@ -1,3 +1,3 @@ (lang dune 2.8) (name dump_blocks) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/logproc/dune-project b/src/app/logproc/dune-project index e42209dad38..95c7cc315ba 100644 --- a/src/app/logproc/dune-project +++ b/src/app/logproc/dune-project @@ -1,3 +1,3 @@ (lang dune 2.8) (name logproc) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/reformat/dune-project b/src/app/reformat/dune-project index 6a975213f4b..a656f0c5296 100644 --- a/src/app/reformat/dune-project +++ b/src/app/reformat/dune-project @@ -1,3 +1,3 @@ (lang dune 3.2) (name reformat) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/rosetta/dune-project b/src/app/rosetta/dune-project index e9c79df2a05..95563d34ea5 100644 --- a/src/app/rosetta/dune-project +++ b/src/app/rosetta/dune-project @@ -1,3 +1,3 @@ (lang dune 3.3) (name rosetta) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/runtime_genesis_ledger/dune-project b/src/app/runtime_genesis_ledger/dune-project index 5991456a2c5..425cc91c4e8 100644 --- a/src/app/runtime_genesis_ledger/dune-project +++ b/src/app/runtime_genesis_ledger/dune-project @@ -1,3 +1,3 @@ (lang dune 2.8) (name runtime_genesis_ledger) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/app/zkapp_test_transaction/dune-project b/src/app/zkapp_test_transaction/dune-project index 2be065f82eb..b316070ac57 100644 --- a/src/app/zkapp_test_transaction/dune-project +++ b/src/app/zkapp_test_transaction/dune-project @@ -1,3 +1,3 @@ (lang dune 2.8) (name zkapp_test_transaction) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/dune-project b/src/dune-project index 7a68f9cdfa8..ed9b508e577 100644 --- a/src/dune-project +++ b/src/dune-project @@ -1,5 +1,5 @@ (lang dune 3.3) -(implicit_transitive_deps false) +(implicit_transitive_deps true) (package (name allocation_functor)) (package (name archive_blocks)) diff --git a/src/lib/transition_frontier/dune-project b/src/lib/transition_frontier/dune-project index 89fce2cde93..b93245aff27 100644 --- a/src/lib/transition_frontier/dune-project +++ b/src/lib/transition_frontier/dune-project @@ -1,3 +1,3 @@ (lang dune 2.8) (name transition_frontier) -(implicit_transitive_deps false) +(implicit_transitive_deps true) diff --git a/src/nonconsensus/dune-project b/src/nonconsensus/dune-project index e99cf75c732..ed3dcb91eab 100644 --- a/src/nonconsensus/dune-project +++ b/src/nonconsensus/dune-project @@ -1,2 +1,2 @@ (lang dune 2.8) -(implicit_transitive_deps false) +(implicit_transitive_deps true) From 25a2e111a9e3cc3156923f48e9474cd18a579c1b Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 4 Jan 2024 00:53:47 +0800 Subject: [PATCH 27/38] add idx for element_ids --- src/app/archive/zkapp_tables.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index 03445a146aa..f9d71b076b5 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -33,6 +33,8 @@ CREATE TABLE zkapp_field_array , element_ids int[] NOT NULL ); +create INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids); + /* Fixed-width arrays of algebraic fields, given as id's from zkapp_field From 2e6514467e230e8cf983c88ec609bcc3f2170c52 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 4 Jan 2024 07:08:36 +0800 Subject: [PATCH 28/38] add an index for zkapp_events --- src/app/archive/zkapp_tables.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index f9d71b076b5..3219004ca71 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -83,6 +83,9 @@ CREATE TABLE zkapp_events , element_ids int[] NOT NULL ); +create index idx_zkapp_events_element_ids ON zkapp_events(element_ids); + + /* field elements derived from verification keys */ CREATE TABLE zkapp_verification_key_hashes ( id serial PRIMARY KEY From db5f1cdd217fa7b6eecf8c1472d25f58d06d97e2 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Jan 2024 03:17:33 +0800 Subject: [PATCH 29/38] use upper case for sql key word --- src/app/archive/zkapp_tables.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index 3219004ca71..ded3a2f3db2 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -33,7 +33,7 @@ CREATE TABLE zkapp_field_array , element_ids int[] NOT NULL ); -create INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids); +CREATE INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids); /* Fixed-width arrays of algebraic fields, given as id's from zkapp_field @@ -83,7 +83,7 @@ CREATE TABLE zkapp_events , element_ids int[] NOT NULL ); -create index idx_zkapp_events_element_ids ON zkapp_events(element_ids); +CREATE INDEX idx_zkapp_events_element_ids ON zkapp_events(element_ids); /* field elements derived from verification keys */ From 7f2844158ebf4d2bd1e20e02e6cfdfded54520e2 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Jan 2024 04:48:23 +0800 Subject: [PATCH 30/38] add sql file to add indexes --- src/app/archive/add_indexes.sql | 3 +++ src/app/archive/zkapp_tables.sql | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 src/app/archive/add_indexes.sql diff --git a/src/app/archive/add_indexes.sql b/src/app/archive/add_indexes.sql new file mode 100644 index 00000000000..12d8cb71e00 --- /dev/null +++ b/src/app/archive/add_indexes.sql @@ -0,0 +1,3 @@ +CREATE INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids); + +CREATE INDEX idx_zkapp_events_element_ids ON zkapp_events(element_ids); \ No newline at end of file diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index ded3a2f3db2..21389206bb1 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -85,7 +85,6 @@ CREATE TABLE zkapp_events CREATE INDEX idx_zkapp_events_element_ids ON zkapp_events(element_ids); - /* field elements derived from verification keys */ CREATE TABLE zkapp_verification_key_hashes ( id serial PRIMARY KEY From 39f8bde53c90b6c3f20e4f4974b4a304590143ac Mon Sep 17 00:00:00 2001 From: dkijania Date: Sun, 7 Jan 2024 21:24:58 +0100 Subject: [PATCH 31/38] Fixed arguments count checks and conditional printing in remove duplicated zkapp precondition row --- ...licated_zkapp_account_precondition_rows.sh | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh b/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh index 04db7b2d7cc..3d016dba157 100755 --- a/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh +++ b/scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh @@ -1,6 +1,8 @@ #!/bin/bash -if [ $# -gt 2 ]; then +set -e + +if [ $# -ne 1 ]; then echo "Usage $0 archive-db" exit 0 fi @@ -33,7 +35,7 @@ echo "SELECT * FROM zkapp_account_update_body" | \ psql -U postgres --csv -q -t "$ARCHIVE_DB" > $BODY_DATA_BCK -echo "Creating temporary file of zkapp_account_preconditions table " "$PRECONDITION_DATA_TMP" +echo "Creating temporary file of zkapp_account_preconditions table with duplicated rows" "$PRECONDITION_DATA_TMP" echo "SELECT MIN(id) as id, $ALL_TABLES, MAX(id) as duplicated_id \ FROM zkapp_account_precondition \ @@ -41,16 +43,23 @@ echo "SELECT MIN(id) as id, $ALL_TABLES, MAX(id) as duplicated_id \ HAVING COUNT(*) > 1" | \ psql -U postgres --csv -q -t "$ARCHIVE_DB" > $PRECONDITION_DATA_TMP -echo "Creating SQL scripts for removing duplications" "$PRECONDITION_UPDATE_SCRIPT" "and updating references" "$BODY_UPDATE_SCRIPT" -cat $PRECONDITION_DATA_TMP | while IFS= read -r line -do - ID_TO_REPLACE=$(echo "$line" | awk -F , '{print $1}'); - ID_TO_REMOVE=$(echo "$line" | awk -F , '{print $10}'); - echo -n . - echo "$ID_TO_REPLACE $ID_TO_REMOVE" | awk '{print "UPDATE zkapp_account_update_body SET zkapp_account_precondition_id = " $1 " WHERE zkapp_account_precondition_id =" $2 ";"}' >> $BODY_UPDATE_SCRIPT - echo "$ID_TO_REMOVE" | awk '{print "DELETE FROM zkapp_account_precondition WHERE id=" $1 ";"}' >> $PRECONDITION_UPDATE_SCRIPT -done +if [[ $(wc -l < $PRECONDITION_DATA_TMP) -gt 0 ]]; then + + echo "Creating SQL scripts for removing duplications" "$PRECONDITION_UPDATE_SCRIPT" "and updating references" "$BODY_UPDATE_SCRIPT" -echo -echo "SQL scripts are ready: $PRECONDITION_UPDATE_SCRIPT, $BODY_UPDATE_SCRIPT !" + cat $PRECONDITION_DATA_TMP | while IFS= read -r line + do + ID_TO_REPLACE=$(echo "$line" | awk -F , '{print $1}'); + ID_TO_REMOVE=$(echo "$line" | awk -F , '{print $10}'); + echo -n . + echo "$ID_TO_REPLACE $ID_TO_REMOVE" | awk '{print "UPDATE zkapp_account_update_body SET zkapp_account_precondition_id = " $1 " WHERE zkapp_account_precondition_id =" $2 ";"}' >> $BODY_UPDATE_SCRIPT + echo "$ID_TO_REMOVE" | awk '{print "DELETE FROM zkapp_account_precondition WHERE id=" $1 ";"}' >> $PRECONDITION_UPDATE_SCRIPT + done + + echo + echo "SQL scripts are ready: $PRECONDITION_UPDATE_SCRIPT, $BODY_UPDATE_SCRIPT !" +else + echo + echo "No duplicated rows found. Database state is correct !" +fi From 81d994d8c565a5209fe122870d5beeaa9bb18955 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 8 Jan 2024 19:15:20 +0100 Subject: [PATCH 32/38] Empty commit to signpost Berkeley 2.0.0rampup8 From 24c39c2f16c9ce9741ba8ae38d968ba6889828ac Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 9 Jan 2024 05:14:12 +0000 Subject: [PATCH 33/38] Fix parsing of URLs With this change, we now support all of the likely URLs, including ``` https://foo:bar@github.com/ORG/REPO.git https://github.com/ORG/REPO.git git://github.com/ORG/REPO.git git@github.com:ORG/REPO.git ``` --- buildkite/scripts/export-git-env-vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index 5b45997339d..3d207d89e9a 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -35,7 +35,7 @@ if [ "${BUILDKITE_REPO}" != "${MINA_REPO}" ]; then # We don't want to use tags (as this can replace our dockers/debian packages). Instead we are using repo name # For example: for given repo 'https://github.com/dkijania/mina.git' we convert it to 'dkijania_mina' - export GITTAG=1.0.0$(echo ${BUILDKITE_REPO} | sed -e "s/https:\/\/github.com\///g" | sed -e "s/.git//g" | sed -e "s/\//-/g") + export GITTAG=1.0.0$(echo ${BUILDKITE_REPO} | sed -e 's/^.*github.com[:\/]\(.*\)\.git$/\1/' -e 's/\//-/') export THIS_COMMIT_TAG="" RELEASE=unstable From b071672a154ceec790a9eff86556764c7c041b26 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 9 Jan 2024 05:23:22 +0000 Subject: [PATCH 34/38] If our regex will fail, exit early --- buildkite/scripts/export-git-env-vars.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index 3d207d89e9a..92c3704e3a6 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -29,6 +29,9 @@ export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye} if [ "${BUILDKITE_REPO}" != "${MINA_REPO}" ]; then + # Abort if `BUILDKITE_REPO` doesn't have the expected format + echo ${BUILDKITE_REPO} | grep -P '^.*github.com[:\/](.*)\.git$' > /dev/null + # We don't want to allow some operations on fork repository which should be done on main repo only. # Publish to docker hub or publish to unstable debian channel should be exclusive to main repo as it can override # packages from main repo (by using the same commit and the same branch from forked repository) From f2ddea6bc1cd60cb18da2c271533031a14376804 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 9 Jan 2024 05:25:40 +0000 Subject: [PATCH 35/38] Add a message on failure --- buildkite/scripts/export-git-env-vars.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index 92c3704e3a6..90df88f660c 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -30,7 +30,8 @@ export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye} if [ "${BUILDKITE_REPO}" != "${MINA_REPO}" ]; then # Abort if `BUILDKITE_REPO` doesn't have the expected format - echo ${BUILDKITE_REPO} | grep -P '^.*github.com[:\/](.*)\.git$' > /dev/null + echo ${BUILDKITE_REPO} | grep -P '^.*github.com[:\/](.*)\.git$' > /dev/null || \ + (echo "BUILDKITE_REPO does not have the expected format" && false) # We don't want to allow some operations on fork repository which should be done on main repo only. # Publish to docker hub or publish to unstable debian channel should be exclusive to main repo as it can override From 9ed15bc9f525810d6611633dc93c3e725798204f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos=20Reis?= Date: Mon, 18 Dec 2023 03:30:20 +0000 Subject: [PATCH 36/38] Extract genesis ledger to tar filename dir --- src/lib/genesis_ledger_helper/genesis_ledger_helper.ml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml index a449f287cb5..5af4331b5c0 100644 --- a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml +++ b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml @@ -47,6 +47,9 @@ module Tar = struct Ok () | Error err -> Error (Error.tag err ~tag:"Error extracting tar file") + + let filename_without_extension = + String.chop_suffix_if_exists ~suffix:".tar.gz" end let file_exists ?follow_symlinks filename = @@ -186,7 +189,9 @@ module Ledger = struct [%log trace] "Loading $ledger from $path" ~metadata: [ ("ledger", `String ledger_name_prefix); ("path", `String filename) ] ; - let dirname = Uuid.to_string (Uuid_unix.create ()) in + let dirname = + Tar.filename_without_extension @@ Filename.basename filename + in (* Unpack the ledger in the autogen directory, since we know that we have write permissions there. *) From 21a2be07ab44d0e0b75e7e3c192710c832629cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santos=20Reis?= Date: Mon, 18 Dec 2023 10:54:14 +0000 Subject: [PATCH 37/38] Delete tar extract dir if it exists --- .../genesis_ledger_helper.ml | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml index 5af4331b5c0..e3c1e5dedf1 100644 --- a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml +++ b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml @@ -192,10 +192,30 @@ module Ledger = struct let dirname = Tar.filename_without_extension @@ Filename.basename filename in - (* Unpack the ledger in the autogen directory, since we know that we have - write permissions there. - *) let dirname = genesis_dir ^/ dirname in + (* remove dir if it exists *) + let%bind () = + if%bind file_exists ~follow_symlinks:true dirname then ( + [%log trace] "Genesis ledger dir $path already exists, removing" + ~metadata:[ ("path", `String dirname) ] ; + let rec remove_dir dir = + let%bind files = Sys.ls_dir dir in + let%bind () = + Deferred.List.iter files ~f:(fun file -> + let file = dir ^/ file in + remove file ) + in + Unix.rmdir dir + and remove file = + match%bind Sys.is_directory file with + | `Yes -> + remove_dir file + | _ -> + Unix.unlink file + in + remove dirname ) + else Deferred.unit + in let%bind () = Unix.mkdir ~p:() dirname in let open Deferred.Or_error.Let_syntax in let%map () = Tar.extract ~root:dirname ~file:filename () in From 95a7c4c400845d228e4f9d064601e372127635be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santos=20Reis?= Date: Mon, 18 Dec 2023 11:03:57 +0000 Subject: [PATCH 38/38] Improvements to tar loading logic --- .../genesis_ledger_helper.ml | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml index e3c1e5dedf1..7537c2e4e69 100644 --- a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml +++ b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml @@ -135,38 +135,36 @@ module Ledger = struct ] ; return None in + let search_local filename = + Deferred.List.find_map ~f:(file_exists filename) search_paths + in + let search_local_and_s3 filename = + match%bind search_local filename with + | Some path -> + return (Some path) + | None -> + load_from_s3 filename + in let%bind hash_filename = match config.hash with - | Some hash -> ( + | Some hash -> let hash_filename = hash_filename hash ~ledger_name_prefix in - let%bind tar_path = - Deferred.List.find_map ~f:(file_exists hash_filename) search_paths - in - match tar_path with - | Some _ -> - return tar_path - | None -> - load_from_s3 hash_filename ) + search_local_and_s3 hash_filename | None -> return None in - let search_local_and_s3 ?other_data name = - let named_filename = - named_filename ~constraint_constants ~num_accounts:config.num_accounts - ~balances:config.balances ~ledger_name_prefix ?other_data name - in - match%bind - Deferred.List.find_map ~f:(file_exists named_filename) search_paths - with - | Some path -> - return (Some path) - | None -> - load_from_s3 named_filename - in match hash_filename with | Some filename -> return (Some filename) | None -> ( + let search_local_and_s3 ?other_data name = + let named_filename = + named_filename ~constraint_constants + ~num_accounts:config.num_accounts ~balances:config.balances + ~ledger_name_prefix ?other_data name + in + search_local_and_s3 named_filename + in match (config.base, config.name) with | Named name, _ -> let named_filename = @@ -174,7 +172,7 @@ module Ledger = struct ~num_accounts:config.num_accounts ~balances:config.balances ~ledger_name_prefix name in - Deferred.List.find_map ~f:(file_exists named_filename) search_paths + search_local named_filename | Accounts accounts, _ -> search_local_and_s3 ~other_data:(accounts_hash accounts) "accounts" | Hash hash, None ->