From 74be2a6b53f57e0d909ace819774633bfc759856 Mon Sep 17 00:00:00 2001 From: "It's me, CI" Date: Mon, 23 Oct 2023 13:53:27 -0700 Subject: [PATCH 1/4] Generate scripts to fix receipt chain hashes --- scripts/migrate-receipt-chain-hashes.sh | 27 +++++++++++++++++++ src/app/receipt_chain_hash_to_b58/dune | 19 +++++++++++++ .../receipt_chain_hash_to_b58.ml | 5 ++++ src/dune-project | 1 + 4 files changed, 52 insertions(+) create mode 100755 scripts/migrate-receipt-chain-hashes.sh create mode 100644 src/app/receipt_chain_hash_to_b58/dune create mode 100644 src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml diff --git a/scripts/migrate-receipt-chain-hashes.sh b/scripts/migrate-receipt-chain-hashes.sh new file mode 100755 index 00000000000..63c7d904f3d --- /dev/null +++ b/scripts/migrate-receipt-chain-hashes.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +if [ ! $# -eq 1 ] ; then + echo "Usage" $0 archive-db + exit 0 +fi + +ARCHIVE_DB=$1 +HASHES_FILE=hashes_file.tmp +UPDATE_SCRIPT=hashes_update.sql + +rm -f $HASHES_FILE +rm -f $UPDATE_SCRIPT + +echo "select id,receipt_chain_hash from zkapp_account_precondition where receipt_chain_hash is not null;" | \ + psql --csv -t -q $ARCHIVE_DB > $HASHES_FILE + +for line in `cat $HASHES_FILE` + do ( + ID=$(echo $line | awk -F , '{print $1}'); + FP=$(echo $line | awk -F , '{print $2}'); + B58=$(echo $FP | _build/default/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.exe); + echo $ID "'"$B58"'" | awk '{print "UPDATE zkapp_account_precondition SET receipt_chain_hash=" $2 " WHERE id=" $1 ";"}' >> $UPDATE_SCRIPT) +done + +echo "Done!" +echo "Now run:" "psql -d" $ARCHIVE_DB "<" $UPDATE_SCRIPT diff --git a/src/app/receipt_chain_hash_to_b58/dune b/src/app/receipt_chain_hash_to_b58/dune new file mode 100644 index 00000000000..8dcf8756f11 --- /dev/null +++ b/src/app/receipt_chain_hash_to_b58/dune @@ -0,0 +1,19 @@ +(executable + (package receipt_chain_hash_to_b58) + (name receipt_chain_hash_to_b58) + (public_name receipt_chain_hash_to_b58) + (libraries + ;; opam libraries + ;; local libraries + mina_base + kimchi_backend + kimchi_backend.pasta + kimchi_backend.pasta.basic + pickles + pickles.backend + pickles_types + snark_params + ) + (preprocessor_deps ../../config.mlh) + (instrumentation (backend bisect_ppx)) + (preprocess (pps ppx_mina ppx_version))) diff --git a/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml b/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml new file mode 100644 index 00000000000..a363c6dffff --- /dev/null +++ b/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml @@ -0,0 +1,5 @@ +let () = + let s = Stdlib.read_line () in + let fp = Kimchi_backend.Pasta.Basic.Fp.of_string s in + let receipt_chain_hash : Mina_base.Receipt.Chain_hash.t = fp in + Format.printf "%s@." (Mina_base.Receipt.Chain_hash.to_base58_check receipt_chain_hash) diff --git a/src/dune-project b/src/dune-project index aeaeae71671..ff60cccc4c4 100644 --- a/src/dune-project +++ b/src/dune-project @@ -163,6 +163,7 @@ (package (name random_oracle_input)) (package (name random_oracle)) (package (name rc_pool)) +(package (name receipt_chain_hash_to_b58)) (package (name replayer)) (package (name rfc3339_time)) (package (name rocksdb)) From af5962b3c120a2d86e6a59ce3559d4e802eb1f07 Mon Sep 17 00:00:00 2001 From: "It's me, CI" Date: Tue, 24 Oct 2023 12:34:40 -0700 Subject: [PATCH 2/4] Allow passing hashes_file and update_script names on command line --- scripts/migrate-receipt-chain-hashes.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/migrate-receipt-chain-hashes.sh b/scripts/migrate-receipt-chain-hashes.sh index 63c7d904f3d..13192a3b546 100755 --- a/scripts/migrate-receipt-chain-hashes.sh +++ b/scripts/migrate-receipt-chain-hashes.sh @@ -1,13 +1,17 @@ #!/bin/bash -if [ ! $# -eq 1 ] ; then - echo "Usage" $0 archive-db +if [ $# -lt 1 ] || [ $# -gt 3 ]; then + echo "Usage" $0 archive-db [hashes_file] [update_script] + echo "'hashes_file' and 'update_script' are created when running this script" exit 0 fi ARCHIVE_DB=$1 -HASHES_FILE=hashes_file.tmp -UPDATE_SCRIPT=hashes_update.sql +HASHES_FILE=${2:-hashes_file.tmp} +UPDATE_SCRIPT=${3:-hashes_update.sql} + +echo "Migrating receipt chain hashes in account preconditions in archive db '"$ARCHIVE_DB"'" +echo "Using temporary file '"$HASHES_FILE"' and creating SQL script '"$UPDATE_SCRIPT"'" rm -f $HASHES_FILE rm -f $UPDATE_SCRIPT From f7c4c3c5fc2d5079987bde98785e8c6dcddb1f64 Mon Sep 17 00:00:00 2001 From: "It's me, CI" Date: Mon, 30 Oct 2023 11:18:50 -0700 Subject: [PATCH 3/4] allow already-migrated hashes --- scripts/migrate-receipt-chain-hashes.sh | 5 ++++- src/app/receipt_chain_hash_to_b58/dune | 1 + .../receipt_chain_hash_to_b58.ml | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/migrate-receipt-chain-hashes.sh b/scripts/migrate-receipt-chain-hashes.sh index 13192a3b546..e2d22db41b6 100755 --- a/scripts/migrate-receipt-chain-hashes.sh +++ b/scripts/migrate-receipt-chain-hashes.sh @@ -11,21 +11,24 @@ HASHES_FILE=${2:-hashes_file.tmp} UPDATE_SCRIPT=${3:-hashes_update.sql} echo "Migrating receipt chain hashes in account preconditions in archive db '"$ARCHIVE_DB"'" -echo "Using temporary file '"$HASHES_FILE"' and creating SQL script '"$UPDATE_SCRIPT"'" rm -f $HASHES_FILE rm -f $UPDATE_SCRIPT +echo "Creating temporary file" "'"$HASHES_FILE"'" echo "select id,receipt_chain_hash from zkapp_account_precondition where receipt_chain_hash is not null;" | \ psql --csv -t -q $ARCHIVE_DB > $HASHES_FILE +echo "Creating SQL script" "'"$UPDATE_SCRIPT"'" for line in `cat $HASHES_FILE` do ( ID=$(echo $line | awk -F , '{print $1}'); FP=$(echo $line | awk -F , '{print $2}'); B58=$(echo $FP | _build/default/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.exe); + echo -n . echo $ID "'"$B58"'" | awk '{print "UPDATE zkapp_account_precondition SET receipt_chain_hash=" $2 " WHERE id=" $1 ";"}' >> $UPDATE_SCRIPT) done +echo echo "Done!" echo "Now run:" "psql -d" $ARCHIVE_DB "<" $UPDATE_SCRIPT diff --git a/src/app/receipt_chain_hash_to_b58/dune b/src/app/receipt_chain_hash_to_b58/dune index 8dcf8756f11..4217bb626ad 100644 --- a/src/app/receipt_chain_hash_to_b58/dune +++ b/src/app/receipt_chain_hash_to_b58/dune @@ -4,6 +4,7 @@ (public_name receipt_chain_hash_to_b58) (libraries ;; opam libraries + core_kernel ;; local libraries mina_base kimchi_backend diff --git a/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml b/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml index a363c6dffff..59a3113cf5b 100644 --- a/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml +++ b/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.ml @@ -1,5 +1,15 @@ +open Core_kernel +open Mina_base + let () = let s = Stdlib.read_line () in - let fp = Kimchi_backend.Pasta.Basic.Fp.of_string s in - let receipt_chain_hash : Mina_base.Receipt.Chain_hash.t = fp in - Format.printf "%s@." (Mina_base.Receipt.Chain_hash.to_base58_check receipt_chain_hash) + let b58 = + match Receipt.Chain_hash.of_base58_check s with + | Ok _ -> + s + | Error _ -> + let fp = Kimchi_backend.Pasta.Basic.Fp.of_string s in + let receipt_chain_hash : Mina_base.Receipt.Chain_hash.t = fp in + Receipt.Chain_hash.to_base58_check receipt_chain_hash + in + Format.printf "%s@." b58 From 142d260ff981898df882eef1612e8c3297c0af55 Mon Sep 17 00:00:00 2001 From: "It's me, CI" Date: Mon, 30 Oct 2023 18:43:09 -0700 Subject: [PATCH 4/4] Migrate last_vrf_output --- scripts/migrate-itn-data.sh | 54 +++++++++++++++++++ scripts/migrate-receipt-chain-hashes.sh | 34 ------------ src/app/last_vrf_output_to_b64/dune | 14 +++++ .../last_vrf_output_to_b64.ml | 23 ++++++++ src/dune-project | 1 + 5 files changed, 92 insertions(+), 34 deletions(-) create mode 100755 scripts/migrate-itn-data.sh delete mode 100755 scripts/migrate-receipt-chain-hashes.sh create mode 100644 src/app/last_vrf_output_to_b64/dune create mode 100644 src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.ml diff --git a/scripts/migrate-itn-data.sh b/scripts/migrate-itn-data.sh new file mode 100755 index 00000000000..41f1e4b0a59 --- /dev/null +++ b/scripts/migrate-itn-data.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +if [ $# -lt 1 ] || [ $# -gt 3 ]; then + echo "Usage" $0 archive-db [data_file] [update_script] + echo "'data_file' and 'update_script' are created when running this script" + exit 0 +fi + +ARCHIVE_DB=$1 +DATA_FILE=${2:-data_file.tmp} +UPDATE_SCRIPT=${3:-data_update.sql} + +echo "Migrating receipt chain hashes in account preconditions in archive db '"$ARCHIVE_DB"'" + +rm -f $DATA_FILE +rm -f $UPDATE_SCRIPT + +echo "Creating temporary file with receipt chain hashes" "'"$DATA_FILE"'" +echo "select id,receipt_chain_hash from zkapp_account_precondition where receipt_chain_hash is not null;" | \ + psql --csv -t -q $ARCHIVE_DB > $DATA_FILE + +echo "Creating SQL script" "'"$UPDATE_SCRIPT"'" +for line in `cat $DATA_FILE` + do ( + ID=$(echo $line | awk -F , '{print $1}'); + FP=$(echo $line | awk -F , '{print $2}'); + B58=$(echo $FP | _build/default/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.exe); + echo -n . + echo $ID "'"$B58"'" | awk '{print "UPDATE zkapp_account_precondition SET receipt_chain_hash=" $2 " WHERE id=" $1 ";"}' >> $UPDATE_SCRIPT) +done + +echo +echo "Receipt chain hash pass done!" + +rm -f $DATA_FILE + +echo "Creating temporary file with last_vrf_ouput" "'"$DATA_FILE"'" +echo "select id,last_vrf_output from blocks;" | \ + psql --csv -t -q $ARCHIVE_DB > $DATA_FILE + +echo "Adding to SQL script" "'"$UPDATE_SCRIPT"'" +for line in `cat $DATA_FILE` + do ( + ID=$(echo $line | awk -F , '{print $1}'); + FP=$(echo $line | awk -F , '{print $2}'); + B64=$(echo $FP | _build/default/src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.exe); + echo -n . + echo $ID "'"$B64"'" | awk '{print "UPDATE blocks SET last_vrf_output=" $2 " WHERE id=" $1 ";"}' >> $UPDATE_SCRIPT) +done + +echo +echo "Last VRF output pass done!" + +echo "Now run:" "psql -d" $ARCHIVE_DB "<" $UPDATE_SCRIPT diff --git a/scripts/migrate-receipt-chain-hashes.sh b/scripts/migrate-receipt-chain-hashes.sh deleted file mode 100755 index e2d22db41b6..00000000000 --- a/scripts/migrate-receipt-chain-hashes.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -if [ $# -lt 1 ] || [ $# -gt 3 ]; then - echo "Usage" $0 archive-db [hashes_file] [update_script] - echo "'hashes_file' and 'update_script' are created when running this script" - exit 0 -fi - -ARCHIVE_DB=$1 -HASHES_FILE=${2:-hashes_file.tmp} -UPDATE_SCRIPT=${3:-hashes_update.sql} - -echo "Migrating receipt chain hashes in account preconditions in archive db '"$ARCHIVE_DB"'" - -rm -f $HASHES_FILE -rm -f $UPDATE_SCRIPT - -echo "Creating temporary file" "'"$HASHES_FILE"'" -echo "select id,receipt_chain_hash from zkapp_account_precondition where receipt_chain_hash is not null;" | \ - psql --csv -t -q $ARCHIVE_DB > $HASHES_FILE - -echo "Creating SQL script" "'"$UPDATE_SCRIPT"'" -for line in `cat $HASHES_FILE` - do ( - ID=$(echo $line | awk -F , '{print $1}'); - FP=$(echo $line | awk -F , '{print $2}'); - B58=$(echo $FP | _build/default/src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.exe); - echo -n . - echo $ID "'"$B58"'" | awk '{print "UPDATE zkapp_account_precondition SET receipt_chain_hash=" $2 " WHERE id=" $1 ";"}' >> $UPDATE_SCRIPT) -done - -echo -echo "Done!" -echo "Now run:" "psql -d" $ARCHIVE_DB "<" $UPDATE_SCRIPT diff --git a/src/app/last_vrf_output_to_b64/dune b/src/app/last_vrf_output_to_b64/dune new file mode 100644 index 00000000000..54c9f34dbc5 --- /dev/null +++ b/src/app/last_vrf_output_to_b64/dune @@ -0,0 +1,14 @@ +(executable + (package last_vrf_output_to_b64) + (name last_vrf_output_to_b64) + (public_name last_vrf_output_to_b64) + (libraries + ;; opam libraries + base64 + core_kernel + hex + ;; local libraries + ) + (preprocessor_deps ../../config.mlh) + (instrumentation (backend bisect_ppx)) + (preprocess (pps ppx_mina ppx_version))) diff --git a/src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.ml b/src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.ml new file mode 100644 index 00000000000..4cf7c7074ec --- /dev/null +++ b/src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.ml @@ -0,0 +1,23 @@ +open Core_kernel + +let () = + let s = Stdlib.read_line () in + let b64_check () = + match Base64.decode ~alphabet:Base64.uri_safe_alphabet s with + | Ok _ -> + (* already base64 *) + s + | Error _ -> + failwith "Bad Base64 encoding" + in + let b64 = + (* try unhexing first, because hex chars are also base64 chars *) + try + match Hex.Safe.of_hex s with + | Some unhexed -> + Base64.encode_exn ~alphabet:Base64.uri_safe_alphabet unhexed + | None -> + b64_check () + with _ -> b64_check () + in + Format.printf "%s@." b64 diff --git a/src/dune-project b/src/dune-project index ff60cccc4c4..7a68f9cdfa8 100644 --- a/src/dune-project +++ b/src/dune-project @@ -79,6 +79,7 @@ (package (name kimchi_backend)) (package (name kimchi_bindings)) (package (name kimchi_types)) +(package (name last_vrf_output_to_b64)) (package (name ledger_catchup)) (package (name ledger_proof)) (package (name libp2p_ipc))