From 57fbdf38bdb643ffcc607e4fe02d7319edff1dc8 Mon Sep 17 00:00:00 2001 From: Jerome P Date: Thu, 6 Feb 2025 07:17:08 -0800 Subject: [PATCH] WIP of make target migration --- Makefile | 53 +++++++---- rosetta_handler.py | 219 +++++++++++++++++++++++++++++++++++++++++++++ script/README.md | 186 ++++++++++++++++++++++++-------------- 3 files changed, 375 insertions(+), 83 deletions(-) create mode 100644 rosetta_handler.py diff --git a/Makefile b/Makefile index 872bc7a..6a98ce0 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -# Set default values +# Configure environment constants ACCOUNT_KEYS_FILENAME = "./account-keys.csv" -FLOW_JSON = "script/flow.json" -FLOW_JSON_NETWORK = "emulator" -FLOW_JSON_SIGNER = emulator-account +FLOW_JSON = script/flow.json +FLOW_JSON_NETWORK = localnet +FLOW_JSON_SIGNER = localnet-service-account FLOW_CLI_FLAGS = -n $(FLOW_JSON_NETWORK) -f $(FLOW_JSON) --signer $(FLOW_JSON_SIGNER) -ROSETTA_NETWORK = "localnet" +ROSETTA_ENV = localnet ROSETTA_HOST_URL = "http://127.0.0.1:8080" COMPILER_FLAGS := CGO_CFLAGS="-O2 -D__BLST_PORTABLE__" @@ -15,7 +15,6 @@ all: build go-build: ${COMPILER_FLAGS} go build -o server cmd/server/server.go - .PHONY: gen-originator-account gen-originator-account: KEYS=$$(go run ./cmd/genkey/genkey.go -csv); \ @@ -37,19 +36,38 @@ gen-originator-account: "hashAlgorithm": "SHA3_256", \ "privateKey": "'$$PRIVATE_KEY'" \ } \ - }' "${FLOW_JSON}" > flow.json.tmp && mv flow.json.tmp "${FLOW_JSON}" || { echo "Failed to update flow.json with jq"; exit 1; }; \ + }' "${FLOW_JSON}" > flow.json.tmp && mv flow.json.tmp "${FLOW_JSON}" || { echo "Failed to update ${FLOW_JSON} with jq"; exit 1; }; \ + jq --arg address "$$address" '.originators += [$$address]' "${ROSETTA_ENV}.json" > env.json.tmp && mv env.json.tmp "${ROSETTA_ENV}.json"; \ echo "$(ACCOUNT_NAME),$$KEYS,$$address" >> $(ACCOUNT_KEYS_FILENAME); \ echo "Updated $(FLOW_JSON) and $(ACCOUNT_KEYS_FILENAME)"; -.PHONY: fund-originator-accounts -fund-originator-accounts: - set -e; \ +.PHONY: fund-accounts +fund-accounts: while IFS=',' read -r col1 col2 col3 col4 address; do \ address=$$(echo $$address | xargs); \ echo "Seeding account with address: $$address"; \ flow transactions send script/cadence/transactions/basic-transfer.cdc $$address 100.0 $(FLOW_CLI_FLAGS); \ done < account-keys.csv +.PHONY: create-originator-derived-account +create-originator-derived-account: + set -e; \ + KEYS=$$(go run ./cmd/genkey/genkey.go -csv); \ + NEW_ACCOUNT_PUBLIC_FLOW_KEY=$$(echo $$KEYS | cut -d',' -f1); \ + NEW_ACCOUNT_PUBLIC_ROSETTA_KEY=$$(echo $$KEYS | cut -d',' -f2); \ + NEW_ACCOUNT_PRIVATE_KEY=$$(echo $$KEYS | cut -d',' -f3); \ + echo "Created keys for $(NEW_ACCOUNT_NAME)"; \ + echo "Flow Key: $$NEW_ACCOUNT_PUBLIC_FLOW_KEY"; \ + echo "Rosetta Key: $$NEW_ACCOUNT_PUBLIC_ROSETTA_KEY"; \ + echo "Private Key: $$NEW_ACCOUNT_PRIVATE_KEY"; \ + ROOT_ORIGINATOR_ADDRESS=$$(grep '$(ORIGINATOR_NAME)' $(ACCOUNT_KEYS_FILENAME) | cut -d ',' -f 5); \ + ROOT_ORIGINATOR_PUBLIC_KEY=$$(grep '$(ORIGINATOR_NAME)' $(ACCOUNT_KEYS_FILENAME) | cut -d ',' -f 5); \ + ROOT_ORIGINATOR_PRIVATE_KEY=$$(grep '$(ORIGINATOR_NAME)' $(ACCOUNT_KEYS_FILENAME) | cut -d ',' -f 5); \ + echo "Originator address: $$ROOT_ORIGINATOR_ADDRESS"; \ + python3 rosetta_handler.py rosetta-create-derived-account $(ROSETTA_HOST_URL) $$ROOT_ORIGINATOR_ADDRESS $$ROOT_ORIGINATOR_PUBLIC_KEY $$ROOT_ORIGINATOR_PRIVATE_KEY $$NEW_ACCOUNT_PUBLIC_ROSETTA_KEY + #echo "$(NEW_ACCOUNT_NAME),$$KEYS,$$address" >> $(ACCOUNT_KEYS_FILENAME); \ + + .PHONY: build build: go-build @@ -72,13 +90,12 @@ proto: @protoc --proto_path=model --go_out=model \ --go_opt=paths=source_relative model/model.proto -.PHONY: integration-test-cleanup -integration-test-cleanup: - rm -f flow.json - rm -f account-keys.csv +.PHONY: test-reset +test-reset: rm -rf data - rm -rf flow-go -.PHONY: integration-test -integration-test: - python3 integration_test.py \ No newline at end of file +.PHONY: test-cleanup +test-cleanup: test-reset + rm -f flow.json + rm -f account-keys.csv + rm -rf flow-go \ No newline at end of file diff --git a/rosetta_handler.py b/rosetta_handler.py new file mode 100644 index 0000000..baab5db --- /dev/null +++ b/rosetta_handler.py @@ -0,0 +1,219 @@ +import json +import subprocess +import requests +import click # pip install click + + +@click.group() +def cli(): + pass + + +# Shared function used internally but not standalone from CLI + +def request_router(target_url, body): + headers = {'Content-type': 'application/json'} + r = requests.post(target_url, data=json.dumps(body), headers=headers) + return r.json() + + +def preprocess_transaction(rosetta_host_url, root_originator, operations, metadata=None): + endpoint = "/construction/preprocess" + target_url = rosetta_host_url + endpoint + data = { + "network_identifier": { + "blockchain": "flow", + "network": "localnet" + }, + "operations": operations, + "metadata": { + "payer": root_originator + } + } + if metadata: + for key in metadata: + data["metadata"][key] = metadata[key] + return request_router(target_url, data) + + +def metadata_transaction(rosetta_host_url, options): + endpoint = "/construction/metadata" + target_url = rosetta_host_url + endpoint + data = { + "network_identifier": { + "blockchain": "flow", + "network": "localnet" + }, + "options": options + } + return request_router(target_url, data) + + +def payloads_transaction(rosetta_host_url, operations, protobuf): + endpoint = "/construction/payloads" + target_url = rosetta_host_url + endpoint + data = { + "network_identifier": { + "blockchain": "flow", + "network": "localnet" + }, + "operations": operations, + "metadata": { + "protobuf": protobuf + } + } + return request_router(target_url, data) + + +def combine_transaction(rosetta_host_url, unsigned_tx, root_originator, hex_bytes, rosetta_key, signed_tx): + endpoint = "/construction/combine" + target_url = rosetta_host_url + endpoint + data = { + "network_identifier": { + "blockchain": "flow", + "network": "localnet" + }, + "unsigned_transaction": unsigned_tx, + "signatures": [ + { + "signing_payload": { + "account_identifier": { + "address": root_originator + }, + "address": root_originator, + "hex_bytes": hex_bytes, + "signature_type": "ecdsa" + }, + "public_key": { + "hex_bytes": rosetta_key, + "curve_type": "secp256k1" + }, + "signature_type": "ecdsa", + "hex_bytes": signed_tx + } + ] + } + return request_router(target_url, data) + + +def submit_transaction(rosetta_host_url, signed_tx): + endpoint = "/construction/submit" + target_url = rosetta_host_url + endpoint + data = { + "network_identifier": { + "blockchain": "flow", + "network": "localnet" + }, + "signed_transaction": signed_tx + } + return request_router(target_url, data) + + +###################################################################################### +# Rosetta Construction helper functions callable from CLI +###################################################################################### + +@click.command() +@click.argument('rosetta_host_url', envvar='ROSETTA_HOST_URL', required=True) +@click.argument('root_originator_address', envvar='ROOT_ORIGINATOR_ADDRESS', required=True) +@click.argument('root_originator_public_rosetta_key', envvar='ROOT_ORIGINATOR_PUBLIC_ROSETTA_KEY', required=True) +@click.argument('root_originator_private_key', envvar='ROOT_ORIGINATOR_PRIVATE_KEY', required=True) +@click.argument('new_account_public_rosetta_key', envvar='NEW_ACCOUNT_PUBLIC_ROSETTA_KEY', required=True) +def rosetta_create_derived_account(rosetta_host_url, root_originator_address, root_originator_public_rosetta_key, + root_originator_private_key, new_account_public_rosetta_key): + print("Enter create: " + root_originator_address + ", new: " + new_account_public_rosetta_key) + transaction = "create_account" + metadata = {"public_key": new_account_public_rosetta_key} + operations = [ + { + "type": transaction, + "operation_identifier": { + "index": 0 + }, + "metadata": metadata + } + ] + preprocess_response = preprocess_transaction(rosetta_host_url, root_originator_address, operations) + print(preprocess_response) + metadata_response = metadata_transaction(rosetta_host_url, preprocess_response["options"]) + payloads_response = payloads_transaction(rosetta_host_url, operations, metadata_response["metadata"]["protobuf"]) + hex_bytes = payloads_response["payloads"][0]["hex_bytes"] + unsigned_tx = payloads_response["unsigned_transaction"] + sign_tx_cmd = "go run cmd/sign/sign.go " + root_originator_private_key + " " + hex_bytes + result = subprocess.run(sign_tx_cmd.split(" "), stdout=subprocess.PIPE) + signed_tx = result.stdout.decode('utf-8')[:-1] + combine_response = combine_transaction(rosetta_host_url, unsigned_tx, root_originator_address, hex_bytes, + root_originator_public_rosetta_key, signed_tx) + submit_transaction_response = submit_transaction(rosetta_host_url, combine_response["signed_transaction"]) + return submit_transaction_response["transaction_identifier"]["hash"] + + +@click.command() +@click.argument('rosetta_host_url', envvar='ROSETTA_HOST_URL', required=True) +@click.argument('root_originator_address', envvar='ROOT_ORIGINATOR_ADDRESS', required=True) +@click.argument('root_originator_public_rosetta_key', envvar='ROOT_ORIGINATOR_PUBLIC_ROSETTA_KEY', required=True) +@click.argument('root_originator_private_key', envvar='ROOT_ORIGINATOR_PRIVATE_KEY', required=True) +@click.argument('recipient_address', envvar='RECIPIENT', required=True) +@click.argument('amount', envvar='AMOUNT', required=True) +def rosetta_transfer_funds(rosetta_host_url, root_originator_address, root_originator_public_rosetta_key, + root_originator_private_key, recipient_address, amount, i=0): + transaction = "transfer" + operations = [ + { + "type": transaction, + "operation_identifier": { + "index": i + }, + "account": { + "address": root_originator_address + }, + "amount": { + "currency": { + "decimals": 8, + "symbol": "FLOW" + }, + "value": str(-1 * amount * 10 ** 7) + } + }, + { + "type": transaction, + "operation_identifier": { + "index": i + 1 + }, + "related_operations": [ + { + "index": i + } + ], + "account": { + "address": recipient_address + }, + "amount": { + "currency": { + "decimals": 8, + "symbol": "FLOW" + }, + "value": str(amount * 10 ** 7) + } + } + ] + preprocess_response = preprocess_transaction(rosetta_host_url, root_originator_address, operations) + metadata_response = metadata_transaction(rosetta_host_url, preprocess_response["options"]) + payloads_response = payloads_transaction(rosetta_host_url, operations, metadata_response["metadata"]["protobuf"]) + hex_bytes = payloads_response["payloads"][0]["hex_bytes"] + unsigned_tx = payloads_response["unsigned_transaction"] + sign_tx_cmd = "go run cmd/sign/sign.go " + root_originator_private_key + " " + hex_bytes + result = subprocess.run(sign_tx_cmd.split(" "), stdout=subprocess.PIPE) + signed_tx = result.stdout.decode('utf-8')[:-1] + combine_response = combine_transaction(rosetta_host_url, unsigned_tx, root_originator_address, hex_bytes, + root_originator_public_rosetta_key, signed_tx) + submit_transaction_response = submit_transaction(rosetta_host_url, combine_response["signed_transaction"]) + return submit_transaction_response["transaction_identifier"]["hash"] + + +# CLI bindings +cli.add_command(rosetta_create_derived_account) +cli.add_command(rosetta_transfer_funds) + +if __name__ == '__main__': + cli() diff --git a/script/README.md b/script/README.md index edce9bc..21b6dfe 100644 --- a/script/README.md +++ b/script/README.md @@ -1,118 +1,174 @@ -This directory contains the transaction scripts used by Flow Rosetta, and the -code for our Proxy contract. +This directory contains Cadence scripts and transactions used by Flow Rosetta. -## Dev Guide +# Rosetta Testing and Upgrade Validation Guide -The following provides an overview of developing with [Cadence], Flow's smart -contract language. +This guide is to support the testing and validation of Rosetta for compatibility with the flow-go module dependency. This +Rosetta implementation integrates core, internal components from the flow-go repo subjecting Rosetta to upstream breaking changes. -First, install Flow CLI: +When using `emulator` or `localnet` it will be necessary to follow all steps. For +testnet or mainnet there are relatively less steps since originator accounts have already been created, are funded and +in active use. + +## Prerequisites + +If contracts, scripts or transactions are being changed in some way, it may be necessary to learn [Cadence](https://cadence-lang.org/), Flow's smart contract language. + +### Install Flow CLI ```bash -$ sh -ci "$(curl -fsSL https://storage.googleapis.com/flow-cli/install.sh)" +sh -ci "$(curl -fsSL https://storage.googleapis.com/flow-cli/install.sh)" +flow version ``` -Init Flow CLI: +### Check python3 and dependency installation + +`python3` is used for testing with one external dependency ```bash -$ flow init +pip install click ``` -Run the emulator: +### Configure target environment + +Before starting ensure that variable constants in the project `Makefile` are updated to reflect the target environment and +desired configuration. + +## First time environment setup + +When testing on `emulator` or `localnet` it will be required to bootstrap and fund originator accounts. If testing +a live network first time setup steps are only required if the network is new. + +If using `emulator` it should be started. ```bash -$ flow emulator --contracts --persist --verbose=true +flow emulator --contracts --persist --verbose=true ``` +Be aware that when using the `--persist` flag the emulator will preserve state between starts locally in `flowdb/` directory. +If the `flowdb/` directory is deleted for a full state reset then bootstrapping will need to be run again -Note down the addresses for the `FlowToken` and `FungibleToken` contracts, and -update the corresponding import addresses used in `FlowColdStorageProxy.cdc`. +### Running localnet -Generate a public/private keypair by running `cmd/genkey`, e.g. +If using `localnet` it will need to be started. Localnet is a full Flow network with 9 containers, each one representing an instance +of one of the Flow node types used in the network. These are automatically built into images and run as containers on Docker. Detailed guidance on +running `localnet` is provided in the [README](https://github.com/onflow/flow-go/tree/master/integration/localnet). Assuming that a specific +release version is being used, you will need to clone `flow-go` and checkout that version. ```bash -$ go run ../cmd/genkey/genkey.go -Public Key (Flow Format): 51eda39f7b5f2da2ebfdb23d7645a6ebf334e49a644e2e736f89d8aefc24994772cd228c26b326fd42547dc799fe81f5f05c4286881c1ce75a81b8ecaa946d42 -Public Key (Rosetta Format): 0251eda39f7b5f2da2ebfdb23d7645a6ebf334e49a644e2e736f89d8aefc249947 -Private Key: dd04d88ef5f2e4025005a953de221b07bbab1937554a44a5ea9052b98be9c93d +git clone git@github.com:onflow/flow-go.git +cd flow-go +git checkout tags/v0.37.26 ## change as required ``` -Create an account with that key on the emulator: +Keep the version number used to hand since it will need to be matched with the `go.mod` flow-go dependency used in Rosetta later on. + +### Bootstrap originator accounts + +Create one or more originator accounts. You will need to set a unique `ACCOUNT_NAME` as an env var for each account created. It may +be necessary to update the configured ${FLOW_JSON} with the appropriate service [account details](https://developers.flow.com/tools/flow-cli/flow.json/configuration#accounts) +for your target environment before proceeding. ```bash -$ flow accounts create --sig-algo ECDSA_secp256k1 --key 51eda39f7b5f2da2ebfdb23d7645a6ebf334e49a644e2e736f89d8aefc24994772cd228c26b326fd42547dc799fe81f5f05c4286881c1ce75a81b8ecaa946d42 -Transaction ID: aa4160d3a2a5882b73f4817c58c0b2c7bc61eec68eba0510c1392a3fd01796b1 -Address 0x01cf0e2f2f715450 -Balance 0.00100000 +ACCOUNT_NAME=root-originator-1 make gen-originator-account ``` +This target undertakes the following for the target environment: -Update the `flow.json` config with a new `contract-account` value in the -`accounts` section using the generated private key and created account address: - -```json -"contract-account": { - "address": "01cf0e2f2f715450", - "key": { - "type": "hex", - "index": 0, - "signatureAlgorithm": "ECDSA_secp256k1", - "hashAlgorithm": "SHA3_256", - "privateKey": "dd04d88ef5f2e4025005a953de221b07bbab1937554a44a5ea9052b98be9c93d" - } -} -``` +* Generates public and private key pairs for use by Flow and Rosetta +* Uses the generated public key to create a new Flow account to serve as an originator +* Updates ${FLOW_JSON} with the new account and private key JSON block +* Adds the new address to the originators list in ${ROSETTA_ENV} config JSON +* Updates ${ACCOUNT_KEYS_FILENAME} with a single CSV row entry with the following columns + * `name, Flow public key, Rosetta public key, Flow private key, account identifier` + +### Fund accounts -Deploy the contract: +Use this target to fund the newly created originator accounts from the network service account. ```bash -$ flow accounts add-contract --signer contract-account FlowColdStorageProxy ./FlowColdStorageProxy.cdc -Transaction ID: f536cd544068cb6976e0226b4146d53b7d3d4aa94e4359df5c1fca66f29eba9a -Contract 'FlowColdStorageProxy' deployed to the account '01cf0e2f2f715450'. +make fund-accounts ``` -If there are any issues, update the contract for changes: +* Iterates row entries in ${ACCOUNT_KEYS_FILENAME} and funds 100 FLOW to each one -```bash -$ flow accounts update-contract --signer contract-account FlowColdStorageProxy ./FlowColdStorageProxy.cdc -Transaction ID: f536cd544068cb6976e0226b4146d53b7d3d4aa94e4359df5c1fca66f29eba9a -Contract 'FlowColdStorageProxy' deployed to the account '01cf0e2f2f715450'. + +## Testing, validation and upgrade sequence + +Once the environment is bootstrapped with funded originator accounts then it's possible to start testing. It's advisable +to start with a confirmed baseline, namely ensuring that the Rosetta version you are starting with works with the `flow-go` +version currently on testnet/mainnet. + +1. Baseline test of main branch Rosetta against current emulator +2. Baseline test of main branch Rosetta against testnet +3. Update `flow-go` and other required dependencies which also need updating +4. Build, deploy and start `localnet` using the same versions of `flow-go` etc +5. Test and validate Rosetta against `localnet` +6. Release new Rosetta version +7. Coordinate with Coinbase to deploy new version during scheduled network upgrade + +### Check Environment before testing + +The remainder of this guide assumes that the required bootstrapping of the target environment is done and that either +the emulator, or localnet, are running (unless testing a live network). If you have previously run Rosetta and encounter this +error on startup: + +```go +ERROR Unexpected parent hash found for block 8345eb03aa4959d3652fb485e3a91f981672276973ecee9b9f2f43ac0cc55aa9 at height 1: expected c77c642ceaa5b3e4b3265da14ead25361c9cd903652b34fc022494fc73f2177b, got 5b105616db0b3c75a7efc4e97ae09d30b48cb0d679221ba9cbdcb7aa29c86dcf ``` -To test out Cadence transaction scripts, place the scripts into files and submit -them with any arguments: +Delete `data/` folder which stores state from previous runs and will cause this error to occur when bringing up Rosetta server: + +### Build Rosetta ```bash -$ flow transactions send --network emulator [ ...] -Transaction ID: b4bf97253c590cb7b4118870efbddb206e776f5fa3dc277281fef493b1445a5c +make build ``` -## Upgrade Notes for the Secure Cadence Release +### Start Rosetta -To validate the code for the [breaking changes introduced by the Secure Cadence -release], first install Flow CLI for the Secure Cadence Emulator: +```bash +./server localnet.json ## use appropriate Rosetta JSON configuration for your target environment +``` + +If successful it should log something like this ```bash -$ sh -ci "$(curl -fsSL https://raw.githubusercontent.com/onflow/flow-cli/master/install.sh)" -- v0.33.2-sc-m6 +➜ ./server localnet.json +2025-02-05T12:42:02.531-0800 INFO [cache/badger] All 0 tables opened in 0s +2025-02-05T12:42:02.542-0800 INFO [cache/badger] Discard stats nextEmptySlot: 0 +2025-02-05T12:42:02.542-0800 INFO [cache/badger] Set nextTxnTs to 0 +2025-02-05T12:42:02.578-0800 INFO Running localnet in online mode +2025-02-05T12:42:02.837-0800 INFO Genesis block set to block b21f01cd3bedc3fb3f716a466c05d4c384169a11091436db6f080afa087fb2f8 at height 1 +2025-02-05T12:42:02.932-0800 INFO Running background loop to validate account balances +2025-02-05T12:42:02.932-0800 INFO Starting Rosetta Server on port 8080 +2025-02-05T12:42:03.192-0800 INFO Retrieved block 20dd925a750399493cf7455f199c32c952e8010a6c0b4424dba00a193fa18e44 at height 2 +2025-02-05T12:42:03.370-0800 INFO Retrieved block 7a4596450e5802ae58407fdf09d429ba74c24fdc146c53cb8d184a678d9f4e7a at height 3 +... ``` +Rosetta will continue syncing blocks from Flow Access nodes until it has caught up, which may take some time on long-lived networks. -Install Cadence Analyzer: +Before continuing to test you must wait for Rosetta to confirm it has reached the tip: ```bash -$ sh -ci "$(curl -fsSL https://storage.googleapis.com/flow-cli/install-cadence-analyzer.sh)" +2025-02-05T17:17:33.701-0800 INFO Indexer is close to tip ``` -To validate deployed contracts code, run `cadence-analyzer` on the address where -the contract was deployed: +### Create originator derived accounts + +If this is the first time testing in this environment, or if the Flow environment has been reset, you will need to create at least two originator +derived accounts. These _must_ be created via Rosetta rather than through the `flow-cli` or other non Rosetta transactions. ```bash -$ cadence-analyzer -network emulator -address 0x01cf0e2f2f715450 +NEW_ACCOUNT_NAME=derived-account-1 ORIGINATOR_NAME=root-originator-1 make rosetta-create-sub-account ``` -To validate transaction scripts, submit transactions with specific scripts, and -then run `cadence-analyzer` on the submitted transaction IDs: +### Transfer funds +Now we use Rosetta to trigger a transfer into the newly created derived account. ```bash -$ cadence-analyzer -network emulator -transaction b4bf97253c590cb7b4118870efbddb206e776f5fa3dc277281fef493b1445a5c +RECIPIENT=derived-account-1 ORIGINATOR_NAME=root-originator-1 make rosetta-transfer-funds ``` -[breaking changes introduced by the Secure Cadence release]: https://forum.onflow.org/t/breaking-changes-coming-with-secure-cadence-release/3052 -[Cadence]: https://docs.onflow.org/cadence/ +## Flow-go update guidance + +In general + +go get github.com/onflow/flow-go@master