diff --git a/.dockerignore b/.dockerignore index c668a76..6244b63 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,9 @@ Dockerfile LICENSE manage README.md +COMPLIANCE.yaml .git* +tmp/ +backup/ +openshift/ +docs/ diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 913a47b..36de7e2 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - master # will be renaming soon. # Ensure only the most recent workflow runs. concurrency: diff --git a/.gitignore b/.gitignore index 7f98992..0cb49c3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ backup/ *_BuildConfig.json tmp/ +# External scripts +genesis_from_files.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile index 18f54c4..bf3d7d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM bcgovimages/von-image:node-1.12-3 +FROM bcgovimages/von-image:node-1.12-4 ENV LOG_LEVEL ${LOG_LEVEL:-info} ENV RUST_LOG ${RUST_LOG:-warning} diff --git a/README.md b/README.md index cc7e110..6da9da3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) +[![Lifecycle:Stable](https://img.shields.io/badge/Lifecycle-Stable-97ca00)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) + # VON Network A portable development level Indy Node network, including a Ledger Browser. The Ledger Browser (for example the BC Gov's [Ledger for the GreenLight Demo Application](http://greenlight.bcovrin.vonx.io/)) allows a user to see the status of the nodes of a network and browse/search/filter the Ledger Transactions. @@ -28,6 +31,8 @@ Want to see a full demo that includes applications and verifiable credentials be This repository includes a fully containerized Indy-Cli environment, allowing you to use the Indy-Cli without having to build or install the Indy-SDK or any of its dependencies on your machine. +For more information refer to [Using the containerized `indy-cli`](./docs/Indy-CLI.md) + ## Ledger Troubleshooting Refer to the [Troubleshooting](./docs/Troubleshooting.md) document for some tips and tools you can use to troubleshoot issues with a ledger. @@ -45,13 +50,31 @@ The [tutorial about using VON Network](docs/UsingVONNetwork.md) has information 1. Run docker to start the ledger, and pass in GENESIS_URL and LEDGER_SEED parameters: +GENESI_URL could be a http(s) URL or path to genesis file inside the container. + +To use the path to genesis file, you need to use the protocol `file://` in GENESIS_URL and the genesis file need to be inside the container. You have two options: + +- Attach a volumen (f.e ./tmp:/tmp) to `webserver` service in volume section inside `docker-compose.yml` file. +- Copy the file inside container using `docker cp` . + For example to connect to the Sovrin Test Network: ```bash ./manage build -GENESIS_URL=https://raw.githubusercontent.com/sovrin-foundation/sovrin/master/sovrin/pool_transactions_sandbox_genesis ./manage start-web +GENESIS_URL=https://raw.githubusercontent.com/sovrin-foundation/sovrin/master/sovrin/pool_transactions_sandbox_genesis +LEDGER_SEED=00000000000000000000000000000012 +./manage start-web ``` +```bash +./manage build +GENESIS_URL=file:///tmp/pool_transactions_sandbox_genesis +LEDGER_SEED=00000000000000000000000000000012 +./manage start-web +``` + + + Note that it takes some time to get the transactions and status from the network. Once the UI appears, try getting the `Genesis Transaction` that the server started up properly. ## Running the web server on your local machine @@ -136,9 +159,9 @@ GENESIS_URL=https://raw.githubusercontent.com/sovrin-foundation/sovrin/master/so 3. Download this repository: ```bash - curl -L https://github.com/bcgov/von-network/archive/master.zip > bcovrin.zip && \ + curl -L https://github.com/bcgov/von-network/archive/main.zip > bcovrin.zip && \ unzip bcovrin.zip && \ - cd von-network-master && \ + cd von-network-main && \ chmod a+w ./server/ ``` diff --git a/bin/read_ledger b/bin/read_ledger index 6c6d227..9f75486 100755 --- a/bin/read_ledger +++ b/bin/read_ledger @@ -1,24 +1,30 @@ #! /usr/bin/env python3 """ Convenience script for reading a ledger transaction data (stored in leveldb) + """ import argparse import logging import os import shutil +from pathlib import Path + from common.serializers.json_serializer import JsonSerializer -from ledger.compact_merkle_tree import CompactMerkleTree from plenum.common.ledger import Ledger -from storage.helper import initHashStore +from plenum.common.constants import HS_ROCKSDB from indy_common.config_util import getConfig -from indy_common.config_helper import ConfigHelper, NodeConfigHelper +from common.serializers.serialization import ledger_txn_serializer logging.root.handlers = [] logger = logging.getLogger() logger.propagate = False logger.disabled = True +_DATA = 'data' + +# TODO: Replace with constant from config +postfix = '_transactions' def read_args(): @@ -39,73 +45,69 @@ def read_args(): parser.add_argument('--count', required=False, action='store_true', help="returns the number of txns in the given ledger") parser.add_argument('--node_name', required=False, help="Node's name") - parser.add_argument('--client_name', required=False, help="Client's name") parser.add_argument('--serializer', required=False, default='json', help="How to represent the data (json by default)") parser.add_argument('--network', required=False, type=str, help="Network name to read ledger from") - parser.add_argument('--base_dir', required=False, type=str, - help="Specifify a directory to read from") return parser.parse_args() -def get_ledger_dir(node_name, client_name, network, base_dir): - - if node_name and client_name: - print("Either 'node_name' or 'client_name' can be specified") - exit() - +def get_ledger_dir(node_name, network): config = getConfig() _network = network if network else config.NETWORK_NAME - if node_name or client_name: - if node_name: - config_helper = NodeConfigHelper(node_name, config) - ledger_data_dir = config_helper.ledger_dir - else: - ledger_data_dir = os.path.join(config.CLI_BASE_DIR, _network, - config.clientDataDir, client_name) - if not os.path.isdir(ledger_data_dir): - print("Specified Node or Client folder not found: {}".format( - ledger_data_dir)) - exit() + ledger_base_dir = config.LEDGER_DIR + if node_name: + # Build path to data if --node_name was specified + ledger_data_dir = os.path.join(ledger_base_dir, _network, _DATA, node_name) else: - - if not base_dir: - base_dir = config.CLI_BASE_DIR if client_name else config.baseDir - data_dir = config.clientDataDir if client_name else 'data' - ledger_data_dir = os.path.join(base_dir, _network, data_dir) - - if not os.path.exists(ledger_data_dir): - print("Can not find the directory with the ledger: {}".format( - ledger_data_dir)) - exit() - - dirs = os.listdir(ledger_data_dir) - ledger_data_dir = os.path.join(ledger_data_dir, dirs[0]) + ledger_data_dir = os.path.join(ledger_base_dir, _network, _DATA) + if os.path.exists(ledger_data_dir): + dirs = os.listdir(ledger_data_dir) + if len(dirs) == 0: + print("Node's 'data' folder not found: {}".format(ledger_data_dir)) + exit() + # --node_name parameter was not set, therefore we can choose first Node name in data dir + ledger_data_dir = os.path.join(ledger_data_dir, dirs[0]) + if not os.path.exists(ledger_data_dir): + print("No such file or directory: {}".format(ledger_data_dir)) + print("Please check, that network: '{}' was used ".format(_network)) + exit() return ledger_data_dir -def get_ledger(type_, ledger_data_dir): +def get_storage(type_, ledger_data_dir): config = getConfig() - ledger_name = None + storage_name = None if type_ == 'pool': - ledger_name = config.poolTransactionsFile + storage_name = config.poolTransactionsFile elif type_ == 'domain': - ledger_name = config.domainTransactionsFile + storage_name = config.domainTransactionsFile elif type_ == 'config': - ledger_name = config.configTransactionsFile + storage_name = config.configTransactionsFile + elif type_ in get_additional_storages(ledger_data_dir): + storage_name = type_ + postfix else: print("Unknown ledger type: {}".format(type_)) exit() - hash_store = initHashStore(ledger_data_dir, type_, config) - return Ledger(CompactMerkleTree(hashStore=hash_store), dataDir=ledger_data_dir, fileName=ledger_name) + return Ledger._defaultStore(dataDir=ledger_data_dir, + logName=storage_name, + ensureDurability=True, + open=True, + config=config, + read_only=True) + +def get_additional_storages(ledger_data_dir): + additional_storages = \ + [name[:name.find(postfix)] for name in os.listdir(ledger_data_dir) if postfix in name] + return additional_storages -def print_txns(ledger, args): + +def print_txns(storage, args): serializer = None if args.serializer == 'json': serializer = JsonSerializer() @@ -116,40 +118,43 @@ def print_txns(ledger, args): # --count count = args.count if count: - print_count(ledger) + print_count(storage) return # --seq_no - seq_no = int(args.seq_no) if args.seq_no is not None else None + seq_no = args.seq_no if seq_no: - print_by_seq_no(ledger, seq_no, serializer) + print_by_seq_no(storage, seq_no, serializer) return # print all (--from --to) - print_all(ledger, serializer) + print_all(storage, serializer) -def print_by_seq_no(ledger, seq_no, serializer): +def print_by_seq_no(storage, seq_no, serializer): try: - txn = ledger.getBySeqNo(seq_no) + print("Getting by seq no:", seq_no) + txn = storage.get(seq_no) except KeyError: print('No transactions found for seq_no={}'.format(seq_no)) return + print("Serializing transactions ...") txn = serializer.serialize(txn, toBytes=False) + print("Printing transactions ...") print(txn) return -def print_count(ledger): - print(ledger.size) +def print_count(storage): + print(storage.size) -def print_all(ledger, serializer): - frm = int(args.frm) if args.frm is not None else None - to = int(args.to) if args.to is not None else None - for txn in ledger.getAllTxn(frm=frm, to=to): - txn = serializer.serialize(txn, toBytes=False) - print(txn) +def print_all(storage, serializer): + frm = int(args.frm) if args.frm else None + to = int(args.to) if args.to else None + for seqNo, txn in storage.iterator(start=frm, end=to): + txn = ledger_txn_serializer.deserialize(txn) + print(serializer.serialize(txn, toBytes=False)) def make_copy_of_ledger(data_dir): @@ -162,20 +167,30 @@ def make_copy_of_ledger(data_dir): if __name__ == '__main__': args = read_args() + config = getConfig() - # TODO: works well only for small ledgers, - ledger_data_dir = get_ledger_dir( - args.node_name, - args.client_name, - args.network, - args.base_dir) - + ledger_data_dir = get_ledger_dir(args.node_name, args.network) read_copy_ledger_data_dir = None try: - read_copy_ledger_data_dir = make_copy_of_ledger(ledger_data_dir) - ledger = get_ledger(args.type, read_copy_ledger_data_dir) - print_txns(ledger, args) + # RocksDB supports real read-only mode and does not need to have a ledger copy. + if config.hashStore['type'].lower() != HS_ROCKSDB: + config.db_transactions_config = None + # NOTE: such approach works well only for small ledgers. + tmp = make_copy_of_ledger(ledger_data_dir) + + # Let's be paranoid to avoid removing of ledger instead of its copy. + ledger_path = Path(ledger_data_dir) + ledger_copy_path = Path(tmp) + assert ledger_path != ledger_copy_path + assert ledger_copy_path not in ledger_path.parents + + read_copy_ledger_data_dir = tmp + ledger_data_dir = read_copy_ledger_data_dir + elif config.db_transactions_config is not None: + # This allows to avoid debug logs creation on each read_ledger run + config.db_transactions_config['db_log_dir'] = '/dev/null' + storage = get_storage(args.type, ledger_data_dir) + print_txns(storage, args) finally: - # TODO be careful about removing original the ledger data dir if read_copy_ledger_data_dir: shutil.rmtree(read_copy_ledger_data_dir) diff --git a/cli-scripts/generate_did.py b/cli-scripts/generate_did.py new file mode 100644 index 0000000..04ea635 --- /dev/null +++ b/cli-scripts/generate_did.py @@ -0,0 +1,38 @@ +import os +import logging +import base64 +import argparse +import asyncio +import sys +# Import server.anchor from the path relative to where the scripts are being executed. +sys.path.insert(1, './server') +from anchor import AnchorHandle + +logging.getLogger().setLevel(logging.ERROR) + +async def generate_did(seed): + TRUST_ANCHOR = AnchorHandle() + did, verkey = await TRUST_ANCHOR.seed_to_did(seed) + print(f"\nSeed: {seed}") + print(f"DID: {did}") + print(f"Verkey: {verkey}") + +def main(seed): + loop = asyncio.get_event_loop() + loop.run_until_complete(generate_did(seed)) + loop.close() + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Generates a DID and Verkey from a Seed.") + parser.add_argument("--seed", required=True, default=os.environ.get('SEED'), help="The seed to use to generate the DID and Verkey.") + args, unknown = parser.parse_known_args() + + testseed = args.seed + if testseed.endswith("="): + testseed = base64.b64decode(testseed).decode("ascii") + + if not len(testseed) == 32: + print("Seed must be 32 characters long.") + exit() + + main(args.seed) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a845f77..1d8c99c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,7 @@ services: volumes: - ./config:/home/indy/config - ./server:/home/indy/server + - ./tmp:/tmp - webserver-cli:/home/indy/.indy-cli - webserver-ledger:/home/indy/ledger diff --git a/docs/Indy-CLI.md b/docs/Indy-CLI.md new file mode 100644 index 0000000..5309dcf --- /dev/null +++ b/docs/Indy-CLI.md @@ -0,0 +1,339 @@ +# Using the containerized `indy-cli` + +This repository includes a fully containerized `indy-cli` environment, allowing you to use the `indy-cli` without ever having to build or install the `indy-sdk` or any of its dependencies on your machine. + +The following `indy-cli` examples are performed using a local instance of `von-network`, however the `indy-cli` environment can be connected to any number of `indy` ledgers. In order to use some `indy-cli` commands you'll need to have a DID already registered on that network. The process of starting the `indy-cli` and generating a local wallet will generate the DID and Verkey you could then have registered on that network. + +## Example Prerequisites + +### Build `von-network` + +``` +./manage build +``` + +### Generate a set of Secrets + +The first thing you'll need is a set of secrets, a `seed`: used to generate your DID, and a `key`: used as the encryption key for your wallet. + +These two values are very sensitive. Make sure you store them in a safe place such as a secure password manager application. If someone gets hold of your `seed`, they can regenerate your `DID` and `Verkey` and impersonate you. If someone gains access to your wallet and your wallet `key`, they can impersonate you AND have control over all of the credentials in your wallet. So, make sure you keep your `seed` and wallet `key` a secret. + +The `./manage` script makes this easy to generate these values randomly, just run: + +``` +./manage generateSecrets +``` +``` +Seed: 2yMTRs+zBBPF75OE5i1WCY5uDcQYUdt4 +Key: b+StO9VtqyvdEeUsUQknR6uB+UoXK09kHzskSjIYIZ0GQmG4O0bLULc7NlY42/Bh +``` + +For the remainder of the examples we'll use the seed and key from above for consistency. + +### Start an instance of `von-network` + +``` +./manage start +``` + +### Register your DID on the network + +1. Browse to http://localhost:9000/. +2. In the **Authenticate a New DID** section enter the seed (created above) in the **Wallet seed (32 characters or base64)** field and `my-did` in the Alias (optional) field. +3. Click **Register DID** + +You should see the following message written to the screen: +``` +Identity successfully registered: +Seed: 2yMTRs+zBBPF75OE5i1WCY5uDcQYUdt4 +DID: 5DHNJswc1LjkCWPYYrzYVX +Verkey: 3J9F2QpresMoukCSJTbvM2YmaovfqveVDgg1P1azMsJP +``` + +_Now on to working with the `indy-cli` ..._ + +## Register the network (ledger) with the `indy-cli` environment + +This command downloads and registeres the network's genisis file with the `indy-cli`. + +Note we're using the internal docker host IP address to access the locally running network since it's also running in docker. This IP can be obtained by running `./manage dockerhost`. + +``` +./manage cli init-pool local_net http://192.168.65.3:9000/genesis +``` +``` +Creating von_client_run ... done + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed +100 3092 100 3092 0 0 1006k 0 --:--:-- --:--:-- --:--:-- 1006k +``` + +To register another network (say Sovrin BuilderNet) you would run the command like this: +``` +./manage cli init-pool builder_net https://raw.githubusercontent.com/sovrin-foundation/sovrin/stable/sovrin/pool_transactions_builder_genesis +``` + +## Create a wallet + +``` +./manage \ + indy-cli create-wallet \ + walletName=local_net_trustee_wallet +``` + +When prompted, enter the wallet encryption key to create the wallet: +``` +Creating von_client_run ... done +load-plugin library=libindystrgpostgres.so initializer=postgresstorage_init +Plugin has been loaded: "libindystrgpostgres.so" + +wallet create local_net_trustee_wallet key storage_type=default storage_config={} storage_credentials={} +Enter value for key: + +``` + +When prompted, enter the wallet encryption key to open the wallet: +``` +Wallet "local_net_trustee_wallet" has been created + +wallet open local_net_trustee_wallet key storage_credentials={} +Enter value for key: + +``` + +When prompted, enter your seed: +``` +Wallet "local_net_trustee_wallet" has been opened + +did new seed +Enter value for seed: + +``` + +``` +Did "5DHNJswc1LjkCWPYYrzYVX" has been created with "~ATfU5eKehb2Y1WkqHbmLvT" verkey + +did list ++------------------------+-------------------------+----------+ +| Did | Verkey | Metadata | ++------------------------+-------------------------+----------+ +| 5DHNJswc1LjkCWPYYrzYVX | ~ATfU5eKehb2Y1WkqHbmLvT | - | ++------------------------+-------------------------+----------+ + +wallet close +Wallet "local_net_trustee_wallet" has been closed + +wallet detach local_net_trustee_wallet +Wallet "local_net_trustee_wallet" has been detached + +exit + +Goodbye... +``` + +## Export (backup) a wallet + +``` +./manage \ + indy-cli export-wallet \ + walletName=local_net_trustee_wallet \ + exportPath=/tmp/local_net_trustee_wallet.export +``` + +When prompted, enter the wallet encryption key to open the wallet: +``` +Creating von_client_run ... done +load-plugin library=libindystrgpostgres.so initializer=postgresstorage_init +Plugin has been loaded: "libindystrgpostgres.so" + +-wallet attach local_net_trustee_wallet storage_type=default storage_config={} +Wallet "local_net_trustee_wallet" has been attached + +wallet open local_net_trustee_wallet key storage_credentials={} +Enter value for key: + +``` + +When prompted, enter the wallet encryption key to encrypt the wallet export: +``` +Wallet "local_net_trustee_wallet" has been opened + +wallet export export_path=/tmp/local_net_trustee_wallet.export export_key +Enter value for export_key: + +``` + +``` +Wallet "local_net_trustee_wallet" has been exported to the file "/tmp/local_net_trustee_wallet.export" + +wallet close +Wallet "local_net_trustee_wallet" has been closed + +wallet detach local_net_trustee_wallet +Wallet "local_net_trustee_wallet" has been detached + +exit + +Goodbye... +``` + +## Import a wallet Export (backup) file + +``` +./manage \ + indy-cli import-wallet \ + walletName=local_net_trustee_wallet \ + importPath=/tmp/local_net_trustee_wallet.export +``` + +When prompted, enter the encryption key of the export (backup) file: +``` +Creating von_client_run ... done +load-plugin library=libindystrgpostgres.so initializer=postgresstorage_init +Plugin has been loaded: "libindystrgpostgres.so" + +wallet import local_net_trustee_wallet key export_path=/tmp/local_net_trustee_wallet.export export_key storage_type=default storage_config={} storage_credentials={} +Enter value for key: + +``` + +When prompted, enter the wallet encryption key for the restore process: +``` +Enter value for export_key: + +``` + +When prompted, enter the wallet encryption key to open the wallet: +``` +Wallet "local_net_trustee_wallet" has been created + +wallet open local_net_trustee_wallet key storage_credentials={} +Enter value for key: + +``` +``` +Wallet "local_net_trustee_wallet" has been opened + +did list ++------------------------+-------------------------+----------+ +| Did | Verkey | Metadata | ++------------------------+-------------------------+----------+ +| 5DHNJswc1LjkCWPYYrzYVX | ~ATfU5eKehb2Y1WkqHbmLvT | - | ++------------------------+-------------------------+----------+ + +wallet close +Wallet "local_net_trustee_wallet" has been closed + +wallet detach local_net_trustee_wallet +Wallet "local_net_trustee_wallet" has been detached + +exit + +Goodbye... +``` + +## Open an interactive `indy-cli` session + +This command starts an `indy-cli` session with a given ledger and opens the specified wallet and sets the session context using the `did use` command. + +The command is setup to use an `indy-cli` config file to indicate the TAA acceptance mechanism. Create the file `./tmp/cliconfig.json` in the `von-network` folder (i.e. `.../von-network/tmp/cliconfig.json`) and save the following content: +``` +{ + "taaAcceptanceMechanism": "for_session" +} +``` + +The `./tmp` folder gets mounted automatically into the docker container's `/tmp` folder. + + +``` +./manage \ + indy-cli --config /tmp/cliconfig.json start-session \ + walletName=local_net_trustee_wallet \ + poolName=local_net \ + useDid=5DHNJswc1LjkCWPYYrzYVX +``` + +When prompted, enter the wallet encryption key to open the wallet: +``` +Creating von_client_run ... done +"for_session" is used as transaction author agreement acceptance mechanism +load-plugin library=libindystrgpostgres.so initializer=postgresstorage_init +Plugin has been loaded: "libindystrgpostgres.so" + +pool connect local_net +Pool "local_net" has been connected + +-wallet attach local_net_trustee_wallet storage_type=default storage_config={} +Wallet "local_net_trustee_wallet" has been attached + +wallet open local_net_trustee_wallet key storage_credentials={} +Enter value for key: + +``` +``` +Wallet "local_net_trustee_wallet" has been opened + +did use 5DHNJswc1LjkCWPYYrzYVX +Did "5DHNJswc1LjkCWPYYrzYVX" has been set as active + +pool(local_net):local_net_trustee_wallet:did(5DH...YVX):indy> +``` + +You are now ready to issue commands/transactions to the ledger using your DID's context. + +### Try it out + +Read your DID from the ledger by entering the following command on the `indy-cli` command prompt: +``` +ledger get-nym did=5DHNJswc1LjkCWPYYrzYVX +``` +``` +Following NYM has been received. +Metadata: ++------------------------+-----------------+---------------------+---------------------+ +| Identifier | Sequence Number | Request ID | Transaction time | ++------------------------+-----------------+---------------------+---------------------+ +| 5DHNJswc1LjkCWPYYrzYVX | 10 | 1633469176534450900 | 2021-10-05 20:02:27 | ++------------------------+-----------------+---------------------+---------------------+ +Data: ++------------------------+------------------------+----------------------------------------------+----------+ +| Identifier | Dest | Verkey | Role | ++------------------------+------------------------+----------------------------------------------+----------+ +| V4SGRU86Z58d6TV7PBUe6f | 5DHNJswc1LjkCWPYYrzYVX | 3J9F2QpresMoukCSJTbvM2YmaovfqveVDgg1P1azMsJP | ENDORSER | ++------------------------+------------------------+----------------------------------------------+----------+ +pool(local_net):local_net_trustee_wallet:did(5DH...YVX):indy> +``` + +When done type `exit` and press enter: +``` +pool(local_net):local_net_trustee_wallet:did(5DH...YVX):indy> exit +Pool "local_net" has been disconnected +Wallet "local_net_trustee_wallet" has been closed +Goodbye... +``` + +## Open an interactive `indy-cli` command line + +``` +./manage indy-cli +``` +``` +Creating von_client_run ... done +indy> +``` + +You are now on an interactive `indy-cli` command line. For `help` with the commands type `help`. This will give you information about the top level commands. To get help about a specific command, or sub-command type the ` [sub-command]` followed by help; for example `ledger help`, or `ledger get-nym help`. + +When done type `exit` and press enter: +``` +indy> exit +Goodbye... +``` + +## Reset your `indy-cli` environment + +**Use with caution. This deletes everything in your `indy-cli` environment. Make sure you have things backed up if needed.** +``` +./manage cli reset +``` diff --git a/docs/UsingVONNetwork.md b/docs/UsingVONNetwork.md index 8dbf421..1f764f6 100644 --- a/docs/UsingVONNetwork.md +++ b/docs/UsingVONNetwork.md @@ -101,17 +101,18 @@ Another thing to notice about the ledger file is the IP addresses for the ledger ## Using the CLI -The VON Network provides a way to get to an Indy command line interface (CLI). To get to the Indy CLI for the network, get to the command prompt (using Ctrl-C if necessary) and run the command: +`von-network` provides a way to get to an Indy command line interface (CLI). To get to the Indy CLI for the network, get to the command prompt (using Ctrl-C if necessary) and run the command: ``` ./manage indy-cli - ``` Run the “help” command to see what you can do, and “exit” to get out of the CLI session. We won’t go into the Indy CLI here, but it has powerful capability that you can use to create scripts for configuring an Indy network and adding objects to the ledger. If you find any Indy CLI tutorials on other sites, you can run them using this mechanism, without any of the bother of getting everything running before you start the CLI. +For more information refer to [Using the containerized `indy-cli`](./Indy-CLI.md) + ## Stopping and Removing a VON Network To stop and delete a running VON Network, get to a command line prompt (using Ctrl-C if necessary), and then run the command: diff --git a/manage b/manage index e7d53b8..07eb1bf 100755 --- a/manage +++ b/manage @@ -1,6 +1,8 @@ #!/bin/bash export MSYS_NO_PATHCONV=1 -export DOCKERHOST=${APPLICATION_URL-$(docker run --rm --net=host eclipse/che-ip)} +# getDockerHost; for details refer to https://github.com/bcgov/DITP-DevOps/tree/main/code/snippets#getdockerhost +. /dev/stdin <<<"$(cat <(curl -s --raw https://raw.githubusercontent.com/bcgov/DITP-DevOps/main/code/snippets/getDockerHost))" +export DOCKERHOST=$(getDockerHost) SCRIPT_HOME="$( cd "$( dirname "$0" )" && pwd )" export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-von}" @@ -10,6 +12,7 @@ export LEDGER_TIMEOUT="${LEDGER_TIMEOUT:-60}" export LEDGER_URL_CONFIG="${LEDGER_URL_CONFIG}" export ROOT_BACKUP_DIR=backup +export SHELL_CMD='bash' # Running on Windows? if [[ "$OSTYPE" == "msys" ]]; then @@ -17,6 +20,23 @@ if [[ "$OSTYPE" == "msys" ]]; then terminalEmu="winpty" fi +# ======================================================================================================== +# Dynamically detect the version of docker compose and adjust the '--log-level' syntax appropriately. +# -------------------------------------------------------------------------------------------------------- +# Default to using the existing syntax +dockerCompose="docker-compose --log-level ERROR" +dockerComposeVersion=$(docker-compose version --short | sed 's~v~~;s~-.*~~') +dockerComposeVersion=${dockerComposeVersion%.*} +if [[ $(awk "BEGIN {print (${dockerComposeVersion} >= 2.0) ? \"true\" : \"false\" }") == "true" ]]; then + # Use the new syntax when version 2.0.0 or greater is detected. + dockerCompose="docker --log-level error compose" +fi + +echo +echo "Using: ${dockerCompose}" +echo +# ======================================================================================================== + # ================================================================================================================= # Usage: # ----------------------------------------------------------------------------------------------------------------- @@ -46,6 +66,7 @@ usage () { start-web - Start the web server to monitor an existing ledger, requires GENESIS_URL and LEDGER_SEED params Example: $0 start-web GENESIS_URL=http://foo.bar/genesis.txt LEDGER_SEED=00000000000000000000000000000012 + $0 start-web GENESIS_URL=file:///tmp/genesis.txt LEDGER_SEED=00000000000000000000000000000012 logs - To tail the logs of running containers (ctrl-c to exit). Use the '--no-tail' option to only print log without tailing. @@ -64,6 +85,29 @@ usage () { generateSecrets - Generate a random set of secrets using openssl; a Seed and a Key. + generateDid - Generates a DID and Verkey from a Seed. + $0 generateDid [seed] + - Optional [seed]; if one is not provided a random one will be generated using openssl. + + generateGenesisFiles - Generates pool and domain genesis files from data input via csv files. + $0 generategenesisfiles + + This is a convenience command wrapped around the Steward Tools script for generating genesis files found here; + https://github.com/sovrin-foundation/steward-tools/tree/master/create_genesis + + The script is downloaded and hosted in a running container which has the required packages installed. + + Examples of the csv files can be downloaded from here; + https://docs.google.com/spreadsheets/d/1LDduIeZp7pansd9deXeVSqGgdf0VdAHNMc7xYli3QAY/edit#gid=0 + Download each sheet separately in csv format and fill them out with the data specific to your network. + + The input csv files must be placed into the ./tmp/ folder. + The resulting output 'pool_transactions_genesis' and 'domain_transactions_genesis' files will be placed + in the ./tmp/ folder. + + Example: + $0 generategenesisfiles "./tmp/CANdy Beta Genesis File Node info - Trustees.csv" "./tmp/CANdy Beta Genesis File Node info - Stewards.csv" + indy-cli - Run Indy-Cli commands in a Indy-Cli container environment. $0 indy-cli -h @@ -77,7 +121,6 @@ usage () { backup [description] - Backup the current von-network environment. Creates a set of tar.gz archives of each of the environment's volumes. Backup sets are stored in a ./backup/date/time folder structure. - Examples: $0 backup $0 backup "The description of my environment's current state." @@ -93,6 +136,26 @@ usage () { - Restore the backup set containing the von_client-data_2021-08-23_08-21-08.tar archive. $0 restore 2021-08-23 - Restore the most recent backup set from 2021-08-23. + + restoreArchive [tarOptions]- Restore a tar.gz archive to a named volume. + Useful for restoring an archive for inspection and debugging purposes. + Examples: + $0 restoreArchive ./backup/Node3-ledger-backup.tar.gz node3-bcovrin-tes + $0 restoreArchive ./backup/Node3-ledger-backup.tar.gz node3-bcovrin-test --strip=1 + - Restore the archive to the named volume, stripping the first level directory from the archive. + - Useful in the scenario where the archive contains additional directory levels that aren’t needed in the restored copy. + + debugVolume [volumeMountFolder] - Mount a named volume into a 'debug' instance of the 'von-network-base' image with an interactive shell. + Provides a containerized environment to perform analysis on the ledger databases and files. + Starting with 'bcgovimages/von-image:node-1.12-4' the base image for von-network conatins the RocksDB sst_dump tool that can be used to verify + and inspect the RocksDB database files; '*.sst' files. + For example the command 'find /debug_volume/ -name "*.sst" | xargs -I {} sst_dump --file={} --command=verify' can be used to do a quick + verification on all the database files once the container starts. + Usage information for sst_dump can be found here; https://github.com/facebook/rocksdb/wiki/Administration-and-Data-Access-Tool + Examples: + $0 debugVolume node3-bcovrin-test + $0 debugVolume node1-bcovrin-test /home/indy/ledger + - Mount the named volume to '/home/indy/ledger' EOF exit 1 } @@ -268,8 +331,8 @@ function runCliCommand() { cliUsage fi - cmd="${terminalEmu} docker-compose \ - --log-level ERROR run " + cmd="${terminalEmu} ${dockerCompose} \ + run " if [ -z "${VOLUMES}" ] && [ -d "${DEFAULT_CLI_SCRIPT_DIR}" ] ; then VOLUMES=$(realpath ${DEFAULT_CLI_SCRIPT_DIR}) @@ -285,14 +348,14 @@ function runCliCommand() { # When running on Windows, you need to prefix the path with an extra '/' path="/${path}" fi - cmd+=" --volume='${path}:/home/indy/${mountPoint}:Z'" + cmd+=" -v '${path}:/home/indy/${mountPoint}:Z'" done fi # Need to escape quotes and commas so they don't get removed along the way ... escapedArgs=$(echo $@ | sed "s~'~\\\'~g" | sed 's~\"~\\"~g') - # Quote the escaped args so docker-compose does not try to perform any processing on them ... + # Quote the escaped args so docker compose does not try to perform any processing on them ... # Separate the command and the args so they don't get treated as one argument by the scripts in the container ... cmd+=" --rm client \ @@ -321,9 +384,9 @@ function logs() { log_args=() (( no_tail != 1 )) && log_args+=( '-f' ) if [ ! -z "${TAIL_LOGS}" ] || [ ! -z "${_force}" ]; then - docker-compose \ - --log-level ERROR logs \ - "${log_args[@]}" "$@" + ${dockerCompose} \ + logs \ + "${log_args[@]}" "$@" fi ) } @@ -414,11 +477,56 @@ function generateSecrets() { echo } +function generateDid() { + seed=${1} + if [ -z ${seed} ]; then + seed=$(generateSeed) + fi + runCliCommand python cli-scripts/generate_did.py --seed ${seed} +} + +function generateGenesisFiles() { + trustee_csv="${1}" + steward_csv="${2}" + genesis_from_files_filename="genesis_from_files.py" + genesis_from_files_url="https://raw.githubusercontent.com/sovrin-foundation/steward-tools/master/create_genesis/genesis_from_files.py" + + if [ -z "${trustee_csv}" ] || [ -z "${steward_csv}" ]; then + echoYellow "You must supply both the trustee and steward csv files." + exit 1 + fi + + if [[ "${trustee_csv}" != ./tmp/* ]]; then + trustee_csv="./tmp/${trustee_csv}" + fi + + if [ ! -f "${trustee_csv}" ]; then + echoYellow "${trustee_csv} not found, please make sure you placed ${trustee_csv} in the ./tmp folder." + exit 1 + fi + + if [[ "${steward_csv}" != ./tmp/* ]]; then + steward_csv="./tmp/${steward_csv}" + fi + + if [ ! -f "${steward_csv}" ]; then + echoYellow "${steward_csv} not found, please make sure you placed ${steward_csv} in the ./tmp folder." + exit 1 + fi + + echo "Downloading the latest version of ${genesis_from_files_filename} from ${genesis_from_files_url} ..." + curl -s -L -o ./cli-scripts/${genesis_from_files_filename} ${genesis_from_files_url} + + # Escape spaces in path ... + trustee_csv_esc=$(echo ${trustee_csv##*/} | sed 's/ /\\ /g') + steward_csv_esc=$(echo ${steward_csv##*/} | sed 's/ /\\ /g') + runCliCommand cli-scripts/${genesis_from_files_filename} --pool /tmp/pool_transactions --domain /tmp/domain_transactions --trustees /tmp/${trustee_csv_esc} --stewards /tmp/${steward_csv_esc} +} + function backup() { ( _msg=$@ - - volumes=$(docker -l ERROR compose config --volumes) + volumes=$(${dockerCompose} config --volumes) timeStamp=`date +\%Y-\%m-\%d_%H-%M-%S` datePart=${timeStamp%%_*} timePart=${timeStamp#*_} @@ -430,6 +538,7 @@ function backup() { fi for volume in ${volumes}; do + volume=$(echo ${volume} |sed 's~\r$~~') sourceVolume=${COMPOSE_PROJECT_NAME}_${volume} archiveName=${sourceVolume}_${timeStamp}.tar.gz archivePath="/${backupDir}/${archiveName}" @@ -450,30 +559,19 @@ function restore() { _fileName=${1} archivePath=$(findBackup ${1}) archiveDirectory=${archivePath%/*} - datePart=$(echo ${archivePath} | awk -F_ '{print $3}') timePart=$(echo ${archivePath} | awk -F_ '{print $4}') archiveSuffix="${datePart}_${timePart}" - timePart=${timePart%%.*} - backupVolumeMount=$(getVolumeMount ./${ROOT_BACKUP_DIR})/ if promptForConfirmation "You are about to restore from the '${archiveDirectory}' backup set.\nYour existing data will be lost if not backed up first."; then - - deleteVolumes - volumes=$(docker -l ERROR compose config --volumes) + volumes=$(${dockerCompose} config --volumes) for volume in ${volumes}; do + volume=$(echo ${volume} |sed 's~\r$~~') targetVolume=${COMPOSE_PROJECT_NAME}_${volume} archiveName=${targetVolume}_${archiveSuffix} - archivePath="/${archiveDirectory}/${archiveName}" - - echoYellow \\n"Restoring ${targetVolume} from ${archivePath} ..." - docker run \ - --rm \ - --name von-network-restore \ - --user root \ - -v ${backupVolumeMount}:/${ROOT_BACKUP_DIR} \ - -v ${targetVolume}:/target_volume von-network-base \ - tar --same-owner -xzvpf ${archivePath} -C /target_volume/ + archivePath="${archiveDirectory}/${archiveName}" + + restoreArchive -q "${archivePath}" "${targetVolume}" done else echo -e \\n"Restore aborted." @@ -481,6 +579,73 @@ function restore() { ) } +function restoreArchive() +{ + ( + local OPTIND + local quiet + unset quiet + while getopts q FLAG; do + case $FLAG in + q ) quiet=1 ;; + esac + done + shift $((OPTIND-1)) + + archive=${1} + volume=${2} + tarOptions=${3} # Example "--strip=1", to remove the first directory level. + if [ -z ${archive} ] || [ -z ${volume} ]; then + echoYellow "You must supply the path to the archive and the name of the volume to which the archive will be restored." + exit 1 + fi + + archiveFolder=${archive%/*} + archiveName=${archive##*/} + archiveToRestore=/${ROOT_BACKUP_DIR}/${archiveName} + archiveVolumeMount=$(getVolumeMount ${archiveFolder}) + + if [ ! -z "${quiet}" ] || promptForConfirmation "You are about to restore '${archive}' to ${volume}.\nYour existing data will be lost if not backed up first." ; then + deleteVolume ${volume} + echoYellow \\n"Restoring ${volume} from ${archive} ..." + docker run \ + --rm \ + --name von-network-restore \ + --user root \ + -v ${archiveVolumeMount}:/${ROOT_BACKUP_DIR} \ + -v ${volume}:/target_volume von-network-base \ + tar --same-owner -xzvpf ${archiveToRestore} -C /target_volume/ ${tarOptions} + else + echo -e \\n"Restore aborted." + fi + ) +} + +function debugVolume() +{ + ( + volume=${1} + volumeMountFolder=${2:-/debug_volume} + if [ -z ${volume} ]; then + echoYellow "You must supply the name of the volume to attach to the debug session." + exit 1 + fi + + backupVolumeMount=$(getVolumeMount ./${ROOT_BACKUP_DIR})/ + + echo -e "\nOpening a debug session with the followig volume mounts:\n - '${volume}':'${volumeMountFolder}'\n - '${backupVolumeMount}':'/${ROOT_BACKUP_DIR}'\n" + docker run \ + --rm \ + -it \ + --network="host" \ + --user root \ + -v ${backupVolumeMount}:/${ROOT_BACKUP_DIR} \ + -v ${volume}:${volumeMountFolder} \ + --entrypoint ${SHELL_CMD} \ + von-network-base + ) +} + function findBackup(){ ( _fileName=${1} @@ -519,12 +684,27 @@ function promptForConfirmation(){ ) } +function deleteVolume() { + ( + volume=${1} + + echoYellow \\n"Deleting volume '${volume}' ..." + containerId=$(docker volume rm ${volume} 2>&1 >/dev/null | sed -e 's~.*\[\(.*\)\]~\1~' | grep -v ${volume}) + if [ ! -z "${containerId}" ]; then + # The volume is in use by a container. Remove the container before deleting the volume. + docker stop ${containerId} > /dev/null 2>&1 + docker rm ${containerId} > /dev/null 2>&1 + docker volume rm ${volume} > /dev/null 2>&1 + fi + ) +} + function deleteVolumes() { ( _projectName=${COMPOSE_PROJECT_NAME:-docker} echoYellow \\n"Stopping and removing any running containers ..." - docker -l ERROR compose down -v + ${dockerCompose} down -v _pattern="^${_projectName}_\|^docker_" _volumes=$(docker volume ls -q | grep ${_pattern}) @@ -555,8 +735,8 @@ case "${COMMAND}" in start|up) initEnv "$@" install_taa - docker-compose \ - --log-level ERROR up \ + ${dockerCompose} \ + up \ -d webserver node1 node2 node3 node4 wait_for_ledger logs @@ -565,8 +745,8 @@ case "${COMMAND}" in start-combined) initEnv "$@" install_taa - docker-compose \ - --log-level ERROR up \ + ${dockerCompose} \ + up \ -d webserver nodes wait_for_ledger logs @@ -576,16 +756,16 @@ case "${COMMAND}" in if [ -z "$LEDGER_SEED" ]; then export ANONYMOUS=1 fi - docker-compose \ - --log-level ERROR up \ + ${dockerCompose} \ + up \ -d webserver wait_for_ledger logs webserver ;; synctest) initEnv "$@" - docker-compose \ - --log-level ERROR up \ + ${dockerCompose} \ + up \ -d synctest node1 node2 node3 node4 logs -f synctest ;; @@ -601,8 +781,8 @@ case "${COMMAND}" in ;; stop) initEnv "$@" - docker-compose \ - --log-level ERROR stop + ${dockerCompose} \ + stop remove_taa ;; down|rm) @@ -614,7 +794,7 @@ case "${COMMAND}" in docker build $(initDockerBuildArgs) -t von-network-base . ;; rebuild) - docker build --no-cache $(initDockerBuildArgs) -t von-network-base . + docker build --no-cache --progress plain $(initDockerBuildArgs) -t von-network-base . ;; dockerhost) echo -e \\n"DockerHost: ${DOCKERHOST}"\\n @@ -622,6 +802,14 @@ case "${COMMAND}" in generatesecrets) generateSecrets ;; + generatedid) + generateDid $@ + ;; + generategenesisfiles) + trustee_csv="${1}" + steward_csv="${2}" + generateGenesisFiles "${trustee_csv}" "${steward_csv}" + ;; backup) backup "$@" @@ -629,6 +817,17 @@ case "${COMMAND}" in restore) restore $@ ;; + restorearchive) + archive=${1} + volume=${2} + tarOptions=${3} + restoreArchive ${archive} ${volume} ${tarOptions} + ;; + debugvolume) + volume=${1} + volumeMountFolder=${2} + debugVolume ${volume} ${volumeMountFolder} + ;; *) usage;; esac diff --git a/server/anchor.py b/server/anchor.py index 375954e..2f1e198 100644 --- a/server/anchor.py +++ b/server/anchor.py @@ -103,7 +103,22 @@ async def _fetch_url(the_url): async def _fetch_genesis_txn(genesis_url: str, target_path: str) -> bool: - (r_status, data) = await _fetch_url(genesis_url) + + # Check if the protocol is http or file + if "http" in genesis_url: + (r_status, data) = await _fetch_url(genesis_url) + elif "file" in genesis_url: + # get the path to the file. + # substring of genesis_url without file:// + genesis_url_path = genesis_url[7:] + # print the path to file + LOGGER.info("Genesis file local path: %s", genesis_url_path) + # read file to data variable + data = open(genesis_url_path,"r").read() + # neither of supported protocols + else: + raise AnchorException("Not used supported protocols https(s) or file for Genesis transaction file url") + # check data is valid json lines = data.splitlines()