From ae234bf69d5d8b8aafde5d8affa50952f9d0f268 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 25 Jul 2024 21:21:06 +0200 Subject: [PATCH 01/48] Initial commit for Typer --- cli.py | 384 +++++++++++++++++++++++++++++++++++++++++++++++++ src/utils.py | 4 + src/wallets.py | 142 ++++++++++++++++++ 3 files changed, 530 insertions(+) create mode 100644 cli.py create mode 100644 src/utils.py create mode 100644 src/wallets.py diff --git a/cli.py b/cli.py new file mode 100644 index 00000000..aa949ce9 --- /dev/null +++ b/cli.py @@ -0,0 +1,384 @@ +import asyncio + +import rich +import typer +from typing import Optional + +from src import wallets + + +# re-usable args +class Options: + wallet_name = typer.Option(None, help="Name of wallet") + wallet_path = typer.Option(None, help="Filepath of wallet") + wallet_hotkey = typer.Option(None, help="Hotkey of wallet") + mnemonic = typer.Option( + None, help="Mnemonic used to regen your key i.e. horse cart dog ..." + ) + seed = typer.Option( + None, help="Seed hex string used to regen your key i.e. 0x1234..." + ) + json = typer.Option( + None, + help="Path to a json file containing the encrypted key backup. (e.g. from PolkadotJS)", + ) + json_password = typer.Option(None, help="Password to decrypt the json file.") + use_password = typer.Option( + False, + help="Set true to protect the generated bittensor key with a password.", + ) + public_hex_key = typer.Option(None, help="The public key in hex format.") + ss58_address = typer.Option(None, help="The SS58 address of the coldkey") + overwrite_coldkey = typer.Option( + False, help="Overwrite the old coldkey with the newly generated coldkey" + ) + overwrite_hotkey = typer.Option( + False, help="Overwrite the old hotkey with the newly generated hotkey" + ) + + +class NotSubtensor: + def __init__(self, network: str, chain: str): + self.network = network + self.chain = chain + + def __str__(self): + return f"NotSubtensor(network={self.network}, chain={self.chain})" + + +def btwallet(wallet_name: str, wallet_path: str, wallet_hotkey: Optional[str] = None): + return "Wallet" + + +def get_n_words(n_words: Optional[int]) -> int: + while n_words not in [12, 15, 18, 21, 24]: + n_words = typer.prompt( + "Choose number of words: 12, 15, 18, 21, 24", type=int, default=12 + ) + return n_words + + +class CLIManager: + def __init__(self): + self.app = typer.Typer() + self.wallet_app = typer.Typer() + self.delegates_app = typer.Typer() + + # wallet aliases + self.app.add_typer(self.wallet_app, name="wallet") + self.app.add_typer(self.wallet_app, name="w", hidden=True) + self.app.add_typer(self.wallet_app, name="wallets", hidden=True) + + # delegates aliases + self.app.add_typer(self.delegates_app, name="delegates") + self.app.add_typer(self.delegates_app, name="d", hidden=True) + + self.wallet_app.command("")(self.wallet_ask) + self.wallet_app.command("list")(self.wallet_list) + self.wallet_app.command("regen-coldkey")(self.wallet_regen_coldkey) + self.delegates_app.command("list")(self.delegates_list) + + self.not_subtensor = None + + def initialize_chain( + self, + network: str = typer.Option("default_network", help="Network name"), + chain: str = typer.Option("default_chain", help="Chain name"), + ): + if not self.not_subtensor: + self.not_subtensor = NotSubtensor(network, chain) + typer.echo(f"Initialized with {self.not_subtensor}") + + @staticmethod + def wallet_ask(wallet_name: str, wallet_path: str, wallet_hotkey: str): + if not any([wallet_name, wallet_path, wallet_hotkey]): + wallet_name = typer.prompt("Enter wallet name:") + wallet = btwallet(wallet_name=wallet_name) + elif wallet_name: + wallet = btwallet(wallet_name=wallet_name) + elif wallet_path: + wallet = btwallet(wallet_path=wallet_path) + elif wallet_hotkey: + wallet = btwallet(wallet_hotkey=wallet_hotkey) + else: + raise typer.BadParameter("Could not create wallet") + return wallet + + def wallet_list(self, network: str = typer.Option("local", help="Network name")): + asyncio.run(wallets.WalletListCommand.run(self.not_subtensor, network)) + + def wallet_regen_coldkey( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + mnemonic: Optional[str] = Options.mnemonic, + seed: Optional[str] = Options.seed, + json: Optional[str] = Options.json, + json_password: Optional[str] = Options.json_password, + use_password: Optional[bool] = Options.use_password, + overwrite_coldkey: Optional[bool] = Options.overwrite_coldkey, + ): + """ + Executes the ``regen_coldkey`` command to regenerate a coldkey for a wallet on the Bittensor network. + + This command is used to create a new coldkey from an existing mnemonic, seed, or JSON file. + + Usage: Users can specify a mnemonic, a seed string, or a JSON file path to regenerate a coldkey. + The command supports optional password protection for the generated key and can overwrite an existing coldkey. + + Example usage: `btcli wallet regen_coldkey --mnemonic "word1 word2 ... word12"` + + Note: This command is critical for users who need to regenerate their coldkey, possibly for recovery or security reasons. + It should be used with caution to avoid overwriting existing keys unintentionally. + """ + + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + if not mnemonic and not seed and not json: + prompt_answer = typer.prompt("Enter mnemonic, seed, or json file location") + if prompt_answer.startswith("0x"): + seed = prompt_answer + elif len(prompt_answer.split(" ")) > 1: + mnemonic = prompt_answer + else: + json = prompt_answer + if json and not json_password: + json_password = typer.prompt("Enter json backup password", hide_input=True) + asyncio.run( + wallets.RegenColdkeyCommand.run( + wallet, + mnemonic, + seed, + json, + json_password, + use_password, + overwrite_coldkey, + ) + ) + + def wallet_regen_coldkey_pub( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + public_key_hex: Optional[str] = Options.public_hex_key, + ss58_address: Optional[str] = Options.ss58_address, + overwrite_coldkeypub: Optional[bool] = typer.Option( + False, help="Overwrites the existing coldkeypub file with the new one." + ), + ): + """ + Executes the ``regen_coldkeypub`` command to regenerate the public part of a coldkey (coldkeypub) for a wallet on the Bittensor network. + + This command is used when a user needs to recreate their coldkeypub from an existing public key or SS58 address. + + Usage: + The command requires either a public key in hexadecimal format or an ``SS58`` address to regenerate the coldkeypub. It optionally allows overwriting an existing coldkeypub file. + + Example usage:: + + btcli wallet regen_coldkeypub --ss58_address 5DkQ4... + + Note: + This command is particularly useful for users who need to regenerate their coldkeypub, perhaps due to file corruption or loss. + It is a recovery-focused utility that ensures continued access to wallet functionalities. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + if not ss58_address and not public_key_hex: + prompt_answer = typer.prompt( + "Enter the ss58_address or the public key in hex" + ) + if prompt_answer.startswith("0x"): + public_key_hex = prompt_answer + else: + ss58_address = prompt_answer + if not bittensor.utils.is_valid_bittensor_address_or_public_key( + address=ss58_address if ss58_address else public_key_hex + ): + rich.print("[red]Error: Invalid SS58 address or public key![/red]") + raise typer.Exit() + asyncio.run( + wallets.RegenColdkeypubCommand.run( + wallet, public_key_hex, ss58_address, overwrite_coldkeypub + ) + ) + + def wallet_regen_hotkey( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + mnemonic: Optional[str] = Options.mnemonic, + seed: Optional[str] = Options.seed, + json: Optional[str] = Options.json, + json_password: Optional[str] = Options.json_password, + use_password: Optional[bool] = Options.use_password, + overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, + ): + """ + Executes the ``regen_hotkey`` command to regenerate a hotkey for a wallet on the Bittensor network. + + Similar to regenerating a coldkey, this command creates a new hotkey from a mnemonic, seed, or JSON file. + + Usage: + Users can provide a mnemonic, seed string, or a JSON file to regenerate the hotkey. + The command supports optional password protection and can overwrite an existing hotkey. + + Example usage:: + + btcli wallet regen_hotkey + btcli wallet regen_hotkey --seed 0x1234... + + Note: + This command is essential for users who need to regenerate their hotkey, possibly for security upgrades or key recovery. + It should be used cautiously to avoid accidental overwrites of existing keys. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + if not wallet_hotkey: # TODO no prompt + # TODO add to wallet object + wallet_hotkey = typer.prompt( + "Enter hotkey name", default=defaults.wallet.hotkey + ) + if not mnemonic and not seed and not json: + prompt_answer = typer.prompt("Enter mnemonic, seed, or json file location") + if prompt_answer.startswith("0x"): + seed = prompt_answer + elif len(prompt_answer.split(" ")) > 1: + mnemonic = prompt_answer + else: + json = prompt_answer + if json and not json_password: + json_password = typer.prompt("Enter json backup password", hide_input=True) + asyncio.run( + wallets.RegenHotkeyCommand.run( + wallet, + mnemonic, + seed, + json, + json_password, + use_password, + overwrite_hotkey, + ) + ) + + def wallet_new_hotkey( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + n_words: Optional[int] = None, + use_password: Optional[bool] = Options.use_password, + overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, + ): + """ + Executes the ``new_hotkey`` command to create a new hotkey under a wallet on the Bittensor network. + + This command is used to generate a new hotkey for managing a neuron or participating in the network. + + Usage: + The command creates a new hotkey with an optional word count for the mnemonic and supports password protection. + It also allows overwriting an existing hotkey. + + Example usage:: + + btcli wallet new_hotkey --n_words 24 + + Note: + This command is useful for users who wish to create additional hotkeys for different purposes, + such as running multiple miners or separating operational roles within the network. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + if not wallet_hotkey: # TODO no prompt + # TODO add to wallet object + wallet_hotkey = typer.prompt( + "Enter hotkey name", default=defaults.wallet.hotkey + ) + n_words = get_n_words(n_words) + asyncio.run( + wallets.NewHotkeyCommand.run(wallet, use_password, overwrite_hotkey) + ) + + def wallet_new_coldkey( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + n_words: Optional[int] = None, + use_password: Optional[bool] = Options.use_password, + overwrite_coldkey: Optional[bool] = typer.Option(), + ): + """ + Executes the ``new_coldkey`` command to create a new coldkey under a wallet on the Bittensor network. + + This command generates a coldkey, which is essential for holding balances and performing high-value transactions. + + Usage: + The command creates a new coldkey with an optional word count for the mnemonic and supports password protection. + It also allows overwriting an existing coldkey. + + Example usage:: + + btcli wallet new_coldkey --n_words 15 + + Note: + This command is crucial for users who need to create a new coldkey for enhanced security or as part of setting up a new wallet. + It's a foundational step in establishing a secure presence on the Bittensor network. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + n_words = get_n_words(n_words) + asyncio.run( + wallets.NewColdkeyCommand.run( + wallet, n_words, use_password, overwrite_coldkey + ) + ) + + def wallet_create_wallet( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + n_words: Optional[int] = None, + use_password: Optional[bool] = Options.use_password, + overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, + overwrite_coldkey: Optional[bool] = Options.overwrite_coldkey, + ): + """ + Executes the ``create`` command to generate both a new coldkey and hotkey under a specified wallet on the Bittensor network. + + This command is a comprehensive utility for creating a complete wallet setup with both cold and hotkeys. + + Usage: + The command facilitates the creation of a new coldkey and hotkey with an optional word count for the mnemonics. + It supports password protection for the coldkey and allows overwriting of existing keys. + + Example usage:: + + btcli wallet create --n_words 21 + + Note: + This command is ideal for new users setting up their wallet for the first time or for those who wish to completely renew their wallet keys. + It ensures a fresh start with new keys for secure and effective participation in the network. + """ + n_words = get_n_words(n_words) + if not wallet_hotkey: # TODO no prompt + # TODO add to wallet object + wallet_hotkey = typer.prompt( + "Enter hotkey name", default=defaults.wallet.hotkey + ) + + def delegates_list( + self, + wallet_name: Optional[str] = typer.Option(None, help="Wallet name"), + network: str = typer.Option("test", help="Network name"), + ): + if not wallet_name: + wallet_name = typer.prompt("Please enter the wallet name") + asyncio.run(delegates.ListDelegatesCommand.run(wallet_name, network)) + + def run(self): + self.app() + + +if __name__ == "__main__": + manager = CLIManager() + manager.run() diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 00000000..0104cd75 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,4 @@ +from rich.console import Console + +console = Console() +err_console = Console(stderr=True) diff --git a/src/wallets.py b/src/wallets.py new file mode 100644 index 00000000..8d4c1362 --- /dev/null +++ b/src/wallets.py @@ -0,0 +1,142 @@ +# The MIT License (MIT) +# Copyright © 2021 Yuma Rao + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +# import argparse + +# import btwallet +import os + +import typer + +# from rich.table import Table +from typing import Optional + +from .utils import err_console +# from . import defaults +# import requests +# from ..utils import RAOPERTAO + + +class RegenColdkeyCommand: + @staticmethod + async def run( + wallet, + mnemonic: Optional[str], + seed: Optional[str] = None, + json_path: Optional[str] = None, + json_password: Optional[str] = "", + use_password: Optional[bool] = True, + overwrite_coldkey: Optional[bool] = False, + ): + wallet = btwallet(wallet) + + json_str: Optional[str] = None + if json_path: + if not os.path.exists(json_path) or not os.path.isfile(json_path): + raise ValueError("File {} does not exist".format(json_path)) + with open(json_path, "r") as f: + json_str = f.read() + wallet.regenerate_coldkey( + mnemonic=mnemonic, + seed=seed, + json=(json_str, json_password), + use_password=use_password, + overwrite=overwrite_coldkey, + ) + + +class RegenColdkeypubCommand: + @staticmethod + async def run( + wallet, ss58_address: str, public_key_hex: str, overwrite_coldkeypub: bool + ): + r"""Creates a new coldkeypub under this wallet.""" + wallet = btwallet(wallet) + wallet.regenerate_coldkeypub( + ss58_address=ss58_address, + public_key=public_key_hex, + overwrite=overwrite_coldkeypub, + ) + + +class RegenHotkeyCommand: + @staticmethod + async def run( + wallet: "btwallet", + mnemonic: Optional[str], + seed: Optional[str], + json_path: Optional[str], + json_password: Optional[str] = "", + use_password: Optional[bool] = True, + overwrite_hotkey: Optional[bool] = False, + ): + json_str: Optional[str] = None + if json_path: + if not os.path.exists(json_path) or not os.path.isfile(json_path): + err_console.print(f"File {json_path} does not exist") + raise typer.Exit() + with open(json_path, "r") as f: + json_str = f.read() + + wallet.regenerate_hotkey( + mnemonic=mnemonic, + seed=seed, + json=(json_str, json_password), + use_password=use_password, + overwrite=overwrite_hotkey, + ) + + +class NewHotkeyCommand: + @staticmethod + async def run(wallet, n_words: int, use_password: bool, overwrite_hotkey: bool): + wallet.create_new_hotkey( + n_words=n_words, + use_password=use_password, + overwrite=overwrite_hotkey, + ) + + +class NewColdkeyCommand: + @staticmethod + async def run(wallet, n_words: int, use_password: bool, overwrite_coldkey: bool): + wallet.create_new_coldkey( + n_words=n_words, + use_password=use_password, + overwrite=overwrite_coldkey, + ) + + +class WalletCreateCommand: + @staticmethod + def run( + wallet, + n_words: int = 12, + use_password: bool = True, + overwrite_coldkey: bool = False, + overwrite_hotkey: bool = False, + ): + wallet.create_new_coldkey( + n_words=n_words, + use_password=use_password, + overwrite=overwrite_coldkey, + ) + wallet.create_new_hotkey( + n_words=n_words, + use_password=False, + overwrite=overwrite_hotkey, + ) From 4082ecff950e9e14935219816f45af8510a94da7 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 25 Jul 2024 21:36:59 +0200 Subject: [PATCH 02/48] Removed wallet overwrite --- src/wallets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/wallets.py b/src/wallets.py index 8d4c1362..ba0f0abd 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -42,8 +42,6 @@ async def run( use_password: Optional[bool] = True, overwrite_coldkey: Optional[bool] = False, ): - wallet = btwallet(wallet) - json_str: Optional[str] = None if json_path: if not os.path.exists(json_path) or not os.path.isfile(json_path): @@ -65,7 +63,6 @@ async def run( wallet, ss58_address: str, public_key_hex: str, overwrite_coldkeypub: bool ): r"""Creates a new coldkeypub under this wallet.""" - wallet = btwallet(wallet) wallet.regenerate_coldkeypub( ss58_address=ss58_address, public_key=public_key_hex, @@ -140,3 +137,4 @@ def run( use_password=False, overwrite=overwrite_hotkey, ) + From e22ff368854d74ac74f3b8294fec6ace03546e81 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 25 Jul 2024 21:44:59 +0200 Subject: [PATCH 03/48] Created mock Wallet object to develop, based on Roman's Wallet --- cli.py | 20 ++++++++++++++------ requirements.txt | 1 + src/wallets.py | 10 +++++----- 3 files changed, 20 insertions(+), 11 deletions(-) create mode 100644 requirements.txt diff --git a/cli.py b/cli.py index aa949ce9..7f437c01 100644 --- a/cli.py +++ b/cli.py @@ -46,8 +46,15 @@ def __str__(self): return f"NotSubtensor(network={self.network}, chain={self.chain})" -def btwallet(wallet_name: str, wallet_path: str, wallet_hotkey: Optional[str] = None): - return "Wallet" +class Wallet: + def __init__( + self, + name: Optional[str] = None, + hotkey: Optional[str] = None, + path: Optional[str] = None, + config: Optional["Config"] = None + ): + pass def get_n_words(n_words: Optional[int]) -> int: @@ -93,13 +100,14 @@ def initialize_chain( def wallet_ask(wallet_name: str, wallet_path: str, wallet_hotkey: str): if not any([wallet_name, wallet_path, wallet_hotkey]): wallet_name = typer.prompt("Enter wallet name:") - wallet = btwallet(wallet_name=wallet_name) + wallet = Wallet(name=wallet_name) elif wallet_name: - wallet = btwallet(wallet_name=wallet_name) + wallet = Wallet(name=wallet_name) elif wallet_path: - wallet = btwallet(wallet_path=wallet_path) + wallet = Wallet(path=wallet_path) elif wallet_hotkey: - wallet = btwallet(wallet_hotkey=wallet_hotkey) + wallet = Wallet(hotkey=wallet_hotkey) + # TODO Wallet(config) else: raise typer.BadParameter("Could not create wallet") return wallet diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7379290e --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +typer \ No newline at end of file diff --git a/src/wallets.py b/src/wallets.py index ba0f0abd..e6cf9fec 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -17,7 +17,7 @@ # import argparse -# import btwallet +# form btwallet import Wallet import os import typer @@ -73,7 +73,7 @@ async def run( class RegenHotkeyCommand: @staticmethod async def run( - wallet: "btwallet", + wallet: "Wallet", mnemonic: Optional[str], seed: Optional[str], json_path: Optional[str], @@ -100,7 +100,7 @@ async def run( class NewHotkeyCommand: @staticmethod - async def run(wallet, n_words: int, use_password: bool, overwrite_hotkey: bool): + async def run(wallet: "Wallet", n_words: int, use_password: bool, overwrite_hotkey: bool): wallet.create_new_hotkey( n_words=n_words, use_password=use_password, @@ -110,7 +110,7 @@ async def run(wallet, n_words: int, use_password: bool, overwrite_hotkey: bool): class NewColdkeyCommand: @staticmethod - async def run(wallet, n_words: int, use_password: bool, overwrite_coldkey: bool): + async def run(wallet: "Wallet", n_words: int, use_password: bool, overwrite_coldkey: bool): wallet.create_new_coldkey( n_words=n_words, use_password=use_password, @@ -121,7 +121,7 @@ async def run(wallet, n_words: int, use_password: bool, overwrite_coldkey: bool) class WalletCreateCommand: @staticmethod def run( - wallet, + wallet: "Wallet", n_words: int = 12, use_password: bool = True, overwrite_coldkey: bool = False, From 99c104c9af828c28a8201acccfc1b9ca281f8362 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 25 Jul 2024 21:45:42 +0200 Subject: [PATCH 04/48] ruff --- cli.py | 10 +++++----- src/wallets.py | 9 ++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cli.py b/cli.py index 7f437c01..97d6bbbd 100644 --- a/cli.py +++ b/cli.py @@ -48,11 +48,11 @@ def __str__(self): class Wallet: def __init__( - self, - name: Optional[str] = None, - hotkey: Optional[str] = None, - path: Optional[str] = None, - config: Optional["Config"] = None + self, + name: Optional[str] = None, + hotkey: Optional[str] = None, + path: Optional[str] = None, + config: Optional["Config"] = None, ): pass diff --git a/src/wallets.py b/src/wallets.py index e6cf9fec..b949c67b 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -100,7 +100,9 @@ async def run( class NewHotkeyCommand: @staticmethod - async def run(wallet: "Wallet", n_words: int, use_password: bool, overwrite_hotkey: bool): + async def run( + wallet: "Wallet", n_words: int, use_password: bool, overwrite_hotkey: bool + ): wallet.create_new_hotkey( n_words=n_words, use_password=use_password, @@ -110,7 +112,9 @@ async def run(wallet: "Wallet", n_words: int, use_password: bool, overwrite_hotk class NewColdkeyCommand: @staticmethod - async def run(wallet: "Wallet", n_words: int, use_password: bool, overwrite_coldkey: bool): + async def run( + wallet: "Wallet", n_words: int, use_password: bool, overwrite_coldkey: bool + ): wallet.create_new_coldkey( n_words=n_words, use_password=use_password, @@ -137,4 +141,3 @@ def run( use_password=False, overwrite=overwrite_hotkey, ) - From 1f13cdb84cb5b3b26b9ef0e5c07d7386e4273d7d Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 25 Jul 2024 23:10:39 +0200 Subject: [PATCH 05/48] Added bittensor-wallet from repo --- cli.py | 14 ++------------ requirements.txt | 3 ++- src/wallets.py | 10 +++++----- 3 files changed, 9 insertions(+), 18 deletions(-) mode change 100644 => 100755 cli.py diff --git a/cli.py b/cli.py old mode 100644 new mode 100755 index 97d6bbbd..13b31749 --- a/cli.py +++ b/cli.py @@ -1,5 +1,6 @@ +#!/usr/bin/env python3 import asyncio - +from bittensor_wallet import Wallet import rich import typer from typing import Optional @@ -46,17 +47,6 @@ def __str__(self): return f"NotSubtensor(network={self.network}, chain={self.chain})" -class Wallet: - def __init__( - self, - name: Optional[str] = None, - hotkey: Optional[str] = None, - path: Optional[str] = None, - config: Optional["Config"] = None, - ): - pass - - def get_n_words(n_words: Optional[int]) -> int: while n_words not in [12, 15, 18, 21, 24]: n_words = typer.prompt( diff --git a/requirements.txt b/requirements.txt index 7379290e..0cecd05f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -typer \ No newline at end of file +typer~=0.12 +git+https://github.com/opentensor/btwallet # bittensor_wallet \ No newline at end of file diff --git a/src/wallets.py b/src/wallets.py index b949c67b..bfe17c03 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -17,7 +17,7 @@ # import argparse -# form btwallet import Wallet +from bittensor_wallet import Wallet import os import typer @@ -73,7 +73,7 @@ async def run( class RegenHotkeyCommand: @staticmethod async def run( - wallet: "Wallet", + wallet: Wallet, mnemonic: Optional[str], seed: Optional[str], json_path: Optional[str], @@ -101,7 +101,7 @@ async def run( class NewHotkeyCommand: @staticmethod async def run( - wallet: "Wallet", n_words: int, use_password: bool, overwrite_hotkey: bool + wallet: Wallet, n_words: int, use_password: bool, overwrite_hotkey: bool ): wallet.create_new_hotkey( n_words=n_words, @@ -113,7 +113,7 @@ async def run( class NewColdkeyCommand: @staticmethod async def run( - wallet: "Wallet", n_words: int, use_password: bool, overwrite_coldkey: bool + wallet: Wallet, n_words: int, use_password: bool, overwrite_coldkey: bool ): wallet.create_new_coldkey( n_words=n_words, @@ -125,7 +125,7 @@ async def run( class WalletCreateCommand: @staticmethod def run( - wallet: "Wallet", + wallet: Wallet, n_words: int = 12, use_password: bool = True, overwrite_coldkey: bool = False, From 671feb0b0632d287883978a5ee23daed2a63342a Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 16:08:57 +0200 Subject: [PATCH 06/48] Added defaults, utils; consolidated some reused code. --- cli.py | 97 +++++++++------------- requirements.txt | 1 + src/__init__.py | 67 +++++++++++++++ src/utils.py | 79 ++++++++++++++++++ src/wallets.py | 208 ++++++++++++++++++++++------------------------- 5 files changed, 284 insertions(+), 168 deletions(-) create mode 100644 src/__init__.py diff --git a/cli.py b/cli.py index 13b31749..d484b221 100755 --- a/cli.py +++ b/cli.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 import asyncio +from typing import Optional + from bittensor_wallet import Wallet import rich import typer -from typing import Optional -from src import wallets +from src import wallets, defaults, utils # re-usable args @@ -13,6 +14,7 @@ class Options: wallet_name = typer.Option(None, help="Name of wallet") wallet_path = typer.Option(None, help="Filepath of wallet") wallet_hotkey = typer.Option(None, help="Hotkey of wallet") + wallet_hk_req = typer.Option(defaults.wallet.hotkey, help="Hotkey name of wallet", prompt=True) mnemonic = typer.Option( None, help="Mnemonic used to regen your key i.e. horse cart dog ..." ) @@ -31,10 +33,12 @@ class Options: public_hex_key = typer.Option(None, help="The public key in hex format.") ss58_address = typer.Option(None, help="The SS58 address of the coldkey") overwrite_coldkey = typer.Option( - False, help="Overwrite the old coldkey with the newly generated coldkey" + False, help="Overwrite the old coldkey with the newly generated coldkey", + prompt=True, ) overwrite_hotkey = typer.Option( - False, help="Overwrite the old hotkey with the newly generated hotkey" + False, help="Overwrite the old hotkey with the newly generated hotkey", + prompt=True ) @@ -55,6 +59,20 @@ def get_n_words(n_words: Optional[int]) -> int: return n_words +def get_creation_data(mnemonic, seed, json, json_password): + if not mnemonic and not seed and not json: + prompt_answer = typer.prompt("Enter mnemonic, seed, or json file location") + if prompt_answer.startswith("0x"): + seed = prompt_answer + elif len(prompt_answer.split(" ")) > 1: + mnemonic = prompt_answer + else: + json = prompt_answer + if json and not json_password: + json_password = typer.prompt("Enter json backup password", hide_input=True) + return mnemonic, seed, json, json_password + + class CLIManager: def __init__(self): self.app = typer.Typer() @@ -70,7 +88,6 @@ def __init__(self): self.app.add_typer(self.delegates_app, name="delegates") self.app.add_typer(self.delegates_app, name="d", hidden=True) - self.wallet_app.command("")(self.wallet_ask) self.wallet_app.command("list")(self.wallet_list) self.wallet_app.command("regen-coldkey")(self.wallet_regen_coldkey) self.delegates_app.command("list")(self.delegates_list) @@ -91,15 +108,9 @@ def wallet_ask(wallet_name: str, wallet_path: str, wallet_hotkey: str): if not any([wallet_name, wallet_path, wallet_hotkey]): wallet_name = typer.prompt("Enter wallet name:") wallet = Wallet(name=wallet_name) - elif wallet_name: - wallet = Wallet(name=wallet_name) - elif wallet_path: - wallet = Wallet(path=wallet_path) - elif wallet_hotkey: - wallet = Wallet(hotkey=wallet_hotkey) - # TODO Wallet(config) else: - raise typer.BadParameter("Could not create wallet") + wallet = Wallet(wallet_name, wallet_path, wallet_hotkey) + # TODO Wallet(config) return wallet def wallet_list(self, network: str = typer.Option("local", help="Network name")): @@ -132,18 +143,9 @@ def wallet_regen_coldkey( """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - if not mnemonic and not seed and not json: - prompt_answer = typer.prompt("Enter mnemonic, seed, or json file location") - if prompt_answer.startswith("0x"): - seed = prompt_answer - elif len(prompt_answer.split(" ")) > 1: - mnemonic = prompt_answer - else: - json = prompt_answer - if json and not json_password: - json_password = typer.prompt("Enter json backup password", hide_input=True) + mnemonic, seed, json, json_password = get_creation_data(mnemonic, seed, json, json_password) asyncio.run( - wallets.RegenColdkeyCommand.run( + wallets.regen_coldkey( wallet, mnemonic, seed, @@ -162,7 +164,8 @@ def wallet_regen_coldkey_pub( public_key_hex: Optional[str] = Options.public_hex_key, ss58_address: Optional[str] = Options.ss58_address, overwrite_coldkeypub: Optional[bool] = typer.Option( - False, help="Overwrites the existing coldkeypub file with the new one." + False, help="Overwrites the existing coldkeypub file with the new one.", + prompt=True ), ): """ @@ -190,13 +193,13 @@ def wallet_regen_coldkey_pub( public_key_hex = prompt_answer else: ss58_address = prompt_answer - if not bittensor.utils.is_valid_bittensor_address_or_public_key( + if not utils.is_valid_bittensor_address_or_public_key( address=ss58_address if ss58_address else public_key_hex ): rich.print("[red]Error: Invalid SS58 address or public key![/red]") raise typer.Exit() asyncio.run( - wallets.RegenColdkeypubCommand.run( + wallets.regen_coldkey_pub( wallet, public_key_hex, ss58_address, overwrite_coldkeypub ) ) @@ -205,7 +208,7 @@ def wallet_regen_hotkey( self, wallet_name: Optional[str] = Options.wallet_name, wallet_path: Optional[str] = Options.wallet_path, - wallet_hotkey: Optional[str] = Options.wallet_hotkey, + wallet_hotkey: Optional[str] = Options.wallet_hk_req, mnemonic: Optional[str] = Options.mnemonic, seed: Optional[str] = Options.seed, json: Optional[str] = Options.json, @@ -232,23 +235,9 @@ def wallet_regen_hotkey( It should be used cautiously to avoid accidental overwrites of existing keys. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - if not wallet_hotkey: # TODO no prompt - # TODO add to wallet object - wallet_hotkey = typer.prompt( - "Enter hotkey name", default=defaults.wallet.hotkey - ) - if not mnemonic and not seed and not json: - prompt_answer = typer.prompt("Enter mnemonic, seed, or json file location") - if prompt_answer.startswith("0x"): - seed = prompt_answer - elif len(prompt_answer.split(" ")) > 1: - mnemonic = prompt_answer - else: - json = prompt_answer - if json and not json_password: - json_password = typer.prompt("Enter json backup password", hide_input=True) + mnemonic, seed, json, json_password = get_creation_data(mnemonic, seed, json, json_password) asyncio.run( - wallets.RegenHotkeyCommand.run( + wallets.regen_hotkey( wallet, mnemonic, seed, @@ -263,7 +252,7 @@ def wallet_new_hotkey( self, wallet_name: Optional[str] = Options.wallet_name, wallet_path: Optional[str] = Options.wallet_path, - wallet_hotkey: Optional[str] = Options.wallet_hotkey, + wallet_hotkey: Optional[str] = Options.wallet_hk_req, n_words: Optional[int] = None, use_password: Optional[bool] = Options.use_password, overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, @@ -286,14 +275,9 @@ def wallet_new_hotkey( such as running multiple miners or separating operational roles within the network. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - if not wallet_hotkey: # TODO no prompt - # TODO add to wallet object - wallet_hotkey = typer.prompt( - "Enter hotkey name", default=defaults.wallet.hotkey - ) n_words = get_n_words(n_words) asyncio.run( - wallets.NewHotkeyCommand.run(wallet, use_password, overwrite_hotkey) + wallets.new_hotkey(wallet, n_words, use_password, overwrite_hotkey) ) def wallet_new_coldkey( @@ -325,7 +309,7 @@ def wallet_new_coldkey( wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) n_words = get_n_words(n_words) asyncio.run( - wallets.NewColdkeyCommand.run( + wallets.new_coldkey( wallet, n_words, use_password, overwrite_coldkey ) ) @@ -334,7 +318,7 @@ def wallet_create_wallet( self, wallet_name: Optional[str] = Options.wallet_name, wallet_path: Optional[str] = Options.wallet_path, - wallet_hotkey: Optional[str] = Options.wallet_hotkey, + wallet_hotkey: Optional[str] = Options.wallet_hk_req, n_words: Optional[int] = None, use_password: Optional[bool] = Options.use_password, overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, @@ -357,12 +341,9 @@ def wallet_create_wallet( This command is ideal for new users setting up their wallet for the first time or for those who wish to completely renew their wallet keys. It ensures a fresh start with new keys for secure and effective participation in the network. """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) n_words = get_n_words(n_words) - if not wallet_hotkey: # TODO no prompt - # TODO add to wallet object - wallet_hotkey = typer.prompt( - "Enter hotkey name", default=defaults.wallet.hotkey - ) + asyncio.run(wallets.wallet_create(wallet, n_words, use_password, overwrite_coldkey, overwrite_hotkey)) def delegates_list( self, diff --git a/requirements.txt b/requirements.txt index 0cecd05f..3dc56257 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ typer~=0.12 +rich~=13.7 git+https://github.com/opentensor/btwallet # bittensor_wallet \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..6fe0ae42 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,67 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class CUDA: + dev_id: list[int] + use_cuda: bool + tpb: int + + +@dataclass +class PoWRegister: + num_processes: Optional[int] + update_interval: int + output_in_place: bool + verbose: bool + cuda: CUDA + + +@dataclass +class Wallet: + name: str + hotkey: str + path: str + + +@dataclass +class Logging: + # likely needs to be changed + debug: bool + trace: bool + record_log: bool + logging_dir: str + + +@dataclass +class Subtensor: + network: str + chain_endpoint: Optional[str] + _mock: bool + + +@dataclass +class Defaults: + netuid: int + subtensor: Subtensor + pow_register: PoWRegister + wallet: Wallet + logging: Logging + + +defaults = Defaults( + netuid=1, + subtensor=Subtensor(network="finney", chain_endpoint=None, _mock=False), + pow_register=PoWRegister( + num_processes=None, + update_interval=50000, + output_in_place=True, + verbose=False, + cuda=CUDA(dev_id=[0], use_cuda=False, tpb=256), + ), + wallet=Wallet(name="default", hotkey="default", path="~/.bittensor/wallets/"), + logging=Logging( + debug=False, trace=False, record_log=False, logging_dir="~/.bittensor/miners" + ), +) diff --git a/src/utils.py b/src/utils.py index 0104cd75..5ff2ef96 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,4 +1,83 @@ +from typing import Union + +from bittensor_wallet.keyfile import Keypair +from bittensor_wallet.utils import SS58_FORMAT, ss58 from rich.console import Console console = Console() err_console = Console(stderr=True) + + +def is_valid_ss58_address(address: str) -> bool: + """ + Checks if the given address is a valid ss58 address. + + Args: + address(str): The address to check. + + Returns: + True if the address is a valid ss58 address for Bittensor, False otherwise. + """ + try: + return ss58.is_valid_ss58_address( + address, valid_ss58_format=SS58_FORMAT + ) or ss58.is_valid_ss58_address( + address, valid_ss58_format=42 + ) # Default substrate ss58 format (legacy) + except IndexError: + return False + + +def is_valid_ed25519_pubkey(public_key: Union[str, bytes]) -> bool: + """ + Checks if the given public_key is a valid ed25519 key. + + Args: + public_key(Union[str, bytes]): The public_key to check. + + Returns: + True if the public_key is a valid ed25519 key, False otherwise. + + """ + try: + if isinstance(public_key, str): + if len(public_key) != 64 and len(public_key) != 66: + raise ValueError("a public_key should be 64 or 66 characters") + elif isinstance(public_key, bytes): + if len(public_key) != 32: + raise ValueError("a public_key should be 32 bytes") + else: + raise ValueError("public_key must be a string or bytes") + + keypair = Keypair(public_key=public_key, ss58_format=SS58_FORMAT) + + ss58_addr = keypair.ss58_address + return ss58_addr is not None + + except (ValueError, IndexError): + return False + + +def is_valid_bittensor_address_or_public_key(address: Union[str, bytes]) -> bool: + """ + Checks if the given address is a valid destination address. + + Args: + address(Union[str, bytes]): The address to check. + + Returns: + True if the address is a valid destination address, False otherwise. + """ + if isinstance(address, str): + # Check if ed25519 + if address.startswith("0x"): + return is_valid_ed25519_pubkey(address) + else: + # Assume ss58 address + return is_valid_ss58_address(address) + elif isinstance(address, bytes): + # Check if ed25519 + return is_valid_ed25519_pubkey(address) + else: + # Invalid address type + return False diff --git a/src/wallets.py b/src/wallets.py index bfe17c03..8fd43c9f 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -31,113 +31,101 @@ # from ..utils import RAOPERTAO -class RegenColdkeyCommand: - @staticmethod - async def run( - wallet, - mnemonic: Optional[str], - seed: Optional[str] = None, - json_path: Optional[str] = None, - json_password: Optional[str] = "", - use_password: Optional[bool] = True, - overwrite_coldkey: Optional[bool] = False, - ): - json_str: Optional[str] = None - if json_path: - if not os.path.exists(json_path) or not os.path.isfile(json_path): - raise ValueError("File {} does not exist".format(json_path)) - with open(json_path, "r") as f: - json_str = f.read() - wallet.regenerate_coldkey( - mnemonic=mnemonic, - seed=seed, - json=(json_str, json_password), - use_password=use_password, - overwrite=overwrite_coldkey, - ) - - -class RegenColdkeypubCommand: - @staticmethod - async def run( - wallet, ss58_address: str, public_key_hex: str, overwrite_coldkeypub: bool - ): - r"""Creates a new coldkeypub under this wallet.""" - wallet.regenerate_coldkeypub( - ss58_address=ss58_address, - public_key=public_key_hex, - overwrite=overwrite_coldkeypub, - ) - - -class RegenHotkeyCommand: - @staticmethod - async def run( - wallet: Wallet, - mnemonic: Optional[str], - seed: Optional[str], - json_path: Optional[str], - json_password: Optional[str] = "", - use_password: Optional[bool] = True, - overwrite_hotkey: Optional[bool] = False, - ): - json_str: Optional[str] = None - if json_path: - if not os.path.exists(json_path) or not os.path.isfile(json_path): - err_console.print(f"File {json_path} does not exist") - raise typer.Exit() - with open(json_path, "r") as f: - json_str = f.read() - - wallet.regenerate_hotkey( - mnemonic=mnemonic, - seed=seed, - json=(json_str, json_password), - use_password=use_password, - overwrite=overwrite_hotkey, - ) - - -class NewHotkeyCommand: - @staticmethod - async def run( - wallet: Wallet, n_words: int, use_password: bool, overwrite_hotkey: bool - ): - wallet.create_new_hotkey( - n_words=n_words, - use_password=use_password, - overwrite=overwrite_hotkey, - ) - - -class NewColdkeyCommand: - @staticmethod - async def run( - wallet: Wallet, n_words: int, use_password: bool, overwrite_coldkey: bool - ): - wallet.create_new_coldkey( - n_words=n_words, - use_password=use_password, - overwrite=overwrite_coldkey, - ) - - -class WalletCreateCommand: - @staticmethod - def run( - wallet: Wallet, - n_words: int = 12, - use_password: bool = True, - overwrite_coldkey: bool = False, - overwrite_hotkey: bool = False, - ): - wallet.create_new_coldkey( - n_words=n_words, - use_password=use_password, - overwrite=overwrite_coldkey, - ) - wallet.create_new_hotkey( - n_words=n_words, - use_password=False, - overwrite=overwrite_hotkey, - ) +async def regen_coldkey( + wallet, + mnemonic: Optional[str], + seed: Optional[str] = None, + json_path: Optional[str] = None, + json_password: Optional[str] = "", + use_password: Optional[bool] = True, + overwrite_coldkey: Optional[bool] = False, +): + json_str: Optional[str] = None + if json_path: + if not os.path.exists(json_path) or not os.path.isfile(json_path): + raise ValueError("File {} does not exist".format(json_path)) + with open(json_path, "r") as f: + json_str = f.read() + wallet.regenerate_coldkey( + mnemonic=mnemonic, + seed=seed, + json=(json_str, json_password), + use_password=use_password, + overwrite=overwrite_coldkey, + ) + + +async def regen_coldkey_pub( + wallet, ss58_address: str, public_key_hex: str, overwrite_coldkeypub: bool +): + r"""Creates a new coldkeypub under this wallet.""" + wallet.regenerate_coldkeypub( + ss58_address=ss58_address, + public_key=public_key_hex, + overwrite=overwrite_coldkeypub, + ) + + +async def regen_hotkey( + wallet: Wallet, + mnemonic: Optional[str], + seed: Optional[str], + json_path: Optional[str], + json_password: Optional[str] = "", + use_password: Optional[bool] = True, + overwrite_hotkey: Optional[bool] = False, +): + json_str: Optional[str] = None + if json_path: + if not os.path.exists(json_path) or not os.path.isfile(json_path): + err_console.print(f"File {json_path} does not exist") + raise typer.Exit() + with open(json_path, "r") as f: + json_str = f.read() + + wallet.regenerate_hotkey( + mnemonic=mnemonic, + seed=seed, + json=(json_str, json_password), + use_password=use_password, + overwrite=overwrite_hotkey, + ) + + +async def new_hotkey( + wallet: Wallet, n_words: int, use_password: bool, overwrite_hotkey: bool +): + wallet.create_new_hotkey( + n_words=n_words, + use_password=use_password, + overwrite=overwrite_hotkey, + ) + + +async def new_coldkey( + wallet: Wallet, n_words: int, use_password: bool, overwrite_coldkey: bool +): + wallet.create_new_coldkey( + n_words=n_words, + use_password=use_password, + overwrite=overwrite_coldkey, + ) + + +def wallet_create( + wallet: Wallet, + n_words: int = 12, + use_password: bool = True, + overwrite_coldkey: bool = False, + overwrite_hotkey: bool = False, +): + wallet.create_new_coldkey( + n_words=n_words, + use_password=use_password, + overwrite=overwrite_coldkey, + ) + wallet.create_new_hotkey( + n_words=n_words, + use_password=False, + overwrite=overwrite_hotkey, + ) From 90d6e8a7b043d252fb0f7961242ab9b417cb7b97 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 17:57:57 +0200 Subject: [PATCH 07/48] Add wallet validation, clean up code. --- cli.py | 215 ++++++++++++++++++++++++++++++++++++++----------- src/utils.py | 22 +++++ src/wallets.py | 127 ++++++++++++++++++++++++++--- 3 files changed, 306 insertions(+), 58 deletions(-) diff --git a/cli.py b/cli.py index d484b221..a3e1975e 100755 --- a/cli.py +++ b/cli.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import asyncio from typing import Optional +from typing_extensions import Annotated from bittensor_wallet import Wallet import rich @@ -11,10 +12,18 @@ # re-usable args class Options: - wallet_name = typer.Option(None, help="Name of wallet") - wallet_path = typer.Option(None, help="Filepath of wallet") - wallet_hotkey = typer.Option(None, help="Hotkey of wallet") - wallet_hk_req = typer.Option(defaults.wallet.hotkey, help="Hotkey name of wallet", prompt=True) + wallet_name = typer.Option(None, "--wallet-name", "-w", help="Name of wallet") + wallet_path = typer.Option( + None, "--wallet-path", "-p", help="Filepath of root of wallets" + ) + wallet_hotkey = typer.Option(None, "--hotkey", "-H", help="Hotkey of wallet") + wallet_hk_req = typer.Option( + defaults.wallet.hotkey, + "--hotkey", + "-H", + help="Hotkey name of wallet", + prompt=True, + ) mnemonic = typer.Option( None, help="Mnemonic used to regen your key i.e. horse cart dog ..." ) @@ -23,22 +32,40 @@ class Options: ) json = typer.Option( None, + "--json", + "-j", help="Path to a json file containing the encrypted key backup. (e.g. from PolkadotJS)", ) - json_password = typer.Option(None, help="Password to decrypt the json file.") + json_password = typer.Option( + None, "--json-password", help="Password to decrypt the json file." + ) use_password = typer.Option( False, + "--use-password", help="Set true to protect the generated bittensor key with a password.", ) - public_hex_key = typer.Option(None, help="The public key in hex format.") - ss58_address = typer.Option(None, help="The SS58 address of the coldkey") + public_hex_key = typer.Option( + None, "--public-hex-key", help="The public key in hex format." + ) + ss58_address = typer.Option(None, "--ss58", help="The SS58 address of the coldkey") overwrite_coldkey = typer.Option( - False, help="Overwrite the old coldkey with the newly generated coldkey", + False, + "--overwrite-coldkey", + help="Overwrite the old coldkey with the newly generated coldkey", prompt=True, ) overwrite_hotkey = typer.Option( - False, help="Overwrite the old hotkey with the newly generated hotkey", - prompt=True + False, + "--overwrite-hotkey", + help="Overwrite the old hotkey with the newly generated hotkey", + prompt=True, + ) + network = typer.Option( + defaults.subtensor.network, help="The subtensor network to connect to." + ) + chain = typer.Option( + defaults.subtensor.chain_endpoint, + help="The subtensor chain endpoint to connect to.", ) @@ -88,8 +115,17 @@ def __init__(self): self.app.add_typer(self.delegates_app, name="delegates") self.app.add_typer(self.delegates_app, name="d", hidden=True) + # wallet commands self.wallet_app.command("list")(self.wallet_list) self.wallet_app.command("regen-coldkey")(self.wallet_regen_coldkey) + self.wallet_app.command("regen-coldkeypub")(self.wallet_regen_coldkey_pub) + self.wallet_app.command("regen-hotkey")(self.wallet_regen_hotkey) + self.wallet_app.command("new-hotkey")(self.wallet_new_hotkey) + self.wallet_app.command("new-coldkey")(self.wallet_new_coldkey) + self.wallet_app.command("create")(self.wallet_create_wallet) + self.wallet_app.command("balance")(self.wallet_balance) + + # delegates commands self.delegates_app.command("list")(self.delegates_list) self.not_subtensor = None @@ -104,13 +140,25 @@ def initialize_chain( typer.echo(f"Initialized with {self.not_subtensor}") @staticmethod - def wallet_ask(wallet_name: str, wallet_path: str, wallet_hotkey: str): + def wallet_ask( + wallet_name: str, + wallet_path: str, + wallet_hotkey: str, + config=None, + validate=True, + ): + # TODO Wallet(config) if not any([wallet_name, wallet_path, wallet_hotkey]): wallet_name = typer.prompt("Enter wallet name:") wallet = Wallet(name=wallet_name) else: wallet = Wallet(wallet_name, wallet_path, wallet_hotkey) - # TODO Wallet(config) + if validate: + if not utils.is_valid_wallet(wallet): + utils.err_console.print( + f"[red]Error: Wallet does not appear valid. Please verify your wallet information: {wallet}[/red]" + ) + raise typer.Exit() return wallet def wallet_list(self, network: str = typer.Option("local", help="Network name")): @@ -129,7 +177,7 @@ def wallet_regen_coldkey( overwrite_coldkey: Optional[bool] = Options.overwrite_coldkey, ): """ - Executes the ``regen_coldkey`` command to regenerate a coldkey for a wallet on the Bittensor network. + Executes the `regen-coldkey` command to regenerate a coldkey for a wallet on the Bittensor network. This command is used to create a new coldkey from an existing mnemonic, seed, or JSON file. @@ -138,12 +186,15 @@ def wallet_regen_coldkey( Example usage: `btcli wallet regen_coldkey --mnemonic "word1 word2 ... word12"` - Note: This command is critical for users who need to regenerate their coldkey, possibly for recovery or security reasons. + Note: This command is critical for users who need to regenerate their coldkey, possibly for recovery or security + reasons. It should be used with caution to avoid overwriting existing keys unintentionally. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - mnemonic, seed, json, json_password = get_creation_data(mnemonic, seed, json, json_password) + mnemonic, seed, json, json_password = get_creation_data( + mnemonic, seed, json, json_password + ) asyncio.run( wallets.regen_coldkey( wallet, @@ -164,25 +215,30 @@ def wallet_regen_coldkey_pub( public_key_hex: Optional[str] = Options.public_hex_key, ss58_address: Optional[str] = Options.ss58_address, overwrite_coldkeypub: Optional[bool] = typer.Option( - False, help="Overwrites the existing coldkeypub file with the new one.", - prompt=True + False, + "--overwrite-coldkeypub", + help="Overwrites the existing coldkeypub file with the new one.", + prompt=True, ), ): """ - Executes the ``regen_coldkeypub`` command to regenerate the public part of a coldkey (coldkeypub) for a wallet on the Bittensor network. + Executes the `regen-coldkeypub` command to regenerate the public part of a coldkey (coldkeypub) for a wallet + on the Bittensor network. This command is used when a user needs to recreate their coldkeypub from an existing public key or SS58 address. Usage: - The command requires either a public key in hexadecimal format or an ``SS58`` address to regenerate the coldkeypub. It optionally allows overwriting an existing coldkeypub file. + The command requires either a public key in hexadecimal format or an ``SS58`` address to regenerate the + coldkeypub. It optionally allows overwriting an existing coldkeypub file. Example usage:: btcli wallet regen_coldkeypub --ss58_address 5DkQ4... Note: - This command is particularly useful for users who need to regenerate their coldkeypub, perhaps due to file corruption or loss. - It is a recovery-focused utility that ensures continued access to wallet functionalities. + This command is particularly useful for users who need to regenerate their coldkeypub, perhaps due to file + corruption or loss. It is a recovery-focused utility that ensures continued access to wallet + functionalities. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) if not ss58_address and not public_key_hex: @@ -217,7 +273,7 @@ def wallet_regen_hotkey( overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, ): """ - Executes the ``regen_hotkey`` command to regenerate a hotkey for a wallet on the Bittensor network. + Executes the `regen-hotkey` command to regenerate a hotkey for a wallet on the Bittensor network. Similar to regenerating a coldkey, this command creates a new hotkey from a mnemonic, seed, or JSON file. @@ -231,11 +287,14 @@ def wallet_regen_hotkey( btcli wallet regen_hotkey --seed 0x1234... Note: - This command is essential for users who need to regenerate their hotkey, possibly for security upgrades or key recovery. + This command is essential for users who need to regenerate their hotkey, possibly for security upgrades or + key recovery. It should be used cautiously to avoid accidental overwrites of existing keys. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - mnemonic, seed, json, json_password = get_creation_data(mnemonic, seed, json, json_password) + mnemonic, seed, json, json_password = get_creation_data( + mnemonic, seed, json, json_password + ) asyncio.run( wallets.regen_hotkey( wallet, @@ -254,17 +313,17 @@ def wallet_new_hotkey( wallet_path: Optional[str] = Options.wallet_path, wallet_hotkey: Optional[str] = Options.wallet_hk_req, n_words: Optional[int] = None, - use_password: Optional[bool] = Options.use_password, - overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, + use_password: bool = Options.use_password, + overwrite_hotkey: bool = Options.overwrite_hotkey, ): """ - Executes the ``new_hotkey`` command to create a new hotkey under a wallet on the Bittensor network. + Executes the `new-hotkey` command to create a new hotkey under a wallet on the Bittensor network. This command is used to generate a new hotkey for managing a neuron or participating in the network. Usage: - The command creates a new hotkey with an optional word count for the mnemonic and supports password protection. - It also allows overwriting an existing hotkey. + The command creates a new hotkey with an optional word count for the mnemonic and supports password + protection. It also allows overwriting an existing hotkey. Example usage:: @@ -274,11 +333,11 @@ def wallet_new_hotkey( This command is useful for users who wish to create additional hotkeys for different purposes, such as running multiple miners or separating operational roles within the network. """ - wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - n_words = get_n_words(n_words) - asyncio.run( - wallets.new_hotkey(wallet, n_words, use_password, overwrite_hotkey) + wallet = self.wallet_ask( + wallet_name, wallet_path, wallet_hotkey, validate=False ) + n_words = get_n_words(n_words) + asyncio.run(wallets.new_hotkey(wallet, n_words, use_password, overwrite_hotkey)) def wallet_new_coldkey( self, @@ -290,28 +349,28 @@ def wallet_new_coldkey( overwrite_coldkey: Optional[bool] = typer.Option(), ): """ - Executes the ``new_coldkey`` command to create a new coldkey under a wallet on the Bittensor network. + Executes the `new-coldkey` command to create a new coldkey under a wallet on the Bittensor network. - This command generates a coldkey, which is essential for holding balances and performing high-value transactions. + This command generates a coldkey, which is essential for holding balances and performing high-value + transactions. Usage: - The command creates a new coldkey with an optional word count for the mnemonic and supports password protection. - It also allows overwriting an existing coldkey. + The command creates a new coldkey with an optional word count for the mnemonic and supports password + protection. It also allows overwriting an existing coldkey. Example usage:: btcli wallet new_coldkey --n_words 15 Note: - This command is crucial for users who need to create a new coldkey for enhanced security or as part of setting up a new wallet. - It's a foundational step in establishing a secure presence on the Bittensor network. + This command is crucial for users who need to create a new coldkey for enhanced security or as part of + setting up a new wallet. It's a foundational step in establishing a secure presence on the Bittensor + network. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) n_words = get_n_words(n_words) asyncio.run( - wallets.new_coldkey( - wallet, n_words, use_password, overwrite_coldkey - ) + wallets.new_coldkey(wallet, n_words, use_password, overwrite_coldkey) ) def wallet_create_wallet( @@ -325,25 +384,83 @@ def wallet_create_wallet( overwrite_coldkey: Optional[bool] = Options.overwrite_coldkey, ): """ - Executes the ``create`` command to generate both a new coldkey and hotkey under a specified wallet on the Bittensor network. + Executes the `create` command to generate both a new coldkey and hotkey under a specified wallet on the + Bittensor network. This command is a comprehensive utility for creating a complete wallet setup with both cold and hotkeys. Usage: - The command facilitates the creation of a new coldkey and hotkey with an optional word count for the mnemonics. - It supports password protection for the coldkey and allows overwriting of existing keys. + The command facilitates the creation of a new coldkey and hotkey with an optional word count for the + mnemonics. It supports password protection for the coldkey and allows overwriting of existing keys. Example usage:: btcli wallet create --n_words 21 Note: - This command is ideal for new users setting up their wallet for the first time or for those who wish to completely renew their wallet keys. - It ensures a fresh start with new keys for secure and effective participation in the network. + This command is ideal for new users setting up their wallet for the first time or for those who wish to + completely renew their wallet keys. It ensures a fresh start with new keys for secure and effective + participation in the network. """ - wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + wallet = self.wallet_ask( + wallet_name, wallet_path, wallet_hotkey, validate=False + ) n_words = get_n_words(n_words) - asyncio.run(wallets.wallet_create(wallet, n_words, use_password, overwrite_coldkey, overwrite_hotkey)) + asyncio.run( + wallets.wallet_create( + wallet, n_words, use_password, overwrite_coldkey, overwrite_hotkey + ) + ) + + def wallet_balance( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + all_balances: Optional[bool] = typer.Option( + False, + "--all", + "-a", + help="Whether to display the balances for all wallets.", + ), + network: Optional[str] = typer.Option( + defaults.subtensor.network, + help="The subtensor network to connect to.", + prompt=True, + ), + chain: Optional[str] = Options.chain, + ): + """ + Executes the `balance` command to check the balance of the wallet on the Bittensor network. + + This command provides a detailed view of the wallet's coldkey balances, including free and staked balances. + + Usage: + The command lists the balances of all wallets in the user's configuration directory, showing the + wallet name, coldkey address, and the respective free and staked balances. + + Example usages: + + - To display the balance of a single wallet, use the command with the `--wallet.name` argument to specify + the wallet name: + + ``` + btcli w balance --wallet.name WALLET + ``` + + ``` + btcli w balance + ``` + + - To display the balances of all wallets, use the `--all` argument: + + ``` + btcli w balance --all + ``` + """ + subtensor = NotSubtensor(network, chain) + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + asyncio.run(wallets.wallet_balance(wallet, subtensor, all_balances)) def delegates_list( self, diff --git a/src/utils.py b/src/utils.py index 5ff2ef96..5d865b1d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,5 +1,7 @@ +import os from typing import Union +from bittensor_wallet import Wallet from bittensor_wallet.keyfile import Keypair from bittensor_wallet.utils import SS58_FORMAT, ss58 from rich.console import Console @@ -8,6 +10,26 @@ err_console = Console(stderr=True) +RAO_PER_TAO = 1e9 +U16_MAX = 65535 +U64_MAX = 18446744073709551615 + + +def is_valid_wallet(wallet: Wallet) -> bool: + """ + Verifies that the wallet with specified parameters. + :param wallet: a Wallet instance + :return: bool, whether the wallet appears valid + """ + return all( + [ + wp := os.path.exists(os.path.expanduser(wallet.path)), + os.path.exists(os.path.join(wp, wallet.name)), + os.path.isfile(os.path.join(wp, wallet.name, "hotkeys", wallet.hotkey)), + ] + ) + + def is_valid_ss58_address(address: str) -> bool: """ Checks if the given address is a valid ss58 address. diff --git a/src/wallets.py b/src/wallets.py index 8fd43c9f..d41d1bb1 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -15,20 +15,17 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -# import argparse -from bittensor_wallet import Wallet import os +from typing import Optional +from bittensor_wallet import Wallet +from bittensor_wallet.keyfile import Keyfile +from rich.table import Table import typer -# from rich.table import Table -from typing import Optional - -from .utils import err_console -# from . import defaults -# import requests -# from ..utils import RAOPERTAO +from .utils import console, err_console, RAO_PER_TAO +from . import defaults async def regen_coldkey( @@ -129,3 +126,115 @@ def wallet_create( use_password=False, overwrite=overwrite_hotkey, ) + + +def _get_coldkey_wallets_for_path(path: str) -> list[Wallet]: + """Get all coldkey wallet names from path.""" + try: + wallet_names = next(os.walk(os.path.expanduser(path)))[1] + return [Wallet(path=path, name=name) for name in wallet_names] + except StopIteration: + # No wallet files found. + wallets = [] + return wallets + + +def _get_coldkey_ss58_addresses_for_path(path: str) -> tuple[list[str], list[str]]: + """Get all coldkey ss58 addresses from path.""" + + abs_path = os.path.abspath(os.path.expanduser(path)) + wallets = [ + name + for name in os.listdir(abs_path) + if os.path.isdir(os.path.join(abs_path, name)) + ] + coldkey_paths = [ + os.path.join(abs_path, wallet, "coldkeypub.txt") + for wallet in wallets + if os.path.exists(os.path.join(abs_path, wallet, "coldkeypub.txt")) + ] + ss58_addresses = [Keyfile(path).keypair.ss58_address for path in coldkey_paths] + + return ss58_addresses, [ + os.path.basename(os.path.dirname(path)) for path in coldkey_paths + ] + + +async def wallet_balance(wallet, subtensor, all_balances): + # TODO make use of new NotSubtensor + if not wallet.coldkeypub_file.exists_on_device(): + err_console.print("[bold red]No wallets found.[/bold red]") + return + + if all_balances: + coldkeys, wallet_names = _get_coldkey_ss58_addresses_for_path(wallet.path) + else: + coldkeys = [wallet.coldkeypub.ss58_address] + wallet_names = [wallet.name] + + free_balances = [subtensor.get_balance(coldkeys[i]) for i in range(len(coldkeys))] + + staked_balances = [ + subtensor.get_total_stake_for_coldkey(coldkeys[i]) for i in range(len(coldkeys)) + ] + + total_free_balance = sum(free_balances) + total_staked_balance = sum(staked_balances) + + balances = { + name: (coldkey, free, staked) + for name, coldkey, free, staked in sorted( + zip(wallet_names, coldkeys, free_balances, staked_balances) + ) + } + + table = Table(show_footer=False) + table.title = "[white]Wallet Coldkey Balances" + table.add_column( + "[white]Wallet Name", + header_style="overline white", + footer_style="overline white", + style="rgb(50,163,219)", + no_wrap=True, + ) + + table.add_column( + "[white]Coldkey Address", + header_style="overline white", + footer_style="overline white", + style="rgb(50,163,219)", + no_wrap=True, + ) + + for type_str in ["Free", "Staked", "Total"]: + table.add_column( + f"[white]{type_str} Balance", + header_style="overline white", + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + + for name, (coldkey, free, staked) in balances.items(): + table.add_row( + name, + coldkey, + str(free), + str(staked), + str(free + staked), + ) + table.add_row() + table.add_row( + "Total Balance Across All Coldkeys", + "", + str(total_free_balance), + str(total_staked_balance), + str(total_free_balance + total_staked_balance), + ) + table.show_footer = True + + table.box = None + table.pad_edge = False + table.width = None + console.print(table) From 5df5cebb586dfb9741ccfe985ca1b84fc4064a51 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 18:42:54 +0200 Subject: [PATCH 08/48] Use correct defaults. --- cli.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/cli.py b/cli.py index a3e1975e..43c2417b 100755 --- a/cli.py +++ b/cli.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import asyncio from typing import Optional -from typing_extensions import Annotated from bittensor_wallet import Wallet import rich @@ -12,11 +11,15 @@ # re-usable args class Options: - wallet_name = typer.Option(None, "--wallet-name", "-w", help="Name of wallet") + wallet_name = typer.Option( + defaults.wallet.name, "--wallet-name", "-w", help="Name of wallet" + ) wallet_path = typer.Option( - None, "--wallet-path", "-p", help="Filepath of root of wallets" + defaults.wallet.path, "--wallet-path", "-p", help="Filepath of root of wallets" + ) + wallet_hotkey = typer.Option( + defaults.wallet.hotkey, "--hotkey", "-H", help="Hotkey of wallet" ) - wallet_hotkey = typer.Option(None, "--hotkey", "-H", help="Hotkey of wallet") wallet_hk_req = typer.Option( defaults.wallet.hotkey, "--hotkey", @@ -40,23 +43,20 @@ class Options: None, "--json-password", help="Password to decrypt the json file." ) use_password = typer.Option( - False, - "--use-password", + None, help="Set true to protect the generated bittensor key with a password.", + is_flag=True, + flag_value=False, ) - public_hex_key = typer.Option( - None, "--public-hex-key", help="The public key in hex format." - ) - ss58_address = typer.Option(None, "--ss58", help="The SS58 address of the coldkey") + public_hex_key = typer.Option(None, help="The public key in hex format.") + ss58_address = typer.Option(None, help="The SS58 address of the coldkey") overwrite_coldkey = typer.Option( False, - "--overwrite-coldkey", help="Overwrite the old coldkey with the newly generated coldkey", prompt=True, ) overwrite_hotkey = typer.Option( False, - "--overwrite-hotkey", help="Overwrite the old hotkey with the newly generated hotkey", prompt=True, ) @@ -216,7 +216,6 @@ def wallet_regen_coldkey_pub( ss58_address: Optional[str] = Options.ss58_address, overwrite_coldkeypub: Optional[bool] = typer.Option( False, - "--overwrite-coldkeypub", help="Overwrites the existing coldkeypub file with the new one.", prompt=True, ), From 36f772d4467b2dad73bab61b581933b013113e8c Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 19:17:32 +0200 Subject: [PATCH 09/48] Updated docstrings. --- cli.py | 140 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 74 insertions(+), 66 deletions(-) diff --git a/cli.py b/cli.py index 43c2417b..4d99a5c3 100755 --- a/cli.py +++ b/cli.py @@ -102,7 +102,7 @@ def get_creation_data(mnemonic, seed, json, json_password): class CLIManager: def __init__(self): - self.app = typer.Typer() + self.app = typer.Typer(rich_markup_mode="markdown") self.wallet_app = typer.Typer() self.delegates_app = typer.Typer() @@ -177,18 +177,21 @@ def wallet_regen_coldkey( overwrite_coldkey: Optional[bool] = Options.overwrite_coldkey, ): """ + # wallet regen-coldkey Executes the `regen-coldkey` command to regenerate a coldkey for a wallet on the Bittensor network. - This command is used to create a new coldkey from an existing mnemonic, seed, or JSON file. - Usage: Users can specify a mnemonic, a seed string, or a JSON file path to regenerate a coldkey. + ## Usage: + Users can specify a mnemonic, a seed string, or a JSON file path to regenerate a coldkey. The command supports optional password protection for the generated key and can overwrite an existing coldkey. - Example usage: `btcli wallet regen_coldkey --mnemonic "word1 word2 ... word12"` + ### Example usage: + ``` + btcli wallet regen_coldkey --mnemonic "word1 word2 ... word12" + ``` - Note: This command is critical for users who need to regenerate their coldkey, possibly for recovery or security - reasons. - It should be used with caution to avoid overwriting existing keys unintentionally. + ### Note: This command is critical for users who need to regenerate their coldkey, possibly for recovery or security + reasons. It should be used with caution to avoid overwriting existing keys unintentionally. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) @@ -221,20 +224,21 @@ def wallet_regen_coldkey_pub( ), ): """ + # wallet regen-coldkeypub Executes the `regen-coldkeypub` command to regenerate the public part of a coldkey (coldkeypub) for a wallet on the Bittensor network. - This command is used when a user needs to recreate their coldkeypub from an existing public key or SS58 address. - Usage: - The command requires either a public key in hexadecimal format or an ``SS58`` address to regenerate the - coldkeypub. It optionally allows overwriting an existing coldkeypub file. + ## Usage: + The command requires either a public key in hexadecimal format or an ``SS58`` address to regenerate the + coldkeypub. It optionally allows overwriting an existing coldkeypub file. - Example usage:: + ### Example usage: + ``` + btcli wallet regen_coldkeypub --ss58_address 5DkQ4... + ``` - btcli wallet regen_coldkeypub --ss58_address 5DkQ4... - - Note: + ### Note: This command is particularly useful for users who need to regenerate their coldkeypub, perhaps due to file corruption or loss. It is a recovery-focused utility that ensures continued access to wallet functionalities. @@ -272,23 +276,23 @@ def wallet_regen_hotkey( overwrite_hotkey: Optional[bool] = Options.overwrite_hotkey, ): """ + # wallet regen-hotkey Executes the `regen-hotkey` command to regenerate a hotkey for a wallet on the Bittensor network. - Similar to regenerating a coldkey, this command creates a new hotkey from a mnemonic, seed, or JSON file. - Usage: - Users can provide a mnemonic, seed string, or a JSON file to regenerate the hotkey. - The command supports optional password protection and can overwrite an existing hotkey. - - Example usage:: + ## Usage: + Users can provide a mnemonic, seed string, or a JSON file to regenerate the hotkey. + The command supports optional password protection and can overwrite an existing hotkey. - btcli wallet regen_hotkey - btcli wallet regen_hotkey --seed 0x1234... + ### Example usage: + ``` + btcli wallet regen_hotkey --seed 0x1234... + ``` - Note: - This command is essential for users who need to regenerate their hotkey, possibly for security upgrades or - key recovery. - It should be used cautiously to avoid accidental overwrites of existing keys. + ### Note: + This command is essential for users who need to regenerate their hotkey, possibly for security upgrades or + key recovery. + It should be used cautiously to avoid accidental overwrites of existing keys. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) mnemonic, seed, json, json_password = get_creation_data( @@ -316,21 +320,23 @@ def wallet_new_hotkey( overwrite_hotkey: bool = Options.overwrite_hotkey, ): """ + # wallet new-hotkey Executes the `new-hotkey` command to create a new hotkey under a wallet on the Bittensor network. - This command is used to generate a new hotkey for managing a neuron or participating in the network. - - Usage: - The command creates a new hotkey with an optional word count for the mnemonic and supports password - protection. It also allows overwriting an existing hotkey. + ## Usage + This command is used to generate a new hotkey for managing a neuron or participating in the network, + with an optional word count for the mnemonic and supports password protection. It also allows overwriting an + existing hotkey. - Example usage:: - btcli wallet new_hotkey --n_words 24 + ### Example usage: + ``` + btcli wallet new-hotkey --n_words 24 + ``` - Note: - This command is useful for users who wish to create additional hotkeys for different purposes, - such as running multiple miners or separating operational roles within the network. + ### Note: + This command is useful for users who wish to create additional hotkeys for different purposes, such as + running multiple miners or separating operational roles within the network. """ wallet = self.wallet_ask( wallet_name, wallet_path, wallet_hotkey, validate=False @@ -348,23 +354,23 @@ def wallet_new_coldkey( overwrite_coldkey: Optional[bool] = typer.Option(), ): """ - Executes the `new-coldkey` command to create a new coldkey under a wallet on the Bittensor network. - - This command generates a coldkey, which is essential for holding balances and performing high-value - transactions. - - Usage: - The command creates a new coldkey with an optional word count for the mnemonic and supports password - protection. It also allows overwriting an existing coldkey. - - Example usage:: - - btcli wallet new_coldkey --n_words 15 - - Note: - This command is crucial for users who need to create a new coldkey for enhanced security or as part of - setting up a new wallet. It's a foundational step in establishing a secure presence on the Bittensor - network. + # wallet new-coldkey + Executes the `new-coldkey` command to create a new coldkey under a wallet on the Bittensor network. This + command generates a coldkey, which is essential for holding balances and performing high-value transactions. + + ## Usage: + The command creates a new coldkey with an optional word count for the mnemonic and supports password + protection. It also allows overwriting an existing coldkey. + + ### Example usage:: + ``` + btcli wallet new_coldkey --n_words 15 + ``` + + ### Note: + This command is crucial for users who need to create a new coldkey for enhanced security or as part of + setting up a new wallet. It's a foundational step in establishing a secure presence on the Bittensor + network. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) n_words = get_n_words(n_words) @@ -383,23 +389,25 @@ def wallet_create_wallet( overwrite_coldkey: Optional[bool] = Options.overwrite_coldkey, ): """ + # wallet create Executes the `create` command to generate both a new coldkey and hotkey under a specified wallet on the - Bittensor network. + Bittensor network. This command is a comprehensive utility for creating a complete wallet setup with both cold + and hotkeys. - This command is a comprehensive utility for creating a complete wallet setup with both cold and hotkeys. - - Usage: - The command facilitates the creation of a new coldkey and hotkey with an optional word count for the - mnemonics. It supports password protection for the coldkey and allows overwriting of existing keys. - Example usage:: + ## Usage: + The command facilitates the creation of a new coldkey and hotkey with an optional word count for the + mnemonics. It supports password protection for the coldkey and allows overwriting of existing keys. - btcli wallet create --n_words 21 + ### Example usage: + ``` + btcli wallet create --n_words 21 + ``` - Note: - This command is ideal for new users setting up their wallet for the first time or for those who wish to - completely renew their wallet keys. It ensures a fresh start with new keys for secure and effective - participation in the network. + ### Note: + This command is ideal for new users setting up their wallet for the first time or for those who wish to + completely renew their wallet keys. It ensures a fresh start with new keys for secure and effective + participation in the network. """ wallet = self.wallet_ask( wallet_name, wallet_path, wallet_hotkey, validate=False From d7ffbdf7617ec7a107d8b48ceec16d2df63b7065 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 20:58:37 +0200 Subject: [PATCH 10/48] [WIP] Check-in --- cli.py | 50 +- requirements.txt | 5 +- src/__init__.py | 12 + src/bittensor/async_substrate_interface.py | 1377 ++++++++++++++++++++ src/bittensor/balances.py | 286 ++++ src/subtensor_interface.py | 64 + src/utils.py | 4 +- src/wallets.py | 13 +- 8 files changed, 1772 insertions(+), 39 deletions(-) create mode 100644 src/bittensor/async_substrate_interface.py create mode 100644 src/bittensor/balances.py create mode 100644 src/subtensor_interface.py diff --git a/cli.py b/cli.py index 4d99a5c3..eb387f55 100755 --- a/cli.py +++ b/cli.py @@ -7,6 +7,7 @@ import typer from src import wallets, defaults, utils +from src.subtensor_interface import SubtensorInterface # re-usable args @@ -69,15 +70,6 @@ class Options: ) -class NotSubtensor: - def __init__(self, network: str, chain: str): - self.network = network - self.chain = chain - - def __str__(self): - return f"NotSubtensor(network={self.network}, chain={self.chain})" - - def get_n_words(n_words: Optional[int]) -> int: while n_words not in [12, 15, 18, 21, 24]: n_words = typer.prompt( @@ -136,7 +128,7 @@ def initialize_chain( chain: str = typer.Option("default_chain", help="Chain name"), ): if not self.not_subtensor: - self.not_subtensor = NotSubtensor(network, chain) + self.not_subtensor = SubtensorInterface(network, chain) typer.echo(f"Initialized with {self.not_subtensor}") @staticmethod @@ -152,7 +144,7 @@ def wallet_ask( wallet_name = typer.prompt("Enter wallet name:") wallet = Wallet(name=wallet_name) else: - wallet = Wallet(wallet_name, wallet_path, wallet_hotkey) + wallet = Wallet(name=wallet_name, hotkey=wallet_hotkey, path=wallet_path) if validate: if not utils.is_valid_wallet(wallet): utils.err_console.print( @@ -438,34 +430,34 @@ def wallet_balance( chain: Optional[str] = Options.chain, ): """ + # wallet balance Executes the `balance` command to check the balance of the wallet on the Bittensor network. - This command provides a detailed view of the wallet's coldkey balances, including free and staked balances. - Usage: - The command lists the balances of all wallets in the user's configuration directory, showing the - wallet name, coldkey address, and the respective free and staked balances. + ## Usage: + The command lists the balances of all wallets in the user's configuration directory, showing the + wallet name, coldkey address, and the respective free and staked balances. - Example usages: + ### Example usages: - - To display the balance of a single wallet, use the command with the `--wallet.name` argument to specify - the wallet name: + - To display the balance of a single wallet, use the command with the `--wallet.name` argument to specify + the wallet name: - ``` - btcli w balance --wallet.name WALLET - ``` + ``` + btcli w balance --wallet.name WALLET + ``` - ``` - btcli w balance - ``` + ``` + btcli w balance + ``` - - To display the balances of all wallets, use the `--all` argument: + - To display the balances of all wallets, use the `--all` argument: - ``` - btcli w balance --all - ``` + ``` + btcli w balance --all + ``` """ - subtensor = NotSubtensor(network, chain) + subtensor = SubtensorInterface(network, chain) wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) asyncio.run(wallets.wallet_balance(wallet, subtensor, all_balances)) diff --git a/requirements.txt b/requirements.txt index 3dc56257..7a456358 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ +scalecodec==1.2.11 +substrate-interface~=1.7.9 typer~=0.12 rich~=13.7 -git+https://github.com/opentensor/btwallet # bittensor_wallet \ No newline at end of file +git+https://github.com/opentensor/btwallet # bittensor_wallet +websockets>=12.0 diff --git a/src/__init__.py b/src/__init__.py index 6fe0ae42..33d494a4 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -2,6 +2,18 @@ from typing import Optional +class Constants: + networks = ["local", "finney", "test", "archive"] + finney_entrypoint = "wss://entrypoint-finney.opentensor.ai:443" + finney_test_entrypoint = "wss://test.finney.opentensor.ai:443/" + archive_entrypoint = "wss://archive.chain.opentensor.ai:443/" + network_map = { + "finney": finney_entrypoint, + "test": finney_test_entrypoint, + "archive": archive_entrypoint, + } + + @dataclass class CUDA: dev_id: list[int] diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py new file mode 100644 index 00000000..a49abfad --- /dev/null +++ b/src/bittensor/async_substrate_interface.py @@ -0,0 +1,1377 @@ +import asyncio +import json +from collections import defaultdict +from dataclasses import dataclass +from typing import Optional, Any, Union, Callable, Awaitable + +import websockets +from scalecodec import GenericExtrinsic +from scalecodec.base import ScaleBytes, ScaleType, RuntimeConfigurationObject +from scalecodec.type_registry import load_type_registry_preset +from scalecodec.types import GenericCall +from substrateinterface import Keypair, ExtrinsicReceipt +from substrateinterface.base import SubstrateInterface, QueryMapResult +from substrateinterface.exceptions import SubstrateRequestException +from substrateinterface.storage import StorageKey + + +ResultHandler = Callable[[dict, Any], Awaitable[tuple[dict, bool]]] + + +@dataclass +class Preprocessed: + queryable: str + method: str + params: list + value_scale_type: str + storage_item: ScaleType + + +class Runtime: + block_hash: str + block_id: int + runtime_version = None + transaction_version = None + cache_region = None + metadata = None + type_registry_preset = None + + def __init__(self, chain, runtime_config, metadata, type_registry): + self.runtime_config = RuntimeConfigurationObject() + self.config = {} + self.chain = chain + self.type_registry = type_registry + self.runtime_config = runtime_config + self.metadata = metadata + + @property + def implements_scaleinfo(self) -> bool: + """ + Returns True if current runtime implementation a `PortableRegistry` (`MetadataV14` and higher) + """ + if self.metadata: + return self.metadata.portable_registry is not None + else: + return False + + def reload_type_registry( + self, use_remote_preset: bool = True, auto_discover: bool = True + ): + """ + Reload type registry and preset used to instantiate the SubstrateInterface object. Useful to periodically apply + changes in type definitions when a runtime upgrade occurred + + Parameters + ---------- + use_remote_preset: When True preset is downloaded from Github master, otherwise use files from local installed + scalecodec package + auto_discover + + Returns + ------- + + """ + self.runtime_config.clear_type_registry() + + self.runtime_config.implements_scale_info = self.implements_scaleinfo + + # Load metadata types in runtime configuration + self.runtime_config.update_type_registry(load_type_registry_preset(name="core")) + self.apply_type_registry_presets( + use_remote_preset=use_remote_preset, auto_discover=auto_discover + ) + + def apply_type_registry_presets( + self, + use_remote_preset: bool = True, + auto_discover: bool = True, + ): + """ + Applies type registry presets to the runtime + :param use_remote_preset: bool, whether to use presets from remote + :param auto_discover: bool, whether to use presets from local installed scalecodec package + """ + if self.type_registry_preset is not None: + # Load type registry according to preset + type_registry_preset_dict = load_type_registry_preset( + name=self.type_registry_preset, use_remote_preset=use_remote_preset + ) + + if not type_registry_preset_dict: + raise ValueError( + f"Type registry preset '{self.type_registry_preset}' not found" + ) + + elif auto_discover: + # Try to auto discover type registry preset by chain name + type_registry_name = self.chain.lower().replace(" ", "-") + try: + type_registry_preset_dict = load_type_registry_preset( + type_registry_name + ) + self.type_registry_preset = type_registry_name + except ValueError: + type_registry_preset_dict = None + + else: + type_registry_preset_dict = None + + if type_registry_preset_dict: + # Load type registries in runtime configuration + if self.implements_scaleinfo is False: + # Only runtime with no embedded types in metadata need the default set of explicit defined types + self.runtime_config.update_type_registry( + load_type_registry_preset( + "legacy", use_remote_preset=use_remote_preset + ) + ) + + if self.type_registry_preset != "legacy": + self.runtime_config.update_type_registry(type_registry_preset_dict) + + if self.type_registry: + # Load type registries in runtime configuration + self.runtime_config.update_type_registry(self.type_registry) + + +class RequestManager: + RequestResults = dict[Union[str, int], list[Union[ScaleType, dict]]] + + def __init__(self, payloads): + self.response_map = {} + self.responses = defaultdict(lambda: {"complete": False, "results": []}) + self.payloads_count = len(payloads) + + def add_request(self, item_id: int, request_id: Any): + """ + Adds an outgoing request to the responses map for later retrieval + """ + self.response_map[item_id] = request_id + + def overwrite_request(self, item_id: int, request_id: Any): + """ + Overwrites an existing request in the responses map with a new request_id. This is used + for multipart responses that generate a subscription id we need to watch, rather than the initial + request_id. + """ + self.response_map[request_id] = self.response_map.pop(item_id) + return request_id + + def add_response(self, item_id: int, response: dict, complete: bool): + """ + Maps a response to the request for later retrieval + """ + request_id = self.response_map[item_id] + self.responses[request_id]["results"].append(response) + self.responses[request_id]["complete"] = complete + + @property + def is_complete(self): + """ + Returns whether all requests in the manager have completed + """ + return ( + all(info["complete"] for info in self.responses.values()) + and len(self.responses) == self.payloads_count + ) + + def get_results(self) -> RequestResults: + """ + Generates a dictionary mapping the requests initiated to the responses received. + """ + return { + request_id: info["results"] for request_id, info in self.responses.items() + } + + +class Websocket: + def __init__( + self, + ws_url: str, + max_subscriptions=1024, + max_connections=100, + shutdown_timer=5, + options: dict = None, + ): + """ + Websocket manager object. Allows for the use of a single websocket connection by multiple + calls. + + :param ws_url: Websocket URL to connect to + :param max_subscriptions: Maximum number of subscriptions per websocket connection + :param max_connections: Maximum number of connections total + :param shutdown_timer: Number of seconds to shut down websocket connection after last use + """ + # TODO allow setting max concurrent connections and rpc subscriptions per connection + # TODO reconnection logic + self.ws_url = ws_url + self.ws = None + self.id = 0 + self.max_subscriptions = max_subscriptions + self.max_connections = max_connections + self.shutdown_timer = shutdown_timer + self._received = {} + self._in_use = 0 + self._receiving_task = None + self._attempts = 0 + self._initialized = False + self._lock = asyncio.Lock() + self._exit_task = None + self._open_subscriptions = 0 + self._options = options if options else {} + + async def __aenter__(self): + async with self._lock: + self._in_use += 1 + if self._exit_task: + self._exit_task.cancel() + if not self._initialized: + self._initialized = True + await self._connect() + self._receiving_task = asyncio.create_task(self._start_receiving()) + return self + + async def _connect(self): + self.ws = await asyncio.wait_for( + websockets.connect(self.ws_url, **self._options), timeout=None + ) + + async def __aexit__(self, exc_type, exc_val, exc_tb): + async with self._lock: + self._in_use -= 1 + if self._exit_task is not None: + self._exit_task.cancel() + try: + await self._exit_task + except asyncio.CancelledError: + pass + if self._in_use == 0 and self.ws is not None: + self.id = 0 + self._open_subscriptions = 0 + self._exit_task = asyncio.create_task(self._exit_with_timer()) + + async def _exit_with_timer(self): + """ + Allows for graceful shutdown of websocket connection after specified number of seconds, allowing + for reuse of the websocket connection. + """ + try: + await asyncio.sleep(self.shutdown_timer) + async with self._lock: + self._receiving_task.cancel() + try: + await self._receiving_task + except asyncio.CancelledError: + pass + await self.ws.close() + self.ws = None + self._initialized = False + self._receiving_task = None + self.id = 0 + except asyncio.CancelledError: + pass + + async def _recv(self) -> None: + try: + response = json.loads(await self.ws.recv()) + async with self._lock: + self._open_subscriptions -= 1 + if "id" in response: + self._received[response["id"]] = response + elif "params" in response: + self._received[response["params"]["subscription"]] = response + else: + raise KeyError(response) + except websockets.ConnectionClosed: + raise + except KeyError as e: + print(f"Unhandled websocket response: {e}") + raise e + + async def _start_receiving(self): + try: + while True: + await self._recv() + except asyncio.CancelledError: + pass + except websockets.ConnectionClosed: + # TODO try reconnect, but only if it's needed + raise + + async def send(self, payload: dict) -> int: + """ + Sends a payload to the websocket connection. + + :param payload: payload, generate a payload with the AsyncSubstrateInterface.make_payload method + """ + async with self._lock: + original_id = self.id + try: + await self.ws.send(json.dumps({**payload, **{"id": original_id}})) + self.id += 1 + self._open_subscriptions += 1 + return original_id + except websockets.ConnectionClosed: + raise + + async def retrieve(self, item_id: int) -> Optional[dict]: + """ + Retrieves a single item from received responses dict queue + + :param item_id: id of the item to retrieve + + :return: retrieved item + """ + while True: + async with self._lock: + if item_id in self._received: + return self._received.pop(item_id) + await asyncio.sleep(0.1) + + +class AsyncSubstrateInterface: + runtime = None + substrate = None + + def __init__( + self, + chain_endpoint: str, + use_remote_preset=False, + auto_discover=True, + auto_reconnect=True, + ss58_format=None, + type_registry=None + ): + """ + The asyncio-compatible version of the subtensor interface commands we use in bittensor + """ + self.chain_endpoint = chain_endpoint + self.__chain = None + self.ws = Websocket( + chain_endpoint, + options={ + "max_size": 2 ** 32, + "read_limit": 2 ** 32, + "write_limit": 2 ** 32, + }, + ) + self._lock = asyncio.Lock() + self.last_block_hash = None + self.config = { + "use_remote_preset": use_remote_preset, + "auto_discover": auto_discover, + "auto_reconnect": auto_reconnect, + "rpc_methods": None, + "strict_scale_decode": True, + } + self.initialized = False + self._forgettable_task = None + self.ss58_format = ss58_format + self.type_registry = type_registry + + async def __aenter__(self): + await self.initialize() + + async def initialize(self): + """ + Initialize the attached substrate object + """ + async with self._lock: + if not self.substrate: + self.substrate = SubstrateInterface( + ss58_format=self.ss58_format, + use_remote_preset=True, + url=self.chain_endpoint, + type_registry=self.type_registry, + ) + self.__chain = (await self.rpc_request("system_chain", [])).get( + "result" + ) + self.initialized = True + + async def __aexit__(self, exc_type, exc_val, exc_tb): + pass + + @property + def chain(self): + """ + Returns the substrate chain currently associated with object + """ + return self.__chain + + async def get_storage_item(self, module: str, storage_function: str): + if not self.substrate.metadata: + self.substrate.init_runtime() + metadata_pallet = self.substrate.metadata.get_metadata_pallet(module) + storage_item = metadata_pallet.get_storage_function(storage_function) + return storage_item + + def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool): + return block_hash if block_hash else (self.last_block_hash if reuse else None) + + async def init_runtime( + self, block_hash: Optional[str] = None, block_id: Optional[int] = None + ) -> Runtime: + """ + This method is used by all other methods that deals with metadata and types defined in the type registry. + It optionally retrieves the block_hash when block_id is given and sets the applicable metadata for that + block_hash. Also, it applies all the versioned types at the time of the block_hash. + + Because parsing of metadata and type registry is quite heavy, the result will be cached per runtime id. + In the future there could be support for caching backends like Redis to make this cache more persistent. + + :param block_hash: optional block hash, should not be specified if block_id is + :param block_id: optional block id, should not be specified if block_hash is + + :returns: Runtime object + """ + async with self._lock: + await asyncio.get_event_loop().run_in_executor( + None, self.substrate.init_runtime, block_hash, block_id + ) + return Runtime( + self.chain, self.substrate.runtime_config, self.substrate.metadata, self.type_registry + ) + + async def get_block_runtime_version(self, block_hash: str) -> dict: + """ + Retrieve the runtime version id of given block_hash + """ + response = await self.rpc_request("state_getRuntimeVersion", [block_hash]) + return response.get("result") + + async def get_block_metadata( + self, block_hash=None, decode=True + ) -> Union[dict, ScaleType]: + """ + A pass-though to existing JSONRPC method `state_getMetadata`. + + Parameters + ---------- + block_hash + decode: True for decoded version + + Returns + ------- + + """ + params = None + if decode and not self.substrate.runtime_config: + raise ValueError( + "Cannot decode runtime configuration without a supplied runtime_config" + ) + + if block_hash: + params = [block_hash] + response = await self.rpc_request("state_getMetadata", params) + + if "error" in response: + raise SubstrateRequestException(response["error"]["message"]) + + if response.get("result") and decode: + metadata_decoder = self.substrate.runtime_config.create_scale_object( + "MetadataVersioned", data=ScaleBytes(response.get("result")) + ) + metadata_decoder.decode() + + return metadata_decoder + + return response + + async def _preprocess( + self, + query_for: Optional[list], + block_hash: str, + storage_function: str, + module: str, + ) -> Preprocessed: + """ + Creates a Preprocessed data object for passing to ``_make_rpc_request`` + """ + params = query_for if query_for else [] + # Search storage call in metadata + metadata_pallet = self.substrate.metadata.get_metadata_pallet(module) + + if not metadata_pallet: + raise Exception(f'Pallet "{module}" not found') + + storage_item = metadata_pallet.get_storage_function(storage_function) + + if not metadata_pallet or not storage_item: + raise Exception(f'Storage function "{module}.{storage_function}" not found') + + # SCALE type string of value + param_types = storage_item.get_params_type_string() + value_scale_type = storage_item.get_value_type_string() + + if len(params) != len(param_types): + raise ValueError( + f"Storage function requires {len(param_types)} parameters, {len(params)} given" + ) + + storage_key = StorageKey.create_from_storage_function( + module, + storage_item.value["name"], + params, + runtime_config=self.substrate.runtime_config, + metadata=self.substrate.metadata, + ) + method = ( + "state_getStorageAt" + if self.substrate.supports_rpc_method("state_getStorageAt") + else "state_getStorage" + ) + return Preprocessed( + str(query_for), + method, + [storage_key.to_hex(), block_hash], + value_scale_type, + storage_item, + ) + + async def _process_response( + self, + response: dict, + subscription_id: Union[int, str], + value_scale_type: str, + storage_item: Optional[ScaleType] = None, + runtime: Optional[Runtime] = None, + result_handler: Optional[ResultHandler] = None, + ) -> tuple[Union[ScaleType, dict], bool]: + obj = response + + if value_scale_type: + if not runtime: + async with self._lock: + runtime = Runtime( + self.chain, + self.substrate.runtime_config, + self.substrate.metadata, + self.type_registry + ) + if response.get("result") is not None: + query_value = response.get("result") + elif storage_item.value["modifier"] == "Default": + # Fallback to default value of storage function if no result + query_value = storage_item.value_object["default"].value_object + else: + # No result is interpreted as an Option<...> result + value_scale_type = f"Option<{value_scale_type}>" + query_value = storage_item.value_object["default"].value_object + + obj = runtime.runtime_config.create_scale_object( + type_string=value_scale_type, + data=ScaleBytes(query_value), + metadata=runtime.metadata, + ) + obj.decode(check_remaining=True) + obj.meta_info = {"result_found": response.get("result") is not None} + if asyncio.iscoroutinefunction(result_handler): + # For multipart responses as a result of subscriptions. + message, bool_result = await result_handler(obj, subscription_id) + return message, bool_result + return obj, True + + async def _make_rpc_request( + self, + payloads: list[dict], + value_scale_type: Optional[str] = None, + storage_item: Optional[ScaleType] = None, + runtime: Optional[Runtime] = None, + result_handler: Optional[ResultHandler] = None, + ) -> RequestManager.RequestResults: + request_manager = RequestManager(payloads) + + subscription_added = False + + async with self.ws as ws: + for item in payloads: + item_id = await ws.send(item["payload"]) + request_manager.add_request(item_id, item["id"]) + + while True: + for item_id in request_manager.response_map.keys(): + if ( + item_id not in request_manager.responses + or asyncio.iscoroutinefunction(result_handler) + ): + if response := await ws.retrieve(item_id): + if ( + asyncio.iscoroutinefunction(result_handler) + and not subscription_added + ): + # handles subscriptions, overwrites the previous mapping of {item_id : payload_id} + # with {subscription_id : payload_id} + item_id = request_manager.overwrite_request( + item_id, response["result"] + ) + decoded_response, complete = await self._process_response( + response, + item_id, + value_scale_type, + storage_item, + runtime, + result_handler, + ) + request_manager.add_response( + item_id, decoded_response, complete + ) + if ( + asyncio.iscoroutinefunction(result_handler) + and not subscription_added + ): + subscription_added = True + break + + if request_manager.is_complete: + break + + return request_manager.get_results() + + @staticmethod + def make_payload(id_: str, method: str, params: list) -> dict: + """ + Creates a payload for making an rpc_request with _make_rpc_request + + :param id_: a unique name you would like to give to this request + :param method: the method in the RPC request + :param params: the params in the RPC request + + :return: the payload dict + """ + return { + "id": id_, + "payload": {"jsonrpc": "2.0", "method": method, "params": params}, + } + + async def rpc_request( + self, + method: str, + params: list, + block_hash: Optional[str] = None, + reuse_block_hash: bool = False, + ) -> Any: + """ + Makes an RPC request to the subtensor. Use this only if ``self.query`` and ``self.query_multiple`` and + ``self.query_map`` do not meet your needs. + + :param method: str the method in the RPC request + :param params: list of the params in the RPC request + :param block_hash: optional str, the hash of the block — only supply this if not supplying the block + hash in the params, and not reusing the block hash + :param reuse_block_hash: optional bool, whether to reuse the block hash in the params — only mark as True + if not supplying the block hash in the params, or via the `block_hash` parameter + + :return: the response from the RPC request + """ + block_hash = self._get_current_block_hash(block_hash, reuse_block_hash) + payloads = [ + self.make_payload( + "rpc_request", + method, + params + [block_hash] if block_hash else params, + ) + ] + runtime = Runtime( + self.chain, self.substrate.runtime_config, self.substrate.metadata, self.type_registry + ) + result = await self._make_rpc_request(payloads, runtime=runtime) + if "error" in result["rpc_request"][0]: + raise SubstrateRequestException( + result["rpc_request"][0]["error"]["message"] + ) + if "result" in result["rpc_request"][0]: + return result["rpc_request"][0] + else: + raise SubstrateRequestException(result["rpc_request"][0]) + + async def get_block_hash(self, block_id: int) -> str: + return (await self.rpc_request("chain_getBlockHash", [block_id]))["result"] + + async def get_chain_head(self) -> str: + return (await self.rpc_request("chain_getHead", []))["result"] + + async def compose_call( + self, + call_module: str, + call_function: str, + call_params: dict = None, + block_hash: str = None, + ) -> GenericCall: + """ + Composes a call payload which can be used in an extrinsic. + + :param call_module: Name of the runtime module e.g. Balances + :param call_function: Name of the call function e.g. transfer + :param call_params: This is a dict containing the params of the call. e.g. + `{'dest': 'EaG2CRhJWPb7qmdcJvy3LiWdh26Jreu9Dx6R1rXxPmYXoDk', 'value': 1000000000000}` + :param block_hash: Use metadata at given block_hash to compose call + + :return: A composed call + """ + return await asyncio.get_event_loop().run_in_executor( + None, + self.substrate.compose_call, + call_module, + call_function, + call_params, + block_hash, + ) + + async def query_multiple( + self, + params: list, + storage_function: str, + module: str, + block_hash: Optional[str] = None, + reuse_block_hash: bool = False, + ) -> RequestManager.RequestResults: + """ + Queries the subtensor. Only use this when making multiple queries, else use ``self.query`` + """ + # By allowing for specifying the block hash, users, if they have multiple query types they want + # to do, can simply query the block hash first, and then pass multiple query_subtensor calls + # into an asyncio.gather, with the specified block hash + block_hash = ( + block_hash + if block_hash + else ( + self.last_block_hash + if reuse_block_hash + else await self.get_chain_head() + ) + ) + self.last_block_hash = block_hash + runtime = await self.init_runtime(block_hash=block_hash) + preprocessed: tuple[Preprocessed] = await asyncio.gather( + *[ + self._preprocess([x], block_hash, storage_function, module) + for x in params + ] + ) + all_info = [ + self.make_payload(item.queryable, item.method, item.params) + for item in preprocessed + ] + # These will always be the same throughout the preprocessed list, so we just grab the first one + value_scale_type = preprocessed[0].value_scale_type + storage_item = preprocessed[0].storage_item + + responses = await self._make_rpc_request( + all_info, value_scale_type, storage_item, runtime + ) + return responses + + async def create_scale_object( + self, + type_string: str, + data: ScaleBytes = None, + block_hash: str = None, + **kwargs, + ) -> "ScaleType": + """ + Convenience method to create a SCALE object of type `type_string`, this will initialize the runtime + automatically at moment of `block_hash`, or chain tip if omitted. + + :param type_string: str Name of SCALE type to create + :param data: ScaleBytes Optional ScaleBytes to decode + :param block_hash: Optional block hash for moment of decoding, when omitted the chain tip will be used + :param kwargs: keyword args for the Scale Type constructor + + :return: The created Scale Type object + """ + runtime = await self.init_runtime(block_hash=block_hash) + if "metadata" not in kwargs: + kwargs["metadata"] = runtime.metadata + + return runtime.runtime_config.create_scale_object( + type_string, data=data, **kwargs + ) + + async def create_signed_extrinsic( + self, + call: GenericCall, + keypair: Keypair, + era: dict = None, + nonce: int = None, + tip: int = 0, + tip_asset_id: int = None, + signature: Union[bytes, str] = None, + ) -> "GenericExtrinsic": + """ + Creates an extrinsic signed by given account details + + :param call: GenericCall to create extrinsic for + :param keypair: Keypair used to sign the extrinsic + :param era: Specify mortality in blocks in follow format: + {'period': [amount_blocks]} If omitted the extrinsic is immortal + :param nonce: nonce to include in extrinsics, if omitted the current nonce is retrieved on-chain + :param tip: The tip for the block author to gain priority during network congestion + :param tip_asset_id: Optional asset ID with which to pay the tip + :param signature: Optionally provide signature if externally signed + + :return: The signed Extrinsic + """ + return await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.substrate.create_signed_extrinsic( + call=call, + keypair=keypair, + era=era, + nonce=nonce, + tip=tip, + tip_asset_id=tip_asset_id, + signature=signature, + ), + ) + + async def runtime_call( + self, + api: str, + method: str, + params: Union[list, dict] = None, + block_hash: str = None, + ) -> ScaleType: + """ + Calls a runtime API method + + :param api: Name of the runtime API e.g. 'TransactionPaymentApi' + :param method: Name of the method e.g. 'query_fee_details' + :param params: List of parameters needed to call the runtime API + :param block_hash: Hash of the block at which to make the runtime API call + + :return: ScaleType from the runtime call + """ + await self.init_runtime() + + if params is None: + params = {} + + async with self._lock: + try: + runtime_call_def = self.substrate.runtime_config.type_registry[ + "runtime_api" + ][api]["methods"][method] + runtime_api_types = self.substrate.runtime_config.type_registry[ + "runtime_api" + ][api].get("types", {}) + except KeyError: + raise ValueError( + f"Runtime API Call '{api}.{method}' not found in registry" + ) + + if isinstance(params, list) and len(params) != len( + runtime_call_def["params"] + ): + raise ValueError( + f"Number of parameter provided ({len(params)}) does not " + f"match definition {len(runtime_call_def['params'])}" + ) + + # Add runtime API types to registry + self.substrate.runtime_config.update_type_registry_types(runtime_api_types) + runtime = Runtime( + self.chain, self.substrate.runtime_config, self.substrate.metadata, self.type_registry + ) + + # Encode params + param_data = ScaleBytes(bytes()) + for idx, param in enumerate(runtime_call_def["params"]): + scale_obj = runtime.runtime_config.create_scale_object(param["type"]) + if isinstance(params, list): + param_data += scale_obj.encode(params[idx]) + else: + if param["name"] not in params: + raise ValueError(f"Runtime Call param '{param['name']}' is missing") + + param_data += scale_obj.encode(params[param["name"]]) + + # RPC request + result_data = await self.rpc_request( + "state_call", [f"{api}_{method}", str(param_data), block_hash] + ) + + # Decode result + result_obj = runtime.runtime_config.create_scale_object( + runtime_call_def["type"] + ) + result_obj.decode( + ScaleBytes(result_data["result"]), + check_remaining=self.config.get("strict_scale_decode"), + ) + + return result_obj + + async def get_account_nonce(self, account_address: str) -> int: + """ + Returns current nonce for given account address + + :param account_address: SS58 formatted address + + :return: Nonce for given account address + """ + nonce_obj = await self.runtime_call( + "AccountNonceApi", "account_nonce", [account_address] + ) + return nonce_obj.value + + async def get_constant( + self, module_name: str, constant_name: str, block_hash: Optional[str] = None + ) -> Optional["ScaleType"]: + """ + Returns the decoded `ScaleType` object of the constant for given module name, call function name and block_hash + (or chaintip if block_hash is omitted) + + Parameters + ---------- + :param module_name: Name of the module to query + :param constant_name: Name of the constant to query + :param block_hash: Hash of the block at which to make the runtime API call + + :return: ScaleType from the runtime call + """ + async with self._lock: + return await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.substrate.get_constant( + module_name, constant_name, block_hash + ), + ) + + async def get_payment_info( + self, call: GenericCall, keypair: Keypair + ) -> dict[str, Any]: + """ + Retrieves fee estimation via RPC for given extrinsic + + Parameters + ---------- + call: Call object to estimate fees for + keypair: Keypair of the sender, does not have to include private key because no valid signature is required + + Returns + ------- + Dict with payment info + + E.g. `{'class': 'normal', 'partialFee': 151000000, 'weight': {'ref_time': 143322000}}` + + """ + + # Check requirements + if not isinstance(call, GenericCall): + raise TypeError("'call' must be of type Call") + + if not isinstance(keypair, Keypair): + raise TypeError("'keypair' must be of type Keypair") + + # No valid signature is required for fee estimation + signature = "0x" + "00" * 64 + + # Create extrinsic + extrinsic = await self.create_signed_extrinsic( + call=call, keypair=keypair, signature=signature + ) + async with self._lock: + extrinsic_len = self.substrate.runtime_config.create_scale_object("u32") + extrinsic_len.encode(len(extrinsic.data)) + + result = await self.runtime_call( + "TransactionPaymentApi", "query_info", [extrinsic, extrinsic_len] + ) + + return result.value + + async def query( + self, + module: str, + storage_function: str, + params: Optional[list] = None, + block_hash: Optional[str] = None, + raw_storage_key: bytes = None, + subscription_handler=None, + reuse_block_hash: bool = False, + ) -> "ScaleType": + """ + Queries subtensor. This should only be used when making a single request. For multiple requests, + you should use ``self.query_multiple`` + """ + block_hash = ( + block_hash + if block_hash + else ( + self.last_block_hash + if reuse_block_hash + else await self.get_chain_head() + ) + ) + self.last_block_hash = block_hash + runtime = await self.init_runtime(block_hash=block_hash) + preprocessed: Preprocessed = await self._preprocess( + params, block_hash, storage_function, module + ) + payload = [ + self.make_payload( + preprocessed.queryable, preprocessed.method, preprocessed.params + ) + ] + value_scale_type = preprocessed.value_scale_type + storage_item = preprocessed.storage_item + + responses = await self._make_rpc_request( + payload, + value_scale_type, + storage_item, + runtime, + result_handler=subscription_handler, + ) + return responses[preprocessed.queryable][0] + + async def query_map( + self, + module: str, + storage_function: str, + params: Optional[list] = None, + block_hash: Optional[str] = None, + max_results: int = None, + start_key: str = None, + page_size: int = 100, + ignore_decoding_errors: bool = True, + reuse_block_hash: bool = False, + ) -> "QueryMapResult": + """ + Iterates over all key-pairs located at the given module and storage_function. The storage + item must be a map. + + Example: + + ``` + result = substrate.query_map('System', 'Account', max_results=100) + + for account, account_info in result: + print(f"Free balance of account '{account.value}': {account_info.value['data']['free']}") + ``` + + :param module: The module name in the metadata, e.g. System or Balances. + :param storage_function: The storage function name, e.g. Account or Locks. + :param params: The input parameters in case of for example a `DoubleMap` storage function + :param block_hash: Optional block hash for result at given block, when left to None the chain tip will be used. + :param max_results: the maximum of results required, if set the query will stop fetching results when number is + reached + :param start_key: The storage key used as offset for the results, for pagination purposes + :param page_size: The results are fetched from the node RPC in chunks of this size + :param ignore_decoding_errors: When set this will catch all decoding errors, set the item to None and continue + decoding + :param reuse_block_hash: use True if you wish to make the query using the last-used block hash. Do not mark True + if supplying a block_hash + + :return: QueryMapResult object + """ + params = params or [] + block_hash = ( + block_hash + if block_hash + else ( + self.last_block_hash + if reuse_block_hash + else await self.get_chain_head() + ) + ) + self.last_block_hash = block_hash + runtime = await self.init_runtime(block_hash=block_hash) + + metadata_pallet = runtime.metadata.get_metadata_pallet(module) + if not metadata_pallet: + raise ValueError(f'Pallet "{module}" not found') + + storage_item = metadata_pallet.get_storage_function(storage_function) + + if not metadata_pallet or not storage_item: + raise ValueError( + f'Storage function "{module}.{storage_function}" not found' + ) + + value_type = storage_item.get_value_type_string() + param_types = storage_item.get_params_type_string() + key_hashers = storage_item.get_param_hashers() + + # Check MapType conditions + if len(param_types) == 0: + raise ValueError("Given storage function is not a map") + if len(params) > len(param_types) - 1: + raise ValueError( + f"Storage function map can accept max {len(param_types) - 1} parameters, {len(params)} given" + ) + + # Generate storage key prefix + storage_key = StorageKey.create_from_storage_function( + module, + storage_item.value["name"], + params, + runtime_config=runtime.runtime_config, + metadata=runtime.metadata, + ) + prefix = storage_key.to_hex() + + if not start_key: + start_key = prefix + + # Make sure if the max result is smaller than the page size, adjust the page size + if max_results is not None and max_results < page_size: + page_size = max_results + + # Retrieve storage keys + response = await self.rpc_request( + method="state_getKeysPaged", + params=[prefix, page_size, start_key, block_hash], + ) + + if "error" in response: + raise SubstrateRequestException(response["error"]["message"]) + + result_keys = response.get("result") + + result = [] + last_key = None + + def concat_hash_len(key_hasher: str) -> int: + """ + Helper function to avoid if statements + """ + if key_hasher == "Blake2_128Concat": + return 16 + elif key_hasher == "Twox64Concat": + return 8 + elif key_hasher == "Identity": + return 0 + else: + raise ValueError("Unsupported hash type") + + if len(result_keys) > 0: + last_key = result_keys[-1] + + # Retrieve corresponding value + response = await self.rpc_request( + method="state_queryStorageAt", params=[result_keys, block_hash] + ) + + if "error" in response: + raise SubstrateRequestException(response["error"]["message"]) + + for result_group in response["result"]: + for item in result_group["changes"]: + try: + # Determine type string + key_type_string = [] + for n in range(len(params), len(param_types)): + key_type_string.append( + f"[u8; {concat_hash_len(key_hashers[n])}]" + ) + key_type_string.append(param_types[n]) + + item_key_obj = self.substrate.decode_scale( + type_string=f"({', '.join(key_type_string)})", + scale_bytes="0x" + item[0][len(prefix):], + return_scale_obj=True, + block_hash=block_hash, + ) + + # strip key_hashers to use as item key + if len(param_types) - len(params) == 1: + item_key = item_key_obj.value_object[1] + else: + item_key = tuple( + item_key_obj.value_object[key + 1] + for key in range(len(params), len(param_types) + 1, 2) + ) + + except Exception as _: + if not ignore_decoding_errors: + raise + item_key = None + + try: + item_value = self.substrate.decode_scale( + type_string=value_type, + scale_bytes=item[1], + return_scale_obj=True, + block_hash=block_hash, + ) + except Exception as _: + if not ignore_decoding_errors: + raise + item_value = None + + result.append([item_key, item_value]) + + return QueryMapResult( + records=result, + page_size=page_size, + module=module, + storage_function=storage_function, + params=params, + block_hash=block_hash, + substrate=self.substrate, + last_key=last_key, + max_results=max_results, + ignore_decoding_errors=ignore_decoding_errors, + ) + + async def submit_extrinsic( + self, + extrinsic: GenericExtrinsic, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + ) -> "ExtrinsicReceipt": + """ + Submit an extrinsic to the connected node, with the possibility to wait until the extrinsic is included + in a block and/or the block is finalized. The receipt returned provided information about the block and + triggered events + + Parameters + ---------- + extrinsic: Extrinsic The extrinsic to be sent to the network + wait_for_inclusion: wait until extrinsic is included in a block (only works for websocket connections) + wait_for_finalization: wait until extrinsic is finalized (only works for websocket connections) + + Returns + ------- + ExtrinsicReceipt + + """ + + # Check requirements + if not isinstance(extrinsic, GenericExtrinsic): + raise TypeError("'extrinsic' must be of type Extrinsics") + + async def result_handler(message: dict, subscription_id) -> tuple[dict, bool]: + """ + Result handler function passed as an arg to _make_rpc_request as the result_handler + to handle the results of the extrinsic rpc call, which are multipart, and require + subscribing to the message + + :param message: message received from the rpc call + :param subscription_id: subscription id received from the initial rpc call for the subscription + + :returns: tuple containing the dict of the block info for the subscription, and bool for whether + the subscription is completed. + """ + # Check if extrinsic is included and finalized + if "params" in message and isinstance(message["params"]["result"], dict): + # Convert result enum to lower for backwards compatibility + message_result = { + k.lower(): v for k, v in message["params"]["result"].items() + } + + if "finalized" in message_result and wait_for_finalization: + # Created as a task because we don't actually care about the result + self._forgettable_task = asyncio.create_task( + self.rpc_request("author_unwatchExtrinsic", [subscription_id]) + ) + return { + "block_hash": message_result["finalized"], + "extrinsic_hash": "0x{}".format(extrinsic.extrinsic_hash.hex()), + "finalized": True, + }, True + elif ( + "inblock" in message_result + and wait_for_inclusion + and not wait_for_finalization + ): + # Created as a task because we don't actually care about the result + self._forgettable_task = asyncio.create_task( + await self.rpc_request( + "author_unwatchExtrinsic", [subscription_id] + ) + ) + return { + "block_hash": message_result["inblock"], + "extrinsic_hash": "0x{}".format(extrinsic.extrinsic_hash.hex()), + "finalized": False, + }, True + return message, False + + if wait_for_inclusion or wait_for_finalization: + responses = ( + await self._make_rpc_request( + [ + self.make_payload( + "rpc_request", + "author_submitAndWatchExtrinsic", + [str(extrinsic.data)], + ) + ], + result_handler=result_handler, + ) + )["rpc_request"] + response = next( + (r for r in responses if "block_hash" in r and "extrinsic_hash" in r), + None + ) + + if not response: + raise SubstrateRequestException(responses) + + # Also, this will be a multipart response, so maybe should change to everything after the first response? + # The following code implies this will be a single response after the initial subscription id. + result = ExtrinsicReceipt( + substrate=self.substrate, + extrinsic_hash=response["extrinsic_hash"], + block_hash=response["block_hash"], + finalized=response["finalized"], + ) + + else: + response = await self.rpc_request( + "author_submitExtrinsic", [str(extrinsic.data)] + ) + + if "result" not in response: + raise SubstrateRequestException(response.get("error")) + + result = ExtrinsicReceipt( + substrate=self.substrate, extrinsic_hash=response["result"] + ) + + return result + + async def get_metadata_call_function( + self, module_name: str, call_function_name: str, block_hash: str = None + ) -> list: + """ + Retrieves a list of all call functions in metadata active for given block_hash (or chaintip if block_hash + is omitted) + + :param module_name: name of the module + :param call_function_name: name of the call function + :param block_hash: optional block hash + + :return: list of call functions + """ + runtime = await self.init_runtime(block_hash=block_hash) + + for pallet in runtime.metadata.pallets: + if pallet.name == module_name and pallet.calls: + for call in pallet.calls: + if call.name == call_function_name: + return call + + async def get_block_number(self, block_hash: str) -> int: + """Async version of `substrateinterface.base.get_block_number` method.""" + response = await self.rpc_request("chain_getHeader", [block_hash]) + + if "error" in response: + raise SubstrateRequestException(response["error"]["message"]) + + elif "result" in response: + if response["result"]: + return int(response["result"]["number"], 16) + + async def close(self): + """ + Closes the substrate connection, and the websocket connection. + """ + self.substrate.close() + try: + await self.ws.ws.close() + except AttributeError: + pass \ No newline at end of file diff --git a/src/bittensor/balances.py b/src/bittensor/balances.py new file mode 100644 index 00000000..10052a04 --- /dev/null +++ b/src/bittensor/balances.py @@ -0,0 +1,286 @@ +# The MIT License (MIT) +# Copyright © 2021-2022 Yuma Rao +# Copyright © 2022 Opentensor Foundation +# Copyright © 2023 Opentensor Technologies Inc + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from typing import Union + + +class Balance: + """ + Represents the bittensor balance of the wallet, stored as rao (int). + This class provides a way to interact with balances in two different units: rao and tao. + It provides methods to convert between these units, as well as to perform arithmetic and comparison operations. + + Attributes: + unit: A string representing the symbol for the tao unit. + rao_unit: A string representing the symbol for the rao unit. + rao: An integer that stores the balance in rao units. + tao: A float property that gives the balance in tao units. + """ + + unit: str = chr(0x03C4) # This is the tao unit + rao_unit: str = chr(0x03C1) # This is the rao unit + rao: int + tao: float + + def __init__(self, balance: Union[int, float]): + """ + Initialize a Balance object. If balance is an int, it's assumed to be in rao. + If balance is a float, it's assumed to be in tao. + + Args: + balance: The initial balance, in either rao (if an int) or tao (if a float). + """ + if isinstance(balance, int): + self.rao = balance + elif isinstance(balance, float): + # Assume tao value for the float + self.rao = int(balance * pow(10, 9)) + else: + raise TypeError("balance must be an int (rao) or a float (tao)") + + @property + def tao(self): + return self.rao / pow(10, 9) + + def __int__(self): + """ + Convert the Balance object to an int. The resulting value is in rao. + """ + return self.rao + + def __float__(self): + """ + Convert the Balance object to a float. The resulting value is in tao. + """ + return self.tao + + def __str__(self): + """ + Returns the Balance object as a string in the format "symbolvalue", where the value is in tao. + """ + return f"{self.unit}{float(self.tao):,.9f}" + + def __rich__(self): + return "[green]{}[/green][green]{}[/green][green].[/green][dim green]{}[/dim green]".format( + self.unit, + format(float(self.tao), "f").split(".")[0], + format(float(self.tao), "f").split(".")[1], + ) + + def __str_rao__(self): + return f"{self.rao_unit}{int(self.rao)}" + + def __rich_rao__(self): + return f"[green]{self.rao_unit}{int(self.rao)}[/green]" + + def __repr__(self): + return self.__str__() + + def __eq__(self, other: Union[int, float, "Balance"]): + if other is None: + return False + + if hasattr(other, "rao"): + return self.rao == other.rao + else: + try: + # Attempt to cast to int from rao + other_rao = int(other) + return self.rao == other_rao + except (TypeError, ValueError): + raise NotImplementedError("Unsupported type") + + def __ne__(self, other: Union[int, float, "Balance"]): + return not self == other + + def __gt__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return self.rao > other.rao + else: + try: + # Attempt to cast to int from rao + other_rao = int(other) + return self.rao > other_rao + except ValueError: + raise NotImplementedError("Unsupported type") + + def __lt__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return self.rao < other.rao + else: + try: + # Attempt to cast to int from rao + other_rao = int(other) + return self.rao < other_rao + except ValueError: + raise NotImplementedError("Unsupported type") + + def __le__(self, other: Union[int, float, "Balance"]): + try: + return self < other or self == other + except TypeError: + raise NotImplementedError("Unsupported type") + + def __ge__(self, other: Union[int, float, "Balance"]): + try: + return self > other or self == other + except TypeError: + raise NotImplementedError("Unsupported type") + + def __add__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return Balance.from_rao(int(self.rao + other.rao)) + else: + try: + # Attempt to cast to int from rao + return Balance.from_rao(int(self.rao + other)) + except (ValueError, TypeError): + raise NotImplementedError("Unsupported type") + + def __radd__(self, other: Union[int, float, "Balance"]): + try: + return self + other + except TypeError: + raise NotImplementedError("Unsupported type") + + def __sub__(self, other: Union[int, float, "Balance"]): + try: + return self + -other + except TypeError: + raise NotImplementedError("Unsupported type") + + def __rsub__(self, other: Union[int, float, "Balance"]): + try: + return -self + other + except TypeError: + raise NotImplementedError("Unsupported type") + + def __mul__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return Balance.from_rao(int(self.rao * other.rao)) + else: + try: + # Attempt to cast to int from rao + return Balance.from_rao(int(self.rao * other)) + except (ValueError, TypeError): + raise NotImplementedError("Unsupported type") + + def __rmul__(self, other: Union[int, float, "Balance"]): + return self * other + + def __truediv__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return Balance.from_rao(int(self.rao / other.rao)) + else: + try: + # Attempt to cast to int from rao + return Balance.from_rao(int(self.rao / other)) + except (ValueError, TypeError): + raise NotImplementedError("Unsupported type") + + def __rtruediv__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return Balance.from_rao(int(other.rao / self.rao)) + else: + try: + # Attempt to cast to int from rao + return Balance.from_rao(int(other / self.rao)) + except (ValueError, TypeError): + raise NotImplementedError("Unsupported type") + + def __floordiv__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return Balance.from_rao(int(self.tao // other.tao)) + else: + try: + # Attempt to cast to int from rao + return Balance.from_rao(int(self.rao // other)) + except (ValueError, TypeError): + raise NotImplementedError("Unsupported type") + + def __rfloordiv__(self, other: Union[int, float, "Balance"]): + if hasattr(other, "rao"): + return Balance.from_rao(int(other.rao // self.rao)) + else: + try: + # Attempt to cast to int from rao + return Balance.from_rao(int(other // self.rao)) + except (ValueError, TypeError): + raise NotImplementedError("Unsupported type") + + def __int__(self) -> int: + return self.rao + + def __float__(self) -> float: + return self.tao + + def __nonzero__(self) -> bool: + return bool(self.rao) + + def __neg__(self): + return Balance.from_rao(-self.rao) + + def __pos__(self): + return Balance.from_rao(self.rao) + + def __abs__(self): + return Balance.from_rao(abs(self.rao)) + + def to_dict(self) -> dict: + return {"rao": self.rao, "tao": self.tao} + + @staticmethod + def from_float(amount: float): + """ + Given tao (float), return Balance object with rao(int) and tao(float), where rao = int(tao*pow(10,9)) + Args: + amount: The amount in tao. + + Returns: + A Balance object representing the given amount. + """ + rao = int(amount * pow(10, 9)) + return Balance(rao) + + @staticmethod + def from_tao(amount: float): + """ + Given tao (float), return Balance object with rao(int) and tao(float), where rao = int(tao*pow(10,9)) + + Args: + amount: The amount in tao. + + Returns: + A Balance object representing the given amount. + """ + rao = int(amount * pow(10, 9)) + return Balance(rao) + + @staticmethod + def from_rao(amount: int): + """ + Given rao (int), return Balance object with rao(int) and tao(float), where rao = int(tao*pow(10,9)) + + Args: + amount: The amount in rao. + + Returns: + A Balance object representing the given amount. + """ + return Balance(amount) \ No newline at end of file diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py new file mode 100644 index 00000000..c2bb16fe --- /dev/null +++ b/src/subtensor_interface.py @@ -0,0 +1,64 @@ +from typing import Optional + +from bittensor_wallet.utils import SS58_FORMAT + +from src.bittensor.async_substrate_interface import AsyncSubstrateInterface +from src.bittensor.balances import Balance +from src import Constants, defaults + + +class SubtensorInterface: + def __init__(self, network, chain_endpoint): + if chain_endpoint and chain_endpoint != defaults.subtensor.chain_endpoint: + self.chain_endpoint = chain_endpoint + self.network = "local" + elif network and network in Constants.network_map: + self.chain_endpoint = Constants.network_map[network] + self.network = network + else: + self.chain_endpoint = chain_endpoint + self.network = "local" + + self.substrate = AsyncSubstrateInterface( + chain_endpoint=self.chain_endpoint, + ss58_format=SS58_FORMAT + ) + + async def __aenter__(self): + async with self.substrate: + return + + async def __aexit__(self, exc_type, exc_val, exc_tb): + pass + + async def get_balance(self, *addresses, block: Optional[int] = None) -> list[Balance]: + """ + Retrieves the balance for given coldkey(s) + :param addresses: coldkey addresses(s) + :param block: the block number, optional, currently unused + :return: list of Balance objects + """ + results = await self.substrate.query_multiple( + params=[a for a in addresses], + storage_function="Account", + module="System", + ) + print([res for res in results]) + return [Balance(result.value["data"]["free"]) for result in results] + + async def get_total_stake_for_coldkey( + self, *ss58_addresses, block: Optional[int] = None + ) -> Optional["Balance"]: + """ + Returns the total stake held on a coldkey. + + :param ss58_addresses: The SS58 address(es) of the coldkey(s) + :param block: The block number to retrieve the stake from. Currently unused. + :return: + """ + results = await self.substrate.query_multiple( + params=[s for s in ss58_addresses], + module="SubtensorModule", + storage_function="TotalColdkeyStake", + ) + return [Balance.from_rao(r.value) if getattr(r, "value", None) else Balance(0) for r in results] diff --git a/src/utils.py b/src/utils.py index 5d865b1d..d56c1951 100644 --- a/src/utils.py +++ b/src/utils.py @@ -23,9 +23,9 @@ def is_valid_wallet(wallet: Wallet) -> bool: """ return all( [ - wp := os.path.exists(os.path.expanduser(wallet.path)), + os.path.exists(wp := os.path.expanduser(wallet.path)), os.path.exists(os.path.join(wp, wallet.name)), - os.path.isfile(os.path.join(wp, wallet.name, "hotkeys", wallet.hotkey)), + os.path.isfile(os.path.join(wp, wallet.name, "hotkeys", wallet.hotkey_str)), ] ) diff --git a/src/wallets.py b/src/wallets.py index d41d1bb1..0768c551 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -26,6 +26,7 @@ from .utils import console, err_console, RAO_PER_TAO from . import defaults +from src.subtensor_interface import SubtensorInterface async def regen_coldkey( @@ -160,8 +161,7 @@ def _get_coldkey_ss58_addresses_for_path(path: str) -> tuple[list[str], list[str ] -async def wallet_balance(wallet, subtensor, all_balances): - # TODO make use of new NotSubtensor +async def wallet_balance(wallet: Wallet, subtensor: SubtensorInterface, all_balances: bool): if not wallet.coldkeypub_file.exists_on_device(): err_console.print("[bold red]No wallets found.[/bold red]") return @@ -172,11 +172,10 @@ async def wallet_balance(wallet, subtensor, all_balances): coldkeys = [wallet.coldkeypub.ss58_address] wallet_names = [wallet.name] - free_balances = [subtensor.get_balance(coldkeys[i]) for i in range(len(coldkeys))] - - staked_balances = [ - subtensor.get_total_stake_for_coldkey(coldkeys[i]) for i in range(len(coldkeys)) - ] + async with subtensor: + # look into gathering + free_balances = await subtensor.get_balance(*coldkeys) + staked_balances = await subtensor.get_total_stake_for_coldkey(*coldkeys) total_free_balance = sum(free_balances) total_staked_balance = sum(staked_balances) From ad8543eed5c27c274e32fba1d45c7565f1244569 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 23:03:40 +0200 Subject: [PATCH 11/48] no op From 31b72a7ccdfac1afccca91da97f55b97be3936a7 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 23:05:22 +0200 Subject: [PATCH 12/48] no op From e09436c234ca5fcd3240123d6973d09f5aa6d2d1 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 26 Jul 2024 23:07:58 +0200 Subject: [PATCH 13/48] no op From 9d8f07671e2b1539c1ecba7d9820b2cc5dd551d8 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Sun, 28 Jul 2024 18:57:53 +0200 Subject: [PATCH 14/48] Added type registry to AsyncSubstrateInterface instantiation to allow us to actually decode the SCALE objects. Refactored the logic of wallet balance to be faster. --- src/__init__.py | 86 ++++++++++++++++++++++ src/bittensor/async_substrate_interface.py | 2 +- src/subtensor_interface.py | 19 +++-- src/wallets.py | 18 ++--- 4 files changed, 107 insertions(+), 18 deletions(-) diff --git a/src/__init__.py b/src/__init__.py index 33d494a4..14874191 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -77,3 +77,89 @@ class Defaults: debug=False, trace=False, record_log=False, logging_dir="~/.bittensor/miners" ), ) + + +TYPE_REGISTRY = { + "types": { + "Balance": "u64", # Need to override default u128 + }, + "runtime_api": { + "NeuronInfoRuntimeApi": { + "methods": { + "get_neuron_lite": { + "params": [ + { + "name": "netuid", + "type": "u16", + }, + { + "name": "uid", + "type": "u16", + }, + ], + "type": "Vec", + }, + "get_neurons_lite": { + "params": [ + { + "name": "netuid", + "type": "u16", + }, + ], + "type": "Vec", + }, + } + }, + "StakeInfoRuntimeApi": { + "methods": { + "get_stake_info_for_coldkey": { + "params": [ + { + "name": "coldkey_account_vec", + "type": "Vec", + }, + ], + "type": "Vec", + }, + "get_stake_info_for_coldkeys": { + "params": [ + { + "name": "coldkey_account_vecs", + "type": "Vec>", + }, + ], + "type": "Vec", + }, + }, + }, + "ValidatorIPRuntimeApi": { + "methods": { + "get_associated_validator_ip_info_for_subnet": { + "params": [ + { + "name": "netuid", + "type": "u16", + }, + ], + "type": "Vec", + }, + }, + }, + "SubnetInfoRuntimeApi": { + "methods": { + "get_subnet_hyperparams": { + "params": [ + { + "name": "netuid", + "type": "u16", + }, + ], + "type": "Vec", + } + } + }, + "SubnetRegistrationRuntimeApi": { + "methods": {"get_network_registration_cost": {"params": [], "type": "u64"}} + }, + }, +} diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index a49abfad..bf84d64c 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -760,7 +760,7 @@ async def query_multiple( responses = await self._make_rpc_request( all_info, value_scale_type, storage_item, runtime ) - return responses + return {param: responses[p.queryable][0] for (param, p) in zip(params, preprocessed)} async def create_scale_object( self, diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index c2bb16fe..133b402d 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -4,7 +4,7 @@ from src.bittensor.async_substrate_interface import AsyncSubstrateInterface from src.bittensor.balances import Balance -from src import Constants, defaults +from src import Constants, defaults, TYPE_REGISTRY class SubtensorInterface: @@ -21,7 +21,8 @@ def __init__(self, network, chain_endpoint): self.substrate = AsyncSubstrateInterface( chain_endpoint=self.chain_endpoint, - ss58_format=SS58_FORMAT + ss58_format=SS58_FORMAT, + type_registry=TYPE_REGISTRY ) async def __aenter__(self): @@ -31,7 +32,7 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): pass - async def get_balance(self, *addresses, block: Optional[int] = None) -> list[Balance]: + async def get_balance(self, *addresses, block: Optional[int] = None, reuse_block: bool = False) -> dict[str, Balance]: """ Retrieves the balance for given coldkey(s) :param addresses: coldkey addresses(s) @@ -42,13 +43,14 @@ async def get_balance(self, *addresses, block: Optional[int] = None) -> list[Bal params=[a for a in addresses], storage_function="Account", module="System", + reuse_block_hash=reuse_block ) - print([res for res in results]) - return [Balance(result.value["data"]["free"]) for result in results] + return {k: Balance(v.value["data"]["free"]) for (k, v) in results.items()} async def get_total_stake_for_coldkey( - self, *ss58_addresses, block: Optional[int] = None - ) -> Optional["Balance"]: + self, *ss58_addresses, block: Optional[int] = None, + reuse_block: bool = False + ) -> dict[str, Balance]: """ Returns the total stake held on a coldkey. @@ -60,5 +62,6 @@ async def get_total_stake_for_coldkey( params=[s for s in ss58_addresses], module="SubtensorModule", storage_function="TotalColdkeyStake", + reuse_block_hash=reuse_block ) - return [Balance.from_rao(r.value) if getattr(r, "value", None) else Balance(0) for r in results] + return {k: Balance.from_rao(r.value) if getattr(r, "value", None) else Balance(0) for (k, r) in results.items()} diff --git a/src/wallets.py b/src/wallets.py index 0768c551..099b90ed 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -15,7 +15,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. - +import asyncio import os from typing import Optional @@ -174,17 +174,17 @@ async def wallet_balance(wallet: Wallet, subtensor: SubtensorInterface, all_bala async with subtensor: # look into gathering - free_balances = await subtensor.get_balance(*coldkeys) - staked_balances = await subtensor.get_total_stake_for_coldkey(*coldkeys) + free_balances, staked_balances = await asyncio.gather( + subtensor.get_balance(*coldkeys, reuse_block=True), + subtensor.get_total_stake_for_coldkey(*coldkeys, reuse_block=True) + ) - total_free_balance = sum(free_balances) - total_staked_balance = sum(staked_balances) + total_free_balance = sum(free_balances.values()) + total_staked_balance = sum(staked_balances.values()) balances = { - name: (coldkey, free, staked) - for name, coldkey, free, staked in sorted( - zip(wallet_names, coldkeys, free_balances, staked_balances) - ) + name: (coldkey, free_balances[coldkey], staked_balances[coldkey]) + for (name, coldkey) in zip(wallet_names, coldkeys) } table = Table(show_footer=False) From d4dbd7c108bd81ae9e3ee620611f22d9b6f5de48 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Sun, 28 Jul 2024 21:28:33 +0200 Subject: [PATCH 15/48] Added spinner animation for wallet balance. --- src/wallets.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/wallets.py b/src/wallets.py index 099b90ed..52f5aa5a 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -22,6 +22,7 @@ from bittensor_wallet import Wallet from bittensor_wallet.keyfile import Keyfile from rich.table import Table + import typer from .utils import console, err_console, RAO_PER_TAO @@ -166,18 +167,18 @@ async def wallet_balance(wallet: Wallet, subtensor: SubtensorInterface, all_bala err_console.print("[bold red]No wallets found.[/bold red]") return - if all_balances: - coldkeys, wallet_names = _get_coldkey_ss58_addresses_for_path(wallet.path) - else: - coldkeys = [wallet.coldkeypub.ss58_address] - wallet_names = [wallet.name] - - async with subtensor: - # look into gathering - free_balances, staked_balances = await asyncio.gather( - subtensor.get_balance(*coldkeys, reuse_block=True), - subtensor.get_total_stake_for_coldkey(*coldkeys, reuse_block=True) - ) + with console.status("Retrieving balances", spinner="aesthetic"): + if all_balances: + coldkeys, wallet_names = _get_coldkey_ss58_addresses_for_path(wallet.path) + else: + coldkeys = [wallet.coldkeypub.ss58_address] + wallet_names = [wallet.name] + + async with subtensor: + free_balances, staked_balances = await asyncio.gather( + subtensor.get_balance(*coldkeys, reuse_block=True), + subtensor.get_total_stake_for_coldkey(*coldkeys, reuse_block=True) + ) total_free_balance = sum(free_balances.values()) total_staked_balance = sum(staked_balances.values()) From 7b7aa066aa7271509b8726e8a02a946cfc7e8028 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Sun, 28 Jul 2024 21:41:41 +0200 Subject: [PATCH 16/48] Reformatted table in wallet balance. Ran ruff. --- src/bittensor/async_substrate_interface.py | 241 +++++++++++---------- src/bittensor/balances.py | 2 +- src/subtensor_interface.py | 18 +- src/wallets.py | 75 ++++--- 4 files changed, 183 insertions(+), 153 deletions(-) diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index bf84d64c..b7478dc7 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -171,8 +171,8 @@ def is_complete(self): Returns whether all requests in the manager have completed """ return ( - all(info["complete"] for info in self.responses.values()) - and len(self.responses) == self.payloads_count + all(info["complete"] for info in self.responses.values()) + and len(self.responses) == self.payloads_count ) def get_results(self) -> RequestResults: @@ -186,12 +186,12 @@ def get_results(self) -> RequestResults: class Websocket: def __init__( - self, - ws_url: str, - max_subscriptions=1024, - max_connections=100, - shutdown_timer=5, - options: dict = None, + self, + ws_url: str, + max_subscriptions=1024, + max_connections=100, + shutdown_timer=5, + options: dict = None, ): """ Websocket manager object. Allows for the use of a single websocket connection by multiple @@ -334,13 +334,13 @@ class AsyncSubstrateInterface: substrate = None def __init__( - self, - chain_endpoint: str, - use_remote_preset=False, - auto_discover=True, - auto_reconnect=True, - ss58_format=None, - type_registry=None + self, + chain_endpoint: str, + use_remote_preset=False, + auto_discover=True, + auto_reconnect=True, + ss58_format=None, + type_registry=None, ): """ The asyncio-compatible version of the subtensor interface commands we use in bittensor @@ -350,9 +350,9 @@ def __init__( self.ws = Websocket( chain_endpoint, options={ - "max_size": 2 ** 32, - "read_limit": 2 ** 32, - "write_limit": 2 ** 32, + "max_size": 2**32, + "read_limit": 2**32, + "write_limit": 2**32, }, ) self._lock = asyncio.Lock() @@ -410,7 +410,7 @@ def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool): return block_hash if block_hash else (self.last_block_hash if reuse else None) async def init_runtime( - self, block_hash: Optional[str] = None, block_id: Optional[int] = None + self, block_hash: Optional[str] = None, block_id: Optional[int] = None ) -> Runtime: """ This method is used by all other methods that deals with metadata and types defined in the type registry. @@ -430,7 +430,10 @@ async def init_runtime( None, self.substrate.init_runtime, block_hash, block_id ) return Runtime( - self.chain, self.substrate.runtime_config, self.substrate.metadata, self.type_registry + self.chain, + self.substrate.runtime_config, + self.substrate.metadata, + self.type_registry, ) async def get_block_runtime_version(self, block_hash: str) -> dict: @@ -441,7 +444,7 @@ async def get_block_runtime_version(self, block_hash: str) -> dict: return response.get("result") async def get_block_metadata( - self, block_hash=None, decode=True + self, block_hash=None, decode=True ) -> Union[dict, ScaleType]: """ A pass-though to existing JSONRPC method `state_getMetadata`. @@ -479,11 +482,11 @@ async def get_block_metadata( return response async def _preprocess( - self, - query_for: Optional[list], - block_hash: str, - storage_function: str, - module: str, + self, + query_for: Optional[list], + block_hash: str, + storage_function: str, + module: str, ) -> Preprocessed: """ Creates a Preprocessed data object for passing to ``_make_rpc_request`` @@ -530,13 +533,13 @@ async def _preprocess( ) async def _process_response( - self, - response: dict, - subscription_id: Union[int, str], - value_scale_type: str, - storage_item: Optional[ScaleType] = None, - runtime: Optional[Runtime] = None, - result_handler: Optional[ResultHandler] = None, + self, + response: dict, + subscription_id: Union[int, str], + value_scale_type: str, + storage_item: Optional[ScaleType] = None, + runtime: Optional[Runtime] = None, + result_handler: Optional[ResultHandler] = None, ) -> tuple[Union[ScaleType, dict], bool]: obj = response @@ -547,7 +550,7 @@ async def _process_response( self.chain, self.substrate.runtime_config, self.substrate.metadata, - self.type_registry + self.type_registry, ) if response.get("result") is not None: query_value = response.get("result") @@ -573,12 +576,12 @@ async def _process_response( return obj, True async def _make_rpc_request( - self, - payloads: list[dict], - value_scale_type: Optional[str] = None, - storage_item: Optional[ScaleType] = None, - runtime: Optional[Runtime] = None, - result_handler: Optional[ResultHandler] = None, + self, + payloads: list[dict], + value_scale_type: Optional[str] = None, + storage_item: Optional[ScaleType] = None, + runtime: Optional[Runtime] = None, + result_handler: Optional[ResultHandler] = None, ) -> RequestManager.RequestResults: request_manager = RequestManager(payloads) @@ -592,13 +595,13 @@ async def _make_rpc_request( while True: for item_id in request_manager.response_map.keys(): if ( - item_id not in request_manager.responses - or asyncio.iscoroutinefunction(result_handler) + item_id not in request_manager.responses + or asyncio.iscoroutinefunction(result_handler) ): if response := await ws.retrieve(item_id): if ( - asyncio.iscoroutinefunction(result_handler) - and not subscription_added + asyncio.iscoroutinefunction(result_handler) + and not subscription_added ): # handles subscriptions, overwrites the previous mapping of {item_id : payload_id} # with {subscription_id : payload_id} @@ -617,8 +620,8 @@ async def _make_rpc_request( item_id, decoded_response, complete ) if ( - asyncio.iscoroutinefunction(result_handler) - and not subscription_added + asyncio.iscoroutinefunction(result_handler) + and not subscription_added ): subscription_added = True break @@ -645,11 +648,11 @@ def make_payload(id_: str, method: str, params: list) -> dict: } async def rpc_request( - self, - method: str, - params: list, - block_hash: Optional[str] = None, - reuse_block_hash: bool = False, + self, + method: str, + params: list, + block_hash: Optional[str] = None, + reuse_block_hash: bool = False, ) -> Any: """ Makes an RPC request to the subtensor. Use this only if ``self.query`` and ``self.query_multiple`` and @@ -673,7 +676,10 @@ async def rpc_request( ) ] runtime = Runtime( - self.chain, self.substrate.runtime_config, self.substrate.metadata, self.type_registry + self.chain, + self.substrate.runtime_config, + self.substrate.metadata, + self.type_registry, ) result = await self._make_rpc_request(payloads, runtime=runtime) if "error" in result["rpc_request"][0]: @@ -692,11 +698,11 @@ async def get_chain_head(self) -> str: return (await self.rpc_request("chain_getHead", []))["result"] async def compose_call( - self, - call_module: str, - call_function: str, - call_params: dict = None, - block_hash: str = None, + self, + call_module: str, + call_function: str, + call_params: dict = None, + block_hash: str = None, ) -> GenericCall: """ Composes a call payload which can be used in an extrinsic. @@ -719,12 +725,12 @@ async def compose_call( ) async def query_multiple( - self, - params: list, - storage_function: str, - module: str, - block_hash: Optional[str] = None, - reuse_block_hash: bool = False, + self, + params: list, + storage_function: str, + module: str, + block_hash: Optional[str] = None, + reuse_block_hash: bool = False, ) -> RequestManager.RequestResults: """ Queries the subtensor. Only use this when making multiple queries, else use ``self.query`` @@ -760,14 +766,16 @@ async def query_multiple( responses = await self._make_rpc_request( all_info, value_scale_type, storage_item, runtime ) - return {param: responses[p.queryable][0] for (param, p) in zip(params, preprocessed)} + return { + param: responses[p.queryable][0] for (param, p) in zip(params, preprocessed) + } async def create_scale_object( - self, - type_string: str, - data: ScaleBytes = None, - block_hash: str = None, - **kwargs, + self, + type_string: str, + data: ScaleBytes = None, + block_hash: str = None, + **kwargs, ) -> "ScaleType": """ Convenience method to create a SCALE object of type `type_string`, this will initialize the runtime @@ -789,14 +797,14 @@ async def create_scale_object( ) async def create_signed_extrinsic( - self, - call: GenericCall, - keypair: Keypair, - era: dict = None, - nonce: int = None, - tip: int = 0, - tip_asset_id: int = None, - signature: Union[bytes, str] = None, + self, + call: GenericCall, + keypair: Keypair, + era: dict = None, + nonce: int = None, + tip: int = 0, + tip_asset_id: int = None, + signature: Union[bytes, str] = None, ) -> "GenericExtrinsic": """ Creates an extrinsic signed by given account details @@ -826,11 +834,11 @@ async def create_signed_extrinsic( ) async def runtime_call( - self, - api: str, - method: str, - params: Union[list, dict] = None, - block_hash: str = None, + self, + api: str, + method: str, + params: Union[list, dict] = None, + block_hash: str = None, ) -> ScaleType: """ Calls a runtime API method @@ -861,7 +869,7 @@ async def runtime_call( ) if isinstance(params, list) and len(params) != len( - runtime_call_def["params"] + runtime_call_def["params"] ): raise ValueError( f"Number of parameter provided ({len(params)}) does not " @@ -871,7 +879,10 @@ async def runtime_call( # Add runtime API types to registry self.substrate.runtime_config.update_type_registry_types(runtime_api_types) runtime = Runtime( - self.chain, self.substrate.runtime_config, self.substrate.metadata, self.type_registry + self.chain, + self.substrate.runtime_config, + self.substrate.metadata, + self.type_registry, ) # Encode params @@ -916,7 +927,7 @@ async def get_account_nonce(self, account_address: str) -> int: return nonce_obj.value async def get_constant( - self, module_name: str, constant_name: str, block_hash: Optional[str] = None + self, module_name: str, constant_name: str, block_hash: Optional[str] = None ) -> Optional["ScaleType"]: """ Returns the decoded `ScaleType` object of the constant for given module name, call function name and block_hash @@ -939,7 +950,7 @@ async def get_constant( ) async def get_payment_info( - self, call: GenericCall, keypair: Keypair + self, call: GenericCall, keypair: Keypair ) -> dict[str, Any]: """ Retrieves fee estimation via RPC for given extrinsic @@ -982,14 +993,14 @@ async def get_payment_info( return result.value async def query( - self, - module: str, - storage_function: str, - params: Optional[list] = None, - block_hash: Optional[str] = None, - raw_storage_key: bytes = None, - subscription_handler=None, - reuse_block_hash: bool = False, + self, + module: str, + storage_function: str, + params: Optional[list] = None, + block_hash: Optional[str] = None, + raw_storage_key: bytes = None, + subscription_handler=None, + reuse_block_hash: bool = False, ) -> "ScaleType": """ Queries subtensor. This should only be used when making a single request. For multiple requests, @@ -1027,16 +1038,16 @@ async def query( return responses[preprocessed.queryable][0] async def query_map( - self, - module: str, - storage_function: str, - params: Optional[list] = None, - block_hash: Optional[str] = None, - max_results: int = None, - start_key: str = None, - page_size: int = 100, - ignore_decoding_errors: bool = True, - reuse_block_hash: bool = False, + self, + module: str, + storage_function: str, + params: Optional[list] = None, + block_hash: Optional[str] = None, + max_results: int = None, + start_key: str = None, + page_size: int = 100, + ignore_decoding_errors: bool = True, + reuse_block_hash: bool = False, ) -> "QueryMapResult": """ Iterates over all key-pairs located at the given module and storage_function. The storage @@ -1170,7 +1181,7 @@ def concat_hash_len(key_hasher: str) -> int: item_key_obj = self.substrate.decode_scale( type_string=f"({', '.join(key_type_string)})", - scale_bytes="0x" + item[0][len(prefix):], + scale_bytes="0x" + item[0][len(prefix) :], return_scale_obj=True, block_hash=block_hash, ) @@ -1217,10 +1228,10 @@ def concat_hash_len(key_hasher: str) -> int: ) async def submit_extrinsic( - self, - extrinsic: GenericExtrinsic, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = False, + self, + extrinsic: GenericExtrinsic, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, ) -> "ExtrinsicReceipt": """ Submit an extrinsic to the connected node, with the possibility to wait until the extrinsic is included @@ -1273,9 +1284,9 @@ async def result_handler(message: dict, subscription_id) -> tuple[dict, bool]: "finalized": True, }, True elif ( - "inblock" in message_result - and wait_for_inclusion - and not wait_for_finalization + "inblock" in message_result + and wait_for_inclusion + and not wait_for_finalization ): # Created as a task because we don't actually care about the result self._forgettable_task = asyncio.create_task( @@ -1305,7 +1316,7 @@ async def result_handler(message: dict, subscription_id) -> tuple[dict, bool]: )["rpc_request"] response = next( (r for r in responses if "block_hash" in r and "extrinsic_hash" in r), - None + None, ) if not response: @@ -1335,7 +1346,7 @@ async def result_handler(message: dict, subscription_id) -> tuple[dict, bool]: return result async def get_metadata_call_function( - self, module_name: str, call_function_name: str, block_hash: str = None + self, module_name: str, call_function_name: str, block_hash: str = None ) -> list: """ Retrieves a list of all call functions in metadata active for given block_hash (or chaintip if block_hash @@ -1374,4 +1385,4 @@ async def close(self): try: await self.ws.ws.close() except AttributeError: - pass \ No newline at end of file + pass diff --git a/src/bittensor/balances.py b/src/bittensor/balances.py index 10052a04..5d7d9347 100644 --- a/src/bittensor/balances.py +++ b/src/bittensor/balances.py @@ -283,4 +283,4 @@ def from_rao(amount: int): Returns: A Balance object representing the given amount. """ - return Balance(amount) \ No newline at end of file + return Balance(amount) diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 133b402d..5addfaf2 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -22,7 +22,7 @@ def __init__(self, network, chain_endpoint): self.substrate = AsyncSubstrateInterface( chain_endpoint=self.chain_endpoint, ss58_format=SS58_FORMAT, - type_registry=TYPE_REGISTRY + type_registry=TYPE_REGISTRY, ) async def __aenter__(self): @@ -32,7 +32,9 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): pass - async def get_balance(self, *addresses, block: Optional[int] = None, reuse_block: bool = False) -> dict[str, Balance]: + async def get_balance( + self, *addresses, block: Optional[int] = None, reuse_block: bool = False + ) -> dict[str, Balance]: """ Retrieves the balance for given coldkey(s) :param addresses: coldkey addresses(s) @@ -43,13 +45,12 @@ async def get_balance(self, *addresses, block: Optional[int] = None, reuse_block params=[a for a in addresses], storage_function="Account", module="System", - reuse_block_hash=reuse_block + reuse_block_hash=reuse_block, ) return {k: Balance(v.value["data"]["free"]) for (k, v) in results.items()} async def get_total_stake_for_coldkey( - self, *ss58_addresses, block: Optional[int] = None, - reuse_block: bool = False + self, *ss58_addresses, block: Optional[int] = None, reuse_block: bool = False ) -> dict[str, Balance]: """ Returns the total stake held on a coldkey. @@ -62,6 +63,9 @@ async def get_total_stake_for_coldkey( params=[s for s in ss58_addresses], module="SubtensorModule", storage_function="TotalColdkeyStake", - reuse_block_hash=reuse_block + reuse_block_hash=reuse_block, ) - return {k: Balance.from_rao(r.value) if getattr(r, "value", None) else Balance(0) for (k, r) in results.items()} + return { + k: Balance.from_rao(r.value) if getattr(r, "value", None) else Balance(0) + for (k, r) in results.items() + } diff --git a/src/wallets.py b/src/wallets.py index 52f5aa5a..7cafaed2 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -21,7 +21,7 @@ from bittensor_wallet import Wallet from bittensor_wallet.keyfile import Keyfile -from rich.table import Table +from rich.table import Table, Column import typer @@ -162,7 +162,9 @@ def _get_coldkey_ss58_addresses_for_path(path: str) -> tuple[list[str], list[str ] -async def wallet_balance(wallet: Wallet, subtensor: SubtensorInterface, all_balances: bool): +async def wallet_balance( + wallet: Wallet, subtensor: SubtensorInterface, all_balances: bool +): if not wallet.coldkeypub_file.exists_on_device(): err_console.print("[bold red]No wallets found.[/bold red]") return @@ -177,7 +179,7 @@ async def wallet_balance(wallet: Wallet, subtensor: SubtensorInterface, all_bala async with subtensor: free_balances, staked_balances = await asyncio.gather( subtensor.get_balance(*coldkeys, reuse_block=True), - subtensor.get_total_stake_for_coldkey(*coldkeys, reuse_block=True) + subtensor.get_total_stake_for_coldkey(*coldkeys, reuse_block=True), ) total_free_balance = sum(free_balances.values()) @@ -188,33 +190,51 @@ async def wallet_balance(wallet: Wallet, subtensor: SubtensorInterface, all_bala for (name, coldkey) in zip(wallet_names, coldkeys) } - table = Table(show_footer=False) - table.title = "[white]Wallet Coldkey Balances" - table.add_column( - "[white]Wallet Name", - header_style="overline white", - footer_style="overline white", - style="rgb(50,163,219)", - no_wrap=True, - ) - - table.add_column( - "[white]Coldkey Address", - header_style="overline white", - footer_style="overline white", - style="rgb(50,163,219)", - no_wrap=True, - ) - - for type_str in ["Free", "Staked", "Total"]: - table.add_column( - f"[white]{type_str} Balance", + table = Table( + Column( + "[white]Wallet Name", + header_style="overline white", + footer_style="overline white", + style="rgb(50,163,219)", + no_wrap=True, + ), + Column( + "[white]Coldkey Address", + header_style="overline white", + footer_style="overline white", + style="rgb(50,163,219)", + no_wrap=True, + ), + Column( + "[white]Free Balance", header_style="overline white", footer_style="overline white", justify="right", style="green", no_wrap=True, - ) + ), + Column( + "[white]Staked Balance", + header_style="overline white", + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ), + Column( + "[white]Total Balance", + header_style="overline white", + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ), + show_footer=True, + title="[white]Wallet Coldkey Balances", + box=None, + pad_edge=False, + width=None, + ) for name, (coldkey, free, staked) in balances.items(): table.add_row( @@ -232,9 +252,4 @@ async def wallet_balance(wallet: Wallet, subtensor: SubtensorInterface, all_bala str(total_staked_balance), str(total_free_balance + total_staked_balance), ) - table.show_footer = True - - table.box = None - table.pad_edge = False - table.width = None console.print(table) From 16ea35e6fd54e383420586ace014d247b695e4ad Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 29 Jul 2024 15:46:21 +0200 Subject: [PATCH 17/48] More wallet commands ported. Improved wallet validation. Improved reuse of block hash in AsyncSubstrateInterface. --- cli.py | 97 ++++++++++-- src/bittensor/async_substrate_interface.py | 5 +- src/subtensor_interface.py | 3 + src/utils.py | 18 ++- src/wallets.py | 166 +++++++++++++++++++++ 5 files changed, 263 insertions(+), 26 deletions(-) diff --git a/cli.py b/cli.py index eb387f55..f9d9187a 100755 --- a/cli.py +++ b/cli.py @@ -4,6 +4,7 @@ from bittensor_wallet import Wallet import rich +from rich.prompt import Confirm, Prompt import typer from src import wallets, defaults, utils @@ -12,17 +13,13 @@ # re-usable args class Options: - wallet_name = typer.Option( - defaults.wallet.name, "--wallet-name", "-w", help="Name of wallet" - ) + wallet_name = typer.Option(None, "--wallet-name", "-w", help="Name of wallet") wallet_path = typer.Option( - defaults.wallet.path, "--wallet-path", "-p", help="Filepath of root of wallets" - ) - wallet_hotkey = typer.Option( - defaults.wallet.hotkey, "--hotkey", "-H", help="Hotkey of wallet" + None, "--wallet-path", "-p", help="Filepath of root of wallets" ) + wallet_hotkey = typer.Option(None, "--hotkey", "-H", help="Hotkey of wallet") wallet_hk_req = typer.Option( - defaults.wallet.hotkey, + None, "--hotkey", "-H", help="Hotkey name of wallet", @@ -72,15 +69,17 @@ class Options: def get_n_words(n_words: Optional[int]) -> int: while n_words not in [12, 15, 18, 21, 24]: - n_words = typer.prompt( - "Choose number of words: 12, 15, 18, 21, 24", type=int, default=12 + n_words: int = Prompt.ask( + "Choose number of words: 12, 15, 18, 21, 24", + choices=[12, 15, 18, 21, 24], + default=12, ) return n_words def get_creation_data(mnemonic, seed, json, json_password): if not mnemonic and not seed and not json: - prompt_answer = typer.prompt("Enter mnemonic, seed, or json file location") + prompt_answer = Prompt.ask("Enter mnemonic, seed, or json file location") if prompt_answer.startswith("0x"): seed = prompt_answer elif len(prompt_answer.split(" ")) > 1: @@ -88,7 +87,7 @@ def get_creation_data(mnemonic, seed, json, json_password): else: json = prompt_answer if json and not json_password: - json_password = typer.prompt("Enter json backup password", hide_input=True) + json_password = Prompt.ask("Enter json backup password", password=True) return mnemonic, seed, json, json_password @@ -116,6 +115,7 @@ def __init__(self): self.wallet_app.command("new-coldkey")(self.wallet_new_coldkey) self.wallet_app.command("create")(self.wallet_create_wallet) self.wallet_app.command("balance")(self.wallet_balance) + self.wallet_app.command("history")(self.wallet_history) # delegates commands self.delegates_app.command("list")(self.delegates_list) @@ -141,20 +141,58 @@ def wallet_ask( ): # TODO Wallet(config) if not any([wallet_name, wallet_path, wallet_hotkey]): - wallet_name = typer.prompt("Enter wallet name:") + wallet_name = typer.prompt("Enter wallet name") wallet = Wallet(name=wallet_name) else: wallet = Wallet(name=wallet_name, hotkey=wallet_hotkey, path=wallet_path) if validate: - if not utils.is_valid_wallet(wallet): + valid = utils.is_valid_wallet(wallet) + print("valid", valid) + if not valid[0]: utils.err_console.print( f"[red]Error: Wallet does not appear valid. Please verify your wallet information: {wallet}[/red]" ) raise typer.Exit() + elif not valid[1]: + if not Confirm.ask( + f"[yellow]Warning: Wallet appears valid, but hotkey '{wallet.hotkey_str}' does not. Proceed?" + ): + raise typer.Exit() return wallet - def wallet_list(self, network: str = typer.Option("local", help="Network name")): - asyncio.run(wallets.WalletListCommand.run(self.not_subtensor, network)) + @staticmethod + def wallet_list( + wallet_path: str = typer.Option( + defaults.wallet.path, + "--wallet-path", + "-p", + help="Filepath of root of wallets", + prompt=True + ), + ): + """ + # wallet list + Executes the `list` command which enumerates all wallets and their respective hotkeys present in the user's + Bittensor configuration directory. The command organizes the information in a tree structure, displaying each + wallet along with the `ss58` addresses for the coldkey public key and any hotkeys associated with it. + The output is presented in a hierarchical tree format, with each wallet as a root node, + and any associated hotkeys as child nodes. The ``ss58`` address is displayed for each + coldkey and hotkey that is not encrypted and exists on the device. + + ## Usage: + Upon invocation, the command scans the wallet directory and prints a list of all wallets, indicating whether the + public keys are available (`?` denotes unavailable or encrypted keys). + + ### Example usage: + ``` + btcli wallet list --path ~/.bittensor + ``` + + #### Note: + This command is read-only and does not modify the filesystem or the network state. It is intended for use within + the Bittensor CLI to provide a quick overview of the user's wallets. + """ + asyncio.run(wallets.wallet_list(wallet_path)) def wallet_regen_coldkey( self, @@ -461,6 +499,33 @@ def wallet_balance( wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) asyncio.run(wallets.wallet_balance(wallet, subtensor, all_balances)) + def wallet_history( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + ): + """ + # wallet history + Executes the `history` command to fetch the latest transfers of the provided wallet on the Bittensor network. + This command provides a detailed view of the transfers carried out on the wallet. + + ## Usage: + The command lists the latest transfers of the provided wallet, showing the From, To, Amount, Extrinsic Id and + Block Number. + + ### Example usage: + ``` + btcli wallet history + ``` + + #### Note: + This command is essential for users to monitor their financial status on the Bittensor network. + It helps in fetching info on all the transfers so that user can easily tally and cross check the transactions. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + asyncio.run(wallets.wallet_history(wallet)) + def delegates_list( self, wallet_name: Optional[str] = typer.Option(None, help="Wallet name"), diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index b7478dc7..cddd99b3 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -695,7 +695,8 @@ async def get_block_hash(self, block_id: int) -> str: return (await self.rpc_request("chain_getBlockHash", [block_id]))["result"] async def get_chain_head(self) -> str: - return (await self.rpc_request("chain_getHead", []))["result"] + self.last_block_hash = (await self.rpc_request("chain_getHead", []))["result"] + return self.last_block_hash async def compose_call( self, @@ -731,7 +732,7 @@ async def query_multiple( module: str, block_hash: Optional[str] = None, reuse_block_hash: bool = False, - ) -> RequestManager.RequestResults: + ) -> dict[str, ScaleType]: """ Queries the subtensor. Only use this when making multiple queries, else use ``self.query`` """ diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 5addfaf2..937622e3 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -32,6 +32,9 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): pass + async def get_chain_head(self): + return await self.substrate.get_chain_head() + async def get_balance( self, *addresses, block: Optional[int] = None, reuse_block: bool = False ) -> dict[str, Balance]: diff --git a/src/utils.py b/src/utils.py index d56c1951..8d828010 100644 --- a/src/utils.py +++ b/src/utils.py @@ -15,18 +15,20 @@ U64_MAX = 18446744073709551615 -def is_valid_wallet(wallet: Wallet) -> bool: +def is_valid_wallet(wallet: Wallet) -> tuple[bool, bool]: """ Verifies that the wallet with specified parameters. :param wallet: a Wallet instance - :return: bool, whether the wallet appears valid + :return: tuple[bool], whether wallet appears valid, whether valid hotkey in wallet """ - return all( - [ - os.path.exists(wp := os.path.expanduser(wallet.path)), - os.path.exists(os.path.join(wp, wallet.name)), - os.path.isfile(os.path.join(wp, wallet.name, "hotkeys", wallet.hotkey_str)), - ] + return ( + all( + [ + os.path.exists(wp := os.path.expanduser(wallet.path)), + os.path.exists(os.path.join(wp, wallet.name)), + ] + ), + os.path.isfile(os.path.join(wp, wallet.name, "hotkeys", wallet.hotkey_str)), ) diff --git a/src/wallets.py b/src/wallets.py index 7cafaed2..6f9bd0eb 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -17,11 +17,14 @@ import asyncio import os +from pathlib import Path from typing import Optional +import aiohttp from bittensor_wallet import Wallet from bittensor_wallet.keyfile import Keyfile from rich.table import Table, Column +from rich.tree import Tree import typer @@ -177,6 +180,7 @@ async def wallet_balance( wallet_names = [wallet.name] async with subtensor: + await subtensor.get_chain_head() free_balances, staked_balances = await asyncio.gather( subtensor.get_balance(*coldkeys, reuse_block=True), subtensor.get_total_stake_for_coldkey(*coldkeys, reuse_block=True), @@ -253,3 +257,165 @@ async def wallet_balance( str(total_free_balance + total_staked_balance), ) console.print(table) + + +async def get_wallet_transfers(wallet_address: str) -> list[dict]: + """Get all transfers associated with the provided wallet address.""" + + api_url = "https://api.subquery.network/sq/TaoStats/bittensor-indexer" + max_txn = 1000 + graphql_query = """ + query ($first: Int!, $after: Cursor, $filter: TransferFilter, $order: [TransfersOrderBy!]!) { + transfers(first: $first, after: $after, filter: $filter, orderBy: $order) { + nodes { + id + from + to + amount + extrinsicId + blockNumber + } + pageInfo { + endCursor + hasNextPage + hasPreviousPage + } + totalCount + } + } + """ + variables = { + "first": max_txn, + "filter": { + "or": [ + {"from": {"equalTo": wallet_address}}, + {"to": {"equalTo": wallet_address}}, + ] + }, + "order": "BLOCK_NUMBER_DESC", + } + async with aiohttp.ClientSession() as session: + response = await session.post( + api_url, json={"query": graphql_query, "variables": variables} + ) + data = await response.json() + + # Extract nodes and pageInfo from the response + transfer_data = data.get("data", {}).get("transfers", {}) + transfers = transfer_data.get("nodes", []) + + return transfers + + +def create_transfer_history_table(transfers: list[dict]) -> Table: + """Get output transfer table""" + + # Define the column names + column_names = [ + "Id", + "From", + "To", + "Amount (Tao)", + "Extrinsic Id", + "Block Number", + "URL (taostats)", + ] + taostats_url_base = "https://x.taostats.io/extrinsic" + + # Create a table + table = Table( + show_footer=True, + box=None, + pad_edge=False, + width=None, + title="[white]Wallet Transfers", + header_style="overline white", + footer_style="overline white", + ) + + column_style = "rgb(50,163,219)" + no_wrap = True + + for column_name in column_names: + table.add_column( + f"[white]{column_name}", + style=column_style, + no_wrap=no_wrap, + justify="left" if column_name == "Id" else "right", + ) + + for item in transfers: + try: + tao_amount = int(item["amount"]) / RAO_PER_TAO + except ValueError: + tao_amount = item["amount"] + table.add_row( + item["id"], + item["from"], + item["to"], + f"{tao_amount:.3f}", + str(item["extrinsicId"]), + item["blockNumber"], + f"{taostats_url_base}/{item['blockNumber']}-{item['extrinsicId']}", + ) + table.add_row() + return table + + +async def wallet_history(wallet: Wallet): + """Check the transfer history of the provided wallet.""" + wallet_address = wallet.get_coldkeypub().ss58_address + transfers = await get_wallet_transfers(wallet_address) + table = create_transfer_history_table(transfers) + console.print(table) + + +async def wallet_list(wallet_path: str): + r"""Lists wallets.""" + wallet_path = Path(wallet_path).expanduser() + wallets = [ + directory.name for directory in wallet_path.iterdir() if directory.is_dir() + ] + if not wallets: + err_console.print(f"[red]No wallets found in dir: {wallet_path}[/red]") + + root = Tree("Wallets") + for w_name in wallets: + wallet_for_name = Wallet(path=str(wallet_path), name=w_name) + if ( + wallet_for_name.coldkeypub_file.exists_on_device() + and not wallet_for_name.coldkeypub_file.is_encrypted() + ): + coldkeypub_str = wallet_for_name.coldkeypub.ss58_address + else: + coldkeypub_str = "?" + + wallet_tree = root.add("\n[bold white]{} ({})".format(w_name, coldkeypub_str)) + hotkeys_path = wallet_path / w_name / "hotkeys" + try: + hotkeys = [entry.name for entry in hotkeys_path.iterdir()] + if len(hotkeys) > 1: + for h_name in hotkeys: + hotkey_for_name = Wallet( + path=str(wallet_path), name=w_name, hotkey=h_name + ) + try: + if ( + hotkey_for_name.hotkey_file.exists_on_device() + and not hotkey_for_name.hotkey_file.is_encrypted() + ): + hotkey_str = hotkey_for_name.hotkey.ss58_address + else: + hotkey_str = "?" + wallet_tree.add(f"[bold grey]{h_name} ({hotkey_str})") + except UnicodeDecodeError: # usually an unrelated file like .DS_Store + continue + + except FileNotFoundError: + # no hotkeys found + continue + + if not wallets: + root.add("[bold red]No wallets found.") + + console.print(root) From 10e692edda266b8531fbb2a08eca203f966f4996 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 29 Jul 2024 22:51:57 +0200 Subject: [PATCH 18/48] [WIP] Check-in. Overview working with all-but encrypted keys. --- cli.py | 121 +++++- src/bittensor/chain_data.py | 448 ++++++++++++++++++++++ src/bittensor/networking.py | 15 + src/subtensor_interface.py | 302 ++++++++++++++- src/utils.py | 87 +++++ src/wallets.py | 714 ++++++++++++++++++++++++++++++++++-- 6 files changed, 1641 insertions(+), 46 deletions(-) create mode 100644 src/bittensor/chain_data.py create mode 100644 src/bittensor/networking.py diff --git a/cli.py b/cli.py index f9d9187a..2e3144bc 100755 --- a/cli.py +++ b/cli.py @@ -116,6 +116,7 @@ def __init__(self): self.wallet_app.command("create")(self.wallet_create_wallet) self.wallet_app.command("balance")(self.wallet_balance) self.wallet_app.command("history")(self.wallet_history) + self.wallet_app.command("overview")(self.wallet_overview) # delegates commands self.delegates_app.command("list")(self.delegates_list) @@ -129,7 +130,7 @@ def initialize_chain( ): if not self.not_subtensor: self.not_subtensor = SubtensorInterface(network, chain) - typer.echo(f"Initialized with {self.not_subtensor}") + # typer.echo(f"Initialized with {self.not_subtensor}") @staticmethod def wallet_ask( @@ -147,7 +148,6 @@ def wallet_ask( wallet = Wallet(name=wallet_name, hotkey=wallet_hotkey, path=wallet_path) if validate: valid = utils.is_valid_wallet(wallet) - print("valid", valid) if not valid[0]: utils.err_console.print( f"[red]Error: Wallet does not appear valid. Please verify your wallet information: {wallet}[/red]" @@ -167,13 +167,15 @@ def wallet_list( "--wallet-path", "-p", help="Filepath of root of wallets", - prompt=True + prompt=True, ), ): """ # wallet list Executes the `list` command which enumerates all wallets and their respective hotkeys present in the user's - Bittensor configuration directory. The command organizes the information in a tree structure, displaying each + Bittensor configuration directory. + + The command organizes the information in a tree structure, displaying each wallet along with the `ss58` addresses for the coldkey public key and any hotkeys associated with it. The output is presented in a hierarchical tree format, with each wallet as a root node, and any associated hotkeys as child nodes. The ``ss58`` address is displayed for each @@ -194,6 +196,110 @@ def wallet_list( """ asyncio.run(wallets.wallet_list(wallet_path)) + def wallet_overview( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + all_wallets: Optional[bool] = typer.Option( + False, "--all", "-a", help="View overview for all wallets" + ), + sort_by: Optional[str] = typer.Option( + None, + help="Sort the hotkeys by the specified column title (e.g. name, uid, axon).", + ), + sort_order: Optional[str] = typer.Option( + None, + help="Sort the hotkeys in the specified ordering. (ascending/asc or descending/desc/reverse)", + ), + include_hotkeys: Optional[list[str]] = typer.Option( + [], + help="Specify the hotkeys to include by name or ss58 address. (e.g. `hk1 hk2 hk3`). " + "If left empty, all hotkeys not excluded will be included.", + ), + exclude_hotkeys: Optional[list[str]] = typer.Option( + [], + help="Specify the hotkeys to exclude by name or ss58 address. (e.g. `hk1 hk2 hk3`). " + "If left empty, and no hotkeys included in --include-hotkeys, all hotkeys will be included.", + ), + netuids: Optional[list[str]] = typer.Option( + [], help="Set the netuid(s) to filter by." + ), + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + ): + """ + # wallet overview + Executes the `overview` command to present a detailed overview of the user's registered accounts on the + Bittensor network. + + This command compiles and displays comprehensive information about each neuron associated with the user's + wallets, including both hotkeys and coldkeys. It is especially useful for users managing multiple accounts or + seeking a summary of their network activities and stake distributions. + + ## Usage: + The command offers various options to customize the output. Users can filter the displayed data by specific + netuids, sort by different criteria, and choose to include all wallets in the user's configuration directory. + The output is presented in a tabular format with the following columns: + + - COLDKEY: The SS58 address of the coldkey. + - HOTKEY: The SS58 address of the hotkey. + - UID: Unique identifier of the neuron. + - ACTIVE: Indicates if the neuron is active. + - STAKE(τ): Amount of stake in the neuron, in Tao. + - RANK: The rank of the neuron within the network. + - TRUST: Trust score of the neuron. + - CONSENSUS: Consensus score of the neuron. + - INCENTIVE: Incentive score of the neuron. + - DIVIDENDS: Dividends earned by the neuron. + - EMISSION(p): Emission received by the neuron, in Rho. + - VTRUST: Validator trust score of the neuron. + - VPERMIT: Indicates if the neuron has a validator permit. + - UPDATED: Time since last update. + - AXON: IP address and port of the neuron. + - HOTKEY_SS58: Human-readable representation of the hotkey. + + ### Example usage: + ``` + btcli wallet overview + ``` + ``` + btcli wallet overview --all --sort-by stake --sort-order descending + ``` + ``` + btcli wallet overview --include-hotkeys hk1 hk2 --sort-by stake + ``` + + #### Note: + This command is read-only and does not modify the network state or account configurations. It provides a quick and + comprehensive view of the user's network presence, making it ideal for monitoring account status, stake distribution, + and overall contribution to the Bittensor network. + """ + # TODO does not yet work with encrypted keys + if include_hotkeys and exclude_hotkeys: + utils.err_console.print( + "[red]You have specified hotkeys for inclusion and exclusion. Pick only one or neither." + ) + raise typer.Exit() + # if all-wallets is entered, ask for path + if all_wallets: + if not wallet_path: + wallet_path = Prompt.ask("Enter the path of the wallets", default=defaults.wallet.path) + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + self.initialize_chain(network, chain) + asyncio.run( + wallets.overview( + wallet, + self.not_subtensor, + all_wallets, + sort_by, + sort_order, + include_hotkeys, + exclude_hotkeys, + netuids_filter=netuids, + ) + ) + def wallet_regen_coldkey( self, wallet_name: Optional[str] = Options.wallet_name, @@ -209,6 +315,7 @@ def wallet_regen_coldkey( """ # wallet regen-coldkey Executes the `regen-coldkey` command to regenerate a coldkey for a wallet on the Bittensor network. + This command is used to create a new coldkey from an existing mnemonic, seed, or JSON file. ## Usage: @@ -257,6 +364,7 @@ def wallet_regen_coldkey_pub( # wallet regen-coldkeypub Executes the `regen-coldkeypub` command to regenerate the public part of a coldkey (coldkeypub) for a wallet on the Bittensor network. + This command is used when a user needs to recreate their coldkeypub from an existing public key or SS58 address. ## Usage: @@ -308,6 +416,7 @@ def wallet_regen_hotkey( """ # wallet regen-hotkey Executes the `regen-hotkey` command to regenerate a hotkey for a wallet on the Bittensor network. + Similar to regenerating a coldkey, this command creates a new hotkey from a mnemonic, seed, or JSON file. ## Usage: @@ -421,7 +530,9 @@ def wallet_create_wallet( """ # wallet create Executes the `create` command to generate both a new coldkey and hotkey under a specified wallet on the - Bittensor network. This command is a comprehensive utility for creating a complete wallet setup with both cold + Bittensor network. + + This command is a comprehensive utility for creating a complete wallet setup with both cold and hotkeys. diff --git a/src/bittensor/chain_data.py b/src/bittensor/chain_data.py new file mode 100644 index 00000000..ef594299 --- /dev/null +++ b/src/bittensor/chain_data.py @@ -0,0 +1,448 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Any, Union + +from scalecodec import ScaleBytes +from scalecodec.base import RuntimeConfiguration +from scalecodec.type_registry import load_type_registry_preset +from scalecodec.utils.ss58 import ss58_encode + +from src.bittensor.balances import Balance +from src.utils import RAO_PER_TAO, SS58_FORMAT, u16_normalized_float + + +class ChainDataType(Enum): + NeuronInfo = 1 + SubnetInfo = 2 + DelegateInfo = 3 + NeuronInfoLite = 4 + DelegatedInfo = 5 + StakeInfo = 6 + IPInfo = 7 + SubnetHyperparameters = 8 + + +def from_scale_encoding( + input_: Union[list[int], bytes, ScaleBytes], + type_name: ChainDataType, + is_vec: bool = False, + is_option: bool = False, +) -> Optional[dict]: + """ + Decodes input_ data from SCALE encoding based on the specified type name and modifiers. + + Args: + input_ (Union[list[int], bytes, ScaleBytes]): The input_ data to decode. + type_name (ChainDataType): The type of data being decoded. + is_vec (bool, optional): Whether the data is a vector of the specified type. Default is ``False``. + is_option (bool, optional): Whether the data is an optional value of the specified type. Default is ``False``. + + Returns: + Optional[Dict]: The decoded data as a dictionary, or ``None`` if the decoding fails. + """ + type_string = type_name.name + if type_name == ChainDataType.DelegatedInfo: + # DelegatedInfo is a tuple of (DelegateInfo, Compact) + type_string = f"({ChainDataType.DelegateInfo.name}, Compact)" + if is_option: + type_string = f"Option<{type_string}>" + if is_vec: + type_string = f"Vec<{type_string}>" + + return from_scale_encoding_using_type_string(input_, type_string) + + +def from_scale_encoding_using_type_string( + input_: Union[list[int], bytes, ScaleBytes], type_string: str +) -> Optional[dict]: + if isinstance(input_, ScaleBytes): + as_scale_bytes = input_ + else: + if isinstance(input_, list) and all([isinstance(i, int) for i in input_]): + vec_u8 = input_ + as_bytes = bytes(vec_u8) + elif isinstance(input_, bytes): + as_bytes = input_ + else: + raise TypeError("input_ must be a list[int], bytes, or ScaleBytes") + + as_scale_bytes = ScaleBytes(as_bytes) + + rpc_runtime_config = RuntimeConfiguration() + rpc_runtime_config.update_type_registry(load_type_registry_preset("legacy")) + rpc_runtime_config.update_type_registry(custom_rpc_type_registry) + + obj = rpc_runtime_config.create_scale_object(type_string, data=as_scale_bytes) + + return obj.decode() + + +@dataclass +class StakeInfo: + """Dataclass for stake info.""" + + hotkey_ss58: str # Hotkey address + coldkey_ss58: str # Coldkey address + stake: Balance # Stake for the hotkey-coldkey pair + + @classmethod + def fix_decoded_values(cls, decoded: Any) -> "StakeInfo": + """Fixes the decoded values.""" + return cls( + hotkey_ss58=ss58_encode(decoded["hotkey"], SS58_FORMAT), + coldkey_ss58=ss58_encode(decoded["coldkey"], SS58_FORMAT), + stake=Balance.from_rao(decoded["stake"]), + ) + + @classmethod + def from_vec_u8(cls, vec_u8: list[int]) -> Optional["StakeInfo"]: + """Returns a StakeInfo object from a ``vec_u8``.""" + if len(vec_u8) == 0: + return None + + decoded = from_scale_encoding(vec_u8, ChainDataType.StakeInfo) + if decoded is None: + return None + + return StakeInfo.fix_decoded_values(decoded) + + @classmethod + def list_of_tuple_from_vec_u8( + cls, vec_u8: list[int] + ) -> dict[str, list["StakeInfo"]]: + """Returns a list of StakeInfo objects from a ``vec_u8``.""" + decoded: Optional[list[tuple[str, list[object]]]] = ( + from_scale_encoding_using_type_string( + input_=vec_u8, type_string="Vec<(AccountId, Vec)>" + ) + ) + + if decoded is None: + return {} + + return { + ss58_encode(address=account_id, ss58_format=SS58_FORMAT): [ + StakeInfo.fix_decoded_values(d) for d in stake_info + ] + for account_id, stake_info in decoded + } + + @classmethod + def list_from_vec_u8(cls, vec_u8: list[int]) -> list["StakeInfo"]: + """Returns a list of StakeInfo objects from a ``vec_u8``.""" + decoded = from_scale_encoding(vec_u8, ChainDataType.StakeInfo, is_vec=True) + if decoded is None: + return [] + + return [StakeInfo.fix_decoded_values(d) for d in decoded] + + +@dataclass +class NeuronInfoLite: + """Dataclass for neuron metadata, but without the weights and bonds.""" + + hotkey: str + coldkey: str + uid: int + netuid: int + active: int + stake: Balance + # mapping of coldkey to amount staked to this Neuron + stake_dict: dict[str, Balance] + total_stake: Balance + rank: float + emission: float + incentive: float + consensus: float + trust: float + validator_trust: float + dividends: float + last_update: int + validator_permit: bool + prometheus_info: Optional["PrometheusInfo"] + axon_info: "axon_info" + pruning_score: int + is_null: bool = False + + @staticmethod + def get_null_neuron() -> "NeuronInfoLite": + neuron = NeuronInfoLite( + uid=0, + netuid=0, + active=0, + stake=Balance.from_rao(0), + stake_dict={}, + total_stake=Balance.from_rao(0), + rank=0, + emission=0, + incentive=0, + consensus=0, + trust=0, + validator_trust=0, + dividends=0, + last_update=0, + validator_permit=False, + prometheus_info=None, + axon_info=None, + is_null=True, + coldkey="000000000000000000000000000000000000000000000000", + hotkey="000000000000000000000000000000000000000000000000", + pruning_score=0, + ) + return neuron + + +@dataclass +class DelegateInfo: + """ + Dataclass for delegate information. For a lighter version of this class, see :func:`DelegateInfoLite`. + + Args: + hotkey_ss58 (str): Hotkey of the delegate for which the information is being fetched. + total_stake (int): Total stake of the delegate. + nominators (list[Tuple[str, int]]): list of nominators of the delegate and their stake. + take (float): Take of the delegate as a percentage. + owner_ss58 (str): Coldkey of the owner. + registrations (list[int]): list of subnets that the delegate is registered on. + validator_permits (list[int]): list of subnets that the delegate is allowed to validate on. + return_per_1000 (int): Return per 1000 TAO, for the delegate over a day. + total_daily_return (int): Total daily return of the delegate. + + """ + + hotkey_ss58: str # Hotkey of delegate + total_stake: Balance # Total stake of the delegate + nominators: list[ + tuple[str, Balance] + ] # list of nominators of the delegate and their stake + owner_ss58: str # Coldkey of owner + take: float # Take of the delegate as a percentage + validator_permits: list[ + int + ] # list of subnets that the delegate is allowed to validate on + registrations: list[int] # list of subnets that the delegate is registered on + return_per_1000: Balance # Return per 1000 tao of the delegate over a day + total_daily_return: Balance # Total daily return of the delegate + + @classmethod + def fix_decoded_values(cls, decoded: Any) -> "DelegateInfo": + """Fixes the decoded values.""" + + return cls( + hotkey_ss58=ss58_encode(decoded["delegate_ss58"], SS58_FORMAT), + owner_ss58=ss58_encode(decoded["owner_ss58"], SS58_FORMAT), + take=u16_normalized_float(decoded["take"]), + nominators=[ + ( + ss58_encode(nom[0], SS58_FORMAT), + Balance.from_rao(nom[1]), + ) + for nom in decoded["nominators"] + ], + total_stake=Balance.from_rao( + sum([nom[1] for nom in decoded["nominators"]]) + ), + validator_permits=decoded["validator_permits"], + registrations=decoded["registrations"], + return_per_1000=Balance.from_rao(decoded["return_per_1000"]), + total_daily_return=Balance.from_rao(decoded["total_daily_return"]), + ) + + @classmethod + def from_vec_u8(cls, vec_u8: list[int]) -> Optional["DelegateInfo"]: + """Returns a DelegateInfo object from a ``vec_u8``.""" + if len(vec_u8) == 0: + return None + + decoded = from_scale_encoding(vec_u8, ChainDataType.DelegateInfo) + if decoded is None: + return None + + return DelegateInfo.fix_decoded_values(decoded) + + @classmethod + def list_from_vec_u8(cls, vec_u8: list[int]) -> list["DelegateInfo"]: + """Returns a list of DelegateInfo objects from a ``vec_u8``.""" + decoded = from_scale_encoding(vec_u8, ChainDataType.DelegateInfo, is_vec=True) + + if decoded is None: + return [] + + return [DelegateInfo.fix_decoded_values(d) for d in decoded] + + @classmethod + def delegated_list_from_vec_u8( + cls, vec_u8: list[int] + ) -> list[tuple["DelegateInfo", Balance]]: + """Returns a list of Tuples of DelegateInfo objects, and Balance, from a ``vec_u8``. + + This is the list of delegates that the user has delegated to, and the amount of stake delegated. + """ + decoded = from_scale_encoding(vec_u8, ChainDataType.DelegatedInfo, is_vec=True) + if decoded is None: + return [] + + return [ + (DelegateInfo.fix_decoded_values(d), Balance.from_rao(s)) + for d, s in decoded + ] + + +custom_rpc_type_registry = { + "types": { + "SubnetInfo": { + "type": "struct", + "type_mapping": [ + ["netuid", "Compact"], + ["rho", "Compact"], + ["kappa", "Compact"], + ["difficulty", "Compact"], + ["immunity_period", "Compact"], + ["max_allowed_validators", "Compact"], + ["min_allowed_weights", "Compact"], + ["max_weights_limit", "Compact"], + ["scaling_law_power", "Compact"], + ["subnetwork_n", "Compact"], + ["max_allowed_uids", "Compact"], + ["blocks_since_last_step", "Compact"], + ["tempo", "Compact"], + ["network_modality", "Compact"], + ["network_connect", "Vec<[u16; 2]>"], + ["emission_values", "Compact"], + ["burn", "Compact"], + ["owner", "AccountId"], + ], + }, + "DelegateInfo": { + "type": "struct", + "type_mapping": [ + ["delegate_ss58", "AccountId"], + ["take", "Compact"], + ["nominators", "Vec<(AccountId, Compact)>"], + ["owner_ss58", "AccountId"], + ["registrations", "Vec>"], + ["validator_permits", "Vec>"], + ["return_per_1000", "Compact"], + ["total_daily_return", "Compact"], + ], + }, + "NeuronInfo": { + "type": "struct", + "type_mapping": [ + ["hotkey", "AccountId"], + ["coldkey", "AccountId"], + ["uid", "Compact"], + ["netuid", "Compact"], + ["active", "bool"], + ["axon_info", "axon_info"], + ["prometheus_info", "PrometheusInfo"], + ["stake", "Vec<(AccountId, Compact)>"], + ["rank", "Compact"], + ["emission", "Compact"], + ["incentive", "Compact"], + ["consensus", "Compact"], + ["trust", "Compact"], + ["validator_trust", "Compact"], + ["dividends", "Compact"], + ["last_update", "Compact"], + ["validator_permit", "bool"], + ["weights", "Vec<(Compact, Compact)>"], + ["bonds", "Vec<(Compact, Compact)>"], + ["pruning_score", "Compact"], + ], + }, + "NeuronInfoLite": { + "type": "struct", + "type_mapping": [ + ["hotkey", "AccountId"], + ["coldkey", "AccountId"], + ["uid", "Compact"], + ["netuid", "Compact"], + ["active", "bool"], + ["axon_info", "axon_info"], + ["prometheus_info", "PrometheusInfo"], + ["stake", "Vec<(AccountId, Compact)>"], + ["rank", "Compact"], + ["emission", "Compact"], + ["incentive", "Compact"], + ["consensus", "Compact"], + ["trust", "Compact"], + ["validator_trust", "Compact"], + ["dividends", "Compact"], + ["last_update", "Compact"], + ["validator_permit", "bool"], + ["pruning_score", "Compact"], + ], + }, + "axon_info": { + "type": "struct", + "type_mapping": [ + ["block", "u64"], + ["version", "u32"], + ["ip", "u128"], + ["port", "u16"], + ["ip_type", "u8"], + ["protocol", "u8"], + ["placeholder1", "u8"], + ["placeholder2", "u8"], + ], + }, + "PrometheusInfo": { + "type": "struct", + "type_mapping": [ + ["block", "u64"], + ["version", "u32"], + ["ip", "u128"], + ["port", "u16"], + ["ip_type", "u8"], + ], + }, + "IPInfo": { + "type": "struct", + "type_mapping": [ + ["ip", "Compact"], + ["ip_type_and_protocol", "Compact"], + ], + }, + "StakeInfo": { + "type": "struct", + "type_mapping": [ + ["hotkey", "AccountId"], + ["coldkey", "AccountId"], + ["stake", "Compact"], + ], + }, + "SubnetHyperparameters": { + "type": "struct", + "type_mapping": [ + ["rho", "Compact"], + ["kappa", "Compact"], + ["immunity_period", "Compact"], + ["min_allowed_weights", "Compact"], + ["max_weights_limit", "Compact"], + ["tempo", "Compact"], + ["min_difficulty", "Compact"], + ["max_difficulty", "Compact"], + ["weights_version", "Compact"], + ["weights_rate_limit", "Compact"], + ["adjustment_interval", "Compact"], + ["activity_cutoff", "Compact"], + ["registration_allowed", "bool"], + ["target_regs_per_interval", "Compact"], + ["min_burn", "Compact"], + ["max_burn", "Compact"], + ["bonds_moving_avg", "Compact"], + ["max_regs_per_block", "Compact"], + ["serving_rate_limit", "Compact"], + ["max_validators", "Compact"], + ["adjustment_alpha", "Compact"], + ["difficulty", "Compact"], + ["commit_reveal_weights_interval", "Compact"], + ["commit_reveal_weights_enabled", "bool"], + ["alpha_high", "Compact"], + ["alpha_low", "Compact"], + ["liquid_alpha_enabled", "bool"], + ], + }, + } +} diff --git a/src/bittensor/networking.py b/src/bittensor/networking.py new file mode 100644 index 00000000..bf19718d --- /dev/null +++ b/src/bittensor/networking.py @@ -0,0 +1,15 @@ +import netaddr + + +def int_to_ip(int_val: int) -> str: + """Maps an integer to a unique ip-string + Args: + int_val (:type:`int128`, `required`): The integer representation of an ip. Must be in the range (0, 3.4028237e+38). + + Returns: + str_val (:type:`str`, `required): The string representation of an ip. Of form *.*.*.* for ipv4 or *::*:*:*:* for ipv6 + + Raises: + netaddr.core.AddrFormatError (Exception): Raised when the passed int_vals is not a valid ip int value. + """ + return str(netaddr.IPAddress(int_val)) diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 937622e3..32a7a0bf 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -1,10 +1,21 @@ -from typing import Optional +import asyncio +from typing import Optional, Any, Union, TypedDict +import scalecodec from bittensor_wallet.utils import SS58_FORMAT +from scalecodec.base import RuntimeConfiguration +from scalecodec.type_registry import load_type_registry_preset from src.bittensor.async_substrate_interface import AsyncSubstrateInterface +from src.bittensor.chain_data import DelegateInfo, custom_rpc_type_registry, StakeInfo from src.bittensor.balances import Balance from src import Constants, defaults, TYPE_REGISTRY +from src.utils import ss58_to_vec_u8 + + +class ParamWithTypes(TypedDict): + name: str # Name of the parameter. + type: str # ScaleType string of the parameter. class SubtensorInterface: @@ -35,6 +46,189 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def get_chain_head(self): return await self.substrate.get_chain_head() + async def encode_params( + self, + call_definition: list["ParamWithTypes"], + params: Union[list[Any], dict[str, Any]], + ) -> str: + """Returns a hex encoded string of the params using their types.""" + param_data = scalecodec.ScaleBytes(b"") + + for i, param in enumerate(call_definition["params"]): # type: ignore + scale_obj = await self.substrate.create_scale_object(param["type"]) + if type(params) is list: + param_data += scale_obj.encode(params[i]) + else: + if param["name"] not in params: + raise ValueError(f"Missing param {param['name']} in params dict.") + + param_data += scale_obj.encode(params[param["name"]]) + + return param_data.to_hex() + + async def get_all_subnet_netuids( + self, block_hash: Optional[str] = None + ) -> list[int]: + """ + Retrieves the list of all subnet unique identifiers (netuids) currently present in the Bittensor network. + + :param block_hash: The hash of the block to retrieve the subnet unique identifiers from. + :return: A list of subnet netuids. + + This function provides a comprehensive view of the subnets within the Bittensor network, + offering insights into its diversity and scale. + """ + result = await self.substrate.query_map( + module="SubtensorModule", + storage_function="NetworksAdded", + block_hash=block_hash, + reuse_block_hash=True, + ) + return ( + [] + if result is None or not hasattr(result, "records") + else [netuid.value for netuid, exists in result if exists] + ) + + async def is_hotkey_delegate( + self, + hotkey_ss58: str, + block_hash: Optional[int] = None, + reuse_block: Optional[bool] = False, + ) -> bool: + """ + Determines whether a given hotkey (public key) is a delegate on the Bittensor network. This function + checks if the neuron associated with the hotkey is part of the network's delegation system. + + Args: + hotkey_ss58 (str): The SS58 address of the neuron's hotkey. + block_hash (Optional[int], optional): The blockchain block number for the query. + + Returns: + bool: ``True`` if the hotkey is a delegate, ``False`` otherwise. + + Being a delegate is a significant status within the Bittensor network, indicating a neuron's + involvement in consensus and governance processes. + """ + return hotkey_ss58 in [ + info.hotkey_ss58 + for info in await self.get_delegates( + block_hash=block_hash, reuse_block=reuse_block + ) + ] + + async def get_delegates( + self, block_hash: Optional[str] = None, reuse_block: Optional[bool] = False + ): + json_body = await self.substrate.rpc_request( + method="delegateInfo_getDelegates", # custom rpc method + params=[block_hash] if block_hash else [], + reuse_block_hash=reuse_block, + ) + + if not (result := json_body.get("result", None)): + return [] + + return DelegateInfo.list_from_vec_u8(result) + + async def get_stake_info_for_coldkey( + self, + coldkey_ss58: str, + block_hash: Optional[str] = None, + reuse_block: bool = False, + ) -> Optional[list[StakeInfo]]: + """ + Retrieves stake information associated with a specific coldkey. This function provides details + about the stakes held by an account, including the staked amounts and associated delegates. + + Args: + coldkey_ss58 (str): The ``SS58`` address of the account's coldkey. + block (Optional[int], optional): The blockchain block number for the query. + + Returns: + List[StakeInfo]: A list of StakeInfo objects detailing the stake allocations for the account. + + Stake information is vital for account holders to assess their investment and participation + in the network's delegation and consensus processes. + """ + encoded_coldkey = ss58_to_vec_u8(coldkey_ss58) + + hex_bytes_result = await self.query_runtime_api( + runtime_api="StakeInfoRuntimeApi", + method="get_stake_info_for_coldkey", + params=[encoded_coldkey], + block_hash=block_hash, + reuse_block=reuse_block, + ) + + if hex_bytes_result is None: + return None + + if hex_bytes_result.startswith("0x"): + bytes_result = bytes.fromhex(hex_bytes_result[2:]) + else: + bytes_result = bytes.fromhex(hex_bytes_result) + # TODO: review if this is the correct type / works + return StakeInfo.list_from_vec_u8(bytes_result) # type: ignore + + async def query_runtime_api( + self, + runtime_api: str, + method: str, + params: Optional[Union[list[int], dict[str, int]]], + block_hash: Optional[str] = None, + reuse_block: Optional[bool] = False, + ) -> Optional[str]: + """ + Queries the runtime API of the Bittensor blockchain, providing a way to interact with the underlying + runtime and retrieve data encoded in Scale Bytes format. This function is essential for advanced users + who need to interact with specific runtime methods and decode complex data types. + + Args: + runtime_api (str): The name of the runtime API to query. + method (str): The specific method within the runtime API to call. + params (Optional[List[ParamWithTypes]], optional): The parameters to pass to the method call. + block (Optional[int]): The blockchain block number at which to perform the query. + + Returns: + Optional[bytes]: The Scale Bytes encoded result from the runtime API call, or ``None`` if the call fails. + + This function enables access to the deeper layers of the Bittensor blockchain, allowing for detailed + and specific interactions with the network's runtime environment. + """ + call_definition = TYPE_REGISTRY["runtime_api"][runtime_api]["methods"][method] + + data = ( + "0x" + if params is None + else await self.encode_params( + call_definition=call_definition, params=params + ) + ) + api_method = f"{runtime_api}_{method}" + + json_result = await self.substrate.rpc_request( + method="state_call", + params=[api_method, data, block_hash] if block_hash else [api_method, data], + ) + + if json_result is None: + return None + + return_type = call_definition["type"] + + as_scale_bytes = scalecodec.ScaleBytes(json_result["result"]) # type: ignore + + rpc_runtime_config = RuntimeConfiguration() + rpc_runtime_config.update_type_registry(load_type_registry_preset("legacy")) + rpc_runtime_config.update_type_registry(custom_rpc_type_registry) + + obj = rpc_runtime_config.create_scale_object(return_type, as_scale_bytes) + if obj.data.to_hex() == "0x0400": # RPC returned None result + return None + + return obj.decode() + async def get_balance( self, *addresses, block: Optional[int] = None, reuse_block: bool = False ) -> dict[str, Balance]: @@ -42,7 +236,8 @@ async def get_balance( Retrieves the balance for given coldkey(s) :param addresses: coldkey addresses(s) :param block: the block number, optional, currently unused - :return: list of Balance objects + :param reuse_block: Whether to reuse the last-used block hash when retrieving info. + :return: dict of {address: Balance objects} """ results = await self.substrate.query_multiple( params=[a for a in addresses], @@ -60,6 +255,7 @@ async def get_total_stake_for_coldkey( :param ss58_addresses: The SS58 address(es) of the coldkey(s) :param block: The block number to retrieve the stake from. Currently unused. + :param reuse_block: Whether to reuse the last-used block hash when retrieving info. :return: """ results = await self.substrate.query_multiple( @@ -69,6 +265,104 @@ async def get_total_stake_for_coldkey( reuse_block_hash=reuse_block, ) return { - k: Balance.from_rao(r.value) if getattr(r, "value", None) else Balance(0) - for (k, r) in results.items() + k: Balance.from_rao(getattr(r, "value", 0)) for (k, r) in results.items() } + + async def get_netuids_for_hotkey( + self, hotkey_ss58: str, block_hash: Optional[str] = None, reuse_block: bool = False + ) -> list[int]: + """ + Retrieves a list of subnet UIDs (netuids) for which a given hotkey is a member. This function + identifies the specific subnets within the Bittensor network where the neuron associated with + the hotkey is active. + + Args: + hotkey_ss58 (str): The ``SS58`` address of the neuron's hotkey. + block (Optional[int]): The blockchain block number at which to perform the query. + + Returns: + List[int]: A list of netuids where the neuron is a member. + """ + result = await self.substrate.query_map( + module="SubtensorModule", + storage_function="IsNetworkMember", + params=[hotkey_ss58], + block_hash=block_hash, + reuse_block_hash=reuse_block + ) + return ( + [record[0].value for record in result.records if record[1]] + if result and hasattr(result, "records") + else [] + ) + + async def subnet_exists( + self, netuid: int, block_hash: Optional[str] = None, reuse_block: bool = False + ) -> bool: + """ + Checks if a subnet with the specified unique identifier (netuid) exists within the Bittensor network. + + :param netuid: The unique identifier of the subnet. + :param block_hash: The hash of the blockchain block number at which to check the subnet existence. + + :return: `True` if the subnet exists, `False` otherwise. + + This function is critical for verifying the presence of specific subnets in the network, + enabling a deeper understanding of the network's structure and composition. + """ + result = await self.substrate.query( + module="SubtensorModule", + storage_function="NetworksAdded", + params=[netuid], + block_hash=block_hash, + reuse_block_hash=reuse_block, + ) + return getattr(result, "value", False) + + async def get_hyperparameter( + self, param_name: str, netuid: int, block_hash: Optional[str] = None + ) -> Optional[Any]: + """ + Retrieves a specified hyperparameter for a specific subnet. + + :param param_name: The name of the hyperparameter to retrieve. + :param netuid: The unique identifier of the subnet. + :param block_hash: The hash of blockchain block number for the query. + + :return: The value of the specified hyperparameter if the subnet exists, or None + """ + if not await self.subnet_exists(netuid, block_hash): + return None + + result = await self.substrate.query( + module="SubtensorModule", + storage_function=param_name, + params=[netuid], + block_hash=block_hash, + ) + + if result is None or not hasattr(result, "value"): + return None + + return result.value + + async def filter_netuids_by_registered_hotkeys( + self, all_netuids, filter_for_netuids, all_hotkeys, reuse_block: bool = False + ) -> list[int]: + netuids_with_registered_hotkeys = [item for sublist in await asyncio.gather( + *[ + self.get_netuids_for_hotkey(wallet.hotkey.ss58_address, reuse_block=reuse_block) + for wallet in all_hotkeys + ] + ) for item in sublist] + + if not filter_for_netuids: + all_netuids = netuids_with_registered_hotkeys + + else: + all_netuids = [ + netuid for netuid in all_netuids if netuid in filter_for_netuids + ] + all_netuids.extend(netuids_with_registered_hotkeys) + + return list(set(all_netuids)) diff --git a/src/utils.py b/src/utils.py index 8d828010..7f5842e3 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,10 +1,14 @@ import os +from pathlib import Path from typing import Union +import scalecodec from bittensor_wallet import Wallet from bittensor_wallet.keyfile import Keypair from bittensor_wallet.utils import SS58_FORMAT, ss58 from rich.console import Console +from scalecodec.base import RuntimeConfiguration +from scalecodec.type_registry import load_type_registry_preset console = Console() err_console = Console(stderr=True) @@ -15,6 +19,54 @@ U64_MAX = 18446744073709551615 +def get_hotkey_wallets_for_wallet( + wallet: Wallet, show_nulls: bool = False +) -> list[Wallet]: + hotkey_wallets = [] + wallet_path = Path(wallet.path).expanduser() + hotkeys_path = wallet_path / wallet.name / "hotkeys" + try: + hotkeys = [entry.name for entry in hotkeys_path.iterdir()] + except FileNotFoundError: + hotkeys = [] + for h_name in hotkeys: + hotkey_for_name = Wallet(path=str(wallet_path), name=wallet.name, hotkey=h_name) + try: + if ( + hotkey_for_name.hotkey_file.exists_on_device() + and not hotkey_for_name.hotkey_file.is_encrypted() + ): + hotkey_wallets.append(hotkey_for_name) + elif show_nulls: + hotkey_wallets.append(None) + except UnicodeDecodeError: # usually an unrelated file like .DS_Store + continue + + return hotkey_wallets + + +def get_coldkey_wallets_for_path(path: str) -> list[Wallet]: + wallet_path = Path(path).expanduser() + wallets = [ + Wallet(name=directory.name, path=path) + for directory in wallet_path.iterdir() + if directory.is_dir() + ] + return wallets + + +def get_all_wallets_for_path(path: str) -> list[Wallet]: + all_wallets = [] + cold_wallets = get_coldkey_wallets_for_path(path) + for cold_wallet in cold_wallets: + if ( + cold_wallet.coldkeypub_file.exists_on_device() + and not cold_wallet.coldkeypub_file.is_encrypted() + ): + all_wallets.extend(get_hotkey_wallets_for_wallet(cold_wallet)) + return all_wallets + + def is_valid_wallet(wallet: Wallet) -> tuple[bool, bool]: """ Verifies that the wallet with specified parameters. @@ -105,3 +157,38 @@ def is_valid_bittensor_address_or_public_key(address: Union[str, bytes]) -> bool else: # Invalid address type return False + + +def u16_normalized_float(x: int) -> float: + return float(x) / float(U16_MAX) + + +def decode_scale_bytes(return_type, scale_bytes, custom_rpc_type_registry): + rpc_runtime_config = RuntimeConfiguration() + rpc_runtime_config.update_type_registry(load_type_registry_preset("legacy")) + rpc_runtime_config.update_type_registry(custom_rpc_type_registry) + obj = rpc_runtime_config.create_scale_object(return_type, scale_bytes) + if obj.data.to_hex() == "0x0400": # RPC returned None result + return None + return obj.decode() + + +def ss58_address_to_bytes(ss58_address: str) -> bytes: + """Converts a ss58 address to a bytes object.""" + account_id_hex: str = scalecodec.ss58_decode(ss58_address, SS58_FORMAT) + return bytes.fromhex(account_id_hex) + + +def ss58_to_vec_u8(ss58_address: str) -> list[int]: + """ + Converts an SS58 address to a list of integers (vector of u8). + + Args: + ss58_address (str): The SS58 address to be converted. + + Returns: + List[int]: A list of integers representing the byte values of the SS58 address. + """ + ss58_bytes: bytes = ss58_address_to_bytes(ss58_address) + encoded_address: list[int] = [int(byte) for byte in ss58_bytes] + return encoded_address diff --git a/src/wallets.py b/src/wallets.py index 6f9bd0eb..eb3c863b 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -17,20 +17,27 @@ import asyncio import os -from pathlib import Path -from typing import Optional +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor +from typing import Optional, Any import aiohttp from bittensor_wallet import Wallet from bittensor_wallet.keyfile import Keyfile +from fuzzywuzzy import fuzz +from rich.align import Align from rich.table import Table, Column from rich.tree import Tree - +import scalecodec import typer -from .utils import console, err_console, RAO_PER_TAO +from src import utils, TYPE_REGISTRY +from src.bittensor.chain_data import NeuronInfoLite, custom_rpc_type_registry, StakeInfo +from src.bittensor.networking import int_to_ip +from src.utils import console, err_console, RAO_PER_TAO, decode_scale_bytes from . import defaults from src.subtensor_interface import SubtensorInterface +from src.bittensor.balances import Balance async def regen_coldkey( @@ -372,50 +379,683 @@ async def wallet_history(wallet: Wallet): async def wallet_list(wallet_path: str): r"""Lists wallets.""" - wallet_path = Path(wallet_path).expanduser() - wallets = [ - directory.name for directory in wallet_path.iterdir() if directory.is_dir() - ] + wallets = utils.get_coldkey_wallets_for_path(wallet_path) if not wallets: err_console.print(f"[red]No wallets found in dir: {wallet_path}[/red]") root = Tree("Wallets") - for w_name in wallets: - wallet_for_name = Wallet(path=str(wallet_path), name=w_name) + for wallet in wallets: if ( - wallet_for_name.coldkeypub_file.exists_on_device() - and not wallet_for_name.coldkeypub_file.is_encrypted() + wallet.coldkeypub_file.exists_on_device() + and not wallet.coldkeypub_file.is_encrypted() ): - coldkeypub_str = wallet_for_name.coldkeypub.ss58_address + coldkeypub_str = wallet.coldkeypub.ss58_address else: coldkeypub_str = "?" - wallet_tree = root.add("\n[bold white]{} ({})".format(w_name, coldkeypub_str)) - hotkeys_path = wallet_path / w_name / "hotkeys" - try: - hotkeys = [entry.name for entry in hotkeys_path.iterdir()] - if len(hotkeys) > 1: - for h_name in hotkeys: - hotkey_for_name = Wallet( - path=str(wallet_path), name=w_name, hotkey=h_name - ) - try: - if ( - hotkey_for_name.hotkey_file.exists_on_device() - and not hotkey_for_name.hotkey_file.is_encrypted() - ): - hotkey_str = hotkey_for_name.hotkey.ss58_address - else: - hotkey_str = "?" - wallet_tree.add(f"[bold grey]{h_name} ({hotkey_str})") - except UnicodeDecodeError: # usually an unrelated file like .DS_Store - continue - - except FileNotFoundError: - # no hotkeys found - continue + wallet_tree = root.add(f"\n[bold white]{wallet.name} ({coldkeypub_str})") + hotkeys = utils.get_hotkey_wallets_for_wallet(wallet, show_nulls=True) + for hkey in hotkeys: + data = f"[bold grey]{hkey.name} (?)" + if hkey: + try: + data = f"[bold grey]{hkey.name} ({hkey.hotkey.ss58_address})" + except UnicodeDecodeError: + pass + wallet_tree.add(data) if not wallets: root.add("[bold red]No wallets found.") console.print(root) + + +async def _get_total_balance( + total_balance: Balance, + subtensor: SubtensorInterface, + wallet: Wallet, + all_wallets: bool = False, +) -> tuple[list[Wallet], Balance]: + if all_wallets: + cold_wallets = utils.get_coldkey_wallets_for_path(wallet.path) + _balance_cold_wallets = [ + cold_wallet + for cold_wallet in cold_wallets + if ( + cold_wallet.coldkeypub_file.exists_on_device() + and not cold_wallet.coldkeypub_file.is_encrypted() + ) + ] + total_balance += sum( + ( + await subtensor.get_balance( + *(x.coldkeypub.ss58_address for x in _balance_cold_wallets), + # reuse_block=True # this breaks it idk why + ) + ).values() + ) + all_hotkeys = utils.get_all_wallets_for_path(wallet.path) + else: + # We are only printing keys for a single coldkey + coldkey_wallet = wallet + if ( + coldkey_wallet.coldkeypub_file.exists_on_device() + and not coldkey_wallet.coldkeypub_file.is_encrypted() + ): + total_balance = sum((await subtensor.get_balance( + coldkey_wallet.coldkeypub.ss58_address, + # reuse_block=True # This breaks it idk why + )).values()) + if not coldkey_wallet.coldkeypub_file.exists_on_device(): + console.print("[bold red]No wallets found.") + return [], None + all_hotkeys = utils.get_hotkey_wallets_for_wallet(coldkey_wallet) + + return all_hotkeys, total_balance + + +async def overview( + wallet: Wallet, + subtensor: SubtensorInterface, + all_wallets: bool = False, + sort_by: Optional[str] = None, + sort_order: Optional[str] = None, + include_hotkeys: Optional[list] = None, + exclude_hotkeys: Optional[list] = None, + netuids_filter: Optional[list] = None, +): + """Prints an overview for the wallet's coldkey.""" + + total_balance = Balance(0) + async with subtensor: + # We are printing for every coldkey. + all_hotkeys, total_balance = await _get_total_balance( + total_balance, subtensor, wallet, all_wallets + ) + + # We are printing for a select number of hotkeys from all_hotkeys. + if include_hotkeys: + all_hotkeys = _get_hotkeys(include_hotkeys, exclude_hotkeys, all_hotkeys) + + # Check we have keys to display. + if not all_hotkeys: + err_console.print("[red]No wallets found.[/red]") + return + + # Pull neuron info for all keys. + neurons: dict[str, list[NeuronInfoLite]] = {} + block, all_netuids, block_hash = await asyncio.gather( + subtensor.substrate.get_block_number(None), + subtensor.get_all_subnet_netuids(), + subtensor.get_chain_head(), + ) + + netuids = await subtensor.filter_netuids_by_registered_hotkeys( + all_netuids, netuids_filter, all_hotkeys, reuse_block=True + ) + # bittensor.logging.debug(f"Netuids to check: {netuids}") + + for netuid in netuids: + neurons[str(netuid)] = [] + + all_wallet_names = {wallet.name for wallet in all_hotkeys} + all_coldkey_wallets = [Wallet(name=wallet_name) for wallet_name in all_wallet_names] + + all_hotkey_addresses, hotkey_coldkey_to_hotkey_wallet = _get_key_address( + all_hotkeys + ) + + with console.status( + f":satellite: Syncing with chain: [white]{subtensor.network}[/white] ..." + ): + # Pull neuron info for all keys.: + + results = await _get_neurons_for_netuids( + subtensor, netuids, all_hotkey_addresses + ) + neurons = await _process_neuron_results(results, neurons, netuids) + total_coldkey_stake_from_metagraph = await _calculate_total_coldkey_stake( + neurons + ) + + alerts_table = Table(show_header=True, header_style="bold magenta") + alerts_table.add_column("🥩 alert!") + + coldkeys_to_check = [] + for coldkey_wallet in all_coldkey_wallets: + # Check if we have any stake with hotkeys that are not registered. + total_coldkey_stake_from_chain = ( + # TODO gathering here may make sense or may not + await subtensor.get_total_stake_for_coldkey( + coldkey_wallet.coldkeypub.ss58_address, + reuse_block=True + ) + ) + difference = ( + total_coldkey_stake_from_chain[coldkey_wallet.coldkeypub.ss58_address] + - total_coldkey_stake_from_metagraph[ + coldkey_wallet.coldkeypub.ss58_address + ] + ) + if difference == 0: + continue # We have all our stake registered. + + coldkeys_to_check.append(coldkey_wallet) + alerts_table.add_row( + "Found {} stake with coldkey {} that is not registered.".format( + difference, coldkey_wallet.coldkeypub.ss58_address + ) + ) + + if coldkeys_to_check: + # We have some stake that is not with a registered hotkey. + if "-1" not in neurons: + neurons["-1"] = [] + + # Check each coldkey wallet for de-registered stake. + + results = await asyncio.gather( + *[ + _get_de_registered_stake_for_coldkey_wallet( + subtensor, all_hotkey_addresses, coldkey_wallet + ) + for coldkey_wallet in coldkeys_to_check + ] + ) + + for result in results: + coldkey_wallet, de_registered_stake, err_msg = result + if err_msg is not None: + err_console.print(err_msg) + + if len(de_registered_stake) == 0: + continue # We have no de-registered stake with this coldkey. + + de_registered_neurons = [] + for hotkey_addr, our_stake in de_registered_stake: + # Make a neuron info lite for this hotkey and coldkey. + de_registered_neuron = NeuronInfoLite.get_null_neuron() + de_registered_neuron.hotkey = hotkey_addr + de_registered_neuron.coldkey = coldkey_wallet.coldkeypub.ss58_address + de_registered_neuron.total_stake = Balance(our_stake) + de_registered_neurons.append(de_registered_neuron) + + # Add this hotkey to the wallets dict + wallet_ = Wallet(name=wallet) + wallet_.hotkey_ss58 = hotkey_addr + wallet.hotkey_str = hotkey_addr[:5] # Max length of 5 characters + # Indicates a hotkey not on local machine but exists in stake_info obj on-chain + if hotkey_coldkey_to_hotkey_wallet.get(hotkey_addr) is None: + hotkey_coldkey_to_hotkey_wallet[hotkey_addr] = {} + hotkey_coldkey_to_hotkey_wallet[hotkey_addr][ + coldkey_wallet.coldkeypub.ss58_address + ] = wallet_ + + # Add neurons to overview. + neurons["-1"].extend(de_registered_neurons) + + # Setup outer table. + grid = Table.grid(pad_edge=False) + + # If there are any alerts, add them to the grid + if len(alerts_table.rows) > 0: + grid.add_row(alerts_table) + + title: str = "" + if not all_wallets: + title = f"[bold white italic]Wallet - {wallet.name}:{wallet.coldkeypub.ss58_address}" + else: + title = "[bold whit italic]All Wallets:" + + # Add title + grid.add_row(Align(title, vertical="middle", align="center")) + + # Generate rows per netuid + hotkeys_seen = set() + total_neurons = 0 + total_stake = 0.0 + tempos = await asyncio.gather( + *[ + subtensor.get_hyperparameter("Tempo", netuid, block_hash) + for netuid in netuids + ] + ) + for netuid, subnet_tempo in zip(netuids, tempos): + last_subnet = netuid == netuids[-1] + table_data = [] + total_rank = 0.0 + total_trust = 0.0 + total_consensus = 0.0 + total_validator_trust = 0.0 + total_incentive = 0.0 + total_dividends = 0.0 + total_emission = 0 + + for nn in neurons[str(netuid)]: + hotwallet = hotkey_coldkey_to_hotkey_wallet.get(nn.hotkey, {}).get( + nn.coldkey, None + ) + if not hotwallet: + # Indicates a mismatch between what the chain says the coldkey + # is for this hotkey and the local wallet coldkey-hotkey pair + hotwallet = Wallet(name=nn.coldkey[:7]) + hotwallet.hotkey_str = nn.hotkey[:7] + + nn: NeuronInfoLite + uid = nn.uid + active = nn.active + stake = nn.total_stake.tao + rank = nn.rank + trust = nn.trust + consensus = nn.consensus + validator_trust = nn.validator_trust + incentive = nn.incentive + dividends = nn.dividends + emission = int(nn.emission / (subnet_tempo + 1) * 1e9) + last_update = int(block - nn.last_update) + validator_permit = nn.validator_permit + row = [ + hotwallet.name, + hotwallet.hotkey_str, + str(uid), + str(active), + "{:.5f}".format(stake), + "{:.5f}".format(rank), + "{:.5f}".format(trust), + "{:.5f}".format(consensus), + "{:.5f}".format(incentive), + "{:.5f}".format(dividends), + "{:_}".format(emission), + "{:.5f}".format(validator_trust), + "*" if validator_permit else "", + str(last_update), + ( + int_to_ip(nn.axon_info.ip) + ":" + str(nn.axon_info.port) + if nn.axon_info.port != 0 + else "[yellow]none[/yellow]" + ), + nn.hotkey, + ] + + total_rank += rank + total_trust += trust + total_consensus += consensus + total_incentive += incentive + total_dividends += dividends + total_emission += emission + total_validator_trust += validator_trust + + if (nn.hotkey, nn.coldkey) not in hotkeys_seen: + # Don't double count stake on hotkey-coldkey pairs. + hotkeys_seen.add((nn.hotkey, nn.coldkey)) + total_stake += stake + + # netuid -1 are neurons that are de-registered. + if netuid != "-1": + total_neurons += 1 + + table_data.append(row) + + # Add subnet header + if netuid == "-1": + grid.add_row("Deregistered Neurons") + else: + grid.add_row(f"Subnet: [bold white]{netuid}[/bold white]") + + table = Table( + show_footer=False, + width=None, + pad_edge=False, + box=None, + ) + if last_subnet: + table.add_column( + "[overline white]COLDKEY", + str(total_neurons), + footer_style="overline white", + style="bold white", + ) + table.add_column( + "[overline white]HOTKEY", + str(total_neurons), + footer_style="overline white", + style="white", + ) + else: + # No footer for non-last subnet. + table.add_column("[overline white]COLDKEY", style="bold white") + table.add_column("[overline white]HOTKEY", style="white") + table.add_column( + "[overline white]UID", + str(total_neurons), + footer_style="overline white", + style="yellow", + ) + table.add_column( + "[overline white]ACTIVE", justify="right", style="green", no_wrap=True + ) + if last_subnet: + table.add_column( + "[overline white]STAKE(\u03c4)", + "\u03c4{:.5f}".format(total_stake), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + else: + # No footer for non-last subnet. + table.add_column( + "[overline white]STAKE(\u03c4)", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column( + "[overline white]RANK", + "{:.5f}".format(total_rank), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column( + "[overline white]TRUST", + "{:.5f}".format(total_trust), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column( + "[overline white]CONSENSUS", + "{:.5f}".format(total_consensus), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column( + "[overline white]INCENTIVE", + "{:.5f}".format(total_incentive), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column( + "[overline white]DIVIDENDS", + "{:.5f}".format(total_dividends), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column( + "[overline white]EMISSION(\u03c1)", + "\u03c1{:_}".format(total_emission), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column( + "[overline white]VTRUST", + "{:.5f}".format(total_validator_trust), + footer_style="overline white", + justify="right", + style="green", + no_wrap=True, + ) + table.add_column("[overline white]VPERMIT", justify="right", no_wrap=True) + table.add_column("[overline white]UPDATED", justify="right", no_wrap=True) + table.add_column( + "[overline white]AXON", justify="left", style="dim blue", no_wrap=True + ) + table.add_column("[overline white]HOTKEY_SS58", style="dim blue", no_wrap=False) + table.show_footer = True + + if sort_by: + column_to_sort_by: int = 0 + highest_matching_ratio: int = 0 + sort_descending: bool = False # Default sort_order to ascending + + for index, column in zip(range(len(table.columns)), table.columns): + # Fuzzy match the column name. Default to the first column. + column_name = column.header.lower().replace("[overline white]", "") + match_ratio = fuzz.ratio(sort_by.lower(), column_name) + # Finds the best matching column + if match_ratio > highest_matching_ratio: + highest_matching_ratio = match_ratio + column_to_sort_by = index + + if sort_order.lower() in {"desc", "descending", "reverse"}: + # Sort descending if the sort_order matches desc, descending, or reverse + sort_descending = True + + def overview_sort_function(row): + data = row[column_to_sort_by] + # Try to convert to number if possible + try: + data = float(data) + except ValueError: + pass + return data + + table_data.sort(key=overview_sort_function, reverse=sort_descending) + + for row in table_data: + table.add_row(*row) + + grid.add_row(table) + + console.clear() + + caption = "[italic][dim][white]Wallet balance: [green]\u03c4" + str( + total_balance.tao + ) + grid.add_row(Align(caption, vertical="middle", align="center")) + + # Print the entire table/grid + console.print(grid, width=None) + + +def _get_hotkeys( + include_hotkeys: list[str], exclude_hotkeys: list[str], all_hotkeys: list[Wallet] +) -> list[Wallet]: + if include_hotkeys: + # We are only showing hotkeys that are specified. + all_hotkeys = [ + hotkey for hotkey in all_hotkeys if hotkey.hotkey_str in include_hotkeys + ] + else: + # We are excluding the specified hotkeys from all_hotkeys. + all_hotkeys = [ + hotkey for hotkey in all_hotkeys if hotkey.hotkey_str not in exclude_hotkeys + ] + return all_hotkeys + + +def _get_key_address(all_hotkeys: list[Wallet]): + hotkey_coldkey_to_hotkey_wallet = {} + for hotkey_wallet in all_hotkeys: + if hotkey_wallet.hotkey.ss58_address not in hotkey_coldkey_to_hotkey_wallet: + hotkey_coldkey_to_hotkey_wallet[hotkey_wallet.hotkey.ss58_address] = {} + + hotkey_coldkey_to_hotkey_wallet[hotkey_wallet.hotkey.ss58_address][ + hotkey_wallet.coldkeypub.ss58_address + ] = hotkey_wallet + + all_hotkey_addresses = list(hotkey_coldkey_to_hotkey_wallet.keys()) + + return all_hotkey_addresses, hotkey_coldkey_to_hotkey_wallet + + +async def _calculate_total_coldkey_stake( + neurons: dict[str, list["NeuronInfoLite"]], +) -> dict[str, Balance]: + total_coldkey_stake_from_metagraph = defaultdict(lambda: Balance(0.0)) + checked_hotkeys = set() + for neuron_list in neurons.values(): + for neuron in neuron_list: + if neuron.hotkey in checked_hotkeys: + continue + total_coldkey_stake_from_metagraph[neuron.coldkey] += neuron.stake_dict[ + neuron.coldkey + ] + checked_hotkeys.add(neuron.hotkey) + return total_coldkey_stake_from_metagraph + + +async def _process_neuron_results( + results: list[tuple[int, list["NeuronInfoLite"], Optional[str]]], + neurons: dict[str, list["NeuronInfoLite"]], + netuids: list[int], +) -> dict[str, list["NeuronInfoLite"]]: + for result in results: + netuid, neurons_result, err_msg = result + if err_msg is not None: + console.print(f"netuid '{netuid}': {err_msg}") + + if len(neurons_result) == 0: + # Remove netuid from overview if no neurons are found. + netuids.remove(netuid) + del neurons[str(netuid)] + else: + # Add neurons to overview. + neurons[str(netuid)] = neurons_result + return neurons + + +def _map_hotkey_to_neurons( + all_neurons: list["NeuronInfoLite"], + hot_wallets: list[str], + netuid: int, +) -> tuple[int, list["NeuronInfoLite"], Optional[str]]: + result: list["NeuronInfoLite"] = [] + hotkey_to_neurons = {n.hotkey: n.uid for n in all_neurons} + try: + for hot_wallet_addr in hot_wallets: + uid = hotkey_to_neurons.get(hot_wallet_addr) + if uid is not None: + nn = all_neurons[uid] + result.append(nn) + except Exception as e: + return netuid, [], f"Error: {e}" + + return netuid, result, None + + +async def _fetch_neuron_for_netuid( + netuid: int, subtensor: SubtensorInterface +) -> tuple[int, dict]: + async def neurons_lite_for_uid(uid: int) -> dict[Any, Any]: + call_definition = TYPE_REGISTRY["runtime_api"]["NeuronInfoRuntimeApi"][ + "methods" + ]["get_neurons_lite"] + async with subtensor: + data = await subtensor.encode_params( + call_definition=call_definition, params=[uid] + ) + block_hash = subtensor.substrate.last_block_hash + hex_bytes_result = await subtensor.substrate.rpc_request( + method="state_call", + params=["NeuronInfoRuntimeApi_get_neurons_lite", data, block_hash], + reuse_block_hash=True + ) + + return hex_bytes_result + + neurons = await neurons_lite_for_uid(uid=netuid) + return netuid, neurons + + +async def _fetch_all_neurons( + netuids: list[int], subtensor +) -> list[tuple[int, list["NeuronInfoLite"]]]: + return list( + await asyncio.gather( + *[_fetch_neuron_for_netuid(netuid, subtensor) for netuid in netuids] + ) + ) + + +def _partial_decode(args): + rt, sb, crtr, nuid = args + return nuid, decode_scale_bytes(rt, sb, crtr) + + +def _process_neurons_for_netuids(netuids_with_all_neurons_hex_bytes): + def make_map(result): + netuid, json_result = result + hex_bytes_result = json_result["result"] + as_scale_bytes = scalecodec.ScaleBytes(hex_bytes_result) + return [return_type, as_scale_bytes, custom_rpc_type_registry, netuid] + + return_type = TYPE_REGISTRY["runtime_api"]["NeuronInfoRuntimeApi"]["methods"][ + "get_neurons_lite" + ]["type"] + + all_results = [] + preprocessed = [make_map(r) for r in netuids_with_all_neurons_hex_bytes] + with ProcessPoolExecutor() as executor: + results = executor.map( + _partial_decode, + preprocessed + ) + for netuid, result in results: + all_results.append( + ( + netuid, + list(results), + ) + ) + + return all_results + + +async def _get_neurons_for_netuids( + subtensor: SubtensorInterface, netuids: list[int], hot_wallets: list[str] +) -> list[tuple[int, list["NeuronInfoLite"], Optional[str]]]: + all_neurons_hex_bytes = await _fetch_all_neurons(netuids, subtensor) + + all_processed_neurons = _process_neurons_for_netuids(all_neurons_hex_bytes) + return [ + _map_hotkey_to_neurons(neurons, hot_wallets, netuid) + for netuid, neurons in all_processed_neurons + ] + + +async def _get_de_registered_stake_for_coldkey_wallet( + subtensor: SubtensorInterface, all_hotkey_addresses, coldkey_wallet +) -> tuple[Wallet, list[tuple[str, Balance]], Optional[str]]: + # Pull all stake for our coldkey + all_stake_info_for_coldkey = await subtensor.get_stake_info_for_coldkey( + coldkey_ss58=coldkey_wallet.coldkeypub.ss58_address, + reuse_block=True + ) + + # Filter out hotkeys that are in our wallets + # Filter out hotkeys that are delegates. + async def _filter_stake_info(stake_info: StakeInfo) -> bool: + if stake_info.stake == 0: + return False # Skip hotkeys that we have no stake with. + if stake_info.hotkey_ss58 in all_hotkey_addresses: + return False # Skip hotkeys that are in our wallets. + return not await subtensor.is_hotkey_delegate( + hotkey_ss58=stake_info.hotkey_ss58, + reuse_block=True + ) + + all_staked_hotkeys = filter(_filter_stake_info, all_stake_info_for_coldkey) + + # List of (hotkey_addr, our_stake) tuples. + result = [ + ( + stake_info.hotkey_ss58, + stake_info.stake.tao, + ) # stake is a Balance object + for stake_info in all_staked_hotkeys + ] + + return coldkey_wallet, result, None From d6c64a0bf5cf2964da1ec7e7b3ed976f992f97d4 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 29 Jul 2024 23:19:05 +0200 Subject: [PATCH 19/48] Overview working with --all --- cli.py | 1 - src/utils.py | 13 ++++++++----- src/wallets.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cli.py b/cli.py index 2e3144bc..43f313da 100755 --- a/cli.py +++ b/cli.py @@ -275,7 +275,6 @@ def wallet_overview( comprehensive view of the user's network presence, making it ideal for monitoring account status, stake distribution, and overall contribution to the Bittensor network. """ - # TODO does not yet work with encrypted keys if include_hotkeys and exclude_hotkeys: utils.err_console.print( "[red]You have specified hotkeys for inclusion and exclusion. Pick only one or neither." diff --git a/src/utils.py b/src/utils.py index 7f5842e3..d7c374cd 100644 --- a/src/utils.py +++ b/src/utils.py @@ -59,11 +59,14 @@ def get_all_wallets_for_path(path: str) -> list[Wallet]: all_wallets = [] cold_wallets = get_coldkey_wallets_for_path(path) for cold_wallet in cold_wallets: - if ( - cold_wallet.coldkeypub_file.exists_on_device() - and not cold_wallet.coldkeypub_file.is_encrypted() - ): - all_wallets.extend(get_hotkey_wallets_for_wallet(cold_wallet)) + try: + if ( + cold_wallet.coldkeypub_file.exists_on_device() + and not cold_wallet.coldkeypub_file.is_encrypted() + ): + all_wallets.extend(get_hotkey_wallets_for_wallet(cold_wallet)) + except UnicodeDecodeError: # usually an incorrect file like .DS_Store + continue return all_wallets diff --git a/src/wallets.py b/src/wallets.py index eb3c863b..88290cc3 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -434,7 +434,7 @@ async def _get_total_balance( ) ).values() ) - all_hotkeys = utils.get_all_wallets_for_path(wallet.path) + all_hotkeys = utils.get_hotkey_wallets_for_wallet(wallet) else: # We are only printing keys for a single coldkey coldkey_wallet = wallet From d3623224bff3e785f9080170dba34548ca3b8159 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 29 Jul 2024 23:54:18 +0200 Subject: [PATCH 20/48] Ruff --- cli.py | 4 +- src/subtensor_interface.py | 25 ++-- src/wallets.py | 241 +++++++++++++++++++------------------ 3 files changed, 142 insertions(+), 128 deletions(-) diff --git a/cli.py b/cli.py index 43f313da..a7fc2d5c 100755 --- a/cli.py +++ b/cli.py @@ -283,7 +283,9 @@ def wallet_overview( # if all-wallets is entered, ask for path if all_wallets: if not wallet_path: - wallet_path = Prompt.ask("Enter the path of the wallets", default=defaults.wallet.path) + wallet_path = Prompt.ask( + "Enter the path of the wallets", default=defaults.wallet.path + ) wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) self.initialize_chain(network, chain) asyncio.run( diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 32a7a0bf..2da6686d 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -269,7 +269,10 @@ async def get_total_stake_for_coldkey( } async def get_netuids_for_hotkey( - self, hotkey_ss58: str, block_hash: Optional[str] = None, reuse_block: bool = False + self, + hotkey_ss58: str, + block_hash: Optional[str] = None, + reuse_block: bool = False, ) -> list[int]: """ Retrieves a list of subnet UIDs (netuids) for which a given hotkey is a member. This function @@ -288,7 +291,7 @@ async def get_netuids_for_hotkey( storage_function="IsNetworkMember", params=[hotkey_ss58], block_hash=block_hash, - reuse_block_hash=reuse_block + reuse_block_hash=reuse_block, ) return ( [record[0].value for record in result.records if record[1]] @@ -349,12 +352,18 @@ async def get_hyperparameter( async def filter_netuids_by_registered_hotkeys( self, all_netuids, filter_for_netuids, all_hotkeys, reuse_block: bool = False ) -> list[int]: - netuids_with_registered_hotkeys = [item for sublist in await asyncio.gather( - *[ - self.get_netuids_for_hotkey(wallet.hotkey.ss58_address, reuse_block=reuse_block) - for wallet in all_hotkeys - ] - ) for item in sublist] + netuids_with_registered_hotkeys = [ + item + for sublist in await asyncio.gather( + *[ + self.get_netuids_for_hotkey( + wallet.hotkey.ss58_address, reuse_block=reuse_block + ) + for wallet in all_hotkeys + ] + ) + for item in sublist + ] if not filter_for_netuids: all_netuids = netuids_with_registered_hotkeys diff --git a/src/wallets.py b/src/wallets.py index 88290cc3..eaa7c30d 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -442,10 +442,14 @@ async def _get_total_balance( coldkey_wallet.coldkeypub_file.exists_on_device() and not coldkey_wallet.coldkeypub_file.is_encrypted() ): - total_balance = sum((await subtensor.get_balance( - coldkey_wallet.coldkeypub.ss58_address, - # reuse_block=True # This breaks it idk why - )).values()) + total_balance = sum( + ( + await subtensor.get_balance( + coldkey_wallet.coldkeypub.ss58_address, + # reuse_block=True # This breaks it idk why + ) + ).values() + ) if not coldkey_wallet.coldkeypub_file.exists_on_device(): console.print("[bold red]No wallets found.") return [], None @@ -467,132 +471,136 @@ async def overview( """Prints an overview for the wallet's coldkey.""" total_balance = Balance(0) - async with subtensor: - # We are printing for every coldkey. - all_hotkeys, total_balance = await _get_total_balance( - total_balance, subtensor, wallet, all_wallets - ) + with console.status( + f":satellite: Synchronizing with chain [white]{subtensor.network}[/white]", + spinner="aesthetic", + ): + async with subtensor: + # We are printing for every coldkey. + all_hotkeys, total_balance = await _get_total_balance( + total_balance, subtensor, wallet, all_wallets + ) - # We are printing for a select number of hotkeys from all_hotkeys. - if include_hotkeys: - all_hotkeys = _get_hotkeys(include_hotkeys, exclude_hotkeys, all_hotkeys) - - # Check we have keys to display. - if not all_hotkeys: - err_console.print("[red]No wallets found.[/red]") - return - - # Pull neuron info for all keys. - neurons: dict[str, list[NeuronInfoLite]] = {} - block, all_netuids, block_hash = await asyncio.gather( - subtensor.substrate.get_block_number(None), - subtensor.get_all_subnet_netuids(), - subtensor.get_chain_head(), - ) + # We are printing for a select number of hotkeys from all_hotkeys. + if include_hotkeys: + all_hotkeys = _get_hotkeys( + include_hotkeys, exclude_hotkeys, all_hotkeys + ) - netuids = await subtensor.filter_netuids_by_registered_hotkeys( - all_netuids, netuids_filter, all_hotkeys, reuse_block=True - ) - # bittensor.logging.debug(f"Netuids to check: {netuids}") + # Check we have keys to display. + if not all_hotkeys: + err_console.print("[red]No wallets found.[/red]") + return + + # Pull neuron info for all keys. + neurons: dict[str, list[NeuronInfoLite]] = {} + block, all_netuids, block_hash = await asyncio.gather( + subtensor.substrate.get_block_number(None), + subtensor.get_all_subnet_netuids(), + subtensor.get_chain_head(), + ) + + netuids = await subtensor.filter_netuids_by_registered_hotkeys( + all_netuids, netuids_filter, all_hotkeys, reuse_block=True + ) + # bittensor.logging.debug(f"Netuids to check: {netuids}") for netuid in netuids: neurons[str(netuid)] = [] all_wallet_names = {wallet.name for wallet in all_hotkeys} - all_coldkey_wallets = [Wallet(name=wallet_name) for wallet_name in all_wallet_names] + all_coldkey_wallets = [ + Wallet(name=wallet_name) for wallet_name in all_wallet_names + ] all_hotkey_addresses, hotkey_coldkey_to_hotkey_wallet = _get_key_address( all_hotkeys ) - with console.status( - f":satellite: Syncing with chain: [white]{subtensor.network}[/white] ..." - ): - # Pull neuron info for all keys.: + # Pull neuron info for all keys.: - results = await _get_neurons_for_netuids( - subtensor, netuids, all_hotkey_addresses - ) - neurons = await _process_neuron_results(results, neurons, netuids) - total_coldkey_stake_from_metagraph = await _calculate_total_coldkey_stake( - neurons - ) + results = await _get_neurons_for_netuids( + subtensor, netuids, all_hotkey_addresses + ) + neurons = await _process_neuron_results(results, neurons, netuids) + total_coldkey_stake_from_metagraph = await _calculate_total_coldkey_stake( + neurons + ) - alerts_table = Table(show_header=True, header_style="bold magenta") - alerts_table.add_column("🥩 alert!") + alerts_table = Table(show_header=True, header_style="bold magenta") + alerts_table.add_column("🥩 alert!") - coldkeys_to_check = [] - for coldkey_wallet in all_coldkey_wallets: - # Check if we have any stake with hotkeys that are not registered. - total_coldkey_stake_from_chain = ( - # TODO gathering here may make sense or may not - await subtensor.get_total_stake_for_coldkey( - coldkey_wallet.coldkeypub.ss58_address, - reuse_block=True - ) - ) - difference = ( - total_coldkey_stake_from_chain[coldkey_wallet.coldkeypub.ss58_address] - - total_coldkey_stake_from_metagraph[ - coldkey_wallet.coldkeypub.ss58_address - ] + coldkeys_to_check = [] + for coldkey_wallet in all_coldkey_wallets: + # Check if we have any stake with hotkeys that are not registered. + total_coldkey_stake_from_chain = ( + # TODO gathering here may make sense or may not + await subtensor.get_total_stake_for_coldkey( + coldkey_wallet.coldkeypub.ss58_address, reuse_block=True ) - if difference == 0: - continue # We have all our stake registered. + ) + difference = ( + total_coldkey_stake_from_chain[coldkey_wallet.coldkeypub.ss58_address] + - total_coldkey_stake_from_metagraph[ + coldkey_wallet.coldkeypub.ss58_address + ] + ) + if difference == 0: + continue # We have all our stake registered. - coldkeys_to_check.append(coldkey_wallet) - alerts_table.add_row( - "Found {} stake with coldkey {} that is not registered.".format( - difference, coldkey_wallet.coldkeypub.ss58_address - ) + coldkeys_to_check.append(coldkey_wallet) + alerts_table.add_row( + "Found {} stake with coldkey {} that is not registered.".format( + difference, coldkey_wallet.coldkeypub.ss58_address ) + ) - if coldkeys_to_check: - # We have some stake that is not with a registered hotkey. - if "-1" not in neurons: - neurons["-1"] = [] + if coldkeys_to_check: + # We have some stake that is not with a registered hotkey. + if "-1" not in neurons: + neurons["-1"] = [] - # Check each coldkey wallet for de-registered stake. + # Check each coldkey wallet for de-registered stake. - results = await asyncio.gather( - *[ - _get_de_registered_stake_for_coldkey_wallet( - subtensor, all_hotkey_addresses, coldkey_wallet - ) - for coldkey_wallet in coldkeys_to_check - ] - ) + results = await asyncio.gather( + *[ + _get_de_registered_stake_for_coldkey_wallet( + subtensor, all_hotkey_addresses, coldkey_wallet + ) + for coldkey_wallet in coldkeys_to_check + ] + ) + + for result in results: + coldkey_wallet, de_registered_stake, err_msg = result + if err_msg is not None: + err_console.print(err_msg) + + if len(de_registered_stake) == 0: + continue # We have no de-registered stake with this coldkey. + + de_registered_neurons = [] + for hotkey_addr, our_stake in de_registered_stake: + # Make a neuron info lite for this hotkey and coldkey. + de_registered_neuron = NeuronInfoLite.get_null_neuron() + de_registered_neuron.hotkey = hotkey_addr + de_registered_neuron.coldkey = coldkey_wallet.coldkeypub.ss58_address + de_registered_neuron.total_stake = Balance(our_stake) + de_registered_neurons.append(de_registered_neuron) + + # Add this hotkey to the wallets dict + wallet_ = Wallet(name=wallet) + wallet_.hotkey_ss58 = hotkey_addr + wallet.hotkey_str = hotkey_addr[:5] # Max length of 5 characters + # Indicates a hotkey not on local machine but exists in stake_info obj on-chain + if hotkey_coldkey_to_hotkey_wallet.get(hotkey_addr) is None: + hotkey_coldkey_to_hotkey_wallet[hotkey_addr] = {} + hotkey_coldkey_to_hotkey_wallet[hotkey_addr][ + coldkey_wallet.coldkeypub.ss58_address + ] = wallet_ - for result in results: - coldkey_wallet, de_registered_stake, err_msg = result - if err_msg is not None: - err_console.print(err_msg) - - if len(de_registered_stake) == 0: - continue # We have no de-registered stake with this coldkey. - - de_registered_neurons = [] - for hotkey_addr, our_stake in de_registered_stake: - # Make a neuron info lite for this hotkey and coldkey. - de_registered_neuron = NeuronInfoLite.get_null_neuron() - de_registered_neuron.hotkey = hotkey_addr - de_registered_neuron.coldkey = coldkey_wallet.coldkeypub.ss58_address - de_registered_neuron.total_stake = Balance(our_stake) - de_registered_neurons.append(de_registered_neuron) - - # Add this hotkey to the wallets dict - wallet_ = Wallet(name=wallet) - wallet_.hotkey_ss58 = hotkey_addr - wallet.hotkey_str = hotkey_addr[:5] # Max length of 5 characters - # Indicates a hotkey not on local machine but exists in stake_info obj on-chain - if hotkey_coldkey_to_hotkey_wallet.get(hotkey_addr) is None: - hotkey_coldkey_to_hotkey_wallet[hotkey_addr] = {} - hotkey_coldkey_to_hotkey_wallet[hotkey_addr][ - coldkey_wallet.coldkeypub.ss58_address - ] = wallet_ - - # Add neurons to overview. - neurons["-1"].extend(de_registered_neurons) + # Add neurons to overview. + neurons["-1"].extend(de_registered_neurons) # Setup outer table. grid = Table.grid(pad_edge=False) @@ -833,8 +841,8 @@ async def overview( # Sort descending if the sort_order matches desc, descending, or reverse sort_descending = True - def overview_sort_function(row): - data = row[column_to_sort_by] + def overview_sort_function(row_): + data = row_[column_to_sort_by] # Try to convert to number if possible try: data = float(data) @@ -961,7 +969,7 @@ async def neurons_lite_for_uid(uid: int) -> dict[Any, Any]: hex_bytes_result = await subtensor.substrate.rpc_request( method="state_call", params=["NeuronInfoRuntimeApi_get_neurons_lite", data, block_hash], - reuse_block_hash=True + reuse_block_hash=True, ) return hex_bytes_result @@ -999,10 +1007,7 @@ def make_map(result): all_results = [] preprocessed = [make_map(r) for r in netuids_with_all_neurons_hex_bytes] with ProcessPoolExecutor() as executor: - results = executor.map( - _partial_decode, - preprocessed - ) + results = executor.map(_partial_decode, preprocessed) for netuid, result in results: all_results.append( ( @@ -1031,8 +1036,7 @@ async def _get_de_registered_stake_for_coldkey_wallet( ) -> tuple[Wallet, list[tuple[str, Balance]], Optional[str]]: # Pull all stake for our coldkey all_stake_info_for_coldkey = await subtensor.get_stake_info_for_coldkey( - coldkey_ss58=coldkey_wallet.coldkeypub.ss58_address, - reuse_block=True + coldkey_ss58=coldkey_wallet.coldkeypub.ss58_address, reuse_block=True ) # Filter out hotkeys that are in our wallets @@ -1043,8 +1047,7 @@ async def _filter_stake_info(stake_info: StakeInfo) -> bool: if stake_info.hotkey_ss58 in all_hotkey_addresses: return False # Skip hotkeys that are in our wallets. return not await subtensor.is_hotkey_delegate( - hotkey_ss58=stake_info.hotkey_ss58, - reuse_block=True + hotkey_ss58=stake_info.hotkey_ss58, reuse_block=True ) all_staked_hotkeys = filter(_filter_stake_info, all_stake_info_for_coldkey) From d440fd71f19730e9cda75b9bb3bb9d3fac99b679 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 29 Jul 2024 23:56:08 +0200 Subject: [PATCH 21/48] TODO --- cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cli.py b/cli.py index a7fc2d5c..837cc0fb 100755 --- a/cli.py +++ b/cli.py @@ -275,6 +275,7 @@ def wallet_overview( comprehensive view of the user's network presence, making it ideal for monitoring account status, stake distribution, and overall contribution to the Bittensor network. """ + # TODO verify this is actually doing all wallets if include_hotkeys and exclude_hotkeys: utils.err_console.print( "[red]You have specified hotkeys for inclusion and exclusion. Pick only one or neither." From cde84864bc2f6252307d4e93624aa1c90e5a6daa Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 30 Jul 2024 14:08:29 +0200 Subject: [PATCH 22/48] Overview now correctly handles `--all` --- cli.py | 29 ++++++++++++++++++++----- src/utils.py | 4 +++- src/wallets.py | 57 +++++++++++++++++++++++++++----------------------- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/cli.py b/cli.py index 837cc0fb..a9cd0cc1 100755 --- a/cli.py +++ b/cli.py @@ -222,8 +222,8 @@ def wallet_overview( help="Specify the hotkeys to exclude by name or ss58 address. (e.g. `hk1 hk2 hk3`). " "If left empty, and no hotkeys included in --include-hotkeys, all hotkeys will be included.", ), - netuids: Optional[list[str]] = typer.Option( - [], help="Set the netuid(s) to filter by." + netuids: Optional[list[int]] = typer.Option( + [], help="Set the netuid(s) to filter by (e.g. `0 1 2`)" ), network: Optional[str] = Options.network, chain: Optional[str] = Options.chain, @@ -243,30 +243,49 @@ def wallet_overview( The output is presented in a tabular format with the following columns: - COLDKEY: The SS58 address of the coldkey. + - HOTKEY: The SS58 address of the hotkey. + - UID: Unique identifier of the neuron. + - ACTIVE: Indicates if the neuron is active. + - STAKE(τ): Amount of stake in the neuron, in Tao. + - RANK: The rank of the neuron within the network. + - TRUST: Trust score of the neuron. + - CONSENSUS: Consensus score of the neuron. + - INCENTIVE: Incentive score of the neuron. + - DIVIDENDS: Dividends earned by the neuron. + - EMISSION(p): Emission received by the neuron, in Rho. + - VTRUST: Validator trust score of the neuron. + - VPERMIT: Indicates if the neuron has a validator permit. + - UPDATED: Time since last update. + - AXON: IP address and port of the neuron. + - HOTKEY_SS58: Human-readable representation of the hotkey. + ### Example usage: - ``` + + - ``` btcli wallet overview ``` - ``` + + - ``` btcli wallet overview --all --sort-by stake --sort-order descending ``` - ``` + + - ``` btcli wallet overview --include-hotkeys hk1 hk2 --sort-by stake ``` diff --git a/src/utils.py b/src/utils.py index d7c374cd..ecca3f86 100644 --- a/src/utils.py +++ b/src/utils.py @@ -35,11 +35,13 @@ def get_hotkey_wallets_for_wallet( if ( hotkey_for_name.hotkey_file.exists_on_device() and not hotkey_for_name.hotkey_file.is_encrypted() + # and hotkey_for_name.coldkeypub.ss58_address + and hotkey_for_name.hotkey.ss58_address ): hotkey_wallets.append(hotkey_for_name) elif show_nulls: hotkey_wallets.append(None) - except UnicodeDecodeError: # usually an unrelated file like .DS_Store + except (UnicodeDecodeError, AttributeError): # usually an unrelated file like .DS_Store continue return hotkey_wallets diff --git a/src/wallets.py b/src/wallets.py index eaa7c30d..e4ce50fd 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -434,7 +434,7 @@ async def _get_total_balance( ) ).values() ) - all_hotkeys = utils.get_hotkey_wallets_for_wallet(wallet) + all_hotkeys = [hk for w in cold_wallets for hk in utils.get_hotkey_wallets_for_wallet(w)] else: # We are only printing keys for a single coldkey coldkey_wallet = wallet @@ -532,28 +532,29 @@ async def overview( coldkeys_to_check = [] for coldkey_wallet in all_coldkey_wallets: - # Check if we have any stake with hotkeys that are not registered. - total_coldkey_stake_from_chain = ( - # TODO gathering here may make sense or may not - await subtensor.get_total_stake_for_coldkey( - coldkey_wallet.coldkeypub.ss58_address, reuse_block=True + if coldkey_wallet.coldkeypub: + # Check if we have any stake with hotkeys that are not registered. + total_coldkey_stake_from_chain = ( + # TODO gathering here may make sense or may not + await subtensor.get_total_stake_for_coldkey( + coldkey_wallet.coldkeypub.ss58_address, reuse_block=True + ) ) - ) - difference = ( - total_coldkey_stake_from_chain[coldkey_wallet.coldkeypub.ss58_address] - - total_coldkey_stake_from_metagraph[ - coldkey_wallet.coldkeypub.ss58_address - ] - ) - if difference == 0: - continue # We have all our stake registered. + difference = ( + total_coldkey_stake_from_chain[coldkey_wallet.coldkeypub.ss58_address] + - total_coldkey_stake_from_metagraph[ + coldkey_wallet.coldkeypub.ss58_address + ] + ) + if difference == 0: + continue # We have all our stake registered. - coldkeys_to_check.append(coldkey_wallet) - alerts_table.add_row( - "Found {} stake with coldkey {} that is not registered.".format( - difference, coldkey_wallet.coldkeypub.ss58_address + coldkeys_to_check.append(coldkey_wallet) + alerts_table.add_row( + "Found {} stake with coldkey {} that is not registered.".format( + difference, coldkey_wallet.coldkeypub.ss58_address + ) ) - ) if coldkeys_to_check: # We have some stake that is not with a registered hotkey. @@ -887,12 +888,16 @@ def _get_hotkeys( def _get_key_address(all_hotkeys: list[Wallet]): hotkey_coldkey_to_hotkey_wallet = {} for hotkey_wallet in all_hotkeys: - if hotkey_wallet.hotkey.ss58_address not in hotkey_coldkey_to_hotkey_wallet: - hotkey_coldkey_to_hotkey_wallet[hotkey_wallet.hotkey.ss58_address] = {} - - hotkey_coldkey_to_hotkey_wallet[hotkey_wallet.hotkey.ss58_address][ - hotkey_wallet.coldkeypub.ss58_address - ] = hotkey_wallet + if hotkey_wallet.coldkeypub: + if hotkey_wallet.hotkey.ss58_address not in hotkey_coldkey_to_hotkey_wallet: + hotkey_coldkey_to_hotkey_wallet[hotkey_wallet.hotkey.ss58_address] = {} + hotkey_coldkey_to_hotkey_wallet[hotkey_wallet.hotkey.ss58_address][ + hotkey_wallet.coldkeypub.ss58_address + ] = hotkey_wallet + else: + # occurs when there is a hotkey without an associated coldkeypub + # TODO log this, maybe display + pass all_hotkey_addresses = list(hotkey_coldkey_to_hotkey_wallet.keys()) From 31d94833104b01947747e225f5e1bdd72b8f5448 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 30 Jul 2024 16:24:03 +0200 Subject: [PATCH 23/48] Async Substrate Interface performance improvements. --- src/bittensor/async_substrate_interface.py | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index cddd99b3..4b22b9a3 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -233,7 +233,7 @@ async def __aenter__(self): async def _connect(self): self.ws = await asyncio.wait_for( - websockets.connect(self.ws_url, **self._options), timeout=None + websockets.connect(self.ws_url, **self._options), timeout=10 ) async def __aexit__(self, exc_type, exc_val, exc_tb): @@ -276,12 +276,12 @@ async def _recv(self) -> None: response = json.loads(await self.ws.recv()) async with self._lock: self._open_subscriptions -= 1 - if "id" in response: - self._received[response["id"]] = response - elif "params" in response: - self._received[response["params"]["subscription"]] = response - else: - raise KeyError(response) + if "id" in response: + self._received[response["id"]] = response + elif "params" in response: + self._received[response["params"]["subscription"]] = response + else: + raise KeyError(response) except websockets.ConnectionClosed: raise except KeyError as e: @@ -306,13 +306,13 @@ async def send(self, payload: dict) -> int: """ async with self._lock: original_id = self.id - try: - await self.ws.send(json.dumps({**payload, **{"id": original_id}})) - self.id += 1 - self._open_subscriptions += 1 - return original_id - except websockets.ConnectionClosed: - raise + self.id += 1 + self._open_subscriptions += 1 + try: + await self.ws.send(json.dumps({**payload, **{"id": original_id}})) + return original_id + except websockets.ConnectionClosed: + raise async def retrieve(self, item_id: int) -> Optional[dict]: """ @@ -351,8 +351,8 @@ def __init__( chain_endpoint, options={ "max_size": 2**32, - "read_limit": 2**32, - "write_limit": 2**32, + "read_limit": 2**16, + "write_limit": 2**16, }, ) self._lock = asyncio.Lock() From 02e6fa3753cac5bdfeb4f829cf316187115dea3e Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 30 Jul 2024 22:17:53 +0200 Subject: [PATCH 24/48] Async Substrate Interface performance improvements. Error handling for CLI. --- cli.py | 32 +++++++----- src/bittensor/async_substrate_interface.py | 58 ++++++++++------------ src/subtensor_interface.py | 3 ++ src/wallets.py | 12 +++-- 4 files changed, 56 insertions(+), 49 deletions(-) diff --git a/cli.py b/cli.py index a9cd0cc1..673eef6f 100755 --- a/cli.py +++ b/cli.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import asyncio -from typing import Optional +from typing import Optional, Coroutine from bittensor_wallet import Wallet import rich @@ -132,6 +132,12 @@ def initialize_chain( self.not_subtensor = SubtensorInterface(network, chain) # typer.echo(f"Initialized with {self.not_subtensor}") + def _run_command(self, cmd: Coroutine): + try: + asyncio.run(cmd) + except ConnectionRefusedError: + typer.echo(f"Connection refused when connecting to chain: {self.not_subtensor}") + @staticmethod def wallet_ask( wallet_name: str, @@ -160,8 +166,8 @@ def wallet_ask( raise typer.Exit() return wallet - @staticmethod def wallet_list( + self, wallet_path: str = typer.Option( defaults.wallet.path, "--wallet-path", @@ -194,7 +200,7 @@ def wallet_list( This command is read-only and does not modify the filesystem or the network state. It is intended for use within the Bittensor CLI to provide a quick overview of the user's wallets. """ - asyncio.run(wallets.wallet_list(wallet_path)) + return self._run_command(wallets.wallet_list(wallet_path)) def wallet_overview( self, @@ -308,7 +314,7 @@ def wallet_overview( ) wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) self.initialize_chain(network, chain) - asyncio.run( + return self._run_command( wallets.overview( wallet, self.not_subtensor, @@ -356,7 +362,7 @@ def wallet_regen_coldkey( mnemonic, seed, json, json_password = get_creation_data( mnemonic, seed, json, json_password ) - asyncio.run( + return self._run_command( wallets.regen_coldkey( wallet, mnemonic, @@ -416,7 +422,7 @@ def wallet_regen_coldkey_pub( ): rich.print("[red]Error: Invalid SS58 address or public key![/red]") raise typer.Exit() - asyncio.run( + return self._run_command( wallets.regen_coldkey_pub( wallet, public_key_hex, ss58_address, overwrite_coldkeypub ) @@ -458,7 +464,7 @@ def wallet_regen_hotkey( mnemonic, seed, json, json_password = get_creation_data( mnemonic, seed, json, json_password ) - asyncio.run( + return self._run_command( wallets.regen_hotkey( wallet, mnemonic, @@ -502,7 +508,7 @@ def wallet_new_hotkey( wallet_name, wallet_path, wallet_hotkey, validate=False ) n_words = get_n_words(n_words) - asyncio.run(wallets.new_hotkey(wallet, n_words, use_password, overwrite_hotkey)) + return self._run_command(wallets.new_hotkey(wallet, n_words, use_password, overwrite_hotkey)) def wallet_new_coldkey( self, @@ -534,7 +540,7 @@ def wallet_new_coldkey( """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) n_words = get_n_words(n_words) - asyncio.run( + return self._run_command( wallets.new_coldkey(wallet, n_words, use_password, overwrite_coldkey) ) @@ -575,7 +581,7 @@ def wallet_create_wallet( wallet_name, wallet_path, wallet_hotkey, validate=False ) n_words = get_n_words(n_words) - asyncio.run( + return self._run_command( wallets.wallet_create( wallet, n_words, use_password, overwrite_coldkey, overwrite_hotkey ) @@ -629,7 +635,7 @@ def wallet_balance( """ subtensor = SubtensorInterface(network, chain) wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - asyncio.run(wallets.wallet_balance(wallet, subtensor, all_balances)) + return self._run_command(wallets.wallet_balance(wallet, subtensor, all_balances)) def wallet_history( self, @@ -656,7 +662,7 @@ def wallet_history( It helps in fetching info on all the transfers so that user can easily tally and cross check the transactions. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - asyncio.run(wallets.wallet_history(wallet)) + return self._run_command(wallets.wallet_history(wallet)) def delegates_list( self, @@ -665,7 +671,7 @@ def delegates_list( ): if not wallet_name: wallet_name = typer.prompt("Please enter the wallet name") - asyncio.run(delegates.ListDelegatesCommand.run(wallet_name, network)) + return self._run_command(delegates.ListDelegatesCommand.run(wallet_name, network)) def run(self): self.app() diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index 4b22b9a3..61866c61 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -406,8 +406,14 @@ async def get_storage_item(self, module: str, storage_function: str): storage_item = metadata_pallet.get_storage_function(storage_function) return storage_item - def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool): - return block_hash if block_hash else (self.last_block_hash if reuse else None) + async def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool): + if block_hash: + self.last_block_hash = block_hash + return block_hash + elif reuse: + if self.last_block_hash: + return self.last_block_hash + return await self.get_chain_head() # also sets the last_block_hash to chain_head async def init_runtime( self, block_hash: Optional[str] = None, block_id: Optional[int] = None @@ -667,7 +673,7 @@ async def rpc_request( :return: the response from the RPC request """ - block_hash = self._get_current_block_hash(block_hash, reuse_block_hash) + block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) payloads = [ self.make_payload( "rpc_request", @@ -695,7 +701,21 @@ async def get_block_hash(self, block_id: int) -> str: return (await self.rpc_request("chain_getBlockHash", [block_id]))["result"] async def get_chain_head(self) -> str: - self.last_block_hash = (await self.rpc_request("chain_getHead", []))["result"] + result = (await self._make_rpc_request( + [ + self.make_payload( + "rpc_request", + "chain_getHead", + [], + ) + ], + runtime=Runtime( + self.chain, + self.substrate.runtime_config, + self.substrate.metadata, + self.type_registry) + )) + self.last_block_hash = result["rpc_request"][0]["result"] return self.last_block_hash async def compose_call( @@ -739,15 +759,7 @@ async def query_multiple( # By allowing for specifying the block hash, users, if they have multiple query types they want # to do, can simply query the block hash first, and then pass multiple query_subtensor calls # into an asyncio.gather, with the specified block hash - block_hash = ( - block_hash - if block_hash - else ( - self.last_block_hash - if reuse_block_hash - else await self.get_chain_head() - ) - ) + block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) self.last_block_hash = block_hash runtime = await self.init_runtime(block_hash=block_hash) preprocessed: tuple[Preprocessed] = await asyncio.gather( @@ -1007,15 +1019,7 @@ async def query( Queries subtensor. This should only be used when making a single request. For multiple requests, you should use ``self.query_multiple`` """ - block_hash = ( - block_hash - if block_hash - else ( - self.last_block_hash - if reuse_block_hash - else await self.get_chain_head() - ) - ) + block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) self.last_block_hash = block_hash runtime = await self.init_runtime(block_hash=block_hash) preprocessed: Preprocessed = await self._preprocess( @@ -1079,15 +1083,7 @@ async def query_map( :return: QueryMapResult object """ params = params or [] - block_hash = ( - block_hash - if block_hash - else ( - self.last_block_hash - if reuse_block_hash - else await self.get_chain_head() - ) - ) + block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) self.last_block_hash = block_hash runtime = await self.init_runtime(block_hash=block_hash) diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 2da6686d..6182fe9a 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -36,6 +36,9 @@ def __init__(self, network, chain_endpoint): type_registry=TYPE_REGISTRY, ) + def __str__(self): + return f"Network: {self.network}, Chain: {self.chain_endpoint}" + async def __aenter__(self): async with self.substrate: return diff --git a/src/wallets.py b/src/wallets.py index e4ce50fd..0eebb454 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -430,7 +430,7 @@ async def _get_total_balance( ( await subtensor.get_balance( *(x.coldkeypub.ss58_address for x in _balance_cold_wallets), - # reuse_block=True # this breaks it idk why + reuse_block=True ) ).values() ) @@ -446,7 +446,7 @@ async def _get_total_balance( ( await subtensor.get_balance( coldkey_wallet.coldkeypub.ss58_address, - # reuse_block=True # This breaks it idk why + reuse_block=True ) ).values() ) @@ -477,6 +477,7 @@ async def overview( ): async with subtensor: # We are printing for every coldkey. + block_hash = await subtensor.get_chain_head() all_hotkeys, total_balance = await _get_total_balance( total_balance, subtensor, wallet, all_wallets ) @@ -494,10 +495,9 @@ async def overview( # Pull neuron info for all keys. neurons: dict[str, list[NeuronInfoLite]] = {} - block, all_netuids, block_hash = await asyncio.gather( + block, all_netuids = await asyncio.gather( subtensor.substrate.get_block_number(None), - subtensor.get_all_subnet_netuids(), - subtensor.get_chain_head(), + subtensor.get_all_subnet_netuids() ) netuids = await subtensor.filter_netuids_by_registered_hotkeys( @@ -623,12 +623,14 @@ async def overview( hotkeys_seen = set() total_neurons = 0 total_stake = 0.0 + print("netuids", netuids) tempos = await asyncio.gather( *[ subtensor.get_hyperparameter("Tempo", netuid, block_hash) for netuid in netuids ] ) + print(tempos) for netuid, subnet_tempo in zip(netuids, tempos): last_subnet = netuid == netuids[-1] table_data = [] From ba95aca1c5fae1c28499379e1d537087c6f70b23 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 30 Jul 2024 23:28:20 +0200 Subject: [PATCH 25/48] [WIP] Transfer almost finished. --- cli.py | 67 ++++++++++- src/bittensor/async_substrate_interface.py | 11 +- src/bittensor/extrinsics/transfer.py | 127 +++++++++++++++++++++ src/utils.py | 5 +- src/wallets.py | 25 +++- 5 files changed, 219 insertions(+), 16 deletions(-) create mode 100644 src/bittensor/extrinsics/transfer.py diff --git a/cli.py b/cli.py index 673eef6f..05efc15a 100755 --- a/cli.py +++ b/cli.py @@ -136,7 +136,9 @@ def _run_command(self, cmd: Coroutine): try: asyncio.run(cmd) except ConnectionRefusedError: - typer.echo(f"Connection refused when connecting to chain: {self.not_subtensor}") + typer.echo( + f"Connection refused when connecting to chain: {self.not_subtensor}" + ) @staticmethod def wallet_ask( @@ -167,7 +169,7 @@ def wallet_ask( return wallet def wallet_list( - self, + self, wallet_path: str = typer.Option( defaults.wallet.path, "--wallet-path", @@ -327,6 +329,55 @@ def wallet_overview( ) ) + def wallet_transfer( + self, + destination: str = typer.Option( + None, + "--destination", + "--dest", + "-d", + prompt=True, + help="Destination address of the wallet.", + ), + amount: float = typer.Option( + None, + "--amount", + "-a", + prompt=True, + help="Amount (in TAO) to transfer.", + ), + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + ): + """ + # wallet transfer + Executes the ``transfer`` command to transfer TAO tokens from one account to another on the Bittensor network. + + This command is used for transactions between different accounts, enabling users to send tokens to other + participants on the network. The command displays the user's current balance before prompting for the amount + to transfer, ensuring transparency and accuracy in the transaction. + + ## Usage: + The command requires specifying the destination address (public key) and the amount of TAO to be transferred. + It checks for sufficient balance and prompts for confirmation before proceeding with the transaction. + + ### Example usage: + ``` + btcli wallet transfer --dest 5Dp8... --amount 100 + ``` + + #### Note: + This command is crucial for executing token transfers within the Bittensor network. Users should verify the destination address and amount before confirming the transaction to avoid errors or loss of funds. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + self.initialize_chain(network, chain) + return self._run_command( + wallets.transfer(wallet, self.not_subtensor, destination, amount) + ) + def wallet_regen_coldkey( self, wallet_name: Optional[str] = Options.wallet_name, @@ -508,7 +559,9 @@ def wallet_new_hotkey( wallet_name, wallet_path, wallet_hotkey, validate=False ) n_words = get_n_words(n_words) - return self._run_command(wallets.new_hotkey(wallet, n_words, use_password, overwrite_hotkey)) + return self._run_command( + wallets.new_hotkey(wallet, n_words, use_password, overwrite_hotkey) + ) def wallet_new_coldkey( self, @@ -635,7 +688,9 @@ def wallet_balance( """ subtensor = SubtensorInterface(network, chain) wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) - return self._run_command(wallets.wallet_balance(wallet, subtensor, all_balances)) + return self._run_command( + wallets.wallet_balance(wallet, subtensor, all_balances) + ) def wallet_history( self, @@ -671,7 +726,9 @@ def delegates_list( ): if not wallet_name: wallet_name = typer.prompt("Please enter the wallet name") - return self._run_command(delegates.ListDelegatesCommand.run(wallet_name, network)) + return self._run_command( + delegates.ListDelegatesCommand.run(wallet_name, network) + ) def run(self): self.app() diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index 61866c61..4abd7cb1 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -413,7 +413,9 @@ async def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool): elif reuse: if self.last_block_hash: return self.last_block_hash - return await self.get_chain_head() # also sets the last_block_hash to chain_head + return ( + await self.get_chain_head() + ) # also sets the last_block_hash to chain_head async def init_runtime( self, block_hash: Optional[str] = None, block_id: Optional[int] = None @@ -701,7 +703,7 @@ async def get_block_hash(self, block_id: int) -> str: return (await self.rpc_request("chain_getBlockHash", [block_id]))["result"] async def get_chain_head(self) -> str: - result = (await self._make_rpc_request( + result = await self._make_rpc_request( [ self.make_payload( "rpc_request", @@ -713,8 +715,9 @@ async def get_chain_head(self) -> str: self.chain, self.substrate.runtime_config, self.substrate.metadata, - self.type_registry) - )) + self.type_registry, + ), + ) self.last_block_hash = result["rpc_request"][0]["result"] return self.last_block_hash diff --git a/src/bittensor/extrinsics/transfer.py b/src/bittensor/extrinsics/transfer.py new file mode 100644 index 00000000..bef239c1 --- /dev/null +++ b/src/bittensor/extrinsics/transfer.py @@ -0,0 +1,127 @@ +import asyncio +from typing import Union + +from bittensor_wallet import Wallet +from rich.prompt import Confirm + +from src.subtensor_interface import SubtensorInterface +from src.bittensor.balances import Balance +from src.utils import console, err_console + + +async def transfer_extrinsic( + subtensor: SubtensorInterface, + wallet: Wallet, + destination: Union[str, bytes], + amount: Balance, + wait_for_inclusion: bool = True, + wait_for_finalization: bool = False, + keep_alive: bool = True, + prompt: bool = False, +) -> bool: + """Transfers funds from this wallet to the destination public key address. + + :param subtensor: SubtensorInterface object used for transfer + :param wallet: Bittensor wallet object to make transfer from. + :param destination: Destination public key address (ss58_address or ed25519) of recipient. + :param amount: Amount to stake as Bittensor balance. + :param wait_for_inclusion: If set, waits for the extrinsic to enter a block before returning `True`, + or returns `False` if the extrinsic fails to enter the block within the timeout. + :param wait_for_finalization: If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + :param keep_alive: If set, keeps the account alive by keeping the balance above the existential deposit. + :param prompt: If `True`, the call waits for confirmation from the user before proceeding. + :return: success: Flag is `True` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is `True`, regardless of its inclusion. + """ + # Validate destination address. + if not is_valid_bittensor_address_or_public_key(destination): # TODO + err_console.print( + f":cross_mark: [red]Invalid destination address[/red]:[bold white]\n {destination}[/bold white]" + ) + return False + + if isinstance(destination, bytes): + # Convert bytes to hex string. + destination = f"0x{destination.hex()}" + + # Unlock wallet coldkey. + wallet.unlock_coldkey() + + # Convert to bittensor.Balance + transfer_balance = amount + + # Check balance. + with console.status(":satellite: Checking balance and fees..."): + # check existential deposit and fee + account_balance, existential_deposit, fee = await asyncio.gather( + subtensor.get_balance(wallet.coldkey.ss58_address), + subtensor.get_existential_deposit(), # TODO + subtensor.get_transfer_fee( + wallet=wallet, dest=destination, value=transfer_balance.rao + ), # TODO + ) + + if not keep_alive: + # Check if the transfer should keep_alive the account + existential_deposit = Balance(0) + + # Check if we have enough balance. + if account_balance < (transfer_balance + fee + existential_deposit): + err_console.print( + ":cross_mark: [red]Not enough balance[/red]:[bold white]\n" + f" balance: {account_balance}\n" + f" amount: {transfer_balance}\n" + f" for fee: {fee}[/bold white]" + ) + return False + + # Ask before moving on. + if prompt: + if not Confirm.ask( + "Do you want to transfer:[bold white]\n" + f" amount: {transfer_balance}\n" + f" from: {wallet.name}:{wallet.coldkey.ss58_address}\n" + f" to: {destination}\n for fee: {fee}[/bold white]" + ): + return False + + with console.status(":satellite: Transferring..."): + success, block_hash, err_msg = await subtensor._do_transfer( # TODO + wallet, + destination, + transfer_balance, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, + ) + + if success: + console.print(":white_heavy_check_mark: [green]Finalized[/green]") + console.print(f"[green]Block Hash: {block_hash}[/green]") + + explorer_urls = bittensor.utils.get_explorer_url_for_network( # TODO + subtensor.network, + block_hash, + bittensor.__network_explorer_map__, # TODO + ) + if explorer_urls != {} and explorer_urls: + console.print( + f"[green]Opentensor Explorer Link: {explorer_urls.get('opentensor')}[/green]" + ) + console.print( + f"[green]Taostats Explorer Link: {explorer_urls.get('taostats')}[/green]" + ) + else: + console.print(f":cross_mark: [red]Failed[/red]: {err_msg}") + + if success: + with console.status(":satellite: Checking Balance..."): + new_balance = await subtensor.get_balance( + wallet.coldkey.ss58_address, reuse_block=False + ) + console.print( + f"Balance:\n [blue]{account_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + ) + return True + + return False diff --git a/src/utils.py b/src/utils.py index ecca3f86..41f5e11f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -41,7 +41,10 @@ def get_hotkey_wallets_for_wallet( hotkey_wallets.append(hotkey_for_name) elif show_nulls: hotkey_wallets.append(None) - except (UnicodeDecodeError, AttributeError): # usually an unrelated file like .DS_Store + except ( + UnicodeDecodeError, + AttributeError, + ): # usually an unrelated file like .DS_Store continue return hotkey_wallets diff --git a/src/wallets.py b/src/wallets.py index 0eebb454..6d9ecc3e 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -38,6 +38,7 @@ from . import defaults from src.subtensor_interface import SubtensorInterface from src.bittensor.balances import Balance +from src.bittensor.extrinsics.transfer import transfer_extrinsic async def regen_coldkey( @@ -430,11 +431,13 @@ async def _get_total_balance( ( await subtensor.get_balance( *(x.coldkeypub.ss58_address for x in _balance_cold_wallets), - reuse_block=True + reuse_block=True, ) ).values() ) - all_hotkeys = [hk for w in cold_wallets for hk in utils.get_hotkey_wallets_for_wallet(w)] + all_hotkeys = [ + hk for w in cold_wallets for hk in utils.get_hotkey_wallets_for_wallet(w) + ] else: # We are only printing keys for a single coldkey coldkey_wallet = wallet @@ -445,8 +448,7 @@ async def _get_total_balance( total_balance = sum( ( await subtensor.get_balance( - coldkey_wallet.coldkeypub.ss58_address, - reuse_block=True + coldkey_wallet.coldkeypub.ss58_address, reuse_block=True ) ).values() ) @@ -497,7 +499,7 @@ async def overview( neurons: dict[str, list[NeuronInfoLite]] = {} block, all_netuids = await asyncio.gather( subtensor.substrate.get_block_number(None), - subtensor.get_all_subnet_netuids() + subtensor.get_all_subnet_netuids(), ) netuids = await subtensor.filter_netuids_by_registered_hotkeys( @@ -541,7 +543,9 @@ async def overview( ) ) difference = ( - total_coldkey_stake_from_chain[coldkey_wallet.coldkeypub.ss58_address] + total_coldkey_stake_from_chain[ + coldkey_wallet.coldkeypub.ss58_address + ] - total_coldkey_stake_from_metagraph[ coldkey_wallet.coldkeypub.ss58_address ] @@ -1069,3 +1073,12 @@ async def _filter_stake_info(stake_info: StakeInfo) -> bool: ] return coldkey_wallet, result, None + + +async def transfer( + wallet: Wallet, subtensor: SubtensorInterface, destination: str, amount: float +): + """Transfer token of amount to destination.""" + await transfer_extrinsic( + subtensor, wallet, destination, Balance.from_tao(amount), prompt=True + ) From 6c7e68b7c9c2773ef36a9bc7ef7a78959947738f Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 30 Jul 2024 23:33:03 +0200 Subject: [PATCH 26/48] aenter subtensor --- src/bittensor/extrinsics/transfer.py | 37 +++++++++++++++------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/bittensor/extrinsics/transfer.py b/src/bittensor/extrinsics/transfer.py index bef239c1..4fe516b6 100644 --- a/src/bittensor/extrinsics/transfer.py +++ b/src/bittensor/extrinsics/transfer.py @@ -53,14 +53,15 @@ async def transfer_extrinsic( # Check balance. with console.status(":satellite: Checking balance and fees..."): + async with subtensor: # check existential deposit and fee - account_balance, existential_deposit, fee = await asyncio.gather( - subtensor.get_balance(wallet.coldkey.ss58_address), - subtensor.get_existential_deposit(), # TODO - subtensor.get_transfer_fee( - wallet=wallet, dest=destination, value=transfer_balance.rao - ), # TODO - ) + account_balance, existential_deposit, fee = await asyncio.gather( + subtensor.get_balance(wallet.coldkey.ss58_address), + subtensor.get_existential_deposit(), # TODO + subtensor.get_transfer_fee( + wallet=wallet, dest=destination, value=transfer_balance.rao + ), # TODO + ) if not keep_alive: # Check if the transfer should keep_alive the account @@ -87,13 +88,14 @@ async def transfer_extrinsic( return False with console.status(":satellite: Transferring..."): - success, block_hash, err_msg = await subtensor._do_transfer( # TODO - wallet, - destination, - transfer_balance, - wait_for_finalization=wait_for_finalization, - wait_for_inclusion=wait_for_inclusion, - ) + async with subtensor: + success, block_hash, err_msg = await subtensor._do_transfer( # TODO + wallet, + destination, + transfer_balance, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, + ) if success: console.print(":white_heavy_check_mark: [green]Finalized[/green]") @@ -116,9 +118,10 @@ async def transfer_extrinsic( if success: with console.status(":satellite: Checking Balance..."): - new_balance = await subtensor.get_balance( - wallet.coldkey.ss58_address, reuse_block=False - ) + async with subtensor: + new_balance = await subtensor.get_balance( + wallet.coldkey.ss58_address, reuse_block=False + ) console.print( f"Balance:\n [blue]{account_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" ) From 060dca2a4a3a1119c75d28bdbfa779d6f527b7ed Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 11:21:13 +0200 Subject: [PATCH 27/48] Wallet transfer finished. --- src/__init__.py | 13 +++ src/bittensor/async_substrate_interface.py | 8 +- src/bittensor/extrinsics/transfer.py | 109 +++++++++++++++------ src/subtensor_interface.py | 28 ++++++ src/utils.py | 98 +++++++++++++++--- 5 files changed, 208 insertions(+), 48 deletions(-) diff --git a/src/__init__.py b/src/__init__.py index 14874191..f72947db 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -163,3 +163,16 @@ class Defaults: }, }, } + +NETWORK_EXPLORER_MAP = { + "opentensor": { + "local": "https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fentrypoint-finney.opentensor.ai%3A443#/explorer", + "endpoint": "https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fentrypoint-finney.opentensor.ai%3A443#/explorer", + "finney": "https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fentrypoint-finney.opentensor.ai%3A443#/explorer", + }, + "taostats": { + "local": "https://x.taostats.io", + "endpoint": "https://x.taostats.io", + "finney": "https://x.taostats.io", + }, +} diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index 4abd7cb1..2bd0b969 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -943,7 +943,11 @@ async def get_account_nonce(self, account_address: str) -> int: return nonce_obj.value async def get_constant( - self, module_name: str, constant_name: str, block_hash: Optional[str] = None + self, + module_name: str, + constant_name: str, + block_hash: Optional[str] = None, + reuse_block_hash: bool = False, ) -> Optional["ScaleType"]: """ Returns the decoded `ScaleType` object of the constant for given module name, call function name and block_hash @@ -954,9 +958,11 @@ async def get_constant( :param module_name: Name of the module to query :param constant_name: Name of the constant to query :param block_hash: Hash of the block at which to make the runtime API call + :param reuse_block_hash: Reuse last-used block hash if set to true :return: ScaleType from the runtime call """ + block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) async with self._lock: return await asyncio.get_event_loop().run_in_executor( None, diff --git a/src/bittensor/extrinsics/transfer.py b/src/bittensor/extrinsics/transfer.py index 4fe516b6..2fa64d0d 100644 --- a/src/bittensor/extrinsics/transfer.py +++ b/src/bittensor/extrinsics/transfer.py @@ -1,18 +1,24 @@ import asyncio -from typing import Union from bittensor_wallet import Wallet from rich.prompt import Confirm +from src import NETWORK_EXPLORER_MAP from src.subtensor_interface import SubtensorInterface from src.bittensor.balances import Balance -from src.utils import console, err_console +from src.utils import ( + console, + err_console, + is_valid_bittensor_address_or_public_key, + get_explorer_url_for_network, + format_error_message, +) async def transfer_extrinsic( subtensor: SubtensorInterface, wallet: Wallet, - destination: Union[str, bytes], + destination: str, amount: Balance, wait_for_inclusion: bool = True, wait_for_finalization: bool = False, @@ -34,33 +40,81 @@ async def transfer_extrinsic( :return: success: Flag is `True` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is `True`, regardless of its inclusion. """ + + async def get_transfer_fee() -> Balance: + """ + Calculates the transaction fee for transferring tokens from a wallet to a specified destination address. + This function simulates the transfer to estimate the associated cost, taking into account the current + network conditions and transaction complexity. + """ + call = await subtensor.substrate.compose_call( + call_module="Balances", + call_function="transfer_allow_death", + call_params={"dest": destination, "value": amount.rao}, + ) + + try: + payment_info = await subtensor.substrate.get_payment_info( + call=call, keypair=wallet.coldkeypub + ) + except Exception as e: + payment_info = {"partialFee": int(2e7)} # assume 0.02 Tao + err_console.print( + f":cross_mark: [red]Failed to get payment info[/red]:[bold white]\n" + f" {e}[/bold white]\n" + f" Defaulting to default transfer fee: {payment_info['partialFee']}" + ) + + return Balance.from_rao(payment_info["partialFee"]) + + async def do_transfer() -> tuple[bool, str, str]: + """ + Makes transfer from wallet to destination public key address. + :return: success, block hash, formatted error message + """ + call = await subtensor.substrate.compose_call( + call_module="Balances", + call_function="transfer_allow_death", + call_params={"dest": destination, "value": amount.rao}, + ) + extrinsic = await subtensor.substrate.create_signed_extrinsic( + call=call, keypair=wallet.coldkey + ) + response = await subtensor.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, "", "" + + # Otherwise continue with finalization. + response.process_events() + if response.is_success: + block_hash_ = response.block_hash + return True, block_hash_, "" + else: + return False, "", format_error_message(response.error_message) + # Validate destination address. - if not is_valid_bittensor_address_or_public_key(destination): # TODO + if not is_valid_bittensor_address_or_public_key(destination): err_console.print( f":cross_mark: [red]Invalid destination address[/red]:[bold white]\n {destination}[/bold white]" ) return False - if isinstance(destination, bytes): - # Convert bytes to hex string. - destination = f"0x{destination.hex()}" - # Unlock wallet coldkey. wallet.unlock_coldkey() - # Convert to bittensor.Balance - transfer_balance = amount - # Check balance. with console.status(":satellite: Checking balance and fees..."): async with subtensor: - # check existential deposit and fee + # check existential deposit and fee account_balance, existential_deposit, fee = await asyncio.gather( subtensor.get_balance(wallet.coldkey.ss58_address), - subtensor.get_existential_deposit(), # TODO - subtensor.get_transfer_fee( - wallet=wallet, dest=destination, value=transfer_balance.rao - ), # TODO + subtensor.get_existential_deposit(reuse_block=True), + get_transfer_fee(), ) if not keep_alive: @@ -68,11 +122,11 @@ async def transfer_extrinsic( existential_deposit = Balance(0) # Check if we have enough balance. - if account_balance < (transfer_balance + fee + existential_deposit): + if account_balance < (amount + fee + existential_deposit): err_console.print( ":cross_mark: [red]Not enough balance[/red]:[bold white]\n" f" balance: {account_balance}\n" - f" amount: {transfer_balance}\n" + f" amount: {amount}\n" f" for fee: {fee}[/bold white]" ) return False @@ -81,7 +135,7 @@ async def transfer_extrinsic( if prompt: if not Confirm.ask( "Do you want to transfer:[bold white]\n" - f" amount: {transfer_balance}\n" + f" amount: {amount}\n" f" from: {wallet.name}:{wallet.coldkey.ss58_address}\n" f" to: {destination}\n for fee: {fee}[/bold white]" ): @@ -89,22 +143,14 @@ async def transfer_extrinsic( with console.status(":satellite: Transferring..."): async with subtensor: - success, block_hash, err_msg = await subtensor._do_transfer( # TODO - wallet, - destination, - transfer_balance, - wait_for_finalization=wait_for_finalization, - wait_for_inclusion=wait_for_inclusion, - ) + success, block_hash, err_msg = await do_transfer() if success: console.print(":white_heavy_check_mark: [green]Finalized[/green]") console.print(f"[green]Block Hash: {block_hash}[/green]") - explorer_urls = bittensor.utils.get_explorer_url_for_network( # TODO - subtensor.network, - block_hash, - bittensor.__network_explorer_map__, # TODO + explorer_urls = get_explorer_url_for_network( + subtensor.network, block_hash, NETWORK_EXPLORER_MAP ) if explorer_urls != {} and explorer_urls: console.print( @@ -123,7 +169,8 @@ async def transfer_extrinsic( wallet.coldkey.ss58_address, reuse_block=False ) console.print( - f"Balance:\n [blue]{account_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + f"Balance:\n" + f" [blue]{account_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" ) return True diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 6182fe9a..0537b1e1 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -378,3 +378,31 @@ async def filter_netuids_by_registered_hotkeys( all_netuids.extend(netuids_with_registered_hotkeys) return list(set(all_netuids)) + + async def get_existential_deposit( + self, block_hash: Optional[str] = None, reuse_block: bool = False + ) -> Optional[Balance]: + """ + Retrieves the existential deposit amount for the Bittensor blockchain. The existential deposit + is the minimum amount of TAO required for an account to exist on the blockchain. Accounts with + balances below this threshold can be reaped to conserve network resources. + + :param block_hash: Block hash at which to query the deposit amount. If `None`, the current block is used. + :param reuse_block: Whether to reuse the last-used blockchain block hash. + + :return: The existential deposit amount + + The existential deposit is a fundamental economic parameter in the Bittensor network, ensuring + efficient use of storage and preventing the proliferation of dust accounts. + """ + result = await self.substrate.get_constant( + module_name="Balances", + constant_name="ExistentialDeposit", + block_hash=block_hash, + reuse_block_hash=reuse_block, + ) + + if result is None or not hasattr(result, "value"): + raise Exception("Unable to retrieve existential deposit amount.") + + return Balance.from_rao(result.value) diff --git a/src/utils.py b/src/utils.py index 41f5e11f..4718ebcb 100644 --- a/src/utils.py +++ b/src/utils.py @@ -78,7 +78,9 @@ def get_all_wallets_for_path(path: str) -> list[Wallet]: def is_valid_wallet(wallet: Wallet) -> tuple[bool, bool]: """ Verifies that the wallet with specified parameters. + :param wallet: a Wallet instance + :return: tuple[bool], whether wallet appears valid, whether valid hotkey in wallet """ return ( @@ -96,11 +98,9 @@ def is_valid_ss58_address(address: str) -> bool: """ Checks if the given address is a valid ss58 address. - Args: - address(str): The address to check. + :param address: The address to check. - Returns: - True if the address is a valid ss58 address for Bittensor, False otherwise. + :return: `True` if the address is a valid ss58 address for Bittensor, `False` otherwise. """ try: return ss58.is_valid_ss58_address( @@ -116,11 +116,9 @@ def is_valid_ed25519_pubkey(public_key: Union[str, bytes]) -> bool: """ Checks if the given public_key is a valid ed25519 key. - Args: - public_key(Union[str, bytes]): The public_key to check. + :param public_key: The public_key to check. - Returns: - True if the public_key is a valid ed25519 key, False otherwise. + :return: True if the public_key is a valid ed25519 key, False otherwise. """ try: @@ -146,11 +144,9 @@ def is_valid_bittensor_address_or_public_key(address: Union[str, bytes]) -> bool """ Checks if the given address is a valid destination address. - Args: - address(Union[str, bytes]): The address to check. + :param address: The address to check. - Returns: - True if the address is a valid destination address, False otherwise. + :return: True if the address is a valid destination address, False otherwise. """ if isinstance(address, str): # Check if ed25519 @@ -191,12 +187,82 @@ def ss58_to_vec_u8(ss58_address: str) -> list[int]: """ Converts an SS58 address to a list of integers (vector of u8). - Args: - ss58_address (str): The SS58 address to be converted. + :param ss58_address: The SS58 address to be converted. - Returns: - List[int]: A list of integers representing the byte values of the SS58 address. + :return: A list of integers representing the byte values of the SS58 address. """ ss58_bytes: bytes = ss58_address_to_bytes(ss58_address) encoded_address: list[int] = [int(byte) for byte in ss58_bytes] return encoded_address + + +def get_explorer_root_url_by_network_from_map( + network: str, network_map: dict[str, dict[str, str]] +) -> dict[str, str]: + r""" + Returns the explorer root url for the given network name from the given network map. + + :param network: The network to get the explorer url for. + :param network_map: The network map to get the explorer url from. + + :return: The explorer url for the given network. + """ + explorer_urls: dict[str, str] = {} + for entity_nm, entity_network_map in network_map.items(): + if network in entity_network_map: + explorer_urls[entity_nm] = entity_network_map[network] + + return explorer_urls + + +def get_explorer_url_for_network( + network: str, block_hash: str, network_map: dict[str, str] +) -> dict[str, str]: + r""" + Returns the explorer url for the given block hash and network. + + :param network: The network to get the explorer url for. + :param block_hash: The block hash to get the explorer url for. + :param network_map: The network maps to get the explorer urls from. + + :return: The explorer url for the given block hash and network + """ + + explorer_urls: dict[str, str] = {} + # Will be None if the network is not known. i.e. not in network_map + explorer_root_urls: dict[str, str] = get_explorer_root_url_by_network_from_map( + network, network_map + ) + + if explorer_root_urls != {}: + # We are on a known network. + explorer_opentensor_url = "{root_url}/query/{block_hash}".format( + root_url=explorer_root_urls.get("opentensor"), block_hash=block_hash + ) + explorer_taostats_url = "{root_url}/extrinsic/{block_hash}".format( + root_url=explorer_root_urls.get("taostats"), block_hash=block_hash + ) + explorer_urls["opentensor"] = explorer_opentensor_url + explorer_urls["taostats"] = explorer_taostats_url + + return explorer_urls + + +def format_error_message(error_message: dict) -> str: + """ + Formats an error message from the Subtensor error information to using in extrinsics. + + :param error_message: A dictionary containing the error information from Subtensor. + + :return: A formatted error message string. + """ + err_type = "UnknownType" + err_name = "UnknownError" + err_description = "Unknown Description" + + if isinstance(error_message, dict): + err_type = error_message.get("type", err_type) + err_name = error_message.get("name", err_name) + err_docs = error_message.get("docs", []) + err_description = err_docs[0] if len(err_docs) > 0 else err_description + return f"Subtensor returned `{err_name} ({err_type})` error. This means: `{err_description}`" From f3bf765004111e826c6f58318079ffd5e1ea57ba Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 17:12:46 +0200 Subject: [PATCH 28/48] [WIP] Check-in --- cli.py | 106 +++++++++++++++++++++++++---- src/__init__.py | 21 ++++++ src/subtensor_interface.py | 79 ++++++++++++++++------ src/utils.py | 20 +++++- src/wallets.py | 135 +++++++++++++++++++++++++++++++++++-- 5 files changed, 321 insertions(+), 40 deletions(-) diff --git a/cli.py b/cli.py index 05efc15a..5e5ff11e 100755 --- a/cli.py +++ b/cli.py @@ -65,6 +65,7 @@ class Options: defaults.subtensor.chain_endpoint, help="The subtensor chain endpoint to connect to.", ) + netuids = typer.Option([], help="Set the netuid(s) to filter by (e.g. `0 1 2`)") def get_n_words(n_words: Optional[int]) -> int: @@ -230,9 +231,7 @@ def wallet_overview( help="Specify the hotkeys to exclude by name or ss58 address. (e.g. `hk1 hk2 hk3`). " "If left empty, and no hotkeys included in --include-hotkeys, all hotkeys will be included.", ), - netuids: Optional[list[int]] = typer.Option( - [], help="Set the netuid(s) to filter by (e.g. `0 1 2`)" - ), + netuids: Optional[list[int]] = Options.netuids, network: Optional[str] = Options.network, chain: Optional[str] = Options.chain, ): @@ -298,11 +297,10 @@ def wallet_overview( ``` #### Note: - This command is read-only and does not modify the network state or account configurations. It provides a quick and - comprehensive view of the user's network presence, making it ideal for monitoring account status, stake distribution, - and overall contribution to the Bittensor network. + This command is read-only and does not modify the network state or account configurations. It provides a quick + and comprehensive view of the user's network presence, making it ideal for monitoring account status, stake + distribution, and overall contribution to the Bittensor network. """ - # TODO verify this is actually doing all wallets if include_hotkeys and exclude_hotkeys: utils.err_console.print( "[red]You have specified hotkeys for inclusion and exclusion. Pick only one or neither." @@ -370,7 +368,8 @@ def wallet_transfer( ``` #### Note: - This command is crucial for executing token transfers within the Bittensor network. Users should verify the destination address and amount before confirming the transaction to avoid errors or loss of funds. + This command is crucial for executing token transfers within the Bittensor network. Users should verify the + destination address and amount before confirming the transaction to avoid errors or loss of funds. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) self.initialize_chain(network, chain) @@ -378,6 +377,87 @@ def wallet_transfer( wallets.transfer(wallet, self.not_subtensor, destination, amount) ) + def wallet_inspect( + self, + all_wallets: bool = typer.Option( + False, + "--all", + "--all-wallets", + "-a", + help="Inspect all wallets within specified path.", + ), + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + netuids: Optional[list[int]] = Options.netuids, + ): + """ + # wallet inspect + Executes the ``inspect`` command, which compiles and displays a detailed report of a user's wallet pairs + (coldkey, hotkey) on the Bittensor network. + + This report includes balance and staking information for both the coldkey and hotkey associated with the wallet. + + The command gathers data on: + + - Coldkey balance and delegated stakes. + - Hotkey stake and emissions per neuron on the network. + - Delegate names and details fetched from the network. + + The resulting table includes columns for: + + - **Coldkey**: The coldkey associated with the user's wallet. + + - **Balance**: The balance of the coldkey. + + - **Delegate**: The name of the delegate to which the coldkey has staked funds. + + - **Stake**: The amount of stake held by both the coldkey and hotkey. + + - **Emission**: The emission or rewards earned from staking. + + - **Netuid**: The network unique identifier of the subnet where the hotkey is active. + + - **Hotkey**: The hotkey associated with the neuron on the network. + + ## Usage: + This command can be used to inspect a single wallet or all wallets located within a + specified path. It is useful for a comprehensive overview of a user's participation + and performance in the Bittensor network. + + #### Example usage:: + ``` + btcli wallet inspect + ``` + + ``` + btcli wallet inspect --all + ``` + + #### Note: + The ``inspect`` command is for displaying information only and does not perform any + transactions or state changes on the Bittensor network. It is intended to be used as + part of the Bittensor CLI and not as a standalone function within user code. + """ + # if all-wallets is entered, ask for path + if all_wallets: + if not wallet_path: + wallet_path = Prompt.ask( + "Enter the path of the wallets", default=defaults.wallet.path + ) + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + self.initialize_chain(network, chain) + self._run_command( + wallets.inspect( + wallet, + self.not_subtensor, + netuids_filter=netuids, + all_wallets=all_wallets, + ) + ) + def wallet_regen_coldkey( self, wallet_name: Optional[str] = Options.wallet_name, @@ -405,8 +485,8 @@ def wallet_regen_coldkey( btcli wallet regen_coldkey --mnemonic "word1 word2 ... word12" ``` - ### Note: This command is critical for users who need to regenerate their coldkey, possibly for recovery or security - reasons. It should be used with caution to avoid overwriting existing keys unintentionally. + ### Note: This command is critical for users who need to regenerate their coldkey, possibly for recovery or + security reasons. It should be used with caution to avoid overwriting existing keys unintentionally. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) @@ -704,8 +784,8 @@ def wallet_history( This command provides a detailed view of the transfers carried out on the wallet. ## Usage: - The command lists the latest transfers of the provided wallet, showing the From, To, Amount, Extrinsic Id and - Block Number. + The command lists the latest transfers of the provided wallet, showing the 'From', 'To', 'Amount', + 'Extrinsic ID' and 'Block Number'. ### Example usage: ``` @@ -714,7 +794,7 @@ def wallet_history( #### Note: This command is essential for users to monitor their financial status on the Bittensor network. - It helps in fetching info on all the transfers so that user can easily tally and cross check the transactions. + It helps in fetching info on all the transfers so that user can easily tally and cross-check the transactions. """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) return self._run_command(wallets.wallet_history(wallet)) diff --git a/src/__init__.py b/src/__init__.py index f72947db..08a1f7bf 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -12,6 +12,27 @@ class Constants: "test": finney_test_entrypoint, "archive": archive_entrypoint, } + delegates_details_url = ( + "https://raw.githubusercontent.com/opentensor/" + "bittensor-delegates/main/public/delegates.json" + ) + + +@dataclass +class DelegatesDetails: + name: str + url: str + description: str + signature: str + + @classmethod + def from_json(cls, json: dict[str, any]) -> "DelegatesDetails": + return cls( + name=json["name"], + url=json["url"], + description=json["description"], + signature=json["signature"], + ) @dataclass diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 0537b1e1..59884a67 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -7,7 +7,7 @@ from scalecodec.type_registry import load_type_registry_preset from src.bittensor.async_substrate_interface import AsyncSubstrateInterface -from src.bittensor.chain_data import DelegateInfo, custom_rpc_type_registry, StakeInfo +from src.bittensor.chain_data import DelegateInfo, custom_rpc_type_registry, StakeInfo, NeuronInfoLite from src.bittensor.balances import Balance from src import Constants, defaults, TYPE_REGISTRY from src.utils import ss58_to_vec_u8 @@ -59,7 +59,7 @@ async def encode_params( for i, param in enumerate(call_definition["params"]): # type: ignore scale_obj = await self.substrate.create_scale_object(param["type"]) - if type(params) is list: + if isinstance(params, list): param_data += scale_obj.encode(params[i]) else: if param["name"] not in params: @@ -104,11 +104,11 @@ async def is_hotkey_delegate( checks if the neuron associated with the hotkey is part of the network's delegation system. Args: - hotkey_ss58 (str): The SS58 address of the neuron's hotkey. - block_hash (Optional[int], optional): The blockchain block number for the query. + :param hotkey_ss58: The SS58 address of the neuron's hotkey. + :param block_hash: The hash of the blockchain block number for the query. + :param reuse_block: Whether to reuse the last-used block hash. - Returns: - bool: ``True`` if the hotkey is a delegate, ``False`` otherwise. + :return: `True` if the hotkey is a delegate, `False` otherwise. Being a delegate is a significant status within the Bittensor network, indicating a neuron's involvement in consensus and governance processes. @@ -145,11 +145,11 @@ async def get_stake_info_for_coldkey( about the stakes held by an account, including the staked amounts and associated delegates. Args: - coldkey_ss58 (str): The ``SS58`` address of the account's coldkey. - block (Optional[int], optional): The blockchain block number for the query. + :param coldkey_ss58: The ``SS58`` address of the account's coldkey. + :param block_hash: The hash of the blockchain block number for the query. + :param reuse_block: Whether to reuse the last-used block hash. - Returns: - List[StakeInfo]: A list of StakeInfo objects detailing the stake allocations for the account. + :return: A list of StakeInfo objects detailing the stake allocations for the account. Stake information is vital for account holders to assess their investment and participation in the network's delegation and consensus processes. @@ -187,14 +187,13 @@ async def query_runtime_api( runtime and retrieve data encoded in Scale Bytes format. This function is essential for advanced users who need to interact with specific runtime methods and decode complex data types. - Args: - runtime_api (str): The name of the runtime API to query. - method (str): The specific method within the runtime API to call. - params (Optional[List[ParamWithTypes]], optional): The parameters to pass to the method call. - block (Optional[int]): The blockchain block number at which to perform the query. + :param runtime_api: The name of the runtime API to query. + :param method: The specific method within the runtime API to call. + :param params: The parameters to pass to the method call. + :param block_hash: The hash of the blockchain block number at which to perform the query. + :param reuse_block: Whether to reuse the last-used block hash. - Returns: - Optional[bytes]: The Scale Bytes encoded result from the runtime API call, or ``None`` if the call fails. + :return: The Scale Bytes encoded result from the runtime API call, or ``None`` if the call fails. This function enables access to the deeper layers of the Bittensor blockchain, allowing for detailed and specific interactions with the network's runtime environment. @@ -259,7 +258,8 @@ async def get_total_stake_for_coldkey( :param ss58_addresses: The SS58 address(es) of the coldkey(s) :param block: The block number to retrieve the stake from. Currently unused. :param reuse_block: Whether to reuse the last-used block hash when retrieving info. - :return: + + :return: {address: Balance objects} """ results = await self.substrate.query_multiple( params=[s for s in ss58_addresses], @@ -283,11 +283,11 @@ async def get_netuids_for_hotkey( the hotkey is active. Args: - hotkey_ss58 (str): The ``SS58`` address of the neuron's hotkey. - block (Optional[int]): The blockchain block number at which to perform the query. + :param hotkey_ss58: The ``SS58`` address of the neuron's hotkey. + :param block_hash: The hash of the blockchain block number at which to perform the query. + :param reuse_block: Whether to reuse the last-used block hash when retrieving info. - Returns: - List[int]: A list of netuids where the neuron is a member. + :return: A list of netuids where the neuron is a member. """ result = await self.substrate.query_map( module="SubtensorModule", @@ -406,3 +406,38 @@ async def get_existential_deposit( raise Exception("Unable to retrieve existential deposit amount.") return Balance.from_rao(result.value) + + async def neurons_lite( + self, netuid: int, block_hash: Optional[str] = None, reuse_block: bool = False + ) -> list[NeuronInfoLite]: + """ + Retrieves a list of neurons in a 'lite' format from a specific subnet of the Bittensor network. + This function provides a streamlined view of the neurons, focusing on key attributes such as stake + and network participation. + + :param netuid: The unique identifier of the subnet. + :param block_hash: The hash of the blockchain block number for the query. + :param reuse_block: Whether to reuse the last-used blockchain block hash. + + :return: A list of simplified neuron information for the subnet. + + This function offers a quick overview of the neuron population within a subnet, facilitating + efficient analysis of the network's decentralized structure and neuron dynamics. + """ + hex_bytes_result = await self.query_runtime_api( + runtime_api="NeuronInfoRuntimeApi", + method="get_neurons_lite", + params=[netuid], + block_hash=block_hash, + reuse_block=reuse_block + ) + + if hex_bytes_result is None: + return [] + + if hex_bytes_result.startswith("0x"): + bytes_result = bytes.fromhex(hex_bytes_result[2:]) + else: + bytes_result = bytes.fromhex(hex_bytes_result) + + return NeuronInfoLite.list_from_vec_u8(bytes_result) # type: ignore diff --git a/src/utils.py b/src/utils.py index 4718ebcb..e1e344eb 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,7 +1,8 @@ import os from pathlib import Path -from typing import Union +from typing import Union, Optional, Any +import aiohttp import scalecodec from bittensor_wallet import Wallet from bittensor_wallet.keyfile import Keypair @@ -10,6 +11,8 @@ from scalecodec.base import RuntimeConfiguration from scalecodec.type_registry import load_type_registry_preset +from src import DelegatesDetails + console = Console() err_console = Console(stderr=True) @@ -266,3 +269,18 @@ def format_error_message(error_message: dict) -> str: err_docs = error_message.get("docs", []) err_description = err_docs[0] if len(err_docs) > 0 else err_description return f"Subtensor returned `{err_name} ({err_type})` error. This means: `{err_description}`" + + +async def get_delegates_details_from_github(url: str) -> dict[str, DelegatesDetails]: + async with aiohttp.ClientSession() as session: + response = await session.get(url) + + all_delegates_details = {} + if response.ok: + all_delegates: dict[str, Any] = await response.json() + for delegate_hotkey, delegates_details in all_delegates.items(): + all_delegates_details[delegate_hotkey] = DelegatesDetails.from_json( + delegates_details + ) + + return all_delegates_details diff --git a/src/wallets.py b/src/wallets.py index 6d9ecc3e..253d5af2 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -31,11 +31,18 @@ import scalecodec import typer -from src import utils, TYPE_REGISTRY -from src.bittensor.chain_data import NeuronInfoLite, custom_rpc_type_registry, StakeInfo +from src import utils, TYPE_REGISTRY, DelegatesDetails, Constants +from src.bittensor.chain_data import NeuronInfoLite, custom_rpc_type_registry, StakeInfo, DelegateInfo from src.bittensor.networking import int_to_ip -from src.utils import console, err_console, RAO_PER_TAO, decode_scale_bytes -from . import defaults +from src.utils import ( + console, + err_console, + RAO_PER_TAO, + decode_scale_bytes, + get_all_wallets_for_path, + get_hotkey_wallets_for_wallet, + get_delegates_details_from_github, +) from src.subtensor_interface import SubtensorInterface from src.bittensor.balances import Balance from src.bittensor.extrinsics.transfer import transfer_extrinsic @@ -1082,3 +1089,123 @@ async def transfer( await transfer_extrinsic( subtensor, wallet, destination, Balance.from_tao(amount), prompt=True ) + + +async def inspect( + wallet: Wallet, + subtensor: SubtensorInterface, + netuids_filter: Optional[list[int]] = None, + all_wallets: bool = False, +): + def delegate_row_maker(delegates_: list[tuple[DelegateInfo, Balance]]) -> list[str]: + for delegate_, staked in delegates_: + if delegate_.hotkey_ss58 in registered_delegate_info: + delegate_name = registered_delegate_info[delegate_.hotkey_ss58].name + else: + delegate_name = delegate_.hotkey_ss58 + yield [ + "", + "", + str(delegate_name), + str(staked), + str( + delegate_.total_daily_return.tao + * (staked.tao / delegate_.total_stake.tao) + ), + "", + "", + "", + "", + ] + + if all_wallets: + wallets = _get_coldkey_wallets_for_path(wallet.path) + all_hotkeys = get_all_wallets_for_path( + wallet.path + ) # TODO verify this is correct + else: + wallets = [wallet] + all_hotkeys = get_hotkey_wallets_for_wallet(wallet) + + all_netuids = subtensor.get_all_subnet_netuids() + async with subtensor: + block, all_netuids = await subtensor.filter_netuids_by_registered_hotkeys( + all_netuids, netuids_filter, all_hotkeys, reuse_block=False + ) + # bittensor.logging.debug(f"Netuids to check: {all_netuids}") + + registered_delegate_info: Optional[ + dict[str, DelegatesDetails] + ] = await get_delegates_details_from_github(url=Constants.delegates_details_url) + if not registered_delegate_info: + console.print( + ":warning:[yellow]Could not get delegate info from chain.[/yellow]" + ) + + neuron_state_dict = {} + for netuid in tqdm(all_netuids): + neurons = subtensor.neurons_lite(netuid) + neuron_state_dict[netuid] = neurons if neurons else [] + + table = Table( + Column("[overline white]Coldkey", style="bold white"), + Column("[overline white]Balance", style="green"), + Column("[overline white]Delegate", style="blue"), + Column("[overline white]Stake", style="green"), + Column("[overline white]Emission", style="green"), + Column("[overline white]Netuid", style="bold white"), + Column("[overline white]Hotkey", style="yellow"), + Column("[overline white]Stake", style="green"), + Column("[overline white]Emission", style="green"), + show_footer=True, + pad_edge=False, + box=None, + expand=True, + footer_style="overline white", + ) + rows = [] + for wallet in wallets: + delegates: list[tuple[DelegateInfo, Balance]] = subtensor.get_delegated( + coldkey_ss58=wallet.coldkeypub.ss58_address + ) + if not wallet.coldkeypub_file.exists_on_device(): + continue + cold_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address) + rows.append([wallet.name, str(cold_balance), "", "", "", "", "", "", ""]) + for row in delegate_row_maker(delegates): + rows.append(row) + + hotkeys = get_hotkey_wallets_for_wallet(wallet) + for netuid in all_netuids: + for neuron in neuron_state_dict[netuid]: + if neuron.coldkey == wallet.coldkeypub.ss58_address: + hotkey_name: str = "" + + hotkey_names: list[str] = [ + wallet.hotkey_str + for wallet in filter( + lambda hotkey: hotkey.hotkey.ss58_address == neuron.hotkey, + hotkeys, + ) + ] + if hotkey_names: + hotkey_name = f"{hotkey_names[0]}-" + + rows.append( + [ + "", + "", + "", + "", + "", + str(netuid), + f"{hotkey_name}{neuron.hotkey}", + str(neuron.stake), + str(Balance.from_tao(neuron.emission)), + ] + ) + + for row in rows: + table.add_row(*row) + + console.print(table) From 3a6c1e14ae31408624a5ae9acd66ab0c41b157e9 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 17:13:17 +0200 Subject: [PATCH 29/48] Ruff --- src/subtensor_interface.py | 11 ++++++++--- src/wallets.py | 7 ++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 59884a67..63f8dd57 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -7,7 +7,12 @@ from scalecodec.type_registry import load_type_registry_preset from src.bittensor.async_substrate_interface import AsyncSubstrateInterface -from src.bittensor.chain_data import DelegateInfo, custom_rpc_type_registry, StakeInfo, NeuronInfoLite +from src.bittensor.chain_data import ( + DelegateInfo, + custom_rpc_type_registry, + StakeInfo, + NeuronInfoLite, +) from src.bittensor.balances import Balance from src import Constants, defaults, TYPE_REGISTRY from src.utils import ss58_to_vec_u8 @@ -407,7 +412,7 @@ async def get_existential_deposit( return Balance.from_rao(result.value) - async def neurons_lite( + async def neurons_lite( self, netuid: int, block_hash: Optional[str] = None, reuse_block: bool = False ) -> list[NeuronInfoLite]: """ @@ -429,7 +434,7 @@ async def neurons_lite( method="get_neurons_lite", params=[netuid], block_hash=block_hash, - reuse_block=reuse_block + reuse_block=reuse_block, ) if hex_bytes_result is None: diff --git a/src/wallets.py b/src/wallets.py index 253d5af2..3a3eac67 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -32,7 +32,12 @@ import typer from src import utils, TYPE_REGISTRY, DelegatesDetails, Constants -from src.bittensor.chain_data import NeuronInfoLite, custom_rpc_type_registry, StakeInfo, DelegateInfo +from src.bittensor.chain_data import ( + NeuronInfoLite, + custom_rpc_type_registry, + StakeInfo, + DelegateInfo, +) from src.bittensor.networking import int_to_ip from src.utils import ( console, From 9dcde05f1b05069b41162c41a2d7a4056093ae0f Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 17:19:35 +0200 Subject: [PATCH 30/48] Clean-up --- requirements.txt | 7 +++++-- src/bittensor/balances.py | 18 ++++++------------ src/bittensor/chain_data.py | 14 ++++++-------- src/bittensor/networking.py | 9 +++------ 4 files changed, 20 insertions(+), 28 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7a456358..3fa37bf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ +aiohttp~=3.9.5 +git+https://github.com/opentensor/btwallet # bittensor_wallet +fuzzywuzzy~=0.18.0 +netaddr~=1.3.0 +rich~=13.7 scalecodec==1.2.11 substrate-interface~=1.7.9 typer~=0.12 -rich~=13.7 -git+https://github.com/opentensor/btwallet # bittensor_wallet websockets>=12.0 diff --git a/src/bittensor/balances.py b/src/bittensor/balances.py index 5d7d9347..922c2ab2 100644 --- a/src/bittensor/balances.py +++ b/src/bittensor/balances.py @@ -249,11 +249,9 @@ def to_dict(self) -> dict: def from_float(amount: float): """ Given tao (float), return Balance object with rao(int) and tao(float), where rao = int(tao*pow(10,9)) - Args: - amount: The amount in tao. + :param amount: The amount in tao. - Returns: - A Balance object representing the given amount. + :return: A Balance object representing the given amount. """ rao = int(amount * pow(10, 9)) return Balance(rao) @@ -263,11 +261,9 @@ def from_tao(amount: float): """ Given tao (float), return Balance object with rao(int) and tao(float), where rao = int(tao*pow(10,9)) - Args: - amount: The amount in tao. + :param amount: The amount in tao. - Returns: - A Balance object representing the given amount. + :return: A Balance object representing the given amount. """ rao = int(amount * pow(10, 9)) return Balance(rao) @@ -277,10 +273,8 @@ def from_rao(amount: int): """ Given rao (int), return Balance object with rao(int) and tao(float), where rao = int(tao*pow(10,9)) - Args: - amount: The amount in rao. + :param amount: The amount in rao. - Returns: - A Balance object representing the given amount. + :return: A Balance object representing the given amount. """ return Balance(amount) diff --git a/src/bittensor/chain_data.py b/src/bittensor/chain_data.py index ef594299..3657eca2 100644 --- a/src/bittensor/chain_data.py +++ b/src/bittensor/chain_data.py @@ -8,7 +8,7 @@ from scalecodec.utils.ss58 import ss58_encode from src.bittensor.balances import Balance -from src.utils import RAO_PER_TAO, SS58_FORMAT, u16_normalized_float +from src.utils import SS58_FORMAT, u16_normalized_float class ChainDataType(Enum): @@ -31,14 +31,12 @@ def from_scale_encoding( """ Decodes input_ data from SCALE encoding based on the specified type name and modifiers. - Args: - input_ (Union[list[int], bytes, ScaleBytes]): The input_ data to decode. - type_name (ChainDataType): The type of data being decoded. - is_vec (bool, optional): Whether the data is a vector of the specified type. Default is ``False``. - is_option (bool, optional): Whether the data is an optional value of the specified type. Default is ``False``. + :param input_: The input data to decode. + :param type_name:The type of data being decoded. + :param is_vec:: Whether the data is a vector of the specified type. + :param is_option: Whether the data is an optional value of the specified type. - Returns: - Optional[Dict]: The decoded data as a dictionary, or ``None`` if the decoding fails. + :return: The decoded data as a dictionary, or `None` if the decoding fails. """ type_string = type_name.name if type_name == ChainDataType.DelegatedInfo: diff --git a/src/bittensor/networking.py b/src/bittensor/networking.py index bf19718d..1073a4f9 100644 --- a/src/bittensor/networking.py +++ b/src/bittensor/networking.py @@ -3,13 +3,10 @@ def int_to_ip(int_val: int) -> str: """Maps an integer to a unique ip-string - Args: - int_val (:type:`int128`, `required`): The integer representation of an ip. Must be in the range (0, 3.4028237e+38). + :param int_val: The integer representation of an ip. Must be in the range (0, 3.4028237e+38). - Returns: - str_val (:type:`str`, `required): The string representation of an ip. Of form *.*.*.* for ipv4 or *::*:*:*:* for ipv6 + :return: The string representation of an ip. Of form *.*.*.* for ipv4 or *::*:*:*:* for ipv6 - Raises: - netaddr.core.AddrFormatError (Exception): Raised when the passed int_vals is not a valid ip int value. + :raises: netaddr.core.AddrFormatError (Exception): Raised when the passed int_vals is not a valid ip int value. """ return str(netaddr.IPAddress(int_val)) From 77b600ea0290ab664116d7d7598b3a2bd54705d0 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 19:05:27 +0200 Subject: [PATCH 31/48] Inspect working --- cli.py | 2 + src/bittensor/async_substrate_interface.py | 4 +- src/bittensor/balances.py | 9 +- src/bittensor/chain_data.py | 19 ++- src/subtensor_interface.py | 40 +++++- src/utils.py | 2 +- src/wallets.py | 135 +++++++++++---------- 7 files changed, 126 insertions(+), 85 deletions(-) diff --git a/cli.py b/cli.py index 5e5ff11e..b07cdc15 100755 --- a/cli.py +++ b/cli.py @@ -118,6 +118,8 @@ def __init__(self): self.wallet_app.command("balance")(self.wallet_balance) self.wallet_app.command("history")(self.wallet_history) self.wallet_app.command("overview")(self.wallet_overview) + self.wallet_app.command("transfer")(self.wallet_transfer) + self.wallet_app.command("inspect")(self.wallet_inspect) # delegates commands self.delegates_app.command("list")(self.delegates_list) diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index 2bd0b969..99f859fa 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -413,9 +413,7 @@ async def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool): elif reuse: if self.last_block_hash: return self.last_block_hash - return ( - await self.get_chain_head() - ) # also sets the last_block_hash to chain_head + return block_hash async def init_runtime( self, block_hash: Optional[str] = None, block_id: Optional[int] = None diff --git a/src/bittensor/balances.py b/src/bittensor/balances.py index 922c2ab2..326b3ed9 100644 --- a/src/bittensor/balances.py +++ b/src/bittensor/balances.py @@ -26,11 +26,10 @@ class Balance: This class provides a way to interact with balances in two different units: rao and tao. It provides methods to convert between these units, as well as to perform arithmetic and comparison operations. - Attributes: - unit: A string representing the symbol for the tao unit. - rao_unit: A string representing the symbol for the rao unit. - rao: An integer that stores the balance in rao units. - tao: A float property that gives the balance in tao units. + :var unit: A string representing the symbol for the tao unit. + :var rao_unit: A string representing the symbol for the rao unit. + :var rao: An integer that stores the balance in rao units. + :var tao: A float property that gives the balance in tao units. """ unit: str = chr(0x03C4) # This is the tao unit diff --git a/src/bittensor/chain_data.py b/src/bittensor/chain_data.py index 3657eca2..e09a7f78 100644 --- a/src/bittensor/chain_data.py +++ b/src/bittensor/chain_data.py @@ -195,16 +195,15 @@ class DelegateInfo: """ Dataclass for delegate information. For a lighter version of this class, see :func:`DelegateInfoLite`. - Args: - hotkey_ss58 (str): Hotkey of the delegate for which the information is being fetched. - total_stake (int): Total stake of the delegate. - nominators (list[Tuple[str, int]]): list of nominators of the delegate and their stake. - take (float): Take of the delegate as a percentage. - owner_ss58 (str): Coldkey of the owner. - registrations (list[int]): list of subnets that the delegate is registered on. - validator_permits (list[int]): list of subnets that the delegate is allowed to validate on. - return_per_1000 (int): Return per 1000 TAO, for the delegate over a day. - total_daily_return (int): Total daily return of the delegate. + :param hotkey_ss58: Hotkey of the delegate for which the information is being fetched. + :param total_stake: Total stake of the delegate. + :param nominators: list of nominators of the delegate and their stake. + :param take: Take of the delegate as a percentage. + :param owner_ss58: Coldkey of the owner. + :param registrations: list of subnets that the delegate is registered on. + :param validator_permits: list of subnets that the delegate is allowed to validate on. + :param return_per_1000: Return per 1000 TAO, for the delegate over a day. + :param total_daily_return: Total daily return of the delegate. """ diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 63f8dd57..2711a2c7 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -432,7 +432,9 @@ async def neurons_lite( hex_bytes_result = await self.query_runtime_api( runtime_api="NeuronInfoRuntimeApi", method="get_neurons_lite", - params=[netuid], + params=[ + netuid + ], # TODO check to see if this can accept more than one at a time block_hash=block_hash, reuse_block=reuse_block, ) @@ -446,3 +448,39 @@ async def neurons_lite( bytes_result = bytes.fromhex(hex_bytes_result) return NeuronInfoLite.list_from_vec_u8(bytes_result) # type: ignore + + async def get_delegated( + self, + coldkey_ss58: str, + block_hash: Optional[int] = None, + reuse_block: bool = False, + ) -> list[tuple[DelegateInfo, Balance]]: + """ + Retrieves a list of delegates and their associated stakes for a given coldkey. This function + identifies the delegates that a specific account has staked tokens on. + + :param coldkey_ss58: The `SS58` address of the account's coldkey. + :param block_hash: The blockchain block number for the query. + :param reuse_block: Whether to reuse the last-used blockchain block hash. + + :return: A list of tuples, each containing a delegate's information and staked amount. + + This function is important for account holders to understand their stake allocations and their + involvement in the network's delegation and consensus mechanisms. + """ + + block_hash = ( + block_hash + if block_hash + else (self.substrate.last_block_hash if reuse_block else None) + ) + encoded_coldkey = ss58_to_vec_u8(coldkey_ss58) + json_body = await self.substrate.rpc_request( + method="delegateInfo_getDelegated", + params=([block_hash, encoded_coldkey] if block_hash else [encoded_coldkey]), + ) + + if not (result := json_body.get("result")): + return [] + + return DelegateInfo.delegated_list_from_vec_u8(result) diff --git a/src/utils.py b/src/utils.py index e1e344eb..0e0f280c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -277,7 +277,7 @@ async def get_delegates_details_from_github(url: str) -> dict[str, DelegatesDeta all_delegates_details = {} if response.ok: - all_delegates: dict[str, Any] = await response.json() + all_delegates: dict[str, Any] = await response.json(content_type=None) for delegate_hotkey, delegates_details in all_delegates.items(): all_delegates_details[delegate_hotkey] = DelegatesDetails.from_json( delegates_details diff --git a/src/wallets.py b/src/wallets.py index 3a3eac67..bc694f6d 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -16,9 +16,10 @@ # DEALINGS IN THE SOFTWARE. import asyncio -import os from collections import defaultdict from concurrent.futures import ProcessPoolExecutor +import itertools +import os from typing import Optional, Any import aiohttp @@ -1099,29 +1100,43 @@ async def transfer( async def inspect( wallet: Wallet, subtensor: SubtensorInterface, - netuids_filter: Optional[list[int]] = None, + netuids_filter: list[int], all_wallets: bool = False, ): def delegate_row_maker(delegates_: list[tuple[DelegateInfo, Balance]]) -> list[str]: - for delegate_, staked in delegates_: - if delegate_.hotkey_ss58 in registered_delegate_info: - delegate_name = registered_delegate_info[delegate_.hotkey_ss58].name + for d_, staked in delegates_: + if d_.hotkey_ss58 in registered_delegate_info: + delegate_name = registered_delegate_info[d_.hotkey_ss58].name else: - delegate_name = delegate_.hotkey_ss58 - yield [ - "", - "", - str(delegate_name), - str(staked), - str( - delegate_.total_daily_return.tao - * (staked.tao / delegate_.total_stake.tao) - ), - "", - "", - "", - "", - ] + delegate_name = d_.hotkey_ss58 + yield ( + [""] * 2 + + [ + str(delegate_name), + str(staked), + str(d_.total_daily_return.tao * (staked.tao / d_.total_stake.tao)), + ] + + [""] * 4 + ) + + def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: + hotkeys = get_hotkey_wallets_for_wallet(wallet_) + for netuid in all_netuids_: + for n in nsd[netuid]: + if n.coldkey == wallet_.coldkeypub.ss58_address: + hotkey_name: str = "" + if hotkey_names := [ + w.hotkey_str + for w in hotkeys + if w.hotkey.ss58_address == n.hotkey + ]: + hotkey_name = f"{hotkey_names[0]}-" + yield [""] * 5 + [ + str(netuid), + f"{hotkey_name}{n.hotkey}", + str(n.stake), + str(Balance.from_tao(n.emission)), + ] if all_wallets: wallets = _get_coldkey_wallets_for_path(wallet.path) @@ -1132,10 +1147,13 @@ def delegate_row_maker(delegates_: list[tuple[DelegateInfo, Balance]]) -> list[s wallets = [wallet] all_hotkeys = get_hotkey_wallets_for_wallet(wallet) - all_netuids = subtensor.get_all_subnet_netuids() async with subtensor: - block, all_netuids = await subtensor.filter_netuids_by_registered_hotkeys( - all_netuids, netuids_filter, all_hotkeys, reuse_block=False + block_hash = await subtensor.substrate.get_chain_head() + all_netuids = await subtensor.filter_netuids_by_registered_hotkeys( + (await subtensor.get_all_subnet_netuids(block_hash)), + netuids_filter, + all_hotkeys, + reuse_block=False ) # bittensor.logging.debug(f"Netuids to check: {all_netuids}") @@ -1147,11 +1165,6 @@ def delegate_row_maker(delegates_: list[tuple[DelegateInfo, Balance]]) -> list[s ":warning:[yellow]Could not get delegate info from chain.[/yellow]" ) - neuron_state_dict = {} - for netuid in tqdm(all_netuids): - neurons = subtensor.neurons_lite(netuid) - neuron_state_dict[netuid] = neurons if neurons else [] - table = Table( Column("[overline white]Coldkey", style="bold white"), Column("[overline white]Balance", style="green"), @@ -1169,46 +1182,38 @@ def delegate_row_maker(delegates_: list[tuple[DelegateInfo, Balance]]) -> list[s footer_style="overline white", ) rows = [] - for wallet in wallets: - delegates: list[tuple[DelegateInfo, Balance]] = subtensor.get_delegated( - coldkey_ss58=wallet.coldkeypub.ss58_address + wallets_with_ckp_file = [ + wallet for wallet in wallets if wallet.coldkeypub_file.exists_on_device() + ] + all_delegates: list[list[tuple[DelegateInfo, Balance]]] + async with subtensor: + balances, all_neurons, all_delegates = await asyncio.gather( + subtensor.get_balance( + *[w.coldkeypub.ss58_address for w in wallets_with_ckp_file] + ), + asyncio.gather( + *[subtensor.neurons_lite(netuid=netuid) for netuid in all_netuids] + ), + asyncio.gather( + *[ + subtensor.get_delegated(w.coldkeypub.ss58_address) + for w in wallets_with_ckp_file + ]), ) - if not wallet.coldkeypub_file.exists_on_device(): - continue - cold_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address) - rows.append([wallet.name, str(cold_balance), "", "", "", "", "", "", ""]) - for row in delegate_row_maker(delegates): - rows.append(row) - hotkeys = get_hotkey_wallets_for_wallet(wallet) - for netuid in all_netuids: - for neuron in neuron_state_dict[netuid]: - if neuron.coldkey == wallet.coldkeypub.ss58_address: - hotkey_name: str = "" - - hotkey_names: list[str] = [ - wallet.hotkey_str - for wallet in filter( - lambda hotkey: hotkey.hotkey.ss58_address == neuron.hotkey, - hotkeys, - ) - ] - if hotkey_names: - hotkey_name = f"{hotkey_names[0]}-" + neuron_state_dict = {} + for netuid, neuron in zip(all_netuids, all_neurons): + neuron_state_dict[netuid] = neuron if neuron else [] - rows.append( - [ - "", - "", - "", - "", - "", - str(netuid), - f"{hotkey_name}{neuron.hotkey}", - str(neuron.stake), - str(Balance.from_tao(neuron.emission)), - ] - ) + for wall, d in zip(wallets_with_ckp_file, all_delegates): + rows.append( + [wall.name, str(balances[wall.coldkeypub.ss58_address])] + [""] * 7 + ) + for row in itertools.chain( + delegate_row_maker(d), + neuron_row_maker(wall, all_netuids, neuron_state_dict), + ): + rows.append(row) for row in rows: table.add_row(*row) From 8fb971fe12b897e86282469e26871b5ce83a47e6 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 19:07:04 +0200 Subject: [PATCH 32/48] Ruff --- src/wallets.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/wallets.py b/src/wallets.py index bc694f6d..15aede29 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1153,7 +1153,7 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: (await subtensor.get_all_subnet_netuids(block_hash)), netuids_filter, all_hotkeys, - reuse_block=False + reuse_block=False, ) # bittensor.logging.debug(f"Netuids to check: {all_netuids}") @@ -1195,10 +1195,11 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: *[subtensor.neurons_lite(netuid=netuid) for netuid in all_netuids] ), asyncio.gather( - *[ - subtensor.get_delegated(w.coldkeypub.ss58_address) - for w in wallets_with_ckp_file - ]), + *[ + subtensor.get_delegated(w.coldkeypub.ss58_address) + for w in wallets_with_ckp_file + ] + ), ) neuron_state_dict = {} @@ -1206,9 +1207,7 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: neuron_state_dict[netuid] = neuron if neuron else [] for wall, d in zip(wallets_with_ckp_file, all_delegates): - rows.append( - [wall.name, str(balances[wall.coldkeypub.ss58_address])] + [""] * 7 - ) + rows.append([wall.name, str(balances[wall.coldkeypub.ss58_address])] + [""] * 7) for row in itertools.chain( delegate_row_maker(d), neuron_row_maker(wall, all_netuids, neuron_state_dict), From db6849ce824abb5cffbfe6ffbc30a2d61d3a5cbe Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 22:04:43 +0200 Subject: [PATCH 33/48] In the arena, trying things. --- cli.py | 3 + src/bittensor/async_substrate_interface.py | 87 +++++++++++++++------- src/subtensor_interface.py | 14 ++-- src/wallets.py | 12 ++- 4 files changed, 78 insertions(+), 38 deletions(-) diff --git a/cli.py b/cli.py index b07cdc15..c97bd280 100755 --- a/cli.py +++ b/cli.py @@ -6,6 +6,7 @@ import rich from rich.prompt import Confirm, Prompt import typer +from websockets import ConnectionClosed from src import wallets, defaults, utils from src.subtensor_interface import SubtensorInterface @@ -142,6 +143,8 @@ def _run_command(self, cmd: Coroutine): typer.echo( f"Connection refused when connecting to chain: {self.not_subtensor}" ) + except ConnectionClosed: + pass @staticmethod def wallet_ask( diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index 99f859fa..97baf4f4 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -27,6 +27,29 @@ class Preprocessed: storage_item: ScaleType +class RuntimeCache: + blocks: dict[int, "Runtime"] + block_hashes: dict[str, "Runtime"] + + def __init__(self): + self.blocks = {} + self.block_hashes = {} + + def add_item(self, block: int, block_hash: str, runtime: "Runtime"): + if block: + self.blocks[block] = runtime + if block_hash: + self.block_hashes[block] = runtime + + def retrieve(self, block: int, block_hash: str): + if block: + return self.blocks.get(block) + elif block_hash: + return self.block_hashes.get(block_hash) + else: + return None + + class Runtime: block_hash: str block_id: int @@ -257,20 +280,23 @@ async def _exit_with_timer(self): """ try: await asyncio.sleep(self.shutdown_timer) - async with self._lock: - self._receiving_task.cancel() - try: - await self._receiving_task - except asyncio.CancelledError: - pass - await self.ws.close() - self.ws = None - self._initialized = False - self._receiving_task = None - self.id = 0 + await self.shutdown() except asyncio.CancelledError: pass + async def shutdown(self): + async with self._lock: + self._receiving_task.cancel() + try: + await self._receiving_task + except asyncio.CancelledError: + pass + await self.ws.close() + self.ws = None + self._initialized = False + self._receiving_task = None + self.id = 0 + async def _recv(self) -> None: try: response = json.loads(await self.ws.recv()) @@ -285,7 +311,6 @@ async def _recv(self) -> None: except websockets.ConnectionClosed: raise except KeyError as e: - print(f"Unhandled websocket response: {e}") raise e async def _start_receiving(self): @@ -368,6 +393,7 @@ def __init__( self._forgettable_task = None self.ss58_format = ss58_format self.type_registry = type_registry + self.runtime_cache = RuntimeCache() async def __aenter__(self): await self.initialize() @@ -401,12 +427,12 @@ def chain(self): async def get_storage_item(self, module: str, storage_function: str): if not self.substrate.metadata: - self.substrate.init_runtime() + await self.init_runtime() metadata_pallet = self.substrate.metadata.get_metadata_pallet(module) storage_item = metadata_pallet.get_storage_function(storage_function) return storage_item - async def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool): + async def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool) -> Optional[str]: if block_hash: self.last_block_hash = block_hash return block_hash @@ -432,15 +458,18 @@ async def init_runtime( :returns: Runtime object """ async with self._lock: - await asyncio.get_event_loop().run_in_executor( - None, self.substrate.init_runtime, block_hash, block_id - ) - return Runtime( - self.chain, - self.substrate.runtime_config, - self.substrate.metadata, - self.type_registry, - ) + if not (runtime := self.runtime_cache.retrieve(block_id, block_hash)): + await asyncio.get_event_loop().run_in_executor( + None, self.substrate.init_runtime, block_hash, block_id + ) + runtime = Runtime( + self.chain, + self.substrate.runtime_config, + self.substrate.metadata, + self.type_registry, + ) + self.runtime_cache.add_item(block_id, block_hash, runtime) + return runtime async def get_block_runtime_version(self, block_hash: str) -> dict: """ @@ -761,7 +790,8 @@ async def query_multiple( # to do, can simply query the block hash first, and then pass multiple query_subtensor calls # into an asyncio.gather, with the specified block hash block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) - self.last_block_hash = block_hash + if block_hash: + self.last_block_hash = block_hash runtime = await self.init_runtime(block_hash=block_hash) preprocessed: tuple[Preprocessed] = await asyncio.gather( *[ @@ -1027,7 +1057,8 @@ async def query( you should use ``self.query_multiple`` """ block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) - self.last_block_hash = block_hash + if block_hash: + self.last_block_hash = block_hash runtime = await self.init_runtime(block_hash=block_hash) preprocessed: Preprocessed = await self._preprocess( params, block_hash, storage_function, module @@ -1091,13 +1122,13 @@ async def query_map( """ params = params or [] block_hash = await self._get_current_block_hash(block_hash, reuse_block_hash) - self.last_block_hash = block_hash + if block_hash: + self.last_block_hash = block_hash runtime = await self.init_runtime(block_hash=block_hash) metadata_pallet = runtime.metadata.get_metadata_pallet(module) if not metadata_pallet: raise ValueError(f'Pallet "{module}" not found') - storage_item = metadata_pallet.get_storage_function(storage_function) if not metadata_pallet or not storage_item: @@ -1387,6 +1418,6 @@ async def close(self): """ self.substrate.close() try: - await self.ws.ws.close() + await self.ws.shutdown() except AttributeError: pass diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 2711a2c7..8da13dda 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -237,12 +237,12 @@ async def query_runtime_api( return obj.decode() async def get_balance( - self, *addresses, block: Optional[int] = None, reuse_block: bool = False + self, *addresses, block_hash: Optional[int] = None, reuse_block: bool = False ) -> dict[str, Balance]: """ Retrieves the balance for given coldkey(s) :param addresses: coldkey addresses(s) - :param block: the block number, optional, currently unused + :param block_hash: the block hash, optional :param reuse_block: Whether to reuse the last-used block hash when retrieving info. :return: dict of {address: Balance objects} """ @@ -250,6 +250,7 @@ async def get_balance( params=[a for a in addresses], storage_function="Account", module="System", + block_hash=block_hash, reuse_block_hash=reuse_block, ) return {k: Balance(v.value["data"]["free"]) for (k, v) in results.items()} @@ -294,6 +295,7 @@ async def get_netuids_for_hotkey( :return: A list of netuids where the neuron is a member. """ + result = await self.substrate.query_map( module="SubtensorModule", storage_function="IsNetworkMember", @@ -358,14 +360,14 @@ async def get_hyperparameter( return result.value async def filter_netuids_by_registered_hotkeys( - self, all_netuids, filter_for_netuids, all_hotkeys, reuse_block: bool = False + self, all_netuids, filter_for_netuids, all_hotkeys, block_hash: str, reuse_block: bool = False ) -> list[int]: netuids_with_registered_hotkeys = [ item for sublist in await asyncio.gather( *[ self.get_netuids_for_hotkey( - wallet.hotkey.ss58_address, reuse_block=reuse_block + wallet.hotkey.ss58_address, reuse_block=reuse_block, block_hash=block_hash ) for wallet in all_hotkeys ] @@ -452,7 +454,7 @@ async def neurons_lite( async def get_delegated( self, coldkey_ss58: str, - block_hash: Optional[int] = None, + block_hash: Optional[str] = None, reuse_block: bool = False, ) -> list[tuple[DelegateInfo, Balance]]: """ @@ -460,7 +462,7 @@ async def get_delegated( identifies the delegates that a specific account has staked tokens on. :param coldkey_ss58: The `SS58` address of the account's coldkey. - :param block_hash: The blockchain block number for the query. + :param block_hash: The hash of the blockchain block number for the query. :param reuse_block: Whether to reuse the last-used blockchain block hash. :return: A list of tuples, each containing a delegate's information and staked amount. diff --git a/src/wallets.py b/src/wallets.py index 15aede29..c89f2809 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1149,11 +1149,13 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: async with subtensor: block_hash = await subtensor.substrate.get_chain_head() + print("BLOCK HASH:", block_hash) + await subtensor.substrate.init_runtime(block_hash=block_hash) all_netuids = await subtensor.filter_netuids_by_registered_hotkeys( (await subtensor.get_all_subnet_netuids(block_hash)), netuids_filter, all_hotkeys, - reuse_block=False, + block_hash=block_hash, ) # bittensor.logging.debug(f"Netuids to check: {all_netuids}") @@ -1189,10 +1191,11 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: async with subtensor: balances, all_neurons, all_delegates = await asyncio.gather( subtensor.get_balance( - *[w.coldkeypub.ss58_address for w in wallets_with_ckp_file] + *[w.coldkeypub.ss58_address for w in wallets_with_ckp_file], + block_hash=block_hash ), asyncio.gather( - *[subtensor.neurons_lite(netuid=netuid) for netuid in all_netuids] + *[subtensor.neurons_lite(netuid=netuid, block_hash=block_hash) for netuid in all_netuids] ), asyncio.gather( *[ @@ -1201,6 +1204,7 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: ] ), ) + await subtensor.substrate.close() neuron_state_dict = {} for netuid, neuron in zip(all_netuids, all_neurons): @@ -1217,4 +1221,4 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: for row in rows: table.add_row(*row) - console.print(table) + return console.print(table) From c7a9489531fa37e94269234e0e76042b0ed25136 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 31 Jul 2024 22:23:10 +0200 Subject: [PATCH 34/48] Initial config port --- cli.py | 31 ++++++++++++++++++---- requirements.txt | 1 + src/bittensor/async_substrate_interface.py | 4 ++- src/subtensor_interface.py | 11 ++++++-- src/wallets.py | 7 +++-- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/cli.py b/cli.py index c97bd280..21f6d7a7 100755 --- a/cli.py +++ b/cli.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import asyncio +import os.path from typing import Optional, Coroutine from bittensor_wallet import Wallet @@ -7,6 +8,7 @@ from rich.prompt import Confirm, Prompt import typer from websockets import ConnectionClosed +from yaml import safe_load from src import wallets, defaults, utils from src.subtensor_interface import SubtensorInterface @@ -95,7 +97,16 @@ def get_creation_data(mnemonic, seed, json, json_password): class CLIManager: def __init__(self): - self.app = typer.Typer(rich_markup_mode="markdown") + self.config = { + "wallet_name": None, + "wallet_path": None, + "wallet_hotkey": None, + "network": None, + "chain": None, + } + self.not_subtensor = None + + self.app = typer.Typer(rich_markup_mode="markdown", callback=self.check_config) self.wallet_app = typer.Typer() self.delegates_app = typer.Typer() @@ -125,16 +136,19 @@ def __init__(self): # delegates commands self.delegates_app.command("list")(self.delegates_list) - self.not_subtensor = None - def initialize_chain( self, network: str = typer.Option("default_network", help="Network name"), chain: str = typer.Option("default_chain", help="Chain name"), ): if not self.not_subtensor: - self.not_subtensor = SubtensorInterface(network, chain) - # typer.echo(f"Initialized with {self.not_subtensor}") + if self.config["chain"] or self.config["chain"]: + self.not_subtensor = SubtensorInterface( + self.config["network"], self.config["chain"] + ) + else: + self.not_subtensor = SubtensorInterface(network, chain) + # typer.echo(f"Initialized with {self.not_subtensor}") def _run_command(self, cmd: Coroutine): try: @@ -146,6 +160,13 @@ def _run_command(self, cmd: Coroutine): except ConnectionClosed: pass + def check_config(self): + with open(os.path.expanduser("~/.bittensor/config.yml"), "r") as f: + config = safe_load(f) + for k, v in config.items(): + if k in self.config.keys(): + self.config[k] = v + @staticmethod def wallet_ask( wallet_name: str, diff --git a/requirements.txt b/requirements.txt index 3fa37bf4..bfea3a56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ scalecodec==1.2.11 substrate-interface~=1.7.9 typer~=0.12 websockets>=12.0 +PyYAML diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index 97baf4f4..9dd4b398 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -432,7 +432,9 @@ async def get_storage_item(self, module: str, storage_function: str): storage_item = metadata_pallet.get_storage_function(storage_function) return storage_item - async def _get_current_block_hash(self, block_hash: Optional[str], reuse: bool) -> Optional[str]: + async def _get_current_block_hash( + self, block_hash: Optional[str], reuse: bool + ) -> Optional[str]: if block_hash: self.last_block_hash = block_hash return block_hash diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 8da13dda..d5b6da1e 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -360,14 +360,21 @@ async def get_hyperparameter( return result.value async def filter_netuids_by_registered_hotkeys( - self, all_netuids, filter_for_netuids, all_hotkeys, block_hash: str, reuse_block: bool = False + self, + all_netuids, + filter_for_netuids, + all_hotkeys, + block_hash: str, + reuse_block: bool = False, ) -> list[int]: netuids_with_registered_hotkeys = [ item for sublist in await asyncio.gather( *[ self.get_netuids_for_hotkey( - wallet.hotkey.ss58_address, reuse_block=reuse_block, block_hash=block_hash + wallet.hotkey.ss58_address, + reuse_block=reuse_block, + block_hash=block_hash, ) for wallet in all_hotkeys ] diff --git a/src/wallets.py b/src/wallets.py index c89f2809..ba01fcad 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1192,10 +1192,13 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: balances, all_neurons, all_delegates = await asyncio.gather( subtensor.get_balance( *[w.coldkeypub.ss58_address for w in wallets_with_ckp_file], - block_hash=block_hash + block_hash=block_hash, ), asyncio.gather( - *[subtensor.neurons_lite(netuid=netuid, block_hash=block_hash) for netuid in all_netuids] + *[ + subtensor.neurons_lite(netuid=netuid, block_hash=block_hash) + for netuid in all_netuids + ] ), asyncio.gather( *[ From a9bcecb8ce923bbcb91ea1fefd1b72bdc6502b00 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 15:10:26 +0200 Subject: [PATCH 35/48] Sets timeout error catch for github pulling. --- cli.py | 2 ++ src/utils.py | 21 ++++++++------ src/wallets.py | 76 +++++++++++++++++++++++++------------------------- 3 files changed, 53 insertions(+), 46 deletions(-) diff --git a/cli.py b/cli.py index 21f6d7a7..20689672 100755 --- a/cli.py +++ b/cli.py @@ -12,6 +12,7 @@ from src import wallets, defaults, utils from src.subtensor_interface import SubtensorInterface +from src.utils import console # re-usable args @@ -149,6 +150,7 @@ def initialize_chain( else: self.not_subtensor = SubtensorInterface(network, chain) # typer.echo(f"Initialized with {self.not_subtensor}") + console.print(f"[yellow] Connected to [/yellow][white]{self.not_subtensor}") def _run_command(self, cmd: Coroutine): try: diff --git a/src/utils.py b/src/utils.py index 0e0f280c..51161579 100644 --- a/src/utils.py +++ b/src/utils.py @@ -272,15 +272,20 @@ def format_error_message(error_message: dict) -> str: async def get_delegates_details_from_github(url: str) -> dict[str, DelegatesDetails]: - async with aiohttp.ClientSession() as session: - response = await session.get(url) - all_delegates_details = {} - if response.ok: - all_delegates: dict[str, Any] = await response.json(content_type=None) - for delegate_hotkey, delegates_details in all_delegates.items(): - all_delegates_details[delegate_hotkey] = DelegatesDetails.from_json( - delegates_details + + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10.0)) as session: + try: + response = await session.get(url) + if response.ok: + all_delegates: dict[str, Any] = await response.json(content_type=None) + for delegate_hotkey, delegates_details in all_delegates.items(): + all_delegates_details[delegate_hotkey] = DelegatesDetails.from_json( + delegates_details + ) + except TimeoutError: + err_console.print( + "Request timed out pulling delegates details from GitHub." ) return all_delegates_details diff --git a/src/wallets.py b/src/wallets.py index ba01fcad..41c745e3 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1146,26 +1146,25 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: else: wallets = [wallet] all_hotkeys = get_hotkey_wallets_for_wallet(wallet) - - async with subtensor: - block_hash = await subtensor.substrate.get_chain_head() - print("BLOCK HASH:", block_hash) - await subtensor.substrate.init_runtime(block_hash=block_hash) - all_netuids = await subtensor.filter_netuids_by_registered_hotkeys( - (await subtensor.get_all_subnet_netuids(block_hash)), - netuids_filter, - all_hotkeys, - block_hash=block_hash, - ) + with console.status("synchronising with chain"): + async with subtensor: + block_hash = await subtensor.substrate.get_chain_head() + await subtensor.substrate.init_runtime(block_hash=block_hash) + all_netuids = await subtensor.filter_netuids_by_registered_hotkeys( + (await subtensor.get_all_subnet_netuids(block_hash)), + netuids_filter, + all_hotkeys, + block_hash=block_hash, + ) # bittensor.logging.debug(f"Netuids to check: {all_netuids}") - - registered_delegate_info: Optional[ - dict[str, DelegatesDetails] - ] = await get_delegates_details_from_github(url=Constants.delegates_details_url) - if not registered_delegate_info: - console.print( - ":warning:[yellow]Could not get delegate info from chain.[/yellow]" - ) + with console.status("Pulling delegates info"): + registered_delegate_info: Optional[ + dict[str, DelegatesDetails] + ] = await get_delegates_details_from_github(url=Constants.delegates_details_url) + if not registered_delegate_info: + console.print( + ":warning:[yellow]Could not get delegate info from chain.[/yellow]" + ) table = Table( Column("[overline white]Coldkey", style="bold white"), @@ -1188,25 +1187,26 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: wallet for wallet in wallets if wallet.coldkeypub_file.exists_on_device() ] all_delegates: list[list[tuple[DelegateInfo, Balance]]] - async with subtensor: - balances, all_neurons, all_delegates = await asyncio.gather( - subtensor.get_balance( - *[w.coldkeypub.ss58_address for w in wallets_with_ckp_file], - block_hash=block_hash, - ), - asyncio.gather( - *[ - subtensor.neurons_lite(netuid=netuid, block_hash=block_hash) - for netuid in all_netuids - ] - ), - asyncio.gather( - *[ - subtensor.get_delegated(w.coldkeypub.ss58_address) - for w in wallets_with_ckp_file - ] - ), - ) + with console.status("Pulling balance data"): + async with subtensor: + balances, all_neurons, all_delegates = await asyncio.gather( + subtensor.get_balance( + *[w.coldkeypub.ss58_address for w in wallets_with_ckp_file], + block_hash=block_hash, + ), + asyncio.gather( + *[ + subtensor.neurons_lite(netuid=netuid, block_hash=block_hash) + for netuid in all_netuids + ] + ), + asyncio.gather( + *[ + subtensor.get_delegated(w.coldkeypub.ss58_address) + for w in wallets_with_ckp_file + ] + ), + ) await subtensor.substrate.close() neuron_state_dict = {} From e2848d8493f7b0b41af3acec5dfc8a8039a62a21 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 18:09:55 +0200 Subject: [PATCH 36/48] Added config setter/getter. WIP on faucet. --- cli.py | 134 +++- src/bittensor/extrinsics/registration.py | 822 +++++++++++++++++++++++ src/wallets.py | 26 + 3 files changed, 981 insertions(+), 1 deletion(-) create mode 100644 src/bittensor/extrinsics/registration.py diff --git a/cli.py b/cli.py index 20689672..232ed101 100755 --- a/cli.py +++ b/cli.py @@ -6,9 +6,10 @@ from bittensor_wallet import Wallet import rich from rich.prompt import Confirm, Prompt +from rich.table import Table, Column import typer from websockets import ConnectionClosed -from yaml import safe_load +from yaml import safe_load, safe_dump from src import wallets, defaults, utils from src.subtensor_interface import SubtensorInterface @@ -108,9 +109,14 @@ def __init__(self): self.not_subtensor = None self.app = typer.Typer(rich_markup_mode="markdown", callback=self.check_config) + self.default_app = typer.Typer() self.wallet_app = typer.Typer() self.delegates_app = typer.Typer() + # default alias + self.app.add_typer(self.default_app, name="default") + self.app.add_typer(self.default_app, name="def", hidden=True) + # wallet aliases self.app.add_typer(self.wallet_app, name="wallet") self.app.add_typer(self.wallet_app, name="w", hidden=True) @@ -120,6 +126,10 @@ def __init__(self): self.app.add_typer(self.delegates_app, name="delegates") self.app.add_typer(self.delegates_app, name="d", hidden=True) + # defaults commands + self.default_app.command("set")(self.set_config) + self.default_app.command("get")(self.get_config) + # wallet commands self.wallet_app.command("list")(self.wallet_list) self.wallet_app.command("regen-coldkey")(self.wallet_regen_coldkey) @@ -169,6 +179,50 @@ def check_config(self): if k in self.config.keys(): self.config[k] = v + def set_config( + self, + wallet_name: Optional[str] = typer.Option( + None, + "--wallet-name", + "--name", + help="Wallet name", + ), + wallet_path: Optional[str] = typer.Option( + None, + "--wallet-path", + "--path", + "-p", + help="Path to root of wallets", + ), + wallet_hotkey: Optional[str] = typer.Option( + None, "--wallet-hotkey", "--hotkey", "-k", help="Path to root of wallets" + ), + network: Optional[str] = typer.Option( + None, + "--network", + "-n", + help="Network name: [finney, test, local]", + ), + chain: Optional[str] = typer.Option( + None, + "--chain", + "-c", + help="Chain name", + ), + ): + args = locals() + for arg in ["wallet_name", "wallet_path", "wallet_hotkey", "network", "chain"]: + if val := args.get(arg): + self.config[arg] = val + with open(os.path.expanduser("~/.bittensor/config.yml"), "w") as f: + safe_dump(self.config, f) + + def get_config(self): + table = Table(Column("Name"), Column("Value")) + for k, v in self.config.items(): + table.add_row(*[k, v]) + console.print(table) + @staticmethod def wallet_ask( wallet_name: str, @@ -486,6 +540,84 @@ def wallet_inspect( ) ) + def wallet_faucet( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + # TODO add the following to config + processors: Optional[int] = typer.Option( + defaults.pow_register.num_processes, + "-processors", + "-p", + help="Number of processors to use for POW registration.", + ), + update_interval: Optional[int] = typer.Option( + defaults.pow_register.update_interval, + "-update-interval", + "-u", + help="The number of nonces to process before checking for next block during registration", + ), + output_in_place: Optional[bool] = typer.Option( + defaults.pow_register.output_in_place, + help="Whether to output the registration statistics in-place.", + ), + verbose: Optional[bool] = typer.Option( + defaults.pow_register.verbose, + "--verbose", + "-v", + help="Whether to output the registration statistics verbosely.", + ), + use_cuda: Optional[bool] = typer.Option( + defaults.pow_register.cuda.use_cuda, + "--use-cuda/--no-use-cuda", + "--cuda/--no-cuda", + help="Set flag to use CUDA to pow_register.", + ), + dev_id: Optional[int] = typer.Option( + defaults.pow_register.cuda.dev_id, + "--dev-id", + "-d", + help="Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).", + ), + threads_per_block: Optional[int] = typer.Option( + defaults.pow_register.cuda.tpb, + "--threads-per-block", + "-tbp", + help="Set the number of Threads Per Block for CUDA.", + ), + max_successes: Optional[int] = typer.Option( + 3, + "--max-successes", + help="Set the maximum number of times to successfully run the faucet for this command.", + ), + ): + """ + # wallet faucet + Executes the `faucet` command to obtain test TAO tokens by performing Proof of Work (PoW). + + This command is particularly useful for users who need test tokens for operations on a local chain. + + ## IMPORTANT: + **THIS COMMAND IS DISABLED ON FINNEY AND TESTNET.** + + ## Usage: + The command uses the PoW mechanism to validate the user's effort and rewards them with test TAO tokens. It is + typically used in local chain environments where real value transactions are not necessary. + + ### Example usage: + ``` + btcli wallet faucet --faucet.num_processes 4 --faucet.cuda.use_cuda + ``` + + #### Note: + This command is meant for use in local environments where users can experiment with the network without using + real TAO tokens. It's important for users to have the necessary hardware setup, especially when opting for + CUDA-based GPU calculations. It is currently disabled on testnet and finney. You must use this on a local chain. + """ + def wallet_regen_coldkey( self, wallet_name: Optional[str] = Options.wallet_name, diff --git a/src/bittensor/extrinsics/registration.py b/src/bittensor/extrinsics/registration.py new file mode 100644 index 00000000..a6f8dab1 --- /dev/null +++ b/src/bittensor/extrinsics/registration.py @@ -0,0 +1,822 @@ +import math +from dataclasses import dataclass +import functools +from multiprocessing.queues import Queue +from multiprocessing import Process, Event, Lock +import os +import time +import typing +from typing import Optional + +from bittensor_wallet import Wallet +from rich.prompt import Confirm + +from src.subtensor_interface import SubtensorInterface +from src.utils import console, err_console, format_error_message + + +def use_torch() -> bool: + """Force the use of torch over numpy for certain operations.""" + return True if os.getenv("USE_TORCH") == "1" else False + + +def legacy_torch_api_compat(func): + """ + Convert function operating on numpy Input&Output to legacy torch Input&Output API if `use_torch()` is True. + + Args: + func (function): + Function with numpy Input/Output to be decorated. + Returns: + decorated (function): + Decorated function. + """ + + @functools.wraps(func) + def decorated(*args, **kwargs): + if use_torch(): + # if argument is a Torch tensor, convert it to numpy + args = [ + arg.cpu().numpy() if isinstance(arg, torch.Tensor) else arg + for arg in args + ] + kwargs = { + key: value.cpu().numpy() if isinstance(value, torch.Tensor) else value + for key, value in kwargs.items() + } + ret = func(*args, **kwargs) + if use_torch(): + # if return value is a numpy array, convert it to Torch tensor + if isinstance(ret, numpy.ndarray): + ret = torch.from_numpy(ret) + return ret + + return decorated + + +@functools.cache +def _get_real_torch(): + try: + import torch as _real_torch + except ImportError: + _real_torch = None + return _real_torch + + +def log_no_torch_error(): + err_console.print( + "This command requires torch. You can install torch for bittensor" + ' with `pip install bittensor[torch]` or `pip install ".[torch]"`' + " if installing from source, and then run the command with USE_TORCH=1 {command}" + ) + + +@dataclass +class POWSolution: + """A solution to the registration PoW problem.""" + + nonce: int + block_number: int + difficulty: int + seal: bytes + + def is_stale(self, subtensor: "bittensor.subtensor") -> bool: + """Returns True if the POW is stale. + This means the block the POW is solved for is within 3 blocks of the current block. + """ + return self.block_number < subtensor.get_current_block() - 3 + + +class LazyLoadedTorch: + def __bool__(self): + return bool(_get_real_torch()) + + def __getattr__(self, name): + if real_torch := _get_real_torch(): + return getattr(real_torch, name) + else: + log_no_torch_error() + raise ImportError("torch not installed") + + +if typing.TYPE_CHECKING: + import torch +else: + torch = LazyLoadedTorch() + + +class MaxSuccessException(Exception): + pass + + +class MaxAttemptsException(Exception): + pass + + +async def run_faucet_extrinsic( + subtensor: SubtensorInterface, + wallet: Wallet, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, + prompt: bool = False, + max_allowed_attempts: int = 3, + output_in_place: bool = True, + cuda: bool = False, + dev_id: int = 0, + tpb: int = 256, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, + log_verbose: bool = False, + max_successes: int = 3, +) -> tuple[bool, str]: + r"""Runs a continual POW to get a faucet of TAO on the test net. + + :param subtensor: The subtensor interface object used to run the extrinsic + :param wallet: Bittensor wallet object. + :param prompt: If `True`, the call waits for confirmation from the user before proceeding. + :param wait_for_inclusion: If set, waits for the extrinsic to enter a block before returning `True`, + or returns `False` if the extrinsic fails to enter the block within the timeout. + :param wait_for_finalization: If set, waits for the extrinsic to be finalized on the chain before returning `True`, + or returns `False` if the extrinsic fails to be finalized within the timeout. + :param max_allowed_attempts: Maximum number of attempts to register the wallet. + :param output_in_place: Whether to output logging data as the process runs. + :param cuda: If `True`, the wallet should be registered using CUDA device(s). + :param dev_id: The CUDA device id to use + :param tpb: The number of threads per block (CUDA). + :param num_processes: The number of processes to use to register. + :param update_interval: The number of nonces to solve between updates. + :param log_verbose: If `True`, the registration process will log more information. + :param max_successes: The maximum number of successful faucet runs for the wallet. + + :return: `True` if extrinsic was finalized or included in the block. If we did not wait for + finalization/inclusion, the response is also `True` + """ + if prompt: + if not Confirm.ask( + "Run Faucet ?\n" + f" coldkey: [bold white]{wallet.coldkeypub.ss58_address}[/bold white]\n" + f" network: [bold white]{subtensor}[/bold white]" + ): + return False, "" + + if not torch: + log_no_torch_error() + return False, "Requires torch" + + # Unlock coldkey + wallet.unlock_coldkey() + + # Get previous balance. + old_balance = await subtensor.get_balance(wallet.coldkeypub.ss58_address) + + # Attempt rolling registration. + attempts = 1 + successes = 1 + while True: + async with subtensor: + try: + pow_result = None + while pow_result is None or pow_result.is_stale(subtensor=subtensor): + # Solve latest POW. + if cuda: + if not torch.cuda.is_available(): + if prompt: + err_console.print("CUDA is not available.") + return False, "CUDA is not available." + pow_result: Optional[POWSolution] = create_pow( + subtensor, + wallet, + -1, + output_in_place, + cuda=cuda, + dev_id=dev_id, + tpb=tpb, + num_processes=num_processes, + update_interval=update_interval, + log_verbose=log_verbose, + ) + else: + pow_result: Optional[POWSolution] = create_pow( + subtensor, + wallet, + -1, + output_in_place, + cuda=cuda, + num_processes=num_processes, + update_interval=update_interval, + log_verbose=log_verbose, + ) + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="faucet", + call_params={ + "block_number": pow_result.block_number, + "nonce": pow_result.nonce, + "work": [int(byte_) for byte_ in pow_result.seal], + }, + ) + extrinsic = subtensor.substrate.create_signed_extrinsic( + call=call, keypair=wallet.coldkey + ) + response = subtensor.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + # process if registration successful, try again if pow is still valid + response.process_events() + if not response.is_success: + err_console.print( + f":cross_mark: [red]Failed[/red]: {format_error_message(response.error_message)}" + ) + if attempts == max_allowed_attempts: + raise MaxAttemptsException + attempts += 1 + # Wait a bit before trying again + time.sleep(1) + + # Successful registration + else: + new_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address) + console.print( + f"Balance: [blue]{old_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + ) + old_balance = new_balance + + if successes == max_successes: + raise MaxSuccessException + + attempts = 1 # Reset attempts on success + successes += 1 + + except KeyboardInterrupt: + return True, "Done" + + except MaxSuccessException: + return True, f"Max successes reached: {3}" + + except MaxAttemptsException: + return False, f"Max attempts reached: {max_allowed_attempts}" + + +def _check_for_newest_block_and_update( + subtensor: "bittensor.subtensor", + netuid: int, + old_block_number: int, + hotkey_bytes: bytes, + curr_diff: multiprocessing.Array, + curr_block: multiprocessing.Array, + curr_block_num: multiprocessing.Value, + update_curr_block: Callable, + check_block: "multiprocessing.Lock", + solvers: List[_Solver], + curr_stats: RegistrationStatistics, +) -> int: + """ + Checks for a new block and updates the current block information if a new block is found. + + Args: + subtensor (:obj:`bittensor.subtensor`, `required`): + The subtensor object to use for getting the current block. + netuid (:obj:`int`, `required`): + The netuid to use for retrieving the difficulty. + old_block_number (:obj:`int`, `required`): + The old block number to check against. + hotkey_bytes (:obj:`bytes`, `required`): + The bytes of the hotkey's pubkey. + curr_diff (:obj:`multiprocessing.Array`, `required`): + The current difficulty as a multiprocessing array. + curr_block (:obj:`multiprocessing.Array`, `required`): + Where the current block is stored as a multiprocessing array. + curr_block_num (:obj:`multiprocessing.Value`, `required`): + Where the current block number is stored as a multiprocessing value. + update_curr_block (:obj:`Callable`, `required`): + A function that updates the current block. + check_block (:obj:`multiprocessing.Lock`, `required`): + A mp lock that is used to check for a new block. + solvers (:obj:`List[_Solver]`, `required`): + A list of solvers to update the current block for. + curr_stats (:obj:`RegistrationStatistics`, `required`): + The current registration statistics to update. + + Returns: + (int) The current block number. + """ + block_number = subtensor.get_current_block() + if block_number != old_block_number: + old_block_number = block_number + # update block information + block_number, difficulty, block_hash = _get_block_with_retry( + subtensor=subtensor, netuid=netuid + ) + block_bytes = bytes.fromhex(block_hash[2:]) + + update_curr_block( + curr_diff, + curr_block, + curr_block_num, + block_number, + block_bytes, + difficulty, + hotkey_bytes, + check_block, + ) + # Set new block events for each solver + + for worker in solvers: + worker.newBlockEvent.set() + + # update stats + curr_stats.block_number = block_number + curr_stats.block_hash = block_hash + curr_stats.difficulty = difficulty + + return old_block_number + + +def _block_solver( + subtensor: SubtensorInterface, + wallet: Wallet, + num_processes: int, + netuid: int, + dev_id: list[int], + tpb: int, + update_interval: int, + curr_block, + curr_block_num, + curr_diff, + n_samples, + cuda: bool, +): + limit = int(math.pow(2, 256)) - 1 + + # Establish communication queues + ## See the _Solver class for more information on the queues. + stop_event = Event() + stop_event.clear() + + solution_queue = Queue() + finished_queues = [Queue() for _ in range(num_processes)] + check_block = Lock() + + hotkey_bytes = ( + wallet.coldkeypub.public_key if netuid == -1 else wallet.hotkey.public_key + ) + + if cuda: + ## Create a worker per CUDA device + num_processes = len(dev_id) + solvers = [ + _CUDASolver( + i, + num_processes, + update_interval, + finished_queues[i], + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, + dev_id[i], + tpb, + ) + for i in range(num_processes) + ] + else: + # Start consumers + solvers = [ + _Solver( + i, + num_processes, + update_interval, + finished_queues[i], + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, + ) + for i in range(num_processes) + ] + + # Get first block + block_number, difficulty, block_hash = _get_block_with_retry( + subtensor=subtensor, netuid=netuid + ) + + block_bytes = bytes.fromhex(block_hash[2:]) + old_block_number = block_number + # Set to current block + _update_curr_block( + curr_diff, + curr_block, + curr_block_num, + block_number, + block_bytes, + difficulty, + hotkey_bytes, + check_block, + ) + + # Set new block events for each solver to start at the initial block + for worker in solvers: + worker.newBlockEvent.set() + + for worker in solvers: + worker.start() # start the solver processes + + start_time = time.time() # time that the registration started + time_last = start_time # time that the last work blocks completed + + curr_stats = RegistrationStatistics( + time_spent_total=0.0, + time_average=0.0, + rounds_total=0, + time_spent=0.0, + hash_rate_perpetual=0.0, + hash_rate=0.0, + difficulty=difficulty, + block_number=block_number, + block_hash=block_hash, + ) + + start_time_perpetual = time.time() + + logger = RegistrationStatisticsLogger(console, output_in_place) + logger.start() + + solution = None + + hash_rates = [0] * n_samples # The last n true hash_rates + weights = [alpha_**i for i in range(n_samples)] # weights decay by alpha + + timeout = 0.15 if cuda else 0.15 + + while netuid == -1 or not subtensor.is_hotkey_registered( + netuid=netuid, hotkey_ss58=wallet.hotkey.ss58_address + ): + # Wait until a solver finds a solution + try: + solution = solution_queue.get(block=True, timeout=timeout) + if solution is not None: + break + except Empty: + # No solution found, try again + pass + + # check for new block + old_block_number = _check_for_newest_block_and_update( + subtensor=subtensor, + netuid=netuid, + hotkey_bytes=hotkey_bytes, + old_block_number=old_block_number, + curr_diff=curr_diff, + curr_block=curr_block, + curr_block_num=curr_block_num, + curr_stats=curr_stats, + update_curr_block=_update_curr_block, + check_block=check_block, + solvers=solvers, + ) + + num_time = 0 + for finished_queue in finished_queues: + try: + proc_num = finished_queue.get(timeout=0.1) + num_time += 1 + + except Empty: + continue + + time_now = time.time() # get current time + time_since_last = time_now - time_last # get time since last work block(s) + if num_time > 0 and time_since_last > 0.0: + # create EWMA of the hash_rate to make measure more robust + + if cuda: + hash_rate_ = (num_time * tpb * update_interval) / time_since_last + else: + hash_rate_ = (num_time * update_interval) / time_since_last + hash_rates.append(hash_rate_) + hash_rates.pop(0) # remove the 0th data point + curr_stats.hash_rate = sum( + [hash_rates[i] * weights[i] for i in range(n_samples)] + ) / (sum(weights)) + + # update time last to now + time_last = time_now + + curr_stats.time_average = ( + curr_stats.time_average * curr_stats.rounds_total + + curr_stats.time_spent + ) / (curr_stats.rounds_total + num_time) + curr_stats.rounds_total += num_time + + # Update stats + curr_stats.time_spent = time_since_last + new_time_spent_total = time_now - start_time_perpetual + if cuda: + curr_stats.hash_rate_perpetual = ( + curr_stats.rounds_total * (tpb * update_interval) + ) / new_time_spent_total + else: + curr_stats.hash_rate_perpetual = ( + curr_stats.rounds_total * update_interval + ) / new_time_spent_total + curr_stats.time_spent_total = new_time_spent_total + + # Update the logger + logger.update(curr_stats, verbose=log_verbose) + + # exited while, solution contains the nonce or wallet is registered + stop_event.set() # stop all other processes + logger.stop() + + # terminate and wait for all solvers to exit + _terminate_workers_and_wait_for_exit(solvers) + + return solution + + +def _solve_for_difficulty_fast_cuda( + subtensor: "bittensor.subtensor", + wallet: "bittensor.wallet", + netuid: int, + output_in_place: bool = True, + update_interval: int = 50_000, + tpb: int = 512, + dev_id: typing.Union[list[int], int] = 0, + n_samples: int = 10, + alpha_: float = 0.80, + log_verbose: bool = False, +) -> Optional[POWSolution]: + """ + Solves the registration fast using CUDA + Args: + subtensor: bittensor.subtensor + The subtensor node to grab blocks + wallet: bittensor.wallet + The wallet to register + netuid: int + The netuid of the subnet to register to. + output_in_place: bool + If true, prints the output in place, otherwise prints to new lines + update_interval: int + The number of nonces to try before checking for more blocks + tpb: int + The number of threads per block. CUDA param that should match the GPU capability + dev_id: Union[List[int], int] + The CUDA device IDs to execute the registration on, either a single device or a list of devices + n_samples: int + The number of samples of the hash_rate to keep for the EWMA + alpha_: float + The alpha for the EWMA for the hash_rate calculation + log_verbose: bool + If true, prints more verbose logging of the registration metrics. + Note: The hash rate is calculated as an exponentially weighted moving average in order to make the measure more robust. + """ + if isinstance(dev_id, int): + dev_id = [dev_id] + elif dev_id is None: + dev_id = [0] + + if update_interval is None: + update_interval = 50_000 + + if not torch.cuda.is_available(): + raise Exception("CUDA not available") + + # Set mp start to use spawn so CUDA doesn't complain + with _UsingSpawnStartMethod(force=True): + curr_block, curr_block_num, curr_diff = _CUDASolver.create_shared_memory() + + solution = _block_solver( + subtensor=subtensor, + wallet=wallet, + num_processes=None, + netuid=netuid, + dev_id=dev_id, + tpb=tpb, + update_interval=update_interval, + curr_block=curr_block, + curr_block_num=curr_block_num, + curr_diff=curr_diff, + n_samples=n_samples, + cuda=True, + ) + + return solution + + +def _solve_for_difficulty_fast( + subtensor, + wallet: "bittensor.wallet", + netuid: int, + output_in_place: bool = True, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, + n_samples: int = 10, + alpha_: float = 0.80, + log_verbose: bool = False, +) -> Optional[POWSolution]: + """ + Solves the POW for registration using multiprocessing. + Args: + subtensor + Subtensor to connect to for block information and to submit. + wallet: + wallet to use for registration. + netuid: int + The netuid of the subnet to register to. + output_in_place: bool + If true, prints the status in place. Otherwise, prints the status on a new line. + num_processes: int + Number of processes to use. + update_interval: int + Number of nonces to solve before updating block information. + n_samples: int + The number of samples of the hash_rate to keep for the EWMA + alpha_: float + The alpha for the EWMA for the hash_rate calculation + log_verbose: bool + If true, prints more verbose logging of the registration metrics. + Note: The hash rate is calculated as an exponentially weighted moving average in order to make the measure more robust. + Note: + - We can also modify the update interval to do smaller blocks of work, + while still updating the block information after a different number of nonces, + to increase the transparency of the process while still keeping the speed. + """ + if not num_processes: + # get the number of allowed processes for this process + num_processes = min(1, get_cpu_count()) + + if update_interval is None: + update_interval = 50_000 + + curr_block, curr_block_num, curr_diff = _Solver.create_shared_memory() + + solution = _block_solver( + subtensor=subtensor, + wallet=wallet, + num_processes=num_processes, + netuid=netuid, + dev_id=None, + tpb=None, + update_interval=update_interval, + curr_block=curr_block, + curr_block_num=curr_block_num, + curr_diff=curr_diff, + n_samples=n_samples, + cuda=True, + ) + + return solution + + +def _terminate_workers_and_wait_for_exit( + workers: list[typing.Union[Process, Queue]], +) -> None: + for worker in workers: + if isinstance(worker, Queue): + worker.join_thread() + else: + worker.join() + worker.close() + + +@backoff.on_exception(backoff.constant, Exception, interval=1, max_tries=3) +def _get_block_with_retry( + subtensor: SubtensorInterface, netuid: int +) -> tuple[int, int, bytes]: + """ + Gets the current block number, difficulty, and block hash from the substrate node. + + Args: + subtensor (:obj:`bittensor.subtensor`, `required`): + The subtensor object to use to get the block number, difficulty, and block hash. + + netuid (:obj:`int`, `required`): + The netuid of the network to get the block number, difficulty, and block hash from. + + Returns: + block_number (:obj:`int`): + The current block number. + + difficulty (:obj:`int`): + The current difficulty of the subnet. + + block_hash (:obj:`bytes`): + The current block hash. + + Raises: + Exception: If the block hash is None. + ValueError: If the difficulty is None. + """ + block_number = subtensor.get_current_block() + difficulty = 1_000_000 if netuid == -1 else subtensor.difficulty(netuid=netuid) + block_hash = subtensor.get_block_hash(block_number) + if block_hash is None: + raise Exception( + "Network error. Could not connect to substrate to get block hash" + ) + if difficulty is None: + raise ValueError("Chain error. Difficulty is None") + return block_number, difficulty, block_hash + + +class _UsingSpawnStartMethod: + def __init__(self, force: bool = False): + self._old_start_method = None + self._force = force + + def __enter__(self): + self._old_start_method = multiprocessing.get_start_method(allow_none=True) + if self._old_start_method == None: + self._old_start_method = "spawn" # default to spawn + + multiprocessing.set_start_method("spawn", force=self._force) + + def __exit__(self, *args): + # restore the old start method + multiprocessing.set_start_method(self._old_start_method, force=True) + + +def create_pow( + subtensor, + wallet, + netuid: int, + output_in_place: bool = True, + cuda: bool = False, + dev_id: Union[List[int], int] = 0, + tpb: int = 256, + num_processes: int = None, + update_interval: int = None, + log_verbose: bool = False, +) -> Optional[Dict[str, Any]]: + """ + Creates a proof of work for the given subtensor and wallet. + Args: + subtensor (:obj:`bittensor.subtensor.subtensor`, `required`): + The subtensor to create a proof of work for. + wallet (:obj:`bittensor.wallet.wallet`, `required`): + The wallet to create a proof of work for. + netuid (:obj:`int`, `required`): + The netuid for the subnet to create a proof of work for. + output_in_place (:obj:`bool`, `optional`, defaults to :obj:`True`): + If true, prints the progress of the proof of work to the console + in-place. Meaning the progress is printed on the same lines. + cuda (:obj:`bool`, `optional`, defaults to :obj:`False`): + If true, uses CUDA to solve the proof of work. + dev_id (:obj:`Union[List[int], int]`, `optional`, defaults to :obj:`0`): + The CUDA device id(s) to use. If cuda is true and dev_id is a list, + then multiple CUDA devices will be used to solve the proof of work. + tpb (:obj:`int`, `optional`, defaults to :obj:`256`): + The number of threads per block to use when solving the proof of work. + Should be a multiple of 32. + num_processes (:obj:`int`, `optional`, defaults to :obj:`None`): + The number of processes to use when solving the proof of work. + If None, then the number of processes is equal to the number of + CPU cores. + update_interval (:obj:`int`, `optional`, defaults to :obj:`None`): + The number of nonces to run before checking for a new block. + log_verbose (:obj:`bool`, `optional`, defaults to :obj:`False`): + If true, prints the progress of the proof of work more verbosely. + Returns: + :obj:`Optional[Dict[str, Any]]`: The proof of work solution or None if + the wallet is already registered or there is a different error. + + Raises: + :obj:`ValueError`: If the subnet does not exist. + """ + if netuid != -1: + if not subtensor.subnet_exists(netuid=netuid): + raise ValueError(f"Subnet {netuid} does not exist") + + if cuda: + solution: Optional[POWSolution] = _solve_for_difficulty_fast_cuda( + subtensor, + wallet, + netuid=netuid, + output_in_place=output_in_place, + dev_id=dev_id, + tpb=tpb, + update_interval=update_interval, + log_verbose=log_verbose, + ) + else: + solution: Optional[POWSolution] = _solve_for_difficulty_fast( + subtensor, + wallet, + netuid=netuid, + output_in_place=output_in_place, + num_processes=num_processes, + update_interval=update_interval, + log_verbose=log_verbose, + ) + + return solution diff --git a/src/wallets.py b/src/wallets.py index 41c745e3..755dd58f 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1225,3 +1225,29 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: table.add_row(*row) return console.print(table) + + +async def faucet( + wallet: Wallet, + subtensor: SubtensorInterface, + threads_per_block: int, + update_interval: int, + processes: int, + use_cuda: bool, + dev_id: int, + output_in_place: bool, + log_verbose: bool, +): + success = await subtensor.run_faucet( + wallet=wallet, + prompt=True, + tpb=threads_per_block, + update_interval=update_interval, + num_processes=processes, + cuda=use_cuda, + dev_id=dev_id, + output_in_place=output_in_place, + log_verbose=log_verbose, + ) + if not success: + err_console.print("Faucet run failed.") From 84e493793a0e40f6b79c30a01ca9106c37026ad7 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 18:42:18 +0200 Subject: [PATCH 37/48] Cleaned up Defaults class --- cli.py | 4 ++- src/__init__.py | 82 +++++++++++++++---------------------------------- 2 files changed, 28 insertions(+), 58 deletions(-) diff --git a/cli.py b/cli.py index 232ed101..367277ef 100755 --- a/cli.py +++ b/cli.py @@ -16,8 +16,10 @@ from src.utils import console -# re-usable args class Options: + """ + Re-usable typer args + """ wallet_name = typer.Option(None, "--wallet-name", "-w", help="Name of wallet") wallet_path = typer.Option( None, "--wallet-path", "-p", help="Filepath of root of wallets" diff --git a/src/__init__.py b/src/__init__.py index 08a1f7bf..a6b94ece 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Optional class Constants: @@ -35,69 +34,38 @@ def from_json(cls, json: dict[str, any]) -> "DelegatesDetails": ) -@dataclass -class CUDA: - dev_id: list[int] - use_cuda: bool - tpb: int - - -@dataclass -class PoWRegister: - num_processes: Optional[int] - update_interval: int - output_in_place: bool - verbose: bool - cuda: CUDA - +class Defaults: + netuid = 1 -@dataclass -class Wallet: - name: str - hotkey: str - path: str + class subtensor: + network = "finney" + chain_endpoint = None + _mock = False + class pow_register: + num_processes = None + update_interval = 50_000 + output_in_place = True + verbose = False -@dataclass -class Logging: - # likely needs to be changed - debug: bool - trace: bool - record_log: bool - logging_dir: str + class cuda: + dev_id = [0] + use_cuda = False + tpb = 256 + class wallet: + name = "default" + hotkey = "default" + path = "~/.bittensor/wallets/" -@dataclass -class Subtensor: - network: str - chain_endpoint: Optional[str] - _mock: bool + class logging: + debug = False + trace = False + record_log = False + logging_dir = "~/.bittensor/miners" -@dataclass -class Defaults: - netuid: int - subtensor: Subtensor - pow_register: PoWRegister - wallet: Wallet - logging: Logging - - -defaults = Defaults( - netuid=1, - subtensor=Subtensor(network="finney", chain_endpoint=None, _mock=False), - pow_register=PoWRegister( - num_processes=None, - update_interval=50000, - output_in_place=True, - verbose=False, - cuda=CUDA(dev_id=[0], use_cuda=False, tpb=256), - ), - wallet=Wallet(name="default", hotkey="default", path="~/.bittensor/wallets/"), - logging=Logging( - debug=False, trace=False, record_log=False, logging_dir="~/.bittensor/miners" - ), -) +defaults = Defaults TYPE_REGISTRY = { From 3afffec6e62b822d9bd3bb7051412f5089049d07 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 20:54:03 +0200 Subject: [PATCH 38/48] Formatting, cleanup, add requirements. --- cli.py | 1 + requirements.txt | 5 +- src/bittensor/extrinsics/registration.py | 794 ++++++++++++++++++----- src/utils.py | 25 +- 4 files changed, 670 insertions(+), 155 deletions(-) diff --git a/cli.py b/cli.py index 367277ef..d40affc8 100755 --- a/cli.py +++ b/cli.py @@ -20,6 +20,7 @@ class Options: """ Re-usable typer args """ + wallet_name = typer.Option(None, "--wallet-name", "-w", help="Name of wallet") wallet_path = typer.Option( None, "--wallet-path", "-p", help="Filepath of root of wallets" diff --git a/requirements.txt b/requirements.txt index bfea3a56..fdd99157 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,13 @@ aiohttp~=3.9.5 +backoff~=2.2.1 git+https://github.com/opentensor/btwallet # bittensor_wallet fuzzywuzzy~=0.18.0 netaddr~=1.3.0 +numpy>=2.0.1 +pycryptodome # Crypto +PyYAML~=6.0.1 rich~=13.7 scalecodec==1.2.11 substrate-interface~=1.7.9 typer~=0.12 websockets>=12.0 -PyYAML diff --git a/src/bittensor/extrinsics/registration.py b/src/bittensor/extrinsics/registration.py index a6f8dab1..5750765c 100644 --- a/src/bittensor/extrinsics/registration.py +++ b/src/bittensor/extrinsics/registration.py @@ -1,18 +1,37 @@ -import math +import binascii +from contextlib import redirect_stdout from dataclasses import dataclass +from datetime import timedelta import functools +import hashlib +import io +import math from multiprocessing.queues import Queue -from multiprocessing import Process, Event, Lock +import multiprocessing as mp +from multiprocessing import Process, Event, Lock, Array, Value import os +from queue import Empty, Full +import random import time import typing from typing import Optional +import backoff from bittensor_wallet import Wallet +from Crypto.Hash import keccak +import numpy as np from rich.prompt import Confirm +from rich.console import Console +from rich.status import Status from src.subtensor_interface import SubtensorInterface -from src.utils import console, err_console, format_error_message +from src.utils import ( + console, + err_console, + format_error_message, + millify, + get_human_readable, +) def use_torch() -> bool: @@ -20,16 +39,13 @@ def use_torch() -> bool: return True if os.getenv("USE_TORCH") == "1" else False -def legacy_torch_api_compat(func): +def legacy_torch_api_compat(func: typing.Callable): """ Convert function operating on numpy Input&Output to legacy torch Input&Output API if `use_torch()` is True. - Args: - func (function): - Function with numpy Input/Output to be decorated. - Returns: - decorated (function): - Decorated function. + :param func: Function with numpy Input/Output to be decorated. + + :return: Decorated function """ @functools.wraps(func) @@ -47,7 +63,7 @@ def decorated(*args, **kwargs): ret = func(*args, **kwargs) if use_torch(): # if return value is a numpy array, convert it to Torch tensor - if isinstance(ret, numpy.ndarray): + if isinstance(ret, np.ndarray): ret = torch.from_numpy(ret) return ret @@ -80,11 +96,285 @@ class POWSolution: difficulty: int seal: bytes - def is_stale(self, subtensor: "bittensor.subtensor") -> bool: + async def is_stale(self, subtensor: SubtensorInterface) -> bool: """Returns True if the POW is stale. This means the block the POW is solved for is within 3 blocks of the current block. """ - return self.block_number < subtensor.get_current_block() - 3 + return self.block_number < await subtensor.get_current_block() - 3 + + +@dataclass +class RegistrationStatistics: + """Statistics for a registration.""" + + time_spent_total: float + rounds_total: int + time_average: float + time_spent: float + hash_rate_perpetual: float + hash_rate: float + difficulty: int + block_number: int + block_hash: str + + +class RegistrationStatisticsLogger: + """Logs statistics for a registration.""" + + console: Console + status: Optional[Status] + + def __init__(self, console_: Console, output_in_place: bool = True) -> None: + self.console = console_ + + if output_in_place: + self.status = self.console.status("Solving") + else: + self.status = None + + def start(self) -> None: + if self.status is not None: + self.status.start() + + def stop(self) -> None: + if self.status is not None: + self.status.stop() + + @classmethod + def get_status_message( + cls, stats: RegistrationStatistics, verbose: bool = False + ) -> str: + message = ( + "Solving\n" + + f"Time Spent (total): [bold white]{timedelta(seconds=stats.time_spent_total)}[/bold white]\n" + + ( + f"Time Spent This Round: {timedelta(seconds=stats.time_spent)}\n" + + f"Time Spent Average: {timedelta(seconds=stats.time_average)}\n" + if verbose + else "" + ) + + f"Registration Difficulty: [bold white]{millify(stats.difficulty)}[/bold white]\n" + + f"Iters (Inst/Perp): [bold white]{get_human_readable(stats.hash_rate, 'H')}/s / " + + f"{get_human_readable(stats.hash_rate_perpetual, 'H')}/s[/bold white]\n" + + f"Block Number: [bold white]{stats.block_number}[/bold white]\n" + + f"Block Hash: [bold white]{stats.block_hash.encode('utf-8')}[/bold white]\n" + ) + return message + + def update(self, stats: RegistrationStatistics, verbose: bool = False) -> None: + if self.status is not None: + self.status.update(self.get_status_message(stats, verbose=verbose)) + else: + self.console.log(self.get_status_message(stats, verbose=verbose)) + + +class _SolverBase(Process): + """ + A process that solves the registration PoW problem. + + :param proc_num: The number of the process being created. + :param num_proc: The total number of processes running. + :param update_interval: The number of nonces to try to solve before checking for a new block. + :param finished_queue: The queue to put the process number when a process finishes each update_interval. + Used for calculating the average time per update_interval across all processes. + :param solution_queue: The queue to put the solution the process has found during the pow solve. + :param stop_event: The event to set by the main process when all the solver processes should stop. + The solver process will check for the event after each update_interval. + The solver process will stop when the event is set. + Used to stop the solver processes when a solution is found. + :param curr_block: The array containing this process's current block hash. + The main process will set the array to the new block hash when a new block is finalized in the network. + The solver process will get the new block hash from this array when newBlockEvent is set. + :param curr_block_num: The value containing this process's current block number. + The main process will set the value to the new block number when a new block is finalized in the network. + The solver process will get the new block number from this value when newBlockEvent is set. + :param curr_diff: The array containing this process's current difficulty. + The main process will set the array to the new difficulty when a new block is finalized in the network. + The solver process will get the new difficulty from this array when newBlockEvent is set. + :param check_block: The lock to prevent this process from getting the new block data while the main process is + updating the data. + :param limit: The limit of the pow solve for a valid solution. + + :var new_block_event: The event to set by the main process when a new block is finalized in the network. + The solver process will check for the event after each update_interval. + The solver process will get the new block hash and difficulty and start solving for a new nonce. + """ + + proc_num: int + num_proc: int + update_interval: int + finished_queue: Queue + solution_queue: Queue + new_block_event: Event + stop_event: Event + hotkey_bytes: bytes + curr_block: Array + curr_block_num: Value + curr_diff: Array + check_block: Lock + limit: int + + def __init__( + self, + proc_num, + num_proc, + update_interval, + finished_queue, + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, + ): + Process.__init__(self, daemon=True) + self.proc_num = proc_num + self.num_proc = num_proc + self.update_interval = update_interval + self.finished_queue = finished_queue + self.solution_queue = solution_queue + self.new_block_event = Event() + self.new_block_event.clear() + self.curr_block = curr_block + self.curr_block_num = curr_block_num + self.curr_diff = curr_diff + self.check_block = check_block + self.stop_event = stop_event + self.limit = limit + + def run(self): + raise NotImplementedError("_SolverBase is an abstract class") + + @staticmethod + def create_shared_memory() -> tuple[Array, Value, Array]: + """Creates shared memory for the solver processes to use.""" + curr_block = Array("h", 32, lock=True) # byte array + curr_block_num = Value("i", 0, lock=True) # int + curr_diff = Array("Q", [0, 0], lock=True) # [high, low] + + return curr_block, curr_block_num, curr_diff + + +class _Solver(_SolverBase): + def run(self): + block_number: int + block_and_hotkey_hash_bytes: bytes + block_difficulty: int + nonce_limit = int(math.pow(2, 64)) - 1 + + # Start at random nonce + nonce_start = random.randint(0, nonce_limit) + nonce_end = nonce_start + self.update_interval + while not self.stop_event.is_set(): + if self.new_block_event.is_set(): + with self.check_block: + block_number = self.curr_block_num.value + block_and_hotkey_hash_bytes = bytes(self.curr_block) + block_difficulty = _registration_diff_unpack(self.curr_diff) + + self.new_block_event.clear() + + # Do a block of nonces + solution = _solve_for_nonce_block( + nonce_start, + nonce_end, + block_and_hotkey_hash_bytes, + block_difficulty, + self.limit, + block_number, + ) + if solution is not None: + self.solution_queue.put(solution) + + try: + # Send time + self.finished_queue.put_nowait(self.proc_num) + except Full: + pass + + nonce_start = random.randint(0, nonce_limit) + nonce_start = nonce_start % nonce_limit + nonce_end = nonce_start + self.update_interval + + +class _CUDASolver(_SolverBase): + dev_id: int + tpb: int + + def __init__( + self, + proc_num, + num_proc, + update_interval, + finished_queue, + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, + dev_id: int, + tpb: int, + ): + super().__init__( + proc_num, + num_proc, + update_interval, + finished_queue, + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, + ) + self.dev_id = dev_id + self.tpb = tpb + + def run(self): + block_number: int = 0 # dummy value + block_and_hotkey_hash_bytes: bytes = b"0" * 32 # dummy value + block_difficulty: int = int(math.pow(2, 64)) - 1 # dummy value + nonce_limit = int(math.pow(2, 64)) - 1 # U64MAX + + # Start at random nonce + nonce_start = random.randint(0, nonce_limit) + while not self.stop_event.is_set(): + if self.new_block_event.is_set(): + with self.check_block: + block_number = self.curr_block_num.value + block_and_hotkey_hash_bytes = bytes(self.curr_block) + block_difficulty = _registration_diff_unpack(self.curr_diff) + + self.new_block_event.clear() + + # Do a block of nonces + solution = _solve_for_nonce_block_cuda( + nonce_start, + self.update_interval, + block_and_hotkey_hash_bytes, + block_difficulty, + self.limit, + block_number, + self.dev_id, + self.tpb, + ) + if solution is not None: + self.solution_queue.put(solution) + + try: + # Signal that a nonce_block was finished using queue + # send our proc_num + self.finished_queue.put(self.proc_num) + except Full: + pass + + # increase nonce by number of nonces processed + nonce_start += self.update_interval * self.tpb + nonce_start = nonce_start % nonce_limit class LazyLoadedTorch: @@ -261,47 +551,34 @@ async def run_faucet_extrinsic( def _check_for_newest_block_and_update( - subtensor: "bittensor.subtensor", + subtensor: SubtensorInterface, netuid: int, old_block_number: int, hotkey_bytes: bytes, - curr_diff: multiprocessing.Array, - curr_block: multiprocessing.Array, - curr_block_num: multiprocessing.Value, - update_curr_block: Callable, - check_block: "multiprocessing.Lock", - solvers: List[_Solver], + curr_diff: Array, + curr_block: Array, + curr_block_num: Value, + update_curr_block: typing.Callable, + check_block: Lock, + solvers: list[_Solver], curr_stats: RegistrationStatistics, ) -> int: """ Checks for a new block and updates the current block information if a new block is found. - Args: - subtensor (:obj:`bittensor.subtensor`, `required`): - The subtensor object to use for getting the current block. - netuid (:obj:`int`, `required`): - The netuid to use for retrieving the difficulty. - old_block_number (:obj:`int`, `required`): - The old block number to check against. - hotkey_bytes (:obj:`bytes`, `required`): - The bytes of the hotkey's pubkey. - curr_diff (:obj:`multiprocessing.Array`, `required`): - The current difficulty as a multiprocessing array. - curr_block (:obj:`multiprocessing.Array`, `required`): - Where the current block is stored as a multiprocessing array. - curr_block_num (:obj:`multiprocessing.Value`, `required`): - Where the current block number is stored as a multiprocessing value. - update_curr_block (:obj:`Callable`, `required`): - A function that updates the current block. - check_block (:obj:`multiprocessing.Lock`, `required`): - A mp lock that is used to check for a new block. - solvers (:obj:`List[_Solver]`, `required`): - A list of solvers to update the current block for. - curr_stats (:obj:`RegistrationStatistics`, `required`): - The current registration statistics to update. - - Returns: - (int) The current block number. + :param subtensor: The subtensor object to use for getting the current block. + :param netuid: The netuid to use for retrieving the difficulty. + :param old_block_number: The old block number to check against. + :param hotkey_bytes: The bytes of the hotkey's pubkey. + :param curr_diff: The current difficulty as a multiprocessing array. + :param curr_block: Where the current block is stored as a multiprocessing array. + :param curr_block_num: Where the current block number is stored as a multiprocessing value. + :param update_curr_block: A function that updates the current block. + :param check_block: A mp lock that is used to check for a new block. + :param solvers: A list of solvers to update the current block for. + :param curr_stats: The current registration statistics to update. + + :return: The current block number. """ block_number = subtensor.get_current_block() if block_number != old_block_number: @@ -325,7 +602,7 @@ def _check_for_newest_block_and_update( # Set new block events for each solver for worker in solvers: - worker.newBlockEvent.set() + worker.new_block_event.set() # update stats curr_stats.block_number = block_number @@ -347,6 +624,9 @@ def _block_solver( curr_block_num, curr_diff, n_samples, + alpha_, + output_in_place, + log_verbose, cuda: bool, ): limit = int(math.pow(2, 256)) - 1 @@ -425,7 +705,7 @@ def _block_solver( # Set new block events for each solver to start at the initial block for worker in solvers: - worker.newBlockEvent.set() + worker.new_block_event.set() for worker in solvers: worker.start() # start the solver processes @@ -544,8 +824,8 @@ def _block_solver( def _solve_for_difficulty_fast_cuda( - subtensor: "bittensor.subtensor", - wallet: "bittensor.wallet", + subtensor: SubtensorInterface, + wallet: Wallet, netuid: int, output_in_place: bool = True, update_interval: int = 50_000, @@ -557,28 +837,20 @@ def _solve_for_difficulty_fast_cuda( ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA - Args: - subtensor: bittensor.subtensor - The subtensor node to grab blocks - wallet: bittensor.wallet - The wallet to register - netuid: int - The netuid of the subnet to register to. - output_in_place: bool - If true, prints the output in place, otherwise prints to new lines - update_interval: int - The number of nonces to try before checking for more blocks - tpb: int - The number of threads per block. CUDA param that should match the GPU capability - dev_id: Union[List[int], int] - The CUDA device IDs to execute the registration on, either a single device or a list of devices - n_samples: int - The number of samples of the hash_rate to keep for the EWMA - alpha_: float - The alpha for the EWMA for the hash_rate calculation - log_verbose: bool - If true, prints more verbose logging of the registration metrics. - Note: The hash rate is calculated as an exponentially weighted moving average in order to make the measure more robust. + + :param subtensor: The subtensor node to grab blocks + :param wallet: The wallet to register + :param netuid: The netuid of the subnet to register to. + :param output_in_place: If true, prints the output in place, otherwise prints to new lines + :param update_interval: The number of nonces to try before checking for more blocks + :param tpb: The number of threads per block. CUDA param that should match the GPU capability + :param dev_id: The CUDA device IDs to execute the registration on, either a single device or a list of devices + :param n_samples: The number of samples of the hash_rate to keep for the EWMA + :param alpha_: The alpha for the EWMA for the hash_rate calculation + :param log_verbose: If true, prints more verbose logging of the registration metrics. + + Note: The hash rate is calculated as an exponentially weighted moving average in order to make the measure more + robust. """ if isinstance(dev_id, int): dev_id = [dev_id] @@ -607,6 +879,9 @@ def _solve_for_difficulty_fast_cuda( curr_block_num=curr_block_num, curr_diff=curr_diff, n_samples=n_samples, + alpha_=alpha_, + output_in_place=output_in_place, + log_verbose=log_verbose, cuda=True, ) @@ -615,7 +890,7 @@ def _solve_for_difficulty_fast_cuda( def _solve_for_difficulty_fast( subtensor, - wallet: "bittensor.wallet", + wallet: Wallet, netuid: int, output_in_place: bool = True, num_processes: Optional[int] = None, @@ -626,30 +901,22 @@ def _solve_for_difficulty_fast( ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. - Args: - subtensor - Subtensor to connect to for block information and to submit. - wallet: - wallet to use for registration. - netuid: int - The netuid of the subnet to register to. - output_in_place: bool - If true, prints the status in place. Otherwise, prints the status on a new line. - num_processes: int - Number of processes to use. - update_interval: int - Number of nonces to solve before updating block information. - n_samples: int - The number of samples of the hash_rate to keep for the EWMA - alpha_: float - The alpha for the EWMA for the hash_rate calculation - log_verbose: bool - If true, prints more verbose logging of the registration metrics. - Note: The hash rate is calculated as an exponentially weighted moving average in order to make the measure more robust. - Note: - - We can also modify the update interval to do smaller blocks of work, - while still updating the block information after a different number of nonces, - to increase the transparency of the process while still keeping the speed. + + :param subtensor: Subtensor to connect to for block information and to submit. + :param wallet: wallet to use for registration. + :param netuid: The netuid of the subnet to register to. + :param output_in_place: If true, prints the status in place. Otherwise, prints the status on a new line. + :param num_processes: Number of processes to use. + :param update_interval: Number of nonces to solve before updating block information. + :param n_samples: The number of samples of the hash_rate to keep for the EWMA + :param alpha_: The alpha for the EWMA for the hash_rate calculation + :param log_verbose: If true, prints more verbose logging of the registration metrics. + + Notes: + + - The hash rate is calculated as an exponentially weighted moving average in order to make the measure more robust. + - We can also modify the update interval to do smaller blocks of work, while still updating the block information + after a different number of nonces, to increase the transparency of the process while still keeping the speed. """ if not num_processes: # get the number of allowed processes for this process @@ -672,6 +939,9 @@ def _solve_for_difficulty_fast( curr_block_num=curr_block_num, curr_diff=curr_diff, n_samples=n_samples, + alpha_=alpha_, + output_in_place=output_in_place, + log_verbose=log_verbose, cuda=True, ) @@ -696,26 +966,13 @@ def _get_block_with_retry( """ Gets the current block number, difficulty, and block hash from the substrate node. - Args: - subtensor (:obj:`bittensor.subtensor`, `required`): - The subtensor object to use to get the block number, difficulty, and block hash. - - netuid (:obj:`int`, `required`): - The netuid of the network to get the block number, difficulty, and block hash from. - - Returns: - block_number (:obj:`int`): - The current block number. + :param subtensor: The subtensor object to use to get the block number, difficulty, and block hash. + :param netuid: The netuid of the network to get the block number, difficulty, and block hash from. - difficulty (:obj:`int`): - The current difficulty of the subnet. + :return: The current block number, difficulty of the subnet, block hash - block_hash (:obj:`bytes`): - The current block hash. - - Raises: - Exception: If the block hash is None. - ValueError: If the difficulty is None. + :raises Exception: If the block hash is None. + :raises ValueError: If the difficulty is None. """ block_number = subtensor.get_current_block() difficulty = 1_000_000 if netuid == -1 else subtensor.difficulty(netuid=netuid) @@ -729,21 +986,32 @@ def _get_block_with_retry( return block_number, difficulty, block_hash +def _registration_diff_unpack(packed_diff: Array) -> int: + """Unpacks the packed two 32-bit integers into one 64-bit integer. Little endian.""" + return int(packed_diff[0] << 32 | packed_diff[1]) + + +def _registration_diff_pack(diff: int, packed_diff: Array): + """Packs the difficulty into two 32-bit integers. Little endian.""" + packed_diff[0] = diff >> 32 + packed_diff[1] = diff & 0xFFFFFFFF # low 32 bits + + class _UsingSpawnStartMethod: def __init__(self, force: bool = False): self._old_start_method = None self._force = force def __enter__(self): - self._old_start_method = multiprocessing.get_start_method(allow_none=True) - if self._old_start_method == None: + self._old_start_method = mp.get_start_method(allow_none=True) + if self._old_start_method is None: self._old_start_method = "spawn" # default to spawn - multiprocessing.set_start_method("spawn", force=self._force) + mp.set_start_method("spawn", force=self._force) def __exit__(self, *args): # restore the old start method - multiprocessing.set_start_method(self._old_start_method, force=True) + mp.set_start_method(self._old_start_method, force=True) def create_pow( @@ -752,46 +1020,32 @@ def create_pow( netuid: int, output_in_place: bool = True, cuda: bool = False, - dev_id: Union[List[int], int] = 0, + dev_id: typing.Union[list[int], int] = 0, tpb: int = 256, num_processes: int = None, update_interval: int = None, log_verbose: bool = False, -) -> Optional[Dict[str, Any]]: +) -> Optional[dict[str, typing.Any]]: """ Creates a proof of work for the given subtensor and wallet. - Args: - subtensor (:obj:`bittensor.subtensor.subtensor`, `required`): - The subtensor to create a proof of work for. - wallet (:obj:`bittensor.wallet.wallet`, `required`): - The wallet to create a proof of work for. - netuid (:obj:`int`, `required`): - The netuid for the subnet to create a proof of work for. - output_in_place (:obj:`bool`, `optional`, defaults to :obj:`True`): - If true, prints the progress of the proof of work to the console - in-place. Meaning the progress is printed on the same lines. - cuda (:obj:`bool`, `optional`, defaults to :obj:`False`): - If true, uses CUDA to solve the proof of work. - dev_id (:obj:`Union[List[int], int]`, `optional`, defaults to :obj:`0`): - The CUDA device id(s) to use. If cuda is true and dev_id is a list, - then multiple CUDA devices will be used to solve the proof of work. - tpb (:obj:`int`, `optional`, defaults to :obj:`256`): - The number of threads per block to use when solving the proof of work. - Should be a multiple of 32. - num_processes (:obj:`int`, `optional`, defaults to :obj:`None`): - The number of processes to use when solving the proof of work. - If None, then the number of processes is equal to the number of - CPU cores. - update_interval (:obj:`int`, `optional`, defaults to :obj:`None`): - The number of nonces to run before checking for a new block. - log_verbose (:obj:`bool`, `optional`, defaults to :obj:`False`): - If true, prints the progress of the proof of work more verbosely. - Returns: - :obj:`Optional[Dict[str, Any]]`: The proof of work solution or None if - the wallet is already registered or there is a different error. - - Raises: - :obj:`ValueError`: If the subnet does not exist. + + :param subtensor: The subtensor to create a proof of work for. + :param wallet: The wallet to create a proof of work for. + :param netuid: The netuid for the subnet to create a proof of work for. + :param output_in_place: If true, prints the progress of the proof of work to the console + in-place. Meaning the progress is printed on the same lines. + :param cuda: If true, uses CUDA to solve the proof of work. + :param dev_id: The CUDA device id(s) to use. If cuda is true and dev_id is a list, + then multiple CUDA devices will be used to solve the proof of work. + :param tpb: The number of threads per block to use when solving the proof of work. Should be a multiple of 32. + :param num_processes: The number of processes to use when solving the proof of work. + If None, then the number of processes is equal to the number of CPU cores. + :param update_interval: The number of nonces to run before checking for a new block. + :param log_verbose: If true, prints the progress of the proof of work more verbosely. + + :return: The proof of work solution or None if the wallet is already registered or there is a different error. + + :raises ValueError: If the subnet does not exist. """ if netuid != -1: if not subtensor.subnet_exists(netuid=netuid): @@ -820,3 +1074,237 @@ def create_pow( ) return solution + + +def _solve_for_nonce_block_cuda( + nonce_start: int, + update_interval: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + block_number: int, + dev_id: int, + tpb: int, +) -> Optional[POWSolution]: + """ + Tries to solve the POW on a CUDA device for a block of nonces (nonce_start, nonce_start + update_interval * tpb + """ + solution, seal = solve_cuda( + nonce_start, + update_interval, + tpb, + block_and_hotkey_hash_bytes, + difficulty, + limit, + dev_id, + ) + + if solution != -1: + # Check if solution is valid (i.e. not -1) + return POWSolution(solution, block_number, difficulty, seal) + + return None + + +def _solve_for_nonce_block( + nonce_start: int, + nonce_end: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + block_number: int, +) -> Optional[POWSolution]: + """ + Tries to solve the POW for a block of nonces (nonce_start, nonce_end) + """ + for nonce in range(nonce_start, nonce_end): + # Create seal. + seal = _create_seal_hash(block_and_hotkey_hash_bytes, nonce) + + # Check if seal meets difficulty + if _seal_meets_difficulty(seal, difficulty, limit): + # Found a solution, save it. + return POWSolution(nonce, block_number, difficulty, seal) + + return None + + +class CUDAException(Exception): + """An exception raised when an error occurs in the CUDA environment.""" + + +def _hex_bytes_to_u8_list(hex_bytes: bytes): + hex_chunks = [int(hex_bytes[i : i + 2], 16) for i in range(0, len(hex_bytes), 2)] + return hex_chunks + + +def _create_seal_hash(block_and_hotkey_hash_bytes: bytes, nonce: int) -> bytes: + nonce_bytes = binascii.hexlify(nonce.to_bytes(8, "little")) + pre_seal = nonce_bytes + binascii.hexlify(block_and_hotkey_hash_bytes)[:64] + seal_sh256 = hashlib.sha256(bytearray(_hex_bytes_to_u8_list(pre_seal))).digest() + kec = keccak.new(digest_bits=256) + seal = kec.update(seal_sh256).digest() + return seal + + +def _seal_meets_difficulty(seal: bytes, difficulty: int, limit: int): + seal_number = int.from_bytes(seal, "big") + product = seal_number * difficulty + return product < limit + + +def _hash_block_with_hotkey(block_bytes: bytes, hotkey_bytes: bytes) -> bytes: + """Hashes the block with the hotkey using Keccak-256 to get 32 bytes""" + kec = keccak.new(digest_bits=256) + kec = kec.update(bytearray(block_bytes + hotkey_bytes)) + block_and_hotkey_hash_bytes = kec.digest() + return block_and_hotkey_hash_bytes + + +def _update_curr_block( + curr_diff: Array, + curr_block: Array, + curr_block_num: Value, + block_number: int, + block_bytes: bytes, + diff: int, + hotkey_bytes: bytes, + lock: Lock, +): + with lock: + curr_block_num.value = block_number + # Hash the block with the hotkey + block_and_hotkey_hash_bytes = _hash_block_with_hotkey(block_bytes, hotkey_bytes) + for i in range(32): + curr_block[i] = block_and_hotkey_hash_bytes[i] + _registration_diff_pack(diff, curr_diff) + + +def get_cpu_count() -> int: + try: + return len(os.sched_getaffinity(0)) + except AttributeError: + # macOS does not have sched_getaffinity + return os.cpu_count() + + +@dataclass +class RegistrationStatistics: + """Statistics for a registration.""" + + time_spent_total: float + rounds_total: int + time_average: float + time_spent: float + hash_rate_perpetual: float + hash_rate: float + difficulty: int + block_number: int + block_hash: bytes + + +def solve_cuda( + nonce_start: np.int64, + update_interval: np.int64, + tpb: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + dev_id: int = 0, +) -> tuple[np.int64, bytes]: + """ + Solves the PoW problem using CUDA. + + :param nonce_start: Starting nonce. + :param update_interval: Number of nonces to solve before updating block information. + :param tpb: Threads per block. + :param block_and_hotkey_hash_bytes: Keccak(Bytes of the block hash + bytes of the hotkey) 64 bytes. + :param difficulty: Difficulty of the PoW problem. + :param limit: Upper limit of the nonce. + :param dev_id: The CUDA device ID + + :return: (nonce, seal) corresponding to the solution. Returns -1 for nonce if no solution is found. + """ + + try: + import cubit + except ImportError: + raise ImportError("Please install cubit") + + upper = int(limit // difficulty) + + upper_bytes = upper.to_bytes(32, byteorder="little", signed=False) + + def _hex_bytes_to_u8_list(hex_bytes: bytes): + hex_chunks = [ + int(hex_bytes[i : i + 2], 16) for i in range(0, len(hex_bytes), 2) + ] + return hex_chunks + + def _create_seal_hash(block_and_hotkey_hash_hex: bytes, nonce: int) -> bytes: + nonce_bytes = binascii.hexlify(nonce.to_bytes(8, "little")) + pre_seal = nonce_bytes + block_and_hotkey_hash_hex + seal_sh256 = hashlib.sha256(bytearray(_hex_bytes_to_u8_list(pre_seal))).digest() + kec = keccak.new(digest_bits=256) + seal = kec.update(seal_sh256).digest() + return seal + + def _seal_meets_difficulty(seal: bytes, difficulty: int): + seal_number = int.from_bytes(seal, "big") + product = seal_number * difficulty + limit = int(math.pow(2, 256)) - 1 + + return product < limit + + # Call cython function + # int blockSize, uint64 nonce_start, uint64 update_interval, const unsigned char[:] limit, + # const unsigned char[:] block_bytes, int dev_id + block_and_hotkey_hash_hex = binascii.hexlify(block_and_hotkey_hash_bytes)[:64] + + solution = cubit.solve_cuda( + tpb, + nonce_start, + update_interval, + upper_bytes, + block_and_hotkey_hash_hex, + dev_id, + ) # 0 is first GPU + seal = None + if solution != -1: + seal = _create_seal_hash(block_and_hotkey_hash_hex, solution) + if _seal_meets_difficulty(seal, difficulty): + return solution, seal + else: + return -1, b"\x00" * 32 + + return solution, seal + + +def reset_cuda(): + """ + Resets the CUDA environment. + """ + try: + import cubit + except ImportError: + raise ImportError("Please install cubit") + + cubit.reset_cuda() + + +def log_cuda_errors() -> str: + """ + Logs any CUDA errors. + """ + try: + import cubit + except ImportError: + raise ImportError("Please install cubit") + + f = io.StringIO() + with redirect_stdout(f): + cubit.log_cuda_errors() + + s = f.getvalue() + + return s diff --git a/src/utils.py b/src/utils.py index 51161579..262cfea5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,6 +1,7 @@ import os +import math from pathlib import Path -from typing import Union, Optional, Any +from typing import Union, Any import aiohttp import scalecodec @@ -289,3 +290,25 @@ async def get_delegates_details_from_github(url: str) -> dict[str, DelegatesDeta ) return all_delegates_details + + +def get_human_readable(num, suffix="H"): + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(num) < 1000.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1000.0 + return f"{num:.1f}Y{suffix}" + + +def millify(n: int): + mill_names = ["", " K", " M", " B", " T"] + n = float(n) + mill_idx = max( + 0, + min( + len(mill_names) - 1, + int(math.floor(0 if n == 0 else math.log10(abs(n)) / 3)), + ), + ) + + return "{:.2f}{}".format(n / 10 ** (3 * mill_idx), mill_names[mill_idx]) From cf2f804734d5280fc1f1dd06d6109e21f4dcd2b7 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 20:55:24 +0200 Subject: [PATCH 39/48] Formatting --- src/bittensor/balances.py | 3 +-- src/subtensor_interface.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/bittensor/balances.py b/src/bittensor/balances.py index 326b3ed9..d0c1945c 100644 --- a/src/bittensor/balances.py +++ b/src/bittensor/balances.py @@ -42,8 +42,7 @@ def __init__(self, balance: Union[int, float]): Initialize a Balance object. If balance is an int, it's assumed to be in rao. If balance is a float, it's assumed to be in tao. - Args: - balance: The initial balance, in either rao (if an int) or tao (if a float). + :param balance: The initial balance, in either rao (if an int) or tao (if a float). """ if isinstance(balance, int): self.rao = balance diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index d5b6da1e..4f193c23 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -108,7 +108,6 @@ async def is_hotkey_delegate( Determines whether a given hotkey (public key) is a delegate on the Bittensor network. This function checks if the neuron associated with the hotkey is part of the network's delegation system. - Args: :param hotkey_ss58: The SS58 address of the neuron's hotkey. :param block_hash: The hash of the blockchain block number for the query. :param reuse_block: Whether to reuse the last-used block hash. @@ -149,7 +148,6 @@ async def get_stake_info_for_coldkey( Retrieves stake information associated with a specific coldkey. This function provides details about the stakes held by an account, including the staked amounts and associated delegates. - Args: :param coldkey_ss58: The ``SS58`` address of the account's coldkey. :param block_hash: The hash of the blockchain block number for the query. :param reuse_block: Whether to reuse the last-used block hash. @@ -288,7 +286,6 @@ async def get_netuids_for_hotkey( identifies the specific subnets within the Bittensor network where the neuron associated with the hotkey is active. - Args: :param hotkey_ss58: The ``SS58`` address of the neuron's hotkey. :param block_hash: The hash of the blockchain block number at which to perform the query. :param reuse_block: Whether to reuse the last-used block hash when retrieving info. From 7aae0f1256107c8b76ed5bb3b9dd72a27392503c Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 22:32:52 +0200 Subject: [PATCH 40/48] Renamed 'default' typer app to 'config' typer app. Faucet working. --- cli.py | 24 +- cuda_requirements.txt | 1 + src/__init__.py | 2 +- src/bittensor/async_substrate_interface.py | 2 +- src/bittensor/extrinsics/registration.py | 542 ++++++++++----------- src/wallets.py | 10 +- 6 files changed, 292 insertions(+), 289 deletions(-) create mode 100644 cuda_requirements.txt diff --git a/cli.py b/cli.py index d40affc8..bb393291 100755 --- a/cli.py +++ b/cli.py @@ -112,26 +112,27 @@ def __init__(self): self.not_subtensor = None self.app = typer.Typer(rich_markup_mode="markdown", callback=self.check_config) - self.default_app = typer.Typer() + self.config_app = typer.Typer() self.wallet_app = typer.Typer() self.delegates_app = typer.Typer() - # default alias - self.app.add_typer(self.default_app, name="default") - self.app.add_typer(self.default_app, name="def", hidden=True) + # config alias + self.app.add_typer(self.config_app, name="config", short_help="Config commands, aliases: `c`, `conf`") + self.app.add_typer(self.config_app, name="conf", hidden=True) + self.app.add_typer(self.config_app, name="c", hidden=True) # wallet aliases - self.app.add_typer(self.wallet_app, name="wallet") + self.app.add_typer(self.wallet_app, name="wallet", short_help="Wallet commands, aliases: `wallets`, `w`") self.app.add_typer(self.wallet_app, name="w", hidden=True) self.app.add_typer(self.wallet_app, name="wallets", hidden=True) # delegates aliases - self.app.add_typer(self.delegates_app, name="delegates") + self.app.add_typer(self.delegates_app, name="delegates", short_help="Delegate commands, alias: `d`") self.app.add_typer(self.delegates_app, name="d", hidden=True) - # defaults commands - self.default_app.command("set")(self.set_config) - self.default_app.command("get")(self.get_config) + # config commands + self.config_app.command("set")(self.set_config) + self.config_app.command("get")(self.get_config) # wallet commands self.wallet_app.command("list")(self.wallet_list) @@ -146,6 +147,7 @@ def __init__(self): self.wallet_app.command("overview")(self.wallet_overview) self.wallet_app.command("transfer")(self.wallet_transfer) self.wallet_app.command("inspect")(self.wallet_inspect) + self.wallet_app.command("faucet")(self.wallet_faucet) # delegates commands self.delegates_app.command("list")(self.delegates_list) @@ -620,6 +622,10 @@ def wallet_faucet( real TAO tokens. It's important for users to have the necessary hardware setup, especially when opting for CUDA-based GPU calculations. It is currently disabled on testnet and finney. You must use this on a local chain. """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + self.initialize_chain(network, chain) + self._run_command(wallets.faucet(wallet, self.not_subtensor, threads_per_block, update_interval, processors, + use_cuda, dev_id, output_in_place, verbose, max_successes)) def wallet_regen_coldkey( self, diff --git a/cuda_requirements.txt b/cuda_requirements.txt new file mode 100644 index 00000000..cda46405 --- /dev/null +++ b/cuda_requirements.txt @@ -0,0 +1 @@ +cubit>=1.1.0 diff --git a/src/__init__.py b/src/__init__.py index a6b94ece..8d2c4cfd 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -49,7 +49,7 @@ class pow_register: verbose = False class cuda: - dev_id = [0] + dev_id = 0 use_cuda = False tpb = 256 diff --git a/src/bittensor/async_substrate_interface.py b/src/bittensor/async_substrate_interface.py index 9dd4b398..85260985 100644 --- a/src/bittensor/async_substrate_interface.py +++ b/src/bittensor/async_substrate_interface.py @@ -1403,7 +1403,7 @@ async def get_metadata_call_function( if call.name == call_function_name: return call - async def get_block_number(self, block_hash: str) -> int: + async def get_block_number(self, block_hash: Optional[str]) -> int: """Async version of `substrateinterface.base.get_block_number` method.""" response = await self.rpc_request("chain_getHeader", [block_hash]) diff --git a/src/bittensor/extrinsics/registration.py b/src/bittensor/extrinsics/registration.py index 5750765c..905a8428 100644 --- a/src/bittensor/extrinsics/registration.py +++ b/src/bittensor/extrinsics/registration.py @@ -6,9 +6,9 @@ import hashlib import io import math -from multiprocessing.queues import Queue import multiprocessing as mp -from multiprocessing import Process, Event, Lock, Array, Value +from multiprocessing.queues import Queue as Queue_Type +from multiprocessing import Process, Event, Lock, Array, Value, Queue import os from queue import Empty, Full import random @@ -23,6 +23,7 @@ from rich.prompt import Confirm from rich.console import Console from rich.status import Status +from substrateinterface.exceptions import SubstrateRequestException from src.subtensor_interface import SubtensorInterface from src.utils import ( @@ -100,7 +101,9 @@ async def is_stale(self, subtensor: SubtensorInterface) -> bool: """Returns True if the POW is stale. This means the block the POW is solved for is within 3 blocks of the current block. """ - return self.block_number < await subtensor.get_current_block() - 3 + async with subtensor: + current_block = await subtensor.substrate.get_block_number(None) + return self.block_number < current_block - 3 @dataclass @@ -142,22 +145,22 @@ def stop(self) -> None: @classmethod def get_status_message( - cls, stats: RegistrationStatistics, verbose: bool = False + cls, stats: RegistrationStatistics, verbose: bool = False ) -> str: message = ( - "Solving\n" - + f"Time Spent (total): [bold white]{timedelta(seconds=stats.time_spent_total)}[/bold white]\n" - + ( - f"Time Spent This Round: {timedelta(seconds=stats.time_spent)}\n" - + f"Time Spent Average: {timedelta(seconds=stats.time_average)}\n" - if verbose - else "" - ) - + f"Registration Difficulty: [bold white]{millify(stats.difficulty)}[/bold white]\n" - + f"Iters (Inst/Perp): [bold white]{get_human_readable(stats.hash_rate, 'H')}/s / " - + f"{get_human_readable(stats.hash_rate_perpetual, 'H')}/s[/bold white]\n" - + f"Block Number: [bold white]{stats.block_number}[/bold white]\n" - + f"Block Hash: [bold white]{stats.block_hash.encode('utf-8')}[/bold white]\n" + "Solving\n" + + f"Time Spent (total): [bold white]{timedelta(seconds=stats.time_spent_total)}[/bold white]\n" + + ( + f"Time Spent This Round: {timedelta(seconds=stats.time_spent)}\n" + + f"Time Spent Average: {timedelta(seconds=stats.time_average)}\n" + if verbose + else "" + ) + + f"Registration Difficulty: [bold white]{millify(stats.difficulty)}[/bold white]\n" + + f"Iters (Inst/Perp): [bold white]{get_human_readable(stats.hash_rate, 'H')}/s / " + + f"{get_human_readable(stats.hash_rate_perpetual, 'H')}/s[/bold white]\n" + + f"Block Number: [bold white]{stats.block_number}[/bold white]\n" + + f"Block Hash: [bold white]{stats.block_hash.encode('utf-8')}[/bold white]\n" ) return message @@ -203,8 +206,8 @@ class _SolverBase(Process): proc_num: int num_proc: int update_interval: int - finished_queue: Queue - solution_queue: Queue + finished_queue: Queue_Type + solution_queue: Queue_Type new_block_event: Event stop_event: Event hotkey_bytes: bytes @@ -215,18 +218,18 @@ class _SolverBase(Process): limit: int def __init__( - self, - proc_num, - num_proc, - update_interval, - finished_queue, - solution_queue, - stop_event, - curr_block, - curr_block_num, - curr_diff, - check_block, - limit, + self, + proc_num, + num_proc, + update_interval, + finished_queue, + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, ): Process.__init__(self, daemon=True) self.proc_num = proc_num @@ -303,20 +306,20 @@ class _CUDASolver(_SolverBase): tpb: int def __init__( - self, - proc_num, - num_proc, - update_interval, - finished_queue, - solution_queue, - stop_event, - curr_block, - curr_block_num, - curr_diff, - check_block, - limit, - dev_id: int, - tpb: int, + self, + proc_num, + num_proc, + update_interval, + finished_queue, + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, + dev_id: int, + tpb: int, ): super().__init__( proc_num, @@ -404,20 +407,20 @@ class MaxAttemptsException(Exception): async def run_faucet_extrinsic( - subtensor: SubtensorInterface, - wallet: Wallet, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - prompt: bool = False, - max_allowed_attempts: int = 3, - output_in_place: bool = True, - cuda: bool = False, - dev_id: int = 0, - tpb: int = 256, - num_processes: Optional[int] = None, - update_interval: Optional[int] = None, - log_verbose: bool = False, - max_successes: int = 3, + subtensor: SubtensorInterface, + wallet: Wallet, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, + prompt: bool = False, + max_allowed_attempts: int = 3, + output_in_place: bool = True, + cuda: bool = False, + dev_id: int = 0, + tpb: int = 256, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, + log_verbose: bool = False, + max_successes: int = 3, ) -> tuple[bool, str]: r"""Runs a continual POW to get a faucet of TAO on the test net. @@ -443,9 +446,9 @@ async def run_faucet_extrinsic( """ if prompt: if not Confirm.ask( - "Run Faucet ?\n" - f" coldkey: [bold white]{wallet.coldkeypub.ss58_address}[/bold white]\n" - f" network: [bold white]{subtensor}[/bold white]" + "Run Faucet ?\n" + f" coldkey: [bold white]{wallet.coldkeypub.ss58_address}[/bold white]\n" + f" network: [bold white]{subtensor}[/bold white]" ): return False, "" @@ -457,7 +460,8 @@ async def run_faucet_extrinsic( wallet.unlock_coldkey() # Get previous balance. - old_balance = await subtensor.get_balance(wallet.coldkeypub.ss58_address) + async with subtensor: + old_balance = await subtensor.get_balance(wallet.coldkeypub.ss58_address) # Attempt rolling registration. attempts = 1 @@ -466,14 +470,14 @@ async def run_faucet_extrinsic( async with subtensor: try: pow_result = None - while pow_result is None or pow_result.is_stale(subtensor=subtensor): + while pow_result is None or await pow_result.is_stale(subtensor=subtensor): # Solve latest POW. if cuda: if not torch.cuda.is_available(): if prompt: err_console.print("CUDA is not available.") return False, "CUDA is not available." - pow_result: Optional[POWSolution] = create_pow( + pow_result: Optional[POWSolution] = await create_pow( subtensor, wallet, -1, @@ -486,7 +490,7 @@ async def run_faucet_extrinsic( log_verbose=log_verbose, ) else: - pow_result: Optional[POWSolution] = create_pow( + pow_result: Optional[POWSolution] = await create_pow( subtensor, wallet, -1, @@ -496,7 +500,7 @@ async def run_faucet_extrinsic( update_interval=update_interval, log_verbose=log_verbose, ) - call = subtensor.substrate.compose_call( + call = await subtensor.substrate.compose_call( call_module="SubtensorModule", call_function="faucet", call_params={ @@ -505,10 +509,10 @@ async def run_faucet_extrinsic( "work": [int(byte_) for byte_ in pow_result.seal], }, ) - extrinsic = subtensor.substrate.create_signed_extrinsic( + extrinsic = await subtensor.substrate.create_signed_extrinsic( call=call, keypair=wallet.coldkey ) - response = subtensor.substrate.submit_extrinsic( + response = await subtensor.substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, @@ -528,9 +532,10 @@ async def run_faucet_extrinsic( # Successful registration else: - new_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address) + new_balance = await subtensor.get_balance(wallet.coldkeypub.ss58_address) console.print( - f"Balance: [blue]{old_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + f"Balance: [blue]{old_balance[wallet.coldkeypub.ss58_address]}[/blue] :arrow_right:" + f" [green]{new_balance[wallet.coldkeypub.ss58_address]}[/green]" ) old_balance = new_balance @@ -550,18 +555,18 @@ async def run_faucet_extrinsic( return False, f"Max attempts reached: {max_allowed_attempts}" -def _check_for_newest_block_and_update( - subtensor: SubtensorInterface, - netuid: int, - old_block_number: int, - hotkey_bytes: bytes, - curr_diff: Array, - curr_block: Array, - curr_block_num: Value, - update_curr_block: typing.Callable, - check_block: Lock, - solvers: list[_Solver], - curr_stats: RegistrationStatistics, +async def _check_for_newest_block_and_update( + subtensor: SubtensorInterface, + netuid: int, + old_block_number: int, + hotkey_bytes: bytes, + curr_diff: Array, + curr_block: Array, + curr_block_num: Value, + update_curr_block: typing.Callable, + check_block: Lock, + solvers: list[_Solver], + curr_stats: RegistrationStatistics, ) -> int: """ Checks for a new block and updates the current block information if a new block is found. @@ -580,11 +585,12 @@ def _check_for_newest_block_and_update( :return: The current block number. """ - block_number = subtensor.get_current_block() + async with subtensor: + block_number = await subtensor.substrate.get_block_number(None) if block_number != old_block_number: old_block_number = block_number # update block information - block_number, difficulty, block_hash = _get_block_with_retry( + block_number, difficulty, block_hash = await _get_block_with_retry( subtensor=subtensor, netuid=netuid ) block_bytes = bytes.fromhex(block_hash[2:]) @@ -612,22 +618,22 @@ def _check_for_newest_block_and_update( return old_block_number -def _block_solver( - subtensor: SubtensorInterface, - wallet: Wallet, - num_processes: int, - netuid: int, - dev_id: list[int], - tpb: int, - update_interval: int, - curr_block, - curr_block_num, - curr_diff, - n_samples, - alpha_, - output_in_place, - log_verbose, - cuda: bool, +async def _block_solver( + subtensor: SubtensorInterface, + wallet: Wallet, + num_processes: int, + netuid: int, + dev_id: list[int], + tpb: int, + update_interval: int, + curr_block, + curr_block_num, + curr_diff, + n_samples, + alpha_, + output_in_place, + log_verbose, + cuda: bool, ): limit = int(math.pow(2, 256)) - 1 @@ -685,7 +691,7 @@ def _block_solver( ] # Get first block - block_number, difficulty, block_hash = _get_block_with_retry( + block_number, difficulty, block_hash = await _get_block_with_retry( subtensor=subtensor, netuid=netuid ) @@ -733,85 +739,88 @@ def _block_solver( solution = None hash_rates = [0] * n_samples # The last n true hash_rates - weights = [alpha_**i for i in range(n_samples)] # weights decay by alpha + weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha timeout = 0.15 if cuda else 0.15 + async with subtensor: - while netuid == -1 or not subtensor.is_hotkey_registered( - netuid=netuid, hotkey_ss58=wallet.hotkey.ss58_address - ): - # Wait until a solver finds a solution - try: - solution = solution_queue.get(block=True, timeout=timeout) - if solution is not None: - break - except Empty: - # No solution found, try again - pass + while netuid == -1 or not await subtensor.substrate.query( + module="SubtensorModule", + storage_function="Uids", + params=[netuid, wallet.hotkey.ss58_address], + ): + # Wait until a solver finds a solution + try: + solution = solution_queue.get(block=True, timeout=timeout) + if solution is not None: + break + except Empty: + # No solution found, try again + pass - # check for new block - old_block_number = _check_for_newest_block_and_update( - subtensor=subtensor, - netuid=netuid, - hotkey_bytes=hotkey_bytes, - old_block_number=old_block_number, - curr_diff=curr_diff, - curr_block=curr_block, - curr_block_num=curr_block_num, - curr_stats=curr_stats, - update_curr_block=_update_curr_block, - check_block=check_block, - solvers=solvers, - ) + # check for new block + old_block_number = await _check_for_newest_block_and_update( + subtensor=subtensor, + netuid=netuid, + hotkey_bytes=hotkey_bytes, + old_block_number=old_block_number, + curr_diff=curr_diff, + curr_block=curr_block, + curr_block_num=curr_block_num, + curr_stats=curr_stats, + update_curr_block=_update_curr_block, + check_block=check_block, + solvers=solvers, + ) - num_time = 0 - for finished_queue in finished_queues: - try: - proc_num = finished_queue.get(timeout=0.1) - num_time += 1 + num_time = 0 + for finished_queue in finished_queues: + try: + finished_queue.get(timeout=0.1) + num_time += 1 - except Empty: - continue + except Empty: + continue - time_now = time.time() # get current time - time_since_last = time_now - time_last # get time since last work block(s) - if num_time > 0 and time_since_last > 0.0: - # create EWMA of the hash_rate to make measure more robust + time_now = time.time() # get current time + time_since_last = time_now - time_last # get time since last work block(s) + if num_time > 0 and time_since_last > 0.0: + # create EWMA of the hash_rate to make measure more robust + if cuda: + hash_rate_ = (num_time * tpb * update_interval) / time_since_last + else: + hash_rate_ = (num_time * update_interval) / time_since_last + hash_rates.append(hash_rate_) + hash_rates.pop(0) # remove the 0th data point + curr_stats.hash_rate = sum( + [hash_rates[i] * weights[i] for i in range(n_samples)] + ) / (sum(weights)) + + # update time last to now + time_last = time_now + + curr_stats.time_average = ( + curr_stats.time_average * curr_stats.rounds_total + + curr_stats.time_spent + ) / (curr_stats.rounds_total + num_time) + curr_stats.rounds_total += num_time + + # Update stats + curr_stats.time_spent = time_since_last + new_time_spent_total = time_now - start_time_perpetual if cuda: - hash_rate_ = (num_time * tpb * update_interval) / time_since_last + curr_stats.hash_rate_perpetual = ( + curr_stats.rounds_total * (tpb * update_interval) + ) / new_time_spent_total else: - hash_rate_ = (num_time * update_interval) / time_since_last - hash_rates.append(hash_rate_) - hash_rates.pop(0) # remove the 0th data point - curr_stats.hash_rate = sum( - [hash_rates[i] * weights[i] for i in range(n_samples)] - ) / (sum(weights)) - - # update time last to now - time_last = time_now - - curr_stats.time_average = ( - curr_stats.time_average * curr_stats.rounds_total - + curr_stats.time_spent - ) / (curr_stats.rounds_total + num_time) - curr_stats.rounds_total += num_time - - # Update stats - curr_stats.time_spent = time_since_last - new_time_spent_total = time_now - start_time_perpetual - if cuda: - curr_stats.hash_rate_perpetual = ( - curr_stats.rounds_total * (tpb * update_interval) - ) / new_time_spent_total - else: - curr_stats.hash_rate_perpetual = ( - curr_stats.rounds_total * update_interval - ) / new_time_spent_total - curr_stats.time_spent_total = new_time_spent_total + curr_stats.hash_rate_perpetual = ( + curr_stats.rounds_total * update_interval + ) / new_time_spent_total + curr_stats.time_spent_total = new_time_spent_total - # Update the logger - logger.update(curr_stats, verbose=log_verbose) + # Update the logger + logger.update(curr_stats, verbose=log_verbose) # exited while, solution contains the nonce or wallet is registered stop_event.set() # stop all other processes @@ -823,17 +832,17 @@ def _block_solver( return solution -def _solve_for_difficulty_fast_cuda( - subtensor: SubtensorInterface, - wallet: Wallet, - netuid: int, - output_in_place: bool = True, - update_interval: int = 50_000, - tpb: int = 512, - dev_id: typing.Union[list[int], int] = 0, - n_samples: int = 10, - alpha_: float = 0.80, - log_verbose: bool = False, +async def _solve_for_difficulty_fast_cuda( + subtensor: SubtensorInterface, + wallet: Wallet, + netuid: int, + output_in_place: bool = True, + update_interval: int = 50_000, + tpb: int = 512, + dev_id: typing.Union[list[int], int] = 0, + n_samples: int = 10, + alpha_: float = 0.80, + log_verbose: bool = False, ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA @@ -867,7 +876,7 @@ def _solve_for_difficulty_fast_cuda( with _UsingSpawnStartMethod(force=True): curr_block, curr_block_num, curr_diff = _CUDASolver.create_shared_memory() - solution = _block_solver( + solution = await _block_solver( subtensor=subtensor, wallet=wallet, num_processes=None, @@ -888,16 +897,16 @@ def _solve_for_difficulty_fast_cuda( return solution -def _solve_for_difficulty_fast( - subtensor, - wallet: Wallet, - netuid: int, - output_in_place: bool = True, - num_processes: Optional[int] = None, - update_interval: Optional[int] = None, - n_samples: int = 10, - alpha_: float = 0.80, - log_verbose: bool = False, +async def _solve_for_difficulty_fast( + subtensor, + wallet: Wallet, + netuid: int, + output_in_place: bool = True, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, + n_samples: int = 10, + alpha_: float = 0.80, + log_verbose: bool = False, ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. @@ -927,7 +936,7 @@ def _solve_for_difficulty_fast( curr_block, curr_block_num, curr_diff = _Solver.create_shared_memory() - solution = _block_solver( + solution = await _block_solver( subtensor=subtensor, wallet=wallet, num_processes=num_processes, @@ -942,26 +951,27 @@ def _solve_for_difficulty_fast( alpha_=alpha_, output_in_place=output_in_place, log_verbose=log_verbose, - cuda=True, + cuda=False, ) return solution def _terminate_workers_and_wait_for_exit( - workers: list[typing.Union[Process, Queue]], + workers: list[typing.Union[Process, Queue_Type]], ) -> None: for worker in workers: - if isinstance(worker, Queue): + if isinstance(worker, Queue_Type): worker.join_thread() else: worker.join() worker.close() +# TODO verify this works with async @backoff.on_exception(backoff.constant, Exception, interval=1, max_tries=3) -def _get_block_with_retry( - subtensor: SubtensorInterface, netuid: int +async def _get_block_with_retry( + subtensor: SubtensorInterface, netuid: int ) -> tuple[int, int, bytes]: """ Gets the current block number, difficulty, and block hash from the substrate node. @@ -974,15 +984,19 @@ def _get_block_with_retry( :raises Exception: If the block hash is None. :raises ValueError: If the difficulty is None. """ - block_number = subtensor.get_current_block() - difficulty = 1_000_000 if netuid == -1 else subtensor.difficulty(netuid=netuid) - block_hash = subtensor.get_block_hash(block_number) - if block_hash is None: - raise Exception( - "Network error. Could not connect to substrate to get block hash" - ) - if difficulty is None: - raise ValueError("Chain error. Difficulty is None") + async with subtensor: + block_number = await subtensor.substrate.get_block_number(None) + block_hash = await subtensor.substrate.get_block_hash(block_number) # TODO check if I need to do all this + try: + difficulty = 1_000_000 if netuid == -1 else int(await subtensor.get_hyperparameter( + param_name="Difficulty", netuid=netuid, block_hash=block_hash + )) + except TypeError: + raise ValueError("Chain error. Difficulty is None") + except SubstrateRequestException: + raise Exception( + "Network error. Could not connect to substrate to get block hash" + ) return block_number, difficulty, block_hash @@ -1014,17 +1028,17 @@ def __exit__(self, *args): mp.set_start_method(self._old_start_method, force=True) -def create_pow( - subtensor, - wallet, - netuid: int, - output_in_place: bool = True, - cuda: bool = False, - dev_id: typing.Union[list[int], int] = 0, - tpb: int = 256, - num_processes: int = None, - update_interval: int = None, - log_verbose: bool = False, +async def create_pow( + subtensor, + wallet, + netuid: int, + output_in_place: bool = True, + cuda: bool = False, + dev_id: typing.Union[list[int], int] = 0, + tpb: int = 256, + num_processes: int = None, + update_interval: int = None, + log_verbose: bool = False, ) -> Optional[dict[str, typing.Any]]: """ Creates a proof of work for the given subtensor and wallet. @@ -1048,11 +1062,12 @@ def create_pow( :raises ValueError: If the subnet does not exist. """ if netuid != -1: - if not subtensor.subnet_exists(netuid=netuid): - raise ValueError(f"Subnet {netuid} does not exist") + async with subtensor: + if not await subtensor.subnet_exists(netuid=netuid): + raise ValueError(f"Subnet {netuid} does not exist") if cuda: - solution: Optional[POWSolution] = _solve_for_difficulty_fast_cuda( + solution: Optional[POWSolution] = await _solve_for_difficulty_fast_cuda( subtensor, wallet, netuid=netuid, @@ -1063,7 +1078,7 @@ def create_pow( log_verbose=log_verbose, ) else: - solution: Optional[POWSolution] = _solve_for_difficulty_fast( + solution: Optional[POWSolution] = await _solve_for_difficulty_fast( subtensor, wallet, netuid=netuid, @@ -1077,14 +1092,14 @@ def create_pow( def _solve_for_nonce_block_cuda( - nonce_start: int, - update_interval: int, - block_and_hotkey_hash_bytes: bytes, - difficulty: int, - limit: int, - block_number: int, - dev_id: int, - tpb: int, + nonce_start: int, + update_interval: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + block_number: int, + dev_id: int, + tpb: int, ) -> Optional[POWSolution]: """ Tries to solve the POW on a CUDA device for a block of nonces (nonce_start, nonce_start + update_interval * tpb @@ -1107,12 +1122,12 @@ def _solve_for_nonce_block_cuda( def _solve_for_nonce_block( - nonce_start: int, - nonce_end: int, - block_and_hotkey_hash_bytes: bytes, - difficulty: int, - limit: int, - block_number: int, + nonce_start: int, + nonce_end: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + block_number: int, ) -> Optional[POWSolution]: """ Tries to solve the POW for a block of nonces (nonce_start, nonce_end) @@ -1134,7 +1149,7 @@ class CUDAException(Exception): def _hex_bytes_to_u8_list(hex_bytes: bytes): - hex_chunks = [int(hex_bytes[i : i + 2], 16) for i in range(0, len(hex_bytes), 2)] + hex_chunks = [int(hex_bytes[i: i + 2], 16) for i in range(0, len(hex_bytes), 2)] return hex_chunks @@ -1162,14 +1177,14 @@ def _hash_block_with_hotkey(block_bytes: bytes, hotkey_bytes: bytes) -> bytes: def _update_curr_block( - curr_diff: Array, - curr_block: Array, - curr_block_num: Value, - block_number: int, - block_bytes: bytes, - diff: int, - hotkey_bytes: bytes, - lock: Lock, + curr_diff: Array, + curr_block: Array, + curr_block_num: Value, + block_number: int, + block_bytes: bytes, + diff: int, + hotkey_bytes: bytes, + lock: Lock, ): with lock: curr_block_num.value = block_number @@ -1204,13 +1219,13 @@ class RegistrationStatistics: def solve_cuda( - nonce_start: np.int64, - update_interval: np.int64, - tpb: int, - block_and_hotkey_hash_bytes: bytes, - difficulty: int, - limit: int, - dev_id: int = 0, + nonce_start: np.int64, + update_interval: np.int64, + tpb: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + dev_id: int = 0, ) -> tuple[np.int64, bytes]: """ Solves the PoW problem using CUDA. @@ -1235,27 +1250,6 @@ def solve_cuda( upper_bytes = upper.to_bytes(32, byteorder="little", signed=False) - def _hex_bytes_to_u8_list(hex_bytes: bytes): - hex_chunks = [ - int(hex_bytes[i : i + 2], 16) for i in range(0, len(hex_bytes), 2) - ] - return hex_chunks - - def _create_seal_hash(block_and_hotkey_hash_hex: bytes, nonce: int) -> bytes: - nonce_bytes = binascii.hexlify(nonce.to_bytes(8, "little")) - pre_seal = nonce_bytes + block_and_hotkey_hash_hex - seal_sh256 = hashlib.sha256(bytearray(_hex_bytes_to_u8_list(pre_seal))).digest() - kec = keccak.new(digest_bits=256) - seal = kec.update(seal_sh256).digest() - return seal - - def _seal_meets_difficulty(seal: bytes, difficulty: int): - seal_number = int.from_bytes(seal, "big") - product = seal_number * difficulty - limit = int(math.pow(2, 256)) - 1 - - return product < limit - # Call cython function # int blockSize, uint64 nonce_start, uint64 update_interval, const unsigned char[:] limit, # const unsigned char[:] block_bytes, int dev_id @@ -1272,7 +1266,7 @@ def _seal_meets_difficulty(seal: bytes, difficulty: int): seal = None if solution != -1: seal = _create_seal_hash(block_and_hotkey_hash_hex, solution) - if _seal_meets_difficulty(seal, difficulty): + if _seal_meets_difficulty(seal, difficulty, limit): return solution, seal else: return -1, b"\x00" * 32 diff --git a/src/wallets.py b/src/wallets.py index 755dd58f..e30c25ab 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -52,6 +52,7 @@ from src.subtensor_interface import SubtensorInterface from src.bittensor.balances import Balance from src.bittensor.extrinsics.transfer import transfer_extrinsic +from src.bittensor.extrinsics.registration import run_faucet_extrinsic async def regen_coldkey( @@ -278,6 +279,7 @@ async def wallet_balance( str(total_free_balance + total_staked_balance), ) console.print(table) + await subtensor.substrate.close() async def get_wallet_transfers(wallet_address: str) -> list[dict]: @@ -1237,17 +1239,17 @@ async def faucet( dev_id: int, output_in_place: bool, log_verbose: bool, + max_successes: int = 3 ): - success = await subtensor.run_faucet( - wallet=wallet, - prompt=True, - tpb=threads_per_block, + success = await run_faucet_extrinsic( + subtensor, wallet, tpb=threads_per_block, prompt=True, update_interval=update_interval, num_processes=processes, cuda=use_cuda, dev_id=dev_id, output_in_place=output_in_place, log_verbose=log_verbose, + max_successes=max_successes ) if not success: err_console.print("Faucet run failed.") From c52d144deda07536365f27c33d0b5b64acf46a9b Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 22:38:32 +0200 Subject: [PATCH 41/48] Ruff --- cli.py | 34 ++- src/bittensor/extrinsics/registration.py | 363 ++++++++++++----------- src/wallets.py | 9 +- 3 files changed, 223 insertions(+), 183 deletions(-) diff --git a/cli.py b/cli.py index bb393291..286e75f3 100755 --- a/cli.py +++ b/cli.py @@ -117,17 +117,29 @@ def __init__(self): self.delegates_app = typer.Typer() # config alias - self.app.add_typer(self.config_app, name="config", short_help="Config commands, aliases: `c`, `conf`") + self.app.add_typer( + self.config_app, + name="config", + short_help="Config commands, aliases: `c`, `conf`", + ) self.app.add_typer(self.config_app, name="conf", hidden=True) self.app.add_typer(self.config_app, name="c", hidden=True) # wallet aliases - self.app.add_typer(self.wallet_app, name="wallet", short_help="Wallet commands, aliases: `wallets`, `w`") + self.app.add_typer( + self.wallet_app, + name="wallet", + short_help="Wallet commands, aliases: `wallets`, `w`", + ) self.app.add_typer(self.wallet_app, name="w", hidden=True) self.app.add_typer(self.wallet_app, name="wallets", hidden=True) # delegates aliases - self.app.add_typer(self.delegates_app, name="delegates", short_help="Delegate commands, alias: `d`") + self.app.add_typer( + self.delegates_app, + name="delegates", + short_help="Delegate commands, alias: `d`", + ) self.app.add_typer(self.delegates_app, name="d", hidden=True) # config commands @@ -624,8 +636,20 @@ def wallet_faucet( """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) self.initialize_chain(network, chain) - self._run_command(wallets.faucet(wallet, self.not_subtensor, threads_per_block, update_interval, processors, - use_cuda, dev_id, output_in_place, verbose, max_successes)) + self._run_command( + wallets.faucet( + wallet, + self.not_subtensor, + threads_per_block, + update_interval, + processors, + use_cuda, + dev_id, + output_in_place, + verbose, + max_successes, + ) + ) def wallet_regen_coldkey( self, diff --git a/src/bittensor/extrinsics/registration.py b/src/bittensor/extrinsics/registration.py index 905a8428..7bd1f5f1 100644 --- a/src/bittensor/extrinsics/registration.py +++ b/src/bittensor/extrinsics/registration.py @@ -145,22 +145,22 @@ def stop(self) -> None: @classmethod def get_status_message( - cls, stats: RegistrationStatistics, verbose: bool = False + cls, stats: RegistrationStatistics, verbose: bool = False ) -> str: message = ( - "Solving\n" - + f"Time Spent (total): [bold white]{timedelta(seconds=stats.time_spent_total)}[/bold white]\n" - + ( - f"Time Spent This Round: {timedelta(seconds=stats.time_spent)}\n" - + f"Time Spent Average: {timedelta(seconds=stats.time_average)}\n" - if verbose - else "" - ) - + f"Registration Difficulty: [bold white]{millify(stats.difficulty)}[/bold white]\n" - + f"Iters (Inst/Perp): [bold white]{get_human_readable(stats.hash_rate, 'H')}/s / " - + f"{get_human_readable(stats.hash_rate_perpetual, 'H')}/s[/bold white]\n" - + f"Block Number: [bold white]{stats.block_number}[/bold white]\n" - + f"Block Hash: [bold white]{stats.block_hash.encode('utf-8')}[/bold white]\n" + "Solving\n" + + f"Time Spent (total): [bold white]{timedelta(seconds=stats.time_spent_total)}[/bold white]\n" + + ( + f"Time Spent This Round: {timedelta(seconds=stats.time_spent)}\n" + + f"Time Spent Average: {timedelta(seconds=stats.time_average)}\n" + if verbose + else "" + ) + + f"Registration Difficulty: [bold white]{millify(stats.difficulty)}[/bold white]\n" + + f"Iters (Inst/Perp): [bold white]{get_human_readable(stats.hash_rate, 'H')}/s / " + + f"{get_human_readable(stats.hash_rate_perpetual, 'H')}/s[/bold white]\n" + + f"Block Number: [bold white]{stats.block_number}[/bold white]\n" + + f"Block Hash: [bold white]{stats.block_hash.encode('utf-8')}[/bold white]\n" ) return message @@ -179,28 +179,30 @@ class _SolverBase(Process): :param num_proc: The total number of processes running. :param update_interval: The number of nonces to try to solve before checking for a new block. :param finished_queue: The queue to put the process number when a process finishes each update_interval. - Used for calculating the average time per update_interval across all processes. + Used for calculating the average time per update_interval across all processes. :param solution_queue: The queue to put the solution the process has found during the pow solve. :param stop_event: The event to set by the main process when all the solver processes should stop. - The solver process will check for the event after each update_interval. - The solver process will stop when the event is set. - Used to stop the solver processes when a solution is found. + The solver process will check for the event after each update_interval. + The solver process will stop when the event is set. + Used to stop the solver processes when a solution is found. :param curr_block: The array containing this process's current block hash. - The main process will set the array to the new block hash when a new block is finalized in the network. - The solver process will get the new block hash from this array when newBlockEvent is set. + The main process will set the array to the new block hash when a new block is finalized in the + network. The solver process will get the new block hash from this array when newBlockEvent is set :param curr_block_num: The value containing this process's current block number. - The main process will set the value to the new block number when a new block is finalized in the network. - The solver process will get the new block number from this value when newBlockEvent is set. - :param curr_diff: The array containing this process's current difficulty. - The main process will set the array to the new difficulty when a new block is finalized in the network. - The solver process will get the new difficulty from this array when newBlockEvent is set. + The main process will set the value to the new block number when a new block is finalized in + the network. The solver process will get the new block number from this value when + new_block_event is set. + :param curr_diff: The array containing this process's current difficulty. The main process will set the array to + the new difficulty when a new block is finalized in the network. The solver process will get the + new difficulty from this array when newBlockEvent is set. :param check_block: The lock to prevent this process from getting the new block data while the main process is updating the data. :param limit: The limit of the pow solve for a valid solution. :var new_block_event: The event to set by the main process when a new block is finalized in the network. - The solver process will check for the event after each update_interval. - The solver process will get the new block hash and difficulty and start solving for a new nonce. + The solver process will check for the event after each update_interval. + The solver process will get the new block hash and difficulty and start solving for a new + nonce. """ proc_num: int @@ -218,18 +220,18 @@ class _SolverBase(Process): limit: int def __init__( - self, - proc_num, - num_proc, - update_interval, - finished_queue, - solution_queue, - stop_event, - curr_block, - curr_block_num, - curr_diff, - check_block, - limit, + self, + proc_num, + num_proc, + update_interval, + finished_queue, + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, ): Process.__init__(self, daemon=True) self.proc_num = proc_num @@ -306,20 +308,20 @@ class _CUDASolver(_SolverBase): tpb: int def __init__( - self, - proc_num, - num_proc, - update_interval, - finished_queue, - solution_queue, - stop_event, - curr_block, - curr_block_num, - curr_diff, - check_block, - limit, - dev_id: int, - tpb: int, + self, + proc_num, + num_proc, + update_interval, + finished_queue, + solution_queue, + stop_event, + curr_block, + curr_block_num, + curr_diff, + check_block, + limit, + dev_id: int, + tpb: int, ): super().__init__( proc_num, @@ -407,20 +409,20 @@ class MaxAttemptsException(Exception): async def run_faucet_extrinsic( - subtensor: SubtensorInterface, - wallet: Wallet, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - prompt: bool = False, - max_allowed_attempts: int = 3, - output_in_place: bool = True, - cuda: bool = False, - dev_id: int = 0, - tpb: int = 256, - num_processes: Optional[int] = None, - update_interval: Optional[int] = None, - log_verbose: bool = False, - max_successes: int = 3, + subtensor: SubtensorInterface, + wallet: Wallet, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, + prompt: bool = False, + max_allowed_attempts: int = 3, + output_in_place: bool = True, + cuda: bool = False, + dev_id: int = 0, + tpb: int = 256, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, + log_verbose: bool = False, + max_successes: int = 3, ) -> tuple[bool, str]: r"""Runs a continual POW to get a faucet of TAO on the test net. @@ -446,9 +448,9 @@ async def run_faucet_extrinsic( """ if prompt: if not Confirm.ask( - "Run Faucet ?\n" - f" coldkey: [bold white]{wallet.coldkeypub.ss58_address}[/bold white]\n" - f" network: [bold white]{subtensor}[/bold white]" + "Run Faucet ?\n" + f" coldkey: [bold white]{wallet.coldkeypub.ss58_address}[/bold white]\n" + f" network: [bold white]{subtensor}[/bold white]" ): return False, "" @@ -470,7 +472,9 @@ async def run_faucet_extrinsic( async with subtensor: try: pow_result = None - while pow_result is None or await pow_result.is_stale(subtensor=subtensor): + while pow_result is None or await pow_result.is_stale( + subtensor=subtensor + ): # Solve latest POW. if cuda: if not torch.cuda.is_available(): @@ -532,7 +536,9 @@ async def run_faucet_extrinsic( # Successful registration else: - new_balance = await subtensor.get_balance(wallet.coldkeypub.ss58_address) + new_balance = await subtensor.get_balance( + wallet.coldkeypub.ss58_address + ) console.print( f"Balance: [blue]{old_balance[wallet.coldkeypub.ss58_address]}[/blue] :arrow_right:" f" [green]{new_balance[wallet.coldkeypub.ss58_address]}[/green]" @@ -556,17 +562,17 @@ async def run_faucet_extrinsic( async def _check_for_newest_block_and_update( - subtensor: SubtensorInterface, - netuid: int, - old_block_number: int, - hotkey_bytes: bytes, - curr_diff: Array, - curr_block: Array, - curr_block_num: Value, - update_curr_block: typing.Callable, - check_block: Lock, - solvers: list[_Solver], - curr_stats: RegistrationStatistics, + subtensor: SubtensorInterface, + netuid: int, + old_block_number: int, + hotkey_bytes: bytes, + curr_diff: Array, + curr_block: Array, + curr_block_num: Value, + update_curr_block: typing.Callable, + check_block: Lock, + solvers: list[_Solver], + curr_stats: RegistrationStatistics, ) -> int: """ Checks for a new block and updates the current block information if a new block is found. @@ -619,21 +625,21 @@ async def _check_for_newest_block_and_update( async def _block_solver( - subtensor: SubtensorInterface, - wallet: Wallet, - num_processes: int, - netuid: int, - dev_id: list[int], - tpb: int, - update_interval: int, - curr_block, - curr_block_num, - curr_diff, - n_samples, - alpha_, - output_in_place, - log_verbose, - cuda: bool, + subtensor: SubtensorInterface, + wallet: Wallet, + num_processes: int, + netuid: int, + dev_id: list[int], + tpb: int, + update_interval: int, + curr_block, + curr_block_num, + curr_diff, + n_samples, + alpha_, + output_in_place, + log_verbose, + cuda: bool, ): limit = int(math.pow(2, 256)) - 1 @@ -739,15 +745,14 @@ async def _block_solver( solution = None hash_rates = [0] * n_samples # The last n true hash_rates - weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha + weights = [alpha_**i for i in range(n_samples)] # weights decay by alpha timeout = 0.15 if cuda else 0.15 async with subtensor: - while netuid == -1 or not await subtensor.substrate.query( - module="SubtensorModule", - storage_function="Uids", - params=[netuid, wallet.hotkey.ss58_address], + module="SubtensorModule", + storage_function="Uids", + params=[netuid, wallet.hotkey.ss58_address], ): # Wait until a solver finds a solution try: @@ -801,9 +806,9 @@ async def _block_solver( time_last = time_now curr_stats.time_average = ( - curr_stats.time_average * curr_stats.rounds_total - + curr_stats.time_spent - ) / (curr_stats.rounds_total + num_time) + curr_stats.time_average * curr_stats.rounds_total + + curr_stats.time_spent + ) / (curr_stats.rounds_total + num_time) curr_stats.rounds_total += num_time # Update stats @@ -811,12 +816,12 @@ async def _block_solver( new_time_spent_total = time_now - start_time_perpetual if cuda: curr_stats.hash_rate_perpetual = ( - curr_stats.rounds_total * (tpb * update_interval) - ) / new_time_spent_total + curr_stats.rounds_total * (tpb * update_interval) + ) / new_time_spent_total else: curr_stats.hash_rate_perpetual = ( - curr_stats.rounds_total * update_interval - ) / new_time_spent_total + curr_stats.rounds_total * update_interval + ) / new_time_spent_total curr_stats.time_spent_total = new_time_spent_total # Update the logger @@ -833,16 +838,16 @@ async def _block_solver( async def _solve_for_difficulty_fast_cuda( - subtensor: SubtensorInterface, - wallet: Wallet, - netuid: int, - output_in_place: bool = True, - update_interval: int = 50_000, - tpb: int = 512, - dev_id: typing.Union[list[int], int] = 0, - n_samples: int = 10, - alpha_: float = 0.80, - log_verbose: bool = False, + subtensor: SubtensorInterface, + wallet: Wallet, + netuid: int, + output_in_place: bool = True, + update_interval: int = 50_000, + tpb: int = 512, + dev_id: typing.Union[list[int], int] = 0, + n_samples: int = 10, + alpha_: float = 0.80, + log_verbose: bool = False, ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA @@ -898,15 +903,15 @@ async def _solve_for_difficulty_fast_cuda( async def _solve_for_difficulty_fast( - subtensor, - wallet: Wallet, - netuid: int, - output_in_place: bool = True, - num_processes: Optional[int] = None, - update_interval: Optional[int] = None, - n_samples: int = 10, - alpha_: float = 0.80, - log_verbose: bool = False, + subtensor, + wallet: Wallet, + netuid: int, + output_in_place: bool = True, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, + n_samples: int = 10, + alpha_: float = 0.80, + log_verbose: bool = False, ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. @@ -958,7 +963,7 @@ async def _solve_for_difficulty_fast( def _terminate_workers_and_wait_for_exit( - workers: list[typing.Union[Process, Queue_Type]], + workers: list[typing.Union[Process, Queue_Type]], ) -> None: for worker in workers: if isinstance(worker, Queue_Type): @@ -971,7 +976,7 @@ def _terminate_workers_and_wait_for_exit( # TODO verify this works with async @backoff.on_exception(backoff.constant, Exception, interval=1, max_tries=3) async def _get_block_with_retry( - subtensor: SubtensorInterface, netuid: int + subtensor: SubtensorInterface, netuid: int ) -> tuple[int, int, bytes]: """ Gets the current block number, difficulty, and block hash from the substrate node. @@ -986,11 +991,19 @@ async def _get_block_with_retry( """ async with subtensor: block_number = await subtensor.substrate.get_block_number(None) - block_hash = await subtensor.substrate.get_block_hash(block_number) # TODO check if I need to do all this + block_hash = await subtensor.substrate.get_block_hash( + block_number + ) # TODO check if I need to do all this try: - difficulty = 1_000_000 if netuid == -1 else int(await subtensor.get_hyperparameter( - param_name="Difficulty", netuid=netuid, block_hash=block_hash - )) + difficulty = ( + 1_000_000 + if netuid == -1 + else int( + await subtensor.get_hyperparameter( + param_name="Difficulty", netuid=netuid, block_hash=block_hash + ) + ) + ) except TypeError: raise ValueError("Chain error. Difficulty is None") except SubstrateRequestException: @@ -1029,16 +1042,16 @@ def __exit__(self, *args): async def create_pow( - subtensor, - wallet, - netuid: int, - output_in_place: bool = True, - cuda: bool = False, - dev_id: typing.Union[list[int], int] = 0, - tpb: int = 256, - num_processes: int = None, - update_interval: int = None, - log_verbose: bool = False, + subtensor, + wallet, + netuid: int, + output_in_place: bool = True, + cuda: bool = False, + dev_id: typing.Union[list[int], int] = 0, + tpb: int = 256, + num_processes: int = None, + update_interval: int = None, + log_verbose: bool = False, ) -> Optional[dict[str, typing.Any]]: """ Creates a proof of work for the given subtensor and wallet. @@ -1092,14 +1105,14 @@ async def create_pow( def _solve_for_nonce_block_cuda( - nonce_start: int, - update_interval: int, - block_and_hotkey_hash_bytes: bytes, - difficulty: int, - limit: int, - block_number: int, - dev_id: int, - tpb: int, + nonce_start: int, + update_interval: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + block_number: int, + dev_id: int, + tpb: int, ) -> Optional[POWSolution]: """ Tries to solve the POW on a CUDA device for a block of nonces (nonce_start, nonce_start + update_interval * tpb @@ -1122,12 +1135,12 @@ def _solve_for_nonce_block_cuda( def _solve_for_nonce_block( - nonce_start: int, - nonce_end: int, - block_and_hotkey_hash_bytes: bytes, - difficulty: int, - limit: int, - block_number: int, + nonce_start: int, + nonce_end: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + block_number: int, ) -> Optional[POWSolution]: """ Tries to solve the POW for a block of nonces (nonce_start, nonce_end) @@ -1149,7 +1162,7 @@ class CUDAException(Exception): def _hex_bytes_to_u8_list(hex_bytes: bytes): - hex_chunks = [int(hex_bytes[i: i + 2], 16) for i in range(0, len(hex_bytes), 2)] + hex_chunks = [int(hex_bytes[i : i + 2], 16) for i in range(0, len(hex_bytes), 2)] return hex_chunks @@ -1177,14 +1190,14 @@ def _hash_block_with_hotkey(block_bytes: bytes, hotkey_bytes: bytes) -> bytes: def _update_curr_block( - curr_diff: Array, - curr_block: Array, - curr_block_num: Value, - block_number: int, - block_bytes: bytes, - diff: int, - hotkey_bytes: bytes, - lock: Lock, + curr_diff: Array, + curr_block: Array, + curr_block_num: Value, + block_number: int, + block_bytes: bytes, + diff: int, + hotkey_bytes: bytes, + lock: Lock, ): with lock: curr_block_num.value = block_number @@ -1219,13 +1232,13 @@ class RegistrationStatistics: def solve_cuda( - nonce_start: np.int64, - update_interval: np.int64, - tpb: int, - block_and_hotkey_hash_bytes: bytes, - difficulty: int, - limit: int, - dev_id: int = 0, + nonce_start: np.int64, + update_interval: np.int64, + tpb: int, + block_and_hotkey_hash_bytes: bytes, + difficulty: int, + limit: int, + dev_id: int = 0, ) -> tuple[np.int64, bytes]: """ Solves the PoW problem using CUDA. diff --git a/src/wallets.py b/src/wallets.py index e30c25ab..95cffbdb 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1239,17 +1239,20 @@ async def faucet( dev_id: int, output_in_place: bool, log_verbose: bool, - max_successes: int = 3 + max_successes: int = 3, ): success = await run_faucet_extrinsic( - subtensor, wallet, tpb=threads_per_block, prompt=True, + subtensor, + wallet, + tpb=threads_per_block, + prompt=True, update_interval=update_interval, num_processes=processes, cuda=use_cuda, dev_id=dev_id, output_in_place=output_in_place, log_verbose=log_verbose, - max_successes=max_successes + max_successes=max_successes, ) if not success: err_console.print("Faucet run failed.") From df138e9b9e974c3ba17f1f82cd262c9df90812db Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 22:50:05 +0200 Subject: [PATCH 42/48] Config loading of wallets --- cli.py | 8 +++++--- src/bittensor/extrinsics/registration.py | 3 ++- src/wallets.py | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cli.py b/cli.py index 286e75f3..1dbe77eb 100755 --- a/cli.py +++ b/cli.py @@ -240,15 +240,17 @@ def get_config(self): table.add_row(*[k, v]) console.print(table) - @staticmethod def wallet_ask( + self, wallet_name: str, wallet_path: str, wallet_hotkey: str, - config=None, validate=True, ): - # TODO Wallet(config) + wallet_name = wallet_name or self.config.get("wallet_name") + wallet_path = wallet_path or self.config.get("wallet_path") + wallet_hotkey = wallet_hotkey or self.config.get("wallet_hotkey") + if not any([wallet_name, wallet_path, wallet_hotkey]): wallet_name = typer.prompt("Enter wallet name") wallet = Wallet(name=wallet_name) diff --git a/src/bittensor/extrinsics/registration.py b/src/bittensor/extrinsics/registration.py index 7bd1f5f1..4d46e9d2 100644 --- a/src/bittensor/extrinsics/registration.py +++ b/src/bittensor/extrinsics/registration.py @@ -448,7 +448,8 @@ async def run_faucet_extrinsic( """ if prompt: if not Confirm.ask( - "Run Faucet ?\n" + "Run Faucet?\n" + f" wallet name: [bold white]{wallet.name}[/bold white]\n" f" coldkey: [bold white]{wallet.coldkeypub.ss58_address}[/bold white]\n" f" network: [bold white]{subtensor}[/bold white]" ): diff --git a/src/wallets.py b/src/wallets.py index 95cffbdb..1cbf89a2 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1256,3 +1256,5 @@ async def faucet( ) if not success: err_console.print("Faucet run failed.") + + await subtensor.substrate.close() From e09bf6c5c9b1f26229cd308bf0a991ee3c2d31e2 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 1 Aug 2024 23:17:26 +0200 Subject: [PATCH 43/48] Swap hotkey added. --- cli.py | 34 ++++++++++++- src/bittensor/extrinsics/registration.py | 61 ++++++++++++++++++++++++ src/wallets.py | 20 +++++++- 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/cli.py b/cli.py index 1dbe77eb..2506a41d 100755 --- a/cli.py +++ b/cli.py @@ -478,6 +478,36 @@ def wallet_transfer( wallets.transfer(wallet, self.not_subtensor, destination, amount) ) + def wallet_swap_hotkey( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + destination_hotkey_name: Optional[str] = typer.Argument( + help="Destination hotkey name." + ), + ): + """ + # wallet swap-hotkey + Executes the `swap_hotkey` command to swap the hotkeys for a neuron on the network. + + ## Usage: + The command is used to swap the hotkey of a wallet for another hotkey on that same wallet. + + ### Example usage: + ``` + btcli wallet swap_hotkey new_hotkey --wallet-name your_wallet_name --wallet-hotkey original_hotkey + ``` + """ + original_wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + new_wallet = self.wallet_ask(wallet_name, wallet_path, destination_hotkey_name) + self.initialize_chain(network, chain) + return self._run_command( + wallets.swap_hotkey(original_wallet, new_wallet, self.not_subtensor) + ) + def wallet_inspect( self, all_wallets: bool = typer.Option( @@ -550,7 +580,7 @@ def wallet_inspect( ) wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) self.initialize_chain(network, chain) - self._run_command( + return self._run_command( wallets.inspect( wallet, self.not_subtensor, @@ -638,7 +668,7 @@ def wallet_faucet( """ wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) self.initialize_chain(network, chain) - self._run_command( + return self._run_command( wallets.faucet( wallet, self.not_subtensor, diff --git a/src/bittensor/extrinsics/registration.py b/src/bittensor/extrinsics/registration.py index 4d46e9d2..0cb54e23 100644 --- a/src/bittensor/extrinsics/registration.py +++ b/src/bittensor/extrinsics/registration.py @@ -1316,3 +1316,64 @@ def log_cuda_errors() -> str: s = f.getvalue() return s + + +async def swap_hotkey_extrinsic( + subtensor: SubtensorInterface, + wallet: Wallet, + new_wallet: Wallet, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, + prompt: bool = False, +) -> bool: + async def _do_swap_hotkey(): + async with subtensor: + call = await subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="swap_hotkey", + call_params={ + "hotkey": wallet.hotkey.ss58_address, + "new_hotkey": new_wallet.hotkey.ss58_address, + }, + ) + extrinsic = await subtensor.substrate.create_signed_extrinsic( + call=call, keypair=wallet.coldkey + ) + response = await subtensor.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, None + + # process if registration successful, try again if pow is still valid + response.process_events() + if not response.is_success: + return False, format_error_message(response.error_message) + # Successful registration + else: + return True, None + + wallet.unlock_coldkey() # unlock coldkey + if prompt: + # Prompt user for confirmation. + if not Confirm.ask( + f"Swap {wallet.hotkey} for new hotkey: {new_wallet.hotkey}?" + ): + return False + + with console.status(":satellite: Swapping hotkeys..."): + success, err_msg = await _do_swap_hotkey() + + if success: + console.print( + f"Hotkey {wallet.hotkey} swapped for new hotkey: {new_wallet.hotkey}" + ) + return True + else: + err_console.print(f":cross_mark: [red]Failed[/red]: {err_msg}") + time.sleep(0.5) + return False diff --git a/src/wallets.py b/src/wallets.py index 1cbf89a2..5f53274a 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -52,7 +52,10 @@ from src.subtensor_interface import SubtensorInterface from src.bittensor.balances import Balance from src.bittensor.extrinsics.transfer import transfer_extrinsic -from src.bittensor.extrinsics.registration import run_faucet_extrinsic +from src.bittensor.extrinsics.registration import ( + run_faucet_extrinsic, + swap_hotkey_extrinsic, +) async def regen_coldkey( @@ -1258,3 +1261,18 @@ async def faucet( err_console.print("Faucet run failed.") await subtensor.substrate.close() + + +async def swap_hotkey( + original_wallet: Wallet, new_wallet: Wallet, subtensor: SubtensorInterface +): + """Swap your hotkey for all registered axons on the network.""" + + return await swap_hotkey_extrinsic( + subtensor, + original_wallet, + new_wallet, + wait_for_finalization=False, + wait_for_inclusion=True, + prompt=True, + ) From 4fb00cd88da2af4ed447b38142380eecf3858164 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 2 Aug 2024 15:28:57 +0200 Subject: [PATCH 44/48] Added get-identity and set-identity --- cli.py | 172 +++++++++++++++++++++++++++++++++++++ src/subtensor_interface.py | 49 ++++++++++- src/wallets.py | 114 +++++++++++++++++++++++- 3 files changed, 330 insertions(+), 5 deletions(-) diff --git a/cli.py b/cli.py index 2506a41d..005d6741 100755 --- a/cli.py +++ b/cli.py @@ -160,6 +160,8 @@ def __init__(self): self.wallet_app.command("transfer")(self.wallet_transfer) self.wallet_app.command("inspect")(self.wallet_inspect) self.wallet_app.command("faucet")(self.wallet_faucet) + self.wallet_app.command("set-identity")(self.wallet_set_id) + self.wallet_app.command("get-identity")(self.wallet_get_id) # delegates commands self.delegates_app.command("list")(self.delegates_list) @@ -1024,6 +1026,176 @@ def wallet_history( wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) return self._run_command(wallets.wallet_history(wallet)) + def wallet_set_id( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + display_name: Optional[str] = typer.Option( + "", + "--display-name", + "--display", + help="The display name for the identity.", + prompt=True, + ), + legal_name: Optional[str] = typer.Option( + "", + "--legal-name", + "--legal", + help="The legal name for the identity.", + prompt=True, + ), + web_url: Optional[str] = typer.Option( + "", "--web-url", "--web", help="The web url for the identity.", prompt=True + ), + riot_handle: Optional[str] = typer.Option( + "", + "--riot-handle", + "--riot", + help="The riot handle for the identity.", + prompt=True, + ), + email: Optional[str] = typer.Option( + "", help="The email address for the identity.", prompt=True + ), + pgp_fingerprint: Optional[str] = typer.Option( + "", + "--pgp-fingerprint", + "--pgp", + help="The pgp fingerprint for the identity.", + prompt=True, + ), + image_url: Optional[str] = typer.Option( + "", + "--image-url", + "--image", + help="The image url for the identity.", + prompt=True, + ), + info_: Optional[str] = typer.Option( + "", "--info", "-i", help="The info for the identity.", prompt=True + ), + twitter_url: Optional[str] = typer.Option( + "", + "-x", + "-𝕏", + "--twitter-url", + "--twitter", + help="The 𝕏 (Twitter) url for the identity.", + prompt=True, + ), + validator_id: Optional[bool] = typer.Option( + "--validator/--not-validator", + help="Are you updating a validator hotkey identity?", + prompt=True, + ), + ): + """ + # wallet set-identity + Executes the `set-identity` command within the Bittensor network, which allows for the creation or update of a + delegate's on-chain identity. + + This identity includes various attributes such as display name, legal name, web URL, PGP fingerprint, and + contact information, among others. + + The command prompts the user for the different identity attributes and validates the + input size for each attribute. It provides an option to update an existing validator + hotkey identity. If the user consents to the transaction cost, the identity is updated + on the blockchain. + + Each field has a maximum size of 64 bytes. The PGP fingerprint field is an exception + and has a maximum size of 20 bytes. The user is prompted to enter the PGP fingerprint + as a hex string, which is then converted to bytes. The user is also prompted to enter + the coldkey or hotkey ``ss58`` address for the identity to be updated. If the user does + not have a hotkey, the coldkey address is used by default. + + If setting a validator identity, the hotkey will be used by default. If the user is + setting an identity for a subnet, the coldkey will be used by default. + + ## Usage: + The user should call this command from the command line and follow the interactive + prompts to enter or update the identity information. The command will display the + updated identity details in a table format upon successful execution. + + ### Example usage: + ``` + btcli wallet set_identity + ``` + + #### Note: + This command should only be used if the user is willing to incur the 1 TAO transaction + fee associated with setting an identity on the blockchain. It is a high-level command + that makes changes to the blockchain state and should not be used programmatically as + part of other scripts or applications. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + self.initialize_chain(network, chain) + return self._run_command( + wallets.set_id( + wallet, + self.not_subtensor, + display_name, + legal_name, + web_url, + pgp_fingerprint, + riot_handle, + email, + image_url, + twitter_url, + info_, + validator_id, + ) + ) + + def wallet_get_id( + self, + key: str = typer.Option( + None, + "--key", + "-k", + "--ss58", + help="The coldkey or hotkey ss58 address to query.", + prompt=True, + ), + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + ): + """ + # wallet get-id + Executes the `get-identity` command, which retrieves and displays the identity details of a user's coldkey or + hotkey associated with the Bittensor network. This function queries the subtensor chain for information such as + the stake, rank, and trust associated with the provided key. + + The command performs the following actions: + + - Connects to the subtensor network and retrieves the identity information. + + - Displays the information in a structured table format. + + The displayed table includes: + + - **Address**: The ``ss58`` address of the queried key. + + - **Item**: Various attributes of the identity such as stake, rank, and trust. + + - **Value**: The corresponding values of the attributes. + + ## Usage: + The user must provide an ss58 address as input to the command. If the address is not + provided in the configuration, the user is prompted to enter one. + + ### Example usage: + ``` + btcli wallet get_identity --key + ``` + + #### Note: + This function is designed for CLI use and should be executed in a terminal. It is + primarily used for informational purposes and has no side effects on the network state. + """ + def delegates_list( self, wallet_name: Optional[str] = typer.Option(None, help="Wallet name"), diff --git a/src/subtensor_interface.py b/src/subtensor_interface.py index 4f193c23..ddcc92e2 100644 --- a/src/subtensor_interface.py +++ b/src/subtensor_interface.py @@ -361,7 +361,7 @@ async def filter_netuids_by_registered_hotkeys( all_netuids, filter_for_netuids, all_hotkeys, - block_hash: str, + block_hash: Optional[str] = None, reuse_block: bool = False, ) -> list[int]: netuids_with_registered_hotkeys = [ @@ -490,3 +490,50 @@ async def get_delegated( return [] return DelegateInfo.delegated_list_from_vec_u8(result) + + async def query_identity( + self, + key: str, + block_hash: Optional[str] = None, + reuse_block: bool = False, + ) -> dict: + """ + Queries the identity of a neuron on the Bittensor blockchain using the given key. This function retrieves + detailed identity information about a specific neuron, which is a crucial aspect of the network's decentralized + identity and governance system. + + Note: + See the `Bittensor CLI documentation `_ for supported identity + parameters. + + :param key: The key used to query the neuron's identity, typically the neuron's SS58 address. + :param block_hash: The hash of the blockchain block number at which to perform the query. + :param reuse_block: Whether to reuse the last-used blockchain block hash. + + :return: An object containing the identity information of the neuron if found, ``None`` otherwise. + + The identity information can include various attributes such as the neuron's stake, rank, and other + network-specific details, providing insights into the neuron's role and status within the Bittensor network. + """ + + def decode_hex_identity_dict(info_dictionary): + for k, v in info_dictionary.items(): + if isinstance(v, dict): + item = list(v.values())[0] + if isinstance(item, str) and item.startswith("0x"): + try: + info_dictionary[k] = bytes.fromhex(item[2:]).decode() + except UnicodeDecodeError: + print(f"Could not decode: {k}: {item}") + else: + info_dictionary[k] = item + return info_dictionary + + identity_info = await self.substrate.query( + module="Registry", + storage_function="IdentityOf", + params=[key], + block_hash=block_hash, + reuse_block_hash=reuse_block, + ) + return decode_hex_identity_dict(identity_info.value["info"]) diff --git a/src/wallets.py b/src/wallets.py index 5f53274a..1052d2ba 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -20,6 +20,7 @@ from concurrent.futures import ProcessPoolExecutor import itertools import os +from sys import getsizeof from typing import Optional, Any import aiohttp @@ -27,6 +28,7 @@ from bittensor_wallet.keyfile import Keyfile from fuzzywuzzy import fuzz from rich.align import Align +from rich.prompt import Confirm from rich.table import Table, Column from rich.tree import Tree import scalecodec @@ -632,7 +634,6 @@ async def overview( if len(alerts_table.rows) > 0: grid.add_row(alerts_table) - title: str = "" if not all_wallets: title = f"[bold white italic]Wallet - {wallet.name}:{wallet.coldkeypub.ss58_address}" else: @@ -1023,11 +1024,11 @@ def _partial_decode(args): def _process_neurons_for_netuids(netuids_with_all_neurons_hex_bytes): - def make_map(result): - netuid, json_result = result + def make_map(res_): + netuid_, json_result = res_ hex_bytes_result = json_result["result"] as_scale_bytes = scalecodec.ScaleBytes(hex_bytes_result) - return [return_type, as_scale_bytes, custom_rpc_type_registry, netuid] + return [return_type, as_scale_bytes, custom_rpc_type_registry, netuid_] return_type = TYPE_REGISTRY["runtime_api"]["NeuronInfoRuntimeApi"]["methods"][ "get_neurons_lite" @@ -1276,3 +1277,108 @@ async def swap_hotkey( wait_for_inclusion=True, prompt=True, ) + + +async def set_id( + wallet: Wallet, + subtensor: SubtensorInterface, + display_name: str, + legal_name: str, + web_url: str, + pgp_fingerprint: str, + riot_handle: str, + email: str, + image: str, + twitter: str, + info_: str, + validator_id, +): + """Create a new or update existing identity on-chain.""" + + identified = wallet.hotkey.ss58_address if validator_id else None + id_dict = { + "display": display_name, + "legal": legal_name, + "web": web_url, + "pgp_fingerprint": pgp_fingerprint, + "riot": riot_handle, + "email": email, + "image": image, + "twitter": twitter, + "info": info_, + "identified": identified, + } + + for field, string in id_dict.items(): + if getsizeof(string) > 113: # 64 + 49 overhead bytes for string + raise ValueError(f"Identity value `{field}` must be <= 64 raw bytes") + + if not Confirm( + "Cost to register an Identity is [bold white italic]0.1 Tao[/bold white italic]," + " are you sure you wish to continue?" + ): + console.print(":cross_mark: Aborted!") + raise typer.Exit() + + wallet.unlock_coldkey() + + with console.status(":satellite: [bold green]Updating identity on-chain..."): + async with subtensor: + try: + call = await subtensor.substrate.compose_call( + call_module="Registry", + call_function="set_identity", + call_params=id_dict, + ) + extrinsic = await subtensor.substrate.create_signed_extrinsic( + call=call, keypair=wallet.coldkey + ) + response = await subtensor.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=True, + wait_for_finalization=False, + ) + response.process_events() + if not response.is_success: + raise Exception(response.error_message) + + except Exception as e: + console.print(f"[red]:cross_mark: Failed![/red] {e}") + typer.Exit(1) + + console.print(":white_heavy_check_mark: Success!") + identity = await subtensor.query_identity( + identified or wallet.coldkey.ss58_address + ) + await subtensor.substrate.close() + + table = Table( + Column("Key", justify="right", style="cyan", no_wrap=True), + Column("Value", style="magenta"), + title="[bold white italic]Updated On-Chain Identity", + ) + + table.add_row("Address", identified or wallet.coldkey.ss58_address) + for key, value in identity.items(): + table.add_row(key, str(value) if value is not None else "~") + + return console.print(table) + + +async def get_id(subtensor: SubtensorInterface, ss58_address: str): + with console.status(":satellite: [bold green]Querying chain identity..."): + async with subtensor: + identity = await subtensor.query_identity(ss58_address) + await subtensor.substrate.close() + + table = Table( + Column("Item", justify="right", style="cyan", no_wrap=True), + Column("Value", style="magenta"), + title="[bold white italic]On-Chain Identity", + ) + + table.add_row("Address", ss58_address) + for key, value in identity.items(): + table.add_row(key, str(value) if value is not None else "~") + + return console.print(table) From 828fb6880b008bfcdbabf3d56a17c136425cbd92 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 2 Aug 2024 15:57:57 +0200 Subject: [PATCH 45/48] Ruff --- src/__init__.py | 7 +++---- src/wallets.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/__init__.py b/src/__init__.py index 8d2c4cfd..f10a2361 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -6,15 +6,14 @@ class Constants: finney_entrypoint = "wss://entrypoint-finney.opentensor.ai:443" finney_test_entrypoint = "wss://test.finney.opentensor.ai:443/" archive_entrypoint = "wss://archive.chain.opentensor.ai:443/" + local_entrypoint = "ws://127.0.0.1:9444" network_map = { "finney": finney_entrypoint, "test": finney_test_entrypoint, "archive": archive_entrypoint, + "local": local_entrypoint, } - delegates_details_url = ( - "https://raw.githubusercontent.com/opentensor/" - "bittensor-delegates/main/public/delegates.json" - ) + delegates_detail_url = "https://raw.githubusercontent.com/opentensor/bittensor-delegates/main/public/delegates.json" @dataclass diff --git a/src/wallets.py b/src/wallets.py index 1052d2ba..6128890b 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -1166,7 +1166,7 @@ def neuron_row_maker(wallet_, all_netuids_, nsd) -> list[str]: with console.status("Pulling delegates info"): registered_delegate_info: Optional[ dict[str, DelegatesDetails] - ] = await get_delegates_details_from_github(url=Constants.delegates_details_url) + ] = await get_delegates_details_from_github(url=Constants.delegates_detail_url) if not registered_delegate_info: console.print( ":warning:[yellow]Could not get delegate info from chain.[/yellow]" From 382bafb810ec243647fda36556ee387f70f97ed8 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 2 Aug 2024 16:59:33 +0200 Subject: [PATCH 46/48] Added wallet check-swap --- cli.py | 28 ++++++++++++++++++++++++++++ src/utils.py | 14 ++++++++++++++ src/wallets.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/cli.py b/cli.py index 005d6741..51dd641d 100755 --- a/cli.py +++ b/cli.py @@ -162,6 +162,7 @@ def __init__(self): self.wallet_app.command("faucet")(self.wallet_faucet) self.wallet_app.command("set-identity")(self.wallet_set_id) self.wallet_app.command("get-identity")(self.wallet_get_id) + self.wallet_app.command("check-swap")(self.wallet_check_ck_swap) # delegates commands self.delegates_app.command("list")(self.delegates_list) @@ -904,6 +905,33 @@ def wallet_new_coldkey( wallets.new_coldkey(wallet, n_words, use_password, overwrite_coldkey) ) + def wallet_check_ck_swap( + self, + wallet_name: Optional[str] = Options.wallet_name, + wallet_path: Optional[str] = Options.wallet_path, + wallet_hotkey: Optional[str] = Options.wallet_hotkey, + network: Optional[str] = Options.network, + chain: Optional[str] = Options.chain, + ): + """ + # wallet check-swap + Executes the `check-swap` command to check swap status of a coldkey in the Bittensor network. + + ## Usage: + Users need to specify the wallet they want to check the swap status of. + + ### Example usage: + ``` + btcli wallet check_coldkey_swap + ``` + + #### Note: + This command is important for users who wish check if swap requests were made against their coldkey. + """ + wallet = self.wallet_ask(wallet_name, wallet_path, wallet_hotkey) + self.initialize_chain(network, chain) + return self._run_command(wallets.check_coldkey_swap(wallet, self.not_subtensor)) + def wallet_create_wallet( self, wallet_name: Optional[str] = Options.wallet_name, diff --git a/src/utils.py b/src/utils.py index 262cfea5..22831a0f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -272,6 +272,20 @@ def format_error_message(error_message: dict) -> str: return f"Subtensor returned `{err_name} ({err_type})` error. This means: `{err_description}`" +def convert_blocks_to_time(blocks: int, block_time: int = 12) -> tuple[int, int, int]: + """ + Converts number of blocks into number of hours, minutes, seconds. + :param blocks: number of blocks + :param block_time: time per block, by default this is 12 + :return: tuple containing number of hours, number of minutes, number of seconds + """ + seconds = blocks * block_time + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + remaining_seconds = seconds % 60 + return hours, minutes, remaining_seconds + + async def get_delegates_details_from_github(url: str) -> dict[str, DelegatesDetails]: all_delegates_details = {} diff --git a/src/wallets.py b/src/wallets.py index 6128890b..a15964a7 100644 --- a/src/wallets.py +++ b/src/wallets.py @@ -50,6 +50,7 @@ get_all_wallets_for_path, get_hotkey_wallets_for_wallet, get_delegates_details_from_github, + convert_blocks_to_time, ) from src.subtensor_interface import SubtensorInterface from src.bittensor.balances import Balance @@ -1382,3 +1383,43 @@ async def get_id(subtensor: SubtensorInterface, ss58_address: str): table.add_row(key, str(value) if value is not None else "~") return console.print(table) + + +async def check_coldkey_swap(wallet: Wallet, subtensor: SubtensorInterface): + async with subtensor: + arbitration_check = len( + ( + await subtensor.substrate.query( + module="SubtensorModule", + storage_function="ColdkeySwapDestinations", + params=[wallet.coldkeypub.ss58_address], + ) + ).decode() + ) + if arbitration_check == 0: + console.print( + "[green]There has been no previous key swap initiated for your coldkey.[/green]" + ) + elif arbitration_check == 1: + arbitration_block = await subtensor.substrate.query( + module="SubtensorModule", + storage_function="ColdkeyArbitrationBlock", + params=[wallet.coldkeypub.ss58_address], + ) + arbitration_remaining = ( + arbitration_block.value + - await subtensor.substrate.get_block_number(None) + ) + + hours, minutes, seconds = convert_blocks_to_time(arbitration_remaining) + console.print( + "[yellow]There has been 1 swap request made for this coldkey already." + " By adding another swap request, the key will enter arbitration." + f" Your key swap is scheduled for {hours} hours, {minutes} minutes, {seconds} seconds" + " from now.[/yellow]" + ) + elif arbitration_check > 1: + console.print( + f"[red]This coldkey is currently in arbitration with a total swaps of {arbitration_check}.[/red]" + ) + await subtensor.substrate.close() From 33b21cb9c9f9f8c8abccf5718e720a4b97fdaa59 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 2 Aug 2024 21:00:50 +0200 Subject: [PATCH 47/48] Versioning with branch name --- cli.py | 21 +++++++++++++++++++-- requirements.txt | 1 + 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/cli.py b/cli.py index 51dd641d..638953f4 100755 --- a/cli.py +++ b/cli.py @@ -4,10 +4,12 @@ from typing import Optional, Coroutine from bittensor_wallet import Wallet +from git import Repo import rich from rich.prompt import Confirm, Prompt from rich.table import Table, Column import typer +from typing_extensions import Annotated from websockets import ConnectionClosed from yaml import safe_load, safe_dump @@ -16,6 +18,9 @@ from src.utils import console +__version__ = "8.0.0" + + class Options: """ Re-usable typer args @@ -100,6 +105,12 @@ def get_creation_data(mnemonic, seed, json, json_password): return mnemonic, seed, json, json_password +def version_callback(value: bool): + if value: + typer.echo(f"BTCLI Version: {__version__}/{Repo('.').active_branch.name}") + raise typer.Exit() + + class CLIManager: def __init__(self): self.config = { @@ -111,7 +122,7 @@ def __init__(self): } self.not_subtensor = None - self.app = typer.Typer(rich_markup_mode="markdown", callback=self.check_config) + self.app = typer.Typer(rich_markup_mode="markdown", callback=self.main_callback) self.config_app = typer.Typer() self.wallet_app = typer.Typer() self.delegates_app = typer.Typer() @@ -192,7 +203,13 @@ def _run_command(self, cmd: Coroutine): except ConnectionClosed: pass - def check_config(self): + def main_callback( + self, + version: Annotated[ + Optional[bool], typer.Option("--version", callback=version_callback) + ] = None, + ): + # check config with open(os.path.expanduser("~/.bittensor/config.yml"), "r") as f: config = safe_load(f) for k, v in config.items(): diff --git a/requirements.txt b/requirements.txt index fdd99157..149cac3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ aiohttp~=3.9.5 backoff~=2.2.1 git+https://github.com/opentensor/btwallet # bittensor_wallet +GitPython>=3.0.0 fuzzywuzzy~=0.18.0 netaddr~=1.3.0 numpy>=2.0.1 From dca2d7aa3f85ec4016bf6008211ddc945205eeb0 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 2 Aug 2024 21:08:37 +0200 Subject: [PATCH 48/48] Versioning with branch name --- cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli.py b/cli.py index 638953f4..2e5b9b11 100755 --- a/cli.py +++ b/cli.py @@ -107,7 +107,9 @@ def get_creation_data(mnemonic, seed, json, json_password): def version_callback(value: bool): if value: - typer.echo(f"BTCLI Version: {__version__}/{Repo('.').active_branch.name}") + typer.echo( + f"BTCLI Version: {__version__}/{Repo(os.path.dirname(__file__)).active_branch.name}" + ) raise typer.Exit()