From 81331fd0bb3092611cd757efe052f5138b9a7198 Mon Sep 17 00:00:00 2001 From: Elias Datler <46360620+fxgst@users.noreply.github.com> Date: Wed, 20 Nov 2024 01:57:23 +0700 Subject: [PATCH] feat: Support PocketIC server 7.0 (#66) - Support for PocketIC server version 7.0 - Load a state directory for any subnet kind with `SubnetConfig.add_subnet_with_state` - Verified Application subnet type - remove `with_nns_state` - replace `PocketIC.topology` with `PocketIC.topology()` --- CHANGELOG.md | 13 ++++ flake.lock | 12 +-- flake.nix | 8 +- pocket_ic/pocket_ic.py | 137 +++++++++++++++++++++++----------- pocket_ic/pocket_ic_server.py | 66 +++++++--------- pyproject.toml | 2 +- tests/pocket_ic_test.py | 41 ++++++---- 7 files changed, 169 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 287e641..b065ffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## 3.0.0 - 2024-11-19 + +### Added +- Support for PocketIC server version 7.0.0 +- Load a state directory for any subnet kind with `SubnetConfig.add_subnet_with_state` +- Verified Application subnet type + +### Removed +- `with_nns_state`. Use `SubnetConfig.add_subnet_with_state` instead +- `PocketIC.topology`. Use `PocketIC.topology()` instead + + + ## 2.1.0 - 2024-02-08 ### Added diff --git a/flake.lock b/flake.lock index 53c7b9c..4ef57cc 100644 --- a/flake.lock +++ b/flake.lock @@ -37,25 +37,25 @@ "pocket-ic-darwin-gz": { "flake": false, "locked": { - "narHash": "sha256-paPv910O9u7JriOWZDSZ1UEy5qqVDjA6ESz5PfvFpmg=", + "narHash": "sha256-tAxziSPN43Utb7Bq6aJheN/uZafOIdWFaq395kw97HE=", "type": "file", - "url": "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-darwin.gz" + "url": "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-darwin.gz" }, "original": { "type": "file", - "url": "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-darwin.gz" + "url": "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-darwin.gz" } }, "pocket-ic-linux-gz": { "flake": false, "locked": { - "narHash": "sha256-eaZJN4gebKFTdzB88hN0mLAbdslH1x1xbYqIi/VeRY8=", + "narHash": "sha256-oks9H8GARq8opVcy+0RH7lffG34lZDuY90iFS0bGhvI=", "type": "file", - "url": "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-linux.gz" + "url": "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-linux.gz" }, "original": { "type": "file", - "url": "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-linux.gz" + "url": "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-linux.gz" } }, "root": { diff --git a/flake.nix b/flake.nix index 2e28ec0..7761d0f 100644 --- a/flake.nix +++ b/flake.nix @@ -1,15 +1,15 @@ { - description = "PocketIC Python Libary"; + description = "PocketIC Python Library"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; pocket-ic-darwin-gz = { - url = "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-darwin.gz"; + url = "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-darwin.gz"; flake = false; }; pocket-ic-linux-gz = { - url = "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-linux.gz"; + url = "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-linux.gz"; flake = false; }; }; @@ -86,7 +86,7 @@ ''; checks.default = pkgs.runCommand "pocketic-py-tests" { - nativeBuildInputs = [ pytest pocketic-py]; + nativeBuildInputs = [ pytest pocketic-py pkgs.cacert]; POCKET_IC_BIN = "${pocket-ic}/bin/pocket-ic"; inherit projectDir; } '' diff --git a/pocket_ic/pocket_ic.py b/pocket_ic/pocket_ic.py index 69497e4..006e66c 100644 --- a/pocket_ic/pocket_ic.py +++ b/pocket_ic/pocket_ic.py @@ -3,11 +3,12 @@ It also contains 'SubnetConfig' and 'SubnetKind', which are used to configure the subnets of a PocketIC instance. """ + import base64 import ic from enum import Enum from ic.candid import Types -from typing import List, Optional, Any +from typing import Optional, Any from pocket_ic.pocket_ic_server import PocketICServer @@ -21,6 +22,7 @@ class SubnetKind(Enum): NNS = "NNS" SNS = "SNS" SYSTEM = "System" + VERIFIED_APPLICATION = "VerifiedApplication" class SubnetConfig: @@ -35,24 +37,26 @@ def __init__( nns=False, sns=False, system=0, + verified_application=0, ) -> None: - self.application = application - self.bitcoin = bitcoin - self.fiduciary = fiduciary - self.ii = ii - self.nns = nns - self.sns = sns - self.system = system + new = {"state_config": "New", "instruction_config": "Production"} + self.application = [new] * application + self.bitcoin = new if bitcoin else None + self.fiduciary = new if fiduciary else None + self.ii = new if ii else None + self.nns = new if nns else None + self.sns = new if sns else None + self.system = [new] * system + self.verified_application = [new] * verified_application def __repr__(self) -> str: - return f"SubnetConfigSet(application={self.application}, bitcoin={self.bitcoin}, fiduciary={self.fiduciary}, ii={self.ii}, nns={self.nns}, sns={self.sns}, system={self.system})" + return f"SubnetConfigSet(application={self.application}, bitcoin={self.bitcoin}, fiduciary={self.fiduciary}, ii={self.ii}, nns={self.nns}, sns={self.sns}, system={self.system}, verified_application={self.verified_application})" def validate(self) -> None: """Validates the subnet configuration. Raises: - ValueError: if no subnet is configured or if the number of application or system - subnets is negative + ValueError: if no subnet is configured """ if not ( self.bitcoin @@ -62,34 +66,77 @@ def validate(self) -> None: or self.sns or self.system or self.application + or self.verified_application ): raise ValueError("At least one subnet must be configured.") - if self.application < 0 or self.system < 0: - raise ValueError( - "The number of application and system subnets must be non-negative." - ) + def add_subnet_with_state( + self, subnet_type: SubnetKind, state_dir_path: str, subnet_id: ic.Principal + ): + """Add a subnet with state loaded form the given state directory. + Note that the provided path must be accessible for the PocketIC server process. + + `state_dir` should point to a directory which is expected to have the following structure: + + state_dir/ + |-- backups + |-- checkpoints + |-- diverged_checkpoints + |-- diverged_state_markers + |-- fs_tmp + |-- page_deltas + |-- states_metadata.pbuf + |-- tip + `-- tmp + + `subnet_id` should be the subnet ID of the subnet in the state to be loaded""" + + raw_subnet_id_bytes = base64.b64encode( + subnet_id.bytes + ) # convert bytes to base64 bytes + raw_subnet_id = raw_subnet_id_bytes.decode() # convert bytes to str + + new_from_path = { + "state_config": { + "FromPath": [state_dir_path, {"subnet_id": raw_subnet_id}] + }, + "instruction_config": "Production", + } - def with_nns_state(self, state_dir_path: str, nns_subnet_id: ic.Principal): - """Provide an NNS state directory and a subnet id. """ - self.nns = (state_dir_path, nns_subnet_id) + match subnet_type: + case SubnetKind.APPLICATION: + self.application.append(new_from_path) + case SubnetKind.BITCOIN: + self.bitcoin = new_from_path + case SubnetKind.FIDUCIARY: + self.fiduciary = new_from_path + case SubnetKind.II: + self.ii = new_from_path + case SubnetKind.NNS: + self.nns = new_from_path + case SubnetKind.SNS: + self.sns = new_from_path + case SubnetKind.SYSTEM: + self.system.append(new_from_path) + case SubnetKind.VERIFIED_APPLICATION: + self.verified_application.append(new_from_path) def _json(self) -> dict: - if isinstance(self.nns, tuple): - raw_subnet_id = base64.b64encode(self.nns[1].bytes).decode() - nns = {"FromPath": (self.nns[0], {"subnet_id": raw_subnet_id})} - elif self.nns: - nns = "New" - else: - nns = None return { - "application": self.application * ["New"], - "bitcoin": "New" if self.bitcoin else None, - "fiduciary": "New" if self.fiduciary else None, - "ii": "New" if self.ii else None, - "nns": nns, - "sns": "New" if self.sns else None, - "system": self.system * ["New"], + "subnet_config_set": { + "application": self.application, + "bitcoin": self.bitcoin, + "fiduciary": self.fiduciary, + "ii": self.ii, + "nns": self.nns, + "sns": self.sns, + "system": self.system, + "verified_application": self.verified_application, + }, + "state_dir": None, + "nonmainnet_features": False, + "log_level": None, + "bitcoind_addr": None, } @@ -111,8 +158,7 @@ def __init__(self, subnet_config: Optional[SubnetConfig] = None) -> None: self.server = PocketICServer() subnet_config = subnet_config if subnet_config else SubnetConfig(application=1) subnet_config.validate() - self.instance_id, topology = self.server.new_instance(subnet_config._json()) - self.topology = self._generate_topology(topology) + self.instance_id = self.server.new_instance(subnet_config._json()) self.sender = ic.Principal.anonymous() def __del__(self) -> None: @@ -131,13 +177,24 @@ def set_sender(self, principal: ic.Principal) -> None: """ self.sender = principal + def topology(self): + """Returns the current topology of the PocketIC instance.""" + res = self._instance_get("read/topology") + t = dict() + subnets = res["subnet_configs"] + for subnet_id, config in subnets.items(): + subnet_id = ic.Principal.from_str(subnet_id) + subnet_kind = SubnetKind(config["subnet_kind"]) + t.update({subnet_id: subnet_kind}) + return t + def get_root_key(self) -> Optional[bytes]: """Get the root key of the IC. If there is no NNS subnet, returns `None`. Returns: Optional[bytes]: the root key of the IC """ - nns_subnet = [k for k, v in self.topology.items() if v == SubnetKind.NNS] + nns_subnet = [k for k, v in self.topology().items() if v == SubnetKind.NNS] if not nns_subnet: return None body = { @@ -448,7 +505,7 @@ def create_and_install_canister_with_candid( arg = [{"type": canister_arguments[0], "value": init_args}] else: raise ValueError("The candid file appears to be malformed") - + self.add_cycles(canister_id, 2_000_000_000_000) self.install_code(canister_id, wasm_module, arg) return canister @@ -499,14 +556,6 @@ def _canister_call( res = self._instance_post(endpoint, body) return self._get_ok_reply(res) - def _generate_topology(self, topology): - t = dict() - for subnet_id, config in topology.items(): - subnet_id = ic.Principal.from_str(subnet_id) - subnet_kind = SubnetKind(config["subnet_kind"]) - t.update({subnet_id: subnet_kind}) - return t - def _get_ok_reply(self, request_result): if "Ok" in request_result: if "Reply" not in request_result["Ok"]: diff --git a/pocket_ic/pocket_ic_server.py b/pocket_ic/pocket_ic_server.py index b4bfe2e..b946fe7 100644 --- a/pocket_ic/pocket_ic_server.py +++ b/pocket_ic/pocket_ic_server.py @@ -9,9 +9,6 @@ from tempfile import gettempdir -HEADERS = {"processing-timeout-ms": "300000"} - - class PocketICServer: """ An object of this class represents a running PocketIC server. During instantiation, @@ -46,24 +43,27 @@ def __init__(self) -> None: """ ) - # Attempt to start the PocketIC server if it's not already running. mute = ( "1> /dev/null 2> /dev/null" if "POCKET_IC_MUTE_SERVER" in os.environ else "" ) - os.system(f"{bin_path} --pid {pid} {mute} &") - self.url = self._get_url(pid) + # Attempt to start the PocketIC server if it's not already running. + tmp_dir = gettempdir() + port_file_path = f"{tmp_dir}/pocket_ic_{pid}.port" + + os.system(f"{bin_path} --port-file {port_file_path} {mute} &") + self.url = self._get_url(port_file_path) self.request_client = requests.session() - def new_instance(self, subnet_config: dict) -> Tuple[int, dict]: + def new_instance(self, subnet_config: dict) -> int: """Creates a new PocketIC instance. Returns: - str: the new instance ID + int: the new instance ID """ url = f"{self.url}/instances" - response = self.request_client.post(url, headers=HEADERS, json=subnet_config) + response = self.request_client.post(url, json=subnet_config) res = self._check_response(response)["Created"] - return res["instance_id"], res["topology"] + return res["instance_id"] def list_instances(self) -> List[str]: """Lists the currently running instances on the PocketIC Server. @@ -72,7 +72,7 @@ def list_instances(self) -> List[str]: List[str]: a list of instance names """ url = f"{self.url}/instances" - response = self.request_client.get(url, headers=HEADERS) + response = self.request_client.get(url) response = self._check_response(response) return response @@ -83,18 +83,18 @@ def delete_instance(self, instance_id: int): instance_id (int): the ID of the instance to delete """ url = f"{self.url}/instances/{instance_id}" - self.request_client.delete(url, headers=HEADERS) + self.request_client.delete(url) def instance_get(self, endpoint: str, instance_id: int): """HTTP get requests for instance endpoints""" url = f"{self.url}/instances/{instance_id}/{endpoint}" - response = self.request_client.get(url, headers=HEADERS) + response = self.request_client.get(url) return self._check_response(response) def instance_post(self, endpoint: str, instance_id: int, body: Optional[dict]): """HTTP post requests for instance endpoints""" url = f"{self.url}/instances/{instance_id}/{endpoint}" - response = self.request_client.post(url, json=body, headers=HEADERS) + response = self.request_client.post(url, json=body) return self._check_response(response) def set_blob_store_entry(self, blob: bytes, compression: Optional[str]) -> str: @@ -109,9 +109,9 @@ def set_blob_store_entry(self, blob: bytes, compression: Optional[str]) -> str: """ url = f"{self.url}/blobstore" if compression is None: - response = self.request_client.post(url, data=blob, headers=HEADERS) + response = self.request_client.post(url, data=blob) elif compression == "gzip": - headers = HEADERS | {"Content-Encoding": "gzip"} + headers = {"Content-Encoding": "gzip"} response = self.request_client.post(url, data=blob, headers=headers) else: raise ValueError('only "gzip" compression is supported') @@ -119,34 +119,22 @@ def set_blob_store_entry(self, blob: bytes, compression: Optional[str]) -> str: self._check_status_code(response) return response.text - def _get_url(self, pid: int) -> str: - tmp_dir = gettempdir() - ready_file_path = f"{tmp_dir}/pocket_ic_{pid}.ready" - port_file_path = f"{tmp_dir}/pocket_ic_{pid}.port" - - stop_at = time.time() + 10 # Wait for the ready file for 10 seconds - - while not os.path.exists(ready_file_path): - if time.time() < stop_at: - time.sleep(0.1) # 100ms - else: - raise TimeoutError("PocketIC failed to start") - - if os.path.isfile(ready_file_path): - with open(port_file_path, "r", encoding="utf-8") as port_file: - port = port_file.readline().strip() - else: - raise ValueError(f"{ready_file_path} is not a file!") - - return f"http://127.0.0.1:{port}" + def _get_url(self, port_file_path: int) -> str: + while True: + if os.path.isfile(port_file_path): + with open(port_file_path, "r", encoding="utf-8") as port_file: + port = port_file.readline() + if "\n" in port: + return f"http://127.0.0.1:{port.strip()}" + time.sleep(0.02) # wait for 20ms - def _check_response(self, response): + def _check_response(self, response: requests.Response): self._check_status_code(response) res_json = response.json() return res_json - def _check_status_code(self, response): + def _check_status_code(self, response: requests.Response): if response.status_code not in [200, 201, 202]: raise ConnectionError( - f'PocketIC server returned status code {response.status_code}: "{response.reason}"' + f'PocketIC server returned status code {response.status_code}: "{response.text}"' ) diff --git a/pyproject.toml b/pyproject.toml index b75017f..c3c17d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pocket_ic" -version = "2.1.0" +version = "3.0.0" description = "PocketIC: A Canister Smart Contract Testing Platform" authors = [ "The Internet Computer Project Developers ", diff --git a/tests/pocket_ic_test.py b/tests/pocket_ic_test.py index 311cb7e..b9156af 100644 --- a/tests/pocket_ic_test.py +++ b/tests/pocket_ic_test.py @@ -32,11 +32,11 @@ def test_create_canister_with_id(self): new_canister_id = pic.create_canister() self.assertNotEqual(new_canister_id.bytes, canister_id.bytes) - # Creating a canister with an ID that is not hosted by any subnet fails. - canister_id = ic.Principal.anonymous() - with self.assertRaises(ValueError) as ex: - pic.create_canister(canister_id=canister_id) - self.assertIn("CanisterNotHostedBySubnet", ex.exception.args[0]) + # Creating a canister with an ID that is not hosted by any subnet creates a new subnet containing the canister. + canister_id = ic.Principal.from_str("zzztf-6qaaa-aaaah-qfsaa-cai") + self.assertEqual(len(pic.topology()), 1) + pic.create_canister(canister_id=canister_id) + self.assertEqual(len(pic.topology()), 2) def test_large_config_and_deduplication(self): pic = PocketIC( @@ -51,23 +51,25 @@ def test_large_config_and_deduplication(self): ) ) app_subnets = [ - k for k, v in pic.topology.items() if v == SubnetKind.APPLICATION + k for k, v in pic.topology().items() if v == SubnetKind.APPLICATION ] self.assertEqual(len(app_subnets), 2) - nns_subnets = [k for k, v in pic.topology.items() if v == SubnetKind.NNS] + nns_subnets = [k for k, v in pic.topology().items() if v == SubnetKind.NNS] self.assertEqual(len(nns_subnets), 1) bitcoin_subnets = [ - k for k, v in pic.topology.items() if v == SubnetKind.BITCOIN + k for k, v in pic.topology().items() if v == SubnetKind.BITCOIN ] self.assertEqual(len(bitcoin_subnets), 1) - system_subnets = [k for k, v in pic.topology.items() if v == SubnetKind.SYSTEM] + system_subnets = [ + k for k, v in pic.topology().items() if v == SubnetKind.SYSTEM + ] self.assertEqual(len(system_subnets), 3) def test_install_canister_on_subnet_and_get_subnet_of_canister(self): pic = PocketIC(SubnetConfig(nns=True, application=1)) - nns_subnet = next(k for k, v in pic.topology.items() if v == SubnetKind.NNS) + nns_subnet = next(k for k, v in pic.topology().items() if v == SubnetKind.NNS) app_subnet = next( - k for k, v in pic.topology.items() if v == SubnetKind.APPLICATION + k for k, v in pic.topology().items() if v == SubnetKind.APPLICATION ) nns_canister = pic.create_canister(subnet=nns_subnet) app_canister = pic.create_canister(subnet=app_subnet) @@ -147,14 +149,21 @@ def test_cycles_balance(self): pic.add_cycles(canister_id, 6_666) self.assertEqual(pic.get_cycles_balance(canister_id), initial_balance + 6_666) - def test_nns_state(self): - principal = "6gvjz-uotju-2ngtj-u2ngt-ju2ng-tju2n-gtju2-ngtjv" + def test_load_state(self): + principal = ic.Principal.from_str( + "6gvjz-uotju-2ngtj-u2ngt-ju2ng-tju2n-gtju2-ngtjv" + ) tmp_dir = tempfile.mkdtemp() - pic = PocketIC(SubnetConfig(nns=(tmp_dir, ic.Principal.from_str(principal)))) - (k,v) = list(pic.topology.items())[0] - self.assertEqual(str(k), principal) + + config = SubnetConfig() + config.add_subnet_with_state(SubnetKind.NNS, tmp_dir, principal) + pic = PocketIC(subnet_config=config) + + (k, v) = list(pic.topology().items())[0] + self.assertEqual(str(k), str(principal)) self.assertEqual(v, SubnetKind.NNS) os.rmdir(tmp_dir) + if __name__ == "__main__": unittest.main()