diff --git a/CHANGELOG.md b/CHANGELOG.md index 631a78a1b6..5dd46fb341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +**0.3.4** +1. Added the possibility to perform a backup of a sidechain non coin-boxes and restore these boxes into a new bootstrapped sidechain of the same type. +2. log4j version updated. + **0.3.3** 1. Mainchain block deserialization fix: CompactSize usage issue. 2. Bootstrapping tool improvement: scgenesisinfo data parsing. diff --git a/README.md b/README.md index b602651b78..f33761afce 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ While we keep monitoring the memory footprint of the proofs generation process, - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows: ``` -LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar:./target/lib/* com.horizen.examples.SimpleApp +LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/ci/run_sc.sh b/ci/run_sc.sh index 1ca672a009..228af7ced5 100755 --- a/ci/run_sc.sh +++ b/ci/run_sc.sh @@ -2,7 +2,7 @@ set -eo pipefail -SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.3}" +SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.4}" if [ -d "$1" ] && [ -f "$2" ]; then path_to_jemalloc="$(ldconfig -p | grep "$(arch)" | grep 'libjemalloc\.so\.1$' | tr -d ' ' | cut -d '>' -f 2)" diff --git a/examples/simpleapp/README.md b/examples/simpleapp/README.md index 89f4528e2b..b9f310fd42 100644 --- a/examples/simpleapp/README.md +++ b/examples/simpleapp/README.md @@ -19,12 +19,12 @@ Otherwise, to run SimpleApp outside the IDE: * (Windows) ``` cd Sidechains-SDK\examples\simpleapp - java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar;./target/lib/* com.horizen.examples.SimpleApp + java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar;./target/lib/* com.horizen.examples.SimpleApp ``` * (Linux) ``` cd ./Sidechains-SDK/examples/simpleapp - java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar:./target/lib/* com.horizen.examples.SimpleApp + java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp ``` On some Linux OSs during backward transfers certificates proofs generation a extremely big RAM consumption may happen, that will lead to the process force killing by the OS. @@ -36,7 +36,7 @@ While we keep monitoring the memory footprint of the proofs generation process, - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows: ``` - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar:./target/lib/* com.horizen.examples.SimpleApp + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/examples/simpleapp/mc_sc_workflow_example.md b/examples/simpleapp/mc_sc_workflow_example.md index e0884b227b..5c635f804c 100644 --- a/examples/simpleapp/mc_sc_workflow_example.md +++ b/examples/simpleapp/mc_sc_workflow_example.md @@ -15,7 +15,7 @@ Build SDK components by using command (in the root of the SDK folder): Run Bootstrapping tool using command: -`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.3.jar` +`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.4.jar` All other commands are performed as commands for Bootstrapping tool in next format: `"command name" "parameters for command in JSON format"`. For any help you could use command `help`, for exit just print `exit` @@ -397,15 +397,15 @@ Run SimpleApp with the `my_settings.conf`: * For Windows: ``` - java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf + java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf ``` * For Linux (Glibc): ``` - java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf + java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf ``` * For Linux (Jemalloc): ``` - LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf + LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/examples/simpleapp/pom.xml b/examples/simpleapp/pom.xml index 2a0f798deb..d04a8472f1 100644 --- a/examples/simpleapp/pom.xml +++ b/examples/simpleapp/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.horizen sidechains-sdk-simpleapp - 0.3.3 + 0.3.4 2018 UTF-8 @@ -16,7 +16,7 @@ io.horizen sidechains-sdk - 0.3.3 + 0.3.4 diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/BoxBackup.java b/examples/simpleapp/src/main/java/com/horizen/examples/BoxBackup.java new file mode 100644 index 0000000000..2ce03e1ca1 --- /dev/null +++ b/examples/simpleapp/src/main/java/com/horizen/examples/BoxBackup.java @@ -0,0 +1,12 @@ +package com.horizen.examples; + +import com.horizen.storage.BoxBackupInterface; +import com.horizen.storage.BackupStorage; +import com.horizen.backup.BoxIterator; + +public class BoxBackup implements BoxBackupInterface { + @Override + public void backup(BoxIterator source, BackupStorage db) throws Exception { + + } +} diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java index e491d0a93a..fb7f85857e 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java @@ -1,5 +1,6 @@ package com.horizen.examples; +import com.horizen.backup.BoxIterator; import com.horizen.block.SidechainBlock; import com.horizen.box.Box; import com.horizen.proposition.Proposition; @@ -91,4 +92,9 @@ public void closeStorages() { appStorage1.close(); appStorage2.close(); } + + + public Try onBackupRestore(BoxIterator i) { + return new Success<>(this); + } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java index ddcd41e94e..8d9a1b961b 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java @@ -1,5 +1,6 @@ package com.horizen.examples; +import com.horizen.backup.BoxIterator; import com.horizen.box.Box; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; @@ -84,4 +85,9 @@ public void closeStorages() { walletStorage1.close(); walletStorage2.close(); } + + @Override + public void onBackupRestore(BoxIterator i) { + + } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java index 545cde3b3d..638d9fb215 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java @@ -65,7 +65,7 @@ public void configureApp() { File stateUtxoMerkleTreeStore = new File(dataDirAbsolutePath + "/stateUtxoMerkleTree"); File historyStore = new File(dataDirAbsolutePath + "/history"); File consensusStore = new File(dataDirAbsolutePath + "/consensusData"); - + File backupStore = new File(dataDirAbsolutePath + "/backupStorage"); // Here I can add my custom rest api and/or override existing one @@ -133,6 +133,9 @@ public void configureApp() { bind(Storage.class) .annotatedWith(Names.named("ConsensusStorage")) .toInstance(new VersionedLevelDbStorageAdapter(consensusStore)); + bind(Storage.class) + .annotatedWith(Names.named("BackupStorage")) + .toInstance(new VersionedLevelDbStorageAdapter(backupStore)); bind(new TypeLiteral> () {}) .annotatedWith(Names.named("CustomApiGroups")) @@ -145,5 +148,6 @@ public void configureApp() { bind(SidechainAppStopper.class) .annotatedWith(Names.named("ApplicationStopper")) .toInstance(applicationStopper); + } } diff --git a/pom.xml b/pom.xml index e7de41a3ad..af1f0c04db 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 com.horizen Sidechains - 0.3.3 + 0.3.4 2018 UTF-8 diff --git a/qa/SidechainTestFramework/sc_test_framework.py b/qa/SidechainTestFramework/sc_test_framework.py index 3d030679f6..a4b4443dca 100644 --- a/qa/SidechainTestFramework/sc_test_framework.py +++ b/qa/SidechainTestFramework/sc_test_framework.py @@ -114,7 +114,7 @@ def main(self): help="Don't stop bitcoinds after the test execution") parser.add_option("--zendir", dest="zendir", default="ZenCore/src", help="Source directory containing zend/zen-cli (default: %default)") - parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future + parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future help="Directory containing .jar file for SC (default: %default)") parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="sc_test"), help="Root directory for datadirs") diff --git a/qa/SidechainTestFramework/scutil.py b/qa/SidechainTestFramework/scutil.py index 79d366951a..dd1c6092a9 100644 --- a/qa/SidechainTestFramework/scutil.py +++ b/qa/SidechainTestFramework/scutil.py @@ -120,7 +120,7 @@ def launch_bootstrap_tool(command_name, json_parameters): json_param = json.dumps(json_parameters) java_ps = subprocess.Popen(["java", "-jar", os.getenv("SIDECHAIN_SDK", - "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.3.jar", + "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.4.jar", command_name, json_param], stdout=subprocess.PIPE) sc_bootstrap_output = java_ps.communicate()[0] try: @@ -142,7 +142,7 @@ def launch_db_tool(dirName, command_name, json_parameters): json_param = json.dumps(json_parameters) java_ps = subprocess.Popen(["java", "-jar", os.getenv("SIDECHAIN_SDK", - "..") + "/tools/dbtool/target/sidechains-sdk-dbtools-0.3.3.jar", + "..") + "/tools/dbtool/target/sidechains-sdk-dbtools-0.3.4.jar", storagesPath, command_name, json_param], stdout=subprocess.PIPE) db_tool_output = java_ps.communicate()[0] try: @@ -468,7 +468,7 @@ def start_sc_node(i, dirname, extra_args=None, rpchost=None, timewait=None, bina lib_separator = ";" if binary is None: - binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp" + binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp" # else if platform.system() == 'Linux': ''' In order to effectively attach a debugger (e.g IntelliJ) to the simpleapp, it is necessary to start the process diff --git a/qa/httpCalls/backup/blockIdForBackup.py b/qa/httpCalls/backup/blockIdForBackup.py new file mode 100644 index 0000000000..f0abf87dae --- /dev/null +++ b/qa/httpCalls/backup/blockIdForBackup.py @@ -0,0 +1,7 @@ +import json + + +# execute a backup/getSidechainBlockIdForBackup call +def getBlockIdForBackup(sidechainNode): + response = sidechainNode.backup_getSidechainBlockIdForBackup() + return response \ No newline at end of file diff --git a/qa/run_sc_tests.py b/qa/run_sc_tests.py index 6076d1e9f0..ee81fe4b73 100644 --- a/qa/run_sc_tests.py +++ b/qa/run_sc_tests.py @@ -33,6 +33,7 @@ from sc_db_tool_cmds import DBToolTest from websocket_server_fee_payments import SCWsServerFeePayments from sc_closed_forger import SidechainClosedForgerTest +from sc_blockid_for_backup import SidechainBlockIdForBackupTest def run_test(test): @@ -139,6 +140,8 @@ def run_tests(log_file): result = run_test(DBToolTest()) assert_equal(0, result, "DBToolTest test failed!") + result = run_test(SidechainBlockIdForBackupTest()) + assert_equal(0, result, "sc_blockid_for_backup test failed!") if __name__ == "__main__": log_file = open("sc_test.log", "w") diff --git a/qa/sc_blockid_for_backup.py b/qa/sc_blockid_for_backup.py new file mode 100644 index 0000000000..72f1bc5d96 --- /dev/null +++ b/qa/sc_blockid_for_backup.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +from SidechainTestFramework.sc_test_framework import SidechainTestFramework +from test_framework.util import assert_equal, assert_true, initialize_chain_clean, start_nodes, connect_nodes_bi, websocket_port_by_mc_node_index, forward_transfer_to_sidechain +from SidechainTestFramework.scutil import start_sc_nodes, generate_next_blocks, bootstrap_sidechain_nodes +from httpCalls.wallet.createPrivateKey25519 import http_wallet_createPrivateKey25519 +from SidechainTestFramework.sc_boostrap_info import SCNodeConfiguration, SCCreationInfo, MCConnectionInfo, \ + SCNetworkConfiguration +from httpCalls.backup.blockIdForBackup import getBlockIdForBackup +import time +from httpCalls.block.best import http_block_best +""" + Setup 1 SC Node. Advanced of some epochs and test the /csw/sidechainBlockItToRollback endpoint + This endpoint should return the sidechain block id containing the mainchain block reference of the MC block with + height = Genesis_MC_block_height + (current_epoch-2) * withdrawalEpochçength -1 +""" +class SidechainBlockIdForBackupTest(SidechainTestFramework): + number_of_mc_nodes = 3 + number_of_sidechain_nodes = 1 + withdrawalEpochLength=10 + + def setup_chain(self): + initialize_chain_clean(self.options.tmpdir, self.number_of_mc_nodes) + + def setup_network(self, split = False): + # Setup nodes and connect them + self.nodes = self.setup_nodes() + connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 0, 2) + self.sync_all() + + def setup_nodes(self): + # Start 3 MC nodes + return start_nodes(self.number_of_mc_nodes, self.options.tmpdir) + + def sc_setup_chain(self): + # Bootstrap new SC, specify SC node 1 connection to MC node 1 + mc_node_1 = self.nodes[0] + + sc_node_1_configuration = SCNodeConfiguration( + MCConnectionInfo(address="ws://{0}:{1}".format(mc_node_1.hostname, websocket_port_by_mc_node_index(0))), + True, + automatic_fee_computation=False, + ) + network = SCNetworkConfiguration(SCCreationInfo(mc_node_1, 600, self.withdrawalEpochLength), + sc_node_1_configuration) + self.sc_nodes_bootstrap_info = bootstrap_sidechain_nodes(self.options, network) + + def sc_setup_nodes(self): + # Start 1 SC node + return start_sc_nodes(self.number_of_sidechain_nodes, self.options.tmpdir) + + def run_test(self): + self.sync_all() + sc_node1 = self.sc_nodes[0] + mc_node1 = self.nodes[0] + + assert_true(sc_node1.submitter_isCertificateSubmitterEnabled()["result"]["enabled"], + "Node 1 submitter expected to be enabled.") + + sc_address_1 = http_wallet_createPrivateKey25519(sc_node1) + + ####################### EPOCH 0 #################### + print("####################### EPOCH 0 ####################") + + # Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + # We need regular coins (the genesis account balance is locked into forging stake), so we perform a + # Forward transfer to sidechain for an amount equals to the genesis_account_balance + forward_transfer_to_sidechain(self.sc_nodes_bootstrap_info.sidechain_id, + mc_node1, + sc_address_1, + self.sc_nodes_bootstrap_info.genesis_account_balance, + mc_node1.getnewaddress()) + self.sc_sync_all() + generate_next_blocks(sc_node1, "first node", 1) + self.sc_sync_all() + + sc_creation_block_height = 450 + sc_creation_block = mc_node1.getblock(str(sc_creation_block_height),2) + assert_true(len(sc_creation_block["tx"][1]["vsc_ccout"]) == 1) + + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch) + print("Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch)") + res = getBlockIdForBackup(sc_node1) + assert_true("error" in res) + assert_equal(res["error"]["code"], "0801") + + #Generate some MC blocks + mc_node1.generate(self.withdrawalEpochLength-2) + + #Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + #This block contains the reference to the MC block 459 that will be the first block available to retrieve with the + # backup/getSidechainBlockIdForBackup endpoint. + blockIdToRollback = http_block_best(sc_node1)["id"] + + # Generate first mc block of the next epoch + mc_node1.generate(1) + generate_next_blocks(sc_node1, "first node", 1) + + # Wait until Certificate will appear in MC node mempool + time.sleep(10) + while mc_node1.getmempoolinfo()["size"] == 0 and sc_node1.submitter_isCertGenerationActive()["result"]["state"]: + print("Wait for certificate in mc mempool...") + time.sleep(2) + sc_node1.block_best() # just a ping to SC node. For some reason, STF can't request SC node API after a while idle. + assert_equal(1, mc_node1.getmempoolinfo()["size"], "Certificate was not added to Mc node mempool.") + + ####################### EPOCH 1 #################### + print("####################### EPOCH 1 ####################") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["state"], "ALIVE") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["epoch"], 1) + + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch) + print("Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch)") + res = getBlockIdForBackup(sc_node1) + assert_true("error" in res) + assert_equal(res["error"]["code"], "0801") + + #Generate some MC blocks + mc_node1.generate(self.withdrawalEpochLength -1) + + #Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + mc_node1.generate(1) + generate_next_blocks(sc_node1, "first node", 1) + + # Wait until Certificate will appear in MC node mempool + time.sleep(10) + while mc_node1.getmempoolinfo()["size"] == 0 and sc_node1.submitter_isCertGenerationActive()["result"]["state"]: + print("Wait for certificate in mc mempool...") + time.sleep(2) + sc_node1.block_best() # just a ping to SC node. For some reason, STF can't request SC node API after a while idle. + assert_equal(1, mc_node1.getmempoolinfo()["size"], "Certificate was not added to Mc node mempool.") + + ####################### EPOCH 2 #################### + print("####################### EPOCH 2 ####################") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["state"], "ALIVE") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["epoch"], 2) + + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we are asking for the MC height 449) + res = getBlockIdForBackup(sc_node1) + assert_true("error" in res) + assert_equal(res["error"]["code"], "0801") + + #Generate some MC blocks + mc_node1.generate(self.withdrawalEpochLength -1) + + #Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + mc_node1.generate(1) + generate_next_blocks(sc_node1, "first node", 1) + + # Wait until Certificate will appear in MC node mempool + time.sleep(10) + while mc_node1.getmempoolinfo()["size"] == 0 and sc_node1.submitter_isCertGenerationActive()["result"]["state"]: + print("Wait for certificate in mc mempool...") + time.sleep(2) + sc_node1.block_best() # just a ping to SC node. For some reason, STF can't request SC node API after a while idle. + assert_equal(1, mc_node1.getmempoolinfo()["size"], "Certificate was not added to Mc node mempool.") + + ####################### EPOCH 3 #################### + print("####################### EPOCH 3 ####################") + + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns the blockIdToRollback + print("Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns the blockIdToRollback") + res = getBlockIdForBackup(sc_node1) + print(res) + assert_equal(res["result"]["blockId"], blockIdToRollback) + +if __name__ == "__main__": + SidechainBlockIdForBackupTest().main() diff --git a/sdk/pom.xml b/sdk/pom.xml index 4c99cdb7c8..0c33579384 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk - 0.3.3 + 0.3.4 ${project.groupId}:${project.artifactId} Zendoo is a unique sidechain and scaling solution developed by Horizen. The Zendoo ${project.artifactId} is a framework that supports the creation of sidechains and their custom business logic, with the Horizen public blockchain as the mainchain. https://github.com/${project.github.organization}/${project.artifactId} @@ -94,7 +94,13 @@ org.apache.logging.log4j log4j-core - 2.17.0 + 2.17.1 + + + + commons-io + commons-io + 2.6 diff --git a/sdk/src/main/java/com/horizen/state/ApplicationState.java b/sdk/src/main/java/com/horizen/state/ApplicationState.java index 1ec65bebd5..7c3b87b943 100644 --- a/sdk/src/main/java/com/horizen/state/ApplicationState.java +++ b/sdk/src/main/java/com/horizen/state/ApplicationState.java @@ -1,5 +1,6 @@ package com.horizen.state; +import com.horizen.backup.BoxIterator; import com.horizen.block.SidechainBlock; import com.horizen.box.Box; import com.horizen.proposition.Proposition; @@ -28,6 +29,8 @@ public interface ApplicationState { // check that all storages of the application which are update by the sdk core, have the version corresponding to the // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); + + Try onBackupRestore(BoxIterator i); } diff --git a/sdk/src/main/java/com/horizen/storage/BoxBackupInterface.java b/sdk/src/main/java/com/horizen/storage/BoxBackupInterface.java new file mode 100644 index 0000000000..608a94ac36 --- /dev/null +++ b/sdk/src/main/java/com/horizen/storage/BoxBackupInterface.java @@ -0,0 +1,8 @@ +package com.horizen.storage; + + +import com.horizen.backup.BoxIterator; + +public interface BoxBackupInterface { + void backup(BoxIterator source, BackupStorage db) throws Exception; +} diff --git a/sdk/src/main/java/com/horizen/storage/Storage.java b/sdk/src/main/java/com/horizen/storage/Storage.java index 61ac503265..9c1dfbb510 100644 --- a/sdk/src/main/java/com/horizen/storage/Storage.java +++ b/sdk/src/main/java/com/horizen/storage/Storage.java @@ -29,4 +29,6 @@ void update(ByteArrayWrapper version, List>, Closeable +{ + /** + * Repositions the iterator so the key of the next BlockElement + * returned greater than or equal to the specified targetKey. + */ + void seek(byte[] key); + + /** + * Repositions the iterator so is is at the beginning of the Database. + */ + void seekToFirst(); + +} \ No newline at end of file diff --git a/sdk/src/main/java/com/horizen/utils/Utils.java b/sdk/src/main/java/com/horizen/utils/Utils.java index cd269e7b9c..1dc61319d7 100644 --- a/sdk/src/main/java/com/horizen/utils/Utils.java +++ b/sdk/src/main/java/com/horizen/utils/Utils.java @@ -1,8 +1,11 @@ package com.horizen.utils; +import scorex.crypto.hash.Blake2b256; + import java.math.BigInteger; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Random; public final class Utils { @@ -106,4 +109,16 @@ public static long readUint32BE(byte[] bytes, int offset) { ((bytes[offset + 2] & 0xffl) << 8) | (bytes[offset + 3] & 0xffl); } + + public static byte[] nextVersion() { + byte[] version = new byte[32]; + Random r = new Random(); + r.nextBytes(version); + return version; + } + + public static ByteArrayWrapper calculateKey(byte[] data) { + return new ByteArrayWrapper((byte[]) Blake2b256.hash(data)); + } + } diff --git a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java index ed473b1777..4a9bea2cdc 100644 --- a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java +++ b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java @@ -2,6 +2,7 @@ import java.util.List; +import com.horizen.backup.BoxIterator; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.box.Box; @@ -16,4 +17,6 @@ public interface ApplicationWallet { // check that all storages of the application which are update by the sdk core, have the version corresponding to the // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); + + void onBackupRestore(BoxIterator i); } diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index ee24a2a5c4..b07416e8ba 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -50,6 +50,7 @@ import com.horizen.transaction.mainchain.SidechainCreation import scorex.core.network.NetworkController.ReceivableMessages.ShutdownNetwork import java.util.concurrent.atomic.AtomicBoolean + import scala.util.{Failure, Success, Try} @@ -70,6 +71,7 @@ class SidechainApp @Inject() @Named("WalletForgingBoxesInfoStorage") val walletForgingBoxesInfoStorage: Storage, @Named("WalletCswDataStorage") val walletCswDataStorage: Storage, @Named("ConsensusStorage") val consensusStorage: Storage, + @Named("BackupStorage") val backUpStorage: Storage, @Named("CustomApiGroups") val customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") val rejectedApiPaths : JList[Pair[String, String]], @Named("ApplicationStopper") val applicationStopper : SidechainAppStopper @@ -259,6 +261,8 @@ class SidechainApp @Inject() sidechainSecretStorage.add(sidechainSecretsCompanion.parseBytes(BytesUtils.fromHexString(secretSchnorr))) } + protected val backupStorage = new BackupStorage(registerStorage(backUpStorage), sidechainBoxesCompanion) + override val nodeViewHolderRef: ActorRef = SidechainNodeViewHolderRef( sidechainSettings, sidechainHistoryStorage, @@ -271,6 +275,7 @@ class SidechainApp @Inject() sidechainWalletTransactionStorage, forgingBoxesMerklePathStorage, sidechainWalletCswDataStorage, + backupStorage, params, timeProvider, applicationWallet, @@ -337,6 +342,7 @@ class SidechainApp @Inject() var applicationApiRoutes : Seq[ApplicationApiRoute] = Seq[ApplicationApiRoute]() customApiGroups.asScala.foreach(apiRoute => applicationApiRoutes = applicationApiRoutes :+ ApplicationApiRoute(settings.restApi, apiRoute, nodeViewHolderRef)) + val boxIterator = backupStorage.getBoxIterator var coreApiRoutes: Seq[SidechainApiRoute] = Seq[SidechainApiRoute]( MainchainBlockApiRoute(settings.restApi, nodeViewHolderRef), SidechainBlockApiRoute(settings.restApi, nodeViewHolderRef, sidechainBlockActorRef, sidechainBlockForgerActorRef), @@ -344,7 +350,8 @@ class SidechainApp @Inject() SidechainTransactionApiRoute(settings.restApi, nodeViewHolderRef, sidechainTransactionActorRef, sidechainTransactionsCompanion, params), SidechainWalletApiRoute(settings.restApi, nodeViewHolderRef), SidechainSubmitterApiRoute(settings.restApi, certificateSubmitterRef, nodeViewHolderRef), - SidechainCswApiRoute(settings.restApi, nodeViewHolderRef, cswManager) + SidechainCswApiRoute(settings.restApi, nodeViewHolderRef, cswManager), + SidechainBackupApiRoute(settings.restApi, nodeViewHolderRef, boxIterator) ) val transactionSubmitProvider : TransactionSubmitProvider = new TransactionSubmitProviderImpl(sidechainTransactionActorRef) @@ -435,4 +442,5 @@ class SidechainApp @Inject() def getSecretSubmitProvider: SecretSubmitProvider = secretSubmitProvider actorSystem.eventStream.publish(SidechainAppEvents.SidechainApplicationStart) + } diff --git a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala index a53757a089..b10fd31f70 100644 --- a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala +++ b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala @@ -2,7 +2,6 @@ package com.horizen import java.lang.{Byte => JByte} import java.util.{HashMap => JHashMap, List => JList} - import com.google.inject.name.Named import com.google.inject.Provides import com.horizen.api.http.ApplicationApiGroup @@ -10,7 +9,7 @@ import com.horizen.box.BoxSerializer import com.horizen.helper.{NodeViewHelper, NodeViewHelperImpl, SecretSubmitHelper, SecretSubmitHelperImpl, TransactionSubmitHelper, TransactionSubmitHelperImpl} import com.horizen.secret.SecretSerializer import com.horizen.state.ApplicationState -import com.horizen.storage.Storage +import com.horizen.storage.{Storage} import com.horizen.transaction.TransactionSerializer import com.horizen.utils.Pair import com.horizen.wallet.ApplicationWallet @@ -53,11 +52,11 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { @Named("WalletForgingBoxesInfoStorage") walletForgingBoxesInfoStorage: Storage, @Named("WalletCswDataStorage") walletCswDataStorage: Storage, @Named("ConsensusStorage") consensusStorage: Storage, + @Named("BackupStorage") backUpStorage: Storage, @Named("CustomApiGroups") customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") rejectedApiPaths : JList[Pair[String, String]], @Named("ApplicationStopper") applicationStopper : SidechainAppStopper - - ): SidechainApp = { + ): SidechainApp = { synchronized { if (app == null) { app = new SidechainApp( @@ -77,6 +76,7 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { walletForgingBoxesInfoStorage, walletCswDataStorage, consensusStorage, + backUpStorage, customApiGroups, rejectedApiPaths, applicationStopper diff --git a/sdk/src/main/scala/com/horizen/SidechainBackup.scala b/sdk/src/main/scala/com/horizen/SidechainBackup.scala new file mode 100644 index 0000000000..91f5076b87 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/SidechainBackup.scala @@ -0,0 +1,69 @@ +package com.horizen + +import com.google.inject.Inject +import com.google.inject.name.Named +import com.horizen.backup.BoxIterator +import com.horizen.box.BoxSerializer +import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.storage._ +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter +import com.horizen.utils.{ByteArrayWrapper, BytesUtils} +import org.apache.commons.io.FileUtils +import scorex.util.ScorexLogging + +import java.io._ +import java.lang.{Byte => JByte} +import java.util.{HashMap => JHashMap} +import scala.util.{Failure, Success} + +class SidechainBackup @Inject() + (@Named("CustomBoxSerializers") val customBoxSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]], + @Named("BackupStorage") val backUpStorage: Storage, + @Named("BackUpper") val backUpper : BoxBackupInterface + ) extends ScorexLogging + { + protected val sidechainBoxesCompanion: SidechainBoxesCompanion = SidechainBoxesCompanion(customBoxSerializers) + protected val backupStorage = new BackupStorage(backUpStorage, sidechainBoxesCompanion) + + + def createBackup(stateStoragePath: String, sidechainBlockIdToRollback: String, copyStateStorage: Boolean): Unit = { + var storagePath = stateStoragePath + + if (copyStateStorage) { + val stateStorage: File = new File(stateStoragePath) + val stateStorageBackup: File = new File(stateStoragePath+"_copy_for_backup") + + try { + FileUtils.copyDirectory(stateStorage, stateStorageBackup) + storagePath = stateStoragePath+"_copy_for_backup" + } catch { + case t: Throwable => + log.error("Error during the copy of the StateStorage: ",t.getMessage) + throw new RuntimeException("Error during the copy of the StateStorage: "+t.getMessage) + } + } + val storage = new VersionedLevelDbStorageAdapter(new File(storagePath)) + val sidechainStateStorage = new SidechainStateStorage(storage, sidechainBoxesCompanion) + sidechainStateStorage.rollback(new ByteArrayWrapper(BytesUtils.fromHexString(sidechainBlockIdToRollback))) match { + case Success(stateStorage) => + log.info(s"Rollback of the SidechainStateStorage completed successfully!") + + //Take an iterator on the sidechainStateStorage + val stateIterator: StorageIterator = stateStorage.getIterator + stateIterator.seekToFirst() + + //Perform the backup in the application level + try { + backUpper.backup(new BoxIterator(stateIterator, sidechainBoxesCompanion), backupStorage) + storage.close() + } catch { + case t: Throwable => + storage.close() + log.error("Error during the Backup generation: ",t.getMessage) + throw new RuntimeException("Error during the Backup generation: "+t.getMessage) + } + case Failure(e) => + log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...", e.getMessage) + } + } + } diff --git a/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala b/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala index 60c5be25b0..8a6f90bb2b 100644 --- a/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala +++ b/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala @@ -7,7 +7,6 @@ import com.horizen.chain.FeePaymentsInfo import com.horizen.consensus._ import com.horizen.node.SidechainNodeView import com.horizen.params.NetworkParams -import com.horizen.proposition.{PublicKey25519Proposition, VrfPublicKey} import com.horizen.state.ApplicationState import com.horizen.storage._ import com.horizen.utils.BytesUtils @@ -39,6 +38,7 @@ class SidechainNodeViewHolder(sidechainSettings: SidechainSettings, walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -226,13 +226,13 @@ class SidechainNodeViewHolder(sidechainSettings: SidechainSettings, override protected def genesisState: (HIS, MS, VL, MP) = { val result = for { - state <- SidechainState.createGenesisState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, applicationState, genesisBlock) + state <- SidechainState.createGenesisState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, backupStorage, params, applicationState, genesisBlock) (_: ModifierId, consensusEpochInfo: ConsensusEpochInfo) <- Success(state.getCurrentConsensusEpochInfo) withdrawalEpochNumber: Int <- Success(state.getWithdrawalEpochInfo.epoch) wallet <- SidechainWallet.createGenesisWallet(sidechainSettings.wallet.seed.getBytes, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, applicationWallet, + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, applicationWallet, genesisBlock, withdrawalEpochNumber, consensusEpochInfo) history <- SidechainHistory.createGenesisHistory(historyStorage, consensusDataStorage, params, genesisBlock, semanticBlockValidators(params), @@ -493,13 +493,14 @@ object SidechainNodeViewHolderRef { walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, applicationState: ApplicationState, genesisBlock: SidechainBlock): Props = Props(new SidechainNodeViewHolder(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) def apply(sidechainSettings: SidechainSettings, historyStorage: SidechainHistoryStorage, @@ -512,6 +513,7 @@ object SidechainNodeViewHolderRef { walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -519,7 +521,7 @@ object SidechainNodeViewHolderRef { genesisBlock: SidechainBlock) (implicit system: ActorSystem): ActorRef = system.actorOf(props(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) def apply(name: String, sidechainSettings: SidechainSettings, @@ -533,6 +535,7 @@ object SidechainNodeViewHolderRef { walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -540,5 +543,5 @@ object SidechainNodeViewHolderRef { genesisBlock: SidechainBlock) (implicit system: ActorSystem): ActorRef = system.actorOf(props(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock), name) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock), name) } diff --git a/sdk/src/main/scala/com/horizen/SidechainState.scala b/sdk/src/main/scala/com/horizen/SidechainState.scala index 7de6c942ec..b4aa0b24a3 100644 --- a/sdk/src/main/scala/com/horizen/SidechainState.scala +++ b/sdk/src/main/scala/com/horizen/SidechainState.scala @@ -1,6 +1,7 @@ package com.horizen import com.google.common.primitives.{Bytes, Ints} +import com.horizen.backup.BoxIterator import java.io.File import java.util @@ -12,7 +13,7 @@ import com.horizen.node.NodeState import com.horizen.params.NetworkParams import com.horizen.proposition.{Proposition, PublicKey25519Proposition, VrfPublicKey} import com.horizen.state.ApplicationState -import com.horizen.storage.{SidechainStateForgerBoxStorage, SidechainStateStorage, SidechainStateUtxoMerkleTreeStorage} +import com.horizen.storage.{BackupStorage, SidechainStateForgerBoxStorage, SidechainStateStorage, SidechainStateUtxoMerkleTreeStorage} import com.horizen.transaction.MC2SCAggregatedTransaction import com.horizen.utils.{BlockFeeInfo, ByteArrayWrapper, BytesUtils, FeePaymentsUtils, MerkleTree, TimeToEpochUtils, WithdrawalEpochInfo, WithdrawalEpochUtils} import scorex.core._ @@ -22,6 +23,7 @@ import scorex.util.{ModifierId, ScorexLogging, bytesToId} import java.math.{BigDecimal, MathContext} import com.horizen.box.data.ZenBoxData +import com.horizen.companion.SidechainBoxesCompanion import com.horizen.cryptolibprovider.CryptoLibProvider import scala.collection.JavaConverters._ @@ -532,6 +534,18 @@ class SidechainState private[horizen] (stateStorage: SidechainStateStorage, new ZenBox(data, nonce) }.filter(box => box.value() > 0) } + + def restoreBackup(backupStorageBoxIterator: BoxIterator, lastVersion: Array[Byte]): Try[SidechainState] = Try { + stateStorage.restoreBackup(backupStorageBoxIterator, lastVersion) + backupStorageBoxIterator.seekToFirst() + applicationState.onBackupRestore(backupStorageBoxIterator) match { + case Success(_) => + this + case Failure(e) => + log.error("Error during the backup restore inside the SidechainState", e) + throw e + } + } } object SidechainState @@ -575,14 +589,18 @@ object SidechainState private[horizen] def createGenesisState(stateStorage: SidechainStateStorage, forgerBoxStorage: SidechainStateForgerBoxStorage, utxoMerkleTreeStorage: SidechainStateUtxoMerkleTreeStorage, + backupStorage: BackupStorage, params: NetworkParams, applicationState: ApplicationState, genesisBlock: SidechainBlock): Try[SidechainState] = Try { - if (stateStorage.isEmpty) - new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) - .applyModifier(genesisBlock).get - else + if (stateStorage.isEmpty) { + var state = new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) + if (!backupStorage.isEmpty) { + state = state.restoreBackup(backupStorage.getBoxIterator, versionToBytes(idToVersion(genesisBlock.parentId))).get + } + state.applyModifier(genesisBlock).get + } else throw new RuntimeException("State storage is not empty!") } diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index 6220db1fbe..7eb7972848 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -1,7 +1,11 @@ package com.horizen -import java.{lang, util} +import java.lang import java.util.{List => JList, Optional => JOptional} +import java.util.{ArrayList => JArrayList} + +import com.horizen.backup.BoxIterator + import com.horizen.block.{MainchainBlockReferenceData, SidechainBlock} import com.horizen.box.{Box, CoinsBox, ForgerBox, ZenBox} import com.horizen.consensus.{ConsensusEpochInfo, ConsensusEpochNumber, ForgingStakeInfo} @@ -19,6 +23,11 @@ import com.horizen.utils._ import scorex.util.{ModifierId, ScorexLogging} import scala.util.{Failure, Success, Try} +import scala.util.{Try} +import scorex.core.block.Block.Timestamp +import scorex.util.ModifierId + +import scala.util.Try import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.language.postfixOps @@ -174,6 +183,40 @@ class SidechainWallet private[horizen] (seed: Array[Byte], this } + /*** + * This function is called at blockchain bootstrap time and preload the SidechainWallletBoxStorage with the boxes taken from the backup storage + * @param backupStorageIterator: iterator on the backup storage + * @param sidechainBoxesCompanion + */ + def scanBackUp(backupStorageBoxIterator: BoxIterator, genesisBlockTimestamp: Timestamp): Try[SidechainWallet] = Try{ + val pubKeys = publicKeys() + val walletBoxes = new JArrayList[WalletBox]() + val removeList = new JArrayList[Array[Byte]]() + var nBoxes = 0 + + var optionalBox = backupStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + val box: SCB = optionalBox.get.getBox + if (pubKeys.contains(box.proposition())) { + walletBoxes.add(new WalletBox(box, genesisBlockTimestamp)) + nBoxes += 1 + if (nBoxes == leveldb.Constants.BatchSize) { + walletBoxStorage.update(new ByteArrayWrapper(Utils.nextVersion), walletBoxes.asScala.toList, removeList.asScala.toList).get + walletBoxes.clear() + nBoxes = 0 + } + } + optionalBox = backupStorageBoxIterator.nextBox + } + if (nBoxes > 0) { + walletBoxStorage.update(new ByteArrayWrapper(Utils.nextVersion), walletBoxes.asScala.toList, removeList.asScala.toList).get + } + backupStorageBoxIterator.seekToFirst + applicationWallet.onBackupRestore(backupStorageBoxIterator) + + this + } + private[horizen] def calculateUtxoCswData(view: UtxoMerkleTreeView): Seq[CswData] = { boxes().filter(wb => wb.box.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]).map(wb => { val box = wb.box @@ -409,6 +452,7 @@ object SidechainWallet walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, params: NetworkParams, applicationWallet: ApplicationWallet, genesisBlock: SidechainBlock, @@ -417,8 +461,9 @@ object SidechainWallet ) : Try[SidechainWallet] = Try { if (walletBoxStorage.isEmpty) { - val genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, + var genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, idToVersion(genesisBlock.parentId), applicationWallet) + genesisWallet = genesisWallet.scanBackUp(backupStorage.getBoxIterator, genesisBlock.timestamp).get genesisWallet.scanPersistent(genesisBlock, withdrawalEpochNumber, Seq(), None).applyConsensusEpochInfo(consensusEpochInfo) } else diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala index bf19eb24cb..ec4c8cdcc8 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala @@ -5,6 +5,9 @@ import com.horizen.node.SidechainNodeView import scorex.core.api.http.{ApiDirectives, ApiRoute} import akka.pattern.ask import akka.http.scaladsl.server.Route +import com.horizen.{SidechainHistory, SidechainMemoryPool, SidechainState, SidechainWallet} +import scorex.core.NodeViewHolder.CurrentView +import scorex.core.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import scala.concurrent.{Await, ExecutionContext, Future} @@ -52,5 +55,14 @@ trait SidechainApiRoute extends ApiRoute with ApiDirectives { .mapTo[SidechainNodeView] } + type View = CurrentView[SidechainHistory, SidechainState, SidechainWallet, SidechainMemoryPool] + + def withView(f: View => Route): Route = onSuccess(sidechainViewAsync())(f) + + protected def sidechainViewAsync(): Future[View] = { + def f(v: View) = v + (sidechainNodeViewHolderRef ? GetDataFromCurrentView(f)).mapTo[View] + } + } diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala new file mode 100644 index 0000000000..5d94a3ac61 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala @@ -0,0 +1,106 @@ +package com.horizen.api.http + +import akka.actor.{ActorRef, ActorRefFactory} +import akka.http.scaladsl.server.Route +import com.fasterxml.jackson.annotation.JsonView +import com.horizen.api.http.SidechainBackupRestScheme.RespSidechainBlockIdForBackup +import com.horizen.serialization.Views +import scorex.core.settings.RESTApiSettings +import com.horizen.api.http.SidechainBackupErrorResponse.{ErrorRetrievingSidechainBlockIdForBackup, GenericBackupApiError} +import com.horizen.utils.BytesUtils + +import java.util.{Optional => JOptional} +import scala.concurrent.ExecutionContext + +import com.horizen.api.http.SidechainBackupRestScheme.{ReqGetInitialBoxes, RespGetInitialBoxes} +import com.horizen.box.Box +import com.horizen.proposition.Proposition + +import scala.util.{Failure, Success, Try} +import com.horizen.api.http.JacksonSupport._ +import com.horizen.backup.BoxIterator +import scala.collection.JavaConverters._ + +case class SidechainBackupApiRoute(override val settings: RESTApiSettings, + sidechainNodeViewHolderRef: ActorRef, + boxIterator: BoxIterator) + (implicit val context: ActorRefFactory, override val ec: ExecutionContext) extends SidechainApiRoute { + override val route: Route = pathPrefix("backup") { + getSidechainBlockIdForBackup ~ getRestoredBoxes + } + + /*** + * Retrieve the SidechainBlockId needed to rollback the SidechainStateStorage for the backup. + * It's calculated by the following formula: + * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochLength -1 + */ + def getSidechainBlockIdForBackup: Route = (post & path("getSidechainBlockIdForBackup")) { + withView { nodeView => + try { + val withdrawalEpochLength = nodeView.state.params.withdrawalEpochLength + val currentEpoch = nodeView.state.getWithdrawalEpochInfo.epoch + val genesisMcBlockHeight = nodeView.history.getMainchainCreationBlockHeight + val blockHeightToRollback = genesisMcBlockHeight + (currentEpoch -2) * withdrawalEpochLength - 1 + val mainchainBlockReferenceInfo = nodeView.history.getMainchainBlockReferenceInfoByMainchainBlockHeight(blockHeightToRollback).get() + ApiResponseUtil.toResponse(RespSidechainBlockIdForBackup(BytesUtils.toHexString(mainchainBlockReferenceInfo.getMainchainReferenceDataSidechainBlockId))) + } catch { + case t: Throwable => + log.error("Failed to retrieve getSidechainBlockIdForBackup.", t.getMessage) + ApiResponseUtil.toResponse(ErrorRetrievingSidechainBlockIdForBackup("Unexpected error during retrieving the sidechain block id to rollback.", JOptional.of(t))) + } + } + } + + + /** + * Return the initial boxes restored in a paginated way. + */ + def getRestoredBoxes: Route = (post & path("getRestoredBoxes")) { + entity(as[ReqGetInitialBoxes]) { body => + def getBoxId: JOptional[Array[Byte]] = body.lastBoxId match { + case Some(boxId) => + if (boxId.equals("")) { + JOptional.empty() + } else { + JOptional.of(BytesUtils.fromHexString(boxId)) + } + case None => + JOptional.empty() + } + + Try { + boxIterator.getNextBoxes(body.numberOfElements, getBoxId) + } match { + case Success(boxes) => + ApiResponseUtil.toResponse(RespGetInitialBoxes(boxes.asScala.toList)) + case Failure(e) => + ApiResponseUtil.toResponse(GenericBackupApiError("GenericBackupApiError", JOptional.of(e))) + } + } + } + +} + +object SidechainBackupRestScheme { + final val MAX_NUMBER_OF_BOX_REQUEST = 100 + + @JsonView(Array(classOf[Views.Default])) + private[api] case class RespSidechainBlockIdForBackup(blockId: String) extends SuccessResponse + + @JsonView(Array(classOf[Views.Default])) + private[api] case class ReqGetInitialBoxes(numberOfElements: Int, lastBoxId: Option[String]) { + require(numberOfElements > 0, s"Invalid numberOfElements $numberOfElements. It should be > 0") + require(numberOfElements <= MAX_NUMBER_OF_BOX_REQUEST, s"Invalid numberOfElements $numberOfElements. It should be <= $MAX_NUMBER_OF_BOX_REQUEST") + } + @JsonView(Array(classOf[Views.Default])) + private[api] case class RespGetInitialBoxes(boxes: List[Box[Proposition]]) extends SuccessResponse +} + +object SidechainBackupErrorResponse { + case class ErrorRetrievingSidechainBlockIdForBackup(description: String, exception: JOptional[Throwable]) extends ErrorResponse { + override val code: String = "0801" + } + case class GenericBackupApiError(description: String, exception: JOptional[Throwable]) extends ErrorResponse { + override val code: String = "0802" + } +} \ No newline at end of file diff --git a/sdk/src/main/scala/com/horizen/backup/BackupBox.java b/sdk/src/main/scala/com/horizen/backup/BackupBox.java new file mode 100644 index 0000000000..6f379e5b17 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/backup/BackupBox.java @@ -0,0 +1,33 @@ +package com.horizen.backup; + +import com.horizen.box.Box; +import com.horizen.proposition.Proposition; + +public class BackupBox { + private Box box; + private byte[] boxKey; + private byte[] boxValue; + + public BackupBox(Box box, byte[] boxKey, byte[] boxValue) { + this.box = box; + this.boxKey = boxKey; + this.boxValue = boxValue; + } + + public byte getBoxTypeId() { + return box.boxTypeId(); + } + + public Box getBox() { + return box; + } + + public byte[] getBoxKey() { + return boxKey; + } + + public byte[] getBoxValue() { + return boxValue; + } + +} diff --git a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java new file mode 100644 index 0000000000..432cd1b721 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java @@ -0,0 +1,79 @@ +package com.horizen.backup; + +import com.horizen.box.Box; +import com.horizen.box.CoinsBox; +import com.horizen.companion.SidechainBoxesCompanion; +import com.horizen.proposition.Proposition; +import com.horizen.storage.StorageIterator; +import com.horizen.utils.Utils; +import scala.util.Try; +import scorex.util.serialization.VLQByteBufferReader; + +import java.nio.ByteBuffer; +import java.util.*; + +public class BoxIterator { + private final StorageIterator iterator; + private final SidechainBoxesCompanion sidechainBoxesCompanion; + + public BoxIterator(StorageIterator iterator, SidechainBoxesCompanion sidechainBoxesCompanion) { + this.iterator = iterator; + this.sidechainBoxesCompanion = sidechainBoxesCompanion; + this.iterator.seekToFirst(); + } + + public void seekToFirst() { + this.iterator.seekToFirst(); + } + + public void seekIterator(byte[] key) { + iterator.seek(key); + } + + public Optional nextBox(boolean ignoreCoinBox) throws RuntimeException { + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + VLQByteBufferReader reader = new VLQByteBufferReader(ByteBuffer.wrap(entry.getValue())); + Try> tryBox = sidechainBoxesCompanion.parseTry(reader); + + if (tryBox.isSuccess() && reader.remaining() == 0) { + Box currBox = tryBox.get(); + if (verifyBox(entry.getKey(), currBox.id())) { + if (!(currBox instanceof CoinsBox)) { + return Optional.of(new BackupBox(currBox, entry.getKey(), entry.getValue())); + } + else { + if (!ignoreCoinBox) + throw new RuntimeException("Coin boxes are not eligible to be restored!"); + } + } + } + } + return Optional.empty(); + } + + public Optional nextBox() throws RuntimeException { + return nextBox(false); + } + + public List> getNextBoxes(int nElement, Optional keyToSeek) { + if (keyToSeek.isPresent()) { + this.seekIterator(Utils.calculateKey(keyToSeek.get()).data()); + this.nextBox(); + } else { + this.seekToFirst(); + } + List> boxes = new ArrayList<>(); + Optional nextBox = this.nextBox(); + while(boxes.size() < nElement && nextBox.isPresent()) { + boxes.add(nextBox.get().getBox()); + nextBox = this.nextBox(); + } + return boxes; + } + + private boolean verifyBox(byte[] recordId, byte[] boxId) { + return Arrays.equals(recordId, Utils.calculateKey(boxId).data()); + } + +} diff --git a/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala b/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala index b52a19a699..3b4d57eb95 100644 --- a/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala +++ b/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala @@ -3,11 +3,11 @@ package com.horizen.consensus import java.util.{ArrayList => JArrayList} import com.horizen.storage.{SidechainStorageInfo, Storage} import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} +import com.horizen.utils.Utils import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging import scala.compat.java8.OptionConverters._ -import scala.util.Random class ConsensusDataStorage(consensusEpochInfoStorage: Storage) extends ScorexLogging @@ -46,12 +46,6 @@ class ConsensusDataStorage(consensusEpochInfoStorage: Storage) .map(byteArray => NonceConsensusEpochInfoSerializer.parseBytes(byteArray.data)) } - private def nextVersion: Array[Byte] = { - val version = new Array[Byte](32) - Random.nextBytes(version) - version - } - private def stakeEpochInfoKey(epochId: ConsensusEpochId): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256(s"stake$epochId")) private def nonceEpochInfoKey(epochId: ConsensusEpochId): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256(s"nonce$epochId")) @@ -60,7 +54,7 @@ class ConsensusDataStorage(consensusEpochInfoStorage: Storage) val listForUpdate = new JArrayList[JPair[ByteArrayWrapper, ByteArrayWrapper]]() val addedData = new JPair(key, new ByteArrayWrapper(value)) listForUpdate.add(addedData) - val version = new ByteArrayWrapper(nextVersion) + val version = new ByteArrayWrapper(Utils.nextVersion) consensusEpochInfoStorage.update(version, listForUpdate, java.util.Collections.emptyList()) log.debug("Consensus data storage updated with version: " + version) } diff --git a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala new file mode 100644 index 0000000000..47aad84a3c --- /dev/null +++ b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala @@ -0,0 +1,37 @@ +package com.horizen.storage + +import com.horizen.backup.BoxIterator +import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} + +import scala.util.Try +import java.util.{ArrayList => JArrayList} + +class BackupStorage (storage : Storage, val sidechainBoxesCompanion: SidechainBoxesCompanion) { + // Version - random number + // Key - byte array box Id + // No remove operation + + require(storage != null, "Storage must be NOT NULL.") + require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") + + def update (version : ByteArrayWrapper, boxToSaveList : java.util.List[JPair[ByteArrayWrapper,ByteArrayWrapper]]) : Try[BackupStorage] = Try { + require(boxToSaveList != null, "List of WalletBoxes to add/update must be NOT NULL.") + require(!boxToSaveList.contains(null), "WalletBox to add/update must be NOT NULL.") + require(!boxToSaveList.isEmpty, "List of WalletBoxes to add/update must be NOT EMPTY.") + + val removeList = new JArrayList[ByteArrayWrapper]() + + storage.update(version, + boxToSaveList, + removeList) + + this + } + + def getBoxIterator: BoxIterator = new BoxIterator(storage.getIterator, sidechainBoxesCompanion) + + def isEmpty: Boolean = storage.isEmpty + + def close: Unit = storage.close() +} diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala index 7f1af78761..2cd221a1c8 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala @@ -14,7 +14,7 @@ import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} import scala.collection.mutable.ArrayBuffer import scala.compat.java8.OptionConverters._ -import scala.util.{Failure, Random, Success, Try} +import scala.util.{Failure, Success, Try} trait SidechainBlockInfoProvider { @@ -64,12 +64,6 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: private def feePaymentsInfoKey(blockId: ModifierId): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256(s"feePaymentsInfo$blockId")) - private def nextVersion: Array[Byte] = { - val version = new Array[Byte](32) - Random.nextBytes(version) - version - } - def height: Int = activeChain.height def heightOf(blockId: ModifierId): Option[Int] = { @@ -250,7 +244,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: toUpdate.add(new JPair(new ByteArrayWrapper(idToBytes(block.id)), new ByteArrayWrapper(block.bytes))) storage.update( - new ByteArrayWrapper(nextVersion), + new ByteArrayWrapper(Utils.nextVersion), toUpdate, new JArrayList[ByteArrayWrapper]()) @@ -259,7 +253,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: def updateFeePaymentsInfo(blockId: ModifierId, feePaymentsInfo: FeePaymentsInfo): Try[SidechainHistoryStorage] = Try { storage.update( - nextVersion, + Utils.nextVersion, java.util.Arrays.asList(new JPair(new ByteArrayWrapper(feePaymentsInfoKey(blockId)), new ByteArrayWrapper(feePaymentsInfo.bytes))), new JArrayList[ByteArrayWrapper]() ) @@ -283,7 +277,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: val blockInfo = oldInfo.copy(semanticValidity = status) storage.update( - new ByteArrayWrapper(nextVersion), + new ByteArrayWrapper(Utils.nextVersion), java.util.Arrays.asList(new JPair(new ByteArrayWrapper(blockInfoKey(block.id)), new ByteArrayWrapper(blockInfo.bytes))), new JArrayList() ) @@ -292,7 +286,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: def setAsBestBlock(block: SidechainBlock, blockInfo: SidechainBlockInfo): Try[SidechainHistoryStorage] = Try { storage.update( - new ByteArrayWrapper(nextVersion), + new ByteArrayWrapper(Utils.nextVersion), java.util.Arrays.asList(new JPair(bestBlockIdKey, new ByteArrayWrapper(idToBytes(block.id)))), new JArrayList() ) diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala index 10ea9980d7..677412948f 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala @@ -1,12 +1,11 @@ package com.horizen.storage +import java.util.{ArrayList => JArrayList} import com.horizen.SidechainTypes import com.horizen.companion.SidechainSecretsCompanion -import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.{ByteArrayWrapper, Utils, Pair => JPair} import scorex.util.ScorexLogging -import java.util.{ArrayList => JArrayList} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.compat.java8.OptionConverters.RichOptionalGeneric @@ -27,7 +26,7 @@ class SidechainSecretStorage(storage: Storage, sidechainSecretsCompanion: Sidech loadSecrets() - def calculateKey(proposition: SidechainTypes#SCP): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256.hash(proposition.bytes)) + def calculateKey(proposition: SidechainTypes#SCP): ByteArrayWrapper = Utils.calculateKey(proposition.bytes) private def loadSecrets(): Unit = { secrets.clear() diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala index 96845a665f..928620633e 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala @@ -1,13 +1,10 @@ package com.horizen.storage import com.horizen.SidechainTypes -import com.horizen.utils.ByteArrayWrapper -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.{ByteArrayWrapper, Utils, Pair => JPair} import scorex.util.ScorexLogging import java.util.{ArrayList => JArrayList} - import com.horizen.box.{ForgerBox, ForgerBoxSerializer} -import com.horizen.utils.{Pair => JPair} import scala.compat.java8.OptionConverters._ import scala.collection.JavaConverters._ @@ -25,12 +22,9 @@ class SidechainStateForgerBoxStorage(storage: Storage) private val forgerBoxSerializer: ForgerBoxSerializer = ForgerBoxSerializer.getSerializer - def calculateKey(boxId: Array[Byte]): ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } def getForgerBox(boxId: Array[Byte]): Option[ForgerBox] = { - storage.get(calculateKey(boxId)).asScala match { + storage.get(Utils.calculateKey(boxId)).asScala match { case Some(baw) => forgerBoxSerializer.parseBytesTry(baw.data) match { case Success(box) => Option(box) @@ -59,10 +53,10 @@ class SidechainStateForgerBoxStorage(storage: Storage) // Update boxes data for (id <- boxIdsRemoveSet) - removeList.add(calculateKey(id.data)) + removeList.add(Utils.calculateKey(id.data)) for (box <- forgerBoxUpdateSeq) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(box.id()), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(box.id()), new ByteArrayWrapper(forgerBoxSerializer.toBytes(box)))) storage.update(version, updateList, removeList) diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index f2c8b856c0..7c05dc20fe 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -3,12 +3,12 @@ package com.horizen.storage import com.google.common.primitives.{Bytes, Ints} import com.horizen.SidechainTypes +import com.horizen.backup.{BoxIterator} import com.horizen.block.{WithdrawalEpochCertificate, WithdrawalEpochCertificateSerializer} import com.horizen.box.{WithdrawalRequestBox, WithdrawalRequestBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ import com.horizen.utils.{ByteArrayWrapper, ListSerializer, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer, Pair => JPair, _} -import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging import java.util.{ArrayList => JArrayList} @@ -28,45 +28,42 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain require(storage != null, "Storage must be NOT NULL.") require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") - private[horizen] val withdrawalEpochInformationKey = calculateKey("withdrawalEpochInformation".getBytes) + private[horizen] val withdrawalEpochInformationKey = Utils.calculateKey("withdrawalEpochInformation".getBytes) private val withdrawalRequestSerializer = new ListSerializer[WithdrawalRequestBox](WithdrawalRequestBoxSerializer.getSerializer) - private[horizen] val consensusEpochKey = calculateKey("consensusEpoch".getBytes) + private[horizen] val consensusEpochKey = Utils.calculateKey("consensusEpoch".getBytes) - private[horizen] val ceasingStateKey = calculateKey("ceasingStateKey".getBytes) + private[horizen] val ceasingStateKey = Utils.calculateKey("ceasingStateKey".getBytes) private val undefinedWithdrawalEpochCounter: Int = -1 private[horizen] def getWithdrawalEpochCounterKey(withdrawalEpoch: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) + Utils.calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) } private[horizen] def getWithdrawalRequestsKey(withdrawalEpoch: Int, counter: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) + Utils.calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) } private[horizen] def getTopQualityCertificateKey(referencedWithdrawalEpoch: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("topQualityCertificate".getBytes, Ints.toByteArray(referencedWithdrawalEpoch))) + Utils.calculateKey(Bytes.concat("topQualityCertificate".getBytes, Ints.toByteArray(referencedWithdrawalEpoch))) } private val undefinedBlockFeeInfoCounter: Int = -1 private[horizen] def getBlockFeeInfoCounterKey(withdrawalEpochNumber: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("blockFeeInfoCounter".getBytes, Ints.toByteArray(withdrawalEpochNumber))) + Utils.calculateKey(Bytes.concat("blockFeeInfoCounter".getBytes, Ints.toByteArray(withdrawalEpochNumber))) } private[horizen] def getBlockFeeInfoKey(withdrawalEpochNumber: Int, counter: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("blockFeeInfo".getBytes, Ints.toByteArray(withdrawalEpochNumber), Ints.toByteArray(counter))) + Utils.calculateKey(Bytes.concat("blockFeeInfo".getBytes, Ints.toByteArray(withdrawalEpochNumber), Ints.toByteArray(counter))) } private[horizen] def getUtxoMerkleTreeRootKey(withdrawalEpochNumber: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("utxoMerkleTreeRoot".getBytes, Ints.toByteArray(withdrawalEpochNumber))) + Utils.calculateKey(Bytes.concat("utxoMerkleTreeRoot".getBytes, Ints.toByteArray(withdrawalEpochNumber))) } - def calculateKey(boxId : Array[Byte]) : ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } def getBox(boxId : Array[Byte]) : Option[SidechainTypes#SCB] = { - storage.get(calculateKey(boxId)) match { + storage.get(Utils.calculateKey(boxId)) match { case v if v.isPresent => sidechainBoxesCompanion.parseBytesTry(v.get().data) match { case Success(box) => Option(box) @@ -207,10 +204,10 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain // Update boxes data for (r <- boxIdsRemoveSet) - removeList.add(calculateKey(r.data)) + removeList.add(Utils.calculateKey(r.data)) for (b <- boxUpdateList) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(b.id()), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(b.id()), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)))) // Update Withdrawal epoch related data @@ -307,4 +304,37 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain def isEmpty: Boolean = storage.isEmpty + def getIterator: StorageIterator = storage.getIterator + + /** + * This function restores the unspent boxes that come from a ceased sidechain by saving + * them into the SidechainStateStorage + * + * @param backupStorage: storage containing the boxes saved from the ceased sidechain + */ + def restoreBackup(backupStorageBoxIterator: BoxIterator, lastVersion: Array[Byte]): Unit = { + val removeList = new JArrayList[ByteArrayWrapper]() + val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() + val lastVersionWrapper = new ByteArrayWrapper(lastVersion) + + var optionalBox = backupStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + val box = optionalBox.get.getBox + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(box.id()), + new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box)))) + log.info("Restore Box id "+box.boxTypeId()) + optionalBox = backupStorageBoxIterator.nextBox + if (updateList.size() == leveldb.Constants.BatchSize) { + if (optionalBox.isPresent) + storage.update(new ByteArrayWrapper(Utils.nextVersion),updateList, removeList) + else + storage.update(lastVersionWrapper,updateList, removeList) + updateList.clear() + } + } + + if (updateList.size() != 0) + storage.update(lastVersionWrapper,updateList, removeList) + log.info("SidechainStateStorage restore completed successfully!") + } } diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala index 5db16d8c23..a5c6540a6f 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala @@ -4,7 +4,7 @@ import com.horizen.SidechainTypes import com.horizen.cryptolibprovider.{CryptoLibProvider, InMemorySparseMerkleTreeWrapper} import com.horizen.librustsidechains.FieldElement import com.horizen.utils.{ByteArrayWrapper, UtxoMerkleTreeLeafInfo, UtxoMerkleTreeLeafInfoSerializer, Pair => JPair} -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.Utils import scorex.util.ScorexLogging import java.util.{List => JList} @@ -43,12 +43,9 @@ class SidechainStateUtxoMerkleTreeStorage(storage: Storage) CryptoLibProvider.cswCircuitFunctions.getUtxoMerkleTreeLeaf(box) } - private[horizen] def calculateKey(boxId: Array[Byte]): ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } def getLeafInfo(boxId: Array[Byte]): Option[UtxoMerkleTreeLeafInfo] = { - storage.get(calculateKey(boxId)) match { + storage.get(Utils.calculateKey(boxId)) match { case v if v.isPresent => UtxoMerkleTreeLeafInfoSerializer.parseBytesTry(v.get().data) match { case Success(leafInfo) => Option(leafInfo) @@ -78,7 +75,7 @@ class SidechainStateUtxoMerkleTreeStorage(storage: Storage) require(boxesToAppend != null, "List of boxes to add must be NOT NULL. Use empty List instead.") require(boxesToRemoveSet != null, "List of Box IDs to remove must be NOT NULL. Use empty List instead.") - val removeList: JList[ByteArrayWrapper] = boxesToRemoveSet.map(id => calculateKey(id.data)).toList.asJava + val removeList: JList[ByteArrayWrapper] = boxesToRemoveSet.map(id => Utils.calculateKey(id.data)).toList.asJava // Remove leaves from inmemory tree require(merkleTreeWrapper.removeLeaves(boxesToRemoveSet.flatMap(id => { @@ -91,7 +88,7 @@ class SidechainStateUtxoMerkleTreeStorage(storage: Storage) throw new IllegalStateException("Not enough empty leaves in the UTXOMerkleTree.") } - val leavesToAppend = boxesToAppend.map(box => (calculateKey(box.id()), calculateLeaf(box))).zip(newLeavesPositions) + val leavesToAppend = boxesToAppend.map(box => (Utils.calculateKey(box.id()), calculateLeaf(box))).zip(newLeavesPositions) // Add leaves to inmemory tree require(merkleTreeWrapper.addLeaves(leavesToAppend.map { diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala index 45c918a6c8..a2af10fff8 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala @@ -1,14 +1,18 @@ package com.horizen.storage -import com.horizen.box.Box + +import java.util.{ArrayList => JArrayList} +import com.horizen.utils.{Pair => JPair} +import com.horizen.utils.ByteArrayWrapper +import java.util.{Optional, ArrayList => JArrayList} +import com.horizen.utils.{ByteArrayWrapper, Utils, Pair => JPair} +import com.horizen.{SidechainTypes, WalletBox, WalletBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.box.Box import com.horizen.proposition.Proposition -import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import com.horizen.{SidechainTypes, WalletBox, WalletBoxSerializer} import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging -import java.util.{ArrayList => JArrayList} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.compat.java8.OptionConverters.RichOptionalGeneric @@ -33,10 +37,6 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid loadWalletBoxes() - def calculateKey(boxId : Array[Byte]) : ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } - private def calculateBoxesBalances() : Unit = { for (bc <-_walletBoxesByType.keys) _walletBoxesBalances.put(bc, _walletBoxesByType(bc).map(_._2.box.value()).sum) @@ -55,7 +55,7 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid private def addWalletBoxByType(walletBox : WalletBox) : Unit = { val bc = walletBox.box.getClass - val key = calculateKey(walletBox.box.id()) + val key = Utils.calculateKey(walletBox.box.id()) val t = _walletBoxesByType.get(bc) if (t.isEmpty) { val m = new mutable.LinkedHashMap[ByteArrayWrapper, WalletBox]() @@ -76,7 +76,7 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid for (wb <- storage.getAll.asScala){ val walletBox = _walletBoxSerializer.parseBytesTry(wb.getValue.data) if (walletBox.isSuccess) { - _walletBoxes.put(calculateKey(walletBox.get.box.id()), walletBox.get) + _walletBoxes.put(Utils.calculateKey(walletBox.get.box.id()), walletBox.get) addWalletBoxByType(walletBox.get) } else log.error("Error while WalletBox parsing.", walletBox) @@ -85,11 +85,11 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid } def get (boxId : Array[Byte]) : Option[WalletBox] = { - _walletBoxes.get(calculateKey(boxId)) + _walletBoxes.get(Utils.calculateKey(boxId)) } def get (boxIds : List[Array[Byte]]) : List[WalletBox] = { - for (id <- boxIds.map(calculateKey) if _walletBoxes.get(id).isDefined) yield _walletBoxes(id) + for (id <- boxIds.map(Utils.calculateKey) if _walletBoxes.get(id).isDefined) yield _walletBoxes(id) } def getAll : List[WalletBox] = { @@ -117,10 +117,10 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid val removeList = new JArrayList[ByteArrayWrapper]() val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() - removeList.addAll(boxIdsRemoveList.map(calculateKey(_)).asJavaCollection) + removeList.addAll(boxIdsRemoveList.map(Utils.calculateKey(_)).asJavaCollection) for (wb <- walletBoxUpdateList) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(wb.box.id()), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(wb.box.id()), new ByteArrayWrapper(_walletBoxSerializer.toBytes(wb)))) storage.update(version, @@ -135,7 +135,7 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid } for (wba <- walletBoxUpdateList) { - val key = calculateKey(wba.box.id()) + val key = Utils.calculateKey(wba.box.id()) val bta = _walletBoxes.put(key, wba) addWalletBoxByType(wba) if (bta.isEmpty) @@ -162,4 +162,6 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid def isEmpty: Boolean = storage.isEmpty + def getIterator: StorageIterator = storage.getIterator + } diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala index 3df6b5d0ea..efe465dbb2 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala @@ -2,8 +2,7 @@ package com.horizen.storage import com.google.common.primitives.{Bytes, Ints} import com.horizen.SidechainTypes -import com.horizen.utils.{ByteArrayWrapper, CswData, CswDataSerializer, ListSerializer, Pair => JPair} -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.{ByteArrayWrapper, CswData, CswDataSerializer, ListSerializer, Utils, Pair => JPair} import scorex.util.ScorexLogging import java.util.{ArrayList => JArrayList} @@ -17,20 +16,16 @@ class SidechainWalletCswDataStorage(storage: Storage) extends ScorexLogging with private val cswDataListSerializer = new ListSerializer[CswData](CswDataSerializer) - private[horizen] val withdrawalEpochKey = calculateKey("withdrawalEpoch".getBytes) + private[horizen] val withdrawalEpochKey = Utils.calculateKey("withdrawalEpoch".getBytes) private val undefinedWithdrawalEpochCounter: Int = -1 private[horizen] def getWithdrawalEpochCounterKey(withdrawalEpoch: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) + Utils.calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) } private[horizen] def getCswDataKey(withdrawalEpoch: Int, counter: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) - } - - private[horizen] def calculateKey(boxId: Array[Byte]): ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) + Utils.calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) } def getWithdrawalEpochCounter(epoch: Int): Int = { diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala index ce0b96cbad..2f8cffdb10 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala @@ -3,11 +3,11 @@ package com.horizen.storage import com.horizen.SidechainTypes import com.horizen.companion.SidechainTransactionsCompanion import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import scorex.crypto.hash.Blake2b256 -import scorex.util.{ModifierId, ScorexLogging, idToBytes} - import java.util.{ArrayList => JArrayList} +import com.horizen.utils.Utils + import scala.collection.JavaConverters._ +import scorex.util.{ModifierId, ScorexLogging, idToBytes} import scala.compat.java8.OptionConverters.RichOptionalGeneric import scala.util.{Failure, Success, Try} @@ -23,12 +23,9 @@ extends SidechainTypes require(storage != null, "Storage must be NOT NULL.") require(sidechainTransactionsCompanion != null, "SidechainTransactionsCompanion must be NOT NULL.") - def calculateKey(transactionId : Array[Byte]) : ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(transactionId)) - } def get (transactionId : Array[Byte]) : Option[SidechainTypes#SCBT] = { - storage.get(calculateKey(transactionId)) match { + storage.get(Utils.calculateKey(transactionId)) match { case v if v.isPresent => { sidechainTransactionsCompanion.parseBytesTry(v.get().data) match { case Success(transaction) => Option(transaction.asInstanceOf[SidechainTypes#SCBT]) @@ -50,7 +47,7 @@ extends SidechainTypes val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() for (tx <- transactionUpdateList) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(idToBytes(ModifierId @@ tx.id)), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(idToBytes(ModifierId @@ tx.id)), new ByteArrayWrapper(sidechainTransactionsCompanion.toBytes(tx)))) storage.update(version, diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/DatabaseIterator.java b/sdk/src/main/scala/com/horizen/storage/leveldb/DatabaseIterator.java new file mode 100644 index 0000000000..787131af33 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/DatabaseIterator.java @@ -0,0 +1,40 @@ +package com.horizen.storage.leveldb; + +import com.horizen.storage.StorageIterator; +import org.iq80.leveldb.DBIterator; + +import java.io.IOException; +import java.util.Map; + +public class DatabaseIterator implements StorageIterator { + DBIterator iterator; + + DatabaseIterator(DBIterator iterator) { + this.iterator = iterator; + } + + @Override + public void seek(byte[] key) { + iterator.seek(key); + } + + @Override + public void seekToFirst() { + iterator.seekToFirst(); + } + + @Override + public void close() throws IOException { + iterator.close(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Map.Entry next() { + return iterator.next(); + } +} diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala index b42310be1a..c685c60651 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala @@ -1,6 +1,8 @@ package com.horizen.storage.leveldb +import com.horizen.storage.StorageIterator import com.horizen.utils.ByteArrayWrapper +import org.fusesource.leveldbjni.internal.JniDBIterator import org.iq80.leveldb.{DB, ReadOptions} import scala.collection.mutable @@ -131,6 +133,9 @@ final class VersionedLDBKVStore(protected val db: DB, keepVersions: Int) extends def versionIdExists(versionId: VersionId): Boolean = versions.exists(new ByteArrayWrapper(_) == new ByteArrayWrapper(versionId)) + def getIterator(): StorageIterator = { + new DatabaseIterator(db.iterator()) + } } object VersionedLDBKVStore { diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala index 59922b5c13..d61c15b8e9 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala @@ -3,11 +3,10 @@ package com.horizen.storage.leveldb import java.io.File import java.util import java.util.{Optional, List => JList} - -import com.horizen.storage.Storage +import com.horizen.storage.{Storage, StorageIterator} import com.horizen.storage.leveldb.LDBFactory.factory import com.horizen.utils.{Pair => JPair, _} -import org.iq80.leveldb.Options +import org.iq80.leveldb.{Options} import scala.collection.JavaConverters._ import scala.compat.java8.OptionConverters._ @@ -97,4 +96,8 @@ class VersionedLevelDbStorageAdapter(pathToDB: File) extends Storage{ override def isEmpty: Boolean = dataBase.versions.isEmpty override def numberOfVersions: Int = dataBase.versions.size + + override def getIterator(): StorageIterator = { + dataBase.getIterator() + } } diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala index e8d610b8fa..1c7ef41e49 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala @@ -10,6 +10,9 @@ import scorex.crypto.hash.Blake2b256 package object leveldb { object Constants { val HashLength: Int = 32 + //Batch size used in the SidechainWallet and SidechainState restore method. + //TODO: Investigate what could be a real good value. + val BatchSize: Int = 10000 } object Algos extends ScorexEncoding { diff --git a/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java new file mode 100644 index 0000000000..ab80f33f2c --- /dev/null +++ b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java @@ -0,0 +1,202 @@ +package com.horizen.backup; + +import com.horizen.box.Box; +import com.horizen.box.BoxSerializer; +import com.horizen.box.ZenBox; +import com.horizen.companion.SidechainBoxesCompanion; +import com.horizen.customtypes.CustomBox; +import com.horizen.customtypes.CustomBoxSerializer; +import com.horizen.fixtures.BoxFixtureClass; +import com.horizen.proposition.Proposition; +import com.horizen.storage.BackupStorage; +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter; +import com.horizen.utils.ByteArrayWrapper; +import com.horizen.utils.Pair; +import com.horizen.utils.Utils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import static org.junit.Assert.fail; + +public class BoxIteratorTest extends BoxFixtureClass { + @Rule + public TemporaryFolder temporaryFolder= new TemporaryFolder(); + + + SidechainBoxesCompanion sidechainBoxesCompanion; + HashMap>> customBoxSerializers = new HashMap<>(); + List customBoxes; + List zenBoxes; + List> customBoxToSave = new ArrayList<>(); + List> zenBoxToSave = new ArrayList<>(); + int nBoxes = 5; + + @Before + public void setup() { + customBoxSerializers.put(CustomBox.BOX_TYPE_ID, (BoxSerializer) CustomBoxSerializer.getSerializer()); + sidechainBoxesCompanion = new SidechainBoxesCompanion(customBoxSerializers); + + customBoxes = getCustomBoxList(nBoxes); + for (CustomBox box : customBoxes) { + ByteArrayWrapper key = Utils.calculateKey(box.id()); + ByteArrayWrapper value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + customBoxToSave.add(new Pair<>(key, value)); + } + zenBoxes = getZenBoxList(nBoxes); + for (ZenBox box : zenBoxes) { + ByteArrayWrapper key = Utils.calculateKey(box.id()); + ByteArrayWrapper value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + zenBoxToSave.add(new Pair<>(key, value)); + } + + } + + @Test + public void BoxIteratorTestCustomBoxes() throws IOException { + //Create temporary BackupStorage + File stateStorageFile = temporaryFolder.newFolder("stateStorage"); + BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); + + //Add an additional random element to customBoxToSave list. + customBoxToSave.add(new Pair(new ByteArrayWrapper("key1".getBytes()), new ByteArrayWrapper("value1".getBytes()))); + + //Populate the BackupStorage + backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), customBoxToSave).get(); + + //Create a BoxIterator + BoxIterator boxIterator = backupStorage.getBoxIterator(); + + //Read the storage using the BoxIterator + ArrayList foundBoxes = readStorage(boxIterator); + + //Test that we read the correct amount of Boxes (nBoxes) and we ignore non-boxes elements. + assert(foundBoxes.size() == nBoxes); + + //Test the content of the boxes. + for (BackupBox backupBox : foundBoxes) { + ByteArrayWrapper newKey = new ByteArrayWrapper(backupBox.getBoxKey()); + ByteArrayWrapper newValue = new ByteArrayWrapper(backupBox.getBoxValue()); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + assert(backupBox.getBoxTypeId() == CustomBox.BOX_TYPE_ID); + assert(customBoxes.contains(backupBox.getBox())); + } + + //Test that the iterator is empty + Optional emptyBox = boxIterator.nextBox(); + assert(emptyBox.isEmpty()); + + //Test the seekToFirst method + boxIterator.seekToFirst(); + foundBoxes = readStorage(boxIterator); + assert(foundBoxes.size() == nBoxes); + } + + @Test + public void BoxIteratorTestCoinBoxes() throws IOException { + //Create temporary BackupStorage + File stateStorageFile = temporaryFolder.newFolder("stateStorage"); + BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); + + //Populate the BackupStorage + backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), zenBoxToSave).get(); + + //Create a BoxIterator + BoxIterator boxIterator = backupStorage.getBoxIterator(); + + //Read the storage using the BoxIterator + try { + boxIterator.nextBox(); + fail("We should not be able to retrieve Coin Boxes!"); + } catch (RuntimeException e) { + System.out.println(e.getMessage()); + assert(e.getMessage().equals("Coin boxes are not eligible to be restored!")); + } + + } + + @Test + public void BoxIteratorNextBoxesTest() throws IOException { + //Create temporary BackupStorage + File stateStorageFile = temporaryFolder.newFolder("stateStorage"); + BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); + + //Add an additional random element to customBoxToSave list. + customBoxToSave.add(new Pair(new ByteArrayWrapper("key1".getBytes()), new ByteArrayWrapper("value1".getBytes()))); + + //Populate the BackupStorage + backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), customBoxToSave).get(); + + //Create a BoxIterator + BoxIterator boxIterator = backupStorage.getBoxIterator(); + + //Test nextBoxes with nElement = 0 and empty key to seek + List> foundBoxes = boxIterator.getNextBoxes(0,Optional.empty()); + assert(foundBoxes.isEmpty()); + + //Test nextBoxes with nElement < 0 and empty key to seek + foundBoxes = boxIterator.getNextBoxes(-1,Optional.empty()); + boxIterator.seekToFirst(); + assert(foundBoxes.isEmpty()); + + //Test nextBoxes with nElement = nBoxes and empty key to seek + boxIterator.seekToFirst(); + foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.empty()); + assert(foundBoxes.size() == nBoxes); + //Test the content of the boxes. + for (Box box : foundBoxes) { + ByteArrayWrapper newKey = Utils.calculateKey(box.id()); + ByteArrayWrapper newValue = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + } + + //Test nextBoxes with nElement > nBoxes and empty key to seek + boxIterator.seekToFirst(); + foundBoxes = boxIterator.getNextBoxes(nBoxes+10,Optional.empty()); + assert(foundBoxes.size() == nBoxes); + //Test the content of the boxes. + for (Box box : foundBoxes) { + ByteArrayWrapper newKey = Utils.calculateKey(box.id()); + ByteArrayWrapper newValue = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + } + + //Test nextBoxes with nElement = nBoxes and keyToSeek=1st box saved + boxIterator.seekToFirst(); + List> boxStoredOrder = foundBoxes; + foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.of(boxStoredOrder.get(0).id())); + assert(foundBoxes.size() == nBoxes-1); + assert(!foundBoxes.contains(boxStoredOrder.get(0))); + //Test the content of the boxes. + for (Box box : foundBoxes) { + ByteArrayWrapper newKey = Utils.calculateKey(box.id()); + ByteArrayWrapper newValue = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + } + + //Test nextBoxes with nElement = nBoxes and keyToSeek=last box saved + boxIterator.seekToFirst(); + foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.of(boxStoredOrder.get(boxStoredOrder.size()-1).id())); + assert(foundBoxes.size() == 0); + + } + + + private ArrayList readStorage(BoxIterator boxIterator) { + ArrayList storedBoxes = new ArrayList<>(); + + Optional optionalBox = boxIterator.nextBox(); + while(optionalBox.isPresent()) { + storedBoxes.add(optionalBox.get()); + optionalBox = boxIterator.nextBox(); + } + return storedBoxes; + } +} diff --git a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java index a66d7d4665..6af2c837ac 100644 --- a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java @@ -1,11 +1,11 @@ package com.horizen.customtypes; +import com.horizen.backup.BoxIterator; import com.horizen.box.Box; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.wallet.ApplicationWallet; -import java.util.Collections; import java.util.List; public class CustomApplicationWallet implements ApplicationWallet { @@ -33,4 +33,9 @@ public void onRollback(byte[] blockId) { public boolean checkStoragesVersion(byte[] blockId) { return true; } + + @Override + public void onBackupRestore(BoxIterator i) { + + } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java index f03a79b643..d514e74e16 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java @@ -1,5 +1,6 @@ package com.horizen.customtypes; +import com.horizen.backup.BoxIterator; import com.horizen.block.SidechainBlock; import com.horizen.box.Box; import com.horizen.proposition.Proposition; @@ -9,7 +10,6 @@ import scala.util.Success; import scala.util.Try; -import java.util.Collections; import java.util.List; public class DefaultApplicationState implements ApplicationState { @@ -35,4 +35,7 @@ public Try onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } + + @Override + public Try onBackupRestore(BoxIterator i) { return new Success<>(this); } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java index e72719c6ef..efca25adea 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java @@ -1,11 +1,11 @@ package com.horizen.customtypes; +import com.horizen.backup.BoxIterator; import com.horizen.box.Box; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.wallet.ApplicationWallet; -import java.util.Collections; import java.util.List; public class DefaultApplicationWallet implements ApplicationWallet { @@ -32,4 +32,9 @@ public void onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } + + @Override + public void onBackupRestore(BoxIterator i) { + + } } diff --git a/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala b/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala new file mode 100644 index 0000000000..9b57d0dd38 --- /dev/null +++ b/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala @@ -0,0 +1,175 @@ +package com.horizen + +import com.horizen.backup.{BackupBox, BoxIterator} +import com.horizen.box.{BoxSerializer, CoinsBox} +import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.customtypes.{CustomBox, CustomBoxSerializer} +import com.horizen.fixtures.{SecretFixture, StoreFixture, TransactionFixture} +import com.horizen.proposition.PublicKey25519Proposition +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter +import com.horizen.storage.{BackupStorage, BoxBackupInterface} +import com.horizen.utils.{ByteArrayWrapper, BytesUtils, Utils, Pair => JPair} +import org.junit.Assert.{assertEquals, assertTrue} +import org.junit.rules.TemporaryFolder +import org.junit.{Before, Rule, Test} +import org.scalatestplus.junit.JUnitSuite +import scorex.crypto.hash.Blake2b256 + +import scala.collection.JavaConverters._ +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, Optional => JOptional} +import java.lang.{Byte => JByte} +import scala.collection.mutable.ListBuffer + +class SidechainBackupTest + extends JUnitSuite + with StoreFixture + with SecretFixture + with TransactionFixture + { + + val customBoxesSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]] = new JHashMap() + customBoxesSerializers.put(CustomBox.BOX_TYPE_ID, CustomBoxSerializer.getSerializer.asInstanceOf[BoxSerializer[SidechainTypes#SCB]]) + val sidechainBoxesCompanion = SidechainBoxesCompanion(customBoxesSerializers) + + val boxListFirstModifier = new ListBuffer[SidechainTypes#SCB]() + val boxListSecondModifier = new ListBuffer[SidechainTypes#SCB]() + val storedBoxListFirstModifier = new ListBuffer[JPair[ByteArrayWrapper, ByteArrayWrapper]]() + val storedBoxListSecondModifier = new ListBuffer[JPair[ByteArrayWrapper, ByteArrayWrapper]]() + val firstModifierBoxLength = 7 + + val firstModifier: ByteArrayWrapper = getVersion + val secondModifier: ByteArrayWrapper = getVersion; + + + val backupper: BoxBackupInterface = new BoxBackupInterface { + override def backup(source: BoxIterator, db: BackupStorage): Unit = { + val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() + + var optionalBox = source.nextBox(true) + while(optionalBox.isPresent) { + val box = optionalBox.get.getBox + if (!box.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) { + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(box.id()), + new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box)))) + } + optionalBox = source.nextBox(true) + } + if (updateList.size() != 0) + db.update(getVersion,updateList) + } + } + + val _temporaryFolder = new TemporaryFolder() + @Rule def temporaryFolder = _temporaryFolder + + @Before + def setup(): Unit = { + boxListFirstModifier ++= getZenBoxList(5).asScala.toList + boxListFirstModifier ++= getCustomBoxList(firstModifierBoxLength).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + + boxListSecondModifier ++= getZenBoxList(5).asScala.toList + boxListSecondModifier ++= getCustomBoxList(5).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + + for (b <- boxListFirstModifier) { + storedBoxListFirstModifier.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new JPair(key, value) + }) + } + + for (b <- boxListSecondModifier) { + storedBoxListSecondModifier.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new JPair(key, value) + }) + } + } + + @Test + def testCreateBackupWithNoCopy(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + var stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new VersionedLevelDbStorageAdapter(backupStorageFile) + + //Update a first time the SidechainStateStorage with some custom and zen boxes + stateStorage.update(firstModifier, storedBoxListFirstModifier.asJava, new JArrayList[ByteArrayWrapper]()) + //Update a second time the SidechainStateStorage with some custom and zen boxes + stateStorage.update(secondModifier, storedBoxListSecondModifier.asJava, new JArrayList[ByteArrayWrapper]()) + stateStorage.close() + + //Instantiate a SidechainBackup class and call createBackup with no Copy option + val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, backUpStorage = backupStorage, backUpper = backupper); + sidechainBakcup.createBackup(stateStorageFile.getPath, BytesUtils.toHexString(firstModifier.data()), false) + + //Read the backup storage created and verify that contains only firstModifierBoxLength elements. (We did a rollback to the first modifier) + val storedBoxes = readStorage(new BoxIterator(backupStorage.getIterator(), sidechainBoxesCompanion)) + assertEquals("BackupStorage should contains only the firstModifierBoxLength CustomBoxes of the storedBoxListFirstModifier!",storedBoxes.size(), firstModifierBoxLength) + + //Verify that the backupped boxes are the ones saved in the first SidechainStateStorage update and they are not CoinBoxes. + storedBoxes.forEach(box => { + val storageElement = new JPair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper(box.getBoxKey), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box.getBox))) + assertTrue("Restored boxes should be inside storedBoxListFirstModifier",storedBoxListFirstModifier.contains(storageElement)) + assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) + }) + + stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + //Verify that the lastVersion of the StateStorage now is the firstModifier + assertEquals(stateStorage.lastVersionID().get().data().deep, firstModifier.data().deep) + } + + @Test + def testCreateBackupWithCopy(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + var stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new VersionedLevelDbStorageAdapter(backupStorageFile) + + //Update a first time the SidechainStateStorage with some custom and zen boxes + val firstModifier = getVersion; + stateStorage.update(firstModifier, storedBoxListFirstModifier.asJava, new JArrayList[ByteArrayWrapper]()) + //Update a second time the SidechainStateStorage with some custom and zen boxes + val secondModifier = getVersion; + stateStorage.update(secondModifier, storedBoxListSecondModifier.asJava, new JArrayList[ByteArrayWrapper]()) + stateStorage.close() + + //Instantiate a SidechainBackup class and call createBackup + val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, backUpStorage = backupStorage, backUpper = backupper); + sidechainBakcup.createBackup(stateStorageFile.getPath, BytesUtils.toHexString(firstModifier.data()), true) + + //Read the backup storage created and verify that contains only firstModifierBoxLength elements. (We did a rollback to the first modifier) + val storedBoxes = readStorage(new BoxIterator(backupStorage.getIterator(), sidechainBoxesCompanion)) + assertEquals("BackupStorage should contains only the firstModifierBoxLength CustomBoxes of the storedBoxListFirstModifier!",storedBoxes.size(), firstModifierBoxLength) + + //Verify that the backupped boxes are the ones saved in the first SidechainStateStorage update and they are not CoinBoxes. + storedBoxes.forEach(box => { + val storageElement = new JPair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper(box.getBoxKey), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box.getBox))) + assertTrue("Restored boxes should be inside storedBoxListFirstModifier",storedBoxListFirstModifier.contains(storageElement)) + assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) + }) + + stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + //Verify that the lastVersion of the StateStorage now is the firstModifier + assertEquals(stateStorage.lastVersionID().get().data().deep, secondModifier.data().deep) + } + + def readStorage(sidechainStateStorageBoxIterator: BoxIterator): JArrayList[BackupBox] = { + val storedBoxes = new JArrayList[BackupBox]() + + var optionalBox = sidechainStateStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + storedBoxes.add(optionalBox.get) + optionalBox = sidechainStateStorageBoxIterator.nextBox + } + storedBoxes + } + +} diff --git a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala index 18d9849766..d240a0ad36 100644 --- a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala @@ -14,12 +14,14 @@ import com.horizen.params.MainNetParams import com.horizen.proposition._ import com.horizen.secret.{PrivateKey25519, Secret, SecretSerializer} import com.horizen.storage._ +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter import com.horizen.transaction.mainchain.{ForwardTransfer, SidechainCreation, SidechainRelatedMainchainOutput} import com.horizen.transaction.{BoxTransaction, MC2SCAggregatedTransaction, RegularTransaction} import com.horizen.utils.{ByteArrayWrapper, BytesUtils, CswData, ForgingStakeMerklePathInfo, ForwardTransferCswData, MerklePath, MerkleTree, Pair, UtxoCswData} import com.horizen.wallet.ApplicationWallet import org.junit.Assert._ import org.junit._ +import org.junit.rules.TemporaryFolder import org.mockito._ import org.scalatestplus.junit.JUnitSuite import org.scalatestplus.mockito._ @@ -48,8 +50,10 @@ class SidechainWalletTest val boxList = new ListBuffer[WalletBox]() val storedBoxList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val boxVersions = new ListBuffer[ByteArrayWrapper]() + val boxToRestoreList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val secretList = new ListBuffer[Secret]() + val customSecretList = new ListBuffer[Secret]() val storedSecretList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val secretVersions = new ListBuffer[ByteArrayWrapper]() @@ -69,6 +73,10 @@ class SidechainWalletTest val params = MainNetParams() + val _temporaryFolder = new TemporaryFolder() + + @Rule def temporaryFolder = _temporaryFolder + def boxIdToMerklePath(boxId: Array[Byte]): Array[Byte] = BytesUtils.reverseBytes(boxId) @Before @@ -76,6 +84,8 @@ class SidechainWalletTest // Set base Secrets data secretList ++= getPrivateKey25519List(5).asScala + customSecretList ++= getCustomPrivateKeyList(5).asScala + secretVersions += getVersion for (s <- secretList) { @@ -123,6 +133,10 @@ class SidechainWalletTest boxList ++= getWalletBoxList(getZenBoxList(secretList.map(_.asInstanceOf[PrivateKey25519]).asJava)).asScala boxList += getWalletBox(getForgerBox(secretList.head.asInstanceOf[PrivateKey25519].publicImage())) + val customBoxList = new ListBuffer[SidechainTypes#SCB]() + customBoxList ++= getCustomBoxListWithPrivateKeys(customSecretList.map(_.asInstanceOf[CustomPrivateKey]).asJava).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + customBoxList += getCustomBox.asInstanceOf[SidechainTypes#SCB] //This box shouldn't be included in the 'scanBackup' test result + boxVersions += getVersion for (b <- boxList) { @@ -134,6 +148,13 @@ class SidechainWalletTest }) } + for (b <- customBoxList) { + boxToRestoreList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new Pair(key,value) + }) + } // Mock get and update methods of BoxStorage Mockito.when(mockedBoxStorage.getAll).thenReturn(storedBoxList.asJava) @@ -377,6 +398,119 @@ class SidechainWalletTest sidechainWallet.scanPersistent(mockedBlock, withdrawalEpochNumber, feePaymentBoxes, Some(utxoMerkleTreeView)) } + @Test + def testScanBackUpNonCoinBoxes(): Unit = { + val mockedSecretStorage: SidechainSecretStorage = mock[SidechainSecretStorage] + val mockedWalletTransactionStorage: SidechainWalletTransactionStorage = mock[SidechainWalletTransactionStorage] + val mockedForgingBoxesInfoStorage: ForgingBoxesInfoStorage = mock[ForgingBoxesInfoStorage] + val mockedCswDataStorage: SidechainWalletCswDataStorage = mock[SidechainWalletCswDataStorage] + val mockedApplicationWallet: ApplicationWallet = mock[ApplicationWallet] + val mockedVersion: VersionTag = bytesToVersion(Array[Byte](32)) + + Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>customSecretList.toList) + + //Create temporary WalletBoxStorage + val walletBoxStorageFile = temporaryFolder.newFolder("walletBoxStorage") + val walletBoxStorage = new SidechainWalletBoxStorage(new VersionedLevelDbStorageAdapter(walletBoxStorageFile), sidechainBoxesCompanion) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) + + boxToRestoreList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) + backupStorage.update(getVersion, boxToRestoreList.asJava).get + + val sidechainWallet = new SidechainWallet("seed".getBytes, + walletBoxStorage, + mockedSecretStorage, + mockedWalletTransactionStorage, + mockedForgingBoxesInfoStorage, + mockedCswDataStorage, + params, + mockedVersion, + mockedApplicationWallet) + + // Mock get and update methods of SecretStorage + sidechainWallet.scanBackUp(backupStorage.getBoxIterator, System.currentTimeMillis()) + + assertTrue("Box stored to the backupStorage should be 7: 5 regular box + 1 new address + 1 fake ",boxToRestoreList.size == 7) + val storedBoxes = readStorage(walletBoxStorage) + + //Verify that we did take only the 5 Boxes + assertEquals("SidechainWalletBoxStorage should contains only the 5 CustomBoxes!",5, storedBoxes.size()) + val publicKeys = customSecretList.map(_.asInstanceOf[CustomPrivateKey].publicImage()).asJava + storedBoxes.forEach(box => { + assertTrue("Restored Boxes propositions should be inside our wallet!", publicKeys.contains(box.box.proposition())) + }) + } + + @Test + def testScanBackUpCoinBoxes(): Unit = { + val mockedSecretStorage: SidechainSecretStorage = mock[SidechainSecretStorage] + val mockedWalletTransactionStorage: SidechainWalletTransactionStorage = mock[SidechainWalletTransactionStorage] + val mockedForgingBoxesInfoStorage: ForgingBoxesInfoStorage = mock[ForgingBoxesInfoStorage] + val mockedCswDataStorage: SidechainWalletCswDataStorage = mock[SidechainWalletCswDataStorage] + val mockedApplicationWallet: ApplicationWallet = mock[ApplicationWallet] + val mockedVersion: VersionTag = bytesToVersion(Array[Byte](32)) + Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>secretList.toList) + + //Create temporary WalletBoxStorage + val walletBoxStorageFile = temporaryFolder.newFolder("walletBoxStorage") + val walletBoxStorage = new SidechainWalletBoxStorage(new VersionedLevelDbStorageAdapter(walletBoxStorageFile), sidechainBoxesCompanion) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) + + //Serialize ZenBoxes + val zenBoxSerializedList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() + for (b <- boxList) { + zenBoxSerializedList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.box.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b.box)) + new Pair(key,value) + }) + } + backupStorage.update(getVersion, zenBoxSerializedList.asJava).get + + val sidechainWallet = new SidechainWallet("seed".getBytes, + walletBoxStorage, + mockedSecretStorage, + mockedWalletTransactionStorage, + mockedForgingBoxesInfoStorage, + mockedCswDataStorage, + params, + mockedVersion, + mockedApplicationWallet) + + var exceptionThrown = false + sidechainWallet.scanBackUp(backupStorage.getBoxIterator, System.currentTimeMillis()) match { + case Failure(_) => + exceptionThrown = true + case Success(_) => + fail() + } + assertTrue("CoinBoxes should not be restored!",exceptionThrown) + } + + def readStorage(walletBoxStorage: SidechainWalletBoxStorage): JArrayList[WalletBox] = { + val walletBoxStorageIterator: StorageIterator = walletBoxStorage.getIterator + walletBoxStorageIterator.seekToFirst() + + val walletBoxSerializer: WalletBoxSerializer = new WalletBoxSerializer(sidechainBoxesCompanion) + val storedBoxes = new JArrayList[WalletBox]() + while(walletBoxStorageIterator.hasNext) { + val entry = walletBoxStorageIterator.next() + val box: Try[WalletBox] = walletBoxSerializer.parseBytesTry(entry.getValue) + + if(box.isSuccess) { + val currBox: WalletBox = box.get + storedBoxes.add(currBox) + } + } + storedBoxes + } + @Test def testRollback(): Unit = { val mockedWalletBoxStorage: SidechainWalletBoxStorage = mock[SidechainWalletBoxStorage] diff --git a/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala b/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala index a77dba1bb4..8b78f3aa35 100644 --- a/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala +++ b/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala @@ -12,6 +12,9 @@ import com.horizen.SidechainNodeViewHolder.ReceivableMessages.{ApplyBiFunctionOn import com.horizen.api.http.SidechainBlockActor.ReceivableMessages.{GenerateSidechainBlocks, SubmitSidechainBlock} import com.horizen.api.http.SidechainTransactionActor.ReceivableMessages.BroadcastTransaction import com.horizen.companion.SidechainTransactionsCompanion +import com.horizen.backup.BoxIterator +import com.horizen.box.BoxSerializer +import com.horizen.companion.{SidechainBoxesCompanion} import com.horizen.consensus.ConsensusEpochAndSlot import com.horizen.fixtures.{CompanionsFixture, SidechainBlockFixture} import com.horizen.forge.Forger @@ -22,7 +25,7 @@ import com.horizen.transaction._ import com.horizen.{SidechainApp, SidechainSettings, SidechainTypes} import org.junit.Assert.{assertEquals, assertTrue} import org.junit.runner.RunWith -import org.mockito.Mockito +import org.mockito.{Mockito} import org.scalatestplus.junit.JUnitRunner import org.scalatestplus.mockito.MockitoSugar import org.scalatest.matchers.should.Matchers @@ -41,7 +44,16 @@ import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} import com.horizen.csw.CswManager.ReceivableMessages.{GenerateCswProof, GetBoxNullifier, GetCeasedStatus, GetCswBoxIds, GetCswInfo} import com.horizen.csw.CswManager.Responses.{Absent, CswInfo, CswProofInfo, NoProofData, ProofCreationFinished} +import com.horizen.customtypes.{CustomBox, CustomBoxSerializer} +import com.horizen.storage.StorageIterator +import com.horizen.utils.{ByteArrayWrapper} import org.bouncycastle.pqc.math.linearalgebra.ByteUtils +import scorex.crypto.hash.Blake2b256 + +import java.lang.{Byte => JByte} +import java.util.{HashMap => JHashMap} +import scala.collection.JavaConverters.asScalaBufferConverter +import scala.collection.mutable.ListBuffer import scala.language.postfixOps @@ -275,6 +287,30 @@ abstract class SidechainApiRouteTest extends AnyWordSpec with Matchers with Scal }) val mockedCswManagerActorRef: ActorRef = mockedCswManagerActor.ref + + val customBoxesSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]] = new JHashMap() + customBoxesSerializers.put(CustomBox.BOX_TYPE_ID, CustomBoxSerializer.getSerializer.asInstanceOf[BoxSerializer[SidechainTypes#SCB]]) + val sidechainBoxesCompanion = SidechainBoxesCompanion(customBoxesSerializers) + val mockedStorageIterator: StorageIterator = mock[StorageIterator] + var boxList:ListBuffer[SidechainTypes#SCB] = new ListBuffer[SidechainTypes#SCB]() + boxList ++= getCustomBoxList(3).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + val storedBoxList = new ListBuffer[util.Map.Entry[Array[Byte], Array[Byte]]]() + for (b <- boxList) { + storedBoxList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())).data() + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)).data() + val entry: util.Map.Entry[Array[Byte], Array[Byte]] = util.Map.entry(key, value) + entry + }) + } + + def mockStorageIterator = { + Mockito.when(mockedStorageIterator.hasNext).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false) + Mockito.when(mockedStorageIterator.next()).thenReturn(storedBoxList(0)).thenReturn(storedBoxList(1)).thenReturn(storedBoxList(2)) + } + mockStorageIterator + val mockedBoxIterator: BoxIterator = new BoxIterator(mockedStorageIterator, sidechainBoxesCompanion) + implicit def default() = RouteTestTimeout(3.second) val params = MainNetParams() @@ -289,6 +325,7 @@ abstract class SidechainApiRouteTest extends AnyWordSpec with Matchers with Scal val mainchainBlockApiRoute: Route = MainchainBlockApiRoute(mockedRESTSettings, mockedSidechainNodeViewHolderRef).route val applicationApiRoute: Route = ApplicationApiRoute(mockedRESTSettings, new SimpleCustomApi(), mockedSidechainNodeViewHolderRef).route val sidechainCswApiRoute: Route = SidechainCswApiRoute(mockedRESTSettings, mockedSidechainNodeViewHolderRef, mockedCswManagerActorRef).route + val sidechainBackupApiRoute: Route = SidechainBackupApiRoute(mockedRESTSettings, mockedSidechainNodeViewHolderRef, mockedBoxIterator).route val walletCoinsBalanceApiRejected: Route = SidechainRejectionApiRoute("wallet", "coinsBalance", mockedRESTSettings, mockedSidechainNodeViewHolderRef).route val walletApiRejected: Route = SidechainRejectionApiRoute("wallet", "", mockedRESTSettings, mockedSidechainNodeViewHolderRef).route diff --git a/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala b/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala new file mode 100644 index 0000000000..539e2f1167 --- /dev/null +++ b/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala @@ -0,0 +1,108 @@ +package com.horizen.api.http + +import akka.http.scaladsl.model.{ContentTypes, HttpMethods, StatusCodes} +import akka.http.scaladsl.server.{MalformedRequestContentRejection, MethodRejection, Route} +import com.horizen.api.http.SidechainBackupRestScheme.ReqGetInitialBoxes +import com.horizen.serialization.SerializationUtil +import com.horizen.utils.BytesUtils +import org.junit.Assert.{assertEquals, assertTrue} + +import scala.collection.JavaConverters.asScalaIteratorConverter + +class SidechainBackupApiRouteTest extends SidechainApiRouteTest{ + override val basePath = "/backup/" + + "The Api should to" should { + + "reject and reply with http error" in { + Get(basePath) ~> sidechainBackupApiRoute ~> check { + rejection shouldBe MethodRejection(HttpMethods.POST) + } + Get(basePath) ~> Route.seal(sidechainBackupApiRoute) ~> check { + status.intValue() shouldBe StatusCodes.MethodNotAllowed.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + } + Post(basePath + "getRestoredBoxes").withEntity("maybe_a_json") ~> sidechainBackupApiRoute ~> check { + rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) + } + //Test with more than max numberOfElements. + Post(basePath + "getRestoredBoxes").withEntity("{\"numberOfElements\": 101}") ~> sidechainBackupApiRoute ~> check { + rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) + } + //Test with negative numberOfElements. + Post(basePath + "getRestoredBoxes").withEntity("{\"numberOfElements\": -1}") ~> sidechainBackupApiRoute ~> check { + rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) + } + } + } + "reply at /getRestoredBoxes" in { + //Test with invalid "lastBoxId". + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("invalid_json")))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("error") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + + val errorCode = result.get("code") + if (errorCode == null) + fail("Result serialization failed") + assertEquals(errorCode.asText(), "0802") + } + //Test with no "lastBoxId". It should return all mocked boxes + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, None))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("result") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + assertEquals(1, result.findValues("boxes").size()) + + val boxes = result.get("boxes") + if (boxes == null) + fail("Result serialization failed") + + assertTrue(boxes.isArray) + assertEquals(storedBoxList.size, boxes.elements().asScala.length) + + mockStorageIterator + } + //Test with empty "lastBoxId". It should return all mocked boxes + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("")))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("result") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + assertEquals(1, result.findValues("boxes").size()) + + val boxes = result.get("boxes") + if (boxes == null) + fail("Result serialization failed") + + assertTrue(boxes.isArray) + assertEquals(storedBoxList.size, boxes.elements().asScala.length) + + mockStorageIterator + } + //Test with "lastBoxId"=mockedBoxes.head. It should skip the first box of the mockedBox list. + //Also test that we return less boxes than "numberOfElement" requested in case of no more boxes. + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some(BytesUtils.toHexString(boxList.head.id()))))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("result") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + assertEquals(1, result.findValues("boxes").size()) + + val boxes = result.get("boxes") + if (boxes == null) + fail("Result serialization failed") + + assertTrue(boxes.isArray) + assertEquals(storedBoxList.size -1, boxes.elements().asScala.length) + + mockStorageIterator + } + } +} \ No newline at end of file diff --git a/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala index 78b69b409a..dd79f570ed 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala @@ -73,6 +73,10 @@ trait BoxFixture new CustomBox(new CustomBoxData(getCustomPrivateKey.publicImage(), Random.nextInt(100)), Random.nextInt(1000)) } + def getCustomBoxWithPrivateKey(proposition: CustomPublicKeyProposition): CustomBox = { + new CustomBox(new CustomBoxData(proposition, Random.nextInt(100)), Random.nextInt(1000)) + } + def getCustomBoxList(count: Int): JList[CustomBox] = { val boxList: JList[CustomBox] = new JArrayList() @@ -82,6 +86,14 @@ trait BoxFixture boxList } + def getCustomBoxListWithPrivateKeys(secretList: JList[CustomPrivateKey]): JList[CustomBox] = { + val boxList: JList[CustomBox] = new JArrayList() + for (s <- secretList.asScala) + boxList.add(getCustomBoxWithPrivateKey(s.publicImage())) + + boxList + } + def getWalletBox(box: SidechainTypes#SCB): WalletBox = { val txId = new Array[Byte](32) Random.nextBytes(txId) diff --git a/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala index 5e8408ea95..d6eec9347a 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala @@ -11,7 +11,7 @@ class MockedSidechainNodeViewHolder(sidechainSettings: SidechainSettings, state: SidechainState, wallet: SidechainWallet, mempool: SidechainMemoryPool) - extends SidechainNodeViewHolder(sidechainSettings, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null) { + extends SidechainNodeViewHolder(sidechainSettings, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null ) { override def dumpStorages: Unit = {} diff --git a/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala index 2d0610a6dc..45b18c76be 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala @@ -100,6 +100,7 @@ trait SidechainNodeViewHolderFixture val sidechainWalletTransactionStorage = new SidechainWalletTransactionStorage(getStorage(), sidechainTransactionsCompanion) val forgingBoxesMerklePathStorage = new ForgingBoxesInfoStorage(getStorage()) val cswDataStorage = new SidechainWalletCswDataStorage(getStorage()) + val backupStorage = new BackupStorage(getStorage(), sidechainBoxesCompanion) // Append genesis secrets if we start the node first time if(sidechainSecretStorage.isEmpty) { @@ -119,6 +120,7 @@ trait SidechainNodeViewHolderFixture sidechainWalletTransactionStorage, forgingBoxesMerklePathStorage, cswDataStorage, + backupStorage, params, timeProvider, defaultApplicationWallet, diff --git a/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala b/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala index 9e319abdc8..3237e42ee0 100644 --- a/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala +++ b/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala @@ -1,7 +1,6 @@ package com.horizen.storage import java.util import java.util.{Optional, List => JList} - import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} import scala.collection.JavaConverters._ @@ -40,4 +39,7 @@ class InMemoryStorageAdapter(hashMap: mutable.HashMap[ByteArrayWrapper, ByteArra override def close(): Unit = {} def copy(): InMemoryStorageAdapter = new InMemoryStorageAdapter(hashMap.clone()) + + override def getIterator: StorageIterator = ??? + } diff --git a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala index 6d05106638..08ee92096a 100644 --- a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala +++ b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala @@ -2,13 +2,15 @@ package com.horizen.storage import com.google.common.primitives.Ints import com.horizen.SidechainTypes -import com.horizen.box.BoxSerializer +import com.horizen.backup.{BackupBox, BoxIterator} +import com.horizen.box.{BoxSerializer, CoinsBox} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus.{ConsensusEpochNumber, intToConsensusEpochNumber} import com.horizen.customtypes.{CustomBox, CustomBoxSerializer} import com.horizen.fixtures.{SecretFixture, StoreFixture, TransactionFixture} +import com.horizen.proposition.PublicKey25519Proposition import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter -import com.horizen.utils.{BlockFeeInfo, BlockFeeInfoSerializer, ByteArrayWrapper, Pair, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer} +import com.horizen.utils.{BlockFeeInfo, BlockFeeInfoSerializer, ByteArrayWrapper, BytesUtils, Pair, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer} import org.junit.Assert._ import org.junit._ import org.mockito.{ArgumentMatchers, Mockito} @@ -21,6 +23,9 @@ import java.util.{ArrayList => JArrayList, HashMap => JHashMap, Optional => JOpt import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.util.Try +import org.junit.Rule +import org.junit.rules.TemporaryFolder + class SidechainStateStorageTest extends JUnitSuite @@ -34,6 +39,7 @@ class SidechainStateStorageTest val boxList = new ListBuffer[SidechainTypes#SCB]() val storedBoxList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() + val customStoredBoxList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val customBoxesSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]] = new JHashMap() customBoxesSerializers.put(CustomBox.BOX_TYPE_ID, CustomBoxSerializer.getSerializer.asInstanceOf[BoxSerializer[SidechainTypes#SCB]]) @@ -43,6 +49,10 @@ class SidechainStateStorageTest val consensusEpoch: ConsensusEpochNumber = intToConsensusEpochNumber(1) + val _temporaryFolder = new TemporaryFolder() + + @Rule def temporaryFolder = _temporaryFolder + @Before def setUp(): Unit = { @@ -56,6 +66,13 @@ class SidechainStateStorageTest val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) new Pair(key,value) }) + if (!b.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) { + customStoredBoxList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new Pair(key,value) + }) + } } Mockito.when(mockedPhysicalStorage.get(ArgumentMatchers.any[ByteArrayWrapper]())) @@ -135,6 +152,69 @@ class SidechainStateStorageTest assertEquals("Storage should return existing Box.", boxList(3), stateStorage.getBox(boxList(3).id()).get) } + @Test + def testRestoreNonCoinBoxes(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + val stateStorage = new SidechainStateStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) + + //Fill BackUpStorage with 5 CustomBoxes and 1 random element + customStoredBoxList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) + backupStorage.update(getVersion, customStoredBoxList.asJava).get + + //Restore the SidechainStateStorage based on the BackupStorage + stateStorage.restoreBackup(backupStorage.getBoxIterator, getVersion.data()) + + //Read the SidechainStateStorage + val storedBoxes = readStorage(new BoxIterator(stateStorage.getIterator, sidechainBoxesCompanion)) + + //Verify that we did take only the 5 CustomBoxes + assertEquals("SidechainStateStorage should contains only the 5 CustomBoxes!",storedBoxes.size(), 5) + storedBoxes.forEach(box => { + val storageElement = new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper(box.getBoxKey), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box.getBox))) + assertTrue("Restored boxes should be inside customStoredBoxList",customStoredBoxList.contains(storageElement)) + assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) + }) + } + + @Test + def testRestoreCoinBoxes(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + val stateStorage = new SidechainStateStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) + + //Fill BackUpStorage with 5 ZenBoxes and 1 random element + storedBoxList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) + backupStorage.update(getVersion, storedBoxList.asJava).get + var exceptionThrown = false + try { + //Restore the SidechainStateStorage based on the BackupStorage + stateStorage.restoreBackup(backupStorage.getBoxIterator, getVersion.data()) + } catch { + case _:RuntimeException => exceptionThrown = true + } + assertTrue("CoinBoxes should not be restored!",exceptionThrown) + } + + def readStorage(sidechainStateStorageBoxIterator: BoxIterator): JArrayList[BackupBox] = { + val storedBoxes = new JArrayList[BackupBox]() + + var optionalBox = sidechainStateStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + storedBoxes.add(optionalBox.get) + optionalBox = sidechainStateStorageBoxIterator.nextBox + } + storedBoxes + } + @Test def testExceptions() : Unit = { var exceptionThrown = false diff --git a/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala b/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala index b11d4830db..5a719a853b 100644 --- a/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala +++ b/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala @@ -6,7 +6,7 @@ import com.horizen.cryptolibprovider.CryptoLibProvider import com.horizen.fixtures.{BoxFixture, StoreFixture} import com.horizen.librustsidechains.FieldElement import com.horizen.proposition.Proposition -import com.horizen.utils.{ByteArrayWrapper, BytesUtils, UtxoMerkleTreeLeafInfo, Pair => JPair} +import com.horizen.utils.{ByteArrayWrapper, BytesUtils, Utils, UtxoMerkleTreeLeafInfo, Pair => JPair} import org.junit.Test import org.mockito.{ArgumentMatchers, Mockito} import org.scalatestplus.junit.JUnitSuite @@ -74,7 +74,7 @@ class SidechainStateUtxoMerkleTreeStorageTest Mockito.when(mockedPhysicalStorage.get(ArgumentMatchers.any[ByteArrayWrapper]())).thenAnswer(answer => { val key: ByteArrayWrapper = answer.getArgument(0) utxoLeafInfoSeq - .find(entry => key.equals(utxoStorage.calculateKey(entry._1.id()))) + .find(entry => key.equals(Utils.calculateKey(entry._1.id()))) .map(entry => new ByteArrayWrapper(entry._2.bytes)) .asJava }) @@ -128,10 +128,10 @@ class SidechainStateUtxoMerkleTreeStorageTest val leafFE = utxoStorage.calculateLeaf(box) val leafInfo = UtxoMerkleTreeLeafInfo(leafFE.serializeFieldElement(), idx) leafFE.freeFieldElement() - new JPair(new ByteArrayWrapper(utxoStorage.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) + new JPair(new ByteArrayWrapper(Utils.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) }.asJava - val expectedToRemove = boxesToRemove.toSeq.map(id => utxoStorage.calculateKey(id.data)).asJava + val expectedToRemove = boxesToRemove.toSeq.map(id => Utils.calculateKey(id.data)).asJava assertEquals("Version is different.", version, actVersion) assertEquals("Update list is different.", expectedToUpdate, actToUpdate) @@ -164,7 +164,7 @@ class SidechainStateUtxoMerkleTreeStorageTest Mockito.when(mockedPhysicalStorage.get(ArgumentMatchers.any[ByteArrayWrapper]())).thenAnswer(answer => { val key: ByteArrayWrapper = answer.getArgument(0) utxoLeafInfoSeq - .find(entry => key.equals(utxoStorage.calculateKey(entry._1.id()))) + .find(entry => key.equals(Utils.calculateKey(entry._1.id()))) .map(entry => new ByteArrayWrapper(entry._2.bytes)) .asJava }) @@ -200,10 +200,10 @@ class SidechainStateUtxoMerkleTreeStorageTest val leafFE = utxoStorage.calculateLeaf(box) val leafInfo = UtxoMerkleTreeLeafInfo(leafFE.serializeFieldElement(), pos) leafFE.freeFieldElement() - new JPair(new ByteArrayWrapper(utxoStorage.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) + new JPair(new ByteArrayWrapper(Utils.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) }.asJava - val expectedToRemove = boxesToRemove.toSeq.map(id => utxoStorage.calculateKey(id.data)).asJava + val expectedToRemove = boxesToRemove.toSeq.map(id => Utils.calculateKey(id.data)).asJava assertEquals("Version is different.", version, actVersion) assertEquals("Update list is different.", expectedToUpdate, actToUpdate) diff --git a/tools/dbtool/pom.xml b/tools/dbtool/pom.xml index 2045907b1f..101d3c7192 100644 --- a/tools/dbtool/pom.xml +++ b/tools/dbtool/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.horizen sidechains-sdk-dbtools - 0.3.3 + 0.3.4 2022 UTF-8 @@ -16,7 +16,7 @@ io.horizen sidechains-sdk - 0.3.3 + 0.3.4 diff --git a/tools/sctool/pom.xml b/tools/sctool/pom.xml index fb0b35637d..f82d1643e8 100644 --- a/tools/sctool/pom.xml +++ b/tools/sctool/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.horizen sidechains-sdk-scbootstrappingtools - 0.3.3 + 0.3.4 2018 UTF-8 @@ -16,7 +16,7 @@ io.horizen sidechains-sdk - 0.3.3 + 0.3.4