From 6e71c2cf0a290afda9d52405d4ebf47acb53b650 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:12:08 +0000 Subject: [PATCH 01/26] Bump log4j-core from 2.17.0 to 2.17.1 in /sdk Bumps log4j-core from 2.17.0 to 2.17.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- sdk/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/pom.xml b/sdk/pom.xml index 10b2e702a0..40ea4bb5f4 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -94,7 +94,7 @@ org.apache.logging.log4j log4j-core - 2.17.0 + 2.17.1 From 0c67d0023e226917558fa7433c5d335194145a58 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Tue, 29 Mar 2022 11:37:30 +0200 Subject: [PATCH 02/26] First commit for backup storage --- .../horizen/storage/BackUpperInterface.java | 8 +++ .../java/com/horizen/storage/Storage.java | 3 ++ .../main/scala/com/horizen/SidechainApp.scala | 22 +++++++- .../com/horizen/SidechainAppModule.scala | 11 ++-- .../com/horizen/storage/BackupStorage.scala | 50 +++++++++++++++++++ .../storage/SidechainStateStorage.scala | 2 + .../storage/leveldb/VersionedLDBKVStore.scala | 5 +- .../VersionedLevelDbStorageAdapter.scala | 7 ++- .../storage/InMemoryStorageAdapter.scala | 5 +- 9 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 sdk/src/main/java/com/horizen/storage/BackUpperInterface.java create mode 100644 sdk/src/main/scala/com/horizen/storage/BackupStorage.scala diff --git a/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java b/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java new file mode 100644 index 0000000000..3230a6f053 --- /dev/null +++ b/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java @@ -0,0 +1,8 @@ +package com.horizen.storage; + +import com.horizen.companion.SidechainBoxesCompanion; +import org.iq80.leveldb.DBIterator; + +public interface BackUpperInterface { + void generateBackUp(DBIterator i, BackupStorage db, SidechainBoxesCompanion sbc); +} diff --git a/sdk/src/main/java/com/horizen/storage/Storage.java b/sdk/src/main/java/com/horizen/storage/Storage.java index 61ac503265..277c394e3a 100644 --- a/sdk/src/main/java/com/horizen/storage/Storage.java +++ b/sdk/src/main/java/com/horizen/storage/Storage.java @@ -4,6 +4,7 @@ import java.util.List; import com.horizen.utils.Pair; import com.horizen.utils.ByteArrayWrapper; +import org.iq80.leveldb.DBIterator; public interface Storage extends AutoCloseable { @@ -29,4 +30,6 @@ void update(ByteArrayWrapper version, List JByte} import java.util.{HashMap => JHashMap, List => JList} - import com.google.inject.name.Named import com.google.inject.Provides import com.horizen.api.http.ApplicationApiGroup @@ -10,7 +9,7 @@ import com.horizen.box.BoxSerializer import com.horizen.helper.{NodeViewHelper, NodeViewHelperImpl, SecretSubmitHelper, SecretSubmitHelperImpl, TransactionSubmitHelper, TransactionSubmitHelperImpl} import com.horizen.secret.SecretSerializer import com.horizen.state.ApplicationState -import com.horizen.storage.Storage +import com.horizen.storage.{BackUpperInterface, Storage} import com.horizen.transaction.TransactionSerializer import com.horizen.utils.Pair import com.horizen.wallet.ApplicationWallet @@ -55,9 +54,10 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { @Named("ConsensusStorage") consensusStorage: Storage, @Named("CustomApiGroups") customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") rejectedApiPaths : JList[Pair[String, String]], - @Named("ApplicationStopper") applicationStopper : SidechainAppStopper + @Named("ApplicationStopper") applicationStopper : SidechainAppStopper, + @Named("BackUpper") backUpper : BackUpperInterface - ): SidechainApp = { + ): SidechainApp = { synchronized { if (app == null) { app = new SidechainApp( @@ -79,7 +79,8 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { consensusStorage, customApiGroups, rejectedApiPaths, - applicationStopper + applicationStopper, + backUpper ) } } diff --git a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala new file mode 100644 index 0000000000..fd9babc99b --- /dev/null +++ b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala @@ -0,0 +1,50 @@ +package com.horizen.storage + +import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.{SidechainTypes} +import com.horizen.utils.ByteArrayWrapper +import org.iq80.leveldb.DBIterator + +import scala.util.Try +import java.util.{ArrayList => JArrayList} +import com.horizen.utils.{Pair => JPair} +import scorex.crypto.hash.Blake2b256 + +class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesCompanion) { + // Version - random number + // Key - byte array box Id + // No remove operation + + require(storage != null, "Storage must be NOT NULL.") + require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") + + def update (version : ByteArrayWrapper, boxToSaveList : java.util.List[SidechainTypes#SCB], + boxIdsRemoveList : java.util.List[Array[Byte]]) : Try[BackupStorage] = Try { + require(boxToSaveList != null, "List of WalletBoxes to add/update must be NOT NULL. Use empty List instead.") + require(boxIdsRemoveList != null, "List of Box IDs to remove must be NOT NULL. Use empty List instead.") + require(!boxToSaveList.contains(null), "WalletBox to add/update must be NOT NULL.") + require(!boxIdsRemoveList.contains(null), "BoxId to remove must be NOT NULL.") + + val removeList = new JArrayList[ByteArrayWrapper]() + val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() + + boxToSaveList.forEach(b => { + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(b.id()), + new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)))) + }) + System.out.println("UPDATE LIST SIZE "+updateList.size()+" VERSION "+version+" REMOVE LIST "+removeList.size()) + storage.update(version, + updateList, + removeList) + + this + } + + def calculateKey(boxId : Array[Byte]) : ByteArrayWrapper = { + new ByteArrayWrapper(Blake2b256.hash(boxId)) + } + + def getIterator: DBIterator = storage.getIterator + + def isEmpty: Boolean = storage.isEmpty +} diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index f2c8b856c0..2544f777b4 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -8,6 +8,7 @@ import com.horizen.box.{WithdrawalRequestBox, WithdrawalRequestBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ import com.horizen.utils.{ByteArrayWrapper, ListSerializer, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer, Pair => JPair, _} +import org.iq80.leveldb.DBIterator import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging @@ -307,4 +308,5 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain def isEmpty: Boolean = storage.isEmpty + def getIterator: DBIterator = storage.getIterator } diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala index b42310be1a..ca25e6c66d 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala @@ -1,7 +1,7 @@ package com.horizen.storage.leveldb import com.horizen.utils.ByteArrayWrapper -import org.iq80.leveldb.{DB, ReadOptions} +import org.iq80.leveldb.{DB, DBIterator, ReadOptions} import scala.collection.mutable import scala.util.{Failure, Success, Try} @@ -131,6 +131,9 @@ final class VersionedLDBKVStore(protected val db: DB, keepVersions: Int) extends def versionIdExists(versionId: VersionId): Boolean = versions.exists(new ByteArrayWrapper(_) == new ByteArrayWrapper(versionId)) + def getIterator(): DBIterator = { + db.iterator() + } } object VersionedLDBKVStore { diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala index 59922b5c13..f73402132d 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala @@ -3,11 +3,10 @@ package com.horizen.storage.leveldb import java.io.File import java.util import java.util.{Optional, List => JList} - import com.horizen.storage.Storage import com.horizen.storage.leveldb.LDBFactory.factory import com.horizen.utils.{Pair => JPair, _} -import org.iq80.leveldb.Options +import org.iq80.leveldb.{DBIterator, Options} import scala.collection.JavaConverters._ import scala.compat.java8.OptionConverters._ @@ -97,4 +96,8 @@ class VersionedLevelDbStorageAdapter(pathToDB: File) extends Storage{ override def isEmpty: Boolean = dataBase.versions.isEmpty override def numberOfVersions: Int = dataBase.versions.size + + override def getIterator(): DBIterator = { + dataBase.getIterator() + } } diff --git a/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala b/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala index 9e319abdc8..ae272970ae 100644 --- a/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala +++ b/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala @@ -1,8 +1,8 @@ package com.horizen.storage import java.util import java.util.{Optional, List => JList} - import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} +import org.iq80.leveldb.DBIterator import scala.collection.JavaConverters._ import scala.collection.mutable @@ -40,4 +40,7 @@ class InMemoryStorageAdapter(hashMap: mutable.HashMap[ByteArrayWrapper, ByteArra override def close(): Unit = {} def copy(): InMemoryStorageAdapter = new InMemoryStorageAdapter(hashMap.clone()) + + override def getIterator: DBIterator = ??? + } From 6d5b72a2229060e1dc7950a95e8cb07a3e407850 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 7 Apr 2022 15:38:29 +0200 Subject: [PATCH 03/26] [SCHAINS-422] Added Restore phase and Unit tests --- .../java/com/horizen/examples/BackUpper.java | 13 +++ .../examples/DefaultApplicationState.java | 6 ++ .../examples/DefaultApplicationWallet.java | 7 ++ .../com/horizen/examples/SimpleAppModule.java | 11 ++- .../com/horizen/state/ApplicationState.java | 4 + .../horizen/storage/BackUpperInterface.java | 2 +- .../main/java/com/horizen/utils/Utils.java | 23 ++++++ .../com/horizen/wallet/ApplicationWallet.java | 4 + .../main/scala/com/horizen/SidechainApp.scala | 17 ++-- .../com/horizen/SidechainAppModule.scala | 2 + .../com/horizen/SidechainNodeViewHolder.scala | 20 +++-- .../scala/com/horizen/SidechainState.scala | 20 ++++- .../scala/com/horizen/SidechainWallet.scala | 41 +++++++++- .../com/horizen/storage/BackupStorage.scala | 19 ++--- .../storage/SidechainStateStorage.scala | 58 ++++++++++++- .../storage/SidechainWalletBoxStorage.scala | 13 ++- .../com/horizen/storage/leveldb/package.scala | 1 + .../customtypes/CustomApplicationWallet.java | 6 +- .../customtypes/DefaultApplicationState.java | 5 +- .../customtypes/DefaultApplicationWallet.java | 6 +- .../com/horizen/SidechainWalletTest.scala | 81 +++++++++++++++++++ ...MockedSidechainNodeViewHolderFixture.scala | 2 +- .../SidechainNodeViewHolderFixture.scala | 3 + .../storage/SidechainStateStorageTest.scala | 55 ++++++++++++- 24 files changed, 373 insertions(+), 46 deletions(-) create mode 100644 examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java b/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java new file mode 100644 index 0000000000..4b2fde8916 --- /dev/null +++ b/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java @@ -0,0 +1,13 @@ +package com.horizen.examples; + +import com.horizen.companion.SidechainBoxesCompanion; +import com.horizen.storage.BackUpperInterface; +import com.horizen.storage.BackupStorage; +import org.iq80.leveldb.DBIterator; + +public class BackUpper implements BackUpperInterface { + @Override + public void generateBackUp(DBIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception { + + } +} diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java index e491d0a93a..990d3469dc 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java @@ -2,6 +2,7 @@ import com.horizen.block.SidechainBlock; import com.horizen.box.Box; +import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.state.ApplicationState; import com.horizen.state.SidechainStateReader; @@ -12,6 +13,7 @@ import com.horizen.utils.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.iq80.leveldb.DBIterator; import scala.util.Success; import scala.util.Try; @@ -91,4 +93,8 @@ public void closeStorages() { appStorage1.close(); appStorage2.close(); } + + public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + return new Success<>(this); + } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java index ddcd41e94e..15d90ec851 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java @@ -1,6 +1,7 @@ package com.horizen.examples; import com.horizen.box.Box; +import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter; @@ -10,6 +11,7 @@ import com.horizen.wallet.ApplicationWallet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.iq80.leveldb.DBIterator; import java.io.File; import java.util.ArrayList; @@ -84,4 +86,9 @@ public void closeStorages() { walletStorage1.close(); walletStorage2.close(); } + + @Override + public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + + } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java index 545cde3b3d..062066262d 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java @@ -18,6 +18,7 @@ import com.horizen.secret.Secret; import com.horizen.secret.SecretSerializer; import com.horizen.settings.SettingsReader; +import com.horizen.storage.BackUpperInterface; import com.horizen.storage.Storage; import com.horizen.state.*; import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter; @@ -65,7 +66,7 @@ public void configureApp() { File stateUtxoMerkleTreeStore = new File(dataDirAbsolutePath + "/stateUtxoMerkleTree"); File historyStore = new File(dataDirAbsolutePath + "/history"); File consensusStore = new File(dataDirAbsolutePath + "/consensusData"); - + File backupStore = new File(dataDirAbsolutePath + "/backupStorage"); // Here I can add my custom rest api and/or override existing one @@ -133,6 +134,9 @@ public void configureApp() { bind(Storage.class) .annotatedWith(Names.named("ConsensusStorage")) .toInstance(new VersionedLevelDbStorageAdapter(consensusStore)); + bind(Storage.class) + .annotatedWith(Names.named("BackupStorage")) + .toInstance(new VersionedLevelDbStorageAdapter(backupStore)); bind(new TypeLiteral> () {}) .annotatedWith(Names.named("CustomApiGroups")) @@ -145,5 +149,10 @@ public void configureApp() { bind(SidechainAppStopper.class) .annotatedWith(Names.named("ApplicationStopper")) .toInstance(applicationStopper); + + BackUpper backUpper = new BackUpper(); + bind(BackUpperInterface.class) + .annotatedWith(Names.named("BackUpper")) + .toInstance(backUpper); } } diff --git a/sdk/src/main/java/com/horizen/state/ApplicationState.java b/sdk/src/main/java/com/horizen/state/ApplicationState.java index 1ec65bebd5..11bc74bab7 100644 --- a/sdk/src/main/java/com/horizen/state/ApplicationState.java +++ b/sdk/src/main/java/com/horizen/state/ApplicationState.java @@ -2,11 +2,13 @@ import com.horizen.block.SidechainBlock; import com.horizen.box.Box; +import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.transaction.BoxTransaction; import java.util.List; +import org.iq80.leveldb.DBIterator; import scala.util.Try; // TO DO: provide access to HistoryReader @@ -28,6 +30,8 @@ public interface ApplicationState { // check that all storages of the application which are update by the sdk core, have the version corresponding to the // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); + + Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i); } diff --git a/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java b/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java index 3230a6f053..464513a78f 100644 --- a/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java +++ b/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java @@ -4,5 +4,5 @@ import org.iq80.leveldb.DBIterator; public interface BackUpperInterface { - void generateBackUp(DBIterator i, BackupStorage db, SidechainBoxesCompanion sbc); + void generateBackUp(DBIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception; } diff --git a/sdk/src/main/java/com/horizen/utils/Utils.java b/sdk/src/main/java/com/horizen/utils/Utils.java index cd269e7b9c..1bb45eb808 100644 --- a/sdk/src/main/java/com/horizen/utils/Utils.java +++ b/sdk/src/main/java/com/horizen/utils/Utils.java @@ -1,5 +1,8 @@ package com.horizen.utils; +import com.google.common.primitives.Longs; +import scorex.crypto.hash.Blake2b256; + import java.math.BigInteger; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -12,6 +15,8 @@ private Utils() {} public static final int SHA256_LENGTH = 32; + private static long latestUniqueVersion = 0; + public static byte[] doubleSHA256Hash(byte[] bytes) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); @@ -106,4 +111,22 @@ public static long readUint32BE(byte[] bytes, int offset) { ((bytes[offset + 2] & 0xffl) << 8) | (bytes[offset + 3] & 0xffl); } + + /* + Generate a uniqueVersion number - simply based to the current time millis + */ + public static synchronized byte[] uniqueVersion() { + long uniqueVersion = System.currentTimeMillis(); + while (latestUniqueVersion == uniqueVersion){ + try { + Utils.uniqueVersion().wait(5); + uniqueVersion = System.currentTimeMillis(); + } catch (InterruptedException e) { + //do nothing + } + } + latestUniqueVersion = uniqueVersion; + return (byte[]) Blake2b256.hash(Longs.toByteArray(uniqueVersion)); //hashed to be sure has 32bytes length + } + } diff --git a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java index ed473b1777..11967e4f37 100644 --- a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java +++ b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java @@ -2,9 +2,11 @@ import java.util.List; +import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.box.Box; +import org.iq80.leveldb.DBIterator; public interface ApplicationWallet { @@ -16,4 +18,6 @@ public interface ApplicationWallet { // check that all storages of the application which are update by the sdk core, have the version corresponding to the // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); + + void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i); } diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index 98c4f464f5..de903f797c 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -46,14 +46,12 @@ import com.horizen.network.SidechainNodeViewSynchronizer import com.horizen.websocket.client.{DefaultWebSocketReconnectionHandler, MainchainNodeChannelImpl, WebSocketChannel, WebSocketCommunicationClient, WebSocketConnector, WebSocketConnectorImpl, WebSocketReconnectionHandler} import com.horizen.websocket.server.WebSocketServerRef import com.horizen.serialization.JsonHorizenPublicKeyHashSerializer -import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter import com.horizen.transaction.mainchain.SidechainCreation import scorex.core.network.NetworkController.ReceivableMessages.ShutdownNetwork import java.util.concurrent.atomic.AtomicBoolean import org.iq80.leveldb.DBIterator -import java.io.File import scala.util.{Failure, Success, Try} @@ -74,6 +72,7 @@ class SidechainApp @Inject() @Named("WalletForgingBoxesInfoStorage") val walletForgingBoxesInfoStorage: Storage, @Named("WalletCswDataStorage") val walletCswDataStorage: Storage, @Named("ConsensusStorage") val consensusStorage: Storage, + @Named("BackupStorage") val backUpStorage: Storage, @Named("CustomApiGroups") val customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") val rejectedApiPaths : JList[Pair[String, String]], @Named("ApplicationStopper") val applicationStopper : SidechainAppStopper, @@ -264,6 +263,8 @@ class SidechainApp @Inject() sidechainSecretStorage.add(sidechainSecretsCompanion.parseBytes(BytesUtils.fromHexString(secretSchnorr))) } + protected val backupStorage = new BackupStorage(registerStorage(backUpStorage), sidechainBoxesCompanion) + override val nodeViewHolderRef: ActorRef = SidechainNodeViewHolderRef( sidechainSettings, sidechainHistoryStorage, @@ -276,6 +277,8 @@ class SidechainApp @Inject() sidechainWalletTransactionStorage, forgingBoxesMerklePathStorage, sidechainWalletCswDataStorage, + backupStorage, + sidechainBoxesCompanion, params, timeProvider, applicationWallet, @@ -441,18 +444,14 @@ class SidechainApp @Inject() actorSystem.eventStream.publish(SidechainAppEvents.SidechainApplicationStart) - def createBackup(storagePath: String): Unit = { - log.info(s"Starting backup at \n$storagePath") - - //Create the backup storage - val backupStore = new File(storagePath) - val backupDataStorage = new BackupStorage(registerStorage( new VersionedLevelDbStorageAdapter(backupStore)), sidechainBoxesCompanion) + def createBackup(): Unit = { + log.info(s"Starting sidechain backup...") //Take an iterator on the sidechainStateStorage val stateIterator: DBIterator = sidechainStateStorage.getIterator stateIterator.seekToFirst() //Perform the backup in the application level - backUpper.generateBackUp(stateIterator, backupDataStorage, sidechainBoxesCompanion) + backUpper.generateBackUp(stateIterator, backupStorage, sidechainBoxesCompanion) } } diff --git a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala index dab59433c6..42a141edeb 100644 --- a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala +++ b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala @@ -52,6 +52,7 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { @Named("WalletForgingBoxesInfoStorage") walletForgingBoxesInfoStorage: Storage, @Named("WalletCswDataStorage") walletCswDataStorage: Storage, @Named("ConsensusStorage") consensusStorage: Storage, + @Named("BackupStorage") backUpStorage: Storage, @Named("CustomApiGroups") customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") rejectedApiPaths : JList[Pair[String, String]], @Named("ApplicationStopper") applicationStopper : SidechainAppStopper, @@ -77,6 +78,7 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { walletForgingBoxesInfoStorage, walletCswDataStorage, consensusStorage, + backUpStorage, customApiGroups, rejectedApiPaths, applicationStopper, diff --git a/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala b/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala index 60c5be25b0..b6c7fbdcbe 100644 --- a/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala +++ b/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala @@ -4,10 +4,10 @@ package com.horizen import akka.actor.{ActorRef, ActorSystem, Props} import com.horizen.block.SidechainBlock import com.horizen.chain.FeePaymentsInfo +import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ import com.horizen.node.SidechainNodeView import com.horizen.params.NetworkParams -import com.horizen.proposition.{PublicKey25519Proposition, VrfPublicKey} import com.horizen.state.ApplicationState import com.horizen.storage._ import com.horizen.utils.BytesUtils @@ -39,6 +39,8 @@ class SidechainNodeViewHolder(sidechainSettings: SidechainSettings, walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, + sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -226,13 +228,13 @@ class SidechainNodeViewHolder(sidechainSettings: SidechainSettings, override protected def genesisState: (HIS, MS, VL, MP) = { val result = for { - state <- SidechainState.createGenesisState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, applicationState, genesisBlock) + state <- SidechainState.createGenesisState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, backupStorage, sidechainBoxesCompanion, params, applicationState, genesisBlock) (_: ModifierId, consensusEpochInfo: ConsensusEpochInfo) <- Success(state.getCurrentConsensusEpochInfo) withdrawalEpochNumber: Int <- Success(state.getWithdrawalEpochInfo.epoch) wallet <- SidechainWallet.createGenesisWallet(sidechainSettings.wallet.seed.getBytes, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, applicationWallet, + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, applicationWallet, genesisBlock, withdrawalEpochNumber, consensusEpochInfo) history <- SidechainHistory.createGenesisHistory(historyStorage, consensusDataStorage, params, genesisBlock, semanticBlockValidators(params), @@ -493,13 +495,15 @@ object SidechainNodeViewHolderRef { walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, + sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, applicationState: ApplicationState, genesisBlock: SidechainBlock): Props = Props(new SidechainNodeViewHolder(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, timeProvider, applicationWallet, applicationState, genesisBlock)) def apply(sidechainSettings: SidechainSettings, historyStorage: SidechainHistoryStorage, @@ -512,6 +516,8 @@ object SidechainNodeViewHolderRef { walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, + sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -519,7 +525,7 @@ object SidechainNodeViewHolderRef { genesisBlock: SidechainBlock) (implicit system: ActorSystem): ActorRef = system.actorOf(props(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, timeProvider, applicationWallet, applicationState, genesisBlock)) def apply(name: String, sidechainSettings: SidechainSettings, @@ -533,6 +539,8 @@ object SidechainNodeViewHolderRef { walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, + sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -540,5 +548,5 @@ object SidechainNodeViewHolderRef { genesisBlock: SidechainBlock) (implicit system: ActorSystem): ActorRef = system.actorOf(props(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock), name) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, timeProvider, applicationWallet, applicationState, genesisBlock), name) } diff --git a/sdk/src/main/scala/com/horizen/SidechainState.scala b/sdk/src/main/scala/com/horizen/SidechainState.scala index 7de6c942ec..a5f847e241 100644 --- a/sdk/src/main/scala/com/horizen/SidechainState.scala +++ b/sdk/src/main/scala/com/horizen/SidechainState.scala @@ -12,7 +12,7 @@ import com.horizen.node.NodeState import com.horizen.params.NetworkParams import com.horizen.proposition.{Proposition, PublicKey25519Proposition, VrfPublicKey} import com.horizen.state.ApplicationState -import com.horizen.storage.{SidechainStateForgerBoxStorage, SidechainStateStorage, SidechainStateUtxoMerkleTreeStorage} +import com.horizen.storage.{BackupStorage, SidechainStateForgerBoxStorage, SidechainStateStorage, SidechainStateUtxoMerkleTreeStorage} import com.horizen.transaction.MC2SCAggregatedTransaction import com.horizen.utils.{BlockFeeInfo, ByteArrayWrapper, BytesUtils, FeePaymentsUtils, MerkleTree, TimeToEpochUtils, WithdrawalEpochInfo, WithdrawalEpochUtils} import scorex.core._ @@ -22,6 +22,7 @@ import scorex.util.{ModifierId, ScorexLogging, bytesToId} import java.math.{BigDecimal, MathContext} import com.horizen.box.data.ZenBoxData +import com.horizen.companion.SidechainBoxesCompanion import com.horizen.cryptolibprovider.CryptoLibProvider import scala.collection.JavaConverters._ @@ -575,14 +576,25 @@ object SidechainState private[horizen] def createGenesisState(stateStorage: SidechainStateStorage, forgerBoxStorage: SidechainStateForgerBoxStorage, utxoMerkleTreeStorage: SidechainStateUtxoMerkleTreeStorage, + backupStorage: BackupStorage, + sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, applicationState: ApplicationState, genesisBlock: SidechainBlock): Try[SidechainState] = Try { - if (stateStorage.isEmpty) - new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) + if (stateStorage.isEmpty) { + if (!backupStorage.isEmpty) { + stateStorage.restoreBackup(backupStorage.getIterator, versionToBytes(idToVersion(genesisBlock.parentId))) + } + //This is for testing purpose + //stateStorage.readStorage() + + val sidechainState = new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) .applyModifier(genesisBlock).get - else + + applicationState.onApplicationRestore(sidechainState, sidechainBoxesCompanion, backupStorage.getIterator) + sidechainState + } else throw new RuntimeException("State storage is not empty!") } diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index 6220db1fbe..1d5a9df6a8 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -1,9 +1,11 @@ package com.horizen -import java.{lang, util} +import java.{lang} import java.util.{List => JList, Optional => JOptional} +import java.util.{ArrayList => JArrayList} import com.horizen.block.{MainchainBlockReferenceData, SidechainBlock} import com.horizen.box.{Box, CoinsBox, ForgerBox, ZenBox} +import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus.{ConsensusEpochInfo, ConsensusEpochNumber, ForgingStakeInfo} import com.horizen.wallet.ApplicationWallet import com.horizen.node.NodeWallet @@ -17,6 +19,7 @@ import com.horizen.utils.{ByteArrayWrapper, BytesUtils, ForgingStakeMerklePathIn import scorex.core.{VersionTag, bytesToVersion, idToVersion, versionToBytes, versionToId} import com.horizen.utils._ import scorex.util.{ModifierId, ScorexLogging} +import org.iq80.leveldb.DBIterator import scala.util.{Failure, Success, Try} import scala.collection.JavaConverters._ @@ -174,6 +177,38 @@ class SidechainWallet private[horizen] (seed: Array[Byte], this } + /*** + * This function is called at blockchain bootstrap time and preload the SidechainWallletBoxStorage with the boxes taken from the backup storage + * @param backupStorageIterator: iterator on the backup storage + * @param sidechainBoxesCompanion + */ + def scanBackUp(backupStorageIterator: DBIterator, sidechainBoxesCompanion: SidechainBoxesCompanion): Unit = { + val pubKeys = publicKeys() + backupStorageIterator.seekToFirst() + val walletBoxes = new JArrayList[WalletBox]() + val removeList = new JArrayList[Array[Byte]]() + + while(backupStorageIterator.hasNext) { + val entry = backupStorageIterator.next() + val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) + var nBoxes = 0 + + if (box.isSuccess) { + val currBox: SCB = box.get + if (pubKeys.contains(currBox.proposition())) { + walletBoxes.add(new WalletBox(currBox, System.currentTimeMillis())) + nBoxes += 1 + if (nBoxes == leveldb.Constants.BatchSize) { + walletBoxStorage.update(new ByteArrayWrapper(Utils.uniqueVersion()), walletBoxes.asScala.toList, removeList.asScala.toList).get + walletBoxes.clear() + nBoxes = 0 + } + } + } + } + walletBoxStorage.update(new ByteArrayWrapper(Utils.uniqueVersion()), walletBoxes.asScala.toList, removeList.asScala.toList).get + } + private[horizen] def calculateUtxoCswData(view: UtxoMerkleTreeView): Seq[CswData] = { boxes().filter(wb => wb.box.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]).map(wb => { val box = wb.box @@ -409,6 +444,8 @@ object SidechainWallet walletTransactionStorage: SidechainWalletTransactionStorage, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, + backupStorage: BackupStorage, + sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, applicationWallet: ApplicationWallet, genesisBlock: SidechainBlock, @@ -419,6 +456,8 @@ object SidechainWallet if (walletBoxStorage.isEmpty) { val genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, idToVersion(genesisBlock.parentId), applicationWallet) + genesisWallet.scanBackUp(backupStorage.getIterator, sidechainBoxesCompanion) + applicationWallet.onApplicationRestore(sidechainBoxesCompanion, backupStorage.getIterator) genesisWallet.scanPersistent(genesisBlock, withdrawalEpochNumber, Seq(), None).applyConsensusEpochInfo(consensusEpochInfo) } else diff --git a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala index fd9babc99b..69183aa27c 100644 --- a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala @@ -1,13 +1,11 @@ package com.horizen.storage import com.horizen.companion.SidechainBoxesCompanion -import com.horizen.{SidechainTypes} -import com.horizen.utils.ByteArrayWrapper +import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} import org.iq80.leveldb.DBIterator import scala.util.Try import java.util.{ArrayList => JArrayList} -import com.horizen.utils.{Pair => JPair} import scorex.crypto.hash.Blake2b256 class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesCompanion) { @@ -18,23 +16,14 @@ class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesC require(storage != null, "Storage must be NOT NULL.") require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") - def update (version : ByteArrayWrapper, boxToSaveList : java.util.List[SidechainTypes#SCB], - boxIdsRemoveList : java.util.List[Array[Byte]]) : Try[BackupStorage] = Try { + def update (version : ByteArrayWrapper, boxToSaveList : java.util.List[JPair[ByteArrayWrapper,ByteArrayWrapper]]) : Try[BackupStorage] = Try { require(boxToSaveList != null, "List of WalletBoxes to add/update must be NOT NULL. Use empty List instead.") - require(boxIdsRemoveList != null, "List of Box IDs to remove must be NOT NULL. Use empty List instead.") require(!boxToSaveList.contains(null), "WalletBox to add/update must be NOT NULL.") - require(!boxIdsRemoveList.contains(null), "BoxId to remove must be NOT NULL.") val removeList = new JArrayList[ByteArrayWrapper]() - val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() - boxToSaveList.forEach(b => { - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(b.id()), - new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)))) - }) - System.out.println("UPDATE LIST SIZE "+updateList.size()+" VERSION "+version+" REMOVE LIST "+removeList.size()) storage.update(version, - updateList, + boxToSaveList, removeList) this @@ -47,4 +36,6 @@ class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesC def getIterator: DBIterator = storage.getIterator def isEmpty: Boolean = storage.isEmpty + + def close: Unit = storage.close() } diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index 2544f777b4..0a33eae009 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -4,9 +4,10 @@ package com.horizen.storage import com.google.common.primitives.{Bytes, Ints} import com.horizen.SidechainTypes import com.horizen.block.{WithdrawalEpochCertificate, WithdrawalEpochCertificateSerializer} -import com.horizen.box.{WithdrawalRequestBox, WithdrawalRequestBoxSerializer} +import com.horizen.box.{CoinsBox, WithdrawalRequestBox, WithdrawalRequestBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ +import com.horizen.proposition.PublicKey25519Proposition import com.horizen.utils.{ByteArrayWrapper, ListSerializer, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer, Pair => JPair, _} import org.iq80.leveldb.DBIterator import scorex.crypto.hash.Blake2b256 @@ -309,4 +310,59 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain def isEmpty: Boolean = storage.isEmpty def getIterator: DBIterator = storage.getIterator + + /** + * This function restores the unspent boxes that come from a ceased sidechain by saving + * them into the SidechainStateStorage + * + * @param backupStorage: storage containing the boxes saved from the ceased sidechain + */ + def restoreBackup(backupStorageIterator: DBIterator, lastVersion: Array[Byte]): Unit = { + backupStorageIterator.seekToFirst() + val removeList = new JArrayList[ByteArrayWrapper]() + val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() + + while(backupStorageIterator.hasNext) { + val entry = backupStorageIterator.next() + val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) + + if (box.isSuccess) { + val currBox: SCB = box.get + if (!currBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) { + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(currBox.id()), + new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(currBox)))) + log.info("Restore Box id "+currBox.boxTypeId()) + if (updateList.size() == leveldb.Constants.BatchSize) { + if (backupStorageIterator.hasNext) + storage.update(new ByteArrayWrapper(Utils.uniqueVersion()),updateList, removeList) + else + storage.update(new ByteArrayWrapper(lastVersion),updateList, removeList) + updateList.clear() + } + } + } + } + if (updateList.size() != 0) + storage.update(new ByteArrayWrapper(lastVersion),updateList, removeList) + log.info("SidechainStateStorage restore completed successfully!") + } + + /** + * Read the box inside the SidechainStateStorage. + * It's used only for testing purpose. + */ + def readStorage(): Unit = { + System.out.println("READ STATE STORAGE..."); + val sidechainStateStorageIterator: DBIterator = getIterator + sidechainStateStorageIterator.seekToFirst() + while(sidechainStateStorageIterator.hasNext) { + val entry = sidechainStateStorageIterator.next() + val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) + + if(box.isSuccess) { + val currBox: SCB = box.get + System.out.println("STATE STORAGE BOX: "+currBox.boxTypeId()+" "+BytesUtils.toHexString(currBox.proposition().bytes())) + } + } + } } diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala index 45c918a6c8..9d436aec72 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala @@ -1,14 +1,17 @@ package com.horizen.storage -import com.horizen.box.Box + +import java.util.{ArrayList => JArrayList} +import com.horizen.utils.{Pair => JPair} +import com.horizen.utils.ByteArrayWrapper +import com.horizen.{SidechainTypes, WalletBox, WalletBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.box.Box import com.horizen.proposition.Proposition -import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import com.horizen.{SidechainTypes, WalletBox, WalletBoxSerializer} +import org.iq80.leveldb.DBIterator import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging -import java.util.{ArrayList => JArrayList} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.compat.java8.OptionConverters.RichOptionalGeneric @@ -162,4 +165,6 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid def isEmpty: Boolean = storage.isEmpty + def getIterator: DBIterator = storage.getIterator + } diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala index e8d610b8fa..e4af7fc242 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala @@ -10,6 +10,7 @@ import scorex.crypto.hash.Blake2b256 package object leveldb { object Constants { val HashLength: Int = 32 + val BatchSize: Int = 10000 } object Algos extends ScorexEncoding { diff --git a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java index a66d7d4665..2291846708 100644 --- a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java @@ -1,11 +1,12 @@ package com.horizen.customtypes; import com.horizen.box.Box; +import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.wallet.ApplicationWallet; +import org.iq80.leveldb.DBIterator; -import java.util.Collections; import java.util.List; public class CustomApplicationWallet implements ApplicationWallet { @@ -33,4 +34,7 @@ public void onRollback(byte[] blockId) { public boolean checkStoragesVersion(byte[] blockId) { return true; } + + public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java index f03a79b643..0062626527 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java @@ -2,14 +2,15 @@ import com.horizen.block.SidechainBlock; import com.horizen.box.Box; +import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.state.ApplicationState; import com.horizen.state.SidechainStateReader; import com.horizen.transaction.BoxTransaction; +import org.iq80.leveldb.DBIterator; import scala.util.Success; import scala.util.Try; -import java.util.Collections; import java.util.List; public class DefaultApplicationState implements ApplicationState { @@ -35,4 +36,6 @@ public Try onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } + + public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { return new Success<>(this); } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java index e72719c6ef..3268640520 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java @@ -1,11 +1,12 @@ package com.horizen.customtypes; import com.horizen.box.Box; +import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.wallet.ApplicationWallet; +import org.iq80.leveldb.DBIterator; -import java.util.Collections; import java.util.List; public class DefaultApplicationWallet implements ApplicationWallet { @@ -32,4 +33,7 @@ public void onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } + + public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + } } diff --git a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala index 18d9849766..e436a993c7 100644 --- a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala @@ -14,18 +14,23 @@ import com.horizen.params.MainNetParams import com.horizen.proposition._ import com.horizen.secret.{PrivateKey25519, Secret, SecretSerializer} import com.horizen.storage._ +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter import com.horizen.transaction.mainchain.{ForwardTransfer, SidechainCreation, SidechainRelatedMainchainOutput} import com.horizen.transaction.{BoxTransaction, MC2SCAggregatedTransaction, RegularTransaction} import com.horizen.utils.{ByteArrayWrapper, BytesUtils, CswData, ForgingStakeMerklePathInfo, ForwardTransferCswData, MerklePath, MerkleTree, Pair, UtxoCswData} import com.horizen.wallet.ApplicationWallet +import org.iq80.leveldb.DBIterator import org.junit.Assert._ import org.junit._ +import org.junit.rules.TemporaryFolder import org.mockito._ import org.scalatestplus.junit.JUnitSuite import org.scalatestplus.mockito._ import scorex.core.{VersionTag, bytesToId, bytesToVersion} import scorex.crypto.hash.Blake2b256 import scorex.util.ModifierId +import com.horizen.WalletBoxSerializer +import scorex.core.transaction.wallet.WalletBox import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer @@ -48,6 +53,7 @@ class SidechainWalletTest val boxList = new ListBuffer[WalletBox]() val storedBoxList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val boxVersions = new ListBuffer[ByteArrayWrapper]() + val boxToRestoreList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val secretList = new ListBuffer[Secret]() val storedSecretList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() @@ -69,6 +75,10 @@ class SidechainWalletTest val params = MainNetParams() + val _temporaryFolder = new TemporaryFolder() + + @Rule def temporaryFolder = _temporaryFolder + def boxIdToMerklePath(boxId: Array[Byte]): Array[Byte] = BytesUtils.reverseBytes(boxId) @Before @@ -123,6 +133,10 @@ class SidechainWalletTest boxList ++= getWalletBoxList(getZenBoxList(secretList.map(_.asInstanceOf[PrivateKey25519]).asJava)).asScala boxList += getWalletBox(getForgerBox(secretList.head.asInstanceOf[PrivateKey25519].publicImage())) + val customBoxList = new ListBuffer[SidechainTypes#SCB]() + customBoxList ++= getZenBoxList(secretList.map(_.asInstanceOf[PrivateKey25519]).asJava).asScala.toList + customBoxList += getZenBox(getPrivateKey25519.publicImage()) //This box shouldn't be included in the 'scanBackup' test result + boxVersions += getVersion for (b <- boxList) { @@ -134,6 +148,13 @@ class SidechainWalletTest }) } + for (b <- customBoxList) { + boxToRestoreList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new Pair(key,value) + }) + } // Mock get and update methods of BoxStorage Mockito.when(mockedBoxStorage.getAll).thenReturn(storedBoxList.asJava) @@ -377,6 +398,66 @@ class SidechainWalletTest sidechainWallet.scanPersistent(mockedBlock, withdrawalEpochNumber, feePaymentBoxes, Some(utxoMerkleTreeView)) } + @Test + def testScanBackUp(): Unit = { + val mockedSecretStorage: SidechainSecretStorage = mock[SidechainSecretStorage] + val mockedWalletTransactionStorage: SidechainWalletTransactionStorage = mock[SidechainWalletTransactionStorage] + val mockedForgingBoxesInfoStorage: ForgingBoxesInfoStorage = mock[ForgingBoxesInfoStorage] + val mockedCswDataStorage: SidechainWalletCswDataStorage = mock[SidechainWalletCswDataStorage] + val mockedApplicationWallet: ApplicationWallet = mock[ApplicationWallet] + Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>secretList.toList) + + //Create temporary WalletBoxStorage + val walletBoxStorageFile = temporaryFolder.newFolder("walletBoxStorage") + val walletBoxStorage = new SidechainWalletBoxStorage(new VersionedLevelDbStorageAdapter(walletBoxStorageFile), sidechainBoxesCompanion) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) + + boxToRestoreList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) + backupStorage.update(getVersion, boxToRestoreList.asJava).get + + val sidechainWallet = new SidechainWallet("seed".getBytes, + walletBoxStorage, + mockedSecretStorage, + mockedWalletTransactionStorage, + mockedForgingBoxesInfoStorage, + mockedCswDataStorage, + params, + mockedApplicationWallet) + + sidechainWallet.scanBackUp(backupStorage.getIterator, sidechainBoxesCompanion) + + assertTrue("Box stored to the backupStorage should be 7",boxToRestoreList.size == 7) + val storedBoxes = readStorage(walletBoxStorage) + + //Verify that we did take only the 5 Boxes + assertEquals("SidechainWalletBoxStorage should contains only the 5 CustomBoxes!",5, storedBoxes.size()) + val publicKeys = secretList.map(_.asInstanceOf[PrivateKey25519].publicImage()).asJava + storedBoxes.forEach(box => { + assertTrue("Restored Boxes propositions should be inside our wallet!", publicKeys.contains(box.box.proposition())) + }) + } + + def readStorage(walletBoxStorage: SidechainWalletBoxStorage): JArrayList[WalletBox] = { + val walletBoxStorageIterator: DBIterator = walletBoxStorage.getIterator + walletBoxStorageIterator.seekToFirst() + + val walletBoxSerializer: WalletBoxSerializer = new WalletBoxSerializer(sidechainBoxesCompanion) + val storedBoxes = new JArrayList[WalletBox]() + while(walletBoxStorageIterator.hasNext) { + val entry = walletBoxStorageIterator.next() + val box: Try[WalletBox] = walletBoxSerializer.parseBytesTry(entry.getValue) + + if(box.isSuccess) { + val currBox: WalletBox = box.get + storedBoxes.add(currBox) + } + } + storedBoxes + } + @Test def testRollback(): Unit = { val mockedWalletBoxStorage: SidechainWalletBoxStorage = mock[SidechainWalletBoxStorage] diff --git a/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala index 5e8408ea95..d9bbee4e59 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala @@ -11,7 +11,7 @@ class MockedSidechainNodeViewHolder(sidechainSettings: SidechainSettings, state: SidechainState, wallet: SidechainWallet, mempool: SidechainMemoryPool) - extends SidechainNodeViewHolder(sidechainSettings, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null) { + extends SidechainNodeViewHolder(sidechainSettings, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null) { override def dumpStorages: Unit = {} diff --git a/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala index 2d0610a6dc..4eb12f2142 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala @@ -100,6 +100,7 @@ trait SidechainNodeViewHolderFixture val sidechainWalletTransactionStorage = new SidechainWalletTransactionStorage(getStorage(), sidechainTransactionsCompanion) val forgingBoxesMerklePathStorage = new ForgingBoxesInfoStorage(getStorage()) val cswDataStorage = new SidechainWalletCswDataStorage(getStorage()) + val backupStorage = new BackupStorage(getStorage(), sidechainBoxesCompanion) // Append genesis secrets if we start the node first time if(sidechainSecretStorage.isEmpty) { @@ -119,6 +120,8 @@ trait SidechainNodeViewHolderFixture sidechainWalletTransactionStorage, forgingBoxesMerklePathStorage, cswDataStorage, + backupStorage, + sidechainBoxesCompanion, params, timeProvider, defaultApplicationWallet, diff --git a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala index 6d05106638..39b8ab7cd8 100644 --- a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala +++ b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala @@ -2,13 +2,15 @@ package com.horizen.storage import com.google.common.primitives.Ints import com.horizen.SidechainTypes -import com.horizen.box.BoxSerializer +import com.horizen.box.{BoxSerializer, CoinsBox} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus.{ConsensusEpochNumber, intToConsensusEpochNumber} import com.horizen.customtypes.{CustomBox, CustomBoxSerializer} import com.horizen.fixtures.{SecretFixture, StoreFixture, TransactionFixture} +import com.horizen.proposition.PublicKey25519Proposition import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter import com.horizen.utils.{BlockFeeInfo, BlockFeeInfoSerializer, ByteArrayWrapper, Pair, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer} +import org.iq80.leveldb.DBIterator import org.junit.Assert._ import org.junit._ import org.mockito.{ArgumentMatchers, Mockito} @@ -21,6 +23,9 @@ import java.util.{ArrayList => JArrayList, HashMap => JHashMap, Optional => JOpt import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.util.Try +import org.junit.Rule +import org.junit.rules.TemporaryFolder + class SidechainStateStorageTest extends JUnitSuite @@ -43,6 +48,10 @@ class SidechainStateStorageTest val consensusEpoch: ConsensusEpochNumber = intToConsensusEpochNumber(1) + val _temporaryFolder = new TemporaryFolder() + + @Rule def temporaryFolder = _temporaryFolder + @Before def setUp(): Unit = { @@ -135,6 +144,50 @@ class SidechainStateStorageTest assertEquals("Storage should return existing Box.", boxList(3), stateStorage.getBox(boxList(3).id()).get) } + @Test + def testRestore(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + val stateStorage = new SidechainStateStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) + + //Fill BackUpStorage with 5 ZenBoxes and 5 CustomBoxes and 1 random element + storedBoxList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) + backupStorage.update(getVersion, storedBoxList.asJava).get + + //Restore the SidechainStateStorage based on the BackupStorage + stateStorage.restoreBackup(backupStorage.getIterator, getVersion.data()) + + //Read the SidechainStateStorage + val storedBoxes = readStorage(stateStorage) + + //Verify that we did take only the 5 CustomBoxes + assertEquals("SidechainStateStorage should contains only the 5 CustomBoxes!",storedBoxes.size(), 5) + storedBoxes.forEach(box => { + assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) + }) + } + + def readStorage(sidechainStateStorage: SidechainStateStorage): JArrayList[SCB] = { + val sidechainStateStorageIterator: DBIterator = sidechainStateStorage.getIterator + sidechainStateStorageIterator.seekToFirst() + + val storedBoxes = new JArrayList[SCB]() + while(sidechainStateStorageIterator.hasNext) { + val entry = sidechainStateStorageIterator.next() + val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) + + if(box.isSuccess) { + val currBox: SCB = box.get + storedBoxes.add(currBox) + } + } + storedBoxes + } + @Test def testExceptions() : Unit = { var exceptionThrown = false From 51b43e6ef7291bb392721bb12b7a4c7e81d28435 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Fri, 8 Apr 2022 14:12:36 +0200 Subject: [PATCH 04/26] [SCHAINS-466] Added SidechainStateStorage rollback --- .../main/scala/com/horizen/SidechainApp.scala | 54 ++++++++++++++++--- .../com/horizen/SidechainWalletTest.scala | 2 - 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index de903f797c..82598aade3 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -7,6 +7,8 @@ import akka.actor.ActorRef import akka.http.scaladsl.Http import akka.http.scaladsl.server.{ExceptionHandler, RejectionHandler} import akka.stream.ActorMaterializer +import akka.pattern.ask +import akka.util.Timeout import com.google.inject.name.Named import com.google.inject.{Inject, _} import com.horizen.api.http.{SidechainSubmitterApiRoute, _} @@ -26,7 +28,7 @@ import com.horizen.secret.SecretSerializer import com.horizen.state.ApplicationState import com.horizen.storage._ import com.horizen.transaction._ -import com.horizen.utils.{BlockUtils, BytesUtils, Pair} +import com.horizen.utils.{BlockUtils, ByteArrayWrapper, BytesUtils, Pair} import com.horizen.wallet.ApplicationWallet import scorex.core.api.http.ApiRoute import scorex.core.app.Application @@ -51,7 +53,11 @@ import scorex.core.network.NetworkController.ReceivableMessages.ShutdownNetwork import java.util.concurrent.atomic.AtomicBoolean import org.iq80.leveldb.DBIterator +import scorex.core.NodeViewHolder.CurrentView +import scorex.core.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView +import scala.concurrent.Future +import scala.concurrent.duration.FiniteDuration import scala.util.{Failure, Success, Try} @@ -85,6 +91,7 @@ class SidechainApp @Inject() override type TX = SidechainTypes#SCBT override type PMOD = SidechainBlock override type NVHT = SidechainNodeViewHolder + type View = CurrentView[SidechainHistory, SidechainState, SidechainWallet, SidechainMemoryPool] override implicit lazy val settings: ScorexSettings = sidechainSettings.scorexSettings @@ -444,14 +451,49 @@ class SidechainApp @Inject() actorSystem.eventStream.publish(SidechainAppEvents.SidechainApplicationStart) + val timeoutDuration: FiniteDuration = settings.restApi.timeout + + implicit val timeout: Timeout = Timeout(timeoutDuration) + + /*** + * Retrieve the SidechainBlockId needed to rollback the SidechainStateStorage for the backup. + * It's calculated by the following formula: + * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochçength -1 + * @param view + * @return sidechainBlockId to rollback or Exception if it's not found + */ + def getSidechainBlockIdForBackUpRollback(view: View):Array[Byte] = { + val currentEpoch = view.state.getWithdrawalEpochInfo.epoch + System.out.println("CURRENT EPOCH "+currentEpoch) + val genesisMcBlockHeight = view.history.getMainchainCreationBlockHeight + val withdrawalEpochLength = sidechainSettings.genesisData.withdrawalEpochLength + val blockHeightToRollback = genesisMcBlockHeight + (currentEpoch -2) * withdrawalEpochLength - 1 + val mainchainBlockReferenceInfo = view.history.getMainchainBlockReferenceInfoByMainchainBlockHeight(blockHeightToRollback).get() + mainchainBlockReferenceInfo.getMainchainReferenceDataSidechainBlockId + } + def createBackup(): Unit = { log.info(s"Starting sidechain backup...") - //Take an iterator on the sidechainStateStorage - val stateIterator: DBIterator = sidechainStateStorage.getIterator - stateIterator.seekToFirst() + val checkAsFuture = (nodeViewHolderRef ? GetDataFromCurrentView(getSidechainBlockIdForBackUpRollback)).asInstanceOf[Future[Try[Array[Byte]]]] + checkAsFuture.onComplete{ + case Success(Success(blockId)) => + log.info(s"Rollback of the SidechainStateStorage to version: ${BytesUtils.toHexString(blockId)}") + if (sidechainStateStorage.rollback(new ByteArrayWrapper(blockId)).isSuccess) { + log.info(s"Rollback of the SidechainStateStorage completed successfully!") + + //Take an iterator on the sidechainStateStorage + val stateIterator: DBIterator = sidechainStateStorage.getIterator + stateIterator.seekToFirst() + + //Perform the backup in the application level + backUpper.generateBackUp(stateIterator, backupStorage, sidechainBoxesCompanion) + } + else + log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...") + case Failure(_) => + log.info("Failed to retrieve the Sidechain block ID fo the SidechainStateStorage rollback! ") - //Perform the backup in the application level - backUpper.generateBackUp(stateIterator, backupStorage, sidechainBoxesCompanion) + } } } diff --git a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala index e436a993c7..0c74ef7930 100644 --- a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala @@ -29,8 +29,6 @@ import org.scalatestplus.mockito._ import scorex.core.{VersionTag, bytesToId, bytesToVersion} import scorex.crypto.hash.Blake2b256 import scorex.util.ModifierId -import com.horizen.WalletBoxSerializer -import scorex.core.transaction.wallet.WalletBox import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer From 62a05f496624b47ca7d635b9bb670aee4da4aa59 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Mon, 11 Apr 2022 11:10:17 +0200 Subject: [PATCH 05/26] [SCHAINS-466] Fix in createBackup --- sdk/src/main/scala/com/horizen/SidechainApp.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index 82598aade3..eb5400b33f 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -464,7 +464,6 @@ class SidechainApp @Inject() */ def getSidechainBlockIdForBackUpRollback(view: View):Array[Byte] = { val currentEpoch = view.state.getWithdrawalEpochInfo.epoch - System.out.println("CURRENT EPOCH "+currentEpoch) val genesisMcBlockHeight = view.history.getMainchainCreationBlockHeight val withdrawalEpochLength = sidechainSettings.genesisData.withdrawalEpochLength val blockHeightToRollback = genesisMcBlockHeight + (currentEpoch -2) * withdrawalEpochLength - 1 @@ -475,9 +474,9 @@ class SidechainApp @Inject() def createBackup(): Unit = { log.info(s"Starting sidechain backup...") - val checkAsFuture = (nodeViewHolderRef ? GetDataFromCurrentView(getSidechainBlockIdForBackUpRollback)).asInstanceOf[Future[Try[Array[Byte]]]] + val checkAsFuture = (nodeViewHolderRef ? GetDataFromCurrentView(getSidechainBlockIdForBackUpRollback)).asInstanceOf[Future[Array[Byte]]] checkAsFuture.onComplete{ - case Success(Success(blockId)) => + case Success(blockId) => log.info(s"Rollback of the SidechainStateStorage to version: ${BytesUtils.toHexString(blockId)}") if (sidechainStateStorage.rollback(new ByteArrayWrapper(blockId)).isSuccess) { log.info(s"Rollback of the SidechainStateStorage completed successfully!") From 2fcd6dfd1d875008a093c1b690d9051af44806eb Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Mon, 11 Apr 2022 11:34:37 +0200 Subject: [PATCH 06/26] [SCHAINS-466] Removed debug functions --- .../scala/com/horizen/SidechainState.scala | 2 -- .../storage/SidechainStateStorage.scala | 19 ------------------- 2 files changed, 21 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/SidechainState.scala b/sdk/src/main/scala/com/horizen/SidechainState.scala index a5f847e241..07ad06dd2a 100644 --- a/sdk/src/main/scala/com/horizen/SidechainState.scala +++ b/sdk/src/main/scala/com/horizen/SidechainState.scala @@ -586,8 +586,6 @@ object SidechainState if (!backupStorage.isEmpty) { stateStorage.restoreBackup(backupStorage.getIterator, versionToBytes(idToVersion(genesisBlock.parentId))) } - //This is for testing purpose - //stateStorage.readStorage() val sidechainState = new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) .applyModifier(genesisBlock).get diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index 0a33eae009..a64275af3d 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -346,23 +346,4 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain storage.update(new ByteArrayWrapper(lastVersion),updateList, removeList) log.info("SidechainStateStorage restore completed successfully!") } - - /** - * Read the box inside the SidechainStateStorage. - * It's used only for testing purpose. - */ - def readStorage(): Unit = { - System.out.println("READ STATE STORAGE..."); - val sidechainStateStorageIterator: DBIterator = getIterator - sidechainStateStorageIterator.seekToFirst() - while(sidechainStateStorageIterator.hasNext) { - val entry = sidechainStateStorageIterator.next() - val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) - - if(box.isSuccess) { - val currBox: SCB = box.get - System.out.println("STATE STORAGE BOX: "+currBox.boxTypeId()+" "+BytesUtils.toHexString(currBox.proposition().bytes())) - } - } - } } From 24df675347c010dfbe42fa51254130e0ba1e28f9 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Fri, 22 Apr 2022 09:43:34 +0200 Subject: [PATCH 07/26] [SCHAINS-422] Fixes after code review --- .../main/scala/com/horizen/SidechainApp.scala | 33 +++++++++++-------- .../scala/com/horizen/SidechainState.scala | 16 ++++++--- .../scala/com/horizen/SidechainWallet.scala | 8 +++-- .../com/horizen/storage/BackupStorage.scala | 3 +- .../storage/SidechainStateStorage.scala | 5 +-- .../com/horizen/storage/leveldb/package.scala | 2 ++ .../com/horizen/SidechainWalletTest.scala | 2 +- 7 files changed, 45 insertions(+), 24 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index eb5400b33f..64a7d1486c 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -478,20 +478,27 @@ class SidechainApp @Inject() checkAsFuture.onComplete{ case Success(blockId) => log.info(s"Rollback of the SidechainStateStorage to version: ${BytesUtils.toHexString(blockId)}") - if (sidechainStateStorage.rollback(new ByteArrayWrapper(blockId)).isSuccess) { - log.info(s"Rollback of the SidechainStateStorage completed successfully!") - - //Take an iterator on the sidechainStateStorage - val stateIterator: DBIterator = sidechainStateStorage.getIterator - stateIterator.seekToFirst() - - //Perform the backup in the application level - backUpper.generateBackUp(stateIterator, backupStorage, sidechainBoxesCompanion) + sidechainStateStorage.rollback(new ByteArrayWrapper(blockId)) match { + case Success(stateStorage) => + log.info(s"Rollback of the SidechainStateStorage completed successfully!") + + //Take an iterator on the sidechainStateStorage + val stateIterator: DBIterator = stateStorage.getIterator + stateIterator.seekToFirst() + + //Perform the backup in the application level + try { + backUpper.generateBackUp(stateIterator, backupStorage, sidechainBoxesCompanion) + } catch { + case t: Throwable => + log.error("Error during the Backup generation: ",t.getMessage) + throw new RuntimeException("Error during the Backup generation: "+t.getMessage) + } + case Failure(e) => + log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...") } - else - log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...") - case Failure(_) => - log.info("Failed to retrieve the Sidechain block ID fo the SidechainStateStorage rollback! ") + case Failure(e) => + log.info("Failed to retrieve the Sidechain block ID fo the SidechainStateStorage rollback! ", e.getMessage) } } diff --git a/sdk/src/main/scala/com/horizen/SidechainState.scala b/sdk/src/main/scala/com/horizen/SidechainState.scala index 07ad06dd2a..402840056e 100644 --- a/sdk/src/main/scala/com/horizen/SidechainState.scala +++ b/sdk/src/main/scala/com/horizen/SidechainState.scala @@ -588,10 +588,18 @@ object SidechainState } val sidechainState = new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) - .applyModifier(genesisBlock).get - - applicationState.onApplicationRestore(sidechainState, sidechainBoxesCompanion, backupStorage.getIterator) - sidechainState + if (!backupStorage.isEmpty) { + applicationState.onApplicationRestore(sidechainState, sidechainBoxesCompanion, backupStorage.getIterator) match { + case Success(updatedState) => + new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), updatedState) + .applyModifier(genesisBlock).get + case Failure(_) => + throw new RuntimeException("State storage is not empty!") + } + } + else { + sidechainState.applyModifier(genesisBlock).get + } } else throw new RuntimeException("State storage is not empty!") } diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index 1d5a9df6a8..86dc8cb51b 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -187,11 +187,11 @@ class SidechainWallet private[horizen] (seed: Array[Byte], backupStorageIterator.seekToFirst() val walletBoxes = new JArrayList[WalletBox]() val removeList = new JArrayList[Array[Byte]]() + var nBoxes = 0 while(backupStorageIterator.hasNext) { val entry = backupStorageIterator.next() val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) - var nBoxes = 0 if (box.isSuccess) { val currBox: SCB = box.get @@ -206,7 +206,9 @@ class SidechainWallet private[horizen] (seed: Array[Byte], } } } - walletBoxStorage.update(new ByteArrayWrapper(Utils.uniqueVersion()), walletBoxes.asScala.toList, removeList.asScala.toList).get + if (nBoxes > 0) { + walletBoxStorage.update(new ByteArrayWrapper(Utils.uniqueVersion()), walletBoxes.asScala.toList, removeList.asScala.toList).get + } } private[horizen] def calculateUtxoCswData(view: UtxoMerkleTreeView): Seq[CswData] = { @@ -454,10 +456,10 @@ object SidechainWallet ) : Try[SidechainWallet] = Try { if (walletBoxStorage.isEmpty) { + applicationWallet.onApplicationRestore(sidechainBoxesCompanion, backupStorage.getIterator) val genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, idToVersion(genesisBlock.parentId), applicationWallet) genesisWallet.scanBackUp(backupStorage.getIterator, sidechainBoxesCompanion) - applicationWallet.onApplicationRestore(sidechainBoxesCompanion, backupStorage.getIterator) genesisWallet.scanPersistent(genesisBlock, withdrawalEpochNumber, Seq(), None).applyConsensusEpochInfo(consensusEpochInfo) } else diff --git a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala index 69183aa27c..78b8b8dd44 100644 --- a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala @@ -17,8 +17,9 @@ class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesC require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") def update (version : ByteArrayWrapper, boxToSaveList : java.util.List[JPair[ByteArrayWrapper,ByteArrayWrapper]]) : Try[BackupStorage] = Try { - require(boxToSaveList != null, "List of WalletBoxes to add/update must be NOT NULL. Use empty List instead.") + require(boxToSaveList != null, "List of WalletBoxes to add/update must be NOT NULL.") require(!boxToSaveList.contains(null), "WalletBox to add/update must be NOT NULL.") + require(!boxToSaveList.isEmpty, "List of WalletBoxes to add/update must be NOT EMPTY.") val removeList = new JArrayList[ByteArrayWrapper]() diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index a64275af3d..fdb58305cc 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -321,6 +321,7 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain backupStorageIterator.seekToFirst() val removeList = new JArrayList[ByteArrayWrapper]() val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() + val lastVersionWrapper = new ByteArrayWrapper(lastVersion) while(backupStorageIterator.hasNext) { val entry = backupStorageIterator.next() @@ -336,14 +337,14 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain if (backupStorageIterator.hasNext) storage.update(new ByteArrayWrapper(Utils.uniqueVersion()),updateList, removeList) else - storage.update(new ByteArrayWrapper(lastVersion),updateList, removeList) + storage.update(lastVersionWrapper,updateList, removeList) updateList.clear() } } } } if (updateList.size() != 0) - storage.update(new ByteArrayWrapper(lastVersion),updateList, removeList) + storage.update(lastVersionWrapper,updateList, removeList) log.info("SidechainStateStorage restore completed successfully!") } } diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala index e4af7fc242..1c7ef41e49 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/package.scala @@ -10,6 +10,8 @@ import scorex.crypto.hash.Blake2b256 package object leveldb { object Constants { val HashLength: Int = 32 + //Batch size used in the SidechainWallet and SidechainState restore method. + //TODO: Investigate what could be a real good value. val BatchSize: Int = 10000 } diff --git a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala index 0c74ef7930..238dd0e45e 100644 --- a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala @@ -427,7 +427,7 @@ class SidechainWalletTest sidechainWallet.scanBackUp(backupStorage.getIterator, sidechainBoxesCompanion) - assertTrue("Box stored to the backupStorage should be 7",boxToRestoreList.size == 7) + assertTrue("Box stored to the backupStorage should be 7: 5 regular box + 1 new address + 1 fake ",boxToRestoreList.size == 7) val storedBoxes = readStorage(walletBoxStorage) //Verify that we did take only the 5 Boxes From e357b493b241127cb51c996ed0da246a2f597eb2 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Fri, 22 Apr 2022 10:50:27 +0200 Subject: [PATCH 08/26] [SCHAINS-422] Fixes after code review --- .../java/com/horizen/examples/BackUpper.java | 4 +- .../examples/DefaultApplicationState.java | 5 ++- .../examples/DefaultApplicationWallet.java | 4 +- .../com/horizen/state/ApplicationState.java | 4 +- .../horizen/storage/BackUpperInterface.java | 3 +- .../java/com/horizen/storage/Storage.java | 3 +- .../com/horizen/storage/StorageIterator.java | 21 ++++++++++ .../com/horizen/wallet/ApplicationWallet.java | 4 +- .../main/scala/com/horizen/SidechainApp.scala | 3 +- .../scala/com/horizen/SidechainWallet.scala | 3 +- .../com/horizen/storage/BackupStorage.scala | 3 +- .../storage/SidechainStateStorage.scala | 5 +-- .../storage/SidechainWalletBoxStorage.scala | 3 +- .../storage/leveldb/DatabaseIterator.java | 40 +++++++++++++++++++ .../storage/leveldb/VersionedLDBKVStore.scala | 8 ++-- .../VersionedLevelDbStorageAdapter.scala | 6 +-- .../customtypes/CustomApplicationWallet.java | 5 ++- .../customtypes/DefaultApplicationState.java | 5 ++- .../customtypes/DefaultApplicationWallet.java | 5 ++- .../com/horizen/SidechainWalletTest.scala | 3 +- .../storage/InMemoryStorageAdapter.scala | 3 +- .../storage/SidechainStateStorageTest.scala | 3 +- 22 files changed, 100 insertions(+), 43 deletions(-) create mode 100644 sdk/src/main/java/com/horizen/storage/StorageIterator.java create mode 100644 sdk/src/main/scala/com/horizen/storage/leveldb/DatabaseIterator.java diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java b/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java index 4b2fde8916..315dcd0e44 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java @@ -3,11 +3,11 @@ import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.storage.BackUpperInterface; import com.horizen.storage.BackupStorage; -import org.iq80.leveldb.DBIterator; +import com.horizen.storage.StorageIterator; public class BackUpper implements BackUpperInterface { @Override - public void generateBackUp(DBIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception { + public void generateBackUp(StorageIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception { } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java index 990d3469dc..e642797243 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java @@ -13,7 +13,7 @@ import com.horizen.utils.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.iq80.leveldb.DBIterator; +import com.horizen.storage.StorageIterator; import scala.util.Success; import scala.util.Try; @@ -94,7 +94,8 @@ public void closeStorages() { appStorage2.close(); } - public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + + public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { return new Success<>(this); } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java index 15d90ec851..39939a4718 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java @@ -11,7 +11,7 @@ import com.horizen.wallet.ApplicationWallet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.iq80.leveldb.DBIterator; +import com.horizen.storage.StorageIterator; import java.io.File; import java.util.ArrayList; @@ -88,7 +88,7 @@ public void closeStorages() { } @Override - public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { } } diff --git a/sdk/src/main/java/com/horizen/state/ApplicationState.java b/sdk/src/main/java/com/horizen/state/ApplicationState.java index 11bc74bab7..896d87cc63 100644 --- a/sdk/src/main/java/com/horizen/state/ApplicationState.java +++ b/sdk/src/main/java/com/horizen/state/ApplicationState.java @@ -4,11 +4,11 @@ import com.horizen.box.Box; import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; +import com.horizen.storage.StorageIterator; import com.horizen.transaction.BoxTransaction; import java.util.List; -import org.iq80.leveldb.DBIterator; import scala.util.Try; // TO DO: provide access to HistoryReader @@ -31,7 +31,7 @@ public interface ApplicationState { // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); - Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i); + Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i); } diff --git a/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java b/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java index 464513a78f..f53b65fb6b 100644 --- a/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java +++ b/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java @@ -1,8 +1,7 @@ package com.horizen.storage; import com.horizen.companion.SidechainBoxesCompanion; -import org.iq80.leveldb.DBIterator; public interface BackUpperInterface { - void generateBackUp(DBIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception; + void generateBackUp(StorageIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception; } diff --git a/sdk/src/main/java/com/horizen/storage/Storage.java b/sdk/src/main/java/com/horizen/storage/Storage.java index 277c394e3a..9c1dfbb510 100644 --- a/sdk/src/main/java/com/horizen/storage/Storage.java +++ b/sdk/src/main/java/com/horizen/storage/Storage.java @@ -4,7 +4,6 @@ import java.util.List; import com.horizen.utils.Pair; import com.horizen.utils.ByteArrayWrapper; -import org.iq80.leveldb.DBIterator; public interface Storage extends AutoCloseable { @@ -31,5 +30,5 @@ void update(ByteArrayWrapper version, List>, Closeable +{ + /** + * Repositions the iterator so the key of the next BlockElement + * returned greater than or equal to the specified targetKey. + */ + void seek(byte[] key); + + /** + * Repositions the iterator so is is at the beginning of the Database. + */ + void seekToFirst(); + +} \ No newline at end of file diff --git a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java index 11967e4f37..f63b9e96d5 100644 --- a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java +++ b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java @@ -6,7 +6,7 @@ import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.box.Box; -import org.iq80.leveldb.DBIterator; +import com.horizen.storage.StorageIterator; public interface ApplicationWallet { @@ -19,5 +19,5 @@ public interface ApplicationWallet { // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); - void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i); + void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i); } diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index 64a7d1486c..84d4a21ef8 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -52,7 +52,6 @@ import com.horizen.transaction.mainchain.SidechainCreation import scorex.core.network.NetworkController.ReceivableMessages.ShutdownNetwork import java.util.concurrent.atomic.AtomicBoolean -import org.iq80.leveldb.DBIterator import scorex.core.NodeViewHolder.CurrentView import scorex.core.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView @@ -483,7 +482,7 @@ class SidechainApp @Inject() log.info(s"Rollback of the SidechainStateStorage completed successfully!") //Take an iterator on the sidechainStateStorage - val stateIterator: DBIterator = stateStorage.getIterator + val stateIterator: StorageIterator = stateStorage.getIterator stateIterator.seekToFirst() //Perform the backup in the application level diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index 86dc8cb51b..abf29afc65 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -19,7 +19,6 @@ import com.horizen.utils.{ByteArrayWrapper, BytesUtils, ForgingStakeMerklePathIn import scorex.core.{VersionTag, bytesToVersion, idToVersion, versionToBytes, versionToId} import com.horizen.utils._ import scorex.util.{ModifierId, ScorexLogging} -import org.iq80.leveldb.DBIterator import scala.util.{Failure, Success, Try} import scala.collection.JavaConverters._ @@ -182,7 +181,7 @@ class SidechainWallet private[horizen] (seed: Array[Byte], * @param backupStorageIterator: iterator on the backup storage * @param sidechainBoxesCompanion */ - def scanBackUp(backupStorageIterator: DBIterator, sidechainBoxesCompanion: SidechainBoxesCompanion): Unit = { + def scanBackUp(backupStorageIterator: StorageIterator, sidechainBoxesCompanion: SidechainBoxesCompanion): Unit = { val pubKeys = publicKeys() backupStorageIterator.seekToFirst() val walletBoxes = new JArrayList[WalletBox]() diff --git a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala index 78b8b8dd44..a48a74c4b0 100644 --- a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala @@ -2,7 +2,6 @@ package com.horizen.storage import com.horizen.companion.SidechainBoxesCompanion import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import org.iq80.leveldb.DBIterator import scala.util.Try import java.util.{ArrayList => JArrayList} @@ -34,7 +33,7 @@ class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesC new ByteArrayWrapper(Blake2b256.hash(boxId)) } - def getIterator: DBIterator = storage.getIterator + def getIterator: StorageIterator = storage.getIterator def isEmpty: Boolean = storage.isEmpty diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index fdb58305cc..6a4481e91c 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -9,7 +9,6 @@ import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ import com.horizen.proposition.PublicKey25519Proposition import com.horizen.utils.{ByteArrayWrapper, ListSerializer, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer, Pair => JPair, _} -import org.iq80.leveldb.DBIterator import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging @@ -309,7 +308,7 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain def isEmpty: Boolean = storage.isEmpty - def getIterator: DBIterator = storage.getIterator + def getIterator: StorageIterator = storage.getIterator /** * This function restores the unspent boxes that come from a ceased sidechain by saving @@ -317,7 +316,7 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain * * @param backupStorage: storage containing the boxes saved from the ceased sidechain */ - def restoreBackup(backupStorageIterator: DBIterator, lastVersion: Array[Byte]): Unit = { + def restoreBackup(backupStorageIterator: StorageIterator, lastVersion: Array[Byte]): Unit = { backupStorageIterator.seekToFirst() val removeList = new JArrayList[ByteArrayWrapper]() val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala index 9d436aec72..fa4228936a 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala @@ -8,7 +8,6 @@ import com.horizen.{SidechainTypes, WalletBox, WalletBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.box.Box import com.horizen.proposition.Proposition -import org.iq80.leveldb.DBIterator import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging @@ -165,6 +164,6 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid def isEmpty: Boolean = storage.isEmpty - def getIterator: DBIterator = storage.getIterator + def getIterator: StorageIterator = storage.getIterator } diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/DatabaseIterator.java b/sdk/src/main/scala/com/horizen/storage/leveldb/DatabaseIterator.java new file mode 100644 index 0000000000..787131af33 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/DatabaseIterator.java @@ -0,0 +1,40 @@ +package com.horizen.storage.leveldb; + +import com.horizen.storage.StorageIterator; +import org.iq80.leveldb.DBIterator; + +import java.io.IOException; +import java.util.Map; + +public class DatabaseIterator implements StorageIterator { + DBIterator iterator; + + DatabaseIterator(DBIterator iterator) { + this.iterator = iterator; + } + + @Override + public void seek(byte[] key) { + iterator.seek(key); + } + + @Override + public void seekToFirst() { + iterator.seekToFirst(); + } + + @Override + public void close() throws IOException { + iterator.close(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Map.Entry next() { + return iterator.next(); + } +} diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala index ca25e6c66d..c685c60651 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLDBKVStore.scala @@ -1,7 +1,9 @@ package com.horizen.storage.leveldb +import com.horizen.storage.StorageIterator import com.horizen.utils.ByteArrayWrapper -import org.iq80.leveldb.{DB, DBIterator, ReadOptions} +import org.fusesource.leveldbjni.internal.JniDBIterator +import org.iq80.leveldb.{DB, ReadOptions} import scala.collection.mutable import scala.util.{Failure, Success, Try} @@ -131,8 +133,8 @@ final class VersionedLDBKVStore(protected val db: DB, keepVersions: Int) extends def versionIdExists(versionId: VersionId): Boolean = versions.exists(new ByteArrayWrapper(_) == new ByteArrayWrapper(versionId)) - def getIterator(): DBIterator = { - db.iterator() + def getIterator(): StorageIterator = { + new DatabaseIterator(db.iterator()) } } diff --git a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala index f73402132d..d61c15b8e9 100644 --- a/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala +++ b/sdk/src/main/scala/com/horizen/storage/leveldb/VersionedLevelDbStorageAdapter.scala @@ -3,10 +3,10 @@ package com.horizen.storage.leveldb import java.io.File import java.util import java.util.{Optional, List => JList} -import com.horizen.storage.Storage +import com.horizen.storage.{Storage, StorageIterator} import com.horizen.storage.leveldb.LDBFactory.factory import com.horizen.utils.{Pair => JPair, _} -import org.iq80.leveldb.{DBIterator, Options} +import org.iq80.leveldb.{Options} import scala.collection.JavaConverters._ import scala.compat.java8.OptionConverters._ @@ -97,7 +97,7 @@ class VersionedLevelDbStorageAdapter(pathToDB: File) extends Storage{ override def isEmpty: Boolean = dataBase.versions.isEmpty override def numberOfVersions: Int = dataBase.versions.size - override def getIterator(): DBIterator = { + override def getIterator(): StorageIterator = { dataBase.getIterator() } } diff --git a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java index 2291846708..d9c969aadb 100644 --- a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java @@ -4,8 +4,8 @@ import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; +import com.horizen.storage.StorageIterator; import com.horizen.wallet.ApplicationWallet; -import org.iq80.leveldb.DBIterator; import java.util.List; @@ -35,6 +35,7 @@ public boolean checkStoragesVersion(byte[] blockId) { return true; } - public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { + } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java index 0062626527..70a349b409 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java @@ -6,8 +6,8 @@ import com.horizen.proposition.Proposition; import com.horizen.state.ApplicationState; import com.horizen.state.SidechainStateReader; +import com.horizen.storage.StorageIterator; import com.horizen.transaction.BoxTransaction; -import org.iq80.leveldb.DBIterator; import scala.util.Success; import scala.util.Try; @@ -37,5 +37,6 @@ public Try onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } - public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { return new Success<>(this); } + + public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { return new Success<>(this); } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java index 3268640520..4f7b1519cd 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java @@ -4,8 +4,8 @@ import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; +import com.horizen.storage.StorageIterator; import com.horizen.wallet.ApplicationWallet; -import org.iq80.leveldb.DBIterator; import java.util.List; @@ -34,6 +34,7 @@ public void onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } - public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, DBIterator i) { + public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { } + } diff --git a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala index 238dd0e45e..3ec8f20690 100644 --- a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala @@ -19,7 +19,6 @@ import com.horizen.transaction.mainchain.{ForwardTransfer, SidechainCreation, Si import com.horizen.transaction.{BoxTransaction, MC2SCAggregatedTransaction, RegularTransaction} import com.horizen.utils.{ByteArrayWrapper, BytesUtils, CswData, ForgingStakeMerklePathInfo, ForwardTransferCswData, MerklePath, MerkleTree, Pair, UtxoCswData} import com.horizen.wallet.ApplicationWallet -import org.iq80.leveldb.DBIterator import org.junit.Assert._ import org.junit._ import org.junit.rules.TemporaryFolder @@ -439,7 +438,7 @@ class SidechainWalletTest } def readStorage(walletBoxStorage: SidechainWalletBoxStorage): JArrayList[WalletBox] = { - val walletBoxStorageIterator: DBIterator = walletBoxStorage.getIterator + val walletBoxStorageIterator: StorageIterator = walletBoxStorage.getIterator walletBoxStorageIterator.seekToFirst() val walletBoxSerializer: WalletBoxSerializer = new WalletBoxSerializer(sidechainBoxesCompanion) diff --git a/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala b/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala index ae272970ae..3237e42ee0 100644 --- a/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala +++ b/sdk/src/test/scala/com/horizen/storage/InMemoryStorageAdapter.scala @@ -2,7 +2,6 @@ package com.horizen.storage import java.util import java.util.{Optional, List => JList} import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import org.iq80.leveldb.DBIterator import scala.collection.JavaConverters._ import scala.collection.mutable @@ -41,6 +40,6 @@ class InMemoryStorageAdapter(hashMap: mutable.HashMap[ByteArrayWrapper, ByteArra def copy(): InMemoryStorageAdapter = new InMemoryStorageAdapter(hashMap.clone()) - override def getIterator: DBIterator = ??? + override def getIterator: StorageIterator = ??? } diff --git a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala index 39b8ab7cd8..315cb187af 100644 --- a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala +++ b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala @@ -10,7 +10,6 @@ import com.horizen.fixtures.{SecretFixture, StoreFixture, TransactionFixture} import com.horizen.proposition.PublicKey25519Proposition import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter import com.horizen.utils.{BlockFeeInfo, BlockFeeInfoSerializer, ByteArrayWrapper, Pair, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer} -import org.iq80.leveldb.DBIterator import org.junit.Assert._ import org.junit._ import org.mockito.{ArgumentMatchers, Mockito} @@ -172,7 +171,7 @@ class SidechainStateStorageTest } def readStorage(sidechainStateStorage: SidechainStateStorage): JArrayList[SCB] = { - val sidechainStateStorageIterator: DBIterator = sidechainStateStorage.getIterator + val sidechainStateStorageIterator: StorageIterator = sidechainStateStorage.getIterator sidechainStateStorageIterator.seekToFirst() val storedBoxes = new JArrayList[SCB]() From 3796969f194b4c431bbfa6713ec612855f49e4e5 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Tue, 26 Apr 2022 16:30:31 +0200 Subject: [PATCH 09/26] [SCHAINS-422] Some changes after code review --- .../java/com/horizen/examples/BackUpper.java | 13 ----- .../java/com/horizen/examples/BoxBackup.java | 12 +++++ .../examples/DefaultApplicationState.java | 5 +- .../examples/DefaultApplicationWallet.java | 5 +- .../com/horizen/examples/SimpleAppModule.java | 6 +-- .../com/horizen/state/ApplicationState.java | 5 +- .../horizen/storage/BackUpperInterface.java | 7 --- .../horizen/storage/BoxBackupInterface.java | 8 +++ .../main/java/com/horizen/utils/Utils.java | 24 +++------ .../com/horizen/wallet/ApplicationWallet.java | 5 +- .../main/scala/com/horizen/SidechainApp.scala | 8 +-- .../com/horizen/SidechainAppModule.scala | 7 ++- .../com/horizen/SidechainNodeViewHolder.scala | 15 ++---- .../scala/com/horizen/SidechainState.scala | 18 +++---- .../scala/com/horizen/SidechainWallet.scala | 17 ++++--- .../scala/com/horizen/backup/BackupBox.java | 33 +++++++++++++ .../scala/com/horizen/backup/BoxIterator.java | 49 +++++++++++++++++++ .../consensus/ConsensusDataStorage.scala | 10 +--- .../com/horizen/storage/BackupStorage.scala | 2 + .../storage/SidechainHistoryStorage.scala | 16 ++---- .../storage/SidechainStateStorage.scala | 2 +- .../customtypes/CustomApplicationWallet.java | 6 +-- .../customtypes/DefaultApplicationState.java | 7 ++- .../customtypes/DefaultApplicationWallet.java | 8 +-- ...MockedSidechainNodeViewHolderFixture.scala | 2 +- .../SidechainNodeViewHolderFixture.scala | 1 - 26 files changed, 169 insertions(+), 122 deletions(-) delete mode 100644 examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java create mode 100644 examples/simpleapp/src/main/java/com/horizen/examples/BoxBackup.java delete mode 100644 sdk/src/main/java/com/horizen/storage/BackUpperInterface.java create mode 100644 sdk/src/main/java/com/horizen/storage/BoxBackupInterface.java create mode 100644 sdk/src/main/scala/com/horizen/backup/BackupBox.java create mode 100644 sdk/src/main/scala/com/horizen/backup/BoxIterator.java diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java b/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java deleted file mode 100644 index 315dcd0e44..0000000000 --- a/examples/simpleapp/src/main/java/com/horizen/examples/BackUpper.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.horizen.examples; - -import com.horizen.companion.SidechainBoxesCompanion; -import com.horizen.storage.BackUpperInterface; -import com.horizen.storage.BackupStorage; -import com.horizen.storage.StorageIterator; - -public class BackUpper implements BackUpperInterface { - @Override - public void generateBackUp(StorageIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception { - - } -} diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/BoxBackup.java b/examples/simpleapp/src/main/java/com/horizen/examples/BoxBackup.java new file mode 100644 index 0000000000..2ce03e1ca1 --- /dev/null +++ b/examples/simpleapp/src/main/java/com/horizen/examples/BoxBackup.java @@ -0,0 +1,12 @@ +package com.horizen.examples; + +import com.horizen.storage.BoxBackupInterface; +import com.horizen.storage.BackupStorage; +import com.horizen.backup.BoxIterator; + +public class BoxBackup implements BoxBackupInterface { + @Override + public void backup(BoxIterator source, BackupStorage db) throws Exception { + + } +} diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java index e642797243..fb7f85857e 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationState.java @@ -1,8 +1,8 @@ package com.horizen.examples; +import com.horizen.backup.BoxIterator; import com.horizen.block.SidechainBlock; import com.horizen.box.Box; -import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.state.ApplicationState; import com.horizen.state.SidechainStateReader; @@ -13,7 +13,6 @@ import com.horizen.utils.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.horizen.storage.StorageIterator; import scala.util.Success; import scala.util.Try; @@ -95,7 +94,7 @@ public void closeStorages() { } - public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { + public Try onBackupRestore(BoxIterator i) { return new Success<>(this); } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java index 39939a4718..8d9a1b961b 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/DefaultApplicationWallet.java @@ -1,7 +1,7 @@ package com.horizen.examples; +import com.horizen.backup.BoxIterator; import com.horizen.box.Box; -import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter; @@ -11,7 +11,6 @@ import com.horizen.wallet.ApplicationWallet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.horizen.storage.StorageIterator; import java.io.File; import java.util.ArrayList; @@ -88,7 +87,7 @@ public void closeStorages() { } @Override - public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { + public void onBackupRestore(BoxIterator i) { } } diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java index 062066262d..2cb774814e 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java @@ -18,7 +18,7 @@ import com.horizen.secret.Secret; import com.horizen.secret.SecretSerializer; import com.horizen.settings.SettingsReader; -import com.horizen.storage.BackUpperInterface; +import com.horizen.storage.BoxBackupInterface; import com.horizen.storage.Storage; import com.horizen.state.*; import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter; @@ -150,8 +150,8 @@ public void configureApp() { .annotatedWith(Names.named("ApplicationStopper")) .toInstance(applicationStopper); - BackUpper backUpper = new BackUpper(); - bind(BackUpperInterface.class) + BoxBackup backUpper = new BoxBackup(); + bind(BoxBackupInterface.class) .annotatedWith(Names.named("BackUpper")) .toInstance(backUpper); } diff --git a/sdk/src/main/java/com/horizen/state/ApplicationState.java b/sdk/src/main/java/com/horizen/state/ApplicationState.java index 896d87cc63..7c3b87b943 100644 --- a/sdk/src/main/java/com/horizen/state/ApplicationState.java +++ b/sdk/src/main/java/com/horizen/state/ApplicationState.java @@ -1,10 +1,9 @@ package com.horizen.state; +import com.horizen.backup.BoxIterator; import com.horizen.block.SidechainBlock; import com.horizen.box.Box; -import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; -import com.horizen.storage.StorageIterator; import com.horizen.transaction.BoxTransaction; import java.util.List; @@ -31,7 +30,7 @@ public interface ApplicationState { // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); - Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i); + Try onBackupRestore(BoxIterator i); } diff --git a/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java b/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java deleted file mode 100644 index f53b65fb6b..0000000000 --- a/sdk/src/main/java/com/horizen/storage/BackUpperInterface.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.horizen.storage; - -import com.horizen.companion.SidechainBoxesCompanion; - -public interface BackUpperInterface { - void generateBackUp(StorageIterator i, BackupStorage db, SidechainBoxesCompanion sbc) throws Exception; -} diff --git a/sdk/src/main/java/com/horizen/storage/BoxBackupInterface.java b/sdk/src/main/java/com/horizen/storage/BoxBackupInterface.java new file mode 100644 index 0000000000..608a94ac36 --- /dev/null +++ b/sdk/src/main/java/com/horizen/storage/BoxBackupInterface.java @@ -0,0 +1,8 @@ +package com.horizen.storage; + + +import com.horizen.backup.BoxIterator; + +public interface BoxBackupInterface { + void backup(BoxIterator source, BackupStorage db) throws Exception; +} diff --git a/sdk/src/main/java/com/horizen/utils/Utils.java b/sdk/src/main/java/com/horizen/utils/Utils.java index 1bb45eb808..dec064c17f 100644 --- a/sdk/src/main/java/com/horizen/utils/Utils.java +++ b/sdk/src/main/java/com/horizen/utils/Utils.java @@ -1,11 +1,9 @@ package com.horizen.utils; -import com.google.common.primitives.Longs; -import scorex.crypto.hash.Blake2b256; - import java.math.BigInteger; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Random; public final class Utils { @@ -112,21 +110,11 @@ public static long readUint32BE(byte[] bytes, int offset) { (bytes[offset + 3] & 0xffl); } - /* - Generate a uniqueVersion number - simply based to the current time millis - */ - public static synchronized byte[] uniqueVersion() { - long uniqueVersion = System.currentTimeMillis(); - while (latestUniqueVersion == uniqueVersion){ - try { - Utils.uniqueVersion().wait(5); - uniqueVersion = System.currentTimeMillis(); - } catch (InterruptedException e) { - //do nothing - } - } - latestUniqueVersion = uniqueVersion; - return (byte[]) Blake2b256.hash(Longs.toByteArray(uniqueVersion)); //hashed to be sure has 32bytes length + public static byte[] nextVersion() { + byte[] version = new byte[32]; + Random r = new Random(); + r.nextBytes(version); + return version; } } diff --git a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java index f63b9e96d5..4a9bea2cdc 100644 --- a/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java +++ b/sdk/src/main/java/com/horizen/wallet/ApplicationWallet.java @@ -2,11 +2,10 @@ import java.util.List; -import com.horizen.companion.SidechainBoxesCompanion; +import com.horizen.backup.BoxIterator; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; import com.horizen.box.Box; -import com.horizen.storage.StorageIterator; public interface ApplicationWallet { @@ -19,5 +18,5 @@ public interface ApplicationWallet { // blockId given. This is useful when checking the alignment of the storages versions at node restart boolean checkStoragesVersion(byte[] blockId); - void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i); + void onBackupRestore(BoxIterator i); } diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index 84d4a21ef8..e4dee3b9ac 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -12,6 +12,7 @@ import akka.util.Timeout import com.google.inject.name.Named import com.google.inject.{Inject, _} import com.horizen.api.http.{SidechainSubmitterApiRoute, _} +import com.horizen.backup.BoxIterator import com.horizen.block.{ProofOfWorkVerifier, SidechainBlock, SidechainBlockSerializer} import com.horizen.box.BoxSerializer import com.horizen.certificatesubmitter.CertificateSubmitterRef @@ -81,7 +82,7 @@ class SidechainApp @Inject() @Named("CustomApiGroups") val customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") val rejectedApiPaths : JList[Pair[String, String]], @Named("ApplicationStopper") val applicationStopper : SidechainAppStopper, - @Named("BackUpper") val backUpper : BackUpperInterface + @Named("BackUpper") val backUpper : BoxBackupInterface, ) extends Application with ScorexLogging { @@ -284,7 +285,6 @@ class SidechainApp @Inject() forgingBoxesMerklePathStorage, sidechainWalletCswDataStorage, backupStorage, - sidechainBoxesCompanion, params, timeProvider, applicationWallet, @@ -487,14 +487,14 @@ class SidechainApp @Inject() //Perform the backup in the application level try { - backUpper.generateBackUp(stateIterator, backupStorage, sidechainBoxesCompanion) + backUpper.backup(new BoxIterator(stateIterator, sidechainBoxesCompanion), backupStorage) } catch { case t: Throwable => log.error("Error during the Backup generation: ",t.getMessage) throw new RuntimeException("Error during the Backup generation: "+t.getMessage) } case Failure(e) => - log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...") + log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...", e.getMessage) } case Failure(e) => log.info("Failed to retrieve the Sidechain block ID fo the SidechainStateStorage rollback! ", e.getMessage) diff --git a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala index 42a141edeb..3c7a812d7e 100644 --- a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala +++ b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala @@ -9,7 +9,7 @@ import com.horizen.box.BoxSerializer import com.horizen.helper.{NodeViewHelper, NodeViewHelperImpl, SecretSubmitHelper, SecretSubmitHelperImpl, TransactionSubmitHelper, TransactionSubmitHelperImpl} import com.horizen.secret.SecretSerializer import com.horizen.state.ApplicationState -import com.horizen.storage.{BackUpperInterface, Storage} +import com.horizen.storage.{BoxBackupInterface, Storage} import com.horizen.transaction.TransactionSerializer import com.horizen.utils.Pair import com.horizen.wallet.ApplicationWallet @@ -56,9 +56,8 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { @Named("CustomApiGroups") customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") rejectedApiPaths : JList[Pair[String, String]], @Named("ApplicationStopper") applicationStopper : SidechainAppStopper, - @Named("BackUpper") backUpper : BackUpperInterface - - ): SidechainApp = { + @Named("BackUpper") backUpper : BoxBackupInterface + ): SidechainApp = { synchronized { if (app == null) { app = new SidechainApp( diff --git a/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala b/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala index b6c7fbdcbe..8a6f90bb2b 100644 --- a/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala +++ b/sdk/src/main/scala/com/horizen/SidechainNodeViewHolder.scala @@ -4,7 +4,6 @@ package com.horizen import akka.actor.{ActorRef, ActorSystem, Props} import com.horizen.block.SidechainBlock import com.horizen.chain.FeePaymentsInfo -import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ import com.horizen.node.SidechainNodeView import com.horizen.params.NetworkParams @@ -40,7 +39,6 @@ class SidechainNodeViewHolder(sidechainSettings: SidechainSettings, forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, backupStorage: BackupStorage, - sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -228,13 +226,13 @@ class SidechainNodeViewHolder(sidechainSettings: SidechainSettings, override protected def genesisState: (HIS, MS, VL, MP) = { val result = for { - state <- SidechainState.createGenesisState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, backupStorage, sidechainBoxesCompanion, params, applicationState, genesisBlock) + state <- SidechainState.createGenesisState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, backupStorage, params, applicationState, genesisBlock) (_: ModifierId, consensusEpochInfo: ConsensusEpochInfo) <- Success(state.getCurrentConsensusEpochInfo) withdrawalEpochNumber: Int <- Success(state.getWithdrawalEpochInfo.epoch) wallet <- SidechainWallet.createGenesisWallet(sidechainSettings.wallet.seed.getBytes, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, applicationWallet, + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, applicationWallet, genesisBlock, withdrawalEpochNumber, consensusEpochInfo) history <- SidechainHistory.createGenesisHistory(historyStorage, consensusDataStorage, params, genesisBlock, semanticBlockValidators(params), @@ -496,14 +494,13 @@ object SidechainNodeViewHolderRef { forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, backupStorage: BackupStorage, - sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, applicationState: ApplicationState, genesisBlock: SidechainBlock): Props = Props(new SidechainNodeViewHolder(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, timeProvider, applicationWallet, applicationState, genesisBlock)) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) def apply(sidechainSettings: SidechainSettings, historyStorage: SidechainHistoryStorage, @@ -517,7 +514,6 @@ object SidechainNodeViewHolderRef { forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, backupStorage: BackupStorage, - sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -525,7 +521,7 @@ object SidechainNodeViewHolderRef { genesisBlock: SidechainBlock) (implicit system: ActorSystem): ActorRef = system.actorOf(props(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, timeProvider, applicationWallet, applicationState, genesisBlock)) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock)) def apply(name: String, sidechainSettings: SidechainSettings, @@ -540,7 +536,6 @@ object SidechainNodeViewHolderRef { forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, backupStorage: BackupStorage, - sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, timeProvider: NetworkTimeProvider, applicationWallet: ApplicationWallet, @@ -548,5 +543,5 @@ object SidechainNodeViewHolderRef { genesisBlock: SidechainBlock) (implicit system: ActorSystem): ActorRef = system.actorOf(props(sidechainSettings, historyStorage, consensusDataStorage, stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, walletBoxStorage, secretStorage, - walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, sidechainBoxesCompanion, params, timeProvider, applicationWallet, applicationState, genesisBlock), name) + walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, backupStorage, params, timeProvider, applicationWallet, applicationState, genesisBlock), name) } diff --git a/sdk/src/main/scala/com/horizen/SidechainState.scala b/sdk/src/main/scala/com/horizen/SidechainState.scala index 402840056e..ca706445a0 100644 --- a/sdk/src/main/scala/com/horizen/SidechainState.scala +++ b/sdk/src/main/scala/com/horizen/SidechainState.scala @@ -1,6 +1,7 @@ package com.horizen import com.google.common.primitives.{Bytes, Ints} +import com.horizen.backup.BoxIterator import java.io.File import java.util @@ -577,7 +578,6 @@ object SidechainState forgerBoxStorage: SidechainStateForgerBoxStorage, utxoMerkleTreeStorage: SidechainStateUtxoMerkleTreeStorage, backupStorage: BackupStorage, - sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, applicationState: ApplicationState, genesisBlock: SidechainBlock): Try[SidechainState] = Try { @@ -585,20 +585,16 @@ object SidechainState if (stateStorage.isEmpty) { if (!backupStorage.isEmpty) { stateStorage.restoreBackup(backupStorage.getIterator, versionToBytes(idToVersion(genesisBlock.parentId))) - } - - val sidechainState = new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) - if (!backupStorage.isEmpty) { - applicationState.onApplicationRestore(sidechainState, sidechainBoxesCompanion, backupStorage.getIterator) match { + applicationState.onBackupRestore(new BoxIterator(backupStorage.getIterator, backupStorage.sBoxesCompanion)) match { case Success(updatedState) => new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), updatedState) .applyModifier(genesisBlock).get - case Failure(_) => - throw new RuntimeException("State storage is not empty!") + case Failure(e) => + throw e } - } - else { - sidechainState.applyModifier(genesisBlock).get + } else{ + new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) + .applyModifier(genesisBlock).get } } else throw new RuntimeException("State storage is not empty!") diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index abf29afc65..0bf8207d22 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -1,8 +1,12 @@ package com.horizen -import java.{lang} +import java.lang import java.util.{List => JList, Optional => JOptional} import java.util.{ArrayList => JArrayList} + +import com.horizen.backup.BoxIterator + +import java.util.{ArrayList => JArrayList, List => JList, Optional => JOptional} import com.horizen.block.{MainchainBlockReferenceData, SidechainBlock} import com.horizen.box.{Box, CoinsBox, ForgerBox, ZenBox} import com.horizen.companion.SidechainBoxesCompanion @@ -21,6 +25,7 @@ import com.horizen.utils._ import scorex.util.{ModifierId, ScorexLogging} import scala.util.{Failure, Success, Try} +import scala.util.{Try} import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.language.postfixOps @@ -198,7 +203,7 @@ class SidechainWallet private[horizen] (seed: Array[Byte], walletBoxes.add(new WalletBox(currBox, System.currentTimeMillis())) nBoxes += 1 if (nBoxes == leveldb.Constants.BatchSize) { - walletBoxStorage.update(new ByteArrayWrapper(Utils.uniqueVersion()), walletBoxes.asScala.toList, removeList.asScala.toList).get + walletBoxStorage.update(new ByteArrayWrapper(Utils.nextVersion), walletBoxes.asScala.toList, removeList.asScala.toList).get walletBoxes.clear() nBoxes = 0 } @@ -206,7 +211,7 @@ class SidechainWallet private[horizen] (seed: Array[Byte], } } if (nBoxes > 0) { - walletBoxStorage.update(new ByteArrayWrapper(Utils.uniqueVersion()), walletBoxes.asScala.toList, removeList.asScala.toList).get + walletBoxStorage.update(new ByteArrayWrapper(Utils.nextVersion), walletBoxes.asScala.toList, removeList.asScala.toList).get } } @@ -446,7 +451,6 @@ object SidechainWallet forgingBoxesInfoStorage: ForgingBoxesInfoStorage, cswDataStorage: SidechainWalletCswDataStorage, backupStorage: BackupStorage, - sidechainBoxesCompanion: SidechainBoxesCompanion, params: NetworkParams, applicationWallet: ApplicationWallet, genesisBlock: SidechainBlock, @@ -455,10 +459,9 @@ object SidechainWallet ) : Try[SidechainWallet] = Try { if (walletBoxStorage.isEmpty) { - applicationWallet.onApplicationRestore(sidechainBoxesCompanion, backupStorage.getIterator) val genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, - forgingBoxesInfoStorage, cswDataStorage, params, idToVersion(genesisBlock.parentId), applicationWallet) - genesisWallet.scanBackUp(backupStorage.getIterator, sidechainBoxesCompanion) + forgingBoxesInfoStorage, cswDataStorage, params, applicationWallet) + genesisWallet.scanBackUp(backupStorage.getIterator, backupStorage.sBoxesCompanion) genesisWallet.scanPersistent(genesisBlock, withdrawalEpochNumber, Seq(), None).applyConsensusEpochInfo(consensusEpochInfo) } else diff --git a/sdk/src/main/scala/com/horizen/backup/BackupBox.java b/sdk/src/main/scala/com/horizen/backup/BackupBox.java new file mode 100644 index 0000000000..6f379e5b17 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/backup/BackupBox.java @@ -0,0 +1,33 @@ +package com.horizen.backup; + +import com.horizen.box.Box; +import com.horizen.proposition.Proposition; + +public class BackupBox { + private Box box; + private byte[] boxKey; + private byte[] boxValue; + + public BackupBox(Box box, byte[] boxKey, byte[] boxValue) { + this.box = box; + this.boxKey = boxKey; + this.boxValue = boxValue; + } + + public byte getBoxTypeId() { + return box.boxTypeId(); + } + + public Box getBox() { + return box; + } + + public byte[] getBoxKey() { + return boxKey; + } + + public byte[] getBoxValue() { + return boxValue; + } + +} diff --git a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java new file mode 100644 index 0000000000..33f37e7cd7 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java @@ -0,0 +1,49 @@ +package com.horizen.backup; + +import com.horizen.box.Box; +import com.horizen.box.CoinsBox; +import com.horizen.companion.SidechainBoxesCompanion; +import com.horizen.proposition.Proposition; +import com.horizen.storage.StorageIterator; +import com.horizen.utils.ByteArrayWrapper; +import scala.util.Try; +import scorex.crypto.hash.Blake2b256; + +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; + +public class BoxIterator { + private final StorageIterator iterator; + private final SidechainBoxesCompanion sidechainBoxesCompanion; + + public BoxIterator(StorageIterator iterator, SidechainBoxesCompanion sidechainBoxesCompanion) { + this.iterator = iterator; + this.sidechainBoxesCompanion = sidechainBoxesCompanion; + this.iterator.seekToFirst(); + } + + public Optional nextBox() { + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + Try> box = sidechainBoxesCompanion.parseBytesTry(entry.getValue()); + if (box.isSuccess()) { + Box currBox = box.get(); + if (verifyBox(entry.getKey(), currBox.id()) && + (!(currBox instanceof CoinsBox)) + ) { + return Optional.of(new BackupBox(currBox, entry.getKey(), entry.getValue())); + } + } + } + return Optional.empty(); + } + + private boolean verifyBox(byte[] recordId, byte[] boxId) { + return Arrays.equals(recordId, calculateKey(boxId).data()); + } + + private ByteArrayWrapper calculateKey(byte[] boxId) { + return new ByteArrayWrapper((byte[]) Blake2b256.hash(boxId)); + } +} diff --git a/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala b/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala index b52a19a699..3b4d57eb95 100644 --- a/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala +++ b/sdk/src/main/scala/com/horizen/consensus/ConsensusDataStorage.scala @@ -3,11 +3,11 @@ package com.horizen.consensus import java.util.{ArrayList => JArrayList} import com.horizen.storage.{SidechainStorageInfo, Storage} import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} +import com.horizen.utils.Utils import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging import scala.compat.java8.OptionConverters._ -import scala.util.Random class ConsensusDataStorage(consensusEpochInfoStorage: Storage) extends ScorexLogging @@ -46,12 +46,6 @@ class ConsensusDataStorage(consensusEpochInfoStorage: Storage) .map(byteArray => NonceConsensusEpochInfoSerializer.parseBytes(byteArray.data)) } - private def nextVersion: Array[Byte] = { - val version = new Array[Byte](32) - Random.nextBytes(version) - version - } - private def stakeEpochInfoKey(epochId: ConsensusEpochId): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256(s"stake$epochId")) private def nonceEpochInfoKey(epochId: ConsensusEpochId): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256(s"nonce$epochId")) @@ -60,7 +54,7 @@ class ConsensusDataStorage(consensusEpochInfoStorage: Storage) val listForUpdate = new JArrayList[JPair[ByteArrayWrapper, ByteArrayWrapper]]() val addedData = new JPair(key, new ByteArrayWrapper(value)) listForUpdate.add(addedData) - val version = new ByteArrayWrapper(nextVersion) + val version = new ByteArrayWrapper(Utils.nextVersion) consensusEpochInfoStorage.update(version, listForUpdate, java.util.Collections.emptyList()) log.debug("Consensus data storage updated with version: " + version) } diff --git a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala index a48a74c4b0..fa75eb1943 100644 --- a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala @@ -15,6 +15,8 @@ class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesC require(storage != null, "Storage must be NOT NULL.") require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") + val sBoxesCompanion = sidechainBoxesCompanion + def update (version : ByteArrayWrapper, boxToSaveList : java.util.List[JPair[ByteArrayWrapper,ByteArrayWrapper]]) : Try[BackupStorage] = Try { require(boxToSaveList != null, "List of WalletBoxes to add/update must be NOT NULL.") require(!boxToSaveList.contains(null), "WalletBox to add/update must be NOT NULL.") diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala index 7f1af78761..2cd221a1c8 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainHistoryStorage.scala @@ -14,7 +14,7 @@ import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} import scala.collection.mutable.ArrayBuffer import scala.compat.java8.OptionConverters._ -import scala.util.{Failure, Random, Success, Try} +import scala.util.{Failure, Success, Try} trait SidechainBlockInfoProvider { @@ -64,12 +64,6 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: private def feePaymentsInfoKey(blockId: ModifierId): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256(s"feePaymentsInfo$blockId")) - private def nextVersion: Array[Byte] = { - val version = new Array[Byte](32) - Random.nextBytes(version) - version - } - def height: Int = activeChain.height def heightOf(blockId: ModifierId): Option[Int] = { @@ -250,7 +244,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: toUpdate.add(new JPair(new ByteArrayWrapper(idToBytes(block.id)), new ByteArrayWrapper(block.bytes))) storage.update( - new ByteArrayWrapper(nextVersion), + new ByteArrayWrapper(Utils.nextVersion), toUpdate, new JArrayList[ByteArrayWrapper]()) @@ -259,7 +253,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: def updateFeePaymentsInfo(blockId: ModifierId, feePaymentsInfo: FeePaymentsInfo): Try[SidechainHistoryStorage] = Try { storage.update( - nextVersion, + Utils.nextVersion, java.util.Arrays.asList(new JPair(new ByteArrayWrapper(feePaymentsInfoKey(blockId)), new ByteArrayWrapper(feePaymentsInfo.bytes))), new JArrayList[ByteArrayWrapper]() ) @@ -283,7 +277,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: val blockInfo = oldInfo.copy(semanticValidity = status) storage.update( - new ByteArrayWrapper(nextVersion), + new ByteArrayWrapper(Utils.nextVersion), java.util.Arrays.asList(new JPair(new ByteArrayWrapper(blockInfoKey(block.id)), new ByteArrayWrapper(blockInfo.bytes))), new JArrayList() ) @@ -292,7 +286,7 @@ class SidechainHistoryStorage(storage: Storage, sidechainTransactionsCompanion: def setAsBestBlock(block: SidechainBlock, blockInfo: SidechainBlockInfo): Try[SidechainHistoryStorage] = Try { storage.update( - new ByteArrayWrapper(nextVersion), + new ByteArrayWrapper(Utils.nextVersion), java.util.Arrays.asList(new JPair(bestBlockIdKey, new ByteArrayWrapper(idToBytes(block.id)))), new JArrayList() ) diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index 6a4481e91c..b04688221a 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -334,7 +334,7 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain log.info("Restore Box id "+currBox.boxTypeId()) if (updateList.size() == leveldb.Constants.BatchSize) { if (backupStorageIterator.hasNext) - storage.update(new ByteArrayWrapper(Utils.uniqueVersion()),updateList, removeList) + storage.update(new ByteArrayWrapper(Utils.nextVersion),updateList, removeList) else storage.update(lastVersionWrapper,updateList, removeList) updateList.clear() diff --git a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java index d9c969aadb..6af2c837ac 100644 --- a/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/CustomApplicationWallet.java @@ -1,10 +1,9 @@ package com.horizen.customtypes; +import com.horizen.backup.BoxIterator; import com.horizen.box.Box; -import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; -import com.horizen.storage.StorageIterator; import com.horizen.wallet.ApplicationWallet; import java.util.List; @@ -35,7 +34,8 @@ public boolean checkStoragesVersion(byte[] blockId) { return true; } - public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { + @Override + public void onBackupRestore(BoxIterator i) { } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java index 70a349b409..d514e74e16 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationState.java @@ -1,12 +1,11 @@ package com.horizen.customtypes; +import com.horizen.backup.BoxIterator; import com.horizen.block.SidechainBlock; import com.horizen.box.Box; -import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.state.ApplicationState; import com.horizen.state.SidechainStateReader; -import com.horizen.storage.StorageIterator; import com.horizen.transaction.BoxTransaction; import scala.util.Success; import scala.util.Try; @@ -37,6 +36,6 @@ public Try onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } - - public Try onApplicationRestore(SidechainStateReader stateReader, SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { return new Success<>(this); } + @Override + public Try onBackupRestore(BoxIterator i) { return new Success<>(this); } } diff --git a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java index 4f7b1519cd..efca25adea 100644 --- a/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java +++ b/sdk/src/test/java/com/horizen/customtypes/DefaultApplicationWallet.java @@ -1,10 +1,9 @@ package com.horizen.customtypes; +import com.horizen.backup.BoxIterator; import com.horizen.box.Box; -import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.secret.Secret; -import com.horizen.storage.StorageIterator; import com.horizen.wallet.ApplicationWallet; import java.util.List; @@ -34,7 +33,8 @@ public void onRollback(byte[] blockId) { @Override public boolean checkStoragesVersion(byte[] blockId) { return true; } - public void onApplicationRestore(SidechainBoxesCompanion sidechainBoxesCompanion, StorageIterator i) { - } + @Override + public void onBackupRestore(BoxIterator i) { + } } diff --git a/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala index d9bbee4e59..d6eec9347a 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/MockedSidechainNodeViewHolderFixture.scala @@ -11,7 +11,7 @@ class MockedSidechainNodeViewHolder(sidechainSettings: SidechainSettings, state: SidechainState, wallet: SidechainWallet, mempool: SidechainMemoryPool) - extends SidechainNodeViewHolder(sidechainSettings, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null) { + extends SidechainNodeViewHolder(sidechainSettings, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null ) { override def dumpStorages: Unit = {} diff --git a/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala index 4eb12f2142..45b18c76be 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/SidechainNodeViewHolderFixture.scala @@ -121,7 +121,6 @@ trait SidechainNodeViewHolderFixture forgingBoxesMerklePathStorage, cswDataStorage, backupStorage, - sidechainBoxesCompanion, params, timeProvider, defaultApplicationWallet, From ce72907838c38dcee24ceba1586e53eeb78502dd Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Wed, 27 Apr 2022 17:45:04 +0200 Subject: [PATCH 10/26] [SCHAINS-422] Moved the backup function outside the SidechainApp --- qa/httpCalls/csw/blockIdToRollback.py | 7 + qa/run_sc_tests.py | 3 + qa/sc_blockid_to_rollback.py | 176 ++++++++++++++++++ .../main/scala/com/horizen/SidechainApp.scala | 64 +------ .../com/horizen/SidechainAppModule.scala | 4 +- .../scala/com/horizen/SidechainBackup.scala | 51 +++++ .../horizen/api/http/SidechainApiRoute.scala | 12 ++ .../api/http/SidechainCswApiRoute.scala | 36 +++- 8 files changed, 285 insertions(+), 68 deletions(-) create mode 100644 qa/httpCalls/csw/blockIdToRollback.py create mode 100644 qa/sc_blockid_to_rollback.py create mode 100644 sdk/src/main/scala/com/horizen/SidechainBackup.scala diff --git a/qa/httpCalls/csw/blockIdToRollback.py b/qa/httpCalls/csw/blockIdToRollback.py new file mode 100644 index 0000000000..533de609f4 --- /dev/null +++ b/qa/httpCalls/csw/blockIdToRollback.py @@ -0,0 +1,7 @@ +import json + + +# execute a csw/getBlockIdToRollback call +def getBlockIdToRollback(sidechainNode): + response = sidechainNode.csw_getSidechainBlockIdToRollback() + return response \ No newline at end of file diff --git a/qa/run_sc_tests.py b/qa/run_sc_tests.py index 6076d1e9f0..52ab38eaa7 100644 --- a/qa/run_sc_tests.py +++ b/qa/run_sc_tests.py @@ -2,6 +2,7 @@ import sys from mc_sc_forging_delegation import MCSCForgingDelegation +from qa.sc_blockid_to_rollback import SidechainBlockIdToRollbackTest from sc_ceased import SCCeased from sc_cert_no_coin_record import SCCertNoCoinRecord from sc_cert_submission_decentralization import SCCertSubmissionDecentralization @@ -139,6 +140,8 @@ def run_tests(log_file): result = run_test(DBToolTest()) assert_equal(0, result, "DBToolTest test failed!") + result = run_test(SidechainBlockIdToRollbackTest()) + assert_equal(0, result, "sc_blockid_to_rollback test failed!") if __name__ == "__main__": log_file = open("sc_test.log", "w") diff --git a/qa/sc_blockid_to_rollback.py b/qa/sc_blockid_to_rollback.py new file mode 100644 index 0000000000..3ade2f8e93 --- /dev/null +++ b/qa/sc_blockid_to_rollback.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +from SidechainTestFramework.sc_test_framework import SidechainTestFramework +from test_framework.util import assert_equal, assert_true, initialize_chain_clean, start_nodes, connect_nodes_bi, websocket_port_by_mc_node_index, forward_transfer_to_sidechain +from SidechainTestFramework.scutil import start_sc_nodes, generate_next_blocks, bootstrap_sidechain_nodes +from httpCalls.wallet.createPrivateKey25519 import http_wallet_createPrivateKey25519 +from SidechainTestFramework.sc_boostrap_info import SCNodeConfiguration, SCCreationInfo, MCConnectionInfo, \ + SCNetworkConfiguration +from httpCalls.csw.blockIdToRollback import getBlockIdToRollback +import time +from httpCalls.block.best import http_block_best +""" + Setup 1 SC Node. Advanced of some epochs and test the /csw/sidechainBlockItToRollback endpoint + This endpoint should return the sidechain block id containing the mainchain block reference of the MC block with + height = Genesis_MC_block_height + (current_epoch-2) * withdrawalEpochçength -1 +""" +class SidechainBlockIdToRollbackTest(SidechainTestFramework): + number_of_mc_nodes = 3 + number_of_sidechain_nodes = 1 + withdrawalEpochLength=10 + + def setup_chain(self): + initialize_chain_clean(self.options.tmpdir, self.number_of_mc_nodes) + + def setup_network(self, split = False): + # Setup nodes and connect them + self.nodes = self.setup_nodes() + connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 0, 2) + self.sync_all() + + def setup_nodes(self): + # Start 3 MC nodes + return start_nodes(self.number_of_mc_nodes, self.options.tmpdir) + + def sc_setup_chain(self): + # Bootstrap new SC, specify SC node 1 connection to MC node 1 + mc_node_1 = self.nodes[0] + + sc_node_1_configuration = SCNodeConfiguration( + MCConnectionInfo(address="ws://{0}:{1}".format(mc_node_1.hostname, websocket_port_by_mc_node_index(0))), + True, + automatic_fee_computation=False, + ) + network = SCNetworkConfiguration(SCCreationInfo(mc_node_1, 600, self.withdrawalEpochLength), + sc_node_1_configuration) + self.sc_nodes_bootstrap_info = bootstrap_sidechain_nodes(self.options, network) + + def sc_setup_nodes(self): + # Start 1 SC node + return start_sc_nodes(self.number_of_sidechain_nodes, self.options.tmpdir) + + def run_test(self): + self.sync_all() + sc_node1 = self.sc_nodes[0] + mc_node1 = self.nodes[0] + + assert_true(sc_node1.submitter_isCertificateSubmitterEnabled()["result"]["enabled"], + "Node 1 submitter expected to be enabled.") + + sc_address_1 = http_wallet_createPrivateKey25519(sc_node1) + + ####################### EPOCH 0 #################### + print("####################### EPOCH 0 ####################") + + # Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + # We need regular coins (the genesis account balance is locked into forging stake), so we perform a + # Forward transfer to sidechain for an amount equals to the genesis_account_balance + forward_transfer_to_sidechain(self.sc_nodes_bootstrap_info.sidechain_id, + mc_node1, + sc_address_1, + self.sc_nodes_bootstrap_info.genesis_account_balance, + mc_node1.getnewaddress()) + self.sc_sync_all() + generate_next_blocks(sc_node1, "first node", 1) + self.sc_sync_all() + + sc_creation_block_height = 450 + sc_creation_block = mc_node1.getblock(str(sc_creation_block_height),2) + assert_true(len(sc_creation_block["tx"][1]["vsc_ccout"]) == 1) + + #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch) + print("Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch)") + res = getBlockIdToRollback(sc_node1) + assert_true("error" in res) + assert_equal(res["error"]["code"], "0706") + + #Generate some MC blocks + mc_node1.generate(self.withdrawalEpochLength-2) + + #Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + #This block contains the reference to the MC block 459 that will be the first block available to retrieve with the + # /csw/getSidechainBlockIdToRollback endpoint. + blockIdToRollback = http_block_best(sc_node1)["id"] + + # Generate first mc block of the next epoch + mc_node1.generate(1) + generate_next_blocks(sc_node1, "first node", 1) + + # Wait until Certificate will appear in MC node mempool + time.sleep(10) + while mc_node1.getmempoolinfo()["size"] == 0 and sc_node1.submitter_isCertGenerationActive()["result"]["state"]: + print("Wait for certificate in mc mempool...") + time.sleep(2) + sc_node1.block_best() # just a ping to SC node. For some reason, STF can't request SC node API after a while idle. + assert_equal(1, mc_node1.getmempoolinfo()["size"], "Certificate was not added to Mc node mempool.") + + ####################### EPOCH 1 #################### + print("####################### EPOCH 1 ####################") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["state"], "ALIVE") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["epoch"], 1) + + #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch) + print("Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch)") + res = getBlockIdToRollback(sc_node1) + assert_true("error" in res) + assert_equal(res["error"]["code"], "0706") + + #Generate some MC blocks + mc_node1.generate(self.withdrawalEpochLength -1) + + #Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + mc_node1.generate(1) + generate_next_blocks(sc_node1, "first node", 1) + + # Wait until Certificate will appear in MC node mempool + time.sleep(10) + while mc_node1.getmempoolinfo()["size"] == 0 and sc_node1.submitter_isCertGenerationActive()["result"]["state"]: + print("Wait for certificate in mc mempool...") + time.sleep(2) + sc_node1.block_best() # just a ping to SC node. For some reason, STF can't request SC node API after a while idle. + assert_equal(1, mc_node1.getmempoolinfo()["size"], "Certificate was not added to Mc node mempool.") + + ####################### EPOCH 2 #################### + print("####################### EPOCH 2 ####################") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["state"], "ALIVE") + assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["epoch"], 2) + + #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we are asking for the MC height 449) + res = getBlockIdToRollback(sc_node1) + assert_true("error" in res) + assert_equal(res["error"]["code"], "0706") + + #Generate some MC blocks + mc_node1.generate(self.withdrawalEpochLength -1) + + #Generate 1 SC block + generate_next_blocks(sc_node1, "first node", 1) + + mc_node1.generate(1) + generate_next_blocks(sc_node1, "first node", 1) + + # Wait until Certificate will appear in MC node mempool + time.sleep(10) + while mc_node1.getmempoolinfo()["size"] == 0 and sc_node1.submitter_isCertGenerationActive()["result"]["state"]: + print("Wait for certificate in mc mempool...") + time.sleep(2) + sc_node1.block_best() # just a ping to SC node. For some reason, STF can't request SC node API after a while idle. + assert_equal(1, mc_node1.getmempoolinfo()["size"], "Certificate was not added to Mc node mempool.") + + ####################### EPOCH 3 #################### + print("####################### EPOCH 3 ####################") + + #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns the blockIdToRollback + print("Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns the blockIdToRollback") + res = getBlockIdToRollback(sc_node1) + print(res) + assert_equal(res["result"]["blockId"], blockIdToRollback) + +if __name__ == "__main__": + SidechainBlockIdToRollbackTest().main() diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index e4dee3b9ac..abb05f47c5 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -7,12 +7,9 @@ import akka.actor.ActorRef import akka.http.scaladsl.Http import akka.http.scaladsl.server.{ExceptionHandler, RejectionHandler} import akka.stream.ActorMaterializer -import akka.pattern.ask -import akka.util.Timeout import com.google.inject.name.Named import com.google.inject.{Inject, _} import com.horizen.api.http.{SidechainSubmitterApiRoute, _} -import com.horizen.backup.BoxIterator import com.horizen.block.{ProofOfWorkVerifier, SidechainBlock, SidechainBlockSerializer} import com.horizen.box.BoxSerializer import com.horizen.certificatesubmitter.CertificateSubmitterRef @@ -29,7 +26,7 @@ import com.horizen.secret.SecretSerializer import com.horizen.state.ApplicationState import com.horizen.storage._ import com.horizen.transaction._ -import com.horizen.utils.{BlockUtils, ByteArrayWrapper, BytesUtils, Pair} +import com.horizen.utils.{BlockUtils, BytesUtils, Pair} import com.horizen.wallet.ApplicationWallet import scorex.core.api.http.ApiRoute import scorex.core.app.Application @@ -53,11 +50,7 @@ import com.horizen.transaction.mainchain.SidechainCreation import scorex.core.network.NetworkController.ReceivableMessages.ShutdownNetwork import java.util.concurrent.atomic.AtomicBoolean -import scorex.core.NodeViewHolder.CurrentView -import scorex.core.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView -import scala.concurrent.Future -import scala.concurrent.duration.FiniteDuration import scala.util.{Failure, Success, Try} @@ -81,8 +74,7 @@ class SidechainApp @Inject() @Named("BackupStorage") val backUpStorage: Storage, @Named("CustomApiGroups") val customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") val rejectedApiPaths : JList[Pair[String, String]], - @Named("ApplicationStopper") val applicationStopper : SidechainAppStopper, - @Named("BackUpper") val backUpper : BoxBackupInterface, + @Named("ApplicationStopper") val applicationStopper : SidechainAppStopper ) extends Application with ScorexLogging { @@ -91,7 +83,6 @@ class SidechainApp @Inject() override type TX = SidechainTypes#SCBT override type PMOD = SidechainBlock override type NVHT = SidechainNodeViewHolder - type View = CurrentView[SidechainHistory, SidechainState, SidechainWallet, SidechainMemoryPool] override implicit lazy val settings: ScorexSettings = sidechainSettings.scorexSettings @@ -450,55 +441,4 @@ class SidechainApp @Inject() actorSystem.eventStream.publish(SidechainAppEvents.SidechainApplicationStart) - val timeoutDuration: FiniteDuration = settings.restApi.timeout - - implicit val timeout: Timeout = Timeout(timeoutDuration) - - /*** - * Retrieve the SidechainBlockId needed to rollback the SidechainStateStorage for the backup. - * It's calculated by the following formula: - * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochçength -1 - * @param view - * @return sidechainBlockId to rollback or Exception if it's not found - */ - def getSidechainBlockIdForBackUpRollback(view: View):Array[Byte] = { - val currentEpoch = view.state.getWithdrawalEpochInfo.epoch - val genesisMcBlockHeight = view.history.getMainchainCreationBlockHeight - val withdrawalEpochLength = sidechainSettings.genesisData.withdrawalEpochLength - val blockHeightToRollback = genesisMcBlockHeight + (currentEpoch -2) * withdrawalEpochLength - 1 - val mainchainBlockReferenceInfo = view.history.getMainchainBlockReferenceInfoByMainchainBlockHeight(blockHeightToRollback).get() - mainchainBlockReferenceInfo.getMainchainReferenceDataSidechainBlockId - } - - def createBackup(): Unit = { - log.info(s"Starting sidechain backup...") - - val checkAsFuture = (nodeViewHolderRef ? GetDataFromCurrentView(getSidechainBlockIdForBackUpRollback)).asInstanceOf[Future[Array[Byte]]] - checkAsFuture.onComplete{ - case Success(blockId) => - log.info(s"Rollback of the SidechainStateStorage to version: ${BytesUtils.toHexString(blockId)}") - sidechainStateStorage.rollback(new ByteArrayWrapper(blockId)) match { - case Success(stateStorage) => - log.info(s"Rollback of the SidechainStateStorage completed successfully!") - - //Take an iterator on the sidechainStateStorage - val stateIterator: StorageIterator = stateStorage.getIterator - stateIterator.seekToFirst() - - //Perform the backup in the application level - try { - backUpper.backup(new BoxIterator(stateIterator, sidechainBoxesCompanion), backupStorage) - } catch { - case t: Throwable => - log.error("Error during the Backup generation: ",t.getMessage) - throw new RuntimeException("Error during the Backup generation: "+t.getMessage) - } - case Failure(e) => - log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...", e.getMessage) - } - case Failure(e) => - log.info("Failed to retrieve the Sidechain block ID fo the SidechainStateStorage rollback! ", e.getMessage) - - } - } } diff --git a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala index 3c7a812d7e..8556fbba18 100644 --- a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala +++ b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala @@ -9,7 +9,7 @@ import com.horizen.box.BoxSerializer import com.horizen.helper.{NodeViewHelper, NodeViewHelperImpl, SecretSubmitHelper, SecretSubmitHelperImpl, TransactionSubmitHelper, TransactionSubmitHelperImpl} import com.horizen.secret.SecretSerializer import com.horizen.state.ApplicationState -import com.horizen.storage.{BoxBackupInterface, Storage} +import com.horizen.storage.{Storage} import com.horizen.transaction.TransactionSerializer import com.horizen.utils.Pair import com.horizen.wallet.ApplicationWallet @@ -56,7 +56,6 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { @Named("CustomApiGroups") customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") rejectedApiPaths : JList[Pair[String, String]], @Named("ApplicationStopper") applicationStopper : SidechainAppStopper, - @Named("BackUpper") backUpper : BoxBackupInterface ): SidechainApp = { synchronized { if (app == null) { @@ -81,7 +80,6 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { customApiGroups, rejectedApiPaths, applicationStopper, - backUpper ) } } diff --git a/sdk/src/main/scala/com/horizen/SidechainBackup.scala b/sdk/src/main/scala/com/horizen/SidechainBackup.scala new file mode 100644 index 0000000000..ba6b67b00b --- /dev/null +++ b/sdk/src/main/scala/com/horizen/SidechainBackup.scala @@ -0,0 +1,51 @@ +package com.horizen + +import com.google.inject.Inject +import com.google.inject.name.Named +import com.horizen.backup.BoxIterator +import com.horizen.box.BoxSerializer +import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.storage._ +import com.horizen.utils.{ByteArrayWrapper, BytesUtils} +import scorex.util.ScorexLogging + +import java.lang.{Byte => JByte} +import java.util.{HashMap => JHashMap} +import scala.util.{Failure, Success} + +class SidechainBackup @Inject() + (@Named("CustomBoxSerializers") val customBoxSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]], + @Named("StateStorage") val stateStorage: Storage, + @Named("BackupStorage") val backUpStorage: Storage, + @Named("BackUpper") val backUpper : BoxBackupInterface + ) extends ScorexLogging + { + protected val sidechainBoxesCompanion: SidechainBoxesCompanion = SidechainBoxesCompanion(customBoxSerializers) + protected val sidechainStateStorage = new SidechainStateStorage( + stateStorage, + sidechainBoxesCompanion) + protected val backupStorage = new BackupStorage(backUpStorage, sidechainBoxesCompanion) + + + def createBackup(sidechainBlockIdToRollback: String, copyStateStorage: Boolean): Unit = { + sidechainStateStorage.rollback(new ByteArrayWrapper(BytesUtils.fromHexString(sidechainBlockIdToRollback))) match { + case Success(stateStorage) => + log.info(s"Rollback of the SidechainStateStorage completed successfully!") + + //Take an iterator on the sidechainStateStorage + val stateIterator: StorageIterator = stateStorage.getIterator + stateIterator.seekToFirst() + + //Perform the backup in the application level + try { + backUpper.backup(new BoxIterator(stateIterator, sidechainBoxesCompanion), backupStorage) + } catch { + case t: Throwable => + log.error("Error during the Backup generation: ",t.getMessage) + throw new RuntimeException("Error during the Backup generation: "+t.getMessage) + } + case Failure(e) => + log.info(s"Rollback of the SidechainStateStorage couldn't end successfully...", e.getMessage) + } + } + } diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala index bf19eb24cb..3bd8d37d6c 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala @@ -5,6 +5,9 @@ import com.horizen.node.SidechainNodeView import scorex.core.api.http.{ApiDirectives, ApiRoute} import akka.pattern.ask import akka.http.scaladsl.server.Route +import com.horizen.{SidechainHistory, SidechainMemoryPool, SidechainState, SidechainWallet} +import scorex.core.NodeViewHolder.CurrentView +import scorex.core.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import scala.concurrent.{Await, ExecutionContext, Future} @@ -52,5 +55,14 @@ trait SidechainApiRoute extends ApiRoute with ApiDirectives { .mapTo[SidechainNodeView] } + type View = CurrentView[SidechainHistory, SidechainState, SidechainWallet, SidechainMemoryPool] + + def withSidechainNodeView(f: View => Route): Route = onSuccess(sidechainViewAsync())(f) + + protected def sidechainViewAsync(): Future[View] = { + def f(v: View) = v + (sidechainNodeViewHolderRef ? GetDataFromCurrentView(f)).mapTo[View] + } + } diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala index 0074d6b03b..3c7032f758 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala @@ -3,12 +3,12 @@ package com.horizen.api.http import akka.actor.{ActorRef, ActorRefFactory} import akka.http.scaladsl.server.Route import com.fasterxml.jackson.annotation.JsonView -import com.horizen.api.http.SidechainCswRestScheme.{ReqCswInfo, ReqGenerationCswState, ReqNullifier, RespCswBoxIds, RespCswHasCeasedState, RespCswInfo, RespGenerationCswState, RespNullifier} +import com.horizen.api.http.SidechainCswRestScheme.{ReqCswInfo, ReqGenerationCswState, ReqNullifier, RespCswBoxIds, RespCswHasCeasedState, RespCswInfo, RespGenerationCswState, RespNullifier, RespSidechainBlockIdToRollback} import com.horizen.serialization.Views import java.util.{Optional => JOptional} import akka.pattern.ask -import com.horizen.api.http.SidechainCswErrorResponse.{ErrorCswGenerationState, ErrorRetrievingCeasingState, ErrorRetrievingCswBoxIds, ErrorRetrievingCswInfo, ErrorRetrievingNullifier} +import com.horizen.api.http.SidechainCswErrorResponse.{ErrorCswGenerationState, ErrorRetrievingCeasingState, ErrorRetrievingCswBoxIds, ErrorRetrievingCswInfo, ErrorRetrievingNullifier, ErrorRetrievingSidechainBlockIdToRollback} import com.horizen.csw.CswManager.ReceivableMessages.{GenerateCswProof, GetBoxNullifier, GetCeasedStatus, GetCswBoxIds, GetCswInfo} import com.horizen.csw.CswManager.Responses.{CswInfo, GenerateCswProofStatus, InvalidAddress, NoProofData, ProofCreationFinished, ProofGenerationInProcess, ProofGenerationStarted, SidechainIsAlive} import com.horizen.api.http.JacksonSupport._ @@ -24,7 +24,7 @@ case class SidechainCswApiRoute(override val settings: RESTApiSettings, (implicit val context: ActorRefFactory, override val ec: ExecutionContext) extends SidechainApiRoute { override val route: Route = pathPrefix("csw") { - hasCeased ~ generateCswProof ~ cswInfo ~ cswBoxIds ~ nullifier + hasCeased ~ generateCswProof ~ cswInfo ~ cswBoxIds ~ nullifier ~ getSidechainBlockIdToRollback } /** @@ -134,6 +134,29 @@ case class SidechainCswApiRoute(override val settings: RESTApiSettings, } } } + + /*** + * Retrieve the SidechainBlockId needed to rollback the SidechainStateStorage for the backup. + * It's calculated by the following formula: + * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochçength -1 + */ + + def getSidechainBlockIdToRollback: Route = (post & path("getSidechainBlockIdToRollback")) { + withSidechainNodeView { nodeView => + try { + val withdrawalEpochLength = nodeView.state.params.withdrawalEpochLength + val currentEpoch = nodeView.state.getWithdrawalEpochInfo.epoch + val genesisMcBlockHeight = nodeView.history.getMainchainCreationBlockHeight + val blockHeightToRollback = genesisMcBlockHeight + (currentEpoch -2) * withdrawalEpochLength - 1 + val mainchainBlockReferenceInfo = nodeView.history.getMainchainBlockReferenceInfoByMainchainBlockHeight(blockHeightToRollback).get() + ApiResponseUtil.toResponse(RespSidechainBlockIdToRollback(BytesUtils.toHexString(mainchainBlockReferenceInfo.getMainchainReferenceDataSidechainBlockId))) + } catch { + case t: Throwable => + log.error("Failed to retrieve SidechainBlockIdToRollback.", t.getMessage) + ApiResponseUtil.toResponse(ErrorRetrievingSidechainBlockIdToRollback("Unexpected error during retrieving the sidechain block id to rollback.", JOptional.of(t))) + } + } + } } object SidechainCswRestScheme { @@ -166,6 +189,9 @@ object SidechainCswRestScheme { @JsonView(Array(classOf[Views.Default])) private[api] case class RespNullifier(nullifier: String) extends SuccessResponse + + @JsonView(Array(classOf[Views.Default])) + private[api] case class RespSidechainBlockIdToRollback(blockId: String) extends SuccessResponse } object SidechainCswErrorResponse { @@ -188,4 +214,8 @@ object SidechainCswErrorResponse { case class ErrorRetrievingNullifier(description: String, exception: JOptional[Throwable]) extends ErrorResponse { override val code: String = "0705" } + + case class ErrorRetrievingSidechainBlockIdToRollback(description: String, exception: JOptional[Throwable]) extends ErrorResponse { + override val code: String = "0706" + } } \ No newline at end of file From 1ed207b86bf19bae972b20faba284eb9facb7bf9 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 28 Apr 2022 10:37:22 +0200 Subject: [PATCH 11/26] [SCHAINS-422] Added the possibility to copy the StateStorage before the rollback --- sdk/pom.xml | 6 ++++++ .../scala/com/horizen/SidechainBackup.scala | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/sdk/pom.xml b/sdk/pom.xml index 4c99cdb7c8..d25dc0fb3f 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -96,6 +96,12 @@ log4j-core 2.17.0 + + + commons-io + commons-io + 2.6 + com.google.guava diff --git a/sdk/src/main/scala/com/horizen/SidechainBackup.scala b/sdk/src/main/scala/com/horizen/SidechainBackup.scala index ba6b67b00b..75e87b01d0 100644 --- a/sdk/src/main/scala/com/horizen/SidechainBackup.scala +++ b/sdk/src/main/scala/com/horizen/SidechainBackup.scala @@ -7,8 +7,10 @@ import com.horizen.box.BoxSerializer import com.horizen.companion.SidechainBoxesCompanion import com.horizen.storage._ import com.horizen.utils.{ByteArrayWrapper, BytesUtils} +import org.apache.commons.io.FileUtils import scorex.util.ScorexLogging +import java.io._ import java.lang.{Byte => JByte} import java.util.{HashMap => JHashMap} import scala.util.{Failure, Success} @@ -27,7 +29,20 @@ class SidechainBackup @Inject() protected val backupStorage = new BackupStorage(backUpStorage, sidechainBoxesCompanion) - def createBackup(sidechainBlockIdToRollback: String, copyStateStorage: Boolean): Unit = { + def createBackup(sidechainBlockIdToRollback: String, copyStateStorage: Boolean, dataDirPath: String): Unit = { + if (copyStateStorage) { + val stateStorage: File = new File(dataDirPath+ "/state") + val stateStorageBackup: File = new File(dataDirPath+ "/state_backup") + + try { + FileUtils.copyDirectory(stateStorage, stateStorageBackup) + } catch { + case t: Throwable => + log.error("Error during the copy of the StateStorage: ",t.getMessage) + throw new RuntimeException("Error during the copy of the StateStorage: "+t.getMessage) + } + } + sidechainStateStorage.rollback(new ByteArrayWrapper(BytesUtils.fromHexString(sidechainBlockIdToRollback))) match { case Success(stateStorage) => log.info(s"Rollback of the SidechainStateStorage completed successfully!") From da3346956cf840b58707c85f657f6bd6daa24231 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 28 Apr 2022 15:25:18 +0200 Subject: [PATCH 12/26] [SCHAINS-422] Fixed StateStorageTest --- sdk/src/main/java/com/horizen/utils/Utils.java | 2 -- .../com/horizen/storage/SidechainStateStorage.scala | 3 +++ .../horizen/storage/SidechainStateStorageTest.scala | 12 ++++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/sdk/src/main/java/com/horizen/utils/Utils.java b/sdk/src/main/java/com/horizen/utils/Utils.java index dec064c17f..694029f8a3 100644 --- a/sdk/src/main/java/com/horizen/utils/Utils.java +++ b/sdk/src/main/java/com/horizen/utils/Utils.java @@ -13,8 +13,6 @@ private Utils() {} public static final int SHA256_LENGTH = 32; - private static long latestUniqueVersion = 0; - public static byte[] doubleSHA256Hash(byte[] bytes) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index b04688221a..9e6dc049ff 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -9,6 +9,7 @@ import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ import com.horizen.proposition.PublicKey25519Proposition import com.horizen.utils.{ByteArrayWrapper, ListSerializer, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer, Pair => JPair, _} +import org.scalacheck.Prop.Exception import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging @@ -339,6 +340,8 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain storage.update(lastVersionWrapper,updateList, removeList) updateList.clear() } + } else { + throw new RuntimeException("Coin boxes are not eligible to be restored!") } } } diff --git a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala index 315cb187af..3098132335 100644 --- a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala +++ b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala @@ -38,6 +38,7 @@ class SidechainStateStorageTest val boxList = new ListBuffer[SidechainTypes#SCB]() val storedBoxList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() + val customStoredBoxList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val customBoxesSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]] = new JHashMap() customBoxesSerializers.put(CustomBox.BOX_TYPE_ID, CustomBoxSerializer.getSerializer.asInstanceOf[BoxSerializer[SidechainTypes#SCB]]) @@ -64,6 +65,13 @@ class SidechainStateStorageTest val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) new Pair(key,value) }) + if (!b.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) { + customStoredBoxList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new Pair(key,value) + }) + } } Mockito.when(mockedPhysicalStorage.get(ArgumentMatchers.any[ByteArrayWrapper]())) @@ -154,8 +162,8 @@ class SidechainStateStorageTest val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) //Fill BackUpStorage with 5 ZenBoxes and 5 CustomBoxes and 1 random element - storedBoxList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) - backupStorage.update(getVersion, storedBoxList.asJava).get + customStoredBoxList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) + backupStorage.update(getVersion, customStoredBoxList.asJava).get //Restore the SidechainStateStorage based on the BackupStorage stateStorage.restoreBackup(backupStorage.getIterator, getVersion.data()) From 1ea9682472705ae6be09f55eee864e35dbb22ae1 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 28 Apr 2022 17:50:00 +0200 Subject: [PATCH 13/26] [SCHAINS-422] Fixed python test import --- qa/run_sc_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/run_sc_tests.py b/qa/run_sc_tests.py index 52ab38eaa7..d2e603db60 100644 --- a/qa/run_sc_tests.py +++ b/qa/run_sc_tests.py @@ -2,7 +2,6 @@ import sys from mc_sc_forging_delegation import MCSCForgingDelegation -from qa.sc_blockid_to_rollback import SidechainBlockIdToRollbackTest from sc_ceased import SCCeased from sc_cert_no_coin_record import SCCertNoCoinRecord from sc_cert_submission_decentralization import SCCertSubmissionDecentralization @@ -34,6 +33,7 @@ from sc_db_tool_cmds import DBToolTest from websocket_server_fee_payments import SCWsServerFeePayments from sc_closed_forger import SidechainClosedForgerTest +from sc_blockid_to_rollback import SidechainBlockIdToRollbackTest def run_test(test): From a399aa4aa5b0458d2900c1bfad22681c8fddaa17 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Tue, 10 May 2022 11:57:01 +0200 Subject: [PATCH 14/26] [SCHAINS-422] Some changes after code review --- .../com/horizen/examples/SimpleAppModule.java | 5 - .../main/java/com/horizen/utils/Utils.java | 6 + .../com/horizen/SidechainAppModule.scala | 4 +- .../scala/com/horizen/SidechainBackup.scala | 25 ++-- .../scala/com/horizen/SidechainState.scala | 25 ++-- .../scala/com/horizen/SidechainWallet.scala | 40 +++--- .../horizen/api/http/SidechainApiRoute.scala | 2 +- .../api/http/SidechainCswApiRoute.scala | 4 +- .../scala/com/horizen/backup/BoxIterator.java | 27 ++-- .../com/horizen/storage/BackupStorage.scala | 12 +- .../storage/SidechainSecretStorage.scala | 7 +- .../SidechainStateForgerBoxStorage.scala | 14 +- .../storage/SidechainStateStorage.scala | 70 ++++----- .../SidechainStateUtxoMerkleTreeStorage.scala | 11 +- .../storage/SidechainWalletBoxStorage.scala | 20 ++- .../SidechainWalletCswDataStorage.scala | 13 +- .../SidechainWalletTransactionStorage.scala | 13 +- .../com/horizen/backup/BoxIteratorTest.java | 136 ++++++++++++++++++ .../com/horizen/SidechainWalletTest.scala | 63 +++++++- .../com/horizen/fixtures/BoxFixture.scala | 12 ++ .../storage/SidechainStateStorageTest.scala | 54 ++++--- ...echainStateUtxoMerkleTreeStorageTest.scala | 14 +- 22 files changed, 388 insertions(+), 189 deletions(-) create mode 100644 sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java diff --git a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java index 2cb774814e..638d9fb215 100644 --- a/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java +++ b/examples/simpleapp/src/main/java/com/horizen/examples/SimpleAppModule.java @@ -18,7 +18,6 @@ import com.horizen.secret.Secret; import com.horizen.secret.SecretSerializer; import com.horizen.settings.SettingsReader; -import com.horizen.storage.BoxBackupInterface; import com.horizen.storage.Storage; import com.horizen.state.*; import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter; @@ -150,9 +149,5 @@ public void configureApp() { .annotatedWith(Names.named("ApplicationStopper")) .toInstance(applicationStopper); - BoxBackup backUpper = new BoxBackup(); - bind(BoxBackupInterface.class) - .annotatedWith(Names.named("BackUpper")) - .toInstance(backUpper); } } diff --git a/sdk/src/main/java/com/horizen/utils/Utils.java b/sdk/src/main/java/com/horizen/utils/Utils.java index 694029f8a3..1dc61319d7 100644 --- a/sdk/src/main/java/com/horizen/utils/Utils.java +++ b/sdk/src/main/java/com/horizen/utils/Utils.java @@ -1,5 +1,7 @@ package com.horizen.utils; +import scorex.crypto.hash.Blake2b256; + import java.math.BigInteger; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -115,4 +117,8 @@ public static byte[] nextVersion() { return version; } + public static ByteArrayWrapper calculateKey(byte[] data) { + return new ByteArrayWrapper((byte[]) Blake2b256.hash(data)); + } + } diff --git a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala index 8556fbba18..b10fd31f70 100644 --- a/sdk/src/main/scala/com/horizen/SidechainAppModule.scala +++ b/sdk/src/main/scala/com/horizen/SidechainAppModule.scala @@ -55,7 +55,7 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { @Named("BackupStorage") backUpStorage: Storage, @Named("CustomApiGroups") customApiGroups: JList[ApplicationApiGroup], @Named("RejectedApiPaths") rejectedApiPaths : JList[Pair[String, String]], - @Named("ApplicationStopper") applicationStopper : SidechainAppStopper, + @Named("ApplicationStopper") applicationStopper : SidechainAppStopper ): SidechainApp = { synchronized { if (app == null) { @@ -79,7 +79,7 @@ abstract class SidechainAppModule extends com.google.inject.AbstractModule { backUpStorage, customApiGroups, rejectedApiPaths, - applicationStopper, + applicationStopper ) } } diff --git a/sdk/src/main/scala/com/horizen/SidechainBackup.scala b/sdk/src/main/scala/com/horizen/SidechainBackup.scala index 75e87b01d0..96bc0efa65 100644 --- a/sdk/src/main/scala/com/horizen/SidechainBackup.scala +++ b/sdk/src/main/scala/com/horizen/SidechainBackup.scala @@ -14,6 +14,7 @@ import java.io._ import java.lang.{Byte => JByte} import java.util.{HashMap => JHashMap} import scala.util.{Failure, Success} +import java.util.{Optional => JOptional} class SidechainBackup @Inject() (@Named("CustomBoxSerializers") val customBoxSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]], @@ -29,17 +30,23 @@ class SidechainBackup @Inject() protected val backupStorage = new BackupStorage(backUpStorage, sidechainBoxesCompanion) - def createBackup(sidechainBlockIdToRollback: String, copyStateStorage: Boolean, dataDirPath: String): Unit = { + def createBackup(sidechainBlockIdToRollback: String, copyStateStorage: Boolean, optionalStateStoragePath: JOptional[String]): Unit = { if (copyStateStorage) { - val stateStorage: File = new File(dataDirPath+ "/state") - val stateStorageBackup: File = new File(dataDirPath+ "/state_backup") + if (optionalStateStoragePath.isEmpty) { + log.error("Error during the copy of the StateStorage: no stateStorage path provided!") + throw new RuntimeException("Error during the copy of the StateStorage: no stateStorage path provided!") + } else { + val stateStoragePath = optionalStateStoragePath.get() + val stateStorage: File = new File(stateStoragePath) + val stateStorageBackup: File = new File(stateStoragePath+"_backup") - try { - FileUtils.copyDirectory(stateStorage, stateStorageBackup) - } catch { - case t: Throwable => - log.error("Error during the copy of the StateStorage: ",t.getMessage) - throw new RuntimeException("Error during the copy of the StateStorage: "+t.getMessage) + try { + FileUtils.copyDirectory(stateStorage, stateStorageBackup) + } catch { + case t: Throwable => + log.error("Error during the copy of the StateStorage: ",t.getMessage) + throw new RuntimeException("Error during the copy of the StateStorage: "+t.getMessage) + } } } diff --git a/sdk/src/main/scala/com/horizen/SidechainState.scala b/sdk/src/main/scala/com/horizen/SidechainState.scala index ca706445a0..ab14b1bac8 100644 --- a/sdk/src/main/scala/com/horizen/SidechainState.scala +++ b/sdk/src/main/scala/com/horizen/SidechainState.scala @@ -534,6 +534,17 @@ class SidechainState private[horizen] (stateStorage: SidechainStateStorage, new ZenBox(data, nonce) }.filter(box => box.value() > 0) } + + def restoreBackup(backupStorageBoxIterator: BoxIterator, lastVersion: Array[Byte]): Try[SidechainState] = Try { + stateStorage.restoreBackup(backupStorageBoxIterator, lastVersion) + backupStorageBoxIterator.seekToFirst() + applicationState.onBackupRestore(backupStorageBoxIterator) match { + case Success(_) => + this + case Failure(e) => + throw e + } + } } object SidechainState @@ -583,19 +594,11 @@ object SidechainState genesisBlock: SidechainBlock): Try[SidechainState] = Try { if (stateStorage.isEmpty) { + var state = new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) if (!backupStorage.isEmpty) { - stateStorage.restoreBackup(backupStorage.getIterator, versionToBytes(idToVersion(genesisBlock.parentId))) - applicationState.onBackupRestore(new BoxIterator(backupStorage.getIterator, backupStorage.sBoxesCompanion)) match { - case Success(updatedState) => - new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), updatedState) - .applyModifier(genesisBlock).get - case Failure(e) => - throw e - } - } else{ - new SidechainState(stateStorage, forgerBoxStorage, utxoMerkleTreeStorage, params, idToVersion(genesisBlock.parentId), applicationState) - .applyModifier(genesisBlock).get + state = state.restoreBackup(backupStorage.getBoxIterator, versionToBytes(idToVersion(genesisBlock.parentId))).get } + state.applyModifier(genesisBlock).get } else throw new RuntimeException("State storage is not empty!") } diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index 0bf8207d22..4d0a327e21 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -6,10 +6,8 @@ import java.util.{ArrayList => JArrayList} import com.horizen.backup.BoxIterator -import java.util.{ArrayList => JArrayList, List => JList, Optional => JOptional} import com.horizen.block.{MainchainBlockReferenceData, SidechainBlock} import com.horizen.box.{Box, CoinsBox, ForgerBox, ZenBox} -import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus.{ConsensusEpochInfo, ConsensusEpochNumber, ForgingStakeInfo} import com.horizen.wallet.ApplicationWallet import com.horizen.node.NodeWallet @@ -26,6 +24,10 @@ import scorex.util.{ModifierId, ScorexLogging} import scala.util.{Failure, Success, Try} import scala.util.{Try} +import scorex.core.block.Block.Timestamp +import scorex.util.ModifierId + +import scala.util.Try import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.language.postfixOps @@ -186,33 +188,33 @@ class SidechainWallet private[horizen] (seed: Array[Byte], * @param backupStorageIterator: iterator on the backup storage * @param sidechainBoxesCompanion */ - def scanBackUp(backupStorageIterator: StorageIterator, sidechainBoxesCompanion: SidechainBoxesCompanion): Unit = { + def scanBackUp(backupStorageBoxIterator: BoxIterator, genesisBlockTimestamp: Timestamp): Try[SidechainWallet] = Try{ val pubKeys = publicKeys() - backupStorageIterator.seekToFirst() val walletBoxes = new JArrayList[WalletBox]() val removeList = new JArrayList[Array[Byte]]() var nBoxes = 0 - while(backupStorageIterator.hasNext) { - val entry = backupStorageIterator.next() - val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) - - if (box.isSuccess) { - val currBox: SCB = box.get - if (pubKeys.contains(currBox.proposition())) { - walletBoxes.add(new WalletBox(currBox, System.currentTimeMillis())) - nBoxes += 1 - if (nBoxes == leveldb.Constants.BatchSize) { - walletBoxStorage.update(new ByteArrayWrapper(Utils.nextVersion), walletBoxes.asScala.toList, removeList.asScala.toList).get - walletBoxes.clear() - nBoxes = 0 - } + var optionalBox = backupStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + val box: SCB = optionalBox.get.getBox + if (pubKeys.contains(box.proposition())) { + walletBoxes.add(new WalletBox(box, genesisBlockTimestamp)) + nBoxes += 1 + if (nBoxes == leveldb.Constants.BatchSize) { + walletBoxStorage.update(new ByteArrayWrapper(Utils.nextVersion), walletBoxes.asScala.toList, removeList.asScala.toList).get + walletBoxes.clear() + nBoxes = 0 } } + optionalBox = backupStorageBoxIterator.nextBox } if (nBoxes > 0) { walletBoxStorage.update(new ByteArrayWrapper(Utils.nextVersion), walletBoxes.asScala.toList, removeList.asScala.toList).get } + backupStorageBoxIterator.seekToFirst + applicationWallet.onBackupRestore(backupStorageBoxIterator) + + this } private[horizen] def calculateUtxoCswData(view: UtxoMerkleTreeView): Seq[CswData] = { @@ -461,7 +463,7 @@ object SidechainWallet if (walletBoxStorage.isEmpty) { val genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, applicationWallet) - genesisWallet.scanBackUp(backupStorage.getIterator, backupStorage.sBoxesCompanion) + genesisWallet.scanBackUp(backupStorage.getBoxIterator, genesisBlock.timestamp).get genesisWallet.scanPersistent(genesisBlock, withdrawalEpochNumber, Seq(), None).applyConsensusEpochInfo(consensusEpochInfo) } else diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala index 3bd8d37d6c..05ef8faf80 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala @@ -57,7 +57,7 @@ trait SidechainApiRoute extends ApiRoute with ApiDirectives { type View = CurrentView[SidechainHistory, SidechainState, SidechainWallet, SidechainMemoryPool] - def withSidechainNodeView(f: View => Route): Route = onSuccess(sidechainViewAsync())(f) + def getNodeView(f: View => Route): Route = onSuccess(sidechainViewAsync())(f) protected def sidechainViewAsync(): Future[View] = { def f(v: View) = v diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala index 3c7032f758..613ab4f8f0 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala @@ -138,11 +138,11 @@ case class SidechainCswApiRoute(override val settings: RESTApiSettings, /*** * Retrieve the SidechainBlockId needed to rollback the SidechainStateStorage for the backup. * It's calculated by the following formula: - * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochçength -1 + * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochLength -1 */ def getSidechainBlockIdToRollback: Route = (post & path("getSidechainBlockIdToRollback")) { - withSidechainNodeView { nodeView => + getNodeView { nodeView => try { val withdrawalEpochLength = nodeView.state.params.withdrawalEpochLength val currentEpoch = nodeView.state.getWithdrawalEpochInfo.epoch diff --git a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java index 33f37e7cd7..44f14971e2 100644 --- a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java +++ b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java @@ -5,9 +5,8 @@ import com.horizen.companion.SidechainBoxesCompanion; import com.horizen.proposition.Proposition; import com.horizen.storage.StorageIterator; -import com.horizen.utils.ByteArrayWrapper; +import com.horizen.utils.Utils; import scala.util.Try; -import scorex.crypto.hash.Blake2b256; import java.util.Arrays; import java.util.Map; @@ -23,16 +22,25 @@ public BoxIterator(StorageIterator iterator, SidechainBoxesCompanion sidechainBo this.iterator.seekToFirst(); } - public Optional nextBox() { + public void seekToFirst() { + this.iterator.seekToFirst(); + } + + public Optional nextBox() throws RuntimeException { while (iterator.hasNext()) { Map.Entry entry = iterator.next(); Try> box = sidechainBoxesCompanion.parseBytesTry(entry.getValue()); if (box.isSuccess()) { Box currBox = box.get(); - if (verifyBox(entry.getKey(), currBox.id()) && - (!(currBox instanceof CoinsBox)) - ) { - return Optional.of(new BackupBox(currBox, entry.getKey(), entry.getValue())); + if (verifyBox(entry.getKey(), currBox.id())) { + if (!(currBox instanceof CoinsBox)) { + return Optional.of(new BackupBox(currBox, entry.getKey(), entry.getValue())); + } + else { + throw new RuntimeException("Coin boxes are not eligible to be restored!"); + } + } else { + throw new RuntimeException("Unable to reconstruct the same box id to restore!"); } } } @@ -40,10 +48,7 @@ public Optional nextBox() { } private boolean verifyBox(byte[] recordId, byte[] boxId) { - return Arrays.equals(recordId, calculateKey(boxId).data()); + return Arrays.equals(recordId, Utils.calculateKey(boxId).data()); } - private ByteArrayWrapper calculateKey(byte[] boxId) { - return new ByteArrayWrapper((byte[]) Blake2b256.hash(boxId)); - } } diff --git a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala index fa75eb1943..47aad84a3c 100644 --- a/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/BackupStorage.scala @@ -1,13 +1,13 @@ package com.horizen.storage +import com.horizen.backup.BoxIterator import com.horizen.companion.SidechainBoxesCompanion import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} import scala.util.Try import java.util.{ArrayList => JArrayList} -import scorex.crypto.hash.Blake2b256 -class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesCompanion) { +class BackupStorage (storage : Storage, val sidechainBoxesCompanion: SidechainBoxesCompanion) { // Version - random number // Key - byte array box Id // No remove operation @@ -15,8 +15,6 @@ class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesC require(storage != null, "Storage must be NOT NULL.") require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") - val sBoxesCompanion = sidechainBoxesCompanion - def update (version : ByteArrayWrapper, boxToSaveList : java.util.List[JPair[ByteArrayWrapper,ByteArrayWrapper]]) : Try[BackupStorage] = Try { require(boxToSaveList != null, "List of WalletBoxes to add/update must be NOT NULL.") require(!boxToSaveList.contains(null), "WalletBox to add/update must be NOT NULL.") @@ -31,11 +29,7 @@ class BackupStorage (storage : Storage, sidechainBoxesCompanion: SidechainBoxesC this } - def calculateKey(boxId : Array[Byte]) : ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } - - def getIterator: StorageIterator = storage.getIterator + def getBoxIterator: BoxIterator = new BoxIterator(storage.getIterator, sidechainBoxesCompanion) def isEmpty: Boolean = storage.isEmpty diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala index 10ea9980d7..677412948f 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainSecretStorage.scala @@ -1,12 +1,11 @@ package com.horizen.storage +import java.util.{ArrayList => JArrayList} import com.horizen.SidechainTypes import com.horizen.companion.SidechainSecretsCompanion -import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.{ByteArrayWrapper, Utils, Pair => JPair} import scorex.util.ScorexLogging -import java.util.{ArrayList => JArrayList} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.compat.java8.OptionConverters.RichOptionalGeneric @@ -27,7 +26,7 @@ class SidechainSecretStorage(storage: Storage, sidechainSecretsCompanion: Sidech loadSecrets() - def calculateKey(proposition: SidechainTypes#SCP): ByteArrayWrapper = new ByteArrayWrapper(Blake2b256.hash(proposition.bytes)) + def calculateKey(proposition: SidechainTypes#SCP): ByteArrayWrapper = Utils.calculateKey(proposition.bytes) private def loadSecrets(): Unit = { secrets.clear() diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala index 96845a665f..928620633e 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateForgerBoxStorage.scala @@ -1,13 +1,10 @@ package com.horizen.storage import com.horizen.SidechainTypes -import com.horizen.utils.ByteArrayWrapper -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.{ByteArrayWrapper, Utils, Pair => JPair} import scorex.util.ScorexLogging import java.util.{ArrayList => JArrayList} - import com.horizen.box.{ForgerBox, ForgerBoxSerializer} -import com.horizen.utils.{Pair => JPair} import scala.compat.java8.OptionConverters._ import scala.collection.JavaConverters._ @@ -25,12 +22,9 @@ class SidechainStateForgerBoxStorage(storage: Storage) private val forgerBoxSerializer: ForgerBoxSerializer = ForgerBoxSerializer.getSerializer - def calculateKey(boxId: Array[Byte]): ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } def getForgerBox(boxId: Array[Byte]): Option[ForgerBox] = { - storage.get(calculateKey(boxId)).asScala match { + storage.get(Utils.calculateKey(boxId)).asScala match { case Some(baw) => forgerBoxSerializer.parseBytesTry(baw.data) match { case Success(box) => Option(box) @@ -59,10 +53,10 @@ class SidechainStateForgerBoxStorage(storage: Storage) // Update boxes data for (id <- boxIdsRemoveSet) - removeList.add(calculateKey(id.data)) + removeList.add(Utils.calculateKey(id.data)) for (box <- forgerBoxUpdateSeq) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(box.id()), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(box.id()), new ByteArrayWrapper(forgerBoxSerializer.toBytes(box)))) storage.update(version, updateList, removeList) diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala index 9e6dc049ff..7c05dc20fe 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateStorage.scala @@ -3,14 +3,12 @@ package com.horizen.storage import com.google.common.primitives.{Bytes, Ints} import com.horizen.SidechainTypes +import com.horizen.backup.{BoxIterator} import com.horizen.block.{WithdrawalEpochCertificate, WithdrawalEpochCertificateSerializer} -import com.horizen.box.{CoinsBox, WithdrawalRequestBox, WithdrawalRequestBoxSerializer} +import com.horizen.box.{WithdrawalRequestBox, WithdrawalRequestBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus._ -import com.horizen.proposition.PublicKey25519Proposition import com.horizen.utils.{ByteArrayWrapper, ListSerializer, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer, Pair => JPair, _} -import org.scalacheck.Prop.Exception -import scorex.crypto.hash.Blake2b256 import scorex.util.ScorexLogging import java.util.{ArrayList => JArrayList} @@ -30,45 +28,42 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain require(storage != null, "Storage must be NOT NULL.") require(sidechainBoxesCompanion != null, "SidechainBoxesCompanion must be NOT NULL.") - private[horizen] val withdrawalEpochInformationKey = calculateKey("withdrawalEpochInformation".getBytes) + private[horizen] val withdrawalEpochInformationKey = Utils.calculateKey("withdrawalEpochInformation".getBytes) private val withdrawalRequestSerializer = new ListSerializer[WithdrawalRequestBox](WithdrawalRequestBoxSerializer.getSerializer) - private[horizen] val consensusEpochKey = calculateKey("consensusEpoch".getBytes) + private[horizen] val consensusEpochKey = Utils.calculateKey("consensusEpoch".getBytes) - private[horizen] val ceasingStateKey = calculateKey("ceasingStateKey".getBytes) + private[horizen] val ceasingStateKey = Utils.calculateKey("ceasingStateKey".getBytes) private val undefinedWithdrawalEpochCounter: Int = -1 private[horizen] def getWithdrawalEpochCounterKey(withdrawalEpoch: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) + Utils.calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) } private[horizen] def getWithdrawalRequestsKey(withdrawalEpoch: Int, counter: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) + Utils.calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) } private[horizen] def getTopQualityCertificateKey(referencedWithdrawalEpoch: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("topQualityCertificate".getBytes, Ints.toByteArray(referencedWithdrawalEpoch))) + Utils.calculateKey(Bytes.concat("topQualityCertificate".getBytes, Ints.toByteArray(referencedWithdrawalEpoch))) } private val undefinedBlockFeeInfoCounter: Int = -1 private[horizen] def getBlockFeeInfoCounterKey(withdrawalEpochNumber: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("blockFeeInfoCounter".getBytes, Ints.toByteArray(withdrawalEpochNumber))) + Utils.calculateKey(Bytes.concat("blockFeeInfoCounter".getBytes, Ints.toByteArray(withdrawalEpochNumber))) } private[horizen] def getBlockFeeInfoKey(withdrawalEpochNumber: Int, counter: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("blockFeeInfo".getBytes, Ints.toByteArray(withdrawalEpochNumber), Ints.toByteArray(counter))) + Utils.calculateKey(Bytes.concat("blockFeeInfo".getBytes, Ints.toByteArray(withdrawalEpochNumber), Ints.toByteArray(counter))) } private[horizen] def getUtxoMerkleTreeRootKey(withdrawalEpochNumber: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("utxoMerkleTreeRoot".getBytes, Ints.toByteArray(withdrawalEpochNumber))) + Utils.calculateKey(Bytes.concat("utxoMerkleTreeRoot".getBytes, Ints.toByteArray(withdrawalEpochNumber))) } - def calculateKey(boxId : Array[Byte]) : ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } def getBox(boxId : Array[Byte]) : Option[SidechainTypes#SCB] = { - storage.get(calculateKey(boxId)) match { + storage.get(Utils.calculateKey(boxId)) match { case v if v.isPresent => sidechainBoxesCompanion.parseBytesTry(v.get().data) match { case Success(box) => Option(box) @@ -209,10 +204,10 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain // Update boxes data for (r <- boxIdsRemoveSet) - removeList.add(calculateKey(r.data)) + removeList.add(Utils.calculateKey(r.data)) for (b <- boxUpdateList) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(b.id()), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(b.id()), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)))) // Update Withdrawal epoch related data @@ -317,34 +312,27 @@ class SidechainStateStorage(storage: Storage, sidechainBoxesCompanion: Sidechain * * @param backupStorage: storage containing the boxes saved from the ceased sidechain */ - def restoreBackup(backupStorageIterator: StorageIterator, lastVersion: Array[Byte]): Unit = { - backupStorageIterator.seekToFirst() + def restoreBackup(backupStorageBoxIterator: BoxIterator, lastVersion: Array[Byte]): Unit = { val removeList = new JArrayList[ByteArrayWrapper]() val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() val lastVersionWrapper = new ByteArrayWrapper(lastVersion) - while(backupStorageIterator.hasNext) { - val entry = backupStorageIterator.next() - val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) - - if (box.isSuccess) { - val currBox: SCB = box.get - if (!currBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) { - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(currBox.id()), - new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(currBox)))) - log.info("Restore Box id "+currBox.boxTypeId()) - if (updateList.size() == leveldb.Constants.BatchSize) { - if (backupStorageIterator.hasNext) - storage.update(new ByteArrayWrapper(Utils.nextVersion),updateList, removeList) - else - storage.update(lastVersionWrapper,updateList, removeList) - updateList.clear() - } - } else { - throw new RuntimeException("Coin boxes are not eligible to be restored!") - } + var optionalBox = backupStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + val box = optionalBox.get.getBox + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(box.id()), + new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box)))) + log.info("Restore Box id "+box.boxTypeId()) + optionalBox = backupStorageBoxIterator.nextBox + if (updateList.size() == leveldb.Constants.BatchSize) { + if (optionalBox.isPresent) + storage.update(new ByteArrayWrapper(Utils.nextVersion),updateList, removeList) + else + storage.update(lastVersionWrapper,updateList, removeList) + updateList.clear() } } + if (updateList.size() != 0) storage.update(lastVersionWrapper,updateList, removeList) log.info("SidechainStateStorage restore completed successfully!") diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala index 5db16d8c23..a5c6540a6f 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorage.scala @@ -4,7 +4,7 @@ import com.horizen.SidechainTypes import com.horizen.cryptolibprovider.{CryptoLibProvider, InMemorySparseMerkleTreeWrapper} import com.horizen.librustsidechains.FieldElement import com.horizen.utils.{ByteArrayWrapper, UtxoMerkleTreeLeafInfo, UtxoMerkleTreeLeafInfoSerializer, Pair => JPair} -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.Utils import scorex.util.ScorexLogging import java.util.{List => JList} @@ -43,12 +43,9 @@ class SidechainStateUtxoMerkleTreeStorage(storage: Storage) CryptoLibProvider.cswCircuitFunctions.getUtxoMerkleTreeLeaf(box) } - private[horizen] def calculateKey(boxId: Array[Byte]): ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } def getLeafInfo(boxId: Array[Byte]): Option[UtxoMerkleTreeLeafInfo] = { - storage.get(calculateKey(boxId)) match { + storage.get(Utils.calculateKey(boxId)) match { case v if v.isPresent => UtxoMerkleTreeLeafInfoSerializer.parseBytesTry(v.get().data) match { case Success(leafInfo) => Option(leafInfo) @@ -78,7 +75,7 @@ class SidechainStateUtxoMerkleTreeStorage(storage: Storage) require(boxesToAppend != null, "List of boxes to add must be NOT NULL. Use empty List instead.") require(boxesToRemoveSet != null, "List of Box IDs to remove must be NOT NULL. Use empty List instead.") - val removeList: JList[ByteArrayWrapper] = boxesToRemoveSet.map(id => calculateKey(id.data)).toList.asJava + val removeList: JList[ByteArrayWrapper] = boxesToRemoveSet.map(id => Utils.calculateKey(id.data)).toList.asJava // Remove leaves from inmemory tree require(merkleTreeWrapper.removeLeaves(boxesToRemoveSet.flatMap(id => { @@ -91,7 +88,7 @@ class SidechainStateUtxoMerkleTreeStorage(storage: Storage) throw new IllegalStateException("Not enough empty leaves in the UTXOMerkleTree.") } - val leavesToAppend = boxesToAppend.map(box => (calculateKey(box.id()), calculateLeaf(box))).zip(newLeavesPositions) + val leavesToAppend = boxesToAppend.map(box => (Utils.calculateKey(box.id()), calculateLeaf(box))).zip(newLeavesPositions) // Add leaves to inmemory tree require(merkleTreeWrapper.addLeaves(leavesToAppend.map { diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala index fa4228936a..a2af10fff8 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletBoxStorage.scala @@ -4,6 +4,8 @@ package com.horizen.storage import java.util.{ArrayList => JArrayList} import com.horizen.utils.{Pair => JPair} import com.horizen.utils.ByteArrayWrapper +import java.util.{Optional, ArrayList => JArrayList} +import com.horizen.utils.{ByteArrayWrapper, Utils, Pair => JPair} import com.horizen.{SidechainTypes, WalletBox, WalletBoxSerializer} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.box.Box @@ -35,10 +37,6 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid loadWalletBoxes() - def calculateKey(boxId : Array[Byte]) : ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) - } - private def calculateBoxesBalances() : Unit = { for (bc <-_walletBoxesByType.keys) _walletBoxesBalances.put(bc, _walletBoxesByType(bc).map(_._2.box.value()).sum) @@ -57,7 +55,7 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid private def addWalletBoxByType(walletBox : WalletBox) : Unit = { val bc = walletBox.box.getClass - val key = calculateKey(walletBox.box.id()) + val key = Utils.calculateKey(walletBox.box.id()) val t = _walletBoxesByType.get(bc) if (t.isEmpty) { val m = new mutable.LinkedHashMap[ByteArrayWrapper, WalletBox]() @@ -78,7 +76,7 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid for (wb <- storage.getAll.asScala){ val walletBox = _walletBoxSerializer.parseBytesTry(wb.getValue.data) if (walletBox.isSuccess) { - _walletBoxes.put(calculateKey(walletBox.get.box.id()), walletBox.get) + _walletBoxes.put(Utils.calculateKey(walletBox.get.box.id()), walletBox.get) addWalletBoxByType(walletBox.get) } else log.error("Error while WalletBox parsing.", walletBox) @@ -87,11 +85,11 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid } def get (boxId : Array[Byte]) : Option[WalletBox] = { - _walletBoxes.get(calculateKey(boxId)) + _walletBoxes.get(Utils.calculateKey(boxId)) } def get (boxIds : List[Array[Byte]]) : List[WalletBox] = { - for (id <- boxIds.map(calculateKey) if _walletBoxes.get(id).isDefined) yield _walletBoxes(id) + for (id <- boxIds.map(Utils.calculateKey) if _walletBoxes.get(id).isDefined) yield _walletBoxes(id) } def getAll : List[WalletBox] = { @@ -119,10 +117,10 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid val removeList = new JArrayList[ByteArrayWrapper]() val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() - removeList.addAll(boxIdsRemoveList.map(calculateKey(_)).asJavaCollection) + removeList.addAll(boxIdsRemoveList.map(Utils.calculateKey(_)).asJavaCollection) for (wb <- walletBoxUpdateList) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(wb.box.id()), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(wb.box.id()), new ByteArrayWrapper(_walletBoxSerializer.toBytes(wb)))) storage.update(version, @@ -137,7 +135,7 @@ class SidechainWalletBoxStorage (storage : Storage, sidechainBoxesCompanion: Sid } for (wba <- walletBoxUpdateList) { - val key = calculateKey(wba.box.id()) + val key = Utils.calculateKey(wba.box.id()) val bta = _walletBoxes.put(key, wba) addWalletBoxByType(wba) if (bta.isEmpty) diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala index 3df6b5d0ea..efe465dbb2 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletCswDataStorage.scala @@ -2,8 +2,7 @@ package com.horizen.storage import com.google.common.primitives.{Bytes, Ints} import com.horizen.SidechainTypes -import com.horizen.utils.{ByteArrayWrapper, CswData, CswDataSerializer, ListSerializer, Pair => JPair} -import scorex.crypto.hash.Blake2b256 +import com.horizen.utils.{ByteArrayWrapper, CswData, CswDataSerializer, ListSerializer, Utils, Pair => JPair} import scorex.util.ScorexLogging import java.util.{ArrayList => JArrayList} @@ -17,20 +16,16 @@ class SidechainWalletCswDataStorage(storage: Storage) extends ScorexLogging with private val cswDataListSerializer = new ListSerializer[CswData](CswDataSerializer) - private[horizen] val withdrawalEpochKey = calculateKey("withdrawalEpoch".getBytes) + private[horizen] val withdrawalEpochKey = Utils.calculateKey("withdrawalEpoch".getBytes) private val undefinedWithdrawalEpochCounter: Int = -1 private[horizen] def getWithdrawalEpochCounterKey(withdrawalEpoch: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) + Utils.calculateKey(Bytes.concat("withdrawalEpochCounter".getBytes, Ints.toByteArray(withdrawalEpoch))) } private[horizen] def getCswDataKey(withdrawalEpoch: Int, counter: Int): ByteArrayWrapper = { - calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) - } - - private[horizen] def calculateKey(boxId: Array[Byte]): ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(boxId)) + Utils.calculateKey(Bytes.concat("withdrawalRequests".getBytes, Ints.toByteArray(withdrawalEpoch), Ints.toByteArray(counter))) } def getWithdrawalEpochCounter(epoch: Int): Int = { diff --git a/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala b/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala index ce0b96cbad..2f8cffdb10 100644 --- a/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala +++ b/sdk/src/main/scala/com/horizen/storage/SidechainWalletTransactionStorage.scala @@ -3,11 +3,11 @@ package com.horizen.storage import com.horizen.SidechainTypes import com.horizen.companion.SidechainTransactionsCompanion import com.horizen.utils.{ByteArrayWrapper, Pair => JPair} -import scorex.crypto.hash.Blake2b256 -import scorex.util.{ModifierId, ScorexLogging, idToBytes} - import java.util.{ArrayList => JArrayList} +import com.horizen.utils.Utils + import scala.collection.JavaConverters._ +import scorex.util.{ModifierId, ScorexLogging, idToBytes} import scala.compat.java8.OptionConverters.RichOptionalGeneric import scala.util.{Failure, Success, Try} @@ -23,12 +23,9 @@ extends SidechainTypes require(storage != null, "Storage must be NOT NULL.") require(sidechainTransactionsCompanion != null, "SidechainTransactionsCompanion must be NOT NULL.") - def calculateKey(transactionId : Array[Byte]) : ByteArrayWrapper = { - new ByteArrayWrapper(Blake2b256.hash(transactionId)) - } def get (transactionId : Array[Byte]) : Option[SidechainTypes#SCBT] = { - storage.get(calculateKey(transactionId)) match { + storage.get(Utils.calculateKey(transactionId)) match { case v if v.isPresent => { sidechainTransactionsCompanion.parseBytesTry(v.get().data) match { case Success(transaction) => Option(transaction.asInstanceOf[SidechainTypes#SCBT]) @@ -50,7 +47,7 @@ extends SidechainTypes val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() for (tx <- transactionUpdateList) - updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](calculateKey(idToBytes(ModifierId @@ tx.id)), + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(idToBytes(ModifierId @@ tx.id)), new ByteArrayWrapper(sidechainTransactionsCompanion.toBytes(tx)))) storage.update(version, diff --git a/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java new file mode 100644 index 0000000000..2d91540707 --- /dev/null +++ b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java @@ -0,0 +1,136 @@ +package com.horizen.backup; + +import com.horizen.box.Box; +import com.horizen.box.BoxSerializer; +import com.horizen.box.ZenBox; +import com.horizen.companion.SidechainBoxesCompanion; +import com.horizen.customtypes.CustomBox; +import com.horizen.customtypes.CustomBoxSerializer; +import com.horizen.fixtures.BoxFixtureClass; +import com.horizen.proposition.Proposition; +import com.horizen.storage.BackupStorage; +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter; +import com.horizen.utils.ByteArrayWrapper; +import com.horizen.utils.Pair; +import com.horizen.utils.Utils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import static org.junit.Assert.fail; + +public class BoxIteratorTest extends BoxFixtureClass { + @Rule + public TemporaryFolder temporaryFolder= new TemporaryFolder(); + + + SidechainBoxesCompanion sidechainBoxesCompanion; + HashMap>> customBoxSerializers = new HashMap<>(); + List customBoxes; + List zenBoxes; + List> customBoxToSave = new ArrayList<>(); + List> zenBoxToSave = new ArrayList<>(); + int nBoxes = 5; + + @Before + public void setup() { + customBoxSerializers.put(CustomBox.BOX_TYPE_ID, (BoxSerializer) CustomBoxSerializer.getSerializer()); + sidechainBoxesCompanion = new SidechainBoxesCompanion(customBoxSerializers); + + customBoxes = getCustomBoxList(nBoxes); + for (CustomBox box : customBoxes) { + ByteArrayWrapper key = Utils.calculateKey(box.id()); + ByteArrayWrapper value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + customBoxToSave.add(new Pair<>(key, value)); + } + zenBoxes = getZenBoxList(nBoxes); + for (ZenBox box : zenBoxes) { + ByteArrayWrapper key = Utils.calculateKey(box.id()); + ByteArrayWrapper value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + zenBoxToSave.add(new Pair<>(key, value)); + } + + } + + @Test + public void BoxIteratorTestCustomBoxes() throws IOException { + //Create temporary BackupStorage + File stateStorageFile = temporaryFolder.newFolder("stateStorage"); + BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); + + //Add an additional random element to customBoxToSave list. + customBoxToSave.add(new Pair(new ByteArrayWrapper("key1".getBytes()), new ByteArrayWrapper("value1".getBytes()))); + + //Popoulate the BackupStorage + backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), customBoxToSave).get(); + + //Create a BoxIterator + BoxIterator boxIterator = backupStorage.getBoxIterator(); + + //Read the storage using the BoxIterator + ArrayList foundBoxes = readStorage(boxIterator); + + //Test that we read the correct amount of Boxes (nBoxes) and we ignore non-boxes elements. + assert(foundBoxes.size() == nBoxes); + + //Test the content of the boxes. + for (BackupBox backupBox : foundBoxes) { + ByteArrayWrapper newKey = new ByteArrayWrapper(backupBox.getBoxKey()); + ByteArrayWrapper newValue = new ByteArrayWrapper(backupBox.getBoxValue()); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + assert(backupBox.getBoxTypeId() == CustomBox.BOX_TYPE_ID); + assert(customBoxes.contains(backupBox.getBox())); + } + + //Test that the iterator is empty + Optional emptyBox = boxIterator.nextBox(); + assert(emptyBox.isEmpty()); + + //Test the seekToFirst method + boxIterator.seekToFirst(); + foundBoxes = readStorage(boxIterator); + assert(foundBoxes.size() == nBoxes); + } + + @Test + public void BoxIteratorTestCoinBoxes() throws IOException { + //Create temporary BackupStorage + File stateStorageFile = temporaryFolder.newFolder("stateStorage"); + BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); + + //Popoulate the BackupStorage + backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), zenBoxToSave).get(); + + //Create a BoxIterator + BoxIterator boxIterator = backupStorage.getBoxIterator(); + + //Read the storage using the BoxIterator + try { + boxIterator.nextBox(); + fail("We should not be able to retrieve Coin Boxes!"); + } catch (RuntimeException e) { + System.out.println(e.getMessage()); + assert(e.getMessage().equals("Coin boxes are not eligible to be restored!")); + } + + } + + + private ArrayList readStorage(BoxIterator boxIterator) { + ArrayList storedBoxes = new ArrayList<>(); + + Optional optionalBox = boxIterator.nextBox(); + while(optionalBox.isPresent()) { + storedBoxes.add(optionalBox.get()); + optionalBox = boxIterator.nextBox(); + } + return storedBoxes; + } +} diff --git a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala index 3ec8f20690..07d021776e 100644 --- a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala @@ -53,6 +53,7 @@ class SidechainWalletTest val boxToRestoreList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val secretList = new ListBuffer[Secret]() + val customSecretList = new ListBuffer[Secret]() val storedSecretList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() val secretVersions = new ListBuffer[ByteArrayWrapper]() @@ -83,6 +84,8 @@ class SidechainWalletTest // Set base Secrets data secretList ++= getPrivateKey25519List(5).asScala + customSecretList ++= getCustomPrivateKeyList(5).asScala + secretVersions += getVersion for (s <- secretList) { @@ -131,8 +134,8 @@ class SidechainWalletTest boxList += getWalletBox(getForgerBox(secretList.head.asInstanceOf[PrivateKey25519].publicImage())) val customBoxList = new ListBuffer[SidechainTypes#SCB]() - customBoxList ++= getZenBoxList(secretList.map(_.asInstanceOf[PrivateKey25519]).asJava).asScala.toList - customBoxList += getZenBox(getPrivateKey25519.publicImage()) //This box shouldn't be included in the 'scanBackup' test result + customBoxList ++= getCustomBoxListWithPrivateKeys(customSecretList.map(_.asInstanceOf[CustomPrivateKey]).asJava).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + customBoxList += getCustomBox.asInstanceOf[SidechainTypes#SCB] //This box shouldn't be included in the 'scanBackup' test result boxVersions += getVersion @@ -396,13 +399,13 @@ class SidechainWalletTest } @Test - def testScanBackUp(): Unit = { + def testScanBackUpNonCoinBoxes(): Unit = { val mockedSecretStorage: SidechainSecretStorage = mock[SidechainSecretStorage] val mockedWalletTransactionStorage: SidechainWalletTransactionStorage = mock[SidechainWalletTransactionStorage] val mockedForgingBoxesInfoStorage: ForgingBoxesInfoStorage = mock[ForgingBoxesInfoStorage] val mockedCswDataStorage: SidechainWalletCswDataStorage = mock[SidechainWalletCswDataStorage] val mockedApplicationWallet: ApplicationWallet = mock[ApplicationWallet] - Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>secretList.toList) + Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>customSecretList.toList) //Create temporary WalletBoxStorage val walletBoxStorageFile = temporaryFolder.newFolder("walletBoxStorage") @@ -424,19 +427,67 @@ class SidechainWalletTest params, mockedApplicationWallet) - sidechainWallet.scanBackUp(backupStorage.getIterator, sidechainBoxesCompanion) + // Mock get and update methods of SecretStorage + sidechainWallet.scanBackUp(backupStorage.getBoxIterator, System.currentTimeMillis()) assertTrue("Box stored to the backupStorage should be 7: 5 regular box + 1 new address + 1 fake ",boxToRestoreList.size == 7) val storedBoxes = readStorage(walletBoxStorage) //Verify that we did take only the 5 Boxes assertEquals("SidechainWalletBoxStorage should contains only the 5 CustomBoxes!",5, storedBoxes.size()) - val publicKeys = secretList.map(_.asInstanceOf[PrivateKey25519].publicImage()).asJava + val publicKeys = customSecretList.map(_.asInstanceOf[CustomPrivateKey].publicImage()).asJava storedBoxes.forEach(box => { assertTrue("Restored Boxes propositions should be inside our wallet!", publicKeys.contains(box.box.proposition())) }) } + @Test + def testScanBackUpCoinBoxes(): Unit = { + val mockedSecretStorage: SidechainSecretStorage = mock[SidechainSecretStorage] + val mockedWalletTransactionStorage: SidechainWalletTransactionStorage = mock[SidechainWalletTransactionStorage] + val mockedForgingBoxesInfoStorage: ForgingBoxesInfoStorage = mock[ForgingBoxesInfoStorage] + val mockedCswDataStorage: SidechainWalletCswDataStorage = mock[SidechainWalletCswDataStorage] + val mockedApplicationWallet: ApplicationWallet = mock[ApplicationWallet] + Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>secretList.toList) + + //Create temporary WalletBoxStorage + val walletBoxStorageFile = temporaryFolder.newFolder("walletBoxStorage") + val walletBoxStorage = new SidechainWalletBoxStorage(new VersionedLevelDbStorageAdapter(walletBoxStorageFile), sidechainBoxesCompanion) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) + + //Serialize ZenBoxes + val zenBoxSerializedList = new ListBuffer[Pair[ByteArrayWrapper, ByteArrayWrapper]]() + for (b <- boxList) { + zenBoxSerializedList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.box.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b.box)) + new Pair(key,value) + }) + } + backupStorage.update(getVersion, zenBoxSerializedList.asJava).get + + val sidechainWallet = new SidechainWallet("seed".getBytes, + walletBoxStorage, + mockedSecretStorage, + mockedWalletTransactionStorage, + mockedForgingBoxesInfoStorage, + mockedCswDataStorage, + params, + mockedApplicationWallet) + + var exceptionThrown = false + sidechainWallet.scanBackUp(backupStorage.getBoxIterator, System.currentTimeMillis()) match { + case Failure(_) => + exceptionThrown = true + case Success(_) => + fail() + } + assertTrue("CoinBoxes should not be restored!",exceptionThrown) + } + def readStorage(walletBoxStorage: SidechainWalletBoxStorage): JArrayList[WalletBox] = { val walletBoxStorageIterator: StorageIterator = walletBoxStorage.getIterator walletBoxStorageIterator.seekToFirst() diff --git a/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala b/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala index 78b69b409a..dd79f570ed 100644 --- a/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala +++ b/sdk/src/test/scala/com/horizen/fixtures/BoxFixture.scala @@ -73,6 +73,10 @@ trait BoxFixture new CustomBox(new CustomBoxData(getCustomPrivateKey.publicImage(), Random.nextInt(100)), Random.nextInt(1000)) } + def getCustomBoxWithPrivateKey(proposition: CustomPublicKeyProposition): CustomBox = { + new CustomBox(new CustomBoxData(proposition, Random.nextInt(100)), Random.nextInt(1000)) + } + def getCustomBoxList(count: Int): JList[CustomBox] = { val boxList: JList[CustomBox] = new JArrayList() @@ -82,6 +86,14 @@ trait BoxFixture boxList } + def getCustomBoxListWithPrivateKeys(secretList: JList[CustomPrivateKey]): JList[CustomBox] = { + val boxList: JList[CustomBox] = new JArrayList() + for (s <- secretList.asScala) + boxList.add(getCustomBoxWithPrivateKey(s.publicImage())) + + boxList + } + def getWalletBox(box: SidechainTypes#SCB): WalletBox = { val txId = new Array[Byte](32) Random.nextBytes(txId) diff --git a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala index 3098132335..08ee92096a 100644 --- a/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala +++ b/sdk/src/test/scala/com/horizen/storage/SidechainStateStorageTest.scala @@ -2,6 +2,7 @@ package com.horizen.storage import com.google.common.primitives.Ints import com.horizen.SidechainTypes +import com.horizen.backup.{BackupBox, BoxIterator} import com.horizen.box.{BoxSerializer, CoinsBox} import com.horizen.companion.SidechainBoxesCompanion import com.horizen.consensus.{ConsensusEpochNumber, intToConsensusEpochNumber} @@ -9,7 +10,7 @@ import com.horizen.customtypes.{CustomBox, CustomBoxSerializer} import com.horizen.fixtures.{SecretFixture, StoreFixture, TransactionFixture} import com.horizen.proposition.PublicKey25519Proposition import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter -import com.horizen.utils.{BlockFeeInfo, BlockFeeInfoSerializer, ByteArrayWrapper, Pair, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer} +import com.horizen.utils.{BlockFeeInfo, BlockFeeInfoSerializer, ByteArrayWrapper, BytesUtils, Pair, WithdrawalEpochInfo, WithdrawalEpochInfoSerializer} import org.junit.Assert._ import org.junit._ import org.mockito.{ArgumentMatchers, Mockito} @@ -152,7 +153,7 @@ class SidechainStateStorageTest } @Test - def testRestore(): Unit = { + def testRestoreNonCoinBoxes(): Unit = { //Create temporary SidechainStateStorage val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") val stateStorage = new SidechainStateStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion) @@ -161,36 +162,55 @@ class SidechainStateStorageTest val backupStorageFile = temporaryFolder.newFolder("backupStorage") val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) - //Fill BackUpStorage with 5 ZenBoxes and 5 CustomBoxes and 1 random element + //Fill BackUpStorage with 5 CustomBoxes and 1 random element customStoredBoxList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) backupStorage.update(getVersion, customStoredBoxList.asJava).get //Restore the SidechainStateStorage based on the BackupStorage - stateStorage.restoreBackup(backupStorage.getIterator, getVersion.data()) + stateStorage.restoreBackup(backupStorage.getBoxIterator, getVersion.data()) //Read the SidechainStateStorage - val storedBoxes = readStorage(stateStorage) + val storedBoxes = readStorage(new BoxIterator(stateStorage.getIterator, sidechainBoxesCompanion)) //Verify that we did take only the 5 CustomBoxes assertEquals("SidechainStateStorage should contains only the 5 CustomBoxes!",storedBoxes.size(), 5) storedBoxes.forEach(box => { - assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) + val storageElement = new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper(box.getBoxKey), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box.getBox))) + assertTrue("Restored boxes should be inside customStoredBoxList",customStoredBoxList.contains(storageElement)) + assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) }) } - def readStorage(sidechainStateStorage: SidechainStateStorage): JArrayList[SCB] = { - val sidechainStateStorageIterator: StorageIterator = sidechainStateStorage.getIterator - sidechainStateStorageIterator.seekToFirst() + @Test + def testRestoreCoinBoxes(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + val stateStorage = new SidechainStateStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion) - val storedBoxes = new JArrayList[SCB]() - while(sidechainStateStorageIterator.hasNext) { - val entry = sidechainStateStorageIterator.next() - val box: Try[SCB] = sidechainBoxesCompanion.parseBytesTry(entry.getValue) + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(backupStorageFile), sidechainBoxesCompanion) - if(box.isSuccess) { - val currBox: SCB = box.get - storedBoxes.add(currBox) - } + //Fill BackUpStorage with 5 ZenBoxes and 1 random element + storedBoxList.append(new Pair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper("key1".getBytes), new ByteArrayWrapper("value1".getBytes))) + backupStorage.update(getVersion, storedBoxList.asJava).get + var exceptionThrown = false + try { + //Restore the SidechainStateStorage based on the BackupStorage + stateStorage.restoreBackup(backupStorage.getBoxIterator, getVersion.data()) + } catch { + case _:RuntimeException => exceptionThrown = true + } + assertTrue("CoinBoxes should not be restored!",exceptionThrown) + } + + def readStorage(sidechainStateStorageBoxIterator: BoxIterator): JArrayList[BackupBox] = { + val storedBoxes = new JArrayList[BackupBox]() + + var optionalBox = sidechainStateStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + storedBoxes.add(optionalBox.get) + optionalBox = sidechainStateStorageBoxIterator.nextBox } storedBoxes } diff --git a/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala b/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala index b11d4830db..5a719a853b 100644 --- a/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala +++ b/sdk/src/test/scala/com/horizen/storage/SidechainStateUtxoMerkleTreeStorageTest.scala @@ -6,7 +6,7 @@ import com.horizen.cryptolibprovider.CryptoLibProvider import com.horizen.fixtures.{BoxFixture, StoreFixture} import com.horizen.librustsidechains.FieldElement import com.horizen.proposition.Proposition -import com.horizen.utils.{ByteArrayWrapper, BytesUtils, UtxoMerkleTreeLeafInfo, Pair => JPair} +import com.horizen.utils.{ByteArrayWrapper, BytesUtils, Utils, UtxoMerkleTreeLeafInfo, Pair => JPair} import org.junit.Test import org.mockito.{ArgumentMatchers, Mockito} import org.scalatestplus.junit.JUnitSuite @@ -74,7 +74,7 @@ class SidechainStateUtxoMerkleTreeStorageTest Mockito.when(mockedPhysicalStorage.get(ArgumentMatchers.any[ByteArrayWrapper]())).thenAnswer(answer => { val key: ByteArrayWrapper = answer.getArgument(0) utxoLeafInfoSeq - .find(entry => key.equals(utxoStorage.calculateKey(entry._1.id()))) + .find(entry => key.equals(Utils.calculateKey(entry._1.id()))) .map(entry => new ByteArrayWrapper(entry._2.bytes)) .asJava }) @@ -128,10 +128,10 @@ class SidechainStateUtxoMerkleTreeStorageTest val leafFE = utxoStorage.calculateLeaf(box) val leafInfo = UtxoMerkleTreeLeafInfo(leafFE.serializeFieldElement(), idx) leafFE.freeFieldElement() - new JPair(new ByteArrayWrapper(utxoStorage.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) + new JPair(new ByteArrayWrapper(Utils.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) }.asJava - val expectedToRemove = boxesToRemove.toSeq.map(id => utxoStorage.calculateKey(id.data)).asJava + val expectedToRemove = boxesToRemove.toSeq.map(id => Utils.calculateKey(id.data)).asJava assertEquals("Version is different.", version, actVersion) assertEquals("Update list is different.", expectedToUpdate, actToUpdate) @@ -164,7 +164,7 @@ class SidechainStateUtxoMerkleTreeStorageTest Mockito.when(mockedPhysicalStorage.get(ArgumentMatchers.any[ByteArrayWrapper]())).thenAnswer(answer => { val key: ByteArrayWrapper = answer.getArgument(0) utxoLeafInfoSeq - .find(entry => key.equals(utxoStorage.calculateKey(entry._1.id()))) + .find(entry => key.equals(Utils.calculateKey(entry._1.id()))) .map(entry => new ByteArrayWrapper(entry._2.bytes)) .asJava }) @@ -200,10 +200,10 @@ class SidechainStateUtxoMerkleTreeStorageTest val leafFE = utxoStorage.calculateLeaf(box) val leafInfo = UtxoMerkleTreeLeafInfo(leafFE.serializeFieldElement(), pos) leafFE.freeFieldElement() - new JPair(new ByteArrayWrapper(utxoStorage.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) + new JPair(new ByteArrayWrapper(Utils.calculateKey(box.id())), new ByteArrayWrapper(leafInfo.bytes)) }.asJava - val expectedToRemove = boxesToRemove.toSeq.map(id => utxoStorage.calculateKey(id.data)).asJava + val expectedToRemove = boxesToRemove.toSeq.map(id => Utils.calculateKey(id.data)).asJava assertEquals("Version is different.", version, actVersion) assertEquals("Update list is different.", expectedToUpdate, actToUpdate) From 51231f94157145573cf4b2bc4142fb5ce2ee5506 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Wed, 11 May 2022 15:12:14 +0200 Subject: [PATCH 15/26] [SCHAINS-422] Added a new section 'backup' in the REST interface and addes some UTs --- qa/httpCalls/backup/blockIdForBackup.py | 7 + qa/httpCalls/csw/blockIdToRollback.py | 7 - qa/run_sc_tests.py | 6 +- ...o_rollback.py => sc_blockid_for_backup.py} | 36 ++-- .../main/scala/com/horizen/SidechainApp.scala | 3 +- .../scala/com/horizen/SidechainBackup.scala | 8 +- .../horizen/api/http/SidechainApiRoute.scala | 2 +- .../api/http/SidechainBackupApiRoute.scala | 56 ++++++ .../api/http/SidechainCswApiRoute.scala | 33 +--- .../scala/com/horizen/backup/BoxIterator.java | 21 ++- .../com/horizen/SidechainBackupTest.scala | 171 ++++++++++++++++++ 11 files changed, 281 insertions(+), 69 deletions(-) create mode 100644 qa/httpCalls/backup/blockIdForBackup.py delete mode 100644 qa/httpCalls/csw/blockIdToRollback.py rename qa/{sc_blockid_to_rollback.py => sc_blockid_for_backup.py} (83%) create mode 100644 sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala create mode 100644 sdk/src/test/scala/com/horizen/SidechainBackupTest.scala diff --git a/qa/httpCalls/backup/blockIdForBackup.py b/qa/httpCalls/backup/blockIdForBackup.py new file mode 100644 index 0000000000..f0abf87dae --- /dev/null +++ b/qa/httpCalls/backup/blockIdForBackup.py @@ -0,0 +1,7 @@ +import json + + +# execute a backup/getSidechainBlockIdForBackup call +def getBlockIdForBackup(sidechainNode): + response = sidechainNode.backup_getSidechainBlockIdForBackup() + return response \ No newline at end of file diff --git a/qa/httpCalls/csw/blockIdToRollback.py b/qa/httpCalls/csw/blockIdToRollback.py deleted file mode 100644 index 533de609f4..0000000000 --- a/qa/httpCalls/csw/blockIdToRollback.py +++ /dev/null @@ -1,7 +0,0 @@ -import json - - -# execute a csw/getBlockIdToRollback call -def getBlockIdToRollback(sidechainNode): - response = sidechainNode.csw_getSidechainBlockIdToRollback() - return response \ No newline at end of file diff --git a/qa/run_sc_tests.py b/qa/run_sc_tests.py index d2e603db60..ee81fe4b73 100644 --- a/qa/run_sc_tests.py +++ b/qa/run_sc_tests.py @@ -33,7 +33,7 @@ from sc_db_tool_cmds import DBToolTest from websocket_server_fee_payments import SCWsServerFeePayments from sc_closed_forger import SidechainClosedForgerTest -from sc_blockid_to_rollback import SidechainBlockIdToRollbackTest +from sc_blockid_for_backup import SidechainBlockIdForBackupTest def run_test(test): @@ -140,8 +140,8 @@ def run_tests(log_file): result = run_test(DBToolTest()) assert_equal(0, result, "DBToolTest test failed!") - result = run_test(SidechainBlockIdToRollbackTest()) - assert_equal(0, result, "sc_blockid_to_rollback test failed!") + result = run_test(SidechainBlockIdForBackupTest()) + assert_equal(0, result, "sc_blockid_for_backup test failed!") if __name__ == "__main__": log_file = open("sc_test.log", "w") diff --git a/qa/sc_blockid_to_rollback.py b/qa/sc_blockid_for_backup.py similarity index 83% rename from qa/sc_blockid_to_rollback.py rename to qa/sc_blockid_for_backup.py index 3ade2f8e93..72f1bc5d96 100644 --- a/qa/sc_blockid_to_rollback.py +++ b/qa/sc_blockid_for_backup.py @@ -5,7 +5,7 @@ from httpCalls.wallet.createPrivateKey25519 import http_wallet_createPrivateKey25519 from SidechainTestFramework.sc_boostrap_info import SCNodeConfiguration, SCCreationInfo, MCConnectionInfo, \ SCNetworkConfiguration -from httpCalls.csw.blockIdToRollback import getBlockIdToRollback +from httpCalls.backup.blockIdForBackup import getBlockIdForBackup import time from httpCalls.block.best import http_block_best """ @@ -13,7 +13,7 @@ This endpoint should return the sidechain block id containing the mainchain block reference of the MC block with height = Genesis_MC_block_height + (current_epoch-2) * withdrawalEpochçength -1 """ -class SidechainBlockIdToRollbackTest(SidechainTestFramework): +class SidechainBlockIdForBackupTest(SidechainTestFramework): number_of_mc_nodes = 3 number_of_sidechain_nodes = 1 withdrawalEpochLength=10 @@ -80,11 +80,11 @@ def run_test(self): sc_creation_block = mc_node1.getblock(str(sc_creation_block_height),2) assert_true(len(sc_creation_block["tx"][1]["vsc_ccout"]) == 1) - #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch) - print("Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch)") - res = getBlockIdToRollback(sc_node1) + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch) + print("Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch)") + res = getBlockIdForBackup(sc_node1) assert_true("error" in res) - assert_equal(res["error"]["code"], "0706") + assert_equal(res["error"]["code"], "0801") #Generate some MC blocks mc_node1.generate(self.withdrawalEpochLength-2) @@ -93,7 +93,7 @@ def run_test(self): generate_next_blocks(sc_node1, "first node", 1) #This block contains the reference to the MC block 459 that will be the first block available to retrieve with the - # /csw/getSidechainBlockIdToRollback endpoint. + # backup/getSidechainBlockIdForBackup endpoint. blockIdToRollback = http_block_best(sc_node1)["id"] # Generate first mc block of the next epoch @@ -113,11 +113,11 @@ def run_test(self): assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["state"], "ALIVE") assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["epoch"], 1) - #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch) - print("Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we still not have 2 epoch)") - res = getBlockIdToRollback(sc_node1) + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch) + print("Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we still not have 2 epoch)") + res = getBlockIdForBackup(sc_node1) assert_true("error" in res) - assert_equal(res["error"]["code"], "0706") + assert_equal(res["error"]["code"], "0801") #Generate some MC blocks mc_node1.generate(self.withdrawalEpochLength -1) @@ -141,10 +141,10 @@ def run_test(self): assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["state"], "ALIVE") assert_equal(mc_node1.getscinfo(self.sc_nodes_bootstrap_info.sidechain_id)["items"][0]["epoch"], 2) - #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns an error (we are asking for the MC height 449) - res = getBlockIdToRollback(sc_node1) + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns an error (we are asking for the MC height 449) + res = getBlockIdForBackup(sc_node1) assert_true("error" in res) - assert_equal(res["error"]["code"], "0706") + assert_equal(res["error"]["code"], "0801") #Generate some MC blocks mc_node1.generate(self.withdrawalEpochLength -1) @@ -166,11 +166,11 @@ def run_test(self): ####################### EPOCH 3 #################### print("####################### EPOCH 3 ####################") - #Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns the blockIdToRollback - print("Call the csw/getSidechainBlockIdToRollback endpoint and verify it returns the blockIdToRollback") - res = getBlockIdToRollback(sc_node1) + #Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns the blockIdToRollback + print("Call the backup/getSidechainBlockIdForBackup endpoint and verify it returns the blockIdToRollback") + res = getBlockIdForBackup(sc_node1) print(res) assert_equal(res["result"]["blockId"], blockIdToRollback) if __name__ == "__main__": - SidechainBlockIdToRollbackTest().main() + SidechainBlockIdForBackupTest().main() diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index abb05f47c5..54687f0101 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -349,7 +349,8 @@ class SidechainApp @Inject() SidechainTransactionApiRoute(settings.restApi, nodeViewHolderRef, sidechainTransactionActorRef, sidechainTransactionsCompanion, params), SidechainWalletApiRoute(settings.restApi, nodeViewHolderRef), SidechainSubmitterApiRoute(settings.restApi, certificateSubmitterRef, nodeViewHolderRef), - SidechainCswApiRoute(settings.restApi, nodeViewHolderRef, cswManager) + SidechainCswApiRoute(settings.restApi, nodeViewHolderRef, cswManager), + SidechainBackupApiRoute(settings.restApi, nodeViewHolderRef) ) val transactionSubmitProvider : TransactionSubmitProvider = new TransactionSubmitProviderImpl(sidechainTransactionActorRef) diff --git a/sdk/src/main/scala/com/horizen/SidechainBackup.scala b/sdk/src/main/scala/com/horizen/SidechainBackup.scala index 96bc0efa65..a178953473 100644 --- a/sdk/src/main/scala/com/horizen/SidechainBackup.scala +++ b/sdk/src/main/scala/com/horizen/SidechainBackup.scala @@ -6,6 +6,7 @@ import com.horizen.backup.BoxIterator import com.horizen.box.BoxSerializer import com.horizen.companion.SidechainBoxesCompanion import com.horizen.storage._ +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter import com.horizen.utils.{ByteArrayWrapper, BytesUtils} import org.apache.commons.io.FileUtils import scorex.util.ScorexLogging @@ -24,7 +25,7 @@ class SidechainBackup @Inject() ) extends ScorexLogging { protected val sidechainBoxesCompanion: SidechainBoxesCompanion = SidechainBoxesCompanion(customBoxSerializers) - protected val sidechainStateStorage = new SidechainStateStorage( + protected var sidechainStateStorage: SidechainStateStorage = new SidechainStateStorage( stateStorage, sidechainBoxesCompanion) protected val backupStorage = new BackupStorage(backUpStorage, sidechainBoxesCompanion) @@ -38,10 +39,13 @@ class SidechainBackup @Inject() } else { val stateStoragePath = optionalStateStoragePath.get() val stateStorage: File = new File(stateStoragePath) - val stateStorageBackup: File = new File(stateStoragePath+"_backup") + val stateStorageBackup: File = new File(stateStoragePath+"_copy_for_backup") try { FileUtils.copyDirectory(stateStorage, stateStorageBackup) + sidechainStateStorage = new SidechainStateStorage( + new VersionedLevelDbStorageAdapter(stateStorageBackup), + sidechainBoxesCompanion) } catch { case t: Throwable => log.error("Error during the copy of the StateStorage: ",t.getMessage) diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala index 05ef8faf80..ec4c8cdcc8 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainApiRoute.scala @@ -57,7 +57,7 @@ trait SidechainApiRoute extends ApiRoute with ApiDirectives { type View = CurrentView[SidechainHistory, SidechainState, SidechainWallet, SidechainMemoryPool] - def getNodeView(f: View => Route): Route = onSuccess(sidechainViewAsync())(f) + def withView(f: View => Route): Route = onSuccess(sidechainViewAsync())(f) protected def sidechainViewAsync(): Future[View] = { def f(v: View) = v diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala new file mode 100644 index 0000000000..10e9bb9476 --- /dev/null +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala @@ -0,0 +1,56 @@ +package com.horizen.api.http + +import akka.actor.{ActorRef, ActorRefFactory} +import akka.http.scaladsl.server.Route +import com.fasterxml.jackson.annotation.JsonView +import com.horizen.api.http.SidechainBackupRestScheme.RespSidechainBlockIdForBackup +import com.horizen.serialization.Views +import scorex.core.settings.RESTApiSettings +import com.horizen.api.http.SidechainBackupErrorResponse.ErrorRetrievingSidechainBlockIdForBackup +import com.horizen.utils.BytesUtils + +import java.util.{Optional => JOptional} +import scala.concurrent.ExecutionContext + + +case class SidechainBackupApiRoute(override val settings: RESTApiSettings, + sidechainNodeViewHolderRef: ActorRef) + (implicit val context: ActorRefFactory, override val ec: ExecutionContext) extends SidechainApiRoute { + override val route: Route = pathPrefix("backup") { + getSidechainBlockIdForBackup + } + + /*** + * Retrieve the SidechainBlockId needed to rollback the SidechainStateStorage for the backup. + * It's calculated by the following formula: + * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochLength -1 + */ + def getSidechainBlockIdForBackup: Route = (post & path("getSidechainBlockIdForBackup")) { + withView { nodeView => + try { + val withdrawalEpochLength = nodeView.state.params.withdrawalEpochLength + val currentEpoch = nodeView.state.getWithdrawalEpochInfo.epoch + val genesisMcBlockHeight = nodeView.history.getMainchainCreationBlockHeight + val blockHeightToRollback = genesisMcBlockHeight + (currentEpoch -2) * withdrawalEpochLength - 1 + val mainchainBlockReferenceInfo = nodeView.history.getMainchainBlockReferenceInfoByMainchainBlockHeight(blockHeightToRollback).get() + ApiResponseUtil.toResponse(RespSidechainBlockIdForBackup(BytesUtils.toHexString(mainchainBlockReferenceInfo.getMainchainReferenceDataSidechainBlockId))) + } catch { + case t: Throwable => + log.error("Failed to retrieve getSidechainBlockIdForBackup.", t.getMessage) + ApiResponseUtil.toResponse(ErrorRetrievingSidechainBlockIdForBackup("Unexpected error during retrieving the sidechain block id to rollback.", JOptional.of(t))) + } + } + } + +} + +object SidechainBackupRestScheme { + @JsonView(Array(classOf[Views.Default])) + private[api] case class RespSidechainBlockIdForBackup(blockId: String) extends SuccessResponse +} + +object SidechainBackupErrorResponse { + case class ErrorRetrievingSidechainBlockIdForBackup(description: String, exception: JOptional[Throwable]) extends ErrorResponse { + override val code: String = "0801" + } +} \ No newline at end of file diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala index 613ab4f8f0..47de502e0f 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala @@ -3,12 +3,12 @@ package com.horizen.api.http import akka.actor.{ActorRef, ActorRefFactory} import akka.http.scaladsl.server.Route import com.fasterxml.jackson.annotation.JsonView -import com.horizen.api.http.SidechainCswRestScheme.{ReqCswInfo, ReqGenerationCswState, ReqNullifier, RespCswBoxIds, RespCswHasCeasedState, RespCswInfo, RespGenerationCswState, RespNullifier, RespSidechainBlockIdToRollback} +import com.horizen.api.http.SidechainCswRestScheme.{ReqCswInfo, ReqGenerationCswState, ReqNullifier, RespCswBoxIds, RespCswHasCeasedState, RespCswInfo, RespGenerationCswState, RespNullifier} import com.horizen.serialization.Views import java.util.{Optional => JOptional} import akka.pattern.ask -import com.horizen.api.http.SidechainCswErrorResponse.{ErrorCswGenerationState, ErrorRetrievingCeasingState, ErrorRetrievingCswBoxIds, ErrorRetrievingCswInfo, ErrorRetrievingNullifier, ErrorRetrievingSidechainBlockIdToRollback} +import com.horizen.api.http.SidechainCswErrorResponse.{ErrorCswGenerationState, ErrorRetrievingCeasingState, ErrorRetrievingCswBoxIds, ErrorRetrievingCswInfo, ErrorRetrievingNullifier} import com.horizen.csw.CswManager.ReceivableMessages.{GenerateCswProof, GetBoxNullifier, GetCeasedStatus, GetCswBoxIds, GetCswInfo} import com.horizen.csw.CswManager.Responses.{CswInfo, GenerateCswProofStatus, InvalidAddress, NoProofData, ProofCreationFinished, ProofGenerationInProcess, ProofGenerationStarted, SidechainIsAlive} import com.horizen.api.http.JacksonSupport._ @@ -24,7 +24,7 @@ case class SidechainCswApiRoute(override val settings: RESTApiSettings, (implicit val context: ActorRefFactory, override val ec: ExecutionContext) extends SidechainApiRoute { override val route: Route = pathPrefix("csw") { - hasCeased ~ generateCswProof ~ cswInfo ~ cswBoxIds ~ nullifier ~ getSidechainBlockIdToRollback + hasCeased ~ generateCswProof ~ cswInfo ~ cswBoxIds ~ nullifier } /** @@ -135,28 +135,6 @@ case class SidechainCswApiRoute(override val settings: RESTApiSettings, } } - /*** - * Retrieve the SidechainBlockId needed to rollback the SidechainStateStorage for the backup. - * It's calculated by the following formula: - * Genesis_MC_block_height + (current_epch-2) * withdrawalEpochLength -1 - */ - - def getSidechainBlockIdToRollback: Route = (post & path("getSidechainBlockIdToRollback")) { - getNodeView { nodeView => - try { - val withdrawalEpochLength = nodeView.state.params.withdrawalEpochLength - val currentEpoch = nodeView.state.getWithdrawalEpochInfo.epoch - val genesisMcBlockHeight = nodeView.history.getMainchainCreationBlockHeight - val blockHeightToRollback = genesisMcBlockHeight + (currentEpoch -2) * withdrawalEpochLength - 1 - val mainchainBlockReferenceInfo = nodeView.history.getMainchainBlockReferenceInfoByMainchainBlockHeight(blockHeightToRollback).get() - ApiResponseUtil.toResponse(RespSidechainBlockIdToRollback(BytesUtils.toHexString(mainchainBlockReferenceInfo.getMainchainReferenceDataSidechainBlockId))) - } catch { - case t: Throwable => - log.error("Failed to retrieve SidechainBlockIdToRollback.", t.getMessage) - ApiResponseUtil.toResponse(ErrorRetrievingSidechainBlockIdToRollback("Unexpected error during retrieving the sidechain block id to rollback.", JOptional.of(t))) - } - } - } } object SidechainCswRestScheme { @@ -190,8 +168,6 @@ object SidechainCswRestScheme { @JsonView(Array(classOf[Views.Default])) private[api] case class RespNullifier(nullifier: String) extends SuccessResponse - @JsonView(Array(classOf[Views.Default])) - private[api] case class RespSidechainBlockIdToRollback(blockId: String) extends SuccessResponse } object SidechainCswErrorResponse { @@ -215,7 +191,4 @@ object SidechainCswErrorResponse { override val code: String = "0705" } - case class ErrorRetrievingSidechainBlockIdToRollback(description: String, exception: JOptional[Throwable]) extends ErrorResponse { - override val code: String = "0706" - } } \ No newline at end of file diff --git a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java index 44f14971e2..367cc96601 100644 --- a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java +++ b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java @@ -7,7 +7,9 @@ import com.horizen.storage.StorageIterator; import com.horizen.utils.Utils; import scala.util.Try; +import scorex.util.serialization.VLQByteBufferReader; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; import java.util.Optional; @@ -26,27 +28,32 @@ public void seekToFirst() { this.iterator.seekToFirst(); } - public Optional nextBox() throws RuntimeException { + public Optional nextBox(boolean ignoreCoinBox) throws RuntimeException { while (iterator.hasNext()) { Map.Entry entry = iterator.next(); - Try> box = sidechainBoxesCompanion.parseBytesTry(entry.getValue()); - if (box.isSuccess()) { - Box currBox = box.get(); + VLQByteBufferReader reader = new VLQByteBufferReader(ByteBuffer.wrap(entry.getValue())); + Try> tryBox = sidechainBoxesCompanion.parseTry(reader); + + if (tryBox.isSuccess() && reader.remaining() == 0) { + Box currBox = tryBox.get(); if (verifyBox(entry.getKey(), currBox.id())) { if (!(currBox instanceof CoinsBox)) { return Optional.of(new BackupBox(currBox, entry.getKey(), entry.getValue())); } else { - throw new RuntimeException("Coin boxes are not eligible to be restored!"); + if (!ignoreCoinBox) + throw new RuntimeException("Coin boxes are not eligible to be restored!"); } - } else { - throw new RuntimeException("Unable to reconstruct the same box id to restore!"); } } } return Optional.empty(); } + public Optional nextBox() throws RuntimeException { + return nextBox(false); + } + private boolean verifyBox(byte[] recordId, byte[] boxId) { return Arrays.equals(recordId, Utils.calculateKey(boxId).data()); } diff --git a/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala b/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala new file mode 100644 index 0000000000..6411f7c052 --- /dev/null +++ b/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala @@ -0,0 +1,171 @@ +package com.horizen + +import com.horizen.backup.{BackupBox, BoxIterator} +import com.horizen.box.{BoxSerializer, CoinsBox} +import com.horizen.companion.SidechainBoxesCompanion +import com.horizen.customtypes.{CustomBox, CustomBoxSerializer} +import com.horizen.fixtures.{SecretFixture, StoreFixture, TransactionFixture} +import com.horizen.proposition.PublicKey25519Proposition +import com.horizen.storage.leveldb.VersionedLevelDbStorageAdapter +import com.horizen.storage.{BackupStorage, BoxBackupInterface} +import com.horizen.utils.{ByteArrayWrapper, BytesUtils, Utils, Pair => JPair} +import org.junit.Assert.{assertEquals, assertTrue} +import org.junit.rules.TemporaryFolder +import org.junit.{Before, Rule, Test} +import org.scalatestplus.junit.JUnitSuite +import scorex.crypto.hash.Blake2b256 + +import scala.collection.JavaConverters._ +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, Optional => JOptional} +import java.lang.{Byte => JByte} +import scala.collection.mutable.ListBuffer + +class SidechainBackupTest + extends JUnitSuite + with StoreFixture + with SecretFixture + with TransactionFixture + { + + val customBoxesSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]] = new JHashMap() + customBoxesSerializers.put(CustomBox.BOX_TYPE_ID, CustomBoxSerializer.getSerializer.asInstanceOf[BoxSerializer[SidechainTypes#SCB]]) + val sidechainBoxesCompanion = SidechainBoxesCompanion(customBoxesSerializers) + + val boxListFirstModifier = new ListBuffer[SidechainTypes#SCB]() + val boxListSecondModifier = new ListBuffer[SidechainTypes#SCB]() + val storedBoxListFirstModifier = new ListBuffer[JPair[ByteArrayWrapper, ByteArrayWrapper]]() + val storedBoxListSecondModifier = new ListBuffer[JPair[ByteArrayWrapper, ByteArrayWrapper]]() + val firstModifierBoxLength = 7 + + val firstModifier: ByteArrayWrapper = getVersion + val secondModifier: ByteArrayWrapper = getVersion; + + + val backupper: BoxBackupInterface = new BoxBackupInterface { + override def backup(source: BoxIterator, db: BackupStorage): Unit = { + val updateList = new JArrayList[JPair[ByteArrayWrapper,ByteArrayWrapper]]() + + var optionalBox = source.nextBox(true) + while(optionalBox.isPresent) { + val box = optionalBox.get.getBox + if (!box.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) { + updateList.add(new JPair[ByteArrayWrapper, ByteArrayWrapper](Utils.calculateKey(box.id()), + new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box)))) + } + optionalBox = source.nextBox(true) + } + if (updateList.size() != 0) + db.update(getVersion,updateList) + } + } + + val _temporaryFolder = new TemporaryFolder() + @Rule def temporaryFolder = _temporaryFolder + + @Before + def setup(): Unit = { + boxListFirstModifier ++= getZenBoxList(5).asScala.toList + boxListFirstModifier ++= getCustomBoxList(firstModifierBoxLength).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + + boxListSecondModifier ++= getZenBoxList(5).asScala.toList + boxListSecondModifier ++= getCustomBoxList(5).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + + for (b <- boxListFirstModifier) { + storedBoxListFirstModifier.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new JPair(key, value) + }) + } + + for (b <- boxListSecondModifier) { + storedBoxListSecondModifier.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())) + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)) + new JPair(key, value) + }) + } + } + + @Test + def testCreateBackupWithNoCopy(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + val stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new VersionedLevelDbStorageAdapter(backupStorageFile) + + //Update a first time the SidechainStateStorage with some custom and zen boxes + stateStorage.update(firstModifier, storedBoxListFirstModifier.asJava, new JArrayList[ByteArrayWrapper]()) + //Update a second time the SidechainStateStorage with some custom and zen boxes + stateStorage.update(secondModifier, storedBoxListSecondModifier.asJava, new JArrayList[ByteArrayWrapper]()) + + //Instantiate a SidechainBackup class and call createBackup with no Copy option + val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, stateStorage = stateStorage, backUpStorage = backupStorage, backUpper = backupper); + sidechainBakcup.createBackup(BytesUtils.toHexString(firstModifier.data()), false, JOptional.empty()) + + //Read the backup storage created and verify that contains only firstModifierBoxLength elements. (We did a rollback to the first modifier) + val storedBoxes = readStorage(new BoxIterator(backupStorage.getIterator(), sidechainBoxesCompanion)) + assertEquals("BackupStorage should contains only the firstModifierBoxLength CustomBoxes of the storedBoxListFirstModifier!",storedBoxes.size(), firstModifierBoxLength) + + //Verify that the backupped boxes are the ones saved in the first SidechainStateStorage update and they are not CoinBoxes. + storedBoxes.forEach(box => { + val storageElement = new JPair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper(box.getBoxKey), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box.getBox))) + assertTrue("Restored boxes should be inside storedBoxListFirstModifier",storedBoxListFirstModifier.contains(storageElement)) + assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) + }) + + //Verify that the lastVersion of the StateStorage now is the firstModifier + assertEquals(stateStorage.lastVersionID().get().data().deep, firstModifier.data().deep) + } + + @Test + def testCreateBackupWithCopy(): Unit = { + //Create temporary SidechainStateStorage + val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") + val stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + + //Create temporary BackupStorage + val backupStorageFile = temporaryFolder.newFolder("backupStorage") + val backupStorage = new VersionedLevelDbStorageAdapter(backupStorageFile) + + //Update a first time the SidechainStateStorage with some custom and zen boxes + val firstModifier = getVersion; + stateStorage.update(firstModifier, storedBoxListFirstModifier.asJava, new JArrayList[ByteArrayWrapper]()) + //Update a second time the SidechainStateStorage with some custom and zen boxes + val secondModifier = getVersion; + stateStorage.update(secondModifier, storedBoxListSecondModifier.asJava, new JArrayList[ByteArrayWrapper]()) + + //Instantiate a SidechainBackup class and call createBackup + val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, stateStorage = stateStorage, backUpStorage = backupStorage, backUpper = backupper); + sidechainBakcup.createBackup(BytesUtils.toHexString(firstModifier.data()), true, JOptional.of(stateStorageFile.getPath)) + + //Read the backup storage created and verify that contains only firstModifierBoxLength elements. (We did a rollback to the first modifier) + val storedBoxes = readStorage(new BoxIterator(backupStorage.getIterator(), sidechainBoxesCompanion)) + assertEquals("BackupStorage should contains only the firstModifierBoxLength CustomBoxes of the storedBoxListFirstModifier!",storedBoxes.size(), firstModifierBoxLength) + + //Verify that the backupped boxes are the ones saved in the first SidechainStateStorage update and they are not CoinBoxes. + storedBoxes.forEach(box => { + val storageElement = new JPair[ByteArrayWrapper, ByteArrayWrapper](new ByteArrayWrapper(box.getBoxKey), new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(box.getBox))) + assertTrue("Restored boxes should be inside storedBoxListFirstModifier",storedBoxListFirstModifier.contains(storageElement)) + assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) + }) + + //Verify that the lastVersion of the StateStorage now is the firstModifier + assertEquals(stateStorage.lastVersionID().get().data().deep, secondModifier.data().deep) + } + + def readStorage(sidechainStateStorageBoxIterator: BoxIterator): JArrayList[BackupBox] = { + val storedBoxes = new JArrayList[BackupBox]() + + var optionalBox = sidechainStateStorageBoxIterator.nextBox + while(optionalBox.isPresent) { + storedBoxes.add(optionalBox.get) + optionalBox = sidechainStateStorageBoxIterator.nextBox + } + storedBoxes + } + +} From bc1a5fdbefb7555869cdd22919fe8560d91fedd9 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Wed, 11 May 2022 15:50:22 +0200 Subject: [PATCH 16/26] [SCHAINS-422] Removed empty lines --- .../main/scala/com/horizen/api/http/SidechainCswApiRoute.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala index 47de502e0f..0074d6b03b 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainCswApiRoute.scala @@ -134,7 +134,6 @@ case class SidechainCswApiRoute(override val settings: RESTApiSettings, } } } - } object SidechainCswRestScheme { @@ -167,7 +166,6 @@ object SidechainCswRestScheme { @JsonView(Array(classOf[Views.Default])) private[api] case class RespNullifier(nullifier: String) extends SuccessResponse - } object SidechainCswErrorResponse { @@ -190,5 +188,4 @@ object SidechainCswErrorResponse { case class ErrorRetrievingNullifier(description: String, exception: JOptional[Throwable]) extends ErrorResponse { override val code: String = "0705" } - } \ No newline at end of file From 4161f891828534149a69c24b0ff1ae13d455099d Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Tue, 17 May 2022 16:26:58 +0200 Subject: [PATCH 17/26] [SCHAINS-422] Removed optionalStateStoragePath from the SidechainBackup interface --- .../scala/com/horizen/SidechainBackup.scala | 39 +++++++------------ .../com/horizen/SidechainBackupTest.scala | 16 +++++--- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/SidechainBackup.scala b/sdk/src/main/scala/com/horizen/SidechainBackup.scala index a178953473..f839120ef2 100644 --- a/sdk/src/main/scala/com/horizen/SidechainBackup.scala +++ b/sdk/src/main/scala/com/horizen/SidechainBackup.scala @@ -15,45 +15,35 @@ import java.io._ import java.lang.{Byte => JByte} import java.util.{HashMap => JHashMap} import scala.util.{Failure, Success} -import java.util.{Optional => JOptional} class SidechainBackup @Inject() (@Named("CustomBoxSerializers") val customBoxSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]], - @Named("StateStorage") val stateStorage: Storage, @Named("BackupStorage") val backUpStorage: Storage, @Named("BackUpper") val backUpper : BoxBackupInterface ) extends ScorexLogging { protected val sidechainBoxesCompanion: SidechainBoxesCompanion = SidechainBoxesCompanion(customBoxSerializers) - protected var sidechainStateStorage: SidechainStateStorage = new SidechainStateStorage( - stateStorage, - sidechainBoxesCompanion) protected val backupStorage = new BackupStorage(backUpStorage, sidechainBoxesCompanion) - def createBackup(sidechainBlockIdToRollback: String, copyStateStorage: Boolean, optionalStateStoragePath: JOptional[String]): Unit = { + def createBackup(stateStoragePath: String, sidechainBlockIdToRollback: String, copyStateStorage: Boolean): Unit = { + var storagePath = stateStoragePath + if (copyStateStorage) { - if (optionalStateStoragePath.isEmpty) { - log.error("Error during the copy of the StateStorage: no stateStorage path provided!") - throw new RuntimeException("Error during the copy of the StateStorage: no stateStorage path provided!") - } else { - val stateStoragePath = optionalStateStoragePath.get() - val stateStorage: File = new File(stateStoragePath) - val stateStorageBackup: File = new File(stateStoragePath+"_copy_for_backup") + val stateStorage: File = new File(stateStoragePath) + val stateStorageBackup: File = new File(stateStoragePath+"_copy_for_backup") - try { - FileUtils.copyDirectory(stateStorage, stateStorageBackup) - sidechainStateStorage = new SidechainStateStorage( - new VersionedLevelDbStorageAdapter(stateStorageBackup), - sidechainBoxesCompanion) - } catch { - case t: Throwable => - log.error("Error during the copy of the StateStorage: ",t.getMessage) - throw new RuntimeException("Error during the copy of the StateStorage: "+t.getMessage) - } + try { + FileUtils.copyDirectory(stateStorage, stateStorageBackup) + storagePath = stateStoragePath+"_copy_for_backup" + } catch { + case t: Throwable => + log.error("Error during the copy of the StateStorage: ",t.getMessage) + throw new RuntimeException("Error during the copy of the StateStorage: "+t.getMessage) } } - + val storage = new VersionedLevelDbStorageAdapter(new File(storagePath)) + val sidechainStateStorage = new SidechainStateStorage(storage, sidechainBoxesCompanion) sidechainStateStorage.rollback(new ByteArrayWrapper(BytesUtils.fromHexString(sidechainBlockIdToRollback))) match { case Success(stateStorage) => log.info(s"Rollback of the SidechainStateStorage completed successfully!") @@ -65,6 +55,7 @@ class SidechainBackup @Inject() //Perform the backup in the application level try { backUpper.backup(new BoxIterator(stateIterator, sidechainBoxesCompanion), backupStorage) + storage.close() } catch { case t: Throwable => log.error("Error during the Backup generation: ",t.getMessage) diff --git a/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala b/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala index 6411f7c052..9b57d0dd38 100644 --- a/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainBackupTest.scala @@ -91,7 +91,7 @@ class SidechainBackupTest def testCreateBackupWithNoCopy(): Unit = { //Create temporary SidechainStateStorage val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") - val stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + var stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) //Create temporary BackupStorage val backupStorageFile = temporaryFolder.newFolder("backupStorage") @@ -101,10 +101,11 @@ class SidechainBackupTest stateStorage.update(firstModifier, storedBoxListFirstModifier.asJava, new JArrayList[ByteArrayWrapper]()) //Update a second time the SidechainStateStorage with some custom and zen boxes stateStorage.update(secondModifier, storedBoxListSecondModifier.asJava, new JArrayList[ByteArrayWrapper]()) + stateStorage.close() //Instantiate a SidechainBackup class and call createBackup with no Copy option - val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, stateStorage = stateStorage, backUpStorage = backupStorage, backUpper = backupper); - sidechainBakcup.createBackup(BytesUtils.toHexString(firstModifier.data()), false, JOptional.empty()) + val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, backUpStorage = backupStorage, backUpper = backupper); + sidechainBakcup.createBackup(stateStorageFile.getPath, BytesUtils.toHexString(firstModifier.data()), false) //Read the backup storage created and verify that contains only firstModifierBoxLength elements. (We did a rollback to the first modifier) val storedBoxes = readStorage(new BoxIterator(backupStorage.getIterator(), sidechainBoxesCompanion)) @@ -117,6 +118,7 @@ class SidechainBackupTest assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) }) + stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) //Verify that the lastVersion of the StateStorage now is the firstModifier assertEquals(stateStorage.lastVersionID().get().data().deep, firstModifier.data().deep) } @@ -125,7 +127,7 @@ class SidechainBackupTest def testCreateBackupWithCopy(): Unit = { //Create temporary SidechainStateStorage val stateStorageFile = temporaryFolder.newFolder("sidechainStateStorage") - val stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) + var stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) //Create temporary BackupStorage val backupStorageFile = temporaryFolder.newFolder("backupStorage") @@ -137,10 +139,11 @@ class SidechainBackupTest //Update a second time the SidechainStateStorage with some custom and zen boxes val secondModifier = getVersion; stateStorage.update(secondModifier, storedBoxListSecondModifier.asJava, new JArrayList[ByteArrayWrapper]()) + stateStorage.close() //Instantiate a SidechainBackup class and call createBackup - val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, stateStorage = stateStorage, backUpStorage = backupStorage, backUpper = backupper); - sidechainBakcup.createBackup(BytesUtils.toHexString(firstModifier.data()), true, JOptional.of(stateStorageFile.getPath)) + val sidechainBakcup = new SidechainBackup(customBoxSerializers = customBoxesSerializers, backUpStorage = backupStorage, backUpper = backupper); + sidechainBakcup.createBackup(stateStorageFile.getPath, BytesUtils.toHexString(firstModifier.data()), true) //Read the backup storage created and verify that contains only firstModifierBoxLength elements. (We did a rollback to the first modifier) val storedBoxes = readStorage(new BoxIterator(backupStorage.getIterator(), sidechainBoxesCompanion)) @@ -153,6 +156,7 @@ class SidechainBackupTest assertTrue("Restored boxes shouldn't be CoinBoxes!",!box.getBox.isInstanceOf[CoinsBox[_ <: PublicKey25519Proposition]]) }) + stateStorage = new VersionedLevelDbStorageAdapter(stateStorageFile) //Verify that the lastVersion of the StateStorage now is the firstModifier assertEquals(stateStorage.lastVersionID().get().data().deep, secondModifier.data().deep) } From a455233b78f8e08f371491ce41dc8399de4ab060 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 26 May 2022 09:59:39 +0200 Subject: [PATCH 18/26] [SCHAINS-422] Reassign genesisWallet after the scanBackup --- sdk/src/main/scala/com/horizen/SidechainWallet.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index 4d0a327e21..26c50295f3 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -461,9 +461,9 @@ object SidechainWallet ) : Try[SidechainWallet] = Try { if (walletBoxStorage.isEmpty) { - val genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, + var genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, forgingBoxesInfoStorage, cswDataStorage, params, applicationWallet) - genesisWallet.scanBackUp(backupStorage.getBoxIterator, genesisBlock.timestamp).get + genesisWallet = genesisWallet.scanBackUp(backupStorage.getBoxIterator, genesisBlock.timestamp).get genesisWallet.scanPersistent(genesisBlock, withdrawalEpochNumber, Seq(), None).applyConsensusEpochInfo(consensusEpochInfo) } else From d1c77d035be51a0f21bba525bd66c87331debcca Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 26 May 2022 14:36:27 +0200 Subject: [PATCH 19/26] [SCHAINS-422] Fixes after rebase --- sdk/src/main/scala/com/horizen/SidechainBackup.scala | 1 + sdk/src/main/scala/com/horizen/SidechainState.scala | 1 + sdk/src/main/scala/com/horizen/SidechainWallet.scala | 2 +- sdk/src/test/scala/com/horizen/SidechainWalletTest.scala | 5 +++++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/src/main/scala/com/horizen/SidechainBackup.scala b/sdk/src/main/scala/com/horizen/SidechainBackup.scala index f839120ef2..91f5076b87 100644 --- a/sdk/src/main/scala/com/horizen/SidechainBackup.scala +++ b/sdk/src/main/scala/com/horizen/SidechainBackup.scala @@ -58,6 +58,7 @@ class SidechainBackup @Inject() storage.close() } catch { case t: Throwable => + storage.close() log.error("Error during the Backup generation: ",t.getMessage) throw new RuntimeException("Error during the Backup generation: "+t.getMessage) } diff --git a/sdk/src/main/scala/com/horizen/SidechainState.scala b/sdk/src/main/scala/com/horizen/SidechainState.scala index ab14b1bac8..b4aa0b24a3 100644 --- a/sdk/src/main/scala/com/horizen/SidechainState.scala +++ b/sdk/src/main/scala/com/horizen/SidechainState.scala @@ -542,6 +542,7 @@ class SidechainState private[horizen] (stateStorage: SidechainStateStorage, case Success(_) => this case Failure(e) => + log.error("Error during the backup restore inside the SidechainState", e) throw e } } diff --git a/sdk/src/main/scala/com/horizen/SidechainWallet.scala b/sdk/src/main/scala/com/horizen/SidechainWallet.scala index 26c50295f3..7eb7972848 100644 --- a/sdk/src/main/scala/com/horizen/SidechainWallet.scala +++ b/sdk/src/main/scala/com/horizen/SidechainWallet.scala @@ -462,7 +462,7 @@ object SidechainWallet if (walletBoxStorage.isEmpty) { var genesisWallet = new SidechainWallet(seed, walletBoxStorage, secretStorage, walletTransactionStorage, - forgingBoxesInfoStorage, cswDataStorage, params, applicationWallet) + forgingBoxesInfoStorage, cswDataStorage, params, idToVersion(genesisBlock.parentId), applicationWallet) genesisWallet = genesisWallet.scanBackUp(backupStorage.getBoxIterator, genesisBlock.timestamp).get genesisWallet.scanPersistent(genesisBlock, withdrawalEpochNumber, Seq(), None).applyConsensusEpochInfo(consensusEpochInfo) } diff --git a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala index 07d021776e..d240a0ad36 100644 --- a/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala +++ b/sdk/src/test/scala/com/horizen/SidechainWalletTest.scala @@ -405,6 +405,8 @@ class SidechainWalletTest val mockedForgingBoxesInfoStorage: ForgingBoxesInfoStorage = mock[ForgingBoxesInfoStorage] val mockedCswDataStorage: SidechainWalletCswDataStorage = mock[SidechainWalletCswDataStorage] val mockedApplicationWallet: ApplicationWallet = mock[ApplicationWallet] + val mockedVersion: VersionTag = bytesToVersion(Array[Byte](32)) + Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>customSecretList.toList) //Create temporary WalletBoxStorage @@ -425,6 +427,7 @@ class SidechainWalletTest mockedForgingBoxesInfoStorage, mockedCswDataStorage, params, + mockedVersion, mockedApplicationWallet) // Mock get and update methods of SecretStorage @@ -448,6 +451,7 @@ class SidechainWalletTest val mockedForgingBoxesInfoStorage: ForgingBoxesInfoStorage = mock[ForgingBoxesInfoStorage] val mockedCswDataStorage: SidechainWalletCswDataStorage = mock[SidechainWalletCswDataStorage] val mockedApplicationWallet: ApplicationWallet = mock[ApplicationWallet] + val mockedVersion: VersionTag = bytesToVersion(Array[Byte](32)) Mockito.when(mockedSecretStorage.getAll).thenAnswer(_=>secretList.toList) //Create temporary WalletBoxStorage @@ -476,6 +480,7 @@ class SidechainWalletTest mockedForgingBoxesInfoStorage, mockedCswDataStorage, params, + mockedVersion, mockedApplicationWallet) var exceptionThrown = false From b0b8bf280c777d491f14c1f1b9c0aded91e0c435 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 26 May 2022 16:25:58 +0200 Subject: [PATCH 20/26] [SCHAINS-474] Added new endpoint getInitialBoxes needed by the Explorer to sync the restored boxes --- .../main/scala/com/horizen/SidechainApp.scala | 3 +- .../api/http/SidechainBackupApiRoute.scala | 56 ++++++++- .../scala/com/horizen/backup/BoxIterator.java | 26 ++++- .../api/http/SidechainApiRouteTest.scala | 39 ++++++- .../http/SidechainBackupApiRouteTest.scala | 108 ++++++++++++++++++ 5 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala diff --git a/sdk/src/main/scala/com/horizen/SidechainApp.scala b/sdk/src/main/scala/com/horizen/SidechainApp.scala index 54687f0101..b07416e8ba 100644 --- a/sdk/src/main/scala/com/horizen/SidechainApp.scala +++ b/sdk/src/main/scala/com/horizen/SidechainApp.scala @@ -342,6 +342,7 @@ class SidechainApp @Inject() var applicationApiRoutes : Seq[ApplicationApiRoute] = Seq[ApplicationApiRoute]() customApiGroups.asScala.foreach(apiRoute => applicationApiRoutes = applicationApiRoutes :+ ApplicationApiRoute(settings.restApi, apiRoute, nodeViewHolderRef)) + val boxIterator = backupStorage.getBoxIterator var coreApiRoutes: Seq[SidechainApiRoute] = Seq[SidechainApiRoute]( MainchainBlockApiRoute(settings.restApi, nodeViewHolderRef), SidechainBlockApiRoute(settings.restApi, nodeViewHolderRef, sidechainBlockActorRef, sidechainBlockForgerActorRef), @@ -350,7 +351,7 @@ class SidechainApp @Inject() SidechainWalletApiRoute(settings.restApi, nodeViewHolderRef), SidechainSubmitterApiRoute(settings.restApi, certificateSubmitterRef, nodeViewHolderRef), SidechainCswApiRoute(settings.restApi, nodeViewHolderRef, cswManager), - SidechainBackupApiRoute(settings.restApi, nodeViewHolderRef) + SidechainBackupApiRoute(settings.restApi, nodeViewHolderRef, boxIterator) ) val transactionSubmitProvider : TransactionSubmitProvider = new TransactionSubmitProviderImpl(sidechainTransactionActorRef) diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala index 10e9bb9476..bcd45e8b27 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala @@ -6,18 +6,27 @@ import com.fasterxml.jackson.annotation.JsonView import com.horizen.api.http.SidechainBackupRestScheme.RespSidechainBlockIdForBackup import com.horizen.serialization.Views import scorex.core.settings.RESTApiSettings -import com.horizen.api.http.SidechainBackupErrorResponse.ErrorRetrievingSidechainBlockIdForBackup +import com.horizen.api.http.SidechainBackupErrorResponse.{ErrorRetrievingSidechainBlockIdForBackup, GenericBackupApiError} import com.horizen.utils.BytesUtils import java.util.{Optional => JOptional} import scala.concurrent.ExecutionContext +import com.horizen.api.http.SidechainBackupRestScheme.{ReqGetInitialBoxes, RespGetInitialBoxes} +import com.horizen.box.Box +import com.horizen.proposition.Proposition + +import scala.util.{Failure, Success, Try} +import com.horizen.api.http.JacksonSupport._ +import com.horizen.backup.BoxIterator +import scala.collection.JavaConverters._ case class SidechainBackupApiRoute(override val settings: RESTApiSettings, - sidechainNodeViewHolderRef: ActorRef) + sidechainNodeViewHolderRef: ActorRef, + boxIterator: BoxIterator) (implicit val context: ActorRefFactory, override val ec: ExecutionContext) extends SidechainApiRoute { override val route: Route = pathPrefix("backup") { - getSidechainBlockIdForBackup + getSidechainBlockIdForBackup ~ getInitialBoxes } /*** @@ -42,15 +51,56 @@ case class SidechainBackupApiRoute(override val settings: RESTApiSettings, } } + + /** + * Return the initial boxes restored in a paginated way. + */ + def getInitialBoxes: Route = (post & path("getInitialBoxes")) { + entity(as[ReqGetInitialBoxes]) { body => + def getBoxId: JOptional[Array[Byte]] = body.lastBoxId match { + case Some(boxId) => + if (boxId.equals("")) { + JOptional.empty() + } else { + JOptional.of(BytesUtils.fromHexString(boxId)) + } + case None => + JOptional.empty() + } + + Try { + boxIterator.getNextBoxes(body.numberOfElements, getBoxId) + } match { + case Success(boxes) => + ApiResponseUtil.toResponse(RespGetInitialBoxes(boxes.asScala.toList)) + case Failure(e) => + ApiResponseUtil.toResponse(GenericBackupApiError("GenericBackupApiError", JOptional.of(e))) + } + } + } + } object SidechainBackupRestScheme { + final val MAX_NUMBER_OF_BOX_REQUEST = 100 + @JsonView(Array(classOf[Views.Default])) private[api] case class RespSidechainBlockIdForBackup(blockId: String) extends SuccessResponse + + @JsonView(Array(classOf[Views.Default])) + private[api] case class ReqGetInitialBoxes(numberOfElements: Int, lastBoxId: Option[String]) { + require(numberOfElements > 0, s"Invalid numberOfElements $numberOfElements. It should be > 0") + require(numberOfElements <= MAX_NUMBER_OF_BOX_REQUEST, s"Invalid numberOfElements $numberOfElements. It should be <= $MAX_NUMBER_OF_BOX_REQUEST") + } + @JsonView(Array(classOf[Views.Default])) + private[api] case class RespGetInitialBoxes(boxes: List[Box[Proposition]]) extends SuccessResponse } object SidechainBackupErrorResponse { case class ErrorRetrievingSidechainBlockIdForBackup(description: String, exception: JOptional[Throwable]) extends ErrorResponse { override val code: String = "0801" } + case class GenericBackupApiError(description: String, exception: JOptional[Throwable]) extends ErrorResponse { + override val code: String = "0802" + } } \ No newline at end of file diff --git a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java index 367cc96601..12df8e673e 100644 --- a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java +++ b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java @@ -10,9 +10,7 @@ import scorex.util.serialization.VLQByteBufferReader; import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; +import java.util.*; public class BoxIterator { private final StorageIterator iterator; @@ -28,6 +26,10 @@ public void seekToFirst() { this.iterator.seekToFirst(); } + public void seekIterator(byte[] key) { + iterator.seek(key); + } + public Optional nextBox(boolean ignoreCoinBox) throws RuntimeException { while (iterator.hasNext()) { Map.Entry entry = iterator.next(); @@ -54,6 +56,24 @@ public Optional nextBox() throws RuntimeException { return nextBox(false); } + public List> getNextBoxes(int nElement, Optional keyToSeek) { + if (keyToSeek.isPresent()) { + nElement += 1; + this.seekIterator(Utils.calculateKey(keyToSeek.get()).data()); + } else { + this.seekToFirst(); + } + List> boxes = new ArrayList<>(); + Optional nextBox = this.nextBox(); + while(boxes.size() < nElement && nextBox.isPresent()) { + boxes.add(nextBox.get().getBox()); + nextBox = this.nextBox(); + } + if (keyToSeek.isPresent()) + boxes.remove(0); + return boxes; + } + private boolean verifyBox(byte[] recordId, byte[] boxId) { return Arrays.equals(recordId, Utils.calculateKey(boxId).data()); } diff --git a/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala b/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala index a77dba1bb4..8b78f3aa35 100644 --- a/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala +++ b/sdk/src/test/scala/com/horizen/api/http/SidechainApiRouteTest.scala @@ -12,6 +12,9 @@ import com.horizen.SidechainNodeViewHolder.ReceivableMessages.{ApplyBiFunctionOn import com.horizen.api.http.SidechainBlockActor.ReceivableMessages.{GenerateSidechainBlocks, SubmitSidechainBlock} import com.horizen.api.http.SidechainTransactionActor.ReceivableMessages.BroadcastTransaction import com.horizen.companion.SidechainTransactionsCompanion +import com.horizen.backup.BoxIterator +import com.horizen.box.BoxSerializer +import com.horizen.companion.{SidechainBoxesCompanion} import com.horizen.consensus.ConsensusEpochAndSlot import com.horizen.fixtures.{CompanionsFixture, SidechainBlockFixture} import com.horizen.forge.Forger @@ -22,7 +25,7 @@ import com.horizen.transaction._ import com.horizen.{SidechainApp, SidechainSettings, SidechainTypes} import org.junit.Assert.{assertEquals, assertTrue} import org.junit.runner.RunWith -import org.mockito.Mockito +import org.mockito.{Mockito} import org.scalatestplus.junit.JUnitRunner import org.scalatestplus.mockito.MockitoSugar import org.scalatest.matchers.should.Matchers @@ -41,7 +44,16 @@ import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} import com.horizen.csw.CswManager.ReceivableMessages.{GenerateCswProof, GetBoxNullifier, GetCeasedStatus, GetCswBoxIds, GetCswInfo} import com.horizen.csw.CswManager.Responses.{Absent, CswInfo, CswProofInfo, NoProofData, ProofCreationFinished} +import com.horizen.customtypes.{CustomBox, CustomBoxSerializer} +import com.horizen.storage.StorageIterator +import com.horizen.utils.{ByteArrayWrapper} import org.bouncycastle.pqc.math.linearalgebra.ByteUtils +import scorex.crypto.hash.Blake2b256 + +import java.lang.{Byte => JByte} +import java.util.{HashMap => JHashMap} +import scala.collection.JavaConverters.asScalaBufferConverter +import scala.collection.mutable.ListBuffer import scala.language.postfixOps @@ -275,6 +287,30 @@ abstract class SidechainApiRouteTest extends AnyWordSpec with Matchers with Scal }) val mockedCswManagerActorRef: ActorRef = mockedCswManagerActor.ref + + val customBoxesSerializers: JHashMap[JByte, BoxSerializer[SidechainTypes#SCB]] = new JHashMap() + customBoxesSerializers.put(CustomBox.BOX_TYPE_ID, CustomBoxSerializer.getSerializer.asInstanceOf[BoxSerializer[SidechainTypes#SCB]]) + val sidechainBoxesCompanion = SidechainBoxesCompanion(customBoxesSerializers) + val mockedStorageIterator: StorageIterator = mock[StorageIterator] + var boxList:ListBuffer[SidechainTypes#SCB] = new ListBuffer[SidechainTypes#SCB]() + boxList ++= getCustomBoxList(3).asScala.map(_.asInstanceOf[SidechainTypes#SCB]) + val storedBoxList = new ListBuffer[util.Map.Entry[Array[Byte], Array[Byte]]]() + for (b <- boxList) { + storedBoxList.append({ + val key = new ByteArrayWrapper(Blake2b256.hash(b.id())).data() + val value = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes(b)).data() + val entry: util.Map.Entry[Array[Byte], Array[Byte]] = util.Map.entry(key, value) + entry + }) + } + + def mockStorageIterator = { + Mockito.when(mockedStorageIterator.hasNext).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false) + Mockito.when(mockedStorageIterator.next()).thenReturn(storedBoxList(0)).thenReturn(storedBoxList(1)).thenReturn(storedBoxList(2)) + } + mockStorageIterator + val mockedBoxIterator: BoxIterator = new BoxIterator(mockedStorageIterator, sidechainBoxesCompanion) + implicit def default() = RouteTestTimeout(3.second) val params = MainNetParams() @@ -289,6 +325,7 @@ abstract class SidechainApiRouteTest extends AnyWordSpec with Matchers with Scal val mainchainBlockApiRoute: Route = MainchainBlockApiRoute(mockedRESTSettings, mockedSidechainNodeViewHolderRef).route val applicationApiRoute: Route = ApplicationApiRoute(mockedRESTSettings, new SimpleCustomApi(), mockedSidechainNodeViewHolderRef).route val sidechainCswApiRoute: Route = SidechainCswApiRoute(mockedRESTSettings, mockedSidechainNodeViewHolderRef, mockedCswManagerActorRef).route + val sidechainBackupApiRoute: Route = SidechainBackupApiRoute(mockedRESTSettings, mockedSidechainNodeViewHolderRef, mockedBoxIterator).route val walletCoinsBalanceApiRejected: Route = SidechainRejectionApiRoute("wallet", "coinsBalance", mockedRESTSettings, mockedSidechainNodeViewHolderRef).route val walletApiRejected: Route = SidechainRejectionApiRoute("wallet", "", mockedRESTSettings, mockedSidechainNodeViewHolderRef).route diff --git a/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala b/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala new file mode 100644 index 0000000000..4cd658c846 --- /dev/null +++ b/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala @@ -0,0 +1,108 @@ +package com.horizen.api.http + +import akka.http.scaladsl.model.{ContentTypes, HttpMethods, StatusCodes} +import akka.http.scaladsl.server.{MalformedRequestContentRejection, MethodRejection, Route} +import com.horizen.api.http.SidechainBackupRestScheme.ReqGetInitialBoxes +import com.horizen.serialization.SerializationUtil +import com.horizen.utils.BytesUtils +import org.junit.Assert.{assertEquals, assertTrue} + +import scala.collection.JavaConverters.asScalaIteratorConverter + +class SidechainBackupApiRouteTest extends SidechainApiRouteTest{ + override val basePath = "/backup/" + + "The Api should to" should { + + "reject and reply with http error" in { + Get(basePath) ~> sidechainBackupApiRoute ~> check { + rejection shouldBe MethodRejection(HttpMethods.POST) + } + Get(basePath) ~> Route.seal(sidechainBackupApiRoute) ~> check { + status.intValue() shouldBe StatusCodes.MethodNotAllowed.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + } + Post(basePath + "getInitialBoxes").withEntity("maybe_a_json") ~> sidechainBackupApiRoute ~> check { + rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) + } + //Test with more than max numberOfElements. + Post(basePath + "getInitialBoxes").withEntity("{\"numberOfElements\": 101}") ~> sidechainBackupApiRoute ~> check { + rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) + } + //Test with negative numberOfElements. + Post(basePath + "getInitialBoxes").withEntity("{\"numberOfElements\": -1}") ~> sidechainBackupApiRoute ~> check { + rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) + } + } + } + "reply at /getInitialBoxes" in { + //Test with invalid "lastBoxId". + Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("invalid_json")))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("error") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + + val errorCode = result.get("code") + if (errorCode == null) + fail("Result serialization failed") + assertEquals(errorCode.asText(), "0802") + } + //Test with no "lastBoxId". It should return all mocked boxes + Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, None))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("result") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + assertEquals(1, result.findValues("boxes").size()) + + val boxes = result.get("boxes") + if (boxes == null) + fail("Result serialization failed") + + assertTrue(boxes.isArray) + assertEquals(storedBoxList.size, boxes.elements().asScala.length) + + mockStorageIterator + } + //Test with empty "lastBoxId". It should return all mocked boxes + Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("")))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("result") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + assertEquals(1, result.findValues("boxes").size()) + + val boxes = result.get("boxes") + if (boxes == null) + fail("Result serialization failed") + + assertTrue(boxes.isArray) + assertEquals(storedBoxList.size, boxes.elements().asScala.length) + + mockStorageIterator + } + //Test with "lastBoxId"=mockedBoxes.head. It should skip the first box of the mockedBox list. + //Also test that we return less boxes than "numberOfElement" requested in case of no more boxes. + Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some(BytesUtils.toHexString(boxList.head.id()))))) ~> sidechainBackupApiRoute ~> check { + status.intValue() shouldBe StatusCodes.OK.intValue + responseEntity.getContentType() shouldEqual ContentTypes.`application/json` + val result = mapper.readTree(entityAs[String]).get("result") + if (result == null) + fail("Serialization failed for object SidechainApiResponseBody") + assertEquals(1, result.findValues("boxes").size()) + + val boxes = result.get("boxes") + if (boxes == null) + fail("Result serialization failed") + + assertTrue(boxes.isArray) + assertEquals(storedBoxList.size -1, boxes.elements().asScala.length) + + mockStorageIterator + } + } +} \ No newline at end of file From c77f8876ee80a88f2197f0b70223bb2595a99f5f Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Thu, 26 May 2022 16:32:33 +0200 Subject: [PATCH 21/26] [SCHAINS-474] Renamed getInitialBoxes endpoint into getRestoredBoxes --- .../api/http/SidechainBackupApiRoute.scala | 4 ++-- .../api/http/SidechainBackupApiRouteTest.scala | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala index bcd45e8b27..5d94a3ac61 100644 --- a/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala +++ b/sdk/src/main/scala/com/horizen/api/http/SidechainBackupApiRoute.scala @@ -26,7 +26,7 @@ case class SidechainBackupApiRoute(override val settings: RESTApiSettings, boxIterator: BoxIterator) (implicit val context: ActorRefFactory, override val ec: ExecutionContext) extends SidechainApiRoute { override val route: Route = pathPrefix("backup") { - getSidechainBlockIdForBackup ~ getInitialBoxes + getSidechainBlockIdForBackup ~ getRestoredBoxes } /*** @@ -55,7 +55,7 @@ case class SidechainBackupApiRoute(override val settings: RESTApiSettings, /** * Return the initial boxes restored in a paginated way. */ - def getInitialBoxes: Route = (post & path("getInitialBoxes")) { + def getRestoredBoxes: Route = (post & path("getRestoredBoxes")) { entity(as[ReqGetInitialBoxes]) { body => def getBoxId: JOptional[Array[Byte]] = body.lastBoxId match { case Some(boxId) => diff --git a/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala b/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala index 4cd658c846..539e2f1167 100644 --- a/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala +++ b/sdk/src/test/scala/com/horizen/api/http/SidechainBackupApiRouteTest.scala @@ -22,22 +22,22 @@ class SidechainBackupApiRouteTest extends SidechainApiRouteTest{ status.intValue() shouldBe StatusCodes.MethodNotAllowed.intValue responseEntity.getContentType() shouldEqual ContentTypes.`application/json` } - Post(basePath + "getInitialBoxes").withEntity("maybe_a_json") ~> sidechainBackupApiRoute ~> check { + Post(basePath + "getRestoredBoxes").withEntity("maybe_a_json") ~> sidechainBackupApiRoute ~> check { rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) } //Test with more than max numberOfElements. - Post(basePath + "getInitialBoxes").withEntity("{\"numberOfElements\": 101}") ~> sidechainBackupApiRoute ~> check { + Post(basePath + "getRestoredBoxes").withEntity("{\"numberOfElements\": 101}") ~> sidechainBackupApiRoute ~> check { rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) } //Test with negative numberOfElements. - Post(basePath + "getInitialBoxes").withEntity("{\"numberOfElements\": -1}") ~> sidechainBackupApiRoute ~> check { + Post(basePath + "getRestoredBoxes").withEntity("{\"numberOfElements\": -1}") ~> sidechainBackupApiRoute ~> check { rejection.getClass.getCanonicalName.contains(MalformedRequestContentRejection.getClass.getCanonicalName.toString) } } } - "reply at /getInitialBoxes" in { + "reply at /getRestoredBoxes" in { //Test with invalid "lastBoxId". - Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("invalid_json")))) ~> sidechainBackupApiRoute ~> check { + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("invalid_json")))) ~> sidechainBackupApiRoute ~> check { status.intValue() shouldBe StatusCodes.OK.intValue responseEntity.getContentType() shouldEqual ContentTypes.`application/json` val result = mapper.readTree(entityAs[String]).get("error") @@ -50,7 +50,7 @@ class SidechainBackupApiRouteTest extends SidechainApiRouteTest{ assertEquals(errorCode.asText(), "0802") } //Test with no "lastBoxId". It should return all mocked boxes - Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, None))) ~> sidechainBackupApiRoute ~> check { + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, None))) ~> sidechainBackupApiRoute ~> check { status.intValue() shouldBe StatusCodes.OK.intValue responseEntity.getContentType() shouldEqual ContentTypes.`application/json` val result = mapper.readTree(entityAs[String]).get("result") @@ -68,7 +68,7 @@ class SidechainBackupApiRouteTest extends SidechainApiRouteTest{ mockStorageIterator } //Test with empty "lastBoxId". It should return all mocked boxes - Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("")))) ~> sidechainBackupApiRoute ~> check { + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some("")))) ~> sidechainBackupApiRoute ~> check { status.intValue() shouldBe StatusCodes.OK.intValue responseEntity.getContentType() shouldEqual ContentTypes.`application/json` val result = mapper.readTree(entityAs[String]).get("result") @@ -87,7 +87,7 @@ class SidechainBackupApiRouteTest extends SidechainApiRouteTest{ } //Test with "lastBoxId"=mockedBoxes.head. It should skip the first box of the mockedBox list. //Also test that we return less boxes than "numberOfElement" requested in case of no more boxes. - Post(basePath + "getInitialBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some(BytesUtils.toHexString(boxList.head.id()))))) ~> sidechainBackupApiRoute ~> check { + Post(basePath + "getRestoredBoxes").withEntity(SerializationUtil.serialize(ReqGetInitialBoxes(3, Some(BytesUtils.toHexString(boxList.head.id()))))) ~> sidechainBackupApiRoute ~> check { status.intValue() shouldBe StatusCodes.OK.intValue responseEntity.getContentType() shouldEqual ContentTypes.`application/json` val result = mapper.readTree(entityAs[String]).get("result") From 9b46f136e3fc9fa5935aa7e467adfa68ed08792d Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Fri, 27 May 2022 09:58:40 +0200 Subject: [PATCH 22/26] [SCHAINS-474] Refactored getNextBoxes --- sdk/src/main/scala/com/horizen/backup/BoxIterator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java index 12df8e673e..432cd1b721 100644 --- a/sdk/src/main/scala/com/horizen/backup/BoxIterator.java +++ b/sdk/src/main/scala/com/horizen/backup/BoxIterator.java @@ -58,8 +58,8 @@ public Optional nextBox() throws RuntimeException { public List> getNextBoxes(int nElement, Optional keyToSeek) { if (keyToSeek.isPresent()) { - nElement += 1; this.seekIterator(Utils.calculateKey(keyToSeek.get()).data()); + this.nextBox(); } else { this.seekToFirst(); } @@ -69,8 +69,6 @@ public List> getNextBoxes(int nElement, Optional keyTo boxes.add(nextBox.get().getBox()); nextBox = this.nextBox(); } - if (keyToSeek.isPresent()) - boxes.remove(0); return boxes; } From 5592574362da80ee76d93f60ee810f5ef52e2826 Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Fri, 27 May 2022 12:54:34 +0200 Subject: [PATCH 23/26] [SCHAINS-474] Added UT for getNextBoxes method --- .../com/horizen/backup/BoxIteratorTest.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java index 2d91540707..571b272476 100644 --- a/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java +++ b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java @@ -122,6 +122,73 @@ public void BoxIteratorTestCoinBoxes() throws IOException { } + @Test + public void BoxIteratorNextBoxesTest() throws IOException { + //Create temporary BackupStorage + File stateStorageFile = temporaryFolder.newFolder("stateStorage"); + BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); + + //Popoulate the BackupStorage + backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), customBoxToSave).get(); + + //Create a BoxIterator + BoxIterator boxIterator = backupStorage.getBoxIterator(); + + //Test nextBoxes with nElement = 0 and empty key to seek + List> foundBoxes = boxIterator.getNextBoxes(0,Optional.empty()); + assert(foundBoxes.isEmpty()); + + //Test nextBoxes with nElement < 0 and empty key to seek + foundBoxes = boxIterator.getNextBoxes(-1,Optional.empty()); + boxIterator.seekToFirst(); + assert(foundBoxes.isEmpty()); + + //Test nextBoxes with nElement = nBoxes and empty key to seek + boxIterator.seekToFirst(); + foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.empty()); + assert(foundBoxes.size() == nBoxes); + //Test the content of the boxes. + for (Box box : foundBoxes) { + ByteArrayWrapper newKey = Utils.calculateKey(box.id()); + ByteArrayWrapper newValue = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + } + + //Test nextBoxes with nElement > nBoxes and empty key to seek + boxIterator.seekToFirst(); + foundBoxes = boxIterator.getNextBoxes(nBoxes+10,Optional.empty()); + assert(foundBoxes.size() == nBoxes); + //Test the content of the boxes. + for (Box box : foundBoxes) { + ByteArrayWrapper newKey = Utils.calculateKey(box.id()); + ByteArrayWrapper newValue = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + } + + //Test nextBoxes with nElement = nBoxes and keyToSeek=1st box saved + boxIterator.seekToFirst(); + List> boxStoredOrder = foundBoxes; + foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.of(boxStoredOrder.get(0).id())); + assert(foundBoxes.size() == nBoxes-1); + assert(!foundBoxes.contains(boxStoredOrder.get(0))); + //Test the content of the boxes. + for (Box box : foundBoxes) { + ByteArrayWrapper newKey = Utils.calculateKey(box.id()); + ByteArrayWrapper newValue = new ByteArrayWrapper(sidechainBoxesCompanion.toBytes((Box) box)); + assert(customBoxToSave.contains(new Pair(newKey, newValue))); + } + + //Test nextBoxes with nElement = nBoxes and keyToSeek=last box saved + boxIterator.seekToFirst(); + foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.of(boxStoredOrder.get(boxStoredOrder.size()-1).id())); + assert(foundBoxes.size() == 0); + + //Test nextBoxes with nElement = nBoxes and keyToSeek=unknown box + boxIterator.seekToFirst(); + foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.of(zenBoxes.get(0).id())); + assert(foundBoxes.size() == 0); + } + private ArrayList readStorage(BoxIterator boxIterator) { ArrayList storedBoxes = new ArrayList<>(); From a22e788ced65e18ae4f00b32ca2d7738725c329f Mon Sep 17 00:00:00 2001 From: MarcoOl94 Date: Fri, 27 May 2022 13:17:46 +0200 Subject: [PATCH 24/26] [SCHAINS-474] Updated UT for getNextBoxes --- .../java/com/horizen/backup/BoxIteratorTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java index 571b272476..ab80f33f2c 100644 --- a/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java +++ b/sdk/src/test/java/com/horizen/backup/BoxIteratorTest.java @@ -68,7 +68,7 @@ public void BoxIteratorTestCustomBoxes() throws IOException { //Add an additional random element to customBoxToSave list. customBoxToSave.add(new Pair(new ByteArrayWrapper("key1".getBytes()), new ByteArrayWrapper("value1".getBytes()))); - //Popoulate the BackupStorage + //Populate the BackupStorage backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), customBoxToSave).get(); //Create a BoxIterator @@ -105,7 +105,7 @@ public void BoxIteratorTestCoinBoxes() throws IOException { File stateStorageFile = temporaryFolder.newFolder("stateStorage"); BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); - //Popoulate the BackupStorage + //Populate the BackupStorage backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), zenBoxToSave).get(); //Create a BoxIterator @@ -128,7 +128,10 @@ public void BoxIteratorNextBoxesTest() throws IOException { File stateStorageFile = temporaryFolder.newFolder("stateStorage"); BackupStorage backupStorage = new BackupStorage(new VersionedLevelDbStorageAdapter(stateStorageFile), sidechainBoxesCompanion); - //Popoulate the BackupStorage + //Add an additional random element to customBoxToSave list. + customBoxToSave.add(new Pair(new ByteArrayWrapper("key1".getBytes()), new ByteArrayWrapper("value1".getBytes()))); + + //Populate the BackupStorage backupStorage.update(new ByteArrayWrapper(Utils.nextVersion()), customBoxToSave).get(); //Create a BoxIterator @@ -183,10 +186,6 @@ public void BoxIteratorNextBoxesTest() throws IOException { foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.of(boxStoredOrder.get(boxStoredOrder.size()-1).id())); assert(foundBoxes.size() == 0); - //Test nextBoxes with nElement = nBoxes and keyToSeek=unknown box - boxIterator.seekToFirst(); - foundBoxes = boxIterator.getNextBoxes(nBoxes,Optional.of(zenBoxes.get(0).id())); - assert(foundBoxes.size() == 0); } From ee7f93d449336a85db69be44b5f463015a10bf30 Mon Sep 17 00:00:00 2001 From: Oleksandr Iozhytsia Date: Mon, 30 May 2022 12:44:05 +0300 Subject: [PATCH 25/26] SDK version updated from 0.3.3 to 0.3.4 --- CHANGELOG.md | 4 ++++ README.md | 2 +- ci/run_sc.sh | 2 +- examples/simpleapp/README.md | 6 +++--- examples/simpleapp/mc_sc_workflow_example.md | 8 ++++---- examples/simpleapp/pom.xml | 4 ++-- pom.xml | 2 +- qa/SidechainTestFramework/sc_test_framework.py | 2 +- qa/SidechainTestFramework/scutil.py | 6 +++--- sdk/pom.xml | 2 +- tools/dbtool/pom.xml | 4 ++-- tools/sctool/pom.xml | 4 ++-- 12 files changed, 25 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 631a78a1b6..d6093352a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +**0.3.4** +1. Added possibility to perform backup and restore non-coin boxes to bootstrap a new sidechain with a history of non-coin data taken from other sidechain. +2. log4j version updated. + **0.3.3** 1. Mainchain block deserialization fix: CompactSize usage issue. 2. Bootstrapping tool improvement: scgenesisinfo data parsing. diff --git a/README.md b/README.md index b602651b78..f33761afce 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ While we keep monitoring the memory footprint of the proofs generation process, - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows: ``` -LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar:./target/lib/* com.horizen.examples.SimpleApp +LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/ci/run_sc.sh b/ci/run_sc.sh index 1ca672a009..228af7ced5 100755 --- a/ci/run_sc.sh +++ b/ci/run_sc.sh @@ -2,7 +2,7 @@ set -eo pipefail -SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.3}" +SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.4}" if [ -d "$1" ] && [ -f "$2" ]; then path_to_jemalloc="$(ldconfig -p | grep "$(arch)" | grep 'libjemalloc\.so\.1$' | tr -d ' ' | cut -d '>' -f 2)" diff --git a/examples/simpleapp/README.md b/examples/simpleapp/README.md index 89f4528e2b..b9f310fd42 100644 --- a/examples/simpleapp/README.md +++ b/examples/simpleapp/README.md @@ -19,12 +19,12 @@ Otherwise, to run SimpleApp outside the IDE: * (Windows) ``` cd Sidechains-SDK\examples\simpleapp - java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar;./target/lib/* com.horizen.examples.SimpleApp + java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar;./target/lib/* com.horizen.examples.SimpleApp ``` * (Linux) ``` cd ./Sidechains-SDK/examples/simpleapp - java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar:./target/lib/* com.horizen.examples.SimpleApp + java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp ``` On some Linux OSs during backward transfers certificates proofs generation a extremely big RAM consumption may happen, that will lead to the process force killing by the OS. @@ -36,7 +36,7 @@ While we keep monitoring the memory footprint of the proofs generation process, - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows: ``` - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.3.jar:./target/lib/* com.horizen.examples.SimpleApp + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/examples/simpleapp/mc_sc_workflow_example.md b/examples/simpleapp/mc_sc_workflow_example.md index e0884b227b..5c635f804c 100644 --- a/examples/simpleapp/mc_sc_workflow_example.md +++ b/examples/simpleapp/mc_sc_workflow_example.md @@ -15,7 +15,7 @@ Build SDK components by using command (in the root of the SDK folder): Run Bootstrapping tool using command: -`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.3.jar` +`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.4.jar` All other commands are performed as commands for Bootstrapping tool in next format: `"command name" "parameters for command in JSON format"`. For any help you could use command `help`, for exit just print `exit` @@ -397,15 +397,15 @@ Run SimpleApp with the `my_settings.conf`: * For Windows: ``` - java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf + java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf ``` * For Linux (Glibc): ``` - java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf + java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf ``` * For Linux (Jemalloc): ``` - LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf + LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/examples/simpleapp/pom.xml b/examples/simpleapp/pom.xml index 2a0f798deb..d04a8472f1 100644 --- a/examples/simpleapp/pom.xml +++ b/examples/simpleapp/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.horizen sidechains-sdk-simpleapp - 0.3.3 + 0.3.4 2018 UTF-8 @@ -16,7 +16,7 @@ io.horizen sidechains-sdk - 0.3.3 + 0.3.4 diff --git a/pom.xml b/pom.xml index e7de41a3ad..af1f0c04db 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 com.horizen Sidechains - 0.3.3 + 0.3.4 2018 UTF-8 diff --git a/qa/SidechainTestFramework/sc_test_framework.py b/qa/SidechainTestFramework/sc_test_framework.py index 3d030679f6..a4b4443dca 100644 --- a/qa/SidechainTestFramework/sc_test_framework.py +++ b/qa/SidechainTestFramework/sc_test_framework.py @@ -114,7 +114,7 @@ def main(self): help="Don't stop bitcoinds after the test execution") parser.add_option("--zendir", dest="zendir", default="ZenCore/src", help="Source directory containing zend/zen-cli (default: %default)") - parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future + parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future help="Directory containing .jar file for SC (default: %default)") parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="sc_test"), help="Root directory for datadirs") diff --git a/qa/SidechainTestFramework/scutil.py b/qa/SidechainTestFramework/scutil.py index 79d366951a..dd1c6092a9 100644 --- a/qa/SidechainTestFramework/scutil.py +++ b/qa/SidechainTestFramework/scutil.py @@ -120,7 +120,7 @@ def launch_bootstrap_tool(command_name, json_parameters): json_param = json.dumps(json_parameters) java_ps = subprocess.Popen(["java", "-jar", os.getenv("SIDECHAIN_SDK", - "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.3.jar", + "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.4.jar", command_name, json_param], stdout=subprocess.PIPE) sc_bootstrap_output = java_ps.communicate()[0] try: @@ -142,7 +142,7 @@ def launch_db_tool(dirName, command_name, json_parameters): json_param = json.dumps(json_parameters) java_ps = subprocess.Popen(["java", "-jar", os.getenv("SIDECHAIN_SDK", - "..") + "/tools/dbtool/target/sidechains-sdk-dbtools-0.3.3.jar", + "..") + "/tools/dbtool/target/sidechains-sdk-dbtools-0.3.4.jar", storagesPath, command_name, json_param], stdout=subprocess.PIPE) db_tool_output = java_ps.communicate()[0] try: @@ -468,7 +468,7 @@ def start_sc_node(i, dirname, extra_args=None, rpchost=None, timewait=None, bina lib_separator = ";" if binary is None: - binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.3.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp" + binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp" # else if platform.system() == 'Linux': ''' In order to effectively attach a debugger (e.g IntelliJ) to the simpleapp, it is necessary to start the process diff --git a/sdk/pom.xml b/sdk/pom.xml index 253c06fe58..0c33579384 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk - 0.3.3 + 0.3.4 ${project.groupId}:${project.artifactId} Zendoo is a unique sidechain and scaling solution developed by Horizen. The Zendoo ${project.artifactId} is a framework that supports the creation of sidechains and their custom business logic, with the Horizen public blockchain as the mainchain. https://github.com/${project.github.organization}/${project.artifactId} diff --git a/tools/dbtool/pom.xml b/tools/dbtool/pom.xml index 2045907b1f..101d3c7192 100644 --- a/tools/dbtool/pom.xml +++ b/tools/dbtool/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.horizen sidechains-sdk-dbtools - 0.3.3 + 0.3.4 2022 UTF-8 @@ -16,7 +16,7 @@ io.horizen sidechains-sdk - 0.3.3 + 0.3.4 diff --git a/tools/sctool/pom.xml b/tools/sctool/pom.xml index fb0b35637d..f82d1643e8 100644 --- a/tools/sctool/pom.xml +++ b/tools/sctool/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.horizen sidechains-sdk-scbootstrappingtools - 0.3.3 + 0.3.4 2018 UTF-8 @@ -16,7 +16,7 @@ io.horizen sidechains-sdk - 0.3.3 + 0.3.4 From 67b4ca794b65ca6ecf04d3e88d72f89dfd8a3eda Mon Sep 17 00:00:00 2001 From: Oleksandr Iozhytsia Date: Mon, 30 May 2022 12:54:29 +0300 Subject: [PATCH 26/26] changelog updated. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6093352a6..5dd46fb341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ **0.3.4** -1. Added possibility to perform backup and restore non-coin boxes to bootstrap a new sidechain with a history of non-coin data taken from other sidechain. +1. Added the possibility to perform a backup of a sidechain non coin-boxes and restore these boxes into a new bootstrapped sidechain of the same type. 2. log4j version updated. **0.3.3**