From 5ea24d5255fd7ddb0fc1f24cc2637eb8eda48c79 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Sat, 25 Sep 2021 11:34:53 +0300
Subject: [PATCH 01/58] unused import, var => val

---
 .../org/ergoplatform/nodeView/wallet/models/ChangeBox.scala    | 2 +-
 .../ergoplatform/nodeView/wallet/models/CollectedBoxes.scala   | 3 ++-
 .../nodeView/wallet/persistence/WalletRegistryBenchmark.scala  | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/models/ChangeBox.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/models/ChangeBox.scala
index 9006e28ebd..769237ecd7 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/models/ChangeBox.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/models/ChangeBox.scala
@@ -1,6 +1,6 @@
 package org.ergoplatform.nodeView.wallet.models
 
-import io.circe.generic.encoding.DerivedObjectEncoder.deriveEncoder
+import io.circe.generic.encoding.DerivedAsObjectEncoder.deriveEncoder
 import io.circe.syntax._
 import io.circe.{Encoder, Json, KeyEncoder}
 import scorex.util.ModifierId
diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/models/CollectedBoxes.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/models/CollectedBoxes.scala
index a8940abe55..2f1041d51e 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/models/CollectedBoxes.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/models/CollectedBoxes.scala
@@ -1,6 +1,6 @@
 package org.ergoplatform.nodeView.wallet.models
 
-import io.circe.generic.encoding.DerivedObjectEncoder.deriveEncoder
+import io.circe.generic.encoding.DerivedAsObjectEncoder.deriveEncoder
 import io.circe.syntax._
 import io.circe.{Encoder, Json}
 import org.ergoplatform.{ErgoBox, JsonCodecs}
@@ -20,4 +20,5 @@ object CollectedBoxes extends JsonCodecs {
       "boxes" -> request.boxes.asJson,
       "changeBoxes" -> request.changeBoxes.asJson
     )
+
 }
diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala
index adc1eae117..005035ff6b 100644
--- a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala
@@ -45,7 +45,7 @@ object WalletRegistryBenchmark extends App with ErgoTestConstants {
   }
 
   val prover = ErgoProvingInterpreter(rootSecret +: derivedSecrets, parameters)
-  var walletVars = WalletVars.apply(storage, settings).withProver(prover)
+  val walletVars = WalletVars.apply(storage, settings).withProver(prover)
 
   val boxes = walletVars.proverOpt.get.hdPubKeys.map { pk =>
     createBox(1000000000, pk.key, 1)

From 95e34fd1cf8d5d7ce3c464a273703652ec229261 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Sat, 25 Sep 2021 11:57:41 +0300
Subject: [PATCH 02/58] proper log levels and minor styling improvs in history
 & history storage

---
 .../nodeView/history/ErgoHistory.scala             | 14 ++++++++------
 .../nodeView/history/storage/HistoryStorage.scala  | 10 ++++++----
 .../modifierprocessors/HeadersProcessor.scala      |  2 +-
 .../BlockSectionValidationSpecification.scala      |  4 ++--
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala
index 1f3006dbf8..8239f99c0c 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala
@@ -85,13 +85,15 @@ trait ErgoHistory
 
         if (nonMarkedIds.nonEmpty) {
           historyStorage.insert(
-            nonMarkedIds.map(id => validityKey(id) -> Array(1.toByte)),
-            Seq.empty).map(_ => this)
-        } else Success(this)
+            indexesToInsert = nonMarkedIds.map(id => validityKey(id) -> Array(1.toByte)),
+            objectsToInsert = Seq.empty).map(_ => this)
+        } else {
+          Success(this)
+        }
       case _ =>
         historyStorage.insert(
-          Seq(validityKey(modifier.id) -> Array(1.toByte)),
-          Seq.empty).map(_ => this)
+          indexesToInsert = Seq(validityKey(modifier.id) -> Array(1.toByte)),
+          objectsToInsert = Seq.empty).map(_ => this)
     }
   }
 
@@ -105,7 +107,7 @@ trait ErgoHistory
   override def reportModifierIsInvalid(modifier: ErgoPersistentModifier,
                                        progressInfo: ProgressInfo[ErgoPersistentModifier]
                                       ): Try[(ErgoHistory, ProgressInfo[ErgoPersistentModifier])] = synchronized {
-    log.debug(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid")
+    log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid")
     correspondingHeader(modifier) match {
       case Some(invalidatedHeader) =>
         val invalidatedHeaders = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala
index 9a452ef2b7..7246487230 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala
@@ -13,8 +13,8 @@ import scala.util.{Failure, Success, Try}
 /**
   * Storage for Ergo history
   *
-  * @param indexStore   - Additional key-value storage for indexes, required by History for efficient work.
-  *                     contains links to bestHeader, bestFullBlock, heights and scores for different blocks, etc.
+  * @param indexStore   - Additional key-value storage for indexes used for efficient queries.
+  *                     Contains links to bestHeader, bestFullBlock, heights and scores for different blocks, etc.
   * @param objectsStore - key-value store, where key is id of ErgoPersistentModifier and value is it's bytes
   * @param config       - cache configs
   */
@@ -23,10 +23,12 @@ class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, config: C
     with AutoCloseable
     with ScorexEncoding {
 
+  // in-memory cache for modifiers
   private val modifiersCache = CacheBuilder.newBuilder()
     .maximumSize(config.modifiersCacheSize)
-    .build[String, ErgoPersistentModifier]
+    .build[ModifierId, ErgoPersistentModifier]
 
+  // in-memory cache for indexes
   private val indexCache = CacheBuilder.newBuilder()
     .maximumSize(config.indexesCacheSize)
     .build[ByteArrayWrapper, Array[Byte]]
@@ -81,7 +83,7 @@ class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, config: C
     }
 
   override def close(): Unit = {
-    log.warn("Closing history storage...")
+    log.info("Closing history storage...")
     indexStore.close()
     objectsStore.close()
   }
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala
index 323db5b6e3..9649d190e9 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala
@@ -47,7 +47,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score
 
   def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity
 
-  // todo for performance reasons we may just use key like s"score$id" but this will require to redownload blockchain
+  // todo for performance reasons we may just use key like s"score$id" but this will require to resync the blockchain
   protected def headerScoreKey(id: ModifierId): ByteArrayWrapper =
     ByteArrayWrapper(Algos.hash("score".getBytes(ErgoHistory.CharsetName) ++ idToBytes(id)))
 
diff --git a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala
index 1234804822..6682e10314 100644
--- a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala
@@ -1,6 +1,6 @@
 package org.ergoplatform.nodeView.history
 
-import org.ergoplatform.modifiers.BlockSection
+import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock}
 import org.ergoplatform.modifiers.history._
 import org.ergoplatform.modifiers.history.extension.Extension
 import org.ergoplatform.modifiers.history.header.Header
@@ -58,7 +58,7 @@ class BlockSectionValidationSpecification extends HistoryTestHelpers {
     commonChecks(history, block.extension, block.header)
   }
 
-  private def init(version: Version = Header.InitialVersion) = {
+  private def init(version: Version = Header.InitialVersion): (ErgoHistory, ErgoFullBlock) = {
     var history = genHistory()
     val chain = genChain(2, history, version)
     history = applyBlock(history, chain.head)

From 61f3d63eaa3402cf8b6d20f07e62ba3dd48f0654 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 27 Sep 2021 12:05:10 +0300
Subject: [PATCH 03/58] script/p2sh back

---
 src/main/resources/api/openapi.yaml           |  2 +-
 .../http/api/ScriptApiRoute.scala             |  7 ++--
 .../http/routes/ScriptApiRouteSpec.scala      | 39 +++++++++----------
 3 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 8087f465b1..dc19748a0d 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -4156,7 +4156,7 @@ paths:
               $ref: '#/components/schemas/SourceHolder'
       responses:
         '200':
-          description: Ergo address derived from source
+          description: P2SH address derived from source
           content:
             application/json:
               schema:
diff --git a/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
index 2c146b711f..17a4718538 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
@@ -23,7 +23,7 @@ import sigmastate.lang.SigmaCompiler
 import sigmastate.serialization.ValueSerializer
 import scala.concurrent.Future
 import scala.concurrent.duration._
-import scala.util.{Failure, Success, Try}
+import scala.util.{Failure, Try}
 
 
 case class ScriptApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
@@ -37,8 +37,8 @@ case class ScriptApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
 
   override val route: Route = pathPrefix("script") {
     toStrictEntity(10.seconds) {
-      // p2shAddressR ~
-      p2sAddressR ~
+      p2shAddressR ~
+        p2sAddressR ~
         addressToTreeR ~
         addressToBytesR ~
         executeWithContextR
@@ -84,7 +84,6 @@ case class ScriptApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
     }
   }
 
-  //todo: temporarily switched off due to https://github.com/ergoplatform/ergo/issues/936
   def p2shAddressR: Route = (path("p2shAddress") & post & source) { source =>
     withWalletOp(_.publicKeys(0, loadMaxKeys)) { addrs =>
       compileSource(source, keysToEnv(addrs.map(_.pubkey))).map(Pay2SHAddress.apply).fold(
diff --git a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala
index 002a1d7d14..d6ba40ad25 100644
--- a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala
@@ -5,7 +5,7 @@ import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.testkit.ScalatestRouteTest
 import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport
 import io.circe.Json
-import org.ergoplatform.Pay2SAddress
+import org.ergoplatform.{Pay2SAddress, Pay2SHAddress}
 import org.ergoplatform.settings.{Args, ErgoSettings}
 import org.ergoplatform.utils.Stubs
 import io.circe.syntax._
@@ -14,8 +14,8 @@ import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import scorex.util.encode.Base16
 import sigmastate.SByte
-import sigmastate.Values.{TrueLeaf, CollectionConstant, ErgoTree}
-import sigmastate.serialization.{ValueSerializer, ErgoTreeSerializer}
+import sigmastate.Values.{CollectionConstant, ErgoTree, TrueLeaf}
+import sigmastate.serialization.{ErgoTreeSerializer, ValueSerializer}
 
 
 class ScriptApiRouteSpec extends AnyFlatSpec
@@ -74,19 +74,17 @@ class ScriptApiRouteSpec extends AnyFlatSpec
       check(assertion(responseAs[Json]))
   }
 
-  //todo: temporarily switched off due to https://github.com/ergoplatform/ergo/issues/936
-//  it should "generate valid P2SHAddress form source" in {
-//    val suffix = "/p2shAddress"
-//    val assertion = (json: Json) => {
-//      status shouldBe StatusCodes.OK
-//      val addressStr = json.hcursor.downField("address").as[String].right.get
-//      ergoAddressEncoder.fromString(addressStr).get.addressTypePrefix shouldEqual Pay2SHAddress.addressTypePrefix
-//    }
-//    Post(prefix + suffix, Json.obj("source" -> scriptSource.asJson)) ~> route ~> check(assertion(responseAs[Json]))
-//    Post(prefix + suffix, Json.obj("source" -> scriptSourceSigProp.asJson)) ~> route ~>
-//      check(assertion(responseAs[Json]))
-//  }
-
+  it should "generate valid P2SHAddress form source" in {
+    val suffix = "/p2shAddress"
+    val assertion = (json: Json) => {
+      status shouldBe StatusCodes.OK
+      val addressStr = json.hcursor.downField("address").as[String].right.get
+      ergoAddressEncoder.fromString(addressStr).get.addressTypePrefix shouldEqual Pay2SHAddress.addressTypePrefix
+    }
+    Post(prefix + suffix, Json.obj("source" -> scriptSource.asJson)) ~> route ~> check(assertion(responseAs[Json]))
+    Post(prefix + suffix, Json.obj("source" -> scriptSourceSigProp.asJson)) ~> route ~>
+      check(assertion(responseAs[Json]))
+  }
 
   it should "get through address <-> ergoTree round-trip" in {
     val suffix = "addressToTree"
@@ -105,12 +103,13 @@ class ScriptApiRouteSpec extends AnyFlatSpec
     val p2pk = "3WvsT2Gm4EpsM9Pg18PdY6XyhNNMqXDsvJTbbf6ihLvAmSb7u5RN"
     Get(s"$prefix/$suffix/$p2pk") ~> route ~> check(assertion(responseAs[Json], p2pk))
 
-    //todo: temporarily switched off due to https://github.com/ergoplatform/ergo/issues/936
-//    val p2sh = "8UmyuJuQ3FS9ts7j72fn3fKChXSGzbL9WC"
-//    Get(s"$prefix/$suffix/$p2sh") ~> route ~> check(assertion(responseAs[Json], p2sh))
-
     val script = TrueLeaf
     val tree = ErgoTree.fromProposition(script)
+
+    val p2sh = Pay2SHAddress.apply(tree).toString()
+    p2sh shouldBe "rbcrmKEYduUvADj9Ts3dSVSG27h54pgrq5fPuwB"
+    Get(s"$prefix/$suffix/$p2sh") ~> route ~> check(assertion(responseAs[Json], p2sh))
+
     val p2s = ergoAddressEncoder.toString(ergoAddressEncoder.fromProposition(tree).get)
     p2s shouldBe "Ms7smJwLGbUAjuWQ"
     Get(s"$prefix/$suffix/$p2s") ~> route ~> check(assertion(responseAs[Json], p2s))

From 69bfc48b2176ebb0d2880e68a79b6aa2b3a0ce7a Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Thu, 30 Sep 2021 13:44:28 +0300
Subject: [PATCH 04/58] minor stylistic improvs

---
 .../scala/org/ergoplatform/http/api/ScriptApiRoute.scala    | 3 ++-
 .../scala/org/ergoplatform/http/api/WalletApiRoute.scala    | 6 +++---
 .../storage/modifierprocessors/ToDownloadProcessor.scala    | 6 +++++-
 .../nodeView/state/UtxoStateSpecification.scala             | 2 +-
 .../org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala   | 4 ++--
 5 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
index 17a4718538..f43ccdc4de 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
@@ -21,9 +21,10 @@ import sigmastate.basics.DLogProtocol.ProveDlog
 import sigmastate.eval.{CompiletimeIRContext, IRContext, RuntimeIRContext}
 import sigmastate.lang.SigmaCompiler
 import sigmastate.serialization.ValueSerializer
+
 import scala.concurrent.Future
 import scala.concurrent.duration._
-import scala.util.{Failure, Try}
+import scala.util.{Failure, Success, Try}
 
 
 case class ScriptApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
diff --git a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala
index 6790f58140..99c4d42c77 100644
--- a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala
@@ -162,17 +162,17 @@ case class WalletApiRoute(readersHolder: ActorRef, nodeViewActorRef: ActorRef, e
 
   def sendTransactionR: Route =
     (path("transaction" / "send") & post & entity(as[RequestsHolder])) { holder =>
-      sendTransaction(holder.withFee, holder.inputsRaw, holder.dataInputsRaw)
+      sendTransaction(holder.withFee(), holder.inputsRaw, holder.dataInputsRaw)
     }
 
   def generateTransactionR: Route =
     (path("transaction" / "generate") & post & entity(as[RequestsHolder])) { holder =>
-      generateTransaction(holder.withFee, holder.inputsRaw, holder.dataInputsRaw)
+      generateTransaction(holder.withFee(), holder.inputsRaw, holder.dataInputsRaw)
     }
 
   def generateUnsignedTransactionR: Route =
     (path("transaction" / "generateUnsigned") & post & entity(as[RequestsHolder])) { holder =>
-      generateUnsignedTransaction(holder.withFee, holder.inputsRaw, holder.dataInputsRaw)
+      generateUnsignedTransaction(holder.withFee(), holder.inputsRaw, holder.dataInputsRaw)
     }
 
   def generateCommitmentsR: Route = (path("generateCommitments")
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala
index f377c427b2..9c4e92048f 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala
@@ -40,6 +40,7 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging {
 
   /**
     * Get modifier ids to download to synchronize full blocks
+    *
     * @param howManyPerType how many ModifierIds per ModifierTypeId to fetch
     * @param condition filter only ModifierIds that pass this condition
     * @return next max howManyPerType ModifierIds by ModifierTypeId to download filtered by condition
@@ -56,7 +57,9 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging {
         if (headersAtThisHeight.nonEmpty) {
           val toDownload = headersAtThisHeight.flatMap(requiredModifiersForHeader).filter(m => condition(m._2))
           // add new modifiers to download to accumulator
-          val newAcc = toDownload.foldLeft(acc) { case (newAcc, (mType, mId)) => newAcc.adjust(mType)(_.fold(Vector(mId))(_ :+ mId)) }
+          val newAcc = toDownload.foldLeft(acc) { case (newAcc, (mType, mId)) =>
+            newAcc.adjust(mType)(_.fold(Vector(mId))(_ :+ mId))
+          }
           continuation(height + 1, newAcc)
         } else {
           acc
@@ -114,6 +117,7 @@ object ToDownloadProcessor {
   implicit class MapPimp[K, V](underlying: Map[K, V]) {
     /**
       * One liner for updating a Map with the possibility to handle case of missing Key
+      *
       * @param k map key
       * @param f function that is passed Option depending on Key being present or missing, returning new Value
       * @return new Map with value updated under given key
diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
index 8221a5fac4..5f7b756b03 100644
--- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
@@ -205,7 +205,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera
     chain.foreach { fb =>
       us2 = us2.applyModifier(fb).get
     }
-    Await.result(f, Duration.Inf);
+    Await.result(f, Duration.Inf)
   }
 
   property("proofsForTransactions() to be deterministic") {
diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala
index ca6c25db2b..54aa8a1e3e 100644
--- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala
@@ -888,7 +888,7 @@ class ErgoWalletSpec extends ErgoPropertyTest with WalletTestOps with Eventually
 
         val txSigned = await(wallet.signTransaction(utx, Seq(es2), hints1, Some(Seq(in)), None)).get
 
-        txSigned.statelessValidity.isSuccess shouldBe true
+        txSigned.statelessValidity().isSuccess shouldBe true
       }
     }
   }
@@ -936,7 +936,7 @@ class ErgoWalletSpec extends ErgoPropertyTest with WalletTestOps with Eventually
         val hints = hintsExtracted.addHintsForInput(0, cmts1.allHintsForInput(0))
 
         val txSigned = await(wallet.signTransaction(utx, Seq(es1), hints, Some(Seq(in)), None)).get
-        txSigned.statelessValidity.isSuccess shouldBe true
+        txSigned.statelessValidity().isSuccess shouldBe true
       }
     }
   }

From 19bd807f8b07382e6786b84a7030f5a627234393 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Sat, 2 Oct 2021 15:41:19 +0300
Subject: [PATCH 05/58] unused inspectAll removed

---
 .../ergoplatform/utils/ErgoTestHelpers.scala  | 23 ++-----------------
 1 file changed, 2 insertions(+), 21 deletions(-)

diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala
index b4f2a20d6a..79119dc8a4 100644
--- a/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala
+++ b/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala
@@ -4,18 +4,13 @@ import java.net.InetSocketAddress
 import java.util.concurrent.Executors
 
 import org.ergoplatform.ErgoBoxCandidate
-import org.ergoplatform.settings.{ErgoSettings, VotingSettings}
+import org.ergoplatform.settings.ErgoSettings
 import org.ergoplatform.utils.generators.ValidBlocksGenerators
-import org.scalactic.{Prettifier, source}
-import org.scalatest.enablers.{Collecting, InspectorAsserting}
-import org.scalatest.{EitherValues, Inspectors, OptionValues}
-import org.scalatest.{EitherValues, OptionValues, TryValues}
-import scorex.core.network.{Incoming, Outgoing}
+import org.scalatest.{EitherValues, OptionValues}
 import scorex.core.network.peer.PeerInfo
 import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding}
 import scorex.util.ScorexLogging
 
-import scala.collection.{GenMap, GenTraversable}
 import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.language.higherKinds
 
@@ -40,20 +35,6 @@ trait ErgoTestHelpers
     }
   }
 
-  def inspectAll[E, C[_], A](xs: C[E])(fun: E => A)
-                            (implicit collecting: Collecting[E, C[E]],
-                             asserting: InspectorAsserting[A],
-                             prettifier: Prettifier,
-                             pos: source.Position): InspectorAsserting[A]#Result =
-    Inspectors.forAll(xs)(fun)
-
-  def inspectAll[K, V, MAP[k, v] <: GenMap[k, v], A](xs: MAP[K, V])(fun: ((K, V)) => A)
-                                                    (implicit collecting: Collecting[(K, V), GenTraversable[(K, V)]],
-                                                     asserting: InspectorAsserting[A],
-                                                     prettifier: Prettifier,
-                                                     pos: source.Position): InspectorAsserting[A]#Result =
-    Inspectors.forAll(xs)(fun)
-
   val inetAddr1 = new InetSocketAddress("92.92.92.92", 27017)
   val inetAddr2 = new InetSocketAddress("93.93.93.93", 27017)
   val ts1 = System.currentTimeMillis() - 100

From 7eda819a037fed24dabfa14c565258da4ebbb9a3 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Thu, 18 Nov 2021 17:59:41 +0300
Subject: [PATCH 06/58] Box and Proposition removed

---
 .../scorex/core/transaction/box/Box.scala     | 20 -------------------
 .../box/proposition/Proposition.scala         |  5 -----
 2 files changed, 25 deletions(-)
 delete mode 100644 src/main/scala/scorex/core/transaction/box/Box.scala
 delete mode 100644 src/main/scala/scorex/core/transaction/box/proposition/Proposition.scala

diff --git a/src/main/scala/scorex/core/transaction/box/Box.scala b/src/main/scala/scorex/core/transaction/box/Box.scala
deleted file mode 100644
index 64bda2c971..0000000000
--- a/src/main/scala/scorex/core/transaction/box/Box.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-package scorex.core.transaction.box
-
-import scorex.core.serialization.BytesSerializable
-import scorex.core.transaction.box.proposition.Proposition
-import scorex.crypto.authds._
-
-/**
-  * Box is a state element locked by some proposition.
-  */
-trait Box[P <: Proposition] extends BytesSerializable {
-  val value: Box.Amount
-  val proposition: P
-
-  val id: ADKey
-}
-
-object Box {
-  type Amount = Long
-}
-
diff --git a/src/main/scala/scorex/core/transaction/box/proposition/Proposition.scala b/src/main/scala/scorex/core/transaction/box/proposition/Proposition.scala
deleted file mode 100644
index 0d9a076fc1..0000000000
--- a/src/main/scala/scorex/core/transaction/box/proposition/Proposition.scala
+++ /dev/null
@@ -1,5 +0,0 @@
-package scorex.core.transaction.box.proposition
-
-import scorex.core.serialization.BytesSerializable
-
-trait Proposition extends BytesSerializable

From eeba0daae040ed4e134729cfd61b8b8d63304194 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 22 Nov 2021 19:05:50 +0300
Subject: [PATCH 07/58] unused code removed from NetworkController and
 ModifierSemanticValidity

---
 .../wallet/requests/AssetIssueRequest.scala   |  7 ++---
 .../consensus/ModifierSemanticValidity.scala  |  5 ---
 .../core/network/NetworkController.scala      | 31 ++-----------------
 .../scala/scorex/core/network/PeerSpec.scala  |  2 --
 4 files changed, 5 insertions(+), 40 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/AssetIssueRequest.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/AssetIssueRequest.scala
index 785d7223e9..714e8cb61f 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/AssetIssueRequest.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/AssetIssueRequest.scala
@@ -7,7 +7,6 @@ import org.ergoplatform.ErgoBox.NonMandatoryRegisterId
 import org.ergoplatform.http.api.ApiCodecs
 import org.ergoplatform.nodeView.wallet.ErgoAddressJsonEncoder
 import org.ergoplatform.settings.ErgoSettings
-import scorex.core.transaction.box.Box.Amount
 import sigmastate.SType
 import sigmastate.Values.EvaluatedValue
 
@@ -24,7 +23,7 @@ import sigmastate.Values.EvaluatedValue
   */
 case class AssetIssueRequest(addressOpt: Option[ErgoAddress],
                              valueOpt: Option[Long],
-                             amount: Amount,
+                             amount: Long,
                              name: String,
                              description: String,
                              decimals: Int,
@@ -35,7 +34,7 @@ object AssetIssueRequest {
 
   def apply(address: ErgoAddress,
             valueOpt: Option[Long],
-            amount: Amount,
+            amount: Long,
             name: String,
             description: String,
             decimals: Int,
@@ -71,7 +70,7 @@ class AssetIssueRequestDecoder(settings: ErgoSettings) extends Decoder[AssetIssu
     for {
       address <- cursor.downField("address").as[Option[ErgoAddress]]
       value <- cursor.downField("ergValue").as[Option[Long]]
-      amount <- cursor.downField("amount").as[Amount]
+      amount <- cursor.downField("amount").as[Long]
       name <- cursor.downField("name").as[String]
       description <- cursor.downField("description").as[String]
       decimals <- cursor.downField("decimals").as[Int]
diff --git a/src/main/scala/scorex/core/consensus/ModifierSemanticValidity.scala b/src/main/scala/scorex/core/consensus/ModifierSemanticValidity.scala
index debd650101..a88acfbae1 100644
--- a/src/main/scala/scorex/core/consensus/ModifierSemanticValidity.scala
+++ b/src/main/scala/scorex/core/consensus/ModifierSemanticValidity.scala
@@ -8,11 +8,6 @@ sealed trait ModifierSemanticValidity {
 }
 
 object ModifierSemanticValidity {
-  def restoreFromCode(code: Byte): ModifierSemanticValidity =
-    if (code == Valid.code) Valid
-    else if (code == Unknown.code) Unknown
-    else if (code == Invalid.code) Invalid
-    else Absent
 
   case object Absent extends ModifierSemanticValidity {
     override val code: Byte = 0
diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala
index 21d7232da0..f4eaabb022 100644
--- a/src/main/scala/scorex/core/network/NetworkController.scala
+++ b/src/main/scala/scorex/core/network/NetworkController.scala
@@ -533,32 +533,6 @@ object NetworkController {
 }
 
 object NetworkControllerRef {
-  def props(settings: NetworkSettings,
-            peerManagerRef: ActorRef,
-            scorexContext: ScorexContext,
-            tcpManager: ActorRef)(implicit ec: ExecutionContext): Props = {
-    Props(new NetworkController(settings, peerManagerRef, scorexContext, tcpManager))
-  }
-
-  def apply(settings: NetworkSettings,
-            peerManagerRef: ActorRef,
-            scorexContext: ScorexContext)
-           (implicit system: ActorSystem, ec: ExecutionContext): ActorRef = {
-    system.actorOf(
-      props(settings, peerManagerRef, scorexContext, IO(Tcp))
-    )
-  }
-
-  def apply(name: String,
-            settings: NetworkSettings,
-            peerManagerRef: ActorRef,
-            scorexContext: ScorexContext,
-            tcpManager: ActorRef)
-           (implicit system: ActorSystem, ec: ExecutionContext): ActorRef = {
-    system.actorOf(
-      props(settings, peerManagerRef, scorexContext, tcpManager),
-      name)
-  }
 
   def apply(name: String,
             settings: NetworkSettings,
@@ -566,9 +540,8 @@ object NetworkControllerRef {
             scorexContext: ScorexContext)
            (implicit system: ActorSystem, ec: ExecutionContext): ActorRef = {
 
-    system.actorOf(
-      props(settings, peerManagerRef, scorexContext, IO(Tcp)),
-      name)
+    val props = Props(new NetworkController(settings, peerManagerRef, scorexContext, IO(Tcp)))
+    system.actorOf(props, name)
   }
 
 }
diff --git a/src/main/scala/scorex/core/network/PeerSpec.scala b/src/main/scala/scorex/core/network/PeerSpec.scala
index 9f2d6edd45..92e14f33ce 100644
--- a/src/main/scala/scorex/core/network/PeerSpec.scala
+++ b/src/main/scala/scorex/core/network/PeerSpec.scala
@@ -30,8 +30,6 @@ case class PeerSpec(agentName: String,
     features.collectFirst { case LocalAddressPeerFeature(addr) => addr }
   }
 
-  def reachablePeer: Boolean = address.isDefined
-
   def address: Option[InetSocketAddress] = declaredAddress orElse localAddressOpt
 
 }

From 2f7f34a00d291d2a5de05f34761caab8731f99c0 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 22 Nov 2021 20:40:25 +0300
Subject: [PATCH 08/58] unused code in ApiError removed

---
 src/main/scala/scorex/core/api/http/ApiDirectives.scala | 7 +++++--
 src/main/scala/scorex/core/api/http/ApiError.scala      | 2 --
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/scorex/core/api/http/ApiDirectives.scala b/src/main/scala/scorex/core/api/http/ApiDirectives.scala
index 7f17b5ceab..b76387c2f9 100644
--- a/src/main/scala/scorex/core/api/http/ApiDirectives.scala
+++ b/src/main/scala/scorex/core/api/http/ApiDirectives.scala
@@ -14,8 +14,11 @@ trait ApiDirectives extends CorsHandler with ScorexEncoding {
     case None => reject(AuthorizationFailedRejection)
     case Some(key) =>
       val keyHashStr: String = encoder.encode(Blake2b256(key))
-      if (settings.apiKeyHash.contains(keyHashStr)) pass
-      else reject(AuthorizationFailedRejection)
+      if (settings.apiKeyHash.contains(keyHashStr)){
+        pass
+      } else{
+        reject(AuthorizationFailedRejection)
+      }
   }
 
 }
diff --git a/src/main/scala/scorex/core/api/http/ApiError.scala b/src/main/scala/scorex/core/api/http/ApiError.scala
index c11153a216..39904c0196 100644
--- a/src/main/scala/scorex/core/api/http/ApiError.scala
+++ b/src/main/scala/scorex/core/api/http/ApiError.scala
@@ -27,9 +27,7 @@ case class ApiError(statusCode: StatusCode, reason: String = "") {
 
 object ApiError {
 
-  def apply(s: String): Route = InternalError(s)
   def apply(e: Throwable): Route = InternalError(safeMessage(e))
-  def apply(causes: Seq[Throwable]): Route = InternalError(mkString(causes))
   def mkString(causes: Seq[Throwable]): String = causes.map(safeMessage).mkString(", ")
   private def safeMessage(e: Throwable): String = Option(e.getMessage).getOrElse(e.toString)
 

From bead7584c9e032f1dc780cc4719571a9de8def5d Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 22 Nov 2021 20:52:05 +0300
Subject: [PATCH 09/58] removing unused code in PeerConnectionHandler

---
 .../core/network/PeerConnectionHandler.scala  | 21 +++++++------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala
index 4854423eb6..9ccfc4b79f 100644
--- a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala
+++ b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala
@@ -237,7 +237,7 @@ class PeerConnectionHandler(val settings: NetworkSettings,
     }
   }
 
-  private def createHandshakeMessage() = {
+  private def createHandshakeMessage(): Handshake = {
     Handshake(
       PeerSpec(
         settings.agentName,
@@ -256,10 +256,6 @@ object PeerConnectionHandler {
 
   object ReceivableMessages {
 
-    private[PeerConnectionHandler] object HandshakeDone
-
-    case object StartInteraction
-
     case object HandshakeTimeout
 
     case object CloseConnection
@@ -271,25 +267,22 @@ object PeerConnectionHandler {
 }
 
 object PeerConnectionHandlerRef {
+
   def props(settings: NetworkSettings,
             networkControllerRef: ActorRef,
             scorexContext: ScorexContext,
             connectionDescription: ConnectionDescription
-           )(implicit ec: ExecutionContext): Props =
+           )(implicit ec: ExecutionContext): Props = {
     Props(new PeerConnectionHandler(settings, networkControllerRef, scorexContext, connectionDescription))
-
-  def apply(settings: NetworkSettings,
-            networkControllerRef: ActorRef,
-            scorexContext: ScorexContext,
-            connectionDescription: ConnectionDescription)
-           (implicit system: ActorSystem, ec: ExecutionContext): ActorRef =
-    system.actorOf(props(settings, networkControllerRef, scorexContext, connectionDescription))
+  }
 
   def apply(name: String,
             settings: NetworkSettings,
             networkControllerRef: ActorRef,
             scorexContext: ScorexContext,
             connectionDescription: ConnectionDescription)
-           (implicit system: ActorSystem, ec: ExecutionContext): ActorRef =
+           (implicit system: ActorSystem, ec: ExecutionContext): ActorRef = {
     system.actorOf(props(settings, networkControllerRef, scorexContext, connectionDescription), name)
+  }
+
 }

From 09349631f518326d5f656679c8a6f6322675db17 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 22 Nov 2021 20:58:39 +0300
Subject: [PATCH 10/58] verifyInput & verifyOutput

---
 .../modifiers/mempool/ErgoTransaction.scala   | 109 +++++++++++-------
 1 file changed, 66 insertions(+), 43 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
index 31a0f74675..8c928f5ef9 100644
--- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
+++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
@@ -18,7 +18,7 @@ import scorex.core.serialization.ScorexSerializer
 import scorex.core.transaction.Transaction
 import scorex.core.utils.ScorexEncoding
 import scorex.core.validation.ValidationResult.fromValidationState
-import scorex.core.validation.{ModifierValidator, ValidationState}
+import scorex.core.validation.{ModifierValidator, ValidationResult, ValidationState}
 import scorex.db.{ByteArrayUtils, ByteArrayWrapper}
 import scorex.util.serialization.{Reader, Writer}
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
@@ -101,6 +101,68 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
   }
 
 
+  private def verifyInput(validationBefore: ValidationState[Long],
+                          boxesToSpend: IndexedSeq[ErgoBox],
+                          dataBoxes: IndexedSeq[ErgoBox],
+                          box: ErgoBox,
+                          inputIndex: Short,
+                          stateContext: ErgoStateContext,
+                          currentTxCost: Long)
+                         (implicit verifier: ErgoInterpreter): ValidationResult[Long] = {
+
+    // Cost limit per block
+    val maxCost = stateContext.currentParameters.maxBlockCost
+
+    val input = inputs(inputIndex)
+
+    // Just in case, should always be true if client implementation is correct.
+    if(!box.id.sameElements(input.boxId)) {
+      log.error("Critical client error: box is not inputs(inputIndex)")
+    }
+
+    val proof = input.spendingProof
+    val transactionContext = TransactionContext(boxesToSpend, dataBoxes, this)
+    val inputContext = InputContext(inputIndex, proof.extension)
+
+    val ctx = new ErgoContext(
+      stateContext, transactionContext, inputContext,
+      costLimit = maxCost - currentTxCost, // remaining cost so far
+      initCost = 0)
+
+    val costTry = verifier.verify(box.ergoTree, ctx, proof, messageToSign)
+    val (isCostValid, scriptCost) =
+      costTry match {
+        case Failure(t) =>
+          log.info(s"Tx verification failed: ${t.getMessage}")
+          (false, maxCost + 1)
+        case Success(result) =>
+          result
+      }
+
+    val currCost = addExact(currentTxCost, scriptCost)
+
+    validationBefore
+      // Check whether input box script interpreter raised exception
+      .validate(txScriptValidation, costTry.isSuccess && isCostValid, s"$id: #$inputIndex => $costTry")
+      // Check that cost of the transaction after checking the input becomes too big
+      .validate(bsBlockTransactionsCost, currCost <= maxCost, s"$id: cost exceeds limit after input #$inputIndex")
+      .map(c => addExact(c, scriptCost))
+  }
+
+  private def verifyOutput(validationBefore: ValidationState[Long],
+                           out: ErgoBox,
+                           stateContext: ErgoStateContext): ValidationResult[Long] = {
+
+    val blockVersion = stateContext.blockVersion
+
+    validationBefore
+      .validate(txDust, out.value >= BoxUtils.minimalErgoAmount(out, stateContext.currentParameters), s"$id, output ${Algos.encode(out.id)}, ${out.value} >= ${BoxUtils.minimalErgoAmount(out, stateContext.currentParameters)}")
+      .validate(txFuture, out.creationHeight <= stateContext.currentHeight, s" ${out.creationHeight} <= ${stateContext.currentHeight} is not true, output id: $id: output $out")
+      .validate(txNegHeight, (blockVersion == 1) || out.creationHeight >= 0, s" ${out.creationHeight} >= 0 is not true, output id: $id: output $out")
+      .validate(txBoxSize, out.bytes.length <= MaxBoxSize.value, s"$id: output $out")
+      .validate(txBoxPropositionSize, out.propositionBytes.length <= MaxPropositionBytes.value, s"$id: output $out")
+  }
+
   /**
     * Checks whether transaction is valid against input boxes to spend, and
     * non-spendable data inputs.
@@ -126,8 +188,6 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
     verifier.IR.resetContext() // ensure there is no garbage in the IRContext
     lazy val inputSumTry = Try(boxesToSpend.map(_.value).reduce(Math.addExact(_, _)))
 
-    val blockVersion = stateContext.blockVersion
-
     // Cost of transaction initialization: we should read and parse all inputs and data inputs,
     // and also iterate through all outputs to check rules
     val initialCost: Long = addExact(
@@ -153,14 +213,7 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
         outputCandidates.forall(_.additionalTokens.forall(_._2 > 0)),
         s"$id: ${outputCandidates.map(_.additionalTokens)}")
       // Check that outputs are not dust, and not created in future
-      .validateSeq(outputs) { case (validationState, out) =>
-      validationState
-        .validate(txDust, out.value >= BoxUtils.minimalErgoAmount(out, stateContext.currentParameters), s"$id, output ${Algos.encode(out.id)}, ${out.value} >= ${BoxUtils.minimalErgoAmount(out, stateContext.currentParameters)}")
-        .validate(txFuture, out.creationHeight <= stateContext.currentHeight, s" ${out.creationHeight} <= ${stateContext.currentHeight} is not true, output id: $id: output $out")
-        .validate(txNegHeight, (blockVersion == 1) || out.creationHeight >= 0, s" ${out.creationHeight} >= 0 is not true, output id: $id: output $out")
-        .validate(txBoxSize, out.bytes.length <= MaxBoxSize.value, s"$id: output $out")
-        .validate(txBoxPropositionSize, out.propositionBytes.length <= MaxPropositionBytes.value, s"$id: output $out")
-    }
+      .validateSeq(outputs) { case (validationState, out) => verifyOutput(validationState, out, stateContext) }
       // Just to be sure, check that all the input boxes to spend (and to read) are presented.
       // Normally, this check should always pass, if the client is implemented properly
       // so it is not part of the protocol really.
@@ -206,38 +259,8 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
       // Check inputs, the most expensive check usually, so done last.
       .validateSeq(boxesToSpend.zipWithIndex) { case (validation, (box, idx)) =>
         val currentTxCost = validation.result.payload.get
-
-        val input = inputs(idx)
-        val proof = input.spendingProof
-        val transactionContext = TransactionContext(boxesToSpend, dataBoxes, this)
-        val inputContext = InputContext(idx.toShort, proof.extension)
-
-        val ctx = new ErgoContext(
-          stateContext, transactionContext, inputContext,
-          costLimit = maxCost - currentTxCost, // remaining cost so far
-          initCost = 0)
-
-        val costTry = verifier.verify(box.ergoTree, ctx, proof, messageToSign)
-        val (isCostValid, scriptCost) =
-          costTry match {
-            case Failure(t) =>
-              log.debug(s"Tx verification failed: ${t.getMessage}")
-              (false, maxCost + 1)
-            case Success(result) =>
-              result
-          }
-
-        val currCost = addExact(currentTxCost, scriptCost)
-
-        validation
-          // Just in case, should always be true if client implementation is correct.
-          .validateEquals(txBoxToSpend, box.id, input.boxId)
-          // Check whether input box script interpreter raised exception
-          .validate(txScriptValidation, costTry.isSuccess && isCostValid, s"$id: #$idx => $costTry")
-          // Check that cost of the transaction after checking the input becomes too big
-          .validate(bsBlockTransactionsCost, currCost <= maxCost, s"$id: cost exceeds limit after input #$idx")
-          .map(c => addExact(c, scriptCost))
-    }
+        verifyInput(validation, boxesToSpend, dataBoxes, box, idx.toShort, stateContext, currentTxCost)
+       }
   }
 
   /**

From 759a46d86e62def3e76006118ddab200acad5c19 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 22 Nov 2021 22:53:16 +0300
Subject: [PATCH 11/58] importing verifyAssets

---
 .../modifiers/mempool/ErgoTransaction.scala   | 73 +++++++++++--------
 1 file changed, 42 insertions(+), 31 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
index 8c928f5ef9..e4a7f7e24a 100644
--- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
+++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
@@ -163,6 +163,47 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
       .validate(txBoxPropositionSize, out.propositionBytes.length <= MaxPropositionBytes.value, s"$id: output $out")
   }
 
+  private def verifyAssets(validationBefore: ValidationState[Long],
+                           outAssets: Map[ByteArrayWrapper, Long],
+                           outAssetsNum: Int,
+                           boxesToSpend: IndexedSeq[ErgoBox],
+                           stateContext: ErgoStateContext): ValidationResult[Long] = {
+    // Cost limit per block
+    val maxCost = stateContext.currentParameters.maxBlockCost
+
+    ErgoTransaction.extractAssets(boxesToSpend) match {
+      case Success((inAssets, inAssetsNum)) =>
+        lazy val newAssetId = ByteArrayWrapper(inputs.head.boxId)
+        val tokenAccessCost = stateContext.currentParameters.tokenAccessCost
+        val currentTxCost = validationBefore.result.payload.get
+
+        // Cost of assets preservation rules checks.
+        // We iterate through all assets to create a map (cost: `(outAssetsNum + inAssetsNum) * tokenAccessCost)`)
+        // and after that we iterate through unique asset ids to check preservation rules (cost: `(inAssets.size + outAssets.size) * tokenAccessCost`)
+        val totalAssetsAccessCost = (outAssetsNum + inAssetsNum) * tokenAccessCost +
+          (inAssets.size + outAssets.size) * tokenAccessCost
+        val newCost = addExact(currentTxCost, totalAssetsAccessCost)
+
+        validationBefore
+          // Check that transaction is not too costly considering all the assets
+          .validate(bsBlockTransactionsCost, maxCost >= newCost, s"$id: assets cost")
+          .validateSeq(outAssets) {
+            case (validationState, (outAssetId, outAmount)) =>
+              val inAmount: Long = inAssets.getOrElse(outAssetId, -1L)
+
+              // Check that for each asset output amount is no more than input amount,
+              // with a possible exception for a new asset created by the transaction
+              validationState.validate(txAssetsPreservation,
+                inAmount >= outAmount || (outAssetId == newAssetId && outAmount > 0),
+                s"$id: Amount in = $inAmount, out = $outAmount. Allowed new asset = $newAssetId, out = $outAssetId")
+          }
+          .payload(newCost)
+      case Failure(e) =>
+        // should never be here as far as we've already checked this when we've created the box
+        ModifierValidator.fatal(e.getMessage)
+    }
+  }
+
   /**
     * Checks whether transaction is valid against input boxes to spend, and
     * non-spendable data inputs.
@@ -224,37 +265,7 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
       // Check that transaction is not creating money out of thin air.
       .validate(txErgPreservation, inputSumTry == outputsSumTry, s"$id: $inputSumTry == $outputsSumTry")
       .validateTry(outAssetsTry, e => ModifierValidator.fatal("Incorrect assets", e)) { case (validation, (outAssets, outAssetsNum)) =>
-        ErgoTransaction.extractAssets(boxesToSpend) match {
-          case Success((inAssets, inAssetsNum)) =>
-            lazy val newAssetId = ByteArrayWrapper(inputs.head.boxId)
-            val tokenAccessCost = stateContext.currentParameters.tokenAccessCost
-            val currentTxCost = validation.result.payload.get
-
-            // Cost of assets preservation rules checks.
-            // We iterate through all assets to create a map (cost: `(outAssetsNum + inAssetsNum) * tokenAccessCost)`)
-            // and after that we iterate through unique asset ids to check preservation rules (cost: `(inAssets.size + outAssets.size) * tokenAccessCost`)
-            val totalAssetsAccessCost = (outAssetsNum + inAssetsNum) * tokenAccessCost +
-              (inAssets.size + outAssets.size) * tokenAccessCost
-            val newCost = addExact(currentTxCost, totalAssetsAccessCost)
-
-            validation
-              // Check that transaction is not too costly considering all the assets
-              .validate(bsBlockTransactionsCost, maxCost >= newCost, s"$id: assets cost")
-              .validateSeq(outAssets) {
-                case (validationState, (outAssetId, outAmount)) =>
-                  val inAmount: Long = inAssets.getOrElse(outAssetId, -1L)
-
-                  // Check that for each asset output amount is no more than input amount,
-                  // with a possible exception for a new asset created by the transaction
-                  validationState.validate(txAssetsPreservation,
-                    inAmount >= outAmount || (outAssetId == newAssetId && outAmount > 0),
-                    s"$id: Amount in = $inAmount, out = $outAmount. Allowed new asset = $newAssetId, out = $outAssetId")
-              }
-              .payload(newCost)
-          case Failure(e) =>
-            // should never be here as far as we've already checked this when we've created the box
-            ModifierValidator.fatal(e.getMessage)
-        }
+        verifyAssets(validation, outAssets, outAssetsNum, boxesToSpend, stateContext)
       }
       // Check inputs, the most expensive check usually, so done last.
       .validateSeq(boxesToSpend.zipWithIndex) { case (validation, (box, idx)) =>

From e5d5fa806cf9c2d60abccd12261b6189c9f6e4e2 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Sun, 28 Nov 2021 22:23:58 +0300
Subject: [PATCH 12/58] post-merging fixes in ErgoTransaction, improving p2sh
 tests

---
 .../ergoplatform/modifiers/mempool/ErgoTransaction.scala    | 6 ++----
 src/main/scala/scorex/core/api/http/ApiDirectives.scala     | 4 ++--
 .../org/ergoplatform/http/routes/ScriptApiRouteSpec.scala   | 5 ++---
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
index 0cad06afa8..c291ef5f50 100644
--- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
+++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
@@ -19,8 +19,6 @@ import scorex.core.serialization.ScorexSerializer
 import scorex.core.transaction.Transaction
 import scorex.core.utils.ScorexEncoding
 import scorex.core.validation.ValidationResult.fromValidationState
-import scorex.core.validation.{ModifierValidator, ValidationState}
-import scorex.db.ByteArrayUtils
 import scorex.core.validation.{ModifierValidator, ValidationResult, ValidationState}
 import scorex.db.{ByteArrayUtils, ByteArrayWrapper}
 import scorex.util.serialization.{Reader, Writer}
@@ -166,14 +164,14 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
   }
 
   private def verifyAssets(validationBefore: ValidationState[Long],
-                           outAssets: Map[ByteArrayWrapper, Long],
+                           outAssets: Map[ByteBuffer, Long],
                            outAssetsNum: Int,
                            boxesToSpend: IndexedSeq[ErgoBox],
                            stateContext: ErgoStateContext): ValidationResult[Long] = {
     // Cost limit per block
     val maxCost = stateContext.currentParameters.maxBlockCost
 
-    ErgoTransaction.extractAssets(boxesToSpend) match {
+    ErgoBoxAssetExtractor.extractAssets(boxesToSpend) match {
       case Success((inAssets, inAssetsNum)) =>
         lazy val newAssetId = ByteArrayWrapper(inputs.head.boxId)
         val tokenAccessCost = stateContext.currentParameters.tokenAccessCost
diff --git a/src/main/scala/scorex/core/api/http/ApiDirectives.scala b/src/main/scala/scorex/core/api/http/ApiDirectives.scala
index b76387c2f9..bebba07910 100644
--- a/src/main/scala/scorex/core/api/http/ApiDirectives.scala
+++ b/src/main/scala/scorex/core/api/http/ApiDirectives.scala
@@ -14,9 +14,9 @@ trait ApiDirectives extends CorsHandler with ScorexEncoding {
     case None => reject(AuthorizationFailedRejection)
     case Some(key) =>
       val keyHashStr: String = encoder.encode(Blake2b256(key))
-      if (settings.apiKeyHash.contains(keyHashStr)){
+      if (settings.apiKeyHash.contains(keyHashStr)) {
         pass
-      } else{
+      } else {
         reject(AuthorizationFailedRejection)
       }
   }
diff --git a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala
index d6ba40ad25..d1b86d36d1 100644
--- a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala
@@ -137,9 +137,8 @@ class ScriptApiRouteSpec extends AnyFlatSpec
     val p2pk = "3WvsT2Gm4EpsM9Pg18PdY6XyhNNMqXDsvJTbbf6ihLvAmSb7u5RN"
     Get(s"$prefix/$suffix/$p2pk") ~> route ~> check(assertion(responseAs[Json], p2pk))
 
-    //todo: temporarily switched off due to https://github.com/ergoplatform/ergo/issues/936
-    //    val p2sh = "8UmyuJuQ3FS9ts7j72fn3fKChXSGzbL9WC"
-    //    Get(s"$prefix/$suffix/$p2sh") ~> route ~> check(assertion(responseAs[Json], p2sh))
+    val p2sh = "rbcrmKEYduUvADj9Ts3dSVSG27h54pgrq5fPuwB"
+    Get(s"$prefix/$suffix/$p2sh") ~> route ~> check(assertion(responseAs[Json], p2sh))
 
     val script = TrueLeaf
     val tree = ErgoTree.fromProposition(script)

From 1475ee680b6fa39230995b61daf0541f5fca4e63 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Sun, 28 Nov 2021 23:08:32 +0300
Subject: [PATCH 13/58] comments for verify* helpers

---
 .../modifiers/mempool/ErgoTransaction.scala   | 34 ++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
index c291ef5f50..03d8f99de0 100644
--- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
+++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala
@@ -101,10 +101,24 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
   }
 
 
+  /**
+    * Helper method to decide whether an input can be spent
+    *
+    *
+    * @param validationBefore - transaction validation before checking the input
+    * @param boxesToSpend - input boxes
+    * @param dataBoxes - data input boxes
+    * @param box - the input to check
+    * @param inputIndex - index of the input to check
+    * @param stateContext - context used during validation
+    * @param currentTxCost - accumulated validation cost before checking the input
+    * @param verifier - verifier used to check spending validity
+    * @return
+    */
   private def verifyInput(validationBefore: ValidationState[Long],
                           boxesToSpend: IndexedSeq[ErgoBox],
                           dataBoxes: IndexedSeq[ErgoBox],
-                          box: ErgoBox,
+                          box: ErgoBox, //todo: replace with boxesToSpend(inputIndex) during next refactoring
                           inputIndex: Short,
                           stateContext: ErgoStateContext,
                           currentTxCost: Long)
@@ -149,6 +163,14 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
       .map(c => addExact(c, scriptCost))
   }
 
+  /**
+    * Helper method to verify an output of the transaction
+    *
+    * @param validationBefore - validation state before checking the output
+    * @param out - the output to check
+    * @param stateContext - context used in validation
+    * @return
+    */
   private def verifyOutput(validationBefore: ValidationState[Long],
                            out: ErgoBox,
                            stateContext: ErgoStateContext): ValidationResult[Long] = {
@@ -163,6 +185,16 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
       .validate(txBoxPropositionSize, out.propositionBytes.length <= MaxPropositionBytes.value, s"$id: output $out")
   }
 
+  /**
+    * Check validity of transaction assets (access cost and preservation rules)
+    *
+    * @param validationBefore - validation state before checking the output
+    * @param outAssets - dictionary of assets in outputs
+    * @param outAssetsNum - number of assets in outputs
+    * @param boxesToSpend - input boxes
+    * @param stateContext - context used in validation
+    * @return
+    */
   private def verifyAssets(validationBefore: ValidationState[Long],
                            outAssets: Map[ByteBuffer, Long],
                            outAssetsNum: Int,

From 136e8fceed25aa1eddf4599c9a90152594771fe6 Mon Sep 17 00:00:00 2001
From: Alex Chepurnoy <alex.chepurnoy@iohk.io>
Date: Tue, 4 Jan 2022 20:52:12 +0300
Subject: [PATCH 14/58] avoiding v1SyncInfo recalc

---
 .../org/ergoplatform/network/ErgoNodeViewSynchronizer.scala    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
index 1f1e2d7442..8e19a3860a 100644
--- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
+++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
@@ -157,7 +157,8 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     val (peersV2, peersV1) = peers.partition(p => syncV2Supported(p))
     log.debug(s"Syncing with ${peersV1.size} peers via sync v1, ${peersV2.size} peers via sync v2")
     if (peersV1.nonEmpty) {
-      networkControllerRef ! SendToNetwork(Message(syncInfoSpec, Right(history.syncInfoV1), None), SendToPeers(peersV1))
+      val v1SyncInfo = history.syncInfoV1
+      networkControllerRef ! SendToNetwork(Message(syncInfoSpec, Right(v1SyncInfo), None), SendToPeers(peersV1))
     }
     if (peersV2.nonEmpty) {
       //todo: send only last header to peers which are equal or younger

From 8edb67627e699505ffe5d8fd3233f2ade63ef1c6 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Wed, 23 Nov 2022 19:03:59 +0300
Subject: [PATCH 15/58] p2sRuleR

---
 .../ergoplatform/http/api/ScanApiRoute.scala  | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
index a6110d1369..7dd3f03028 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
@@ -5,7 +5,7 @@ import akka.http.scaladsl.server.Route
 import io.circe.Encoder
 import org.ergoplatform._
 import org.ergoplatform.nodeView.wallet._
-import org.ergoplatform.nodeView.wallet.scanning.ScanRequest
+import org.ergoplatform.nodeView.wallet.scanning.{EqualsScanningPredicate, ScanRequest}
 import org.ergoplatform.settings.ErgoSettings
 import scorex.core.api.http.ApiError.BadRequest
 import scorex.core.api.http.ApiResponse
@@ -13,7 +13,10 @@ import scorex.core.settings.RESTApiSettings
 
 import scala.util.{Failure, Success}
 import ScanEntities._
+import org.ergoplatform.ErgoBox.R1
 import org.ergoplatform.wallet.Constants.ScanId
+import sigmastate.Values.ByteArrayConstant
+import sigmastate.serialization.ValueSerializer
 
 /**
   * This class contains methods to register / deregister and list external scans, and also to serve them.
@@ -40,7 +43,8 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
       unspentR ~
       spentR ~
       stopTrackingR ~
-      addBoxR
+      addBoxR ~
+      p2sRuleR
   }
 
   def registerR: Route = (path("register") & post & entity(as[ScanRequest])) { request =>
@@ -91,4 +95,18 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
       case Success(_) => ApiResponse(scanIdsBox.box.id)
     }
   }
+
+  def p2sRuleR: Route = (path("p2sRule") & post & entity(as[String])) { p2s =>
+    addressEncoder.fromString(p2s) match {
+      case Success(p2sAddr) =>
+        val scriptBytes = ByteArrayConstant(ValueSerializer.serialize(p2sAddr.script.toProposition(replaceConstants = true).propBytes))
+        val trackingRule = EqualsScanningPredicate(R1, scriptBytes)
+        val request = ScanRequest(p2s, trackingRule, None, None)
+        withWalletOp(_.addScan(request).map(_.response)) {
+          case Failure(e) => BadRequest(s"Bad request $request. ${Option(e.getMessage).getOrElse(e.toString)}")
+          case Success(app) => ApiResponse(ScanIdWrapper(app.scanId))
+        }
+      case Failure(e) => BadRequest(s"Can't parse $p2s. ${Option(e.getMessage).getOrElse(e.toString)}")
+    }
+  }
 }

From 7f409fd4e1b57ff65565e692f01d7954fc6f1203 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 16 Jan 2023 14:18:38 +0300
Subject: [PATCH 16/58] openapi.yaml updated with /scan/p2srule

---
 src/main/resources/api/openapi.yaml           | 37 ++++++++++++++++++-
 .../ergoplatform/http/api/ScanApiRoute.scala  |  3 ++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index f193638740..c16fbde333 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.2"
 
 info:
-  version: "5.0.4"
+  version: "5.0.6"
   title: Ergo Node API
   description: API docs for Ergo Node. Models are shared between all Ergo products
   contact:
@@ -5296,6 +5296,41 @@ paths:
               schema:
                 $ref: '#/components/schemas/ApiError'
 
+  /scan/p2sRule:
+    post:
+      security:
+        - ApiKeyAuth: [api_key]
+      summary: Create custom scan to track P2S address
+      operationId: scriptP2SRule
+      tags:
+        - scan
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+                type: string
+                example: '02c9e71790399816b3e40b2207e9ade19a9b7fe0600186cfb8e2b115bfdb34b57f38cd3c9f2890d11720eb3bb993993f00ededf812a590d2993df094a7ca4f0213e4820e1ab831eed5dc5c72665396d3a01d2a12900f1c3ab77700b284ae24fa8e8f7754f86f2282c795db6b0b17df1c29cc0552e59d01f7d777c638a813333277271c2f8b4d99d01ff0e6ee8695697bdd5b568089395620d7198c6093ce8bc59b928611b1b12452c05addaa42f4beff6a0a6fe90000000380d0dbc3f40210090402040005c801040205c8010500040004000e2003faf2cb329f2e90d6d23b58d91bbb6c046aa143261cc21f52fbe2824bfcbf04d807d601e4c6a70408d602b2a5730000d603e4c6a70601d604e4c6a7080ed605e4c6a70505d606e4c6a70705d60795720399c1a7c1720299c17202c1a7eb027201d1ededededededededed93c27202c2a793e4c672020408720193e4c6720205059572039d9c72057eb272047301000573029d9c72057eb2720473030005730494e4c672020601720393e4c672020705720693e4c67202080e720493e4c67202090ec5a79572039072079c720672059272079c72067205917207730595ef720393b1db630872027306d801d608b2db63087202730700ed938c7208017308938c7208027206c8df35000508cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba0405c8a8c105010105dc8b020e0266608cdea8baf0380008cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba04c8df350000c0843d1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304c8df350000'
+      responses:
+        '200':
+          description: Id of custom scan generated and registered
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScanId'
+        '400':
+          description: Bad source
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+        default:
+          description: Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+
   /wallet/generateCommitments:
     post:
       security:
diff --git a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
index 7dd3f03028..e59ac2739f 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
@@ -96,6 +96,9 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
     }
   }
 
+  /**
+    * API method to get tracking rule corresponding to p2s address
+    */
   def p2sRuleR: Route = (path("p2sRule") & post & entity(as[String])) { p2s =>
     addressEncoder.fromString(p2s) match {
       case Success(p2sAddr) =>

From 8faca9696a349b4d07d603f52d83e8a6b0d54ada Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 16 Jan 2023 16:37:15 +0300
Subject: [PATCH 17/58] true address as an example, no interaction with wallet
 and not removing spent offchain for p2s

---
 src/main/resources/api/openapi.yaml                         | 2 +-
 src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index c16fbde333..8c083784fe 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -5310,7 +5310,7 @@ paths:
           application/json:
             schema:
                 type: string
-                example: '02c9e71790399816b3e40b2207e9ade19a9b7fe0600186cfb8e2b115bfdb34b57f38cd3c9f2890d11720eb3bb993993f00ededf812a590d2993df094a7ca4f0213e4820e1ab831eed5dc5c72665396d3a01d2a12900f1c3ab77700b284ae24fa8e8f7754f86f2282c795db6b0b17df1c29cc0552e59d01f7d777c638a813333277271c2f8b4d99d01ff0e6ee8695697bdd5b568089395620d7198c6093ce8bc59b928611b1b12452c05addaa42f4beff6a0a6fe90000000380d0dbc3f40210090402040005c801040205c8010500040004000e2003faf2cb329f2e90d6d23b58d91bbb6c046aa143261cc21f52fbe2824bfcbf04d807d601e4c6a70408d602b2a5730000d603e4c6a70601d604e4c6a7080ed605e4c6a70505d606e4c6a70705d60795720399c1a7c1720299c17202c1a7eb027201d1ededededededededed93c27202c2a793e4c672020408720193e4c6720205059572039d9c72057eb272047301000573029d9c72057eb2720473030005730494e4c672020601720393e4c672020705720693e4c67202080e720493e4c67202090ec5a79572039072079c720672059272079c72067205917207730595ef720393b1db630872027306d801d608b2db63087202730700ed938c7208017308938c7208027206c8df35000508cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba0405c8a8c105010105dc8b020e0266608cdea8baf0380008cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba04c8df350000c0843d1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304c8df350000'
+                example: '4MQyML64GnzMxZgm'
       responses:
         '200':
           description: Id of custom scan generated and registered
diff --git a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
index e59ac2739f..7272d5ef86 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
@@ -5,7 +5,7 @@ import akka.http.scaladsl.server.Route
 import io.circe.Encoder
 import org.ergoplatform._
 import org.ergoplatform.nodeView.wallet._
-import org.ergoplatform.nodeView.wallet.scanning.{EqualsScanningPredicate, ScanRequest}
+import org.ergoplatform.nodeView.wallet.scanning.{EqualsScanningPredicate, ScanRequest, ScanWalletInteraction}
 import org.ergoplatform.settings.ErgoSettings
 import scorex.core.api.http.ApiError.BadRequest
 import scorex.core.api.http.ApiResponse
@@ -104,7 +104,7 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
       case Success(p2sAddr) =>
         val scriptBytes = ByteArrayConstant(ValueSerializer.serialize(p2sAddr.script.toProposition(replaceConstants = true).propBytes))
         val trackingRule = EqualsScanningPredicate(R1, scriptBytes)
-        val request = ScanRequest(p2s, trackingRule, None, None)
+        val request = ScanRequest(p2s, trackingRule, Some(ScanWalletInteraction.Off), Some(true))
         withWalletOp(_.addScan(request).map(_.response)) {
           case Failure(e) => BadRequest(s"Bad request $request. ${Option(e.getMessage).getOrElse(e.toString)}")
           case Success(app) => ApiResponse(ScanIdWrapper(app.scanId))

From 3b4db242ec97fefeb458a4818496f10300fa340a Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 21 Aug 2023 14:39:57 +0300
Subject: [PATCH 18/58] 5.0.15  version set

---
 src/main/resources/api/openapi.yaml | 2 +-
 src/main/resources/application.conf | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 04a5a8d24a..554dd75303 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.2"
 
 info:
-  version: "5.0.14"
+  version: "5.0.15"
   title: Ergo Node API
   description: API docs for Ergo Node. Models are shared between all Ergo products
   contact:
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index c80a917058..5cde542d65 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -434,7 +434,7 @@ scorex {
     nodeName = "ergo-node"
 
     # Network protocol version to be sent in handshakes
-    appVersion = 5.0.14
+    appVersion = 5.0.15
 
     # Network agent name. May contain information about client code
     # stack, starting from core code-base up to the end graphical interface.

From f95da3a1593252e927fa06ffd280ac55287558b8 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Thu, 24 Aug 2023 03:44:57 +0200
Subject: [PATCH 19/58] Added API specification

---
 src/main/resources/api/openapi.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 2b537d4fb7..0db682ba5a 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -6066,6 +6066,13 @@ paths:
           schema:
             type: string
             default: desc
+        - in: query
+          name: includeUnconfirmed
+          required: false
+          description: if true include unconfirmed transactions from mempool
+          schema:
+            type: boolean
+            default: false
       responses:
         '200':
           description: unspent boxes associated with wanted address
@@ -6221,6 +6228,13 @@ paths:
           schema:
             type: string
             default: desc
+        - in: query
+          name: includeUnconfirmed
+          required: false
+          description: if true include unconfirmed transactions from mempool
+          schema:
+            type: boolean
+            default: false
       responses:
         '200':
           description: unspent boxes with wanted ergotree

From ca410acdac83697dcc9ed192d9dd6bd7fe280dea Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Thu, 24 Aug 2023 04:37:29 +0200
Subject: [PATCH 20/58] Updated and refactored endpoints

---
 .../http/api/BlockchainApiRoute.scala         | 36 ++++++++-----------
 .../history/extra/IndexedErgoAddress.scala    | 28 ++++++++++++---
 2 files changed, 38 insertions(+), 26 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
index cdc4d7feca..71f729457d 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
@@ -46,6 +46,8 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
   }
   private val sortDir: Directive[Tuple1[Direction]] = parameters("sortDirection".as(sortMarshaller) ? DESC)
 
+  private val unconfirmed: Directive[Tuple1[Boolean]] = parameters("includeUnconfirmed".as[Boolean] ? false)
+
   /**
     * Total number of boxes/transactions that can be requested at once to avoid too heavy requests ([[BlocksApiRoute.MaxHeaders]])
     */
@@ -90,11 +92,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
   private def getHistoryWithMempool: Future[(ErgoHistoryReader,ErgoMemPoolReader)] =
     (readersHolder ? GetReaders).mapTo[Readers].map(r => (r.h, r.m))
 
-  private def getAddress(tree: ErgoTree)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = {
+  private def getAddress(tree: ErgoTree)(history: ErgoHistoryReader): Option[IndexedErgoAddress] =
     history.typedExtraIndexById[IndexedErgoAddress](hashErgoTree(tree))
-  }
 
-  private def getAddress(addr: ErgoAddress)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = getAddress(addr.script)(history)
+  private def getAddress(addr: ErgoAddress)(history: ErgoHistoryReader): Option[IndexedErgoAddress] =
+    getAddress(addr.script)(history)
 
   private def getTxById(id: ModifierId)(history: ErgoHistoryReader): Option[IndexedErgoTransaction] =
     history.typedExtraIndexById[IndexedErgoTransaction](id) match {
@@ -213,21 +215,13 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     }
   }
 
-  private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int, sortDir: Direction): Future[Seq[IndexedErgoBox]] =
-    getHistory.map { history =>
-      getAddress(addr)(history) match {
-        case Some(addr) => addr.retrieveUtxos(history, offset, limit, sortDir)
-        case None       => Seq.empty[IndexedErgoBox]
-      }
-    }
-
-  private def getBoxesByAddressUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir) { (address, offset, limit, dir) =>
+  private def getBoxesByAddressUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir & unconfirmed) { (address, offset, limit, dir, unconfirmed) =>
     if(limit > MaxItems) {
       BadRequest(s"No more than $MaxItems boxes can be requested")
     }else if(dir == SortDirection.INVALID) {
-      BadRequest("Invalid parameter for sort direction, valid values are \"ASC\" and \"DESC\"")
+      BadRequest("Invalid parameter for sort direction, valid values are 'ASC' and 'DESC'")
     }else {
-      ApiResponse(getBoxesByAddressUnspent(address, offset, limit, dir))
+      ApiResponse(getBoxesByErgoTreeUnspent(address.script, offset, limit, dir, unconfirmed))
     }
   }
 
@@ -265,21 +259,21 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     }
   }
 
-  private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Int, limit: Int, sortDir: Direction): Future[Seq[IndexedErgoBox]] =
-    getHistory.map { history =>
+  private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Int, limit: Int, sortDir: Direction, unconfirmed: Boolean): Future[Seq[IndexedErgoBox]] =
+    getHistoryWithMempool.map { case (history, mempool) =>
       getAddress(tree)(history) match {
-        case Some(iEa) => iEa.retrieveUtxos(history, offset, limit, sortDir)
-        case None      => Seq.empty[IndexedErgoBox]
+        case Some(addr) => addr.retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed)
+        case None       => Seq.empty[IndexedErgoBox]
       }
     }
 
-  private def getBoxesByErgoTreeUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byErgoTree") & ergoTree & paging & sortDir) { (tree, offset, limit, dir) =>
+  private def getBoxesByErgoTreeUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byErgoTree") & ergoTree & paging & sortDir & unconfirmed) { (tree, offset, limit, dir, unconfirmed) =>
     if(limit > MaxItems) {
       BadRequest(s"No more than $MaxItems boxes can be requested")
     }else if (dir == SortDirection.INVALID) {
-      BadRequest("Invalid parameter for sort direction, valid values are \"ASC\" and \"DESC\"")
+      BadRequest("Invalid parameter for sort direction, valid values are 'ASC' and 'DESC'")
     }else {
-      ApiResponse(getBoxesByErgoTreeUnspent(tree, offset, limit, dir))
+      ApiResponse(getBoxesByErgoTreeUnspent(tree, offset, limit, dir, unconfirmed))
     }
   }
 
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
index 0965922603..8abd05bba1 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
@@ -5,7 +5,8 @@ import org.ergoplatform.http.api.SortDirection.{ASC, DESC, Direction}
 import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes}
 import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getFromSegments, getTxs}
-import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId}
+import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, hashErgoTree, txSegmentId}
+import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
 import org.ergoplatform.settings.Algos
 import scorex.core.serialization.ErgoSerializer
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
@@ -141,14 +142,21 @@ case class IndexedErgoAddress(treeHash: ModifierId,
   /**
     * Get a range of the boxes associated with this address that are NOT spent
     * @param history - history to use
+    * @param mempool - mempool to use, if unconfirmed is true
     * @param offset  - items to skip from the start
     * @param limit   - items to retrieve
     * @param sortDir - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]])
+    * @param unconfirmed - whether to include unconfirmed boxes
     * @return array of unspent boxes
     */
-  def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int, sortDir: Direction): Array[IndexedErgoBox] = {
+  def retrieveUtxos(history: ErgoHistoryReader,
+                    mempool: ErgoMemPoolReader,
+                    offset: Int,
+                    limit: Int,
+                    sortDir: Direction,
+                    unconfirmed: Boolean): Seq[IndexedErgoBox] = {
     val data: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox]
-    sortDir match {
+    val confirmedBoxes: Seq[IndexedErgoBox] = sortDir match {
       case DESC =>
         data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
         var segment: Int = boxSegmentCount
@@ -157,7 +165,7 @@ case class IndexedErgoAddress(treeHash: ModifierId,
           history.typedExtraIndexById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes
             .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) ++=: data
         }
-        data.reverse.slice(offset, offset + limit).toArray
+        data.reverse.slice(offset, offset + limit)
       case ASC =>
         var segment: Int = 0
         while (data.length < (limit + offset) && segment < boxSegmentCount) {
@@ -167,8 +175,18 @@ case class IndexedErgoAddress(treeHash: ModifierId,
         }
         if(data.length < (limit + offset))
           data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
-        data.slice(offset, offset + limit).toArray
+        data.slice(offset, offset + limit)
     }
+    if(unconfirmed) {
+      val mempoolBoxes = mempool.getAll.flatMap(_.transaction.outputs)
+        .filter(box => hashErgoTree(box.ergoTree) == treeHash)
+      val unconfirmedBoxes = mempoolBoxes.map(new IndexedErgoBox(0, None, None, _, 0))
+      sortDir match {
+        case DESC => unconfirmedBoxes ++ confirmedBoxes
+        case ASC => confirmedBoxes ++ unconfirmedBoxes
+      }
+    } else
+      confirmedBoxes
   }
 
   /**

From 659fd4dea15943f2b0658cd81b367f04c039d17b Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 4 Sep 2023 19:22:59 +0300
Subject: [PATCH 21/58] Seq.empty in ErgoValidationSettingsUpdate.empty

---
 .../ergoplatform/settings/ErgoValidationSettingsUpdate.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala b/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala
index 9dccb5d9fa..50d41af36c 100644
--- a/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala
+++ b/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala
@@ -18,7 +18,7 @@ case class ErgoValidationSettingsUpdate(rulesToDisable: Seq[Short],
 }
 
 object ErgoValidationSettingsUpdate {
-  val empty: ErgoValidationSettingsUpdate = ErgoValidationSettingsUpdate(Seq(), Seq())
+  val empty: ErgoValidationSettingsUpdate = ErgoValidationSettingsUpdate(Seq.empty, Seq.empty)
 }
 
 object ErgoValidationSettingsUpdateSerializer extends ErgoSerializer[ErgoValidationSettingsUpdate] {

From 9d0a6d8605fb18f0296d5a6704fd9db3f570b0f1 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 4 Sep 2023 19:50:56 +0300
Subject: [PATCH 22/58] simplified upcoming(), validate mempool txs as in next
 block, TransactionValidation elimination

---
 .../http/api/ErgoBaseApiRoute.scala             |  3 ++-
 .../org/ergoplatform/local/CleanupWorker.scala  |  3 ++-
 .../network/ErgoNodeViewSynchronizer.scala      |  2 +-
 .../nodeView/mempool/ErgoMemPool.scala          | 13 ++-----------
 .../nodeView/state/ErgoStateContext.scala       | 15 +++++++++++++++
 .../ergoplatform/nodeView/state/UtxoState.scala |  2 --
 .../nodeView/state/UtxoStateReader.scala        | 16 ++--------------
 .../core/transaction/state/StateFeature.scala   | 17 +----------------
 .../nodeView/mempool/ErgoMemPoolSpec.scala      |  3 ++-
 .../nodeView/state/UtxoStateSpecification.scala | 11 ++++++-----
 10 files changed, 33 insertions(+), 52 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
index 498e73fa91..694ee183d9 100644
--- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
@@ -97,8 +97,9 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
       .map {
         case (utxo: UtxoStateReader, mp: ErgoMemPoolReader) =>
           val maxTxCost = ergoSettings.nodeSettings.maxTransactionCost
+          val validationContext = utxo.stateContext.simplifiedUpcoming()
           utxo.withMempool(mp)
-            .validateWithCost(tx, maxTxCost)
+            .validateWithCost(tx, Some(validationContext), maxTxCost, None)
             .map(cost => new UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None))
         case _ =>
           tx.statelessValidity().map(_ => new UnconfirmedTransaction(tx, None, now, now, bytes, source = None))
diff --git a/src/main/scala/org/ergoplatform/local/CleanupWorker.scala b/src/main/scala/org/ergoplatform/local/CleanupWorker.scala
index 01232ad607..6ab159c1f9 100644
--- a/src/main/scala/org/ergoplatform/local/CleanupWorker.scala
+++ b/src/main/scala/org/ergoplatform/local/CleanupWorker.scala
@@ -86,7 +86,8 @@ class CleanupWorker(nodeViewHolderRef: ActorRef,
                       ): (mutable.ArrayBuilder[UnconfirmedTransaction], mutable.ArrayBuilder[ModifierId]) = {
       txs match {
         case head :: tail if costAcc < CostLimit =>
-          state.validateWithCost(head.transaction, nodeSettings.maxTransactionCost) match {
+          val validationContext = state.stateContext.simplifiedUpcoming()
+          state.validateWithCost(head.transaction, Some(validationContext), nodeSettings.maxTransactionCost, None) match {
             case Success(txCost) =>
               val updTx = head.withCost(txCost)
               validationLoop(tail, validated += updTx, invalidated, txCost + costAcc)
diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
index 70061fc910..c69d554db7 100644
--- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
+++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
@@ -34,7 +34,7 @@ import scorex.core.validation.MalformedModifierError
 import scorex.util.{ModifierId, ScorexLogging}
 import scorex.core.network.DeliveryTracker
 import scorex.core.network.peer.PenaltyType
-import scorex.core.transaction.state.TransactionValidation.TooHighCostError
+import scorex.core.transaction.state.TooHighCostError
 import scorex.core.app.Version
 import scorex.crypto.hash.Digest32
 import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId}
diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala
index 8e5431c369..0cf8a38f1e 100644
--- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala
@@ -6,7 +6,6 @@ import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransacti
 import org.ergoplatform.nodeView.mempool.OrderedTxPool.WeightedTxId
 import org.ergoplatform.nodeView.state.{ErgoState, UtxoState}
 import org.ergoplatform.settings.{ErgoSettings, MonetarySettings, NodeConfigurationSettings}
-import scorex.core.transaction.state.TransactionValidation
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
 import OrderedTxPool.weighted
 import org.ergoplatform.nodeView.mempool.ErgoMemPool.SortingOption
@@ -228,7 +227,8 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool,
               // Allow proceeded transaction to spend outputs of pooled transactions.
               val utxoWithPool = utxo.withUnconfirmedTransactions(getAll)
               if (tx.inputIds.forall(inputBoxId => utxoWithPool.boxById(inputBoxId).isDefined)) {
-                utxoWithPool.validateWithCost(tx, Some(utxo.stateContext), costLimit, None) match {
+                val validationContext = utxo.stateContext.simplifiedUpcoming()
+                utxoWithPool.validateWithCost(tx, Some(validationContext), costLimit, None) match {
                   case Success(cost) =>
                     acceptIfNoDoubleSpend(unconfirmedTx.withCost(cost), validationStartTime)
                   case Failure(ex) =>
@@ -238,15 +238,6 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool,
                 val exc = new Exception("not all utxos in place yet")
                 this -> new ProcessingOutcome.Declined(exc, validationStartTime)
               }
-            case validator: TransactionValidation =>
-              // transaction validation currently works only for UtxoState, so this branch currently
-              // will not be triggered probably
-              validator.validateWithCost(tx, costLimit) match {
-                case Success(cost) =>
-                  acceptIfNoDoubleSpend(unconfirmedTx.withCost(cost), validationStartTime)
-                case Failure(ex) =>
-                  this.invalidate(unconfirmedTx) -> new ProcessingOutcome.Invalidated(ex, validationStartTime)
-              }
             case _ =>
               // Accept transaction in case of "digest" state. Transactions are not downloaded in this mode from other
               // peers though, so such transactions can come from the local wallet only.
diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
index b6495e3815..163908931a 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
@@ -123,6 +123,21 @@ class ErgoStateContext(val lastHeaders: Seq[Header],
                           calculatedValidationSettings, votingData)
   }
 
+  def simplifiedUpcoming(): UpcomingStateContext = {
+    val minerPk = org.ergoplatform.mining.group.generator
+    val version = lastHeaderOpt.map(_.version).getOrElse(Header.InitialVersion)
+    val nBits = lastHeaderOpt.map(_.nBits).getOrElse(ergoSettings.chainSettings.initialNBits)
+    val timestamp = System.currentTimeMillis()
+    val votes = Array.emptyByteArray
+    val proposedUpdate = ErgoValidationSettingsUpdate.empty
+    val upcomingHeader = PreHeader(lastHeaderOpt, version, minerPk, timestamp, nBits, votes)
+    val height = ErgoHistory.heightOf(lastHeaderOpt) + 1
+    val (calculatedParams, updated) = currentParameters.update(height, forkVote = false, votingData.epochVotes, proposedUpdate, votingSettings)
+    val calculatedValidationSettings = validationSettings.updated(updated)
+    UpcomingStateContext(lastHeaders, lastExtensionOpt, upcomingHeader, genesisStateDigest, calculatedParams,
+      calculatedValidationSettings, votingData)
+  }
+
   protected def checkForkVote(height: Height): Unit = {
     if (currentParameters.softForkStartingHeight.nonEmpty) {
       val startingHeight = currentParameters.softForkStartingHeight.get
diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala
index 37003d4241..a4254f07b0 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala
@@ -13,7 +13,6 @@ import org.ergoplatform.settings.{Algos, ErgoSettings, Parameters}
 import org.ergoplatform.utils.LoggingUtil
 import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedModifier
 import scorex.core._
-import scorex.core.transaction.state.TransactionValidation
 import scorex.core.utils.ScorexEncoding
 import scorex.core.validation.ModifierValidator
 import scorex.crypto.authds.avltree.batch._
@@ -38,7 +37,6 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32
                 override val store: LDBVersionedStore,
                 override protected val ergoSettings: ErgoSettings)
   extends ErgoState[UtxoState]
-    with TransactionValidation
     with UtxoStateReader
     with ScorexEncoding {
 
diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
index b856643302..2f2e420ad7 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
@@ -9,8 +9,7 @@ import org.ergoplatform.settings.{Algos, ErgoSettings}
 import org.ergoplatform.settings.Algos.HF
 import org.ergoplatform.wallet.boxes.ErgoBoxSerializer
 import org.ergoplatform.wallet.interpreter.ErgoInterpreter
-import scorex.core.transaction.state.TransactionValidation
-import scorex.core.transaction.state.TransactionValidation.TooHighCostError
+import scorex.core.transaction.state.TooHighCostError
 import scorex.core.validation.MalformedModifierError
 import scorex.crypto.authds.avltree.batch.{Lookup, PersistentBatchAVLProver, VersionedLDBAVLStorage}
 import scorex.crypto.authds.{ADDigest, ADKey, SerializedAdProof}
@@ -23,7 +22,7 @@ import scala.util.{Failure, Success, Try}
   * state representation (so functions to generate UTXO set modifiction proofs, do stateful transaction validation,
   * get UTXOs are there
   */
-trait UtxoStateReader extends ErgoStateReader with UtxoSetSnapshotPersistence with TransactionValidation {
+trait UtxoStateReader extends ErgoStateReader with UtxoSetSnapshotPersistence {
 
   protected implicit val hf: HF = Algos.hash
 
@@ -71,17 +70,6 @@ trait UtxoStateReader extends ErgoStateReader with UtxoSetSnapshotPersistence wi
     }
   }
 
-  /**
-    * Validate transaction as if it was included at the end of the last block.
-    * This validation does not guarantee that transaction will be valid in future
-    * as soon as state (both UTXO set and state context) will change.
-    *
-    * Used in mempool.
-    */
-  override def validateWithCost(tx: ErgoTransaction, maxTxCost: Int): Try[Int] = {
-    validateWithCost(tx, None, maxTxCost, None)
-  }
-
   /**
     *
     * @param fb - ergo full block
diff --git a/src/main/scala/scorex/core/transaction/state/StateFeature.scala b/src/main/scala/scorex/core/transaction/state/StateFeature.scala
index ebccd8a23f..681aaaf167 100644
--- a/src/main/scala/scorex/core/transaction/state/StateFeature.scala
+++ b/src/main/scala/scorex/core/transaction/state/StateFeature.scala
@@ -1,21 +1,6 @@
 package scorex.core.transaction.state
 
-import org.ergoplatform.modifiers.mempool.ErgoTransaction
-import scala.util.Try
 
-/**
-  * Basic trait for features supported by state representation
-  */
-trait StateFeature
+case class TooHighCostError(message: String) extends Exception(message)
 
-/**
-  * Instance of this trait supports stateful validation of any transaction
-  */
-trait TransactionValidation extends StateFeature {
-  def validateWithCost(tx: ErgoTransaction, maxTxCost: Int): Try[Int]
-}
-
-object TransactionValidation {
-  case class TooHighCostError(message: String) extends Exception(message)
-}
 
diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala
index d08408e7cf..4ab6f0d98f 100644
--- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala
@@ -72,7 +72,8 @@ class ErgoMemPoolSpec extends AnyFlatSpec
 
     var poolCost = ErgoMemPool.empty(sortByCostSettings)
     poolCost = poolCost.process(UnconfirmedTransaction(tx, None), wus)._1
-    val cost = wus.validateWithCost(tx, Int.MaxValue).get
+    val validationContext = wus.stateContext.simplifiedUpcoming()
+    val cost = wus.validateWithCost(tx, Some(validationContext), Int.MaxValue, None).get
     poolCost.pool.orderedTransactions.firstKey.weight shouldBe OrderedTxPool.weighted(tx, cost).weight
   }
 
diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
index a0cd911b10..d5f3ad9226 100644
--- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
@@ -15,7 +15,7 @@ import org.ergoplatform.settings.Constants
 import org.ergoplatform.utils.{ErgoPropertyTest, RandomWrapper}
 import org.ergoplatform.utils.generators.ErgoTransactionGenerators
 import scorex.core._
-import scorex.core.transaction.state.TransactionValidation.TooHighCostError
+import scorex.core.transaction.state.TooHighCostError
 import scorex.crypto.authds.ADKey
 import scorex.db.ByteArrayWrapper
 import scorex.util.{ModifierId, bytesToId}
@@ -94,17 +94,18 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera
       )
       val unsignedTx = new UnsignedErgoTransaction(inputs, IndexedSeq(), newBoxes)
       val tx = ErgoTransaction(defaultProver.sign(unsignedTx, IndexedSeq(foundersBox), emptyDataBoxes, us.stateContext).get)
-      val validationRes1 = us.validateWithCost(tx, 100000)
+      val validationContext = us.stateContext.simplifiedUpcoming()
+      val validationRes1 = us.validateWithCost(tx, Some(validationContext), 100000, None)
       validationRes1 shouldBe 'success
       val txCost = validationRes1.get
 
-      val validationRes2 = us.validateWithCost(tx, txCost - 1)
+      val validationRes2 = us.validateWithCost(tx, Some(validationContext), txCost - 1, None)
       validationRes2 shouldBe 'failure
       validationRes2.toEither.left.get.isInstanceOf[TooHighCostError] shouldBe true
 
-      us.validateWithCost(tx, txCost + 1) shouldBe 'success
+      us.validateWithCost(tx, Some(validationContext), txCost + 1, None) shouldBe 'success
 
-      us.validateWithCost(tx, txCost) shouldBe 'success
+      us.validateWithCost(tx, Some(validationContext), txCost, None) shouldBe 'success
 
       height = height + 1
     }

From 84ca2e9499c32060553cd2a39a7e46003ebc0640 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Wed, 6 Sep 2023 15:46:00 +0300
Subject: [PATCH 23/58] p2sRule test

---
 .../org/ergoplatform/http/api/ScanApiRoute.scala    |  5 ++++-
 .../ergoplatform/http/routes/ScanApiRouteSpec.scala | 13 ++++++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
index 7272d5ef86..157f0726ad 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
@@ -99,7 +99,8 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
   /**
     * API method to get tracking rule corresponding to p2s address
     */
-  def p2sRuleR: Route = (path("p2sRule") & post & entity(as[String])) { p2s =>
+  def p2sRuleR: Route = (path("p2sRule") & post & entity(as[String])) { p2sRaw =>
+    val p2s = fromJsonOrPlain(p2sRaw)
     addressEncoder.fromString(p2s) match {
       case Success(p2sAddr) =>
         val scriptBytes = ByteArrayConstant(ValueSerializer.serialize(p2sAddr.script.toProposition(replaceConstants = true).propBytes))
@@ -113,3 +114,5 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
     }
   }
 }
+
+
diff --git a/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala
index afee4a72b3..fb4956190f 100644
--- a/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala
@@ -252,7 +252,6 @@ class ScanApiRouteSpec extends AnyFlatSpec
     }
   }
 
-
   it should "stop tracking a box" in {
     val scanIdBoxId = ScanIdBoxId(ScanId @@ (51: Short), ADKey @@ Random.randomBytes(32))
 
@@ -261,4 +260,16 @@ class ScanApiRouteSpec extends AnyFlatSpec
     }
   }
 
+  it should "generate scan for p2s rule" in {
+    Post(prefix + "/p2sRule", "Ms7smJmdbakqfwNo") ~> route ~> check {
+      status shouldBe StatusCodes.OK
+      val res = responseAs[Json]
+      res.hcursor.downField("scanId").as[Int].toOption.isDefined shouldBe true
+    }
+
+    Post(prefix + "/p2sRule", "s7smJmdbakqfwNo") ~> route ~> check {
+      status shouldBe StatusCodes.BadRequest
+    }
+  }
+
 }

From 5e3d80a710173a0b8bf84a1f58f40fe840f0ad61 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Fri, 8 Sep 2023 20:21:55 +0300
Subject: [PATCH 24/58] fixing scan

---
 src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
index 157f0726ad..446f13284b 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
@@ -16,7 +16,7 @@ import ScanEntities._
 import org.ergoplatform.ErgoBox.R1
 import org.ergoplatform.wallet.Constants.ScanId
 import sigmastate.Values.ByteArrayConstant
-import sigmastate.serialization.ValueSerializer
+import sigmastate.serialization.{ErgoTreeSerializer, ValueSerializer}
 
 /**
   * This class contains methods to register / deregister and list external scans, and also to serve them.
@@ -103,7 +103,8 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings)
     val p2s = fromJsonOrPlain(p2sRaw)
     addressEncoder.fromString(p2s) match {
       case Success(p2sAddr) =>
-        val scriptBytes = ByteArrayConstant(ValueSerializer.serialize(p2sAddr.script.toProposition(replaceConstants = true).propBytes))
+        val script = p2sAddr.script
+        val scriptBytes = ByteArrayConstant(ErgoTreeSerializer.DefaultSerializer.serializeErgoTree(script))
         val trackingRule = EqualsScanningPredicate(R1, scriptBytes)
         val request = ScanRequest(p2s, trackingRule, Some(ScanWalletInteraction.Off), Some(true))
         withWalletOp(_.addScan(request).map(_.response)) {

From 66c0d99252eb1eddbf5edd606aeb60e134248d21 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Fri, 8 Sep 2023 20:40:48 +0300
Subject: [PATCH 25/58] api method description improved

---
 src/main/resources/api/openapi.yaml                         | 2 +-
 src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 92965223aa..af4b22671b 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -5554,7 +5554,7 @@ paths:
     post:
       security:
         - ApiKeyAuth: [api_key]
-      summary: Create custom scan to track P2S address
+      summary: Create and register a scan to track P2S address provided
       operationId: scriptP2SRule
       tags:
         - scan
diff --git a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
index 446f13284b..f99af46995 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala
@@ -16,7 +16,7 @@ import ScanEntities._
 import org.ergoplatform.ErgoBox.R1
 import org.ergoplatform.wallet.Constants.ScanId
 import sigmastate.Values.ByteArrayConstant
-import sigmastate.serialization.{ErgoTreeSerializer, ValueSerializer}
+import sigmastate.serialization.ErgoTreeSerializer
 
 /**
   * This class contains methods to register / deregister and list external scans, and also to serve them.

From c713aefd34e90c38073a91f5d5ee05598b5d694b Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Tue, 12 Sep 2023 12:32:11 +0300
Subject: [PATCH 26/58] TooHighCostError rework

---
 .../ergoplatform/network/ErgoNodeViewSynchronizer.scala  | 4 ++--
 .../ergoplatform/nodeView/state/UtxoStateReader.scala    | 6 +++---
 .../scala/scorex/core/transaction/TooHighCostError.scala | 9 +++++++++
 .../scorex/core/transaction/state/StateFeature.scala     | 6 ------
 .../nodeView/state/UtxoStateSpecification.scala          | 2 +-
 5 files changed, 15 insertions(+), 12 deletions(-)
 create mode 100644 src/main/scala/scorex/core/transaction/TooHighCostError.scala
 delete mode 100644 src/main/scala/scorex/core/transaction/state/StateFeature.scala

diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
index c69d554db7..f304e4224e 100644
--- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
+++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
@@ -34,12 +34,12 @@ import scorex.core.validation.MalformedModifierError
 import scorex.util.{ModifierId, ScorexLogging}
 import scorex.core.network.DeliveryTracker
 import scorex.core.network.peer.PenaltyType
-import scorex.core.transaction.state.TooHighCostError
 import scorex.core.app.Version
 import scorex.crypto.hash.Digest32
 import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId}
 import org.ergoplatform.ErgoLikeContext.Height
 import scorex.core.serialization.{ErgoSerializer, ManifestSerializer, SubtreeSerializer}
+import scorex.core.transaction.TooHighCostError
 import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage.splitDigest
 
 import scala.annotation.tailrec
@@ -1412,7 +1412,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
       utx.source.foreach { peer =>
         // no need to call deliveryTracker.setInvalid, as mempool will consider invalidated tx in contains()
         error match {
-          case TooHighCostError(_) =>
+          case TooHighCostError(_, _) =>
             log.info(s"Penalize spamming peer $peer for too costly transaction $id")
             penalizeSpammingPeer(peer)
           case _ =>
diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
index 2f2e420ad7..a1679587c0 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
@@ -9,7 +9,7 @@ import org.ergoplatform.settings.{Algos, ErgoSettings}
 import org.ergoplatform.settings.Algos.HF
 import org.ergoplatform.wallet.boxes.ErgoBoxSerializer
 import org.ergoplatform.wallet.interpreter.ErgoInterpreter
-import scorex.core.transaction.state.TooHighCostError
+import scorex.core.transaction.TooHighCostError
 import scorex.core.validation.MalformedModifierError
 import scorex.crypto.authds.avltree.batch.{Lookup, PersistentBatchAVLProver, VersionedLDBAVLStorage}
 import scorex.crypto.authds.{ADDigest, ADKey, SerializedAdProof}
@@ -60,11 +60,11 @@ trait UtxoStateReader extends ErgoStateReader with UtxoSetSnapshotPersistence {
         context,
         accumulatedCost = 0L)(verifier) match {
         case Success(txCost) if txCost > costLimit =>
-          Failure(TooHighCostError(s"Transaction $tx has too high cost $txCost"))
+          Failure(TooHighCostError(tx, Some(txCost)))
         case Success(txCost) =>
           Success(txCost)
         case Failure(mme: MalformedModifierError) if mme.message.contains("CostLimitException") =>
-          Failure(TooHighCostError(s"Transaction $tx has too high cost"))
+          Failure(TooHighCostError(tx, None))
         case f: Failure[_] => f
       }
     }
diff --git a/src/main/scala/scorex/core/transaction/TooHighCostError.scala b/src/main/scala/scorex/core/transaction/TooHighCostError.scala
new file mode 100644
index 0000000000..e86d3d6189
--- /dev/null
+++ b/src/main/scala/scorex/core/transaction/TooHighCostError.scala
@@ -0,0 +1,9 @@
+package scorex.core.transaction
+
+import org.ergoplatform.modifiers.mempool.ErgoTransaction
+
+/**
+  * Exception which is indicating that transaction had too high cost during validation
+  */
+case class TooHighCostError(tx: ErgoTransaction, txCost: Option[Int])
+  extends Exception(s"Transaction $tx has too high cost ${txCost.map(_.toString).getOrElse("")}")
diff --git a/src/main/scala/scorex/core/transaction/state/StateFeature.scala b/src/main/scala/scorex/core/transaction/state/StateFeature.scala
deleted file mode 100644
index 681aaaf167..0000000000
--- a/src/main/scala/scorex/core/transaction/state/StateFeature.scala
+++ /dev/null
@@ -1,6 +0,0 @@
-package scorex.core.transaction.state
-
-
-case class TooHighCostError(message: String) extends Exception(message)
-
-
diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
index d5f3ad9226..5b4aa9cba5 100644
--- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
@@ -15,7 +15,7 @@ import org.ergoplatform.settings.Constants
 import org.ergoplatform.utils.{ErgoPropertyTest, RandomWrapper}
 import org.ergoplatform.utils.generators.ErgoTransactionGenerators
 import scorex.core._
-import scorex.core.transaction.state.TooHighCostError
+import scorex.core.transaction.TooHighCostError
 import scorex.crypto.authds.ADKey
 import scorex.db.ByteArrayWrapper
 import scorex.util.{ModifierId, bytesToId}

From 2e1a8f02498ad7bbae4b013bae394d3f7d6afe06 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Tue, 12 Sep 2023 13:01:29 +0300
Subject: [PATCH 27/58] timestamp fix in simplifiedUpcoming(), ScalaDoc

---
 .../nodeView/state/ErgoStateContext.scala             | 11 +++++++++--
 .../settings/ErgoValidationSettingsUpdate.scala       |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
index 163908931a..b842f5cacc 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
@@ -108,6 +108,9 @@ class ErgoStateContext(val lastHeaders: Seq[Header],
 
   override def serializer: ErgoSerializer[M] = ErgoStateContextSerializer(ergoSettings)
 
+  /**
+    * @return state context corresponding to a block after last known one with fields provided
+    */
   def upcoming(minerPk: EcPointType,
                timestamp: Long,
                nBits: Long,
@@ -123,11 +126,15 @@ class ErgoStateContext(val lastHeaders: Seq[Header],
                           calculatedValidationSettings, votingData)
   }
 
-  def simplifiedUpcoming(): UpcomingStateContext = {
+  /**
+    * @return state context corresponding to a block after last known one
+    *         with fields filled with (kind of) default values
+    */
+  def c(): UpcomingStateContext = {
     val minerPk = org.ergoplatform.mining.group.generator
     val version = lastHeaderOpt.map(_.version).getOrElse(Header.InitialVersion)
     val nBits = lastHeaderOpt.map(_.nBits).getOrElse(ergoSettings.chainSettings.initialNBits)
-    val timestamp = System.currentTimeMillis()
+    val timestamp = lastHeaderOpt.map(_.timestamp + 1).getOrElse(System.currentTimeMillis())
     val votes = Array.emptyByteArray
     val proposedUpdate = ErgoValidationSettingsUpdate.empty
     val upcomingHeader = PreHeader(lastHeaderOpt, version, minerPk, timestamp, nBits, votes)
diff --git a/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala b/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala
index 50d41af36c..9dccb5d9fa 100644
--- a/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala
+++ b/src/main/scala/org/ergoplatform/settings/ErgoValidationSettingsUpdate.scala
@@ -18,7 +18,7 @@ case class ErgoValidationSettingsUpdate(rulesToDisable: Seq[Short],
 }
 
 object ErgoValidationSettingsUpdate {
-  val empty: ErgoValidationSettingsUpdate = ErgoValidationSettingsUpdate(Seq.empty, Seq.empty)
+  val empty: ErgoValidationSettingsUpdate = ErgoValidationSettingsUpdate(Seq(), Seq())
 }
 
 object ErgoValidationSettingsUpdateSerializer extends ErgoSerializer[ErgoValidationSettingsUpdate] {

From f11cc3c11b919b50c9e28e7da84b604e0d6b8d12 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Tue, 12 Sep 2023 13:08:34 +0300
Subject: [PATCH 28/58] simplifiedUpcoming name back

---
 .../org/ergoplatform/nodeView/state/ErgoStateContext.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
index b842f5cacc..d300939fb4 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
@@ -130,7 +130,7 @@ class ErgoStateContext(val lastHeaders: Seq[Header],
     * @return state context corresponding to a block after last known one
     *         with fields filled with (kind of) default values
     */
-  def c(): UpcomingStateContext = {
+  def simplifiedUpcoming(): UpcomingStateContext = {
     val minerPk = org.ergoplatform.mining.group.generator
     val version = lastHeaderOpt.map(_.version).getOrElse(Header.InitialVersion)
     val nBits = lastHeaderOpt.map(_.nBits).getOrElse(ergoSettings.chainSettings.initialNBits)

From 1f23bd62f04a1479746367b60646caa8d0db4f62 Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Wed, 13 Sep 2023 01:12:45 +0300
Subject: [PATCH 29/58] #1959 Remove lastActivity from ConnectedPeer

---
 .../http/api/ErgoPeersApiRoute.scala          |  2 +-
 .../scorex/core/network/ConnectedPeer.scala   |  3 +-
 .../core/network/NetworkController.scala      | 33 ++++++++++---------
 .../core/network/PeerConnectionHandler.scala  |  2 +-
 ...rgoNodeViewSynchronizerSpecification.scala |  2 --
 .../ErgoSyncTrackerSpecification.scala        |  4 +--
 .../PeerFilteringRuleSpecification.scala      |  4 +--
 .../sanity/ErgoSanityDigest.scala             |  3 +-
 .../ergoplatform/sanity/ErgoSanityUTXO.scala  |  3 +-
 .../testkit/generators/ObjectGenerators.scala |  4 +--
 10 files changed, 28 insertions(+), 32 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoPeersApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoPeersApiRoute.scala
index 988b02135a..6efe66f3cf 100644
--- a/src/main/scala/org/ergoplatform/http/api/ErgoPeersApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ErgoPeersApiRoute.scala
@@ -56,7 +56,7 @@ class ErgoPeersApiRoute(peerManager: ActorRef,
         con.peerInfo.map { peerInfo =>
           PeerInfoResponse(
             address = peerInfo.peerSpec.declaredAddress.map(_.toString).getOrElse(""),
-            lastMessage = con.lastMessage,
+            lastMessage = peerInfo.lastStoredActivityTime,
             lastHandshake = peerInfo.lastHandshake,
             name = peerInfo.peerSpec.nodeName,
             connectionType = peerInfo.connectionType.map(_.toString),
diff --git a/src/main/scala/scorex/core/network/ConnectedPeer.scala b/src/main/scala/scorex/core/network/ConnectedPeer.scala
index 55ac009a17..711e6b89b6 100644
--- a/src/main/scala/scorex/core/network/ConnectedPeer.scala
+++ b/src/main/scala/scorex/core/network/ConnectedPeer.scala
@@ -15,7 +15,6 @@ import scorex.core.network.peer.PeerInfo
   */
 case class ConnectedPeer(connectionId: ConnectionId,
                          handlerRef: ActorRef,
-                         var lastMessage: Long,
                          peerInfo: Option[PeerInfo]) {
 
   override def hashCode(): Int = connectionId.remoteAddress.hashCode()
@@ -45,7 +44,7 @@ object ConnectedPeer {
     val optionalFields =
       List(
         peer.peerInfo.map(_.peerSpec.protocolVersion.toString).map("version" -> _.asJson),
-        Option(peer.lastMessage).filter(_ != 0L).map("lastMessage" -> _.asJson)
+        peer.peerInfo.map(_.lastStoredActivityTime).filter(_ != 0L).map("lastMessage" -> _.asJson),
       ).flatten
     val fields = addressField :: optionalFields
     Json.obj(fields:_*)
diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala
index 2ac5376e1c..bee88fb2a1 100644
--- a/src/main/scala/scorex/core/network/NetworkController.scala
+++ b/src/main/scala/scorex/core/network/NetworkController.scala
@@ -116,25 +116,24 @@ class NetworkController(ergoSettings: ErgoSettings,
   private def time(): Time = System.currentTimeMillis()
 
   private def businessLogic: Receive = {
-    //a message coming in from another peer
+    // a message coming in from another peer
     case msg@Message(spec, _, Some(remote)) =>
       messageHandlers.get(spec.messageCode) match {
         case Some(handler) => handler ! msg // forward the message to the appropriate handler for processing
         case None => log.error(s"No handlers found for message $remote: " + spec.messageCode)
       }
 
-      // Update last seen message timestamps, global and peer's, with the message timestamp
+      // Update last seen message timestamps with the message timestamp
       val remoteAddress = remote.connectionId.remoteAddress
       connections.get(remoteAddress) match {
         case Some(cp) =>
           val now = time()
           lastIncomingMessageTime = now
-          cp.lastMessage = now
-          // Update peer's last activity time every X minutes inside PeerInfo
-          cp.peerInfo.foreach { x =>
-            if ((now - x.lastStoredActivityTime) > activityDelta) {
-              val peerInfo = x.copy(lastStoredActivityTime = now)
-              peerManagerRef ! AddOrUpdatePeer(peerInfo)
+          // Update peer's last activity time every ${activityDelta} minutes inside PeerInfo
+          cp.peerInfo.foreach { peerInfo =>
+            if ((now - peerInfo.lastStoredActivityTime) > activityDelta) {
+              val peerInfoUpdated = peerInfo.copy(lastStoredActivityTime = now)
+              peerManagerRef ! AddOrUpdatePeer(peerInfoUpdated)
             }
           }
 
@@ -297,13 +296,15 @@ class NetworkController(ergoSettings: ErgoSettings,
       () => {
         // Drop connections with peers if they seem to be inactive
         val now = time()
-        connections.values.foreach { cp =>
-          val lastSeen = cp.lastMessage
-          val timeout = networkSettings.inactiveConnectionDeadline.toMillis
-          val delta = now - lastSeen
-          if (delta > timeout) {
-            log.info(s"Dropping connection with ${cp.peerInfo}, last seen ${delta / 1000.0} seconds ago")
-            cp.handlerRef ! CloseConnection
+        connections.values.foreach { cp => 
+          cp.peerInfo.foreach { peerInfo =>
+            val lastSeen = peerInfo.lastStoredActivityTime
+            val timeout = networkSettings.inactiveConnectionDeadline.toMillis
+            val delta = now - lastSeen
+            if (delta > timeout) {
+              log.info(s"Dropping connection with ${peerInfo}, last seen ${delta / 1000.0} seconds ago")
+              cp.handlerRef ! CloseConnection
+            }
           }
         }
       }
@@ -400,7 +401,7 @@ class NetworkController(ergoSettings: ErgoSettings,
 
     val handler = context.actorOf(handlerProps) // launch connection handler
     context.watch(handler)
-    val connectedPeer = ConnectedPeer(connectionId, handler, time(), None)
+    val connectedPeer = ConnectedPeer(connectionId, handler, None)
     connections += connectionId.remoteAddress -> connectedPeer
     unconfirmedConnections -= connectionId.remoteAddress
   }
diff --git a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala
index 80cbbce51b..84bb3ac2f4 100644
--- a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala
+++ b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala
@@ -72,7 +72,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings,
       log.info(s"Got a Handshake from $connectionId")
 
       val peerInfo = PeerInfo(receivedHandshake.peerSpec, System.currentTimeMillis(), Some(direction))
-      val peer = ConnectedPeer(connectionDescription.connectionId, self, 0, Some(peerInfo))
+      val peer = ConnectedPeer(connectionDescription.connectionId, self, Some(peerInfo))
       selfPeer = Some(peer)
 
       networkControllerRef ! Handshaked(peerInfo)
diff --git a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala
index 6c5055ed5e..2430aa7728 100644
--- a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala
+++ b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala
@@ -157,7 +157,6 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc
     val p: ConnectedPeer = ConnectedPeer(
       connectionIdGen.sample.get,
       pchProbe.ref,
-      lastMessage = 0,
       Some(peerInfo)
     )
     synchronizerMockRef ! ChangedHistory(history)
@@ -197,7 +196,6 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc
     val peer: ConnectedPeer = ConnectedPeer(
       connectionIdGen.sample.get,
       pchProbe.ref,
-      lastMessage = 0,
       Some(peerInfo)
     )
   }
diff --git a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala
index 0f9580ce63..b96081117e 100644
--- a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala
+++ b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala
@@ -8,9 +8,9 @@ import scorex.core.network.peer.PeerInfo
 class ErgoSyncTrackerSpecification extends ErgoPropertyTest {
   property("getters test") {
     val time = 10L
-    val peerInfo = PeerInfo(defaultPeerSpec, time, Some(Incoming))
+    val peerInfo = PeerInfo(defaultPeerSpec, time, Some(Incoming), 5L)
     val cid = ConnectionId(inetAddr1, inetAddr2, Incoming)
-    val connectedPeer = ConnectedPeer(cid, handlerRef = null, lastMessage = 5L, Some(peerInfo))
+    val connectedPeer = ConnectedPeer(cid, handlerRef = null, Some(peerInfo))
     val syncTracker = ErgoSyncTracker(settings.scorexSettings.network)
 
     val height = 1000
diff --git a/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala b/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala
index 028d9af7df..cd7ee1e71e 100644
--- a/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala
+++ b/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala
@@ -12,8 +12,8 @@ class PeerFilteringRuleSpecification extends ErgoPropertyTest {
   private def peerWithVersion(version: Version): ConnectedPeer = {
     val ref = ActorRef.noSender
     val peerSpec = PeerSpec("", version, "", None, Seq.empty)
-    val peerInfo = PeerInfo(peerSpec, lastHandshake = 0L, None)
-    ConnectedPeer(ConnectionId(null, null, null), ref, lastMessage = 0L, Some(peerInfo))
+    val peerInfo = PeerInfo(peerSpec, lastHandshake = 0L, None, 0L)
+    ConnectedPeer(ConnectionId(null, null, null), ref, Some(peerInfo))
   }
 
   property("syncv2 filter") {
diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala
index 674f851227..d9b07a50e7 100644
--- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala
+++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala
@@ -84,13 +84,12 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] {
     val tx = validErgoTransactionGenTemplate(minAssets = 0, maxAssets = 0).sample.get._2
 
 
-    val peerInfo = PeerInfo(defaultPeerSpec, Long.MaxValue)
+    val peerInfo = PeerInfo(defaultPeerSpec, Long.MaxValue, None, 0L)
 
     @SuppressWarnings(Array("org.wartremover.warts.OptionPartial"))
     val p: ConnectedPeer = ConnectedPeer(
       connectionIdGen.sample.get,
       pchProbe.ref,
-      lastMessage = 0,
       Some(peerInfo)
     )
     ref ! ChangedHistory(h)
diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala
index 52cb2fbaab..ef6107e35b 100644
--- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala
+++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala
@@ -78,12 +78,11 @@ class ErgoSanityUTXO extends ErgoSanity[UTXO_ST] with ErgoTestHelpers {
     val tx = validErgoTransactionGenTemplate(minAssets = 0, maxAssets = 0).sample.get._2
 
 
-    val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis())
+    val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis(), None, 0L)
     @SuppressWarnings(Array("org.wartremover.warts.OptionPartial"))
     val p: ConnectedPeer = ConnectedPeer(
       connectionIdGen.sample.get,
       pchProbe.ref,
-      lastMessage = 0,
       Some(peerInfo)
     )
     ref ! ChangedHistory(h)
diff --git a/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala b/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala
index 7c9dddf961..f99112fab7 100644
--- a/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala
+++ b/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala
@@ -100,12 +100,12 @@ trait ObjectGenerators {
 
   def peerInfoGen: Gen[PeerInfo] = for {
     peerSpec <- peerSpecGen
-  } yield PeerInfo(peerSpec, 0L, Some(Incoming))
+  } yield PeerInfo(peerSpec, 0L, Some(Incoming), 0L)
 
   def connectedPeerGen(peerRef: ActorRef): Gen[ConnectedPeer] = for {
     connectionId <- connectionIdGen
     peerInfo <- peerInfoGen
-  } yield ConnectedPeer(connectionId, peerRef, 0, Some(peerInfo))
+  } yield ConnectedPeer(connectionId, peerRef, Some(peerInfo))
 
   def peerSpecGen: Gen[PeerSpec] = for {
     declaredAddress <- Gen.frequency(5 -> const(None), 5 -> some(inetSocketAddressGen))

From 285892d712c15c7466151cd884055576c340617c Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Tue, 19 Sep 2023 08:22:24 +0200
Subject: [PATCH 30/58] Fixed IndexedErgoBox API schema

---
 src/main/resources/api/openapi.yaml | 90 ++++++++++++-----------------
 1 file changed, 36 insertions(+), 54 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 563abdb14e..0a05fdbea0 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -243,60 +243,42 @@ components:
                 description: Name of the token, if any
 
     IndexedErgoBox:
-      type: object
-      description: Box indexed with extra information
-      required:
-        - box
-        - confirmationsNum
-        - address
-        - creationTransaction
-        - spendingTransaction
-        - spendingHeight
-        - inclusionHeight
-        - spent
-        - globalIndex
-      properties:
-        box:
-          $ref: '#/components/schemas/ErgoTransactionOutput'
-        confirmationsNum:
-          description: Number of confirmations, if the box is included into the blockchain
-          type: integer
-          format: int32
-          minimum: 0
-          example: 147
-          nullable: true
-        address:
-          $ref: '#/components/schemas/ErgoAddress'
-        creationTransaction:
-          description: Transaction which created the box
-          $ref: '#/components/schemas/ModifierId'
-        spendingTransaction:
-          description: Transaction which created the box
-          nullable: true
-          $ref: '#/components/schemas/ModifierId'
-        spendingHeight:
-          description: The height the box was spent at
-          type: integer
-          format: int32
-          minimum: 0
-          example: 147
-          nullable: true
-        inclusionHeight:
-          description: The height the transaction containing the box was included in a block at
-          type: integer
-          format: int32
-          minimum: 0
-          example: 147
-        spent:
-          description: A flag signalling whether the box was spent
-          type: boolean
-          example: false
-        globalIndex:
-          description: Global index of the output in the blockchain
-          type: integer
-          format: int64
-          minimum: 0
-          example: 83927
+      allOf:
+        - $ref: '#/components/schemas/ErgoTransactionOutput'
+        - type: object
+          description: Box indexed with extra information
+          required:
+            - address
+            - spentTransactionId
+            - spendingHeight
+            - inclusionHeight
+            - globalIndex
+          properties:
+            address:
+              $ref: '#/components/schemas/ErgoAddress'
+            spentTransactionId:
+              description: Transaction which spent the box
+              nullable: true
+              $ref: '#/components/schemas/ModifierId'
+            spendingHeight:
+              description: The height the box was spent at
+              type: integer
+              format: int32
+              minimum: 0
+              example: 147
+              nullable: true
+            inclusionHeight:
+              description: The height the transaction containing the box was included in a block at
+              type: integer
+              format: int32
+              minimum: 0
+              example: 147
+            globalIndex:
+              description: Global index of the output in the blockchain
+              type: integer
+              format: int64
+              minimum: 0
+              example: 83927
 
     IndexedToken:
       type: object

From f51ebde4623c0cd562adf0abdf9b150a9f5385ff Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Mon, 25 Sep 2023 02:08:42 +0200
Subject: [PATCH 31/58] [WIP] Separated segment handling logic from
 IndexedErgoAddress

---
 .../nodeView/history/extra/ExtraIndexer.scala |   4 +-
 .../history/extra/IndexedErgoAddress.scala    | 387 ++---------------
 .../nodeView/history/extra/Segment.scala      | 405 ++++++++++++++++++
 3 files changed, 436 insertions(+), 360 deletions(-)
 create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
index e7547aa3b8..28862146f5 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
@@ -65,7 +65,7 @@ trait ExtraIndexerBase extends ScorexLogging {
   protected val general: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex]
   protected val boxes: mutable.HashMap[ModifierId,IndexedErgoBox] = mutable.HashMap.empty[ModifierId,IndexedErgoBox]
   protected val trees: mutable.HashMap[ModifierId,IndexedErgoAddress] = mutable.HashMap.empty[ModifierId,IndexedErgoAddress]
-  protected val segments: mutable.HashMap[ModifierId,IndexedErgoAddress] = mutable.HashMap.empty[ModifierId,IndexedErgoAddress]
+  protected val segments: mutable.HashMap[ModifierId,Segment[_]] = mutable.HashMap.empty[ModifierId,Segment[_]]
 
   // input tokens in a tx
   protected val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)]
@@ -117,7 +117,7 @@ trait ExtraIndexerBase extends ScorexLogging {
       case None => // address not found at all
         spendOrReceive match {
           case Left(iEb) => log.warn(s"Unknown address spent box ${bytesToId(iEb.box.id)}") // spend box should never happen by an unknown address
-          case Right(iEb) => trees.put(id, IndexedErgoAddress(id, ArrayBuffer(globalTxIndex), ArrayBuffer.empty[Long], Some(BalanceInfo())).addBox(iEb)) // receive box
+          case Right(iEb) => trees.put(id, IndexedErgoAddress(id).initBalance.addTx(globalTxIndex).addBox(iEb)) // receive box, new address
         }
     }
   }
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
index 0965922603..8ee0a3149a 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
@@ -1,183 +1,33 @@
 package org.ergoplatform.nodeView.history.extra
 
 import org.ergoplatform.ErgoAddressEncoder
-import org.ergoplatform.http.api.SortDirection.{ASC, DESC, Direction}
-import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
+import org.ergoplatform.nodeView.history.ErgoHistoryReader
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes}
-import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getFromSegments, getTxs}
-import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId}
 import org.ergoplatform.settings.Algos
 import scorex.core.serialization.ErgoSerializer
-import scorex.util.{ModifierId, ScorexLogging, bytesToId}
+import scorex.util.{ModifierId, bytesToId}
 import scorex.util.serialization.{Reader, Writer}
 import sigmastate.Values.ErgoTree
-import spire.ClassTag
-
-import scala.collection.mutable.ArrayBuffer
-import spire.syntax.all.cfor
-
-import java.lang.Math.abs
 
 /**
   * An index of an address (ErgoTree)
   * @param treeHash    - hash of the corresponding ErgoTree
   * @param txs         - list of numberic transaction indexes associated with this address
   * @param boxes       - list of numberic box indexes associated with this address, negative values indicate the box is spent
-  * @param balanceInfo - balance information (Optional because fragments do not contain it)
   */
-case class IndexedErgoAddress(treeHash: ModifierId,
-                              txs: ArrayBuffer[Long],
-                              boxes: ArrayBuffer[Long],
-                              balanceInfo: Option[BalanceInfo]) extends ExtraIndex with ScorexLogging {
+case class IndexedErgoAddress(treeHash: ModifierId)
+  extends Segment[IndexedErgoAddress](treeHash, id => IndexedErgoAddress(id)) with ExtraIndex {
 
   override lazy val id: ModifierId = treeHash
-  override def serializedId: Array[Byte] = fastIdToBytes(id)
-
-  // Internal segment buffer used when spending boxes
-  private[extra] val segments: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress]
-
-  private[extra] var boxSegmentCount: Int = 0
-  private[extra] var txSegmentCount: Int = 0
-
-  /**
-    * @return total number of transactions associated with this address
-    */
-  def txCount(implicit segmentTreshold: Int): Long = segmentTreshold * txSegmentCount + txs.length
+  override def serializedId: Array[Byte] = fastIdToBytes(treeHash)
 
   /**
-    * @return total number of boxes associated with this address
-    */
-  def boxCount(implicit segmentTreshold: Int): Long = segmentTreshold * boxSegmentCount + boxes.length
+   * Balance information (Optional because fragments do not contain it)
+   */
+  var balanceInfo: Option[BalanceInfo] = None
 
-  /**
-    * Copied from [[java.util.Arrays.binarySearch]]
-    */
-  private def binarySearch(a: ArrayBuffer[Long], key: Long): Int = {
-    var low: Int = 0
-    var high: Int = a.length - 1
-
-    while (low <= high) {
-      val mid = (low + high) >>> 1
-      val midVal = abs(a(mid)) // ignore negativity
-
-      if (midVal < key)
-        low = mid + 1
-      else if (midVal > key)
-        high = mid - 1
-      else
-        return mid // key found
-    }
-    -1 // key not found.
-  }
-
-  /**
-    * Retrieve segment with specified id from buffer or database
-    * @param history - database handle to search, if segment is not found in buffer
-    * @param id - address segment to search for
-    * @return
-    */
-  private def getSegmentFromBufferOrHistroy(history: ErgoHistoryReader, id: ModifierId): Int = {
-    cfor(segments.length - 1)(_ >= 0, _ - 1) { i =>
-      if(segments(i).id.equals(id)) return i
-    }
-    segments += history.typedExtraIndexById[IndexedErgoAddress](id).get
-    segments.length - 1
-  }
-
-  /**
-    * Locate which segment the given box number is in and change its sign, meaning it spends unspent boxes and vice versa.
-    * @param boxNum - box number to locate
-    * @param history - database to retrieve segments from
-    */
-  private[extra] def findAndModBox(boxNum: Long, history: ErgoHistoryReader): Unit = {
-    val boxNumAbs = abs(boxNum)
-    val inCurrent: Int = binarySearch(boxes, boxNumAbs)
-    if(inCurrent >= 0) { // box found in current box array
-      boxes(inCurrent) = -boxes(inCurrent)
-    } else { // box is in another segment, use binary search to locate
-      var n = 0
-      var low = 0
-      var high = boxSegmentCount - 1
-      while(low <= high) {
-        val mid = (low + high) >>> 1
-        n = getSegmentFromBufferOrHistroy(history, boxSegmentId(treeHash, mid))
-        if(abs(segments(n).boxes.head) < boxNumAbs &&
-           abs(segments(n).boxes.last) < boxNumAbs)
-          low = mid + 1
-        else if(abs(segments(n).boxes.head) > boxNumAbs &&
-                abs(segments(n).boxes.last) > boxNumAbs)
-          high = mid - 1
-        else
-          low = high + 1 // break
-      }
-      val i: Int = binarySearch(segments(n).boxes, boxNumAbs)
-      if(i >= 0)
-        segments(n).boxes(i) = -segments(n).boxes(i)
-      else
-        log.warn(s"Box $boxNum not found in any segment of parent address when trying to spend")
-    }
-  }
-
-  /**
-    * Get a range of the transactions associated with this address
-    * @param history - history to use
-    * @param offset  - items to skip from the start
-    * @param limit   - items to retrieve
-    * @return array of transactions with full bodies
-    */
-  def retrieveTxs(history: ErgoHistoryReader, offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[IndexedErgoTransaction] =
-    getFromSegments(history, treeHash, offset, limit, txSegmentCount, txs, txSegmentId, _.txs, getTxs)
-
-  /**
-    * Get a range of the boxes associated with this address
-    * @param history - history to use
-    * @param offset  - items to skip from the start
-    * @param limit   - items to retrieve
-    * @return array of boxes
-    */
-  def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[IndexedErgoBox] =
-    getFromSegments(history, treeHash, offset, limit, boxSegmentCount, boxes, boxSegmentId, _.boxes, getBoxes)
-
-  /**
-    * Get a range of the boxes associated with this address that are NOT spent
-    * @param history - history to use
-    * @param offset  - items to skip from the start
-    * @param limit   - items to retrieve
-    * @param sortDir - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]])
-    * @return array of unspent boxes
-    */
-  def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int, sortDir: Direction): Array[IndexedErgoBox] = {
-    val data: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox]
-    sortDir match {
-      case DESC =>
-        data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
-        var segment: Int = boxSegmentCount
-        while (data.length < (limit + offset) && segment > 0) {
-          segment -= 1
-          history.typedExtraIndexById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes
-            .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) ++=: data
-        }
-        data.reverse.slice(offset, offset + limit).toArray
-      case ASC =>
-        var segment: Int = 0
-        while (data.length < (limit + offset) && segment < boxSegmentCount) {
-          data ++= history.typedExtraIndexById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes
-            .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
-          segment += 1
-        }
-        if(data.length < (limit + offset))
-          data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
-        data.slice(offset, offset + limit).toArray
-    }
-  }
-
-  /**
-    * Associate transaction index with this address
-    * @param tx - numeric transaction index
-    * @return this address
-    */
-  private[extra] def addTx(tx: Long): IndexedErgoAddress = {
-    if(txs.lastOption.getOrElse(-1) != tx) txs += tx // check for duplicates
+  private[extra] def initBalance: IndexedErgoAddress = {
+    balanceInfo = Some(BalanceInfo())
     this
   }
 
@@ -187,9 +37,9 @@ case class IndexedErgoAddress(treeHash: ModifierId,
     * @param record - whether to add box to boxes list, used in rollbacks (true by default)
     * @return this address
     */
-  private[extra] def addBox(iEb: IndexedErgoBox, record: Boolean = true): IndexedErgoAddress = {
+  override private[extra] def addBox(iEb: IndexedErgoBox, record: Boolean = true): IndexedErgoAddress = {
     if(record) boxes += iEb.globalIndex
-    balanceInfo.get.add(iEb.box)
+    balanceInfo.foreach(_.add(iEb.box))
     this
   }
 
@@ -199,98 +49,24 @@ case class IndexedErgoAddress(treeHash: ModifierId,
     * @param historyOpt - history handle to update address fragment if spent box is old
     * @return this address
     */
-  private[extra] def spendBox(iEb: IndexedErgoBox, historyOpt: Option[ErgoHistoryReader] = None)(implicit ae: ErgoAddressEncoder): IndexedErgoAddress = {
-    balanceInfo.get.subtract(iEb.box)
-
-    if(historyOpt.isEmpty)
-      return this
-
-    findAndModBox(iEb.globalIndex, historyOpt.get)
-
+  override private[extra] def spendBox(iEb: IndexedErgoBox, historyOpt: Option[ErgoHistoryReader] = None)(implicit ae: ErgoAddressEncoder): IndexedErgoAddress = {
+    if(historyOpt.isDefined)
+      findAndModBox(iEb.globalIndex, historyOpt.get)
+    balanceInfo.foreach(_.subtract(iEb.box))
     this
   }
 
   /**
-    * Rollback the state of this address and of boxes associted with it
-    * @param txTarget  - remove transaction numbers above this number
-    * @param boxTarget - remove box numbers above this number and revert the balance
-    * @param _history  - history handle to update address in database
-    * @return modifier ids to remove
-    */
-  private[extra] def rollback(txTarget: Long, boxTarget: Long, _history: ErgoHistory)(implicit segmentTreshold: Int): Array[ModifierId] = {
-
-    if((txCount == 0 && boxCount == 0) || // address already rolled back
-       (txs.last <= txTarget && abs(boxes.last) <= boxTarget)) // no rollback needed
-      return Array.empty[ModifierId]
-
-    def history: ErgoHistoryReader = _history.getReader
-
-    val toSave: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex]
-    val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId]
-
-    // filter tx numbers
-    do {
-      val tmp = txs.takeWhile(_ <= txTarget)
-      txs.clear()
-      txs ++= tmp
-      if(txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible
-        val id = txSegmentId(treeHash, txSegmentCount - 1)
-        txs ++= history.typedExtraIndexById[IndexedErgoAddress](id).get.txs
-        toRemove += id
-        txSegmentCount -= 1
-      }
-    }while(txCount > 0 && txs.last > txTarget)
-
-    // filter box numbers
-    do {
-      val tmp = boxes.takeWhile(abs(_) <= boxTarget)
-      boxes.clear()
-      boxes ++= tmp
-      if(boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible
-        val id = boxSegmentId(treeHash, boxSegmentCount - 1)
-        boxes ++= history.typedExtraIndexById[IndexedErgoAddress](id).get.boxes
-        toRemove += id
-        boxSegmentCount -= 1
-      }
-    }while(boxCount > 0 && abs(boxes.last) > boxTarget)
-
-    if(txCount == 0 && boxCount == 0)
-      toRemove += this.id // address is empty after rollback, delete
-    else
-      toSave += this // save the changes made to this address
-
-    // Save changes
-    _history.historyStorage.insertExtra(Array.empty, toSave.toArray)
-
-    toRemove.toArray
+   * Add transaction index
+   *
+   * @param tx - numeric transaction index
+   * @return this
+   */
+  override private[extra] def addTx(tx: Long): IndexedErgoAddress = {
+    if (txs.lastOption.getOrElse(-1) != tx) txs += tx // check for duplicates
+    this
   }
 
-  /**
-    * Create an array addresses each containing a "segmentTreshold" number of this address's transaction and box indexes.
-    * These special addresses have their ids calculated by "txSegmentId" and "boxSegmentId" respectively.
-    * @return array of addresses
-    */
-  private[extra] def splitToSegments(implicit segmentTreshold: Int): Array[IndexedErgoAddress] = {
-    val data: Array[IndexedErgoAddress] = new Array[IndexedErgoAddress]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold))
-    var i: Int = 0
-
-    // Split txs until under segmentTreshold
-    while(txs.length >= segmentTreshold) {
-      data(i) = new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ArrayBuffer.empty[Long], None)
-      i += 1
-      txSegmentCount += 1
-      txs.remove(0, segmentTreshold)
-    }
-
-    // Split boxes until under segmentTreshold
-    while(boxes.length >= segmentTreshold) {
-      data(i) = new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ArrayBuffer.empty[Long], boxes.take(segmentTreshold), None)
-      i += 1
-      boxSegmentCount += 1
-      boxes.remove(0, segmentTreshold)
-    }
-    data
-  }
 }
 
 object IndexedErgoAddressSerializer extends ErgoSerializer[IndexedErgoAddress] {
@@ -302,45 +78,17 @@ object IndexedErgoAddressSerializer extends ErgoSerializer[IndexedErgoAddress] {
     */
   def hashErgoTree(tree: ErgoTree): ModifierId = bytesToId(Algos.hash(tree.bytes))
 
-  /**
-    * Calculates id of an address segment containing box indexes.
-    * @param hash       - hash of the parent addresses ErgoTree
-    * @param segmentNum - numberic identifier of the segment
-    * @return calculated ModifierId
-    */
-  def boxSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " box segment " + segmentNum))
-
-  /**
-    * Calculates id of an address segment transaction indexes.
-    * @param hash       - hash of the parent addresses ErgoTree
-    * @param segmentNum - numberic identifier of the segment
-    * @return calculated ModifierId
-    */
-  def txSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " tx segment " + segmentNum))
-
   override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = {
-    w.putBytes(fastIdToBytes(iEa.treeHash))
-    w.putUInt(iEa.txs.length)
-    cfor(0)(_ < iEa.txs.length, _ + 1) { i => w.putLong(iEa.txs(i))}
-    w.putUInt(iEa.boxes.length)
-    cfor(0)(_ < iEa.boxes.length, _ + 1) { i => w.putLong(iEa.boxes(i))}
+    w.putBytes(iEa.serializedId)
     w.putOption[BalanceInfo](iEa.balanceInfo)((ww, bI) => BalanceInfoSerializer.serialize(bI, ww))
-    w.putInt(iEa.boxSegmentCount)
-    w.putInt(iEa.txSegmentCount)
+    SegmentSerializer.serialize(iEa, w)
   }
 
   override def parse(r: Reader): IndexedErgoAddress = {
     val addressHash: ModifierId = bytesToId(r.getBytes(32))
-    val txnsLen: Long = r.getUInt()
-    val txns: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
-    cfor(0)(_ < txnsLen, _ + 1) { _ => txns += r.getLong()}
-    val boxesLen: Long = r.getUInt()
-    val boxes: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
-    cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += r.getLong()}
-    val balanceInfo: Option[BalanceInfo] = r.getOption[BalanceInfo](BalanceInfoSerializer.parse(r))
-    val iEa: IndexedErgoAddress = new IndexedErgoAddress(addressHash, txns, boxes, balanceInfo)
-    iEa.boxSegmentCount = r.getInt()
-    iEa.txSegmentCount = r.getInt()
+    val iEa: IndexedErgoAddress = new IndexedErgoAddress(addressHash)
+    iEa.balanceInfo = r.getOption[BalanceInfo](BalanceInfoSerializer.parse(r))
+    SegmentSerializer.parse(r, iEa)
     iEa
   }
 }
@@ -349,81 +97,4 @@ object IndexedErgoAddress {
 
   val extraIndexTypeId: ExtraIndexTypeId = 15.toByte
 
-  /**
-    * Calculate the segment offsets for the given range.
-    * @param offset - items to skip from the start
-    * @param limit  - items to retrieve
-    * @return array of offsets
-    */
-  private def getSegmentsForRange(offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[Int] =
-    (math.max(math.floor(offset * 1F / segmentTreshold).toInt, 1) to math.ceil((offset + limit) * 1F / segmentTreshold).toInt).toArray
-
-  /**
-    * Get a range of elements from an ArrayBuffer by removing the last "offset" elements,
-    * then getting the last "limit" elements reversed.
-    * @param arr    - array to get range from
-    * @param offset - number of items to skip from the end
-    * @param limit  - number of items to retrieve
-    * @return a reversed range in "arr" ArrayBuffer
-    */
-  private def sliceReversed(arr: ArrayBuffer[Long], offset: Int, limit: Int): ArrayBuffer[Long] =
-    arr.slice(arr.length - limit - offset, arr.length - offset).reverse
-
-  /**
-    * Get an array of transactions with full bodies from an array of numeric transaction indexes
-    * @param arr     - array of numeric transaction indexes to retrieve
-    * @param history - database handle
-    * @return array of transactions with full bodies
-    */
-  private def getTxs(arr: ArrayBuffer[Long], history: ErgoHistoryReader): Array[IndexedErgoTransaction] = // sorted to match explorer
-    arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray.sortBy(tx => (-tx.height, tx.id))
-
-  /**
-    * Get an array of boxes from an array of numeric box indexes
-    * @param arr     - array of numeric box indexes to retrieve
-    * @param history - database handle
-    * @return array of boxes
-    */
-  private def getBoxes(arr: ArrayBuffer[Long], history: ErgoHistoryReader): Array[IndexedErgoBox] =
-    arr.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray
-
-  /**
-    * Get a set of address segments from database containing numeric transaction or box indexes. Then actually retreive these indexes.
-    * @param history - database handle
-    * @param treeHash - hash of parent ErgoTree
-    * @param offset - number of items to skip from the start
-    * @param limit - max number of item to be returned
-    * @param segmentCount - number of segments of the parent address
-    * @param array - the indexes already in memory
-    * @param idOf - function to calculate segment ids, either [[txSegmentId]] or [[boxSegmentId]]
-    * @param arraySelector - function to select index array from retreived segments
-    * @param retreive - function to retreive indexes from database
-    * @tparam T - type of desired indexes, either [[IndexedErgoTransaction]] or [[IndexedErgoBox]]
-    * @return
-    */
-  private def getFromSegments[T : ClassTag](history: ErgoHistoryReader,
-                                             treeHash: ModifierId,
-                                             offset: Int,
-                                             limit: Int,
-                                             segmentCount: Int,
-                                             array: ArrayBuffer[Long],
-                                             idOf: (ModifierId, Int) => ModifierId,
-                                             arraySelector: IndexedErgoAddress => ArrayBuffer[Long],
-                                             retreive: (ArrayBuffer[Long], ErgoHistoryReader) => Array[T])
-                                            (implicit segmentTreshold: Int): Array[T] = {
-    val total: Int = segmentTreshold * segmentCount + array.length
-    if(offset >= total)
-      return Array.empty[T] // return empty array if all elements are skipped
-    if (offset + limit > array.length && segmentCount > 0) {
-      val data: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
-      getSegmentsForRange(offset, limit).map(n => math.max(segmentCount - n, 0)).distinct.foreach { num =>
-        arraySelector(
-          history.typedExtraIndexById[IndexedErgoAddress](idOf(treeHash, num)).get
-        ) ++=: data
-      }
-      data ++= (if (offset < array.length) array else Nil)
-      retreive(sliceReversed(data, offset % segmentTreshold, math.min(total - offset, limit)), history)
-    } else
-      retreive(sliceReversed(array, offset, limit), history)
-  }
 }
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
new file mode 100644
index 0000000000..8fac581b17
--- /dev/null
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -0,0 +1,405 @@
+package org.ergoplatform.nodeView.history.extra
+
+import org.ergoplatform.ErgoAddressEncoder
+import org.ergoplatform.http.api.SortDirection.{ASC, DESC, Direction}
+import org.ergoplatform.nodeView.history.extra.ExtraIndexer.fastIdToBytes
+import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
+import org.ergoplatform.nodeView.history.extra.SegmentSerializer._
+import org.ergoplatform.sdk.JavaHelpers.Algos
+import scorex.util.serialization.{Reader, Writer}
+import scorex.util.{ModifierId, ScorexLogging, bytesToId}
+import spire.implicits.cfor
+
+import scala.collection.mutable.ArrayBuffer
+import java.lang.Math.abs
+import scala.reflect.ClassTag
+
+abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory: ModifierId => T) extends ExtraIndex with ScorexLogging {
+
+  override lazy val id: ModifierId = parentId
+  override def serializedId: Array[Byte] = fastIdToBytes(parentId)
+
+  /**
+   * Internal segment buffer
+   */
+  private[extra] val segments: ArrayBuffer[T] = ArrayBuffer.empty[T]
+
+  val txs: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
+  val boxes: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
+
+  private[extra] var boxSegmentCount: Int = 0
+  private[extra] var txSegmentCount: Int = 0
+
+  /**
+   * @return total number of boxes associated with this address
+   */
+  def boxCount(implicit segmentTreshold: Int): Long = segmentTreshold * boxSegmentCount + boxes.length
+
+  /**
+   * @return total number of transactions associated with this address
+   */
+  def txCount(implicit segmentTreshold: Int): Long = segmentTreshold * txSegmentCount + txs.length
+
+  /**
+   * Retrieve segment with specified id from buffer or database
+   *
+   * @param history - database handle to search, if segment is not found in buffer
+   * @param searchId- address segment to search for
+   * @return
+   */
+  protected def getSegmentFromBufferOrHistroy(history: ErgoHistoryReader, searchId: ModifierId): Int = {
+    cfor(segments.length - 1)(_ >= 0, _ - 1) { i =>
+      if(segments(i).id.equals(searchId)) return i
+    }
+    segments += history.typedExtraIndexById[T](id).get
+    segments.length - 1
+  }
+
+  /**
+   * Locate which segment the given box number is in and change its sign, meaning it spends unspent boxes and vice versa.
+   *
+   * @param boxNum  - box number to locate
+   * @param history - database to retrieve segments from
+   */
+  private[extra] def findAndModBox(boxNum: Long, history: ErgoHistoryReader): Unit = {
+    val boxNumAbs = abs(boxNum)
+    val inCurrent: Int = binarySearch(boxes, boxNumAbs)
+    if(inCurrent >= 0) { // box found in current box array
+      boxes(inCurrent) = -boxes(inCurrent)
+    } else { // box is in another segment, use binary search to locate
+      var n = 0
+      var low = 0
+      var high = boxSegmentCount - 1
+      while (low <= high) {
+        val mid = (low + high) >>> 1
+        n = getSegmentFromBufferOrHistroy(history, boxSegmentId(id, mid))
+        if (abs(segments(n).boxes.head) < boxNumAbs &&
+          abs(segments(n).boxes.last) < boxNumAbs)
+          low = mid + 1
+        else if (abs(segments(n).boxes.head) > boxNumAbs &&
+          abs(segments(n).boxes.last) > boxNumAbs)
+          high = mid - 1
+        else
+          low = high + 1 // break
+      }
+      val i: Int = binarySearch(segments(n).boxes, boxNumAbs)
+      if (i >= 0)
+        segments(n).boxes(i) = -segments(n).boxes(i)
+      else
+        log.warn(s"Box $boxNum not found in any segment of parent address when trying to spend")
+    }
+  }
+
+  /**
+   * Create an array addresses each containing a "segmentTreshold" number of this address's transaction and box indexes.
+   * These special addresses have their ids calculated by "txSegmentId" and "boxSegmentId" respectively.
+   *
+   * @return array of addresses
+   */
+  private[extra] def splitToSegments(implicit segmentTreshold: Int): Array[T] = {
+    val data: Array[T] = new Array[T]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold))
+    var i: Int = 0
+
+    // Split txs until under segmentTreshold
+    while(txs.length >= segmentTreshold) {
+      data(i) = factory(txSegmentId(id, txSegmentCount))
+      data(i).txs ++= txs.take(segmentTreshold)
+      i += 1
+      txSegmentCount += 1
+      txs.remove(0, segmentTreshold)
+    }
+
+    // Split boxes until under segmentTreshold
+    while(boxes.length >= segmentTreshold) {
+      data(i) = factory(boxSegmentId(id, boxSegmentCount))
+      data(i).boxes ++= boxes.take(segmentTreshold)
+      i += 1
+      boxSegmentCount += 1
+      boxes.remove(0, segmentTreshold)
+    }
+    data
+  }
+
+  /**
+   * Calculate the segment offsets for the given range.
+   *
+   * @param offset - items to skip from the start
+   * @param limit  - items to retrieve
+   * @return array of offsets
+   */
+  private def getSegmentsForRange(offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[Int] =
+    (math.max(math.floor(offset * 1F / segmentTreshold).toInt, 1) to math.ceil((offset + limit) * 1F / segmentTreshold).toInt).toArray
+
+  /**
+   * Get a range of elements from an ArrayBuffer by removing the last "offset" elements,
+   * then getting the last "limit" elements reversed.
+   *
+   * @param arr    - array to get range from
+   * @param offset - number of items to skip from the end
+   * @param limit  - number of items to retrieve
+   * @return a reversed range in "arr" ArrayBuffer
+   */
+  private def sliceReversed(arr: ArrayBuffer[Long], offset: Int, limit: Int): ArrayBuffer[Long] =
+    arr.slice(arr.length - limit - offset, arr.length - offset).reverse
+
+  /**
+   * Get an array of transactions with full bodies from an array of numeric transaction indexes
+   *
+   * @param arr     - array of numeric transaction indexes to retrieve
+   * @param history - database handle
+   * @return array of transactions with full bodies
+   */
+  private def getTxs(arr: ArrayBuffer[Long], history: ErgoHistoryReader): Array[IndexedErgoTransaction] = // sorted to match explorer
+    arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray.sortBy(tx => (-tx.height, tx.id))
+
+  /**
+   * Get an array of boxes from an array of numeric box indexes
+   *
+   * @param arr     - array of numeric box indexes to retrieve
+   * @param history - database handle
+   * @return array of boxes
+   */
+  private def getBoxes(arr: ArrayBuffer[Long], history: ErgoHistoryReader): Array[IndexedErgoBox] =
+    arr.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray
+
+  /**
+   * Get a set of address segments from database containing numeric transaction or box indexes. Then actually retreive these indexes.
+   *
+   * @param history       - database handle
+   * @param offset        - number of items to skip from the start
+   * @param limit         - max number of item to be returned
+   * @param segmentCount  - number of segments of the parent address
+   * @param array         - the indexes already in memory
+   * @param idOf          - function to calculate segment ids, either [[txSegmentId]] or [[boxSegmentId]]
+   * @param arraySelector - function to select index array from retreived segments
+   * @param retreive      - function to retreive indexes from database
+   * @tparam B - type of desired indexes, either [[IndexedErgoTransaction]] or [[IndexedErgoBox]]
+   * @return
+   */
+  private def getFromSegments[B: ClassTag](history: ErgoHistoryReader,
+                                           offset: Int,
+                                           limit: Int,
+                                           segmentCount: Int,
+                                           array: ArrayBuffer[Long],
+                                           idOf: (ModifierId, Int) => ModifierId,
+                                           arraySelector: T => ArrayBuffer[Long],
+                                           retreive: (ArrayBuffer[Long], ErgoHistoryReader) => Array[B])
+                                          (implicit segmentTreshold: Int): Array[B] = {
+    val total: Int = segmentTreshold * segmentCount + array.length
+    if(offset >= total)
+      return Array.empty[B] // return empty array if all elements are skipped
+    if(offset + limit > array.length && segmentCount > 0) {
+      val data: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
+      getSegmentsForRange(offset, limit).map(n => math.max(segmentCount - n, 0)).distinct.foreach { num =>
+        arraySelector(
+          history.typedExtraIndexById[T](idOf(id, num)).get
+        ) ++=: data
+      }
+      data ++= (if(offset < array.length) array else Nil)
+      retreive(sliceReversed(data, offset % segmentTreshold, math.min(total - offset, limit)), history)
+    } else
+      retreive(sliceReversed(array, offset, limit), history)
+  }
+
+  /**
+   * Get a range of the transactions associated with this address
+   *
+   * @param history - history to use
+   * @param offset  - items to skip from the start
+   * @param limit   - items to retrieve
+   * @return array of transactions with full bodies
+   */
+  def retrieveTxs(history: ErgoHistoryReader, offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[IndexedErgoTransaction] =
+    getFromSegments(history, offset, limit, txSegmentCount, txs, txSegmentId, _.txs, getTxs)
+
+  /**
+   * Get a range of the boxes associated with this address
+   *
+   * @param history - history to use
+   * @param offset  - items to skip from the start
+   * @param limit   - items to retrieve
+   * @return array of boxes
+   */
+  def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[IndexedErgoBox] =
+    getFromSegments(history, offset, limit, boxSegmentCount, boxes, boxSegmentId, _.boxes, getBoxes)
+
+  /**
+   * Get a range of the boxes associated with this address that are NOT spent
+   *
+   * @param history - history to use
+   * @param offset  - items to skip from the start
+   * @param limit   - items to retrieve
+   * @param sortDir - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]])
+   * @return array of unspent boxes
+   */
+  def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int, sortDir: Direction): Array[IndexedErgoBox] = {
+    val data: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox]
+    sortDir match {
+      case DESC =>
+        data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
+        var segment: Int = boxSegmentCount
+        while(data.length < (limit + offset) && segment > 0) {
+          segment -= 1
+          history.typedExtraIndexById[T](boxSegmentId(id, segment)).get.boxes
+            .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) ++=: data
+        }
+        data.reverse.slice(offset, offset + limit).toArray
+      case ASC =>
+        var segment: Int = 0
+        while(data.length < (limit + offset) && segment < boxSegmentCount) {
+          data ++= history.typedExtraIndexById[T](boxSegmentId(id, segment)).get.boxes
+            .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
+          segment += 1
+        }
+        if (data.length < (limit + offset))
+          data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
+        data.slice(offset, offset + limit).toArray
+    }
+  }
+
+  /**
+   * Rollback the state of segments in memory and in db
+   *
+   * @param txTarget  - remove transaction numbers above this number
+   * @param boxTarget - remove box numbers above this number and revert the balance
+   * @param _history  - history handle to update segment in database
+   * @return modifier ids to remove
+   */
+  private[extra] def rollback(txTarget: Long, boxTarget: Long, _history: ErgoHistory)(implicit segmentTreshold: Int): Array[ModifierId] = {
+
+    if((txCount == 0 && boxCount == 0) || // already rolled back
+      (txs.last <= txTarget && abs(boxes.last) <= boxTarget)) // no rollback needed
+      return Array.empty[ModifierId]
+
+    def history: ErgoHistoryReader = _history.getReader
+
+    val toSave: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex]
+    val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId]
+
+    // filter tx numbers
+    do {
+      val tmp = txs.takeWhile(_ <= txTarget)
+      txs.clear()
+      txs ++= tmp
+      if(txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible
+        val id = txSegmentId(parentId, txSegmentCount - 1)
+        txs ++= history.typedExtraIndexById[T](id).get.txs
+        toRemove += id
+        txSegmentCount -= 1
+      }
+    } while (txCount > 0 && txs.last > txTarget)
+
+    // filter box numbers
+    do {
+      val tmp = boxes.takeWhile(abs(_) <= boxTarget)
+      boxes.clear()
+      boxes ++= tmp
+      if(boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible
+        val id = boxSegmentId(parentId, boxSegmentCount - 1)
+        boxes ++= history.typedExtraIndexById[T](id).get.boxes
+        toRemove += id
+        boxSegmentCount -= 1
+      }
+    } while (boxCount > 0 && abs(boxes.last) > boxTarget)
+
+    if (txCount == 0 && boxCount == 0)
+      toRemove += this.id // all segments empty after rollback, delete
+    else // TODO this will not work
+      toSave += this // save the changes made to this address
+
+    // Save changes
+    _history.historyStorage.insertExtra(Array.empty, toSave.toArray)
+
+    toRemove.toArray
+  }
+
+
+
+  /**
+   * Add transaction index
+   *
+   * @param tx - numeric transaction index
+   * @return this
+   */
+  private[extra] def addTx(tx: Long): T
+
+  /**
+   * Add box index
+   *
+   * @param iEb    - box to use
+   * @param record - whether to add box to boxes list, used in rollbacks (true by default)
+   * @return this
+   */
+  private[extra] def addBox(iEb: IndexedErgoBox, record: Boolean = true): T
+
+  /**
+   * Update segments in memory or in database by spending a box
+   *
+   * @param iEb        - box to spend
+   * @param historyOpt - history handle to update segment in db if spent box is old
+   * @return this
+   */
+  private[extra] def spendBox(iEb: IndexedErgoBox, historyOpt: Option[ErgoHistoryReader] = None)(implicit ae: ErgoAddressEncoder): T
+
+}
+
+object SegmentSerializer {
+
+  /**
+   * Calculates the id of a segment containing box indexes.
+   *
+   * @param parentId   - parent class identifier
+   * @param segmentNum - numberic identifier of the segment
+   * @return calculated ModifierId
+   */
+  def boxSegmentId(parentId: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(parentId + " box segment " + segmentNum))
+
+  /**
+   * Calculates the id of a segment containing transaction indexes.
+   *
+   * @param parentId   - parent class identifier
+   * @param segmentNum - numberic identifier of the segment
+   * @return calculated ModifierId
+   */
+  def txSegmentId(parentId: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(parentId + " tx segment " + segmentNum))
+
+  /**
+   * Copied from [[java.util.Arrays.binarySearch]]
+   */
+  private[extra] def binarySearch(a: ArrayBuffer[Long], key: Long): Int = {
+    var low: Int = 0
+    var high: Int = a.length - 1
+
+    while (low <= high) {
+      val mid = (low + high) >>> 1
+      val midVal = abs(a(mid)) // ignore negativity
+
+      if (midVal < key)
+        low = mid + 1
+      else if (midVal > key)
+        high = mid - 1
+      else
+        return mid // key found
+    }
+    -1 // key not found.
+  }
+
+  def serialize(s: Segment[_], w: Writer): Unit = {
+    w.putUInt(s.txs.length)
+    cfor(0)(_ < s.txs.length, _ + 1) { i => w.putLong(s.txs(i)) }
+    w.putUInt(s.boxes.length)
+    cfor(0)(_ < s.boxes.length, _ + 1) { i => w.putLong(s.boxes(i)) }
+    w.putInt(s.boxSegmentCount)
+    w.putInt(s.txSegmentCount)
+  }
+
+  def parse(r: Reader, s: Segment[_]): Unit = {
+    val txnsLen: Long = r.getUInt()
+    cfor(0)(_ < txnsLen, _ + 1) { _ => s.txs += r.getLong() }
+    val boxesLen: Long = r.getUInt()
+    cfor(0)(_ < boxesLen, _ + 1) { _ => s.boxes += r.getLong() }
+    s.boxSegmentCount = r.getInt()
+    s.txSegmentCount = r.getInt()
+  }
+
+}

From 494dd783bbb35ad9fbaa1fd2e0e96c196ced01cf Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Mon, 25 Sep 2023 16:25:30 +0200
Subject: [PATCH 32/58] Fixed breaking changes, eliminated bug caused by binary
 search due to improper ordering

---
 .../nodeView/history/extra/ExtraIndexer.scala |  8 +-
 .../history/extra/IndexedErgoAddress.scala    | 25 +++++-
 .../nodeView/history/extra/Segment.scala      | 84 +++++++++----------
 .../extra/ExtraIndexerSpecification.scala     |  3 +-
 4 files changed, 68 insertions(+), 52 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
index 28862146f5..82ff86eab4 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
@@ -136,9 +136,9 @@ trait ExtraIndexerBase extends ScorexLogging {
 
     // perform segmentation on big addresses and save their internal segment buffer
     trees.values.foreach { tree =>
-      if(tree.segments.nonEmpty) {
-        tree.segments.foreach(seg => segments.put(seg.id, seg))
-        tree.segments.clear()
+      if(tree.buffer.nonEmpty) {
+        tree.buffer.foreach(seg => segments.put(seg.id, seg))
+        tree.buffer.clear()
       }
       if(tree.txs.length > segmentTreshold || tree.boxes.length > segmentTreshold)
         tree.splitToSegments.foreach(seg => segments.put(seg.id, seg))
@@ -292,7 +292,7 @@ trait ExtraIndexerBase extends ScorexLogging {
         iEb.spendingTxIdOpt = None
         val address = history.typedExtraIndexById[IndexedErgoAddress](hashErgoTree(iEb.box.ergoTree)).get.addBox(iEb, record = false)
         address.findAndModBox(iEb.globalIndex, history)
-        historyStorage.insertExtra(Array.empty, Array[ExtraIndex](iEb, address) ++ address.segments)
+        historyStorage.insertExtra(Array.empty, Array[ExtraIndex](iEb, address) ++ address.buffer)
       })
       toRemove += tx.id // tx by id
       toRemove += bytesToId(NumericTxIndex.indexToBytes(globalTxIndex)) // tx id by number
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
index 8ee0a3149a..99b83104cf 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
@@ -1,7 +1,7 @@
 package org.ergoplatform.nodeView.history.extra
 
 import org.ergoplatform.ErgoAddressEncoder
-import org.ergoplatform.nodeView.history.ErgoHistoryReader
+import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes}
 import org.ergoplatform.settings.Algos
 import scorex.core.serialization.ErgoSerializer
@@ -9,11 +9,11 @@ import scorex.util.{ModifierId, bytesToId}
 import scorex.util.serialization.{Reader, Writer}
 import sigmastate.Values.ErgoTree
 
+import scala.collection.mutable.ArrayBuffer
+
 /**
   * An index of an address (ErgoTree)
   * @param treeHash    - hash of the corresponding ErgoTree
-  * @param txs         - list of numberic transaction indexes associated with this address
-  * @param boxes       - list of numberic box indexes associated with this address, negative values indicate the box is spent
   */
 case class IndexedErgoAddress(treeHash: ModifierId)
   extends Segment[IndexedErgoAddress](treeHash, id => IndexedErgoAddress(id)) with ExtraIndex {
@@ -67,6 +67,25 @@ case class IndexedErgoAddress(treeHash: ModifierId)
     this
   }
 
+  /**
+   * Rollback the state of segments in memory and in db
+   *
+   * @param txTarget  - remove transaction numbers above this number
+   * @param boxTarget - remove box numbers above this number
+   * @param history   - history handle to update segment in database
+   * @return modifier ids to remove
+   */
+  override private[extra] def rollback(txTarget: Long, boxTarget: Long, history: ErgoHistory)(implicit segmentTreshold: Int): Array[ModifierId] = {
+
+    val toRemove: ArrayBuffer[ModifierId] = rollbackState(txTarget, boxTarget, history.getReader)
+
+    if (txCount == 0 && boxCount == 0)
+      toRemove += treeHash // all segments empty after rollback, delete parent
+    else
+      history.historyStorage.insertExtra(Array.empty, Array(this)) // save the changes made to this address
+
+    toRemove.toArray
+  }
 }
 
 object IndexedErgoAddressSerializer extends ErgoSerializer[IndexedErgoAddress] {
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
index 8fac581b17..9cfebfc5be 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -22,9 +22,15 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
   /**
    * Internal segment buffer
    */
-  private[extra] val segments: ArrayBuffer[T] = ArrayBuffer.empty[T]
+  private[extra] val buffer: ArrayBuffer[T] = ArrayBuffer.empty[T]
 
+  /**
+   * List of numberic transaction indexes
+   */
   val txs: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
+  /**
+   * List of numberic box indexes, negative values indicate the box is spent
+   */
   val boxes: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
 
   private[extra] var boxSegmentCount: Int = 0
@@ -47,12 +53,12 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
    * @param searchId- address segment to search for
    * @return
    */
-  protected def getSegmentFromBufferOrHistroy(history: ErgoHistoryReader, searchId: ModifierId): Int = {
-    cfor(segments.length - 1)(_ >= 0, _ - 1) { i =>
-      if(segments(i).id.equals(searchId)) return i
+  private def getSegmentFromBufferOrHistroy(history: ErgoHistoryReader, searchId: ModifierId): Int = {
+    cfor(buffer.length - 1)(_ >= 0, _ - 1) { i =>
+      if(buffer(i).id.equals(searchId)) return i
     }
-    segments += history.typedExtraIndexById[T](id).get
-    segments.length - 1
+    buffer += history.typedExtraIndexById[T](searchId).get
+    buffer.length - 1
   }
 
   /**
@@ -72,21 +78,21 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
       var high = boxSegmentCount - 1
       while (low <= high) {
         val mid = (low + high) >>> 1
-        n = getSegmentFromBufferOrHistroy(history, boxSegmentId(id, mid))
-        if (abs(segments(n).boxes.head) < boxNumAbs &&
-          abs(segments(n).boxes.last) < boxNumAbs)
+        n = getSegmentFromBufferOrHistroy(history, boxSegmentId(parentId, mid))
+        if (abs(buffer(n).boxes.head) < boxNumAbs &&
+          abs(buffer(n).boxes.last) < boxNumAbs)
           low = mid + 1
-        else if (abs(segments(n).boxes.head) > boxNumAbs &&
-          abs(segments(n).boxes.last) > boxNumAbs)
+        else if (abs(buffer(n).boxes.head) > boxNumAbs &&
+          abs(buffer(n).boxes.last) > boxNumAbs)
           high = mid - 1
         else
           low = high + 1 // break
       }
-      val i: Int = binarySearch(segments(n).boxes, boxNumAbs)
+      val i: Int = binarySearch(buffer(n).boxes, boxNumAbs)
       if (i >= 0)
-        segments(n).boxes(i) = -segments(n).boxes(i)
+        buffer(n).boxes(i) = -buffer(n).boxes(i)
       else
-        log.warn(s"Box $boxNum not found in any segment of parent address when trying to spend")
+        log.warn(s"Box $boxNum not found in any segment of parent when trying to spend")
     }
   }
 
@@ -102,7 +108,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
 
     // Split txs until under segmentTreshold
     while(txs.length >= segmentTreshold) {
-      data(i) = factory(txSegmentId(id, txSegmentCount))
+      data(i) = factory(txSegmentId(parentId, txSegmentCount))
       data(i).txs ++= txs.take(segmentTreshold)
       i += 1
       txSegmentCount += 1
@@ -111,7 +117,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
 
     // Split boxes until under segmentTreshold
     while(boxes.length >= segmentTreshold) {
-      data(i) = factory(boxSegmentId(id, boxSegmentCount))
+      data(i) = factory(boxSegmentId(parentId, boxSegmentCount))
       data(i).boxes ++= boxes.take(segmentTreshold)
       i += 1
       boxSegmentCount += 1
@@ -257,23 +263,13 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
     }
   }
 
-  /**
-   * Rollback the state of segments in memory and in db
-   *
-   * @param txTarget  - remove transaction numbers above this number
-   * @param boxTarget - remove box numbers above this number and revert the balance
-   * @param _history  - history handle to update segment in database
-   * @return modifier ids to remove
-   */
-  private[extra] def rollback(txTarget: Long, boxTarget: Long, _history: ErgoHistory)(implicit segmentTreshold: Int): Array[ModifierId] = {
+  protected def rollbackState(txTarget: Long, boxTarget: Long, history: ErgoHistoryReader)
+                             (implicit segmentTreshold: Int): ArrayBuffer[ModifierId] = {
 
-    if((txCount == 0 && boxCount == 0) || // already rolled back
+    if ((txCount == 0 && boxCount == 0) || // already rolled back
       (txs.last <= txTarget && abs(boxes.last) <= boxTarget)) // no rollback needed
-      return Array.empty[ModifierId]
+      return ArrayBuffer.empty[ModifierId]
 
-    def history: ErgoHistoryReader = _history.getReader
-
-    val toSave: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex]
     val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId]
 
     // filter tx numbers
@@ -281,9 +277,9 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
       val tmp = txs.takeWhile(_ <= txTarget)
       txs.clear()
       txs ++= tmp
-      if(txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible
+      if (txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible
         val id = txSegmentId(parentId, txSegmentCount - 1)
-        txs ++= history.typedExtraIndexById[T](id).get.txs
+        history.typedExtraIndexById[T](id).get.txs ++=: txs
         toRemove += id
         txSegmentCount -= 1
       }
@@ -294,26 +290,26 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
       val tmp = boxes.takeWhile(abs(_) <= boxTarget)
       boxes.clear()
       boxes ++= tmp
-      if(boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible
+      if (boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible
         val id = boxSegmentId(parentId, boxSegmentCount - 1)
-        boxes ++= history.typedExtraIndexById[T](id).get.boxes
+        history.typedExtraIndexById[T](id).get.boxes ++=: boxes
         toRemove += id
         boxSegmentCount -= 1
       }
     } while (boxCount > 0 && abs(boxes.last) > boxTarget)
 
-    if (txCount == 0 && boxCount == 0)
-      toRemove += this.id // all segments empty after rollback, delete
-    else // TODO this will not work
-      toSave += this // save the changes made to this address
-
-    // Save changes
-    _history.historyStorage.insertExtra(Array.empty, toSave.toArray)
-
-    toRemove.toArray
+    toRemove
   }
 
-
+  /**
+   * Rollback the state of segments in memory and in db
+   *
+   * @param txTarget  - remove transaction numbers above this number
+   * @param boxTarget - remove box numbers above this number
+   * @param history  - history handle to update segment in database
+   * @return modifier ids to remove
+   */
+  private[extra] def rollback(txTarget: Long, boxTarget: Long, history: ErgoHistory)(implicit segmentTreshold: Int): Array[ModifierId]
 
   /**
    * Add transaction index
diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
index c26f9cf451..27af9bf610 100644
--- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
@@ -11,7 +11,8 @@ import org.ergoplatform.modifiers.history.header.Header
 import org.ergoplatform.modifiers.history.popow.NipopowAlgos
 import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction}
 import org.ergoplatform.nodeView.history.ErgoHistory
-import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, hashErgoTree, txSegmentId}
+import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree
+import org.ergoplatform.nodeView.history.extra.SegmentSerializer.{boxSegmentId, txSegmentId}
 import org.ergoplatform.nodeView.mempool.ErgoMemPool.SortingOption
 import org.ergoplatform.nodeView.state._
 import org.ergoplatform.settings.{ErgoSettings, NetworkType, NipopowSettings, NodeConfigurationSettings, UtxoSettings}

From d85c20a7dccc2b0813a42aac54eb29e9624eee83 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Tue, 26 Sep 2023 10:28:28 +0200
Subject: [PATCH 33/58] [WIP] Added box tracking functionality to tokens

---
 .../nodeView/history/extra/ExtraIndexer.scala | 123 ++++++++++++++----
 .../history/extra/IndexedErgoAddress.scala    |  10 +-
 .../nodeView/history/extra/IndexedToken.scala |  95 +++++++++++---
 .../nodeView/history/extra/Segment.scala      |  56 +++++---
 .../extra/ExtraIndexerSpecification.scala     |   1 +
 5 files changed, 220 insertions(+), 65 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
index 82ff86eab4..966024ef8e 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
@@ -2,7 +2,7 @@ package org.ergoplatform.nodeView.history.extra
 
 import akka.actor.{Actor, ActorRef, ActorSystem, Props}
 import org.ergoplatform.ErgoBox.TokenId
-import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, ErgoBox, Pay2SAddress}
+import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, Pay2SAddress}
 import org.ergoplatform.modifiers.history.BlockTransactions
 import org.ergoplatform.modifiers.history.header.Header
 import org.ergoplatform.modifiers.mempool.ErgoTransaction
@@ -11,10 +11,12 @@ import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{GlobalBoxIndexKey,
 import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.ReceivableMessages.{GetSegmentTreshold, StartExtraIndexer}
 import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree
+import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId
 import org.ergoplatform.nodeView.history.storage.HistoryStorage
 import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings}
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
 import sigmastate.Values.ErgoTree
+import special.collection.Extensions._
 
 import java.nio.ByteBuffer
 import scala.collection.mutable.ArrayBuffer
@@ -27,34 +29,54 @@ import scala.collection.mutable
   */
 trait ExtraIndexerBase extends ScorexLogging {
 
-  // Indexed block height
+  /**
+   * Indexed block height
+   */
   protected var indexedHeight: Int = 0
 
-  // Indexed transaction count
+  /**
+   * Indexed transaction count
+   */
   protected var globalTxIndex: Long = 0L
 
-  // Indexed box count
+  /**
+   * Indexed box count
+   */
   protected var globalBoxIndex: Long = 0L
 
-  // Last block height when buffer contents were saved to database
+  /**
+   * Last block height when buffer contents were saved to database
+   */
   protected var lastWroteToDB: Int = 0
 
-  // Max buffer size (determined by config)
+  /**
+   * Max buffer size (determined by config)
+   */
   protected val saveLimit: Int
 
-  // Size of box and tx number containing segment addresses
+  /**
+   * Number of transaction/box numberic indexes object segments contain
+   */
   protected implicit val segmentTreshold: Int
 
-  // Address encoder instance
+  /**
+   * Address encoder instance
+   */
   protected implicit val addressEncoder: ErgoAddressEncoder
 
-  // Flag to signal when indexer has reached current block height
+  /**
+   * Flag to signal when indexer has reached current block height
+   */
   protected var caughtUp: Boolean = false
 
-  // Flag to signal a rollback
+  /**
+   * Flag to signal a rollback
+   */
   protected var rollback: Boolean = false
 
-  // Database handle
+  /**
+   * Database handle
+   */
   protected var _history: ErgoHistory = _
 
   protected def chainHeight: Int = _history.fullBlockHeight
@@ -65,10 +87,13 @@ trait ExtraIndexerBase extends ScorexLogging {
   protected val general: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex]
   protected val boxes: mutable.HashMap[ModifierId,IndexedErgoBox] = mutable.HashMap.empty[ModifierId,IndexedErgoBox]
   protected val trees: mutable.HashMap[ModifierId,IndexedErgoAddress] = mutable.HashMap.empty[ModifierId,IndexedErgoAddress]
+  protected val tokens: mutable.HashMap[ModifierId,IndexedToken] = mutable.HashMap.empty[ModifierId,IndexedToken]
   protected val segments: mutable.HashMap[ModifierId,Segment[_]] = mutable.HashMap.empty[ModifierId,Segment[_]]
 
-  // input tokens in a tx
-  protected val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)]
+  /**
+   * Input tokens in a transaction
+   */
+  private val inputTokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)]
 
   /**
     * Spend an IndexedErgoBox from buffer or database. Also record tokens for later use in balance tracking logic.
@@ -80,13 +105,13 @@ trait ExtraIndexerBase extends ScorexLogging {
     */
   private def findAndSpendBox(id: ModifierId, txId: ModifierId, height: Int): Boolean = {
     boxes.get(id).map(box => {
-      tokens ++= box.asSpent(txId, height).box.additionalTokens.toArray
+      inputTokens ++= box.asSpent(txId, height).box.additionalTokens.toArray
       return true
     })
     history.typedExtraIndexById[IndexedErgoBox](id) match { // box not found in last saveLimit modifiers
       case Some(x) => // box found in DB, update
         boxes.put(id, x.asSpent(txId, height))
-        tokens ++= x.box.additionalTokens.toArray
+        inputTokens ++= x.box.additionalTokens.toArray
         true
       case None => // box not found at all (this shouldn't happen)
         log.warn(s"Unknown box used as input: $id")
@@ -98,7 +123,7 @@ trait ExtraIndexerBase extends ScorexLogging {
     * Add or subtract a box from an address in the buffer or in database.
     *
     * @param id             - hash of the (ergotree) address
-    * @param spendOrReceive - ErgoBox to spend or IndexedErgoBox to receive
+    * @param spendOrReceive - IndexedErgoBox to receive (Right) or spend (Left)
     */
   private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[IndexedErgoBox, IndexedErgoBox]): Unit = {
     trees.get(id).map(tree => {
@@ -111,7 +136,7 @@ trait ExtraIndexerBase extends ScorexLogging {
     history.typedExtraIndexById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers
       case Some(x) =>
         spendOrReceive match {
-          case Left(box) => trees.put(id, x.addTx(globalTxIndex).spendBox(box, Some(history))) // spend box
+          case Left(iEb) => trees.put(id, x.addTx(globalTxIndex).spendBox(iEb, Some(history))) // spend box
           case Right(iEb) => trees.put(id, x.addTx(globalTxIndex).addBox(iEb)) // receive box
         }
       case None => // address not found at all
@@ -122,6 +147,34 @@ trait ExtraIndexerBase extends ScorexLogging {
     }
   }
 
+  /**
+   * Add or subtract a box from a token in the buffer or in database.
+   *
+   * @param id             - token id
+   * @param spendOrReceive - IndexedErgoBox to receive (Right) or spend (Left)
+   */
+  private def findAndUpdateToken(id: ModifierId, spendOrReceive: Either[IndexedErgoBox, IndexedErgoBox]): Unit = {
+    tokens.get(id).map(token => {
+      spendOrReceive match {
+        case Left(iEb) => token.spendBox(iEb, Some(history)) // spend box
+        case Right(iEb) => token.addBox(iEb) // receive box
+      }
+      return
+    })
+    history.typedExtraIndexById[IndexedToken](uniqueId(id)) match { // token not found in last saveLimit modifiers
+      case Some(x) =>
+        spendOrReceive match {
+          case Left(iEb) => tokens.put(id, x.spendBox(iEb, Some(history))) // spend box
+          case Right(iEb) => tokens.put(id, x.addBox(iEb)) // receive box
+        }
+      case None => // token not found at all
+        spendOrReceive match {
+          case Left(iEb) => log.warn(s"Unknown address spent box ${bytesToId(iEb.box.id)}") // spend box should never happen by an unknown address
+          case Right(iEb) => trees.put(id, IndexedErgoAddress(id).initBalance.addTx(globalTxIndex).addBox(iEb)) // receive box, new address
+        }
+    }
+  }
+
   /**
     * @return number of indexes in all buffers
     */
@@ -144,11 +197,21 @@ trait ExtraIndexerBase extends ScorexLogging {
         tree.splitToSegments.foreach(seg => segments.put(seg.id, seg))
     }
 
+    // perform segmentation on big tokens and save their internal segment buffer
+    tokens.values.foreach { token =>
+      if(token.buffer.nonEmpty) {
+        token.buffer.foreach(seg => segments.put(seg.id, seg))
+        token.buffer.clear()
+      }
+      if(token.txs.length > segmentTreshold || token.boxes.length > segmentTreshold)
+        token.splitToSegments.foreach(seg => segments.put(seg.id, seg))
+    }
+
     // insert modifiers and progress info to db
     historyStorage.insertExtra(Array((IndexedHeightKey, ByteBuffer.allocate(4).putInt(indexedHeight).array),
                                      (GlobalTxIndexKey, ByteBuffer.allocate(8).putLong(globalTxIndex).array),
                                      (GlobalBoxIndexKey,ByteBuffer.allocate(8).putLong(globalBoxIndex).array)),
-                              (((general ++= boxes.values) ++= trees.values) ++= segments.values).toArray)
+      ((((general ++= boxes.values) ++= trees.values) ++= tokens.values) ++= segments.values).toArray)
 
     if (writeLog)
       log.info(s"Processed ${trees.size} ErgoTrees with ${boxes.size} boxes and inserted them to database in ${System.currentTimeMillis - start}ms")
@@ -182,7 +245,7 @@ trait ExtraIndexerBase extends ScorexLogging {
       val tx: ErgoTransaction = bt.txs(n)
       val inputs: Array[Long] = Array.ofDim[Long](tx.inputs.length)
 
-      tokens.clear()
+      inputTokens.clear()
 
       //process transaction inputs
       if (height != 1) { //only after 1st block (skip genesis box)
@@ -191,6 +254,9 @@ trait ExtraIndexerBase extends ScorexLogging {
           if (findAndSpendBox(boxId, tx.id, height)) { // spend box and add tx
             val iEb = boxes(boxId)
             findAndUpdateTree(hashErgoTree(iEb.box.ergoTree), Left(iEb))
+            cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { j =>
+              findAndUpdateToken(iEb.box.additionalTokens(j)._1.toModifierId, Left(iEb))
+            }
             inputs(i) = iEb.globalIndex
           }
         }
@@ -198,19 +264,20 @@ trait ExtraIndexerBase extends ScorexLogging {
 
       //process transaction outputs
       cfor(0)(_ < tx.outputs.size, _ + 1) { i =>
-        val box: ErgoBox = tx.outputs(i)
-        val boxId = bytesToId(box.id)
-        boxes.put(boxId, new IndexedErgoBox(height, None, None, box, globalBoxIndex)) // box by id
-        general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number
+        val iEb: IndexedErgoBox = new IndexedErgoBox(height, None, None, tx.outputs(i), globalBoxIndex)
+        boxes.put(iEb.id, iEb) // box by id
+        general += NumericBoxIndex(globalBoxIndex, iEb.id) // box id by global box number
 
         // box by address
-        findAndUpdateTree(hashErgoTree(box.ergoTree), Right(boxes(boxId)))
+        findAndUpdateTree(hashErgoTree(iEb.box.ergoTree), Right(boxes(iEb.id)))
 
         // check if box is creating new tokens, if yes record them
-        cfor(0)(_ < box.additionalTokens.length, _ + 1) { j =>
-          if (!tokens.exists(x => x._1 == box.additionalTokens(j)._1)) {
-            general += IndexedToken.fromBox(box, j)
+        cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { j =>
+          if (!inputTokens.exists(x => x._1 == iEb.box.additionalTokens(j)._1)) {
+            val token = IndexedToken.fromBox(iEb, j)
+            tokens.put(token.tokenId, token)
           }
+          findAndUpdateToken(iEb.box.additionalTokens(j)._1.toModifierId, Right(iEb))
         }
 
         globalBoxIndex += 1
@@ -305,7 +372,7 @@ trait ExtraIndexerBase extends ScorexLogging {
     while(globalBoxIndex > boxTarget) {
       val iEb: IndexedErgoBox = NumericBoxIndex.getBoxByNumber(history, globalBoxIndex).get
       cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { i =>
-        history.typedExtraIndexById[IndexedToken](IndexedToken.fromBox(iEb.box, i).id) match {
+        history.typedExtraIndexById[IndexedToken](IndexedToken.fromBox(iEb, i).id) match {
           case Some(token) if token.boxId == iEb.id =>
             toRemove += token.id // token created, delete
           case _ => // no token created
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
index 99b83104cf..230f17f88b 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
@@ -13,10 +13,14 @@ import scala.collection.mutable.ArrayBuffer
 
 /**
   * An index of an address (ErgoTree)
-  * @param treeHash    - hash of the corresponding ErgoTree
+  * @param treeHash - hash of the corresponding ErgoTree
+  * @param txs      - list of numberic transaction indexes
+  * @param boxes    - list of numberic box indexes, negative values indicate the box is spent
   */
-case class IndexedErgoAddress(treeHash: ModifierId)
-  extends Segment[IndexedErgoAddress](treeHash, id => IndexedErgoAddress(id)) with ExtraIndex {
+case class IndexedErgoAddress(treeHash: ModifierId,
+                              override val txs: ArrayBuffer[Long] = new ArrayBuffer[Long],
+                              override val boxes: ArrayBuffer[Long] = new ArrayBuffer[Long])
+  extends Segment[IndexedErgoAddress](treeHash, id => IndexedErgoAddress(id), txs, boxes) with ExtraIndex {
 
   override lazy val id: ModifierId = treeHash
   override def serializedId: Array[Byte] = fastIdToBytes(treeHash)
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
index 19c6d18d8e..6c4a356376 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
@@ -1,17 +1,20 @@
 package org.ergoplatform.nodeView.history.extra
 
-import org.ergoplatform.ErgoBox
+import org.ergoplatform.ErgoAddressEncoder
 import org.ergoplatform.ErgoBox.{R4, R5, R6}
+import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes}
 import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.{ByteColl, uniqueId}
 import org.ergoplatform.settings.Algos
 import scorex.core.serialization.ErgoSerializer
-import scorex.util.{ByteArrayOps, ModifierId, bytesToId}
+import scorex.util.{ModifierId, bytesToId}
 import scorex.util.serialization.{Reader, Writer}
 import sigmastate.Values.CollectionConstant
 import sigmastate.SByte
 import special.collection.Extensions._
 
+import scala.collection.mutable.ArrayBuffer
+
 /**
   * Index of a token containing creation information.
   * @param tokenId     - id of this token
@@ -20,17 +23,73 @@ import special.collection.Extensions._
   * @param name        - name of this token (UTF-8)
   * @param description - description of this token (UTF-8)
   * @param decimals    - number of decimal places
+  * @param boxes       - list of numberic box indexes, negative values indicate the box is spent
   */
 case class IndexedToken(tokenId: ModifierId,
-                        boxId: ModifierId,
-                        amount: Long,
-                        name: String,
-                        description: String,
-                        decimals: Int) extends ExtraIndex {
+                        boxId: ModifierId = ModifierId @@ "",
+                        amount: Long = 0L,
+                        name: String = "",
+                        description: String = "",
+                        decimals: Int = 0,
+                        override val boxes: ArrayBuffer[Long] = new ArrayBuffer[Long])
+  extends Segment[IndexedToken](uniqueId(tokenId), id => IndexedToken(id), new ArrayBuffer[Long], boxes) with ExtraIndex {
 
   override lazy val id: ModifierId = uniqueId(tokenId)
   override def serializedId: Array[Byte] = fastIdToBytes(id)
 
+  /**
+   * Rollback the state of segments in memory and in db
+   *
+   * @param txTarget  - remove transaction numbers above this number
+   * @param boxTarget - remove box numbers above this number
+   * @param history   - history handle to update segment(s) in database
+   * @return modifier ids to remove
+   */
+  override private[extra] def rollback(txTarget: Long, boxTarget: Long, history: ErgoHistory)(implicit segmentTreshold: Int): Array[ModifierId] = {
+
+    val toRemove: ArrayBuffer[ModifierId] = rollbackState(txTarget, boxTarget, history.getReader)
+
+    if (txCount == 0 && boxCount == 0)
+      toRemove += id // all segments empty after rollback, delete parent
+    else
+      history.historyStorage.insertExtra(Array.empty, Array(this)) // save the changes made to this address
+
+    toRemove.toArray
+  }
+
+  /**
+   * Add transaction index
+   *
+   * @param tx - numeric transaction index
+   * @return this
+   */
+  @deprecated("Indexed tokens do not track transactions", "")
+  override private[extra] def addTx(tx: Long): IndexedToken = this
+
+  /**
+   * Add box index
+   *
+   * @param iEb    - box to use
+   * @param record - whether to add box to boxes list, used in rollbacks (true by default)
+   * @return this
+   */
+  override private[extra] def addBox(iEb: IndexedErgoBox, record: Boolean = true): IndexedToken = {
+    if(record) boxes += iEb.globalIndex
+    this
+  }
+
+  /**
+   * Update segments in memory or in database by spending a box
+   *
+   * @param iEb        - box to spend
+   * @param historyOpt - history handle to update segment in db if spent box is old
+   * @return this
+   */
+  override private[extra] def spendBox(iEb: IndexedErgoBox, historyOpt: Option[ErgoHistoryReader])(implicit ae: ErgoAddressEncoder): IndexedToken = {
+    if(historyOpt.isDefined)
+      findAndModBox(iEb.globalIndex, historyOpt.get)
+    this
+  }
 }
 
 object IndexedTokenSerializer extends ErgoSerializer[IndexedToken] {
@@ -56,6 +115,7 @@ object IndexedTokenSerializer extends ErgoSerializer[IndexedToken] {
     w.putUShort(description.length)
     w.putBytes(description)
     w.putInt(iT.decimals)
+    SegmentSerializer.serialize(iT, w)
   }
 
   override def parse(r: Reader): IndexedToken = {
@@ -67,7 +127,9 @@ object IndexedTokenSerializer extends ErgoSerializer[IndexedToken] {
     val descLen: Int = r.getUShort()
     val description: String = new String(r.getBytes(descLen), "UTF-8")
     val decimals: Int = r.getInt()
-    IndexedToken(tokenId, boxId, amount, name, description, decimals)
+    val iT = IndexedToken(tokenId, boxId, amount, name, description, decimals)
+    SegmentSerializer.parse(r, iT)
+    iT
   }
 }
 
@@ -80,13 +142,13 @@ object IndexedToken {
     * Tokens can be created without setting registers or something other than token information can be in them,
     * so they are checked with Try-catches.
     *
-    * @param box - box to use
+    * @param iEb - box to use
     * @param tokenIndex - token index to check in [[ErgoBox.additionalTokens]]
     * @return token index
     */
-  def fromBox(box: ErgoBox, tokenIndex: Int): IndexedToken = {
+  def fromBox(iEb: IndexedErgoBox, tokenIndex: Int): IndexedToken = {
     val name: String =
-      box.additionalRegisters.get(R4) match {
+      iEb.box.additionalRegisters.get(R4) match {
         case Some(reg) =>
           try {
             new String(reg.asInstanceOf[ByteColl].value.toArray, "UTF-8")
@@ -97,7 +159,7 @@ object IndexedToken {
       }
 
     val description: String =
-      box.additionalRegisters.get(R5) match {
+      iEb.box.additionalRegisters.get(R5) match {
         case Some(reg) =>
           try {
             new String(reg.asInstanceOf[ByteColl].value.toArray, "UTF-8")
@@ -109,7 +171,7 @@ object IndexedToken {
 
 
     val decimals: Int =
-      box.additionalRegisters.get(R6) match {
+      iEb.box.additionalRegisters.get(R6) match {
         case Some(reg) =>
           try {
             new String(reg.asInstanceOf[ByteColl].value.toArray, "UTF-8").toInt
@@ -124,11 +186,12 @@ object IndexedToken {
         case None => 0
       }
 
-    IndexedToken(box.additionalTokens(tokenIndex)._1.toModifierId,
-                 box.id.toModifierId,
-                 box.additionalTokens(tokenIndex)._2,
+    IndexedToken(iEb.box.additionalTokens(tokenIndex)._1.toModifierId,
+                 iEb.id,
+                 iEb.box.additionalTokens(tokenIndex)._2,
                  name,
                  description,
                  decimals)
+      .addBox(iEb)
   }
 }
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
index 9cfebfc5be..f5e86a6954 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -14,7 +14,21 @@ import scala.collection.mutable.ArrayBuffer
 import java.lang.Math.abs
 import scala.reflect.ClassTag
 
-abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory: ModifierId => T) extends ExtraIndex with ScorexLogging {
+/**
+ * Class to manage the tracking of transactions/boxes in relation to some other object (ErgoTree/token).
+ * When [[ExtraIndexerBase.segmentTreshold]] number of transaction/box indexes are accumulated, new instances of the parent object are created to contain them.
+ * This mechanism is used to prevent excessive serialization/deserialization delays caused by objects with a lot of transaction/box indexes.
+ * @param parentId - identifier of parent object
+ * @param factory  - parent object factory
+ * @param txs      - list of numberic transaction indexes
+ * @param boxes    - list of numberic box indexes, negative values indicate the box is spent
+ * @tparam T       - type of parent object
+ */
+abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
+                                                   val factory: ModifierId => T,
+                                                   val txs: ArrayBuffer[Long],
+                                                   val boxes: ArrayBuffer[Long])
+  extends ExtraIndex with ScorexLogging {
 
   override lazy val id: ModifierId = parentId
   override def serializedId: Array[Byte] = fastIdToBytes(parentId)
@@ -25,24 +39,22 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
   private[extra] val buffer: ArrayBuffer[T] = ArrayBuffer.empty[T]
 
   /**
-   * List of numberic transaction indexes
+   * Number of segments in database containing box numbers
    */
-  val txs: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
+  private[extra] var boxSegmentCount: Int = 0
+
   /**
-   * List of numberic box indexes, negative values indicate the box is spent
+   * Number of segments in database containing transaction numbers
    */
-  val boxes: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
-
-  private[extra] var boxSegmentCount: Int = 0
   private[extra] var txSegmentCount: Int = 0
 
   /**
-   * @return total number of boxes associated with this address
+   * @return total number of boxes associated with the parent object
    */
   def boxCount(implicit segmentTreshold: Int): Long = segmentTreshold * boxSegmentCount + boxes.length
 
   /**
-   * @return total number of transactions associated with this address
+   * @return total number of transactions associated with the parent object
    */
   def txCount(implicit segmentTreshold: Int): Long = segmentTreshold * txSegmentCount + txs.length
 
@@ -50,7 +62,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
    * Retrieve segment with specified id from buffer or database
    *
    * @param history - database handle to search, if segment is not found in buffer
-   * @param searchId- address segment to search for
+   * @param searchId- segment id to search for
    * @return
    */
   private def getSegmentFromBufferOrHistroy(history: ErgoHistoryReader, searchId: ModifierId): Int = {
@@ -97,10 +109,10 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
   }
 
   /**
-   * Create an array addresses each containing a "segmentTreshold" number of this address's transaction and box indexes.
-   * These special addresses have their ids calculated by "txSegmentId" and "boxSegmentId" respectively.
+   * Create an array of parent objects each containing [[ExtraIndexerBase.segmentTreshold]] number of transaction/box indexes.
+   * These objects have their ids calculated by "txSegmentId" and "boxSegmentId" respectively.
    *
-   * @return array of addresses
+   * @return array of parent objects
    */
   private[extra] def splitToSegments(implicit segmentTreshold: Int): Array[T] = {
     val data: Array[T] = new Array[T]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold))
@@ -169,7 +181,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
     arr.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray
 
   /**
-   * Get a set of address segments from database containing numeric transaction or box indexes. Then actually retreive these indexes.
+   * Get a set of segments from database containing numeric transaction or box indexes. Then actually retreive these indexes.
    *
    * @param history       - database handle
    * @param offset        - number of items to skip from the start
@@ -208,7 +220,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
   }
 
   /**
-   * Get a range of the transactions associated with this address
+   * Get a range of the transactions associated with the parent object
    *
    * @param history - history to use
    * @param offset  - items to skip from the start
@@ -219,7 +231,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
     getFromSegments(history, offset, limit, txSegmentCount, txs, txSegmentId, _.txs, getTxs)
 
   /**
-   * Get a range of the boxes associated with this address
+   * Get a range of the boxes associated with the parent object
    *
    * @param history - history to use
    * @param offset  - items to skip from the start
@@ -230,7 +242,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
     getFromSegments(history, offset, limit, boxSegmentCount, boxes, boxSegmentId, _.boxes, getBoxes)
 
   /**
-   * Get a range of the boxes associated with this address that are NOT spent
+   * Get a range of the boxes associated with the parent that are NOT spent
    *
    * @param history - history to use
    * @param offset  - items to skip from the start
@@ -263,6 +275,14 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
     }
   }
 
+  /**
+   * Logic for [[Segment.rollback]]
+   *
+   * @param txTarget  - remove transaction numbers above this number
+   * @param boxTarget - remove box numbers above this number
+   * @param history   - history handle to update segment(s) in database
+   * @return modifier ids to remove
+   */
   protected def rollbackState(txTarget: Long, boxTarget: Long, history: ErgoHistoryReader)
                              (implicit segmentTreshold: Int): ArrayBuffer[ModifierId] = {
 
@@ -306,7 +326,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](parentId: ModifierId, factory
    *
    * @param txTarget  - remove transaction numbers above this number
    * @param boxTarget - remove box numbers above this number
-   * @param history  - history handle to update segment in database
+   * @param history  - history handle to update segment(s) in database
    * @return modifier ids to remove
    */
   private[extra] def rollback(txTarget: Long, boxTarget: Long, history: ErgoHistory)(implicit segmentTreshold: Int): Array[ModifierId]
diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
index 27af9bf610..aaf2786235 100644
--- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
@@ -68,6 +68,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
     general.clear()
     boxes.clear()
     trees.clear()
+    segments.clear()
   }
 
   def getAddresses(limit: Int): (mutable.HashMap[ModifierId,(Long,Long)],Int,Int) = {

From c9e3c51c810f2d9cc32555f014859e2868786946 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Tue, 26 Sep 2023 10:38:53 +0200
Subject: [PATCH 34/58] Added missing clear statements

---
 .../org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala   | 1 +
 .../nodeView/history/extra/ExtraIndexerSpecification.scala       | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
index 966024ef8e..ed0a65c82e 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
@@ -220,6 +220,7 @@ trait ExtraIndexerBase extends ScorexLogging {
     general.clear()
     boxes.clear()
     trees.clear()
+    tokens.clear()
     segments.clear()
 
     lastWroteToDB = indexedHeight
diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
index aaf2786235..56e0bb8f11 100644
--- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
@@ -68,6 +68,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
     general.clear()
     boxes.clear()
     trees.clear()
+    tokens.clear()
     segments.clear()
   }
 

From 6febeb2600fefd999cba9bb1a47adbdab87c74aa Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Tue, 26 Sep 2023 16:46:32 +0300
Subject: [PATCH 35/58] validateWithCost args simplification

---
 .../org/ergoplatform/http/api/ErgoBaseApiRoute.scala   |  2 +-
 .../scala/org/ergoplatform/local/CleanupWorker.scala   |  2 +-
 .../org/ergoplatform/mining/CandidateGenerator.scala   |  2 +-
 .../network/ErgoNodeViewSynchronizer.scala             |  1 -
 .../ergoplatform/nodeView/mempool/ErgoMemPool.scala    |  2 +-
 .../ergoplatform/nodeView/state/UtxoStateReader.scala  |  3 +--
 .../mining/CandidateGeneratorPropSpec.scala            |  2 +-
 .../scala/org/ergoplatform/mining/ErgoMinerSpec.scala  |  2 +-
 .../nodeView/mempool/ErgoMemPoolSpec.scala             |  2 +-
 .../nodeView/state/UtxoStateSpecification.scala        | 10 +++++-----
 10 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
index 694ee183d9..67614c099f 100644
--- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
@@ -99,7 +99,7 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
           val maxTxCost = ergoSettings.nodeSettings.maxTransactionCost
           val validationContext = utxo.stateContext.simplifiedUpcoming()
           utxo.withMempool(mp)
-            .validateWithCost(tx, Some(validationContext), maxTxCost, None)
+            .validateWithCost(tx, validationContext, maxTxCost, None)
             .map(cost => new UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None))
         case _ =>
           tx.statelessValidity().map(_ => new UnconfirmedTransaction(tx, None, now, now, bytes, source = None))
diff --git a/src/main/scala/org/ergoplatform/local/CleanupWorker.scala b/src/main/scala/org/ergoplatform/local/CleanupWorker.scala
index 6ab159c1f9..0db0c9fce8 100644
--- a/src/main/scala/org/ergoplatform/local/CleanupWorker.scala
+++ b/src/main/scala/org/ergoplatform/local/CleanupWorker.scala
@@ -87,7 +87,7 @@ class CleanupWorker(nodeViewHolderRef: ActorRef,
       txs match {
         case head :: tail if costAcc < CostLimit =>
           val validationContext = state.stateContext.simplifiedUpcoming()
-          state.validateWithCost(head.transaction, Some(validationContext), nodeSettings.maxTransactionCost, None) match {
+          state.validateWithCost(head.transaction, validationContext, nodeSettings.maxTransactionCost, None) match {
             case Success(txCost) =>
               val updTx = head.withCost(txCost)
               validationLoop(tail, validated += updTx, invalidated, txCost + costAcc)
diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala
index 61fee2f123..89059abc0f 100644
--- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala
+++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala
@@ -819,7 +819,7 @@ object CandidateGenerator extends ScorexLogging {
             // check validity and calculate transaction cost
             stateWithTxs.validateWithCost(
               tx,
-              Some(upcomingContext),
+              upcomingContext,
               maxBlockCost,
               Some(verifier)
             ) match {
diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
index 16179c95b1..352a41ab70 100644
--- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
+++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala
@@ -34,7 +34,6 @@ import scorex.core.validation.MalformedModifierError
 import scorex.util.{ModifierId, ScorexLogging}
 import scorex.core.network.DeliveryTracker
 import scorex.core.network.peer.PenaltyType
-import scorex.core.app.Version
 import scorex.crypto.hash.Digest32
 import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId}
 import org.ergoplatform.ErgoLikeContext.Height
diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala
index 0cf8a38f1e..75f626db88 100644
--- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala
@@ -228,7 +228,7 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool,
               val utxoWithPool = utxo.withUnconfirmedTransactions(getAll)
               if (tx.inputIds.forall(inputBoxId => utxoWithPool.boxById(inputBoxId).isDefined)) {
                 val validationContext = utxo.stateContext.simplifiedUpcoming()
-                utxoWithPool.validateWithCost(tx, Some(validationContext), costLimit, None) match {
+                utxoWithPool.validateWithCost(tx, validationContext, costLimit, None) match {
                   case Success(cost) =>
                     acceptIfNoDoubleSpend(unconfirmedTx.withCost(cost), validationStartTime)
                   case Failure(ex) =>
diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
index a1679587c0..f717464185 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala
@@ -45,10 +45,9 @@ trait UtxoStateReader extends ErgoStateReader with UtxoSetSnapshotPersistence {
     * or state context from the previous block if not
     */
   def validateWithCost(tx: ErgoTransaction,
-                       stateContextOpt: Option[ErgoStateContext],
+                       context: ErgoStateContext,
                        costLimit: Int,
                        interpreterOpt: Option[ErgoInterpreter]): Try[Int] = {
-    val context = stateContextOpt.getOrElse(stateContext)
     val parameters = context.currentParameters.withBlockCost(costLimit)
     val verifier = interpreterOpt.getOrElse(ErgoInterpreter(parameters))
 
diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala
index 66aa6a8c05..7bca0659bf 100644
--- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala
+++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala
@@ -167,7 +167,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest {
 
       val newBoxes = fromBigMempool.flatMap(_.outputs)
       val costs: Seq[Int] = fromBigMempool.map { tx =>
-        us.validateWithCost(tx, Some(upcomingContext), Int.MaxValue, Some(verifier)).getOrElse {
+        us.validateWithCost(tx, upcomingContext, Int.MaxValue, Some(verifier)).getOrElse {
           val boxesToSpend =
             tx.inputs.map(i => newBoxes.find(b => b.id sameElements i.boxId).get)
           tx.statefulValidity(boxesToSpend, IndexedSeq(), upcomingContext).get
diff --git a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala
index 16e83f5d00..f2ce95c9b7 100644
--- a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala
+++ b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala
@@ -114,7 +114,7 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen
     val txCost =
       state.validateWithCost(
         ErgoTransaction(costlyTx.inputs, costlyTx.dataInputs, costlyTx.outputCandidates),
-        Some(r.s.stateContext),
+        r.s.stateContext,
         costLimit = 440000,
         None
       ).get
diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala
index 4ab6f0d98f..38aaa799d3 100644
--- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala
@@ -73,7 +73,7 @@ class ErgoMemPoolSpec extends AnyFlatSpec
     var poolCost = ErgoMemPool.empty(sortByCostSettings)
     poolCost = poolCost.process(UnconfirmedTransaction(tx, None), wus)._1
     val validationContext = wus.stateContext.simplifiedUpcoming()
-    val cost = wus.validateWithCost(tx, Some(validationContext), Int.MaxValue, None).get
+    val cost = wus.validateWithCost(tx, validationContext, Int.MaxValue, None).get
     poolCost.pool.orderedTransactions.firstKey.weight shouldBe OrderedTxPool.weighted(tx, cost).weight
   }
 
diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
index 5b4aa9cba5..d24a152ed7 100644
--- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
@@ -51,7 +51,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera
       val unsignedTx = new UnsignedErgoTransaction(inputs, IndexedSeq(), newBoxes)
       val tx: ErgoTransaction = ErgoTransaction(defaultProver.sign(unsignedTx, IndexedSeq(foundersBox), emptyDataBoxes, us.stateContext).get)
       val txCostLimit     = initSettings.nodeSettings.maxTransactionCost
-      us.validateWithCost(tx, None, txCostLimit, None).get should be <= 100000
+      us.validateWithCost(tx, us.stateContext.simplifiedUpcoming(), txCostLimit, None).get should be <= 100000
       val block1 = validFullBlock(Some(lastBlock), us, Seq(ErgoTransaction(tx)))
       us = us.applyModifier(block1, None)(_ => ()).get
       foundersBox = tx.outputs.head
@@ -95,17 +95,17 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera
       val unsignedTx = new UnsignedErgoTransaction(inputs, IndexedSeq(), newBoxes)
       val tx = ErgoTransaction(defaultProver.sign(unsignedTx, IndexedSeq(foundersBox), emptyDataBoxes, us.stateContext).get)
       val validationContext = us.stateContext.simplifiedUpcoming()
-      val validationRes1 = us.validateWithCost(tx, Some(validationContext), 100000, None)
+      val validationRes1 = us.validateWithCost(tx, validationContext, 100000, None)
       validationRes1 shouldBe 'success
       val txCost = validationRes1.get
 
-      val validationRes2 = us.validateWithCost(tx, Some(validationContext), txCost - 1, None)
+      val validationRes2 = us.validateWithCost(tx, validationContext, txCost - 1, None)
       validationRes2 shouldBe 'failure
       validationRes2.toEither.left.get.isInstanceOf[TooHighCostError] shouldBe true
 
-      us.validateWithCost(tx, Some(validationContext), txCost + 1, None) shouldBe 'success
+      us.validateWithCost(tx, validationContext, txCost + 1, None) shouldBe 'success
 
-      us.validateWithCost(tx, Some(validationContext), txCost, None) shouldBe 'success
+      us.validateWithCost(tx, validationContext, txCost, None) shouldBe 'success
 
       height = height + 1
     }

From 343d0905cb598e8ee58b5ccd78c8fac20ace91a5 Mon Sep 17 00:00:00 2001
From: Alexander Slesarenko <avslesarenko@gmail.com>
Date: Tue, 26 Sep 2023 16:43:03 +0200
Subject: [PATCH 36/58] with-sigma-v5.0.12: upgrade code

---
 build.sbt                                     |  2 +-
 .../interface4j/crypto/ErgoUnsafeProver.java  |  2 +-
 .../contracts/ReemissionContracts.scala       |  2 +-
 .../wallet/boxes/ErgoBoxAssetExtractor.scala  |  2 +-
 .../wallet/boxes/TrackedBox.scala             |  2 +-
 .../wallet/crypto/ErgoSignature.scala         |  2 +-
 .../wallet/interpreter/ErgoInterpreter.scala  |  8 +--
 .../interpreter/ErgoProvingInterpreter.scala  | 27 +++++-----
 .../wallet/interpreter/ErgoUnsafeProver.scala |  2 +-
 .../transactions/TransactionBuilder.scala     |  4 +-
 .../wallet/CreateTransactionDemo.java         |  2 +-
 .../wallet/boxes/DefaultBoxSelectorSpec.scala |  2 +-
 .../wallet/crypto/ErgoSignatureSpec.scala     |  2 +-
 .../ErgoProvingInterpreterSpec.scala          |  1 +
 .../interpreter/InterpreterSpecCommon.scala   | 14 +++---
 .../wallet/utils/Generators.scala             |  2 +-
 .../org/ergoplatform/it/WalletSpec.scala      |  1 +
 .../org/ergoplatform/http/api/ApiCodecs.scala | 50 +++++++++++++------
 .../http/api/ErgoUtilsApiRoute.scala          |  2 +-
 .../http/api/MiningApiRoute.scala             |  2 +-
 .../http/api/ScriptApiRoute.scala             |  2 +-
 .../mining/AutolykosPowScheme.scala           |  2 +-
 .../mining/AutolykosSolution.scala            |  4 +-
 .../mining/CandidateGenerator.scala           |  6 +--
 .../mining/DefaultFakePowScheme.scala         |  2 +-
 .../org/ergoplatform/mining/ErgoMiner.scala   |  2 +-
 .../org/ergoplatform/mining/WorkMessage.scala |  2 +-
 .../org/ergoplatform/mining/mining.scala      |  6 +--
 .../modifiers/history/PreHeader.scala         |  4 +-
 .../modifiers/history/header/Header.scala     |  4 +-
 .../nodeView/history/extra/BalanceInfo.scala  |  2 +-
 .../nodeView/history/extra/IndexedToken.scala |  2 +-
 .../nodeView/state/ErgoState.scala            |  2 +-
 .../nodeView/state/ErgoStateContext.scala     | 16 +++---
 .../nodeView/wallet/ErgoWalletActor.scala     |  2 +-
 .../nodeView/wallet/ErgoWalletReader.scala    |  2 +-
 .../nodeView/wallet/ErgoWalletService.scala   |  4 +-
 .../nodeView/wallet/ErgoWalletSupport.scala   |  3 +-
 .../wallet/scanning/ScanningPredicate.scala   |  2 +-
 .../ScanningPredicateJsonCodecs.scala         |  2 +-
 .../reemission/ReemissionRules.scala          |  2 +-
 .../ergoplatform/settings/ErgoSettings.scala  |  2 +-
 .../ergoplatform/settings/Parameters.scala    |  4 +-
 .../settings/ReemissionSettings.scala         |  2 +-
 .../org/ergoplatform/utils/BoxUtils.scala     | 11 ++--
 .../http/routes/TransactionApiRouteSpec.scala |  8 +--
 .../mining/CandidateGeneratorPropSpec.scala   |  2 +-
 .../mining/CandidateGeneratorSpec.scala       |  4 +-
 .../ergoplatform/mining/ErgoMinerSpec.scala   |  4 +-
 .../mempool/ErgoTransactionSpec.scala         |  8 +--
 .../mempool/ExpirationSpecification.scala     |  1 +
 .../HeaderSerializationSpecification.scala    |  2 +-
 .../extra/ExtraIndexerSpecification.scala     |  4 +-
 .../nodeView/mempool/ScriptsSpec.scala        |  4 +-
 .../state/UtxoStateSpecification.scala        |  2 +-
 .../nodeView/wallet/ErgoWalletSpec.scala      |  2 +-
 .../reemission/ReemissionRulesSpec.scala      |  3 +-
 .../serialization/JsonSerializationSpec.scala |  4 +-
 .../ergoplatform/tools/ChainGenerator.scala   |  2 +-
 .../org/ergoplatform/tools/FeeSimulator.scala |  3 +-
 .../utils/ErgoTestConstants.scala             |  4 +-
 .../scala/org/ergoplatform/utils/Stubs.scala  |  2 +-
 .../ergoplatform/utils/WalletTestOps.scala    |  5 +-
 .../utils/generators/ChainGenerator.scala     |  3 +-
 .../utils/generators/ErgoGenerators.scala     |  6 +--
 .../ErgoTransactionGenerators.scala           |  2 +-
 66 files changed, 162 insertions(+), 136 deletions(-)

diff --git a/build.sbt b/build.sbt
index b6116b4644..dc5c2e79f4 100644
--- a/build.sbt
+++ b/build.sbt
@@ -37,7 +37,7 @@ val circeVersion = "0.13.0"
 val akkaVersion = "2.6.10"
 val akkaHttpVersion = "10.2.4"
 
-val sigmaStateVersion = "5.0.10"
+val sigmaStateVersion = "5.0.11-37-2750749c-SNAPSHOT"
 
 // for testing current sigmastate build (see sigmastate-ergo-it jenkins job)
 val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion)
diff --git a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java
index 4493e490e5..f12f4cb457 100644
--- a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java
+++ b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java
@@ -3,7 +3,7 @@
 import org.ergoplatform.ErgoLikeTransaction;
 import org.ergoplatform.UnsignedErgoLikeTransaction;
 import scala.collection.JavaConverters;
-import sigmastate.basics.DLogProtocol;
+import sigmastate.crypto.DLogProtocol;
 import java.util.Map;
 
 /**
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala b/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala
index 55a70b4016..7569b8c00c 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala
@@ -8,7 +8,7 @@ import org.ergoplatform.{Height, MinerPubkey, Outputs, Self}
 import sigmastate.Values.{ByteArrayConstant, ErgoTree, IntConstant, LongConstant, SigmaPropValue, Value}
 import sigmastate.utxo._
 import sigmastate._
-import special.collection.Coll
+import sigma.Coll
 
 /**
   * Container for re-emission related contracts. Contains re-emission contract and pay-to-reemission contract.
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala
index 8c2fdf5384..34dcdc1c74 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala
@@ -2,7 +2,7 @@ package org.ergoplatform.wallet.boxes
 
 import java7.compat.Math
 import org.ergoplatform.ErgoBoxCandidate
-import special.collection.Extensions._
+import sigma.Extensions._
 
 import scala.collection.compat.immutable.ArraySeq
 import scala.collection.mutable
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala
index 2cfec5ad29..472b54e9f8 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala
@@ -8,7 +8,7 @@ import org.ergoplatform.wallet.serialization.ErgoWalletSerializer
 import org.ergoplatform.{ErgoBox, ErgoBoxAssets, ErgoLikeTransaction}
 import scorex.util.serialization.{Reader, Writer}
 import scorex.util.{ModifierId, bytesToId, idToBytes}
-import special.collection.Extensions._
+import sigma.Extensions._
 
 /**
   * A box tracked by a wallet that contains Ergo box itself as well as
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala
index 03f144ea2d..430fc8c894 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala
@@ -3,7 +3,7 @@ package org.ergoplatform.wallet.crypto
 import org.bouncycastle.util.BigIntegers
 import scorex.crypto.hash.Blake2b256
 import scorex.util.encode.Base16
-import sigmastate.basics.CryptoConstants
+import sigmastate.crypto.CryptoConstants
 import sigmastate.serialization.GroupElementSerializer
 
 import scala.annotation.tailrec
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala
index e429ecbdad..bdd423615f 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala
@@ -1,14 +1,14 @@
 package org.ergoplatform.wallet.interpreter
 
 import org.ergoplatform.ErgoLikeContext.Height
-import org.ergoplatform.sdk.wallet.protocol.context.ErgoLikeParameters
+import org.ergoplatform.sdk.BlockchainParameters
 import org.ergoplatform.wallet.protocol.Constants
 import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoLikeContext, ErgoLikeInterpreter}
 import scorex.util.ScorexLogging
 import sigmastate.Values.ErgoTree
 import sigmastate.interpreter.Interpreter.{ScriptEnv, VerificationResult}
 import sigmastate.{AvlTreeData, AvlTreeFlags}
-import special.collection.Coll
+import sigma.Coll
 
 import scala.util.Try
 
@@ -18,7 +18,7 @@ import scala.util.Try
   *
   * @param params - current values of adjustable blockchain settings
   */
-class ErgoInterpreter(params: ErgoLikeParameters)
+class ErgoInterpreter(params: BlockchainParameters)
   extends ErgoLikeInterpreter with ScorexLogging {
 
   /** Override default logging for all Ergo interpreters. */
@@ -96,7 +96,7 @@ object ErgoInterpreter {
   val interpreterInitCost = 10000
 
   /** Creates an interpreter with the given parameters. */
-  def apply(params: ErgoLikeParameters): ErgoInterpreter =
+  def apply(params: BlockchainParameters): ErgoInterpreter =
     new ErgoInterpreter(params)
 
   /** Create [[AvlTreeData]] with the given digest and all operations enabled. */
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala
index 4d1d62ed6f..62732ab788 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala
@@ -1,19 +1,18 @@
 package org.ergoplatform.wallet.interpreter
 
 import org.ergoplatform._
+import org.ergoplatform.sdk.BlockchainParameters
 import org.ergoplatform.sdk.utils.ArithUtils.{addExact, multiplyExact}
-import org.ergoplatform.sdk.wallet.protocol.context.{ErgoLikeParameters, ErgoLikeStateContext}
+import org.ergoplatform.sdk.wallet.protocol.context.BlockchainStateContext
 import org.ergoplatform.sdk.wallet.secrets.{ExtendedPublicKey, ExtendedSecretKey, SecretKey}
 import org.ergoplatform.validation.{SigmaValidationSettings, ValidationRules}
 import org.ergoplatform.wallet.boxes.ErgoBoxAssetExtractor
-import scorex.crypto.authds.ADDigest
 import scorex.util.encode.Base16
 import sigmastate.AvlTreeData
 import sigmastate.Values.SigmaBoolean
-import sigmastate.basics.SigmaProtocolPrivateInput
+import sigmastate.crypto.SigmaProtocolPrivateInput
 import sigmastate.interpreter.{ContextExtension, ProverInterpreter}
-import special.collection.Coll
-import special.sigma.{Header, PreHeader}
+import sigma.{Coll, Header, PreHeader}
 
 import java.util
 import scala.util.{Failure, Success, Try}
@@ -39,14 +38,14 @@ import scala.util.{Failure, Success, Try}
   *                           (to not to recompute them)
   */
 class ErgoProvingInterpreter(val secretKeys: IndexedSeq[SecretKey],
-                             params: ErgoLikeParameters,
+                             params: BlockchainParameters,
                              val cachedHdPubKeysOpt: Option[IndexedSeq[ExtendedPublicKey]] = None)
   extends ErgoInterpreter(params) with ProverInterpreter {
 
   /**
     * Interpreter's secrets, in form of sigma protocols private inputs
     */
-  val secrets: IndexedSeq[SigmaProtocolPrivateInput[_, _]] = secretKeys.map(_.privateInput)
+  val secrets: IndexedSeq[SigmaProtocolPrivateInput[_]] = secretKeys.map(_.privateInput)
 
   /**
     * Only secrets corresponding to hierarchical deterministic scheme (BIP-32 impl)
@@ -93,14 +92,14 @@ class ErgoProvingInterpreter(val secretKeys: IndexedSeq[SecretKey],
     * @param newParams - updated parameters
     * @return modified prover
     */
-  def withNewParameters(newParams: ErgoLikeParameters): ErgoProvingInterpreter = {
+  def withNewParameters(newParams: BlockchainParameters): ErgoProvingInterpreter = {
     new ErgoProvingInterpreter(this.secretKeys, newParams, this.cachedHdPubKeysOpt)
   }
 
   def signInputs(unsignedTx: UnsignedErgoLikeTransaction,
                  boxesToSpend: IndexedSeq[ErgoBox],
                  dataBoxes: IndexedSeq[ErgoBox],
-                 stateContext: ErgoLikeStateContext,
+                 stateContext: BlockchainStateContext,
                  txHints: TransactionHintsBag): Try[(IndexedSeq[Input], Long)] = {
     if (unsignedTx.inputs.length != boxesToSpend.length) {
       Failure(new Exception("Not enough boxes to spend"))
@@ -164,7 +163,7 @@ class ErgoProvingInterpreter(val secretKeys: IndexedSeq[SecretKey],
   def sign(unsignedTx: UnsignedErgoLikeTransaction,
            boxesToSpend: IndexedSeq[ErgoBox],
            dataBoxes: IndexedSeq[ErgoBox],
-           stateContext: ErgoLikeStateContext,
+           stateContext: BlockchainStateContext,
            txHints: TransactionHintsBag = TransactionHintsBag.empty): Try[ErgoLikeTransaction] = {
 
     val signedInputs: Try[(IndexedSeq[Input], Long)] =
@@ -190,7 +189,7 @@ class ErgoProvingInterpreter(val secretKeys: IndexedSeq[SecretKey],
   def generateCommitmentsFor(unsignedTx: UnsignedErgoLikeTransaction,
                              boxesToSpend: IndexedSeq[ErgoBox],
                              dataBoxes: IndexedSeq[ErgoBox],
-                             stateContext: ErgoLikeStateContext): Try[TransactionHintsBag] = Try {
+                             stateContext: BlockchainStateContext): Try[TransactionHintsBag] = Try {
     val inputCmts = unsignedTx.inputs.zipWithIndex.map { case (unsignedInput, inpIndex) =>
 
       val inputBox = boxesToSpend(inpIndex)
@@ -230,7 +229,7 @@ class ErgoProvingInterpreter(val secretKeys: IndexedSeq[SecretKey],
   def bagForTransaction(tx: ErgoLikeTransaction,
                         boxesToSpend: IndexedSeq[ErgoBox],
                         dataBoxes: IndexedSeq[ErgoBox],
-                        stateContext: ErgoLikeStateContext,
+                        stateContext: BlockchainStateContext,
                         realSecretsToExtract: Seq[SigmaBoolean],
                         simulatedSecretsToExtract: Seq[SigmaBoolean]): TransactionHintsBag = {
     val augmentedInputs = tx.inputs.zipWithIndex.zip(boxesToSpend)
@@ -262,11 +261,11 @@ class ErgoProvingInterpreter(val secretKeys: IndexedSeq[SecretKey],
 object ErgoProvingInterpreter {
 
   def apply(secrets: IndexedSeq[SecretKey],
-            params: ErgoLikeParameters): ErgoProvingInterpreter =
+            params: BlockchainParameters): ErgoProvingInterpreter =
     new ErgoProvingInterpreter(secrets, params)
 
   def apply(rootSecret: ExtendedSecretKey,
-            params: ErgoLikeParameters): ErgoProvingInterpreter =
+            params: BlockchainParameters): ErgoProvingInterpreter =
     new ErgoProvingInterpreter(IndexedSeq(rootSecret), params)
 
 }
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProver.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProver.scala
index ad1f780404..c2b763cb65 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProver.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProver.scala
@@ -2,7 +2,7 @@ package org.ergoplatform.wallet.interpreter
 
 import org.ergoplatform.{ErgoLikeTransaction, Input, UnsignedErgoLikeTransaction}
 import scorex.util.encode.Base16
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 import sigmastate.interpreter.{ContextExtension, ProverResult}
 
 /**
diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala
index 6ad66b5423..745002eacf 100644
--- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala
+++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala
@@ -10,8 +10,8 @@ import scorex.util.{ModifierId, bytesToId}
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
 import sigmastate.utils.Extensions._
-import special.collection.Coll
-import special.collection.Extensions._
+import sigma.Coll
+import sigma.Extensions._
 
 import scala.collection.JavaConverters._
 import scala.util.Try
diff --git a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java
index 11ac65f6b0..70b39eea0e 100644
--- a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java
+++ b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java
@@ -8,7 +8,7 @@
 import org.ergoplatform.wallet.transactions.TransactionBuilder;
 import org.ergoplatform.wallet.transactions.TransactionBuilder.Payment;
 import scorex.util.Random;
-import sigmastate.basics.DLogProtocol;
+import sigmastate.crypto.DLogProtocol;
 
 import java.util.ArrayList;
 import java.util.HashMap;
diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala
index 311275b06f..f34f050501 100644
--- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala
+++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala
@@ -16,7 +16,7 @@ import sigmastate.Values.SigmaPropValue
 import sigmastate.eval.Extensions._
 import sigmastate.helpers.TestingHelpers._
 import sigmastate.utils.Extensions._
-import special.collection.Extensions._
+import sigma.Extensions._
 
 import scala.util.Random
 
diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/crypto/ErgoSignatureSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/crypto/ErgoSignatureSpec.scala
index 42993ec9df..5c35807d5e 100644
--- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/crypto/ErgoSignatureSpec.scala
+++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/crypto/ErgoSignatureSpec.scala
@@ -4,7 +4,7 @@ import org.ergoplatform.wallet.utils.Generators
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.propspec.AnyPropSpec
 import scorex.util.Random
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 
 class ErgoSignatureSpec extends AnyPropSpec with Matchers with Generators {
 
diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala
index 0d96c4c8cc..e821987d01 100644
--- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala
+++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala
@@ -9,6 +9,7 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
 import scorex.util.{ModifierId, Random}
 import scorex.util.encode.Base16
+import sigma.Colls
 import sigmastate.CTHRESHOLD
 import sigmastate.Values.{GroupElementConstant, SigmaBoolean}
 import sigmastate.interpreter.{ContextExtension, HintsBag}
diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala
index 484c7fb47a..5f797a27d2 100644
--- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala
+++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala
@@ -1,16 +1,16 @@
 package org.ergoplatform.wallet.interpreter
 
-import org.ergoplatform.sdk.wallet.protocol.context.{ErgoLikeParameters, ErgoLikeStateContext}
+import org.ergoplatform.sdk.BlockchainParameters
+import org.ergoplatform.sdk.wallet.protocol.context.BlockchainStateContext
 import scorex.util.encode.Base16
-import sigmastate.basics.CryptoConstants
+import sigmastate.crypto.CryptoConstants
 import sigmastate.eval.Extensions.ArrayOps
-import sigmastate.eval.{CGroupElement, CPreHeader, Colls}
-import special.collection.Coll
-import special.sigma.{Header, PreHeader}
+import sigmastate.eval.{CGroupElement, CPreHeader}
+import sigma.{Coll, Colls, Header, PreHeader}
 
 trait InterpreterSpecCommon {
 
-  protected val parameters = new ErgoLikeParameters {
+  protected val parameters = new BlockchainParameters {
 
     override def storageFeeFactor: Int = 1250000
 
@@ -35,7 +35,7 @@ trait InterpreterSpecCommon {
     override def blockVersion: Byte = 1
   }
 
-  protected val stateContext = new ErgoLikeStateContext {
+  protected val stateContext = new BlockchainStateContext {
 
     override def sigmaLastHeaders: Coll[Header] = Colls.emptyColl
 
diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala
index fc8d6d3ab3..8ba31c3c27 100644
--- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala
+++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala
@@ -13,7 +13,7 @@ import org.scalacheck.{Arbitrary, Gen}
 import scorex.crypto.authds.ADKey
 import scorex.util._
 import sigmastate.Values.{ByteArrayConstant, CollectionConstant, ErgoTree, EvaluatedValue, FalseLeaf, TrueLeaf}
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.crypto.CryptoFacade.SecretKeyLength
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
diff --git a/src/it/scala/org/ergoplatform/it/WalletSpec.scala b/src/it/scala/org/ergoplatform/it/WalletSpec.scala
index a3c6b4012a..3d22140ea0 100644
--- a/src/it/scala/org/ergoplatform/it/WalletSpec.scala
+++ b/src/it/scala/org/ergoplatform/it/WalletSpec.scala
@@ -20,6 +20,7 @@ import org.ergoplatform.{ErgoBox, P2PKAddress}
 import org.scalatest.wordspec.AsyncWordSpec
 import scorex.util.ModifierId
 import scorex.util.encode.Base16
+import sigma.Colls
 import sigmastate.Values.{ErgoTree, TrueLeaf}
 
 import scala.concurrent.ExecutionContext
diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala
index f293daa633..424c279c99 100644
--- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala
@@ -27,16 +27,17 @@ import scorex.crypto.hash.Digest
 import scorex.util.encode.Base16
 import sigmastate.Values.SigmaBoolean
 import sigmastate._
-import sigmastate.basics.CryptoConstants.EcPointType
-import sigmastate.basics.DLogProtocol.{DLogProverInput, FirstDLogProverMessage, ProveDlog}
-import sigmastate.basics.VerifierMessage.Challenge
-import sigmastate.basics._
+import sigmastate.crypto.CryptoConstants.EcPointType
+import sigmastate.crypto.DLogProtocol.{DLogProverInput, FirstDLogProverMessage, ProveDlog}
+import sigmastate.crypto.VerifierMessage.Challenge
+import sigmastate.crypto._
 import sigmastate.interpreter._
 import sigmastate.serialization.OpCodes
-import special.sigma.AnyValue
+import sigma.AnyValue
 import org.ergoplatform.nodeView.state.SnapshotsInfo
 import org.ergoplatform.nodeView.state.UtxoState.ManifestId
 import org.ergoplatform.sdk.JsonCodecs
+import sigmastate.eval.Extensions.ArrayOps
 
 import java.math.BigInteger
 import scala.util.{Failure, Success, Try}
@@ -200,7 +201,14 @@ trait ApiCodecs extends JsonCodecs {
     } yield ErgoTransaction(ergoLikeTx)
   }
 
-
+  implicit val sigmaLeafEncoder: Encoder[SigmaLeaf] = {
+    leaf =>
+      val op = leaf.opCode.toByte.asJson
+      leaf match {
+        case dlog: ProveDlog => Map("op" -> op, "h" -> dlog.value.asJson).asJson
+        case dht: ProveDHTuple => Map("op" -> op, "g" -> dht.g.asJson, "h" -> dht.h.asJson, "u" -> dht.u.asJson, "v" -> dht.v.asJson).asJson
+      }
+  }
 
   implicit val sigmaBooleanEncoder: Encoder[SigmaBoolean] = {
     sigma =>
@@ -218,6 +226,16 @@ trait ApiCodecs extends JsonCodecs {
       }
   }
 
+  implicit val sigmaLeafDecoder: Decoder[SigmaLeaf] = Decoder.instance { c =>
+    c.downField("op").as[Byte].flatMap {
+      case b: Byte if b == OpCodes.ProveDlogCode =>
+        c.downField("h").as[EcPointType].map(h => ProveDlog(h))
+      case _ =>
+        //only dlog is supported for now
+        Left(DecodingFailure("Unsupported value", List()))
+    }
+  }
+
   implicit val sigmaBooleanDecoder: Decoder[SigmaBoolean] = Decoder.instance { c =>
     c.downField("op").as[Byte].flatMap {
       case b: Byte if b == OpCodes.ProveDlogCode =>
@@ -251,7 +269,7 @@ trait ApiCodecs extends JsonCodecs {
   implicit val firstProverMessageEncoder: Encoder[FirstProverMessage] = {
     case cmtDlog: FirstDLogProverMessage =>
       Json.obj("type" -> "dlog".asJson, "a" -> cmtDlog.ecData.asJson)
-    case cmtDht: FirstDiffieHellmanTupleProverMessage =>
+    case cmtDht: FirstDHTupleProverMessage =>
       Json.obj("type" -> "dht".asJson, "a" -> cmtDht.a.asJson, "b" -> cmtDht.b.asJson)
     case _ => ???
   }
@@ -266,7 +284,7 @@ trait ApiCodecs extends JsonCodecs {
         for {
           a <- c.downField("a").as[EcPointType]
           b <- c.downField("b").as[EcPointType]
-        } yield FirstDiffieHellmanTupleProverMessage(a, b)
+        } yield FirstDHTupleProverMessage(a, b)
       case _ =>
         Left(DecodingFailure("Unsupported sigma-protocol type value", List()))
     }
@@ -304,20 +322,20 @@ trait ApiCodecs extends JsonCodecs {
       case h: String if h == "cmtWithSecret" =>
         for {
           secret <- c.downField("secret").as[BigInteger]
-          pubkey <- c.downField("pubkey").as[SigmaBoolean]
+          pubkey <- c.downField("pubkey").as[SigmaLeaf]
           position <- c.downField("position").as[NodePosition]
           firstMsg <- firstProverMessageDecoder.tryDecode(c)
         } yield OwnCommitment(pubkey, secret, firstMsg, position)
       case h: String if h == "cmtReal" =>
         for {
-          pubkey <- c.downField("pubkey").as[SigmaBoolean]
+          pubkey <- c.downField("pubkey").as[SigmaLeaf]
           position <- c.downField("position").as[NodePosition]
           firstMsg <- firstProverMessageDecoder.tryDecode(c)
         } yield RealCommitment(pubkey, firstMsg, position)
       case h: String if h == "cmtSimulated" =>
         for {
           position <- c.downField("position").as[NodePosition]
-          pubkey <- c.downField("pubkey").as[SigmaBoolean]
+          pubkey <- c.downField("pubkey").as[SigmaLeaf]
           firstMsg <- firstProverMessageDecoder.tryDecode(c)
         } yield SimulatedCommitment(pubkey, firstMsg, position)
       case _ =>
@@ -334,7 +352,7 @@ trait ApiCodecs extends JsonCodecs {
 
     Json.obj(
       "hint" -> proofType.asJson,
-      "challenge" -> Base16.encode(sp.challenge).asJson,
+      "challenge" -> Base16.encode(sp.challenge.toArray).asJson,
       "pubkey" -> sp.image.asJson,
       "proof" -> SigSerializer.toProofBytes(sp.uncheckedTree).asJson,
       "position" -> sp.position.asJson
@@ -346,26 +364,26 @@ trait ApiCodecs extends JsonCodecs {
       case h: String if h == "proofReal" =>
         for {
           challenge <- c.downField("challenge").as[String]
-          pubkey <- c.downField("pubkey").as[SigmaBoolean]
+          pubkey <- c.downField("pubkey").as[SigmaLeaf]
           proof <- c.downField("proof").as[String]
           position <- c.downField("position").as[NodePosition]
         } yield
           RealSecretProof(
             pubkey,
-            Challenge @@ Base16.decode(challenge).get,
+            Challenge @@ Base16.decode(challenge).get.toColl,
             SigSerializer.parseAndComputeChallenges(pubkey, Base16.decode(proof).get)(null),
             position
           )
       case h: String if h == "proofSimulated" =>
         for {
           challenge <- c.downField("challenge").as[String]
-          pubkey <- c.downField("pubkey").as[SigmaBoolean]
+          pubkey <- c.downField("pubkey").as[SigmaLeaf]
           proof <- c.downField("proof").as[String]
           position <- c.downField("position").as[NodePosition]
         } yield
           SimulatedSecretProof(
             pubkey,
-            Challenge @@ Base16.decode(challenge).get,
+            Challenge @@ Base16.decode(challenge).get.toColl,
             SigSerializer.parseAndComputeChallenges(pubkey, Base16.decode(proof).get)(null),
             position
           )
diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala
index 5715e4d5aa..8684526663 100644
--- a/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala
@@ -13,7 +13,7 @@ import scorex.core.settings.RESTApiSettings
 import scorex.core.utils.ScorexEncoding
 import scorex.crypto.hash.Blake2b256
 import scorex.util.encode.Base16
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 
 import java.security.SecureRandom
 import scala.util.Failure
diff --git a/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala
index 03e7d543a2..c9ca7c1b8c 100644
--- a/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala
@@ -13,7 +13,7 @@ import org.ergoplatform.settings.ErgoSettings
 import org.ergoplatform.{ErgoAddress, ErgoTreePredef, Pay2SAddress}
 import scorex.core.api.http.ApiResponse
 import scorex.core.settings.RESTApiSettings
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 
 import scala.concurrent.Future
 
diff --git a/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
index d7a0bf8f37..87e20bfe74 100644
--- a/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ScriptApiRoute.scala
@@ -17,7 +17,7 @@ import scorex.core.settings.RESTApiSettings
 import scorex.util.encode.Base16
 import sigmastate.Values.{ByteArrayConstant, ErgoTree}
 import sigmastate._
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.eval.CompiletimeIRContext
 import sigmastate.interpreter.Interpreter
 import sigmastate.lang.{CompilerResult, SigmaCompiler}
diff --git a/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala b/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala
index 858bc51d7b..ad5af554a6 100644
--- a/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala
+++ b/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala
@@ -15,7 +15,7 @@ import org.ergoplatform.nodeView.mempool.TransactionMembershipProof
 import scorex.crypto.authds.{ADDigest, SerializedAdProof}
 import scorex.crypto.hash.{Blake2b256, Digest32}
 import scorex.util.{ModifierId, ScorexLogging}
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.crypto.CryptoFacade
 
 import scala.annotation.tailrec
diff --git a/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala b/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala
index 13917b06a7..adeff25edc 100644
--- a/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala
+++ b/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala
@@ -8,8 +8,8 @@ import org.ergoplatform.modifiers.history.header.Header.Version
 import org.ergoplatform.settings.Algos
 import scorex.core.serialization.ErgoSerializer
 import scorex.util.serialization.{Reader, Writer}
-import sigmastate.basics.CryptoConstants
-import sigmastate.basics.CryptoConstants.EcPointType
+import sigmastate.crypto.CryptoConstants
+import sigmastate.crypto.CryptoConstants.EcPointType
 
 /**
   * Solution for an Autolykos PoW puzzle.
diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala
index 61fee2f123..5f25ee8d33 100644
--- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala
+++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala
@@ -26,13 +26,13 @@ import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoTreePredef, Input}
 import scorex.crypto.hash.Digest32
 import scorex.util.encode.Base16
 import scorex.util.{ModifierId, ScorexLogging}
-import sigmastate.SType.ErgoBoxRType
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.ErgoBoxRType
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.crypto.CryptoFacade
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
 import sigmastate.interpreter.ProverResult
-import special.collection.Coll
+import sigma.{Coll, Colls}
 
 import scala.annotation.tailrec
 import scala.concurrent.duration._
diff --git a/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala b/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala
index f318a4f426..732bf25805 100644
--- a/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala
+++ b/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala
@@ -3,7 +3,7 @@ package org.ergoplatform.mining
 import org.ergoplatform.modifiers.history.header.Header
 import scorex.crypto.authds.ADDigest
 import scorex.crypto.hash.Digest32
-import sigmastate.basics.CryptoConstants.EcPointType
+import sigmastate.crypto.CryptoConstants.EcPointType
 
 import scala.util.{Random, Success, Try}
 
diff --git a/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala b/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala
index 887845cd20..4379ebd26b 100644
--- a/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala
+++ b/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala
@@ -10,7 +10,7 @@ import org.ergoplatform.settings.ErgoSettings
 import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.GetDataFromCurrentView
 import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied
 import scorex.util.ScorexLogging
-import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog}
+import sigmastate.crypto.DLogProtocol.{DLogProverInput, ProveDlog}
 
 import scala.concurrent.duration._
 import scala.util.{Failure, Success}
diff --git a/src/main/scala/org/ergoplatform/mining/WorkMessage.scala b/src/main/scala/org/ergoplatform/mining/WorkMessage.scala
index 19d758d961..3c815d44fc 100644
--- a/src/main/scala/org/ergoplatform/mining/WorkMessage.scala
+++ b/src/main/scala/org/ergoplatform/mining/WorkMessage.scala
@@ -4,7 +4,7 @@ import io.circe.syntax._
 import io.circe.{Encoder, Json}
 import org.ergoplatform.http.api.ApiCodecs
 import org.ergoplatform.nodeView.history.ErgoHistory.Height
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 
 
 /**
diff --git a/src/main/scala/org/ergoplatform/mining/mining.scala b/src/main/scala/org/ergoplatform/mining/mining.scala
index 3fc4be9f3e..526d2eeba2 100644
--- a/src/main/scala/org/ergoplatform/mining/mining.scala
+++ b/src/main/scala/org/ergoplatform/mining/mining.scala
@@ -2,9 +2,9 @@ package org.ergoplatform
 
 import org.bouncycastle.util.BigIntegers
 import scorex.crypto.hash.Blake2b256
-import sigmastate.basics.CryptoConstants.EcPointType
-import sigmastate.basics.{BcDlogGroup, CryptoConstants}
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigmastate.crypto.CryptoConstants.EcPointType
+import sigmastate.crypto.{BcDlogGroup, CryptoConstants}
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 import sigmastate.serialization.{GroupElementSerializer, SigmaSerializer}
 
 package object mining {
diff --git a/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala b/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala
index 8387a14291..da183f33eb 100644
--- a/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala
+++ b/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala
@@ -6,7 +6,7 @@ import org.ergoplatform.modifiers.history.header.Header._
 import org.ergoplatform.nodeView.history.ErgoHistory
 import org.ergoplatform.settings.Constants
 import scorex.util._
-import sigmastate.basics.CryptoConstants.EcPointType
+import sigmastate.crypto.CryptoConstants.EcPointType
 import sigmastate.eval.CGroupElement
 import sigmastate.eval.Extensions._
 
@@ -35,7 +35,7 @@ case class CPreHeader(version: Version,
 
 object PreHeader {
 
-  def toSigma(preHeader: PreHeader): special.sigma.PreHeader =
+  def toSigma(preHeader: PreHeader): sigma.PreHeader =
     sigmastate.eval.CPreHeader(
       version = preHeader.version,
       parentId = preHeader.parentId.toBytes.toColl,
diff --git a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala
index cd15b11ae7..df31f80686 100644
--- a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala
+++ b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala
@@ -16,7 +16,7 @@ import scorex.core.serialization.ErgoSerializer
 import scorex.crypto.authds.ADDigest
 import scorex.crypto.hash.Digest32
 import scorex.util._
-import sigmastate.basics.CryptoConstants.EcPointType
+import sigmastate.crypto.CryptoConstants.EcPointType
 import sigmastate.eval.Extensions._
 import sigmastate.eval.{CAvlTree, CBigInt, CGroupElement, CHeader}
 
@@ -136,7 +136,7 @@ object Header extends ApiCodecs {
   val Interpreter50Version: Byte = 3
 
 
-  def toSigma(header: Header): special.sigma.Header =
+  def toSigma(header: Header): sigma.Header =
     CHeader(
       id = header.id.toBytes.toColl,
       version = header.version,
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala
index a6002c1b0c..67a6fb1a1d 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala
@@ -8,7 +8,7 @@ import scorex.core.serialization.ErgoSerializer
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
 import scorex.util.serialization.{Reader, Writer}
 import spire.implicits.cfor
-import special.collection.Extensions._
+import sigma.Extensions._
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
index 19c6d18d8e..c87a4fc600 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
@@ -10,7 +10,7 @@ import scorex.util.{ByteArrayOps, ModifierId, bytesToId}
 import scorex.util.serialization.{Reader, Writer}
 import sigmastate.Values.CollectionConstant
 import sigmastate.SByte
-import special.collection.Extensions._
+import sigma.Extensions._
 
 /**
   * Index of a token containing creation information.
diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala
index dc3644897f..4290d96ecc 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala
@@ -24,7 +24,7 @@ import scorex.util.encode.Base16
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
 import sigmastate.AtLeast
 import sigmastate.Values.{ByteArrayConstant, ErgoTree, IntConstant, SigmaPropConstant}
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.serialization.ValueSerializer
 import spire.syntax.all.cfor
 
diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
index 83f302a3b5..82fe0ff772 100644
--- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala
@@ -8,7 +8,7 @@ import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer}
 import org.ergoplatform.modifiers.history.popow.NipopowAlgos
 import org.ergoplatform.nodeView.history.ErgoHistory
 import org.ergoplatform.nodeView.history.storage.modifierprocessors.ExtensionValidator
-import org.ergoplatform.sdk.wallet.protocol.context.ErgoLikeStateContext
+import org.ergoplatform.sdk.wallet.protocol.context.BlockchainStateContext
 import org.ergoplatform.settings.ValidationRules._
 import org.ergoplatform.settings._
 import scorex.core.serialization.{BytesSerializable, ErgoSerializer}
@@ -17,10 +17,10 @@ import scorex.core.validation.{InvalidModifier, ModifierValidator, ValidationSta
 import scorex.crypto.authds.ADDigest
 import scorex.util.ScorexLogging
 import scorex.util.serialization.{Reader, Writer}
-import sigmastate.basics.CryptoConstants.EcPointType
+import sigmastate.crypto.CryptoConstants.EcPointType
 import sigmastate.eval.Extensions.ArrayOps
 import sigmastate.eval.SigmaDsl
-import special.collection.Coll
+import sigma.Coll
 
 import scala.util.{Failure, Success, Try}
 
@@ -38,9 +38,9 @@ case class UpcomingStateContext(override val lastHeaders: Seq[Header],
   extends ErgoStateContext(lastHeaders, lastExtensionOpt, genesisStateDigest, currentParameters,
                             validationSettings, votingData)(ergoSettings) {
 
-  override def sigmaPreHeader: special.sigma.PreHeader = PreHeader.toSigma(predictedHeader)
+  override def sigmaPreHeader: sigma.PreHeader = PreHeader.toSigma(predictedHeader)
 
-  override def sigmaLastHeaders: Coll[special.sigma.Header] = {
+  override def sigmaLastHeaders: Coll[sigma.Header] = {
     SigmaDsl.Colls.fromArray(lastHeaders.map(h => Header.toSigma(h)).toArray)
   }
 
@@ -66,7 +66,7 @@ class ErgoStateContext(val lastHeaders: Seq[Header],
                        val validationSettings: ErgoValidationSettings,
                        val votingData: VotingData)
                       (implicit val ergoSettings: ErgoSettings)
-  extends ErgoLikeStateContext
+  extends BlockchainStateContext
     with BytesSerializable
     with ScorexEncoding
     with ScorexLogging {
@@ -76,10 +76,10 @@ class ErgoStateContext(val lastHeaders: Seq[Header],
   private val votingSettings = ergoSettings.chainSettings.voting
   private val popowAlgos = new NipopowAlgos(ergoSettings.chainSettings)
 
-  override def sigmaPreHeader: special.sigma.PreHeader =
+  override def sigmaPreHeader: sigma.PreHeader =
     PreHeader.toSigma(lastHeaders.headOption.getOrElse(PreHeader.fake))
 
-  override def sigmaLastHeaders: Coll[special.sigma.Header] =
+  override def sigmaLastHeaders: Coll[sigma.Header] =
     SigmaDsl.Colls.fromArray(lastHeaders.drop(1).map(h => Header.toSigma(h)).toArray)
 
   // todo remove from ErgoLikeContext and from ErgoStateContext
diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala
index 0d9732964d..3c1594e091 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala
@@ -25,7 +25,7 @@ import scorex.core.VersionTag
 import scorex.core.utils.ScorexEncoding
 import scorex.util.{ModifierId, ScorexLogging}
 import sigmastate.Values.SigmaBoolean
-import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog}
+import sigmastate.crypto.DLogProtocol.{DLogProverInput, ProveDlog}
 
 import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala
index dab82fe87b..fd2d38ba76 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala
@@ -20,7 +20,7 @@ import org.ergoplatform.{ErgoBox, P2PKAddress}
 import scorex.core.NodeViewComponent
 import scorex.util.ModifierId
 import sigmastate.Values.SigmaBoolean
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 
 import java.util.concurrent.TimeUnit
 import scala.concurrent.Future
diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala
index 34c82fcded..7ff6ab2d72 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala
@@ -24,8 +24,8 @@ import org.ergoplatform.wallet.utils.FileUtils
 import scorex.util.encode.Base16
 import scorex.util.{ModifierId}
 import sigmastate.Values.SigmaBoolean
-import sigmastate.basics.DLogProtocol.DLogProverInput
-import special.collection.Extensions.CollBytesOps
+import sigmastate.crypto.DLogProtocol.DLogProverInput
+import sigma.Extensions.CollBytesOps
 
 import java.io.FileNotFoundException
 import scala.collection.compat.immutable.ArraySeq
diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala
index 669f69da39..247c6cd116 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala
@@ -20,8 +20,9 @@ import org.ergoplatform.wallet.interpreter.ErgoProvingInterpreter
 import org.ergoplatform.wallet.mnemonic.Mnemonic
 import org.ergoplatform.wallet.transactions.TransactionBuilder
 import scorex.util.ScorexLogging
+import sigma.Colls
 import sigmastate.Values.ByteArrayConstant
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
 import sigmastate.utils.Extensions._
diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala
index 402475a8aa..133c1736ef 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala
@@ -3,7 +3,7 @@ package org.ergoplatform.nodeView.wallet.scanning
 import org.ergoplatform.ErgoBox
 import sigmastate.Values.EvaluatedValue
 import sigmastate.{SType, Values}
-import special.collection.Extensions._
+import sigma.Extensions._
 
 /**
   * Basic interface for box scanning predicate functionality
diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala
index e8305fb331..c57b4923ff 100644
--- a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala
@@ -7,7 +7,7 @@ import org.ergoplatform.ErgoBox.RegisterId
 import org.ergoplatform.http.api.ApiCodecs
 import sigmastate.SType
 import sigmastate.Values.EvaluatedValue
-import special.collection.Extensions._
+import sigma.Extensions._
 
 
 object ScanningPredicateJsonCodecs extends ApiCodecs {
diff --git a/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala b/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala
index 7dc87a66b0..2660693fac 100644
--- a/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala
+++ b/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala
@@ -9,7 +9,7 @@ import sigmastate.SBoolean
 import sigmastate.Values.Value
 import sigmastate.eval.CompiletimeIRContext
 import sigmastate.lang.{CompilerSettings, SigmaCompiler, TransformingSigmaBuilder}
-import special.collection.Coll
+import sigma.Coll
 
 import scala.util.Try
 
diff --git a/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala b/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala
index 5ceff6e42a..41bdd67a43 100644
--- a/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala
+++ b/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala
@@ -13,7 +13,7 @@ import org.ergoplatform.{ErgoAddressEncoder, ErgoApp, P2PKAddress}
 import scorex.core.settings.{ScorexSettings, SettingsReaders}
 import scorex.util.ScorexLogging
 import scorex.util.encode.Base16
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 
 import java.net.{InetAddress, URL}
 import scala.util.Try
diff --git a/src/main/scala/org/ergoplatform/settings/Parameters.scala b/src/main/scala/org/ergoplatform/settings/Parameters.scala
index f2f896335c..5a12fa813d 100644
--- a/src/main/scala/org/ergoplatform/settings/Parameters.scala
+++ b/src/main/scala/org/ergoplatform/settings/Parameters.scala
@@ -12,7 +12,7 @@ import scala.util.Try
 import org.ergoplatform.http.api.ApiCodecs
 import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionCandidate}
 import Extension.SystemParametersPrefix
-import org.ergoplatform.sdk.wallet.protocol.context.ErgoLikeParameters
+import org.ergoplatform.sdk.BlockchainParameters
 
 /**
   * System parameters which could be readjusted via collective miners decision.
@@ -20,7 +20,7 @@ import org.ergoplatform.sdk.wallet.protocol.context.ErgoLikeParameters
 class Parameters(val height: Height,
                  val parametersTable: Map[Byte, Int],
                  val proposedUpdate: ErgoValidationSettingsUpdate)
-  extends ErgoLikeParameters {
+  extends BlockchainParameters {
 
   import Parameters._
 
diff --git a/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala b/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala
index 1acc476597..aa869754a9 100644
--- a/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala
+++ b/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala
@@ -6,7 +6,7 @@ import org.ergoplatform.wallet.boxes.ErgoBoxSerializer
 import scorex.util.ModifierId
 import scorex.util.encode.Base16
 import sigmastate.utils.Extensions.ModifierIdOps
-import special.collection.Coll
+import sigma.Coll
 /**
   * Configuration section for re-emission (EIP27) parameters
   *
diff --git a/src/main/scala/org/ergoplatform/utils/BoxUtils.scala b/src/main/scala/org/ergoplatform/utils/BoxUtils.scala
index f6655aca85..37931c9c7f 100644
--- a/src/main/scala/org/ergoplatform/utils/BoxUtils.scala
+++ b/src/main/scala/org/ergoplatform/utils/BoxUtils.scala
@@ -1,13 +1,12 @@
 package org.ergoplatform.utils
 
-import org.ergoplatform.ErgoBox.{NonMandatoryRegisterId, TokenId}
-import org.ergoplatform.{ErgoBox, ErgoBoxCandidate}
+import org.ergoplatform.ErgoBox.{AdditionalRegisters, TokenId}
 import org.ergoplatform.settings.{Algos, Parameters}
+import org.ergoplatform.{ErgoBox, ErgoBoxCandidate}
 import scorex.util.ModifierId
-import sigmastate.SType
-import sigmastate.Values.{ErgoTree, EvaluatedValue}
+import sigma.{Coll, Colls}
+import sigmastate.Values.ErgoTree
 import sigmastate.eval._
-import special.collection.Coll
 
 object BoxUtils {
 
@@ -19,7 +18,7 @@ object BoxUtils {
   @inline
   def minimalErgoAmountSimulated(script: ErgoTree,
                                  tokens: Coll[(TokenId, Long)] = Colls.emptyColl,
-                                 additionalRegisters: Map[NonMandatoryRegisterId, _ <: EvaluatedValue[_ <: SType]] = Map(),
+                                 additionalRegisters: AdditionalRegisters = Map(),
                                  parameters: Parameters): Long = {
     val candidateMock = new ErgoBoxCandidate(value = Long.MaxValue, script, creationHeight = Int.MaxValue, tokens, additionalRegisters)
     val mockId = ModifierId @@ Algos.encode(scorex.util.Random.randomBytes(32))
diff --git a/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala
index c13bd6cb0c..1e7860907f 100644
--- a/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala
@@ -7,7 +7,7 @@ import akka.http.scaladsl.testkit.ScalatestRouteTest
 import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport
 import io.circe.Json
 import io.circe.syntax._
-import org.ergoplatform.ErgoBox.{NonMandatoryRegisterId, TokenId}
+import org.ergoplatform.ErgoBox.{AdditionalRegisters, NonMandatoryRegisterId, TokenId}
 import org.ergoplatform.http.api.{ApiCodecs, TransactionsApiRoute}
 import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction}
 import org.ergoplatform.nodeView.ErgoReadersHolder.{GetDataFromHistory, GetReaders, Readers}
@@ -22,7 +22,7 @@ import sigmastate.SType
 import sigmastate.Values.{ByteArrayConstant, EvaluatedValue}
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
-import special.collection.Extensions._
+import sigma.Extensions._
 
 import java.net.InetSocketAddress
 import scala.concurrent.duration._
@@ -243,7 +243,7 @@ class TransactionApiRouteSpec extends AnyFlatSpec
       Map(
         ErgoBox.R4 -> ByteArrayConstant("name".getBytes("UTF-8")),
         ErgoBox.R5 -> ByteArrayConstant("4".getBytes("UTF-8")),
-      ).asJson
+      ).asInstanceOf[AdditionalRegisters].asJson
 
     Post(prefix + s"/unconfirmed/outputs/byRegisters", searchedRegs) ~> chainedRoute ~> check {
       status shouldBe StatusCodes.OK
@@ -271,7 +271,7 @@ class TransactionApiRouteSpec extends AnyFlatSpec
         ErgoBox.R4 -> ByteArrayConstant("name".getBytes("UTF-8")),
         ErgoBox.R5 -> ByteArrayConstant("4".getBytes("UTF-8")),
         ErgoBox.R6 -> ByteArrayConstant("description".getBytes("UTF-8")),
-      ).asJson
+      ).asInstanceOf[AdditionalRegisters].asJson
 
     Post(prefix + s"/unconfirmed/outputs/byRegisters", searchedRegs) ~> chainedRoute ~> check {
       status shouldBe StatusCodes.OK
diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala
index 66aa6a8c05..bbdee81ce2 100644
--- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala
+++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala
@@ -7,7 +7,7 @@ import org.ergoplatform.settings.MonetarySettings
 import org.ergoplatform.utils.{ErgoPropertyTest, RandomWrapper}
 import org.ergoplatform.wallet.interpreter.ErgoInterpreter
 import org.scalacheck.Gen
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 
 import scala.concurrent.duration._
 
diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala
index 0fddaf5750..148d5ebffb 100644
--- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala
+++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala
@@ -19,8 +19,8 @@ import org.ergoplatform.utils.ErgoTestHelpers
 import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoTreePredef, Input}
 import org.scalatest.concurrent.Eventually
 import org.scalatest.flatspec.AnyFlatSpec
-import sigmastate.basics.DLogProtocol
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigmastate.crypto.DLogProtocol
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 
 import scala.concurrent.duration._
 
diff --git a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala
index 16e83f5d00..b6e4a6d073 100644
--- a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala
+++ b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala
@@ -26,8 +26,8 @@ import org.scalatest.concurrent.Eventually
 import org.scalatest.flatspec.AnyFlatSpec
 import sigmastate.SigmaAnd
 import sigmastate.Values.{ErgoTree, SigmaPropConstant}
-import sigmastate.basics.DLogProtocol
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigmastate.crypto.DLogProtocol
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 
 import scala.annotation.tailrec
 import scala.concurrent.ExecutionContext.Implicits.global
diff --git a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala
index ee110f90fc..4901e7526b 100644
--- a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala
+++ b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala
@@ -14,19 +14,21 @@ import org.ergoplatform.wallet.interpreter.{ErgoInterpreter, TransactionHintsBag
 import org.ergoplatform.wallet.protocol.context.InputContext
 import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, Input}
 import org.scalacheck.Gen
-import scalan.util.BenchmarkUtil
+import sigma.util.BenchmarkUtil
 import scorex.crypto.authds.ADKey
 import scorex.crypto.hash.Blake2b256
 import scorex.util.encode.Base16
 import scorex.util.{ModifierId, bytesToId}
+import sigma.Colls
 import sigmastate.AND
 import sigmastate.Values.{ByteArrayConstant, ByteConstant, IntConstant, LongArrayConstant, SigmaPropConstant, TrueLeaf}
-import sigmastate.basics.CryptoConstants
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.CryptoConstants
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.eval._
 import sigmastate.helpers.TestingHelpers._
 import sigmastate.interpreter.{ContextExtension, ProverResult}
 import sigmastate.eval.Extensions._
+
 import scala.util.{Random, Try}
 
 class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants {
diff --git a/src/test/scala/org/ergoplatform/modifiers/mempool/ExpirationSpecification.scala b/src/test/scala/org/ergoplatform/modifiers/mempool/ExpirationSpecification.scala
index ae85bade3b..7398a9ce5f 100644
--- a/src/test/scala/org/ergoplatform/modifiers/mempool/ExpirationSpecification.scala
+++ b/src/test/scala/org/ergoplatform/modifiers/mempool/ExpirationSpecification.scala
@@ -6,6 +6,7 @@ import org.ergoplatform.utils.ErgoPropertyTest
 import org.ergoplatform.wallet.interpreter.ErgoInterpreter
 import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, Input}
 import org.scalatest.Assertion
+import sigma.Colls
 import sigmastate.Values.ShortConstant
 import sigmastate.interpreter.{ContextExtension, ProverResult}
 import sigmastate.eval._
diff --git a/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala b/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala
index 1196482d4c..8270804c48 100644
--- a/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala
+++ b/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala
@@ -9,7 +9,7 @@ import scorex.crypto.authds.ADDigest
 import scorex.crypto.hash.{Blake2b256, Digest32}
 import scorex.util.ModifierId
 import scorex.util.encode.Base16
-import sigmastate.basics.CryptoConstants.EcPointType
+import sigmastate.crypto.CryptoConstants.EcPointType
 
 import java.nio.ByteBuffer
 
diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
index c26f9cf451..7fd4541e53 100644
--- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
@@ -18,10 +18,10 @@ import org.ergoplatform.settings.{ErgoSettings, NetworkType, NipopowSettings, No
 import org.ergoplatform.utils.{ErgoPropertyTest, ErgoTestHelpers, HistoryTestHelpers}
 import scorex.util.{ModifierId, bytesToId}
 import sigmastate.Values
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
-import special.collection.Coll
+import sigma.{Coll, Colls}
 import spire.implicits.cfor
 
 import java.io.File
diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala
index ce643b96ef..7f2f6b7cca 100644
--- a/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala
@@ -9,8 +9,8 @@ import org.ergoplatform.{ErgoBox, ErgoTreePredef, Height, Self}
 import scorex.crypto.authds.avltree.batch.Remove
 import sigmastate.Values._
 import sigmastate._
-import sigmastate.basics.CryptoConstants.dlogGroup
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.CryptoConstants.dlogGroup
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.eval.{CompiletimeIRContext, IRContext}
 import sigmastate.lang.Terms._
 import sigmastate.lang.{CompilerSettings, SigmaCompiler, TransformingSigmaBuilder}
diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
index a0cd911b10..535c11d7c1 100644
--- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala
@@ -21,7 +21,7 @@ import scorex.db.ByteArrayWrapper
 import scorex.util.{ModifierId, bytesToId}
 import scorex.util.encode.Base16
 import sigmastate.Values.ByteArrayConstant
-import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog}
+import sigmastate.crypto.DLogProtocol.{DLogProverInput, ProveDlog}
 import sigmastate.interpreter.ProverResult
 import sigmastate.helpers.TestingHelpers._
 
diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala
index 9659837cb8..5f5967d4d3 100644
--- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala
@@ -17,7 +17,7 @@ import org.scalacheck.Gen
 import org.scalatest.concurrent.Eventually
 import scorex.util.ModifierId
 import scorex.util.encode.Base16
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
 import sigmastate.{CAND, CTHRESHOLD}
diff --git a/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala b/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala
index 2d5eca8bf7..46a670cab6 100644
--- a/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala
+++ b/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala
@@ -5,9 +5,10 @@ import org.ergoplatform.settings.{MonetarySettings, ReemissionSettings}
 import org.ergoplatform.utils.{ErgoPropertyTest, ErgoTestConstants}
 import scorex.crypto.hash.Blake2b256
 import scorex.util.ModifierId
+import sigma.Colls
 import sigmastate.AvlTreeData
 import sigmastate.TrivialProp.TrueProp
-import sigmastate.eval.{Colls, Digest32Coll}
+import sigmastate.eval.Digest32Coll
 import sigmastate.helpers.TestingHelpers.testBox
 import sigmastate.helpers.{ContextEnrichingTestProvingInterpreter, ErgoLikeContextTesting, ErgoLikeTestInterpreter}
 import sigmastate.interpreter.Interpreter.emptyEnv
diff --git a/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala b/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala
index 571586ca26..016696d708 100644
--- a/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala
+++ b/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala
@@ -3,7 +3,7 @@ package org.ergoplatform.serialization
 import io.circe.syntax._
 import io.circe.{ACursor, Decoder, Encoder, Json}
 import org.ergoplatform.ErgoBox
-import org.ergoplatform.ErgoBox.NonMandatoryRegisterId
+import org.ergoplatform.ErgoBox.{AdditionalRegisters, NonMandatoryRegisterId}
 import org.ergoplatform.http.api.ApiCodecs
 import org.ergoplatform.http.api.ApiEncoderOption.HideDetails.implicitValue
 import org.ergoplatform.http.api.ApiEncoderOption.{Detalization, ShowDetails}
@@ -187,7 +187,7 @@ class JsonSerializationSpec extends ErgoPropertyTest with WalletGenerators with
     stringify(decodedAssets) should contain theSameElementsAs stringify(assets)
   }
 
-  private def checkRegisters(c: ACursor, registers: Map[NonMandatoryRegisterId, _ <: EvaluatedValue[_ <: SType]]) = {
+  private def checkRegisters(c: ACursor, registers: AdditionalRegisters) = {
     val Right(decodedRegs) = c.as[Map[NonMandatoryRegisterId, EvaluatedValue[SType]]]
     decodedRegs should contain theSameElementsAs registers
   }
diff --git a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala
index 6db85c433d..7fb07b62f9 100644
--- a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala
+++ b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala
@@ -16,7 +16,7 @@ import org.ergoplatform.settings._
 import org.ergoplatform.utils.{ErgoTestHelpers, HistoryTestHelpers}
 import org.ergoplatform.wallet.boxes.{BoxSelector, ReplaceCompactCollectBoxSelector}
 import scorex.util.ModifierId
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 
 import java.io.File
 import scala.annotation.tailrec
diff --git a/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala b/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala
index c09e9c6bd9..9eebe3f4b7 100644
--- a/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala
+++ b/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala
@@ -6,7 +6,8 @@ import org.ergoplatform.settings.LaunchParameters._
 import org.ergoplatform.{ErgoBoxCandidate, Input}
 import scorex.crypto.authds.ADKey
 import scorex.utils.Random
-import sigmastate.basics.DLogProtocol.DLogProverInput
+import sigma.Colls
+import sigmastate.crypto.DLogProtocol.DLogProverInput
 import sigmastate.eval.Extensions.ArrayByteOps
 import sigmastate.eval._
 import sigmastate.interpreter.{ContextExtension, ProverResult}
diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala
index cd24b2d52e..51d7e80597 100644
--- a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala
+++ b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala
@@ -22,8 +22,8 @@ import scorex.crypto.authds.ADDigest
 import scorex.crypto.hash.Digest32
 import scorex.util.ScorexLogging
 import sigmastate.Values.ErgoTree
-import sigmastate.basics.CryptoConstants.EcPointType
-import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog}
+import sigmastate.crypto.CryptoConstants.EcPointType
+import sigmastate.crypto.DLogProtocol.{DLogProverInput, ProveDlog}
 import sigmastate.interpreter.{ContextExtension, ProverResult}
 
 import scala.concurrent.duration._
diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala
index 078719ccb5..b68886a573 100644
--- a/src/test/scala/org/ergoplatform/utils/Stubs.scala
+++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala
@@ -42,7 +42,7 @@ import scorex.crypto.authds.ADDigest
 import scorex.crypto.hash.Digest32
 import scorex.db.ByteArrayWrapper
 import scorex.util.Random
-import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog}
+import sigmastate.crypto.DLogProtocol.{DLogProverInput, ProveDlog}
 
 import scala.collection.mutable
 import scala.concurrent.duration._
diff --git a/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala b/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala
index b76ac7485b..42407311f2 100644
--- a/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala
+++ b/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala
@@ -16,12 +16,13 @@ import org.ergoplatform.utils.fixtures.WalletFixture
 import scorex.crypto.authds.ADKey
 import scorex.crypto.hash.Blake2b256
 import scorex.util.ModifierId
+import sigma.Colls
 import sigmastate.Values.ErgoTree
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
 import sigmastate.interpreter.ProverResult
-import special.collection.Extensions._
+import sigma.Extensions._
 
 trait WalletTestOps extends NodeViewBaseOps {
 
diff --git a/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala b/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala
index 441bd039f2..cfe35ca205 100644
--- a/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala
+++ b/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala
@@ -7,12 +7,13 @@ import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionCandida
 import org.ergoplatform.modifiers.history.header.Header
 import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, PoPowHeader}
 import org.ergoplatform.modifiers.mempool.ErgoTransaction
-import org.ergoplatform.modifiers.{NonHeaderBlockSection, ErgoFullBlock, BlockSection}
+import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection}
 import org.ergoplatform.nodeView.history.ErgoHistory
 import org.ergoplatform.settings.Constants
 import org.ergoplatform.utils.{BoxUtils, ErgoTestConstants}
 import scorex.crypto.authds.{ADKey, SerializedAdProof}
 import scorex.crypto.hash.Digest32
+import sigma.Colls
 import sigmastate.eval._
 import sigmastate.helpers.TestingHelpers._
 import sigmastate.interpreter.{ContextExtension, ProverResult}
diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala
index 9c0eed8a88..ad49b7557a 100644
--- a/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala
+++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala
@@ -23,9 +23,9 @@ import scorex.crypto.authds.{ADDigest, SerializedAdProof}
 import scorex.crypto.hash.Digest32
 import scorex.testkit.generators.CoreGenerators
 import sigmastate.Values.ErgoTree
-import sigmastate.basics.CryptoConstants.EcPointType
-import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog}
-import sigmastate.basics.{CryptoConstants, DiffieHellmanTupleProverInput, ProveDHTuple}
+import sigmastate.crypto.CryptoConstants.EcPointType
+import sigmastate.crypto.DLogProtocol.{DLogProverInput, ProveDlog}
+import sigmastate.crypto.{CryptoConstants, DiffieHellmanTupleProverInput, ProveDHTuple}
 import sigmastate.interpreter.ProverResult
 
 import scala.util.Random
diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala
index 7630d2cdbe..978617e43d 100644
--- a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala
+++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala
@@ -24,7 +24,7 @@ import scorex.crypto.hash.Blake2b256
 import scorex.db.ByteArrayWrapper
 import scorex.util.encode.Base16
 import sigmastate.Values.ErgoTree
-import sigmastate.basics.DLogProtocol.ProveDlog
+import sigmastate.crypto.DLogProtocol.ProveDlog
 import sigmastate.eval.Extensions._
 import sigmastate.eval._
 import sigmastate.helpers.TestingHelpers._

From 34fea49951c5e79fd643cfaeb935b0a2be1ba288 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Tue, 26 Sep 2023 21:18:04 +0200
Subject: [PATCH 37/58] Fixed conflicts

---
 .../http/api/BlockchainApiRoute.scala         | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
index ce154e3ac4..df2bfbbb71 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
@@ -239,10 +239,10 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     validateAndGetBoxesByAddress(address, offset, limit)
   }
 
-  private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int, sortDir: Direction): Future[Seq[IndexedErgoBox]] =
-    getHistory.map { history =>
+  private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int, sortDir: Direction, unconfirmed: Boolean): Future[Seq[IndexedErgoBox]] =
+    getHistoryWithMempool.map { case (history, mempool) =>
       getAddress(addr)(history) match {
-        case Some(addr) => addr.retrieveUtxos(history, offset, limit, sortDir)
+        case Some(addr) => addr.retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed)
         case None       => Seq.empty[IndexedErgoBox]
       }
     }
@@ -250,26 +250,27 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
   private def validateAndGetBoxesByAddressUnspent(address: ErgoAddress,
                                                   offset: Int,
                                                   limit: Int,
-                                                  dir: Direction): Route = {
+                                                  dir: Direction,
+                                                  unconfirmed: Boolean): Route = {
     if (limit > MaxItems) {
       BadRequest(s"No more than $MaxItems boxes can be requested")
     } else if (dir == SortDirection.INVALID) {
       BadRequest("Invalid parameter for sort direction, valid values are \"ASC\" and \"DESC\"")
     } else {
-      ApiResponse(getBoxesByAddressUnspent(address, offset, limit, dir))
+      ApiResponse(getBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed))
     }
   }
 
   private def getBoxesByAddressUnspentR: Route =
-    (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir) {
-      (address, offset, limit, dir) =>
-        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir)
+    (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir & unconfirmed) {
+      (address, offset, limit, dir, unconfirmed) =>
+        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed)
     }
 
   private def getBoxesByAddressUnspentGetRoute: Route =
-    (pathPrefix("box" / "unspent" / "byAddress") & get & addressPass & paging & sortDir) {
-      (address, offset, limit, dir) =>
-        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir)
+    (pathPrefix("box" / "unspent" / "byAddress") & get & addressPass & paging & sortDir & unconfirmed) {
+      (address, offset, limit, dir, unconfirmed) =>
+        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed)
     }
 
   private def getBoxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] =

From a25c71825b93c71117458227ddb3564dacd04fa0 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Wed, 27 Sep 2023 20:07:15 +0200
Subject: [PATCH 38/58] Small refactor

---
 .../nodeView/history/extra/ExtraIndexer.scala | 31 ++++++++++++-------
 .../nodeView/history/extra/IndexedToken.scala |  3 +-
 .../nodeView/history/extra/Segment.scala      | 12 +++----
 3 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
index ed0a65c82e..1412926488 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
@@ -168,17 +168,14 @@ trait ExtraIndexerBase extends ScorexLogging {
           case Right(iEb) => tokens.put(id, x.addBox(iEb)) // receive box
         }
       case None => // token not found at all
-        spendOrReceive match {
-          case Left(iEb) => log.warn(s"Unknown address spent box ${bytesToId(iEb.box.id)}") // spend box should never happen by an unknown address
-          case Right(iEb) => trees.put(id, IndexedErgoAddress(id).initBalance.addTx(globalTxIndex).addBox(iEb)) // receive box, new address
-        }
+        log.warn(s"Unknown token $id") // spend box should never happen by an unknown token
     }
   }
 
   /**
     * @return number of indexes in all buffers
     */
-  private def modCount: Int = general.length + boxes.size + trees.size
+  private def modCount: Int = general.length + boxes.size + trees.size + tokens.size
 
   /**
     * Write buffered indexes to database and clear buffers.
@@ -203,7 +200,7 @@ trait ExtraIndexerBase extends ScorexLogging {
         token.buffer.foreach(seg => segments.put(seg.id, seg))
         token.buffer.clear()
       }
-      if(token.txs.length > segmentTreshold || token.boxes.length > segmentTreshold)
+      if(token.boxes.length > segmentTreshold)
         token.splitToSegments.foreach(seg => segments.put(seg.id, seg))
     }
 
@@ -355,13 +352,22 @@ trait ExtraIndexerBase extends ScorexLogging {
     globalTxIndex -= 1
     while(globalTxIndex > txTarget) {
       val tx: IndexedErgoTransaction = NumericTxIndex.getTxByNumber(history, globalTxIndex).get
-      tx.inputNums.map(NumericBoxIndex.getBoxByNumber(history, _).get).foreach(iEb => { // undo all spendings
+      tx.inputNums.map(NumericBoxIndex.getBoxByNumber(history, _).get).foreach { iEb => // undo all spendings
+
         iEb.spendingHeightOpt = None
         iEb.spendingTxIdOpt = None
+
         val address = history.typedExtraIndexById[IndexedErgoAddress](hashErgoTree(iEb.box.ergoTree)).get.addBox(iEb, record = false)
         address.findAndModBox(iEb.globalIndex, history)
         historyStorage.insertExtra(Array.empty, Array[ExtraIndex](iEb, address) ++ address.buffer)
-      })
+
+        cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { i =>
+          history.typedExtraIndexById[IndexedToken](IndexedToken.fromBox(iEb, i).id).map { token =>
+            token.findAndModBox(iEb.globalIndex, history)
+            historyStorage.insertExtra(Array.empty, Array[ExtraIndex](token) ++ token.buffer)
+          }
+        }
+      }
       toRemove += tx.id // tx by id
       toRemove += bytesToId(NumericTxIndex.indexToBytes(globalTxIndex)) // tx id by number
       globalTxIndex -= 1
@@ -373,10 +379,11 @@ trait ExtraIndexerBase extends ScorexLogging {
     while(globalBoxIndex > boxTarget) {
       val iEb: IndexedErgoBox = NumericBoxIndex.getBoxByNumber(history, globalBoxIndex).get
       cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { i =>
-        history.typedExtraIndexById[IndexedToken](IndexedToken.fromBox(iEb, i).id) match {
-          case Some(token) if token.boxId == iEb.id =>
-            toRemove += token.id // token created, delete
-          case _ => // no token created
+        history.typedExtraIndexById[IndexedToken](IndexedToken.fromBox(iEb, i).id).map { token =>
+          if(token.boxId == iEb.id) // token created, delete
+            toRemove += token.id
+          else // no token created, update
+            toRemove ++= token.rollback(txTarget, boxTarget, _history)
         }
       }
       history.typedExtraIndexById[IndexedErgoAddress](hashErgoTree(iEb.box.ergoTree)).map { address =>
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
index 6c4a356376..5652ae9c2a 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
@@ -49,7 +49,7 @@ case class IndexedToken(tokenId: ModifierId,
 
     val toRemove: ArrayBuffer[ModifierId] = rollbackState(txTarget, boxTarget, history.getReader)
 
-    if (txCount == 0 && boxCount == 0)
+    if (boxCount == 0)
       toRemove += id // all segments empty after rollback, delete parent
     else
       history.historyStorage.insertExtra(Array.empty, Array(this)) // save the changes made to this address
@@ -192,6 +192,5 @@ object IndexedToken {
                  name,
                  description,
                  decimals)
-      .addBox(iEb)
   }
 }
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
index f5e86a6954..2e4aab8127 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -298,9 +298,9 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
       txs.clear()
       txs ++= tmp
       if (txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible
-        val id = txSegmentId(parentId, txSegmentCount - 1)
-        history.typedExtraIndexById[T](id).get.txs ++=: txs
-        toRemove += id
+        val segmentId = txSegmentId(parentId, txSegmentCount - 1)
+        history.typedExtraIndexById[T](segmentId).get.txs ++=: txs
+        toRemove += segmentId
         txSegmentCount -= 1
       }
     } while (txCount > 0 && txs.last > txTarget)
@@ -311,9 +311,9 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
       boxes.clear()
       boxes ++= tmp
       if (boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible
-        val id = boxSegmentId(parentId, boxSegmentCount - 1)
-        history.typedExtraIndexById[T](id).get.boxes ++=: boxes
-        toRemove += id
+        val segmentId = boxSegmentId(parentId, boxSegmentCount - 1)
+        history.typedExtraIndexById[T](segmentId).get.boxes ++=: boxes
+        toRemove += segmentId
         boxSegmentCount -= 1
       }
     } while (boxCount > 0 && abs(boxes.last) > boxTarget)

From 9cb9b964e3c8484331285d6cf62db846f10139d1 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Thu, 28 Sep 2023 00:07:42 +0200
Subject: [PATCH 39/58] Fixed a lot of bugs and reworked tests

---
 .../nodeView/history/extra/IndexedToken.scala |   8 +-
 .../nodeView/history/extra/Segment.scala      |  40 ++---
 .../extra/ExtraIndexerSpecification.scala     | 137 +++++++++++-------
 3 files changed, 113 insertions(+), 72 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
index 5652ae9c2a..725b068fda 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
@@ -18,7 +18,7 @@ import scala.collection.mutable.ArrayBuffer
 /**
   * Index of a token containing creation information.
   * @param tokenId     - id of this token
-  * @param boxId       - id of the box that created th is token
+  * @param boxId       - id of the box that created this token
   * @param amount      - emission amount
   * @param name        - name of this token (UTF-8)
   * @param description - description of this token (UTF-8)
@@ -32,7 +32,7 @@ case class IndexedToken(tokenId: ModifierId,
                         description: String = "",
                         decimals: Int = 0,
                         override val boxes: ArrayBuffer[Long] = new ArrayBuffer[Long])
-  extends Segment[IndexedToken](uniqueId(tokenId), id => IndexedToken(id), new ArrayBuffer[Long], boxes) with ExtraIndex {
+  extends Segment[IndexedToken](tokenId, id => IndexedToken(id), new ArrayBuffer[Long], boxes, uniqueId) with ExtraIndex {
 
   override lazy val id: ModifierId = uniqueId(tokenId)
   override def serializedId: Array[Byte] = fastIdToBytes(id)
@@ -106,6 +106,7 @@ object IndexedTokenSerializer extends ErgoSerializer[IndexedToken] {
 
   override def serialize(iT: IndexedToken, w: Writer): Unit = {
     w.putBytes(fastIdToBytes(iT.tokenId))
+    w.putUByte(iT.boxId.length / 2)
     w.putBytes(fastIdToBytes(iT.boxId))
     w.putULong(iT.amount)
     val name: Array[Byte] = iT.name.getBytes("UTF-8")
@@ -120,7 +121,8 @@ object IndexedTokenSerializer extends ErgoSerializer[IndexedToken] {
 
   override def parse(r: Reader): IndexedToken = {
     val tokenId: ModifierId = bytesToId(r.getBytes(32))
-    val boxId: ModifierId = bytesToId(r.getBytes(32))
+    val boxIdLen: Int = r.getUByte()
+    val boxId: ModifierId = bytesToId(r.getBytes(boxIdLen))
     val amount: Long = r.getULong()
     val nameLen: Int = r.getUShort()
     val name: String = new String(r.getBytes(nameLen), "UTF-8")
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
index 2e4aab8127..deef2da190 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -22,12 +22,14 @@ import scala.reflect.ClassTag
  * @param factory  - parent object factory
  * @param txs      - list of numberic transaction indexes
  * @param boxes    - list of numberic box indexes, negative values indicate the box is spent
+ * @param idMod    - function to apply to ids during segmentation and db lookup
  * @tparam T       - type of parent object
  */
 abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
                                                    val factory: ModifierId => T,
                                                    val txs: ArrayBuffer[Long],
-                                                   val boxes: ArrayBuffer[Long])
+                                                   val boxes: ArrayBuffer[Long],
+                                                   val idMod: ModifierId => ModifierId = id => id)
   extends ExtraIndex with ScorexLogging {
 
   override lazy val id: ModifierId = parentId
@@ -69,7 +71,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
     cfor(buffer.length - 1)(_ >= 0, _ - 1) { i =>
       if(buffer(i).id.equals(searchId)) return i
     }
-    buffer += history.typedExtraIndexById[T](searchId).get
+    buffer += history.typedExtraIndexById[T](idMod(searchId)).get
     buffer.length - 1
   }
 
@@ -115,27 +117,24 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
    * @return array of parent objects
    */
   private[extra] def splitToSegments(implicit segmentTreshold: Int): Array[T] = {
-    val data: Array[T] = new Array[T]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold))
-    var i: Int = 0
+    val data: ArrayBuffer[T] = new ArrayBuffer[T]
 
     // Split txs until under segmentTreshold
-    while(txs.length >= segmentTreshold) {
-      data(i) = factory(txSegmentId(parentId, txSegmentCount))
-      data(i).txs ++= txs.take(segmentTreshold)
-      i += 1
+    while(txs.length > segmentTreshold) {
+      data += factory(txSegmentId(parentId, txSegmentCount))
+      data.last.txs ++= txs.take(segmentTreshold)
       txSegmentCount += 1
       txs.remove(0, segmentTreshold)
     }
 
     // Split boxes until under segmentTreshold
-    while(boxes.length >= segmentTreshold) {
-      data(i) = factory(boxSegmentId(parentId, boxSegmentCount))
-      data(i).boxes ++= boxes.take(segmentTreshold)
-      i += 1
+    while(boxes.length > segmentTreshold) {
+      data += factory(boxSegmentId(parentId, boxSegmentCount))
+      data.last.boxes ++= boxes.take(segmentTreshold)
       boxSegmentCount += 1
       boxes.remove(0, segmentTreshold)
     }
-    data
+    data.toArray
   }
 
   /**
@@ -210,7 +209,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
       val data: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
       getSegmentsForRange(offset, limit).map(n => math.max(segmentCount - n, 0)).distinct.foreach { num =>
         arraySelector(
-          history.typedExtraIndexById[T](idOf(id, num)).get
+          history.typedExtraIndexById[T](idMod(idOf(id, num))).get
         ) ++=: data
       }
       data ++= (if(offset < array.length) array else Nil)
@@ -258,14 +257,14 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
         var segment: Int = boxSegmentCount
         while(data.length < (limit + offset) && segment > 0) {
           segment -= 1
-          history.typedExtraIndexById[T](boxSegmentId(id, segment)).get.boxes
+          history.typedExtraIndexById[T](idMod(boxSegmentId(id, segment))).get.boxes
             .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) ++=: data
         }
         data.reverse.slice(offset, offset + limit).toArray
       case ASC =>
         var segment: Int = 0
         while(data.length < (limit + offset) && segment < boxSegmentCount) {
-          data ++= history.typedExtraIndexById[T](boxSegmentId(id, segment)).get.boxes
+          data ++= history.typedExtraIndexById[T](idMod(boxSegmentId(id, segment))).get.boxes
             .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
           segment += 1
         }
@@ -286,8 +285,9 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
   protected def rollbackState(txTarget: Long, boxTarget: Long, history: ErgoHistoryReader)
                              (implicit segmentTreshold: Int): ArrayBuffer[ModifierId] = {
 
-    if ((txCount == 0 && boxCount == 0) || // already rolled back
-      (txs.last <= txTarget && abs(boxes.last) <= boxTarget)) // no rollback needed
+    if((txCount == 0 && boxCount == 0) || // already rolled back
+      (txs.lastOption.getOrElse[Long](0) <= txTarget &&
+        abs(boxes.lastOption.getOrElse[Long](0)) <= boxTarget)) // no rollback needed
       return ArrayBuffer.empty[ModifierId]
 
     val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId]
@@ -299,7 +299,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
       txs ++= tmp
       if (txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible
         val segmentId = txSegmentId(parentId, txSegmentCount - 1)
-        history.typedExtraIndexById[T](segmentId).get.txs ++=: txs
+        history.typedExtraIndexById[T](idMod(segmentId)).get.txs ++=: txs
         toRemove += segmentId
         txSegmentCount -= 1
       }
@@ -312,7 +312,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
       boxes ++= tmp
       if (boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible
         val segmentId = boxSegmentId(parentId, boxSegmentCount - 1)
-        history.typedExtraIndexById[T](segmentId).get.boxes ++=: boxes
+        history.typedExtraIndexById[T](idMod(segmentId)).get.boxes ++=: boxes
         toRemove += segmentId
         boxSegmentCount -= 1
       }
diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
index 56e0bb8f11..1f0fcd7c6c 100644
--- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
+++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala
@@ -30,17 +30,21 @@ import scala.annotation.tailrec
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.duration.{DurationInt, FiniteDuration}
+import scala.reflect.ClassTag
 import scala.util.{Random, Try}
 
 class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase with HistoryTestHelpers {
 
+  type ID_LL = mutable.HashMap[ModifierId,(Long,Long)]
+
   override protected val saveLimit: Int = 1 // save every block
   override protected implicit val segmentTreshold: Int = 8 // split to smaller segments
   override protected implicit val addressEncoder: ErgoAddressEncoder = initSettings.chainSettings.addressEncoder
 
   val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true,
-    -1, UtxoSettings(false, 0, 2), NipopowSettings(false, 1), mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false,
-    internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false,
+    -1, UtxoSettings(utxoBootstrap = false, 0, 2), NipopowSettings(nipopowBootstrap = false, 1), mining = false,
+    ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false, internalMinersCount = 1,
+    internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false,
     200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20,
     1000000, 100, adProofsSuffixLength = 112 * 1024, extraIndex = false)
 
@@ -72,74 +76,98 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
     segments.clear()
   }
 
-  def getAddresses(limit: Int): (mutable.HashMap[ModifierId,(Long,Long)],Int,Int) = {
+  def manualIndex(limit: Int): (ID_LL, // address -> (erg,tokenSum)
+                                ID_LL, // tokenId -> (boxesCount,_)
+                                Int, // txs indexed
+                                Int) = { // boxes indexed
     var txsIndexed = 0
     var boxesIndexed = 0
-    val addresses: mutable.HashMap[ModifierId,(Long,Long)] = mutable.HashMap[ModifierId,(Long,Long)]()
+    val addresses: ID_LL = mutable.HashMap[ModifierId,(Long,Long)]()
+    val indexedTokens: ID_LL = mutable.HashMap[ModifierId,(Long,Long)]()
     cfor(1)(_ <= limit, _ + 1) { i =>
-      _history.getReader.bestBlockTransactionsAt(i).get.txs.foreach(tx => {
+      _history.getReader.bestBlockTransactionsAt(i).get.txs.foreach { tx =>
         txsIndexed += 1
         if (i != 1) {
-          tx.inputs.foreach(input => {
+          tx.inputs.foreach { input =>
             val iEb: IndexedErgoBox = _history.getReader.typedExtraIndexById[IndexedErgoBox](bytesToId(input.boxId)).get
             val address = hashErgoTree(ExtraIndexer.getAddress(iEb.box.ergoTree)(addressEncoder).script)
             val prev = addresses(address)
             addresses.put(address, (prev._1 - iEb.box.value, prev._2 - iEb.box.additionalTokens.toArray.map(_._2).sum))
-          })
+          }
         }
-        tx.outputs.foreach(output => {
+        tx.outputs.foreach { output =>
           boxesIndexed += 1
           val address = hashErgoTree(addressEncoder.fromProposition(output.ergoTree).get.script)
           val prev = addresses.getOrElse(address, (0L, 0L))
           addresses.put(address, (prev._1 + output.value, prev._2 + output.additionalTokens.toArray.map(_._2).sum))
-        })
-      })
+          cfor(0)(_ < output.additionalTokens.length, _ + 1) { j =>
+            val token = IndexedToken.fromBox(new IndexedErgoBox(i, None, None, output, 0), j)
+            val prev2 = indexedTokens.getOrElse(token.id, (0L, 0L))
+            indexedTokens.put(token.id, (prev2._1 + 1, 0))
+          }
+        }
+      }
     }
-    (addresses, txsIndexed, boxesIndexed)
+    (addresses, indexedTokens, txsIndexed, boxesIndexed)
   }
 
-  def checkAddresses(addresses: mutable.HashMap[ModifierId,(Long,Long)], isSegment: Boolean = false): Int = {
-    var mismatches: Int = 0
-    addresses.foreach(addr => {
-      history.typedExtraIndexById[IndexedErgoAddress](addr._1) match {
-        case Some(iEa) =>
-          if(!isSegment && (iEa.balanceInfo.get.nanoErgs != addr._2._1 || iEa.balanceInfo.get.tokens.map(_._2).sum != addr._2._2))
-            mismatches += 1
-          if(!isSegment) {
+  def checkSegmentables[T <: Segment[_] : ClassTag](segmentables: ID_LL,
+                                                    isChild: Boolean = false,
+                                                    check: ((T, (Long, Long))) => Boolean
+                                                   ): Int = {
+    var errors: Int = 0
+    segmentables.foreach { segmentable =>
+      history.typedExtraIndexById[T](segmentable._1) match {
+        case Some(obj: T) =>
+          if(isChild) { // this is a segment
             // check tx segments
-            val txSegments: mutable.HashMap[ModifierId,(Long,Long)] = mutable.HashMap.empty[ModifierId, (Long,Long)]
-            txSegments ++= (0 until iEa.txSegmentCount).map(txSegmentId(iEa.treeHash, _)).map(Tuple2(_, (0L, 0L)))
-            checkAddresses(txSegments, isSegment = true) shouldBe 0
+            val txSegments: ID_LL = mutable.HashMap.empty[ModifierId,(Long,Long)]
+            txSegments ++= (0 until obj.txSegmentCount).map(n => obj.idMod(txSegmentId(obj.parentId, n))).map(Tuple2(_, (0L, 0L)))
+            checkSegmentables(txSegments, isChild = true, check) shouldBe 0
             // check box segments
-            val boxSegments: mutable.HashMap[ModifierId,(Long,Long)] = mutable.HashMap.empty[ModifierId,(Long,Long)]
-            boxSegments ++= (0 until iEa.boxSegmentCount).map(boxSegmentId(iEa.treeHash, _)).map(Tuple2(_, (0L, 0L)))
-            checkAddresses(boxSegments, isSegment = true) shouldBe 0
-          }
-          // check boxes in memory
-          iEa.boxes.foreach(boxNum =>
-            NumericBoxIndex.getBoxByNumber(history, boxNum) match {
-              case Some(iEb) =>
-                if (iEb.isSpent)
-                  boxNum.toInt should be <= 0
-                else
-                  boxNum.toInt should be >= 0
-              case None => System.err.println(s"Box $boxNum not found in database")
+            val boxSegments: ID_LL = mutable.HashMap.empty[ModifierId,(Long,Long)]
+            boxSegments ++= (0 until obj.boxSegmentCount).map(n => obj.idMod(boxSegmentId(obj.parentId, n))).map(Tuple2(_, (0L, 0L)))
+            checkSegmentables(boxSegments, isChild = true, check) shouldBe 0
+          }else { // this is the parent object
+            // check properties of object
+            if(!check((obj, segmentable._2)))
+              errors += 1
+            // check boxes in memory
+            obj.boxes.foreach { boxNum =>
+              NumericBoxIndex.getBoxByNumber(history, boxNum) match {
+                case Some(iEb) =>
+                  if (iEb.isSpent)
+                    boxNum.toInt should be <= 0
+                  else
+                    boxNum.toInt should be >= 0
+                case None =>
+                  System.err.println(s"Box $boxNum not found in database")
+                  errors += 1
+              }
+            }
+            // check txs in memory
+            obj.txs.foreach { txNum =>
+              NumericTxIndex.getTxByNumber(history, txNum) shouldNot be(empty)
             }
-          )
-          // check txs in memory
-          iEa.txs.foreach(txNum =>
-            NumericTxIndex.getTxByNumber(history, txNum) shouldNot be(empty)
-          )
-        case None =>
-          if (addr._2._1 != 0L && addr._2._2 != 0L) {
-            mismatches += 1
-            System.err.println(s"Address ${addr._1} should exist, but was not found")
           }
+        case None =>
+          System.err.println(s"Segmentable object ${segmentable._1} should exist, but was not found")
+          errors += 1
       }
-    })
-    mismatches
+    }
+    errors
   }
 
+  def checkAddresses(addresses: ID_LL): Int =
+    checkSegmentables[IndexedErgoAddress](addresses, isChild = false, seg => {
+      seg._1.balanceInfo.get.nanoErgs == seg._2._1 && seg._1.balanceInfo.get.tokens.map(_._2).sum == seg._2._2
+    })
+
+  def checkTokens(indexedTokens: ID_LL): Int =
+    checkSegmentables[IndexedToken](indexedTokens, isChild = false, seg => {
+      seg._1.boxCount == seg._2._1
+    })
+
   property("extra indexer transactions") {
     createDB()
     run()
@@ -163,10 +191,17 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
   property("extra indexer addresses") {
     createDB()
     run()
-    val (addresses, _, _) = getAddresses(HEIGHT)
+    val (addresses, _, _, _) = manualIndex(HEIGHT)
     checkAddresses(addresses) shouldBe 0
   }
 
+  property("extra indexer tokens") {
+    createDB()
+    run()
+    val (_, indexedTokens, _, _) = manualIndex(HEIGHT)
+    checkTokens(indexedTokens) shouldBe 0
+  }
+
   property("extra indexer rollback") {
     createDB()
 
@@ -176,7 +211,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
     val boxIndexBefore = globalBoxIndex
 
     // manually count balances
-    val (addresses, txsIndexed, boxesIndexed) = getAddresses(BRANCHPOINT)
+    val (addresses, indexedTokens, txsIndexed, boxesIndexed) = manualIndex(BRANCHPOINT)
 
     // perform rollback
     removeAfter(BRANCHPOINT)
@@ -184,6 +219,9 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
     // address balances
     checkAddresses(addresses) shouldBe 0
 
+    // token indexes
+    checkTokens(indexedTokens) shouldBe 0
+
     // check indexnumbers
     globalTxIndex shouldBe txsIndexed
     globalBoxIndex shouldBe boxesIndexed
@@ -211,8 +249,9 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
     run()
 
     // Check addresses again
-    val (addresses2, _, _) = getAddresses(HEIGHT)
+    val (addresses2, indexedTokens2, _, _) = manualIndex(HEIGHT)
     checkAddresses(addresses2) shouldBe 0
+    checkTokens(indexedTokens2) shouldBe 0
 
     // check indexnumbers again
     globalTxIndex shouldBe txIndexBefore

From e9e146e82681688f97970528312df84c7218f53f Mon Sep 17 00:00:00 2001
From: Alexander Slesarenko <avslesarenko@gmail.com>
Date: Thu, 28 Sep 2023 17:30:46 +0200
Subject: [PATCH 40/58] with-sigma-v5.0.12: updated sigma

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index dc5c2e79f4..b784648934 100644
--- a/build.sbt
+++ b/build.sbt
@@ -37,7 +37,7 @@ val circeVersion = "0.13.0"
 val akkaVersion = "2.6.10"
 val akkaHttpVersion = "10.2.4"
 
-val sigmaStateVersion = "5.0.11-37-2750749c-SNAPSHOT"
+val sigmaStateVersion = "5.0.11-43-321b3b04-SNAPSHOT"
 
 // for testing current sigmastate build (see sigmastate-ergo-it jenkins job)
 val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion)

From a48966c012a3252db518b6634590e8fb4a0069f4 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Fri, 29 Sep 2023 04:15:35 +0200
Subject: [PATCH 41/58] Fixed token detection problems, made some speed
 improvements by avoiding boxing

---
 .../nodeView/history/extra/ExtraIndexer.scala | 26 +++++---
 .../nodeView/history/extra/Segment.scala      | 62 ++++++++-----------
 .../history/storage/HistoryStorage.scala      |  8 +--
 3 files changed, 47 insertions(+), 49 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
index 1412926488..e257e0365d 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
@@ -126,13 +126,13 @@ trait ExtraIndexerBase extends ScorexLogging {
     * @param spendOrReceive - IndexedErgoBox to receive (Right) or spend (Left)
     */
   private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[IndexedErgoBox, IndexedErgoBox]): Unit = {
-    trees.get(id).map(tree => {
+    trees.get(id).map { tree =>
       spendOrReceive match {
         case Left(iEb) => tree.addTx(globalTxIndex).spendBox(iEb, Some(history)) // spend box
         case Right(iEb) => tree.addTx(globalTxIndex).addBox(iEb) // receive box
       }
       return
-    })
+    }
     history.typedExtraIndexById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers
       case Some(x) =>
         spendOrReceive match {
@@ -154,13 +154,13 @@ trait ExtraIndexerBase extends ScorexLogging {
    * @param spendOrReceive - IndexedErgoBox to receive (Right) or spend (Left)
    */
   private def findAndUpdateToken(id: ModifierId, spendOrReceive: Either[IndexedErgoBox, IndexedErgoBox]): Unit = {
-    tokens.get(id).map(token => {
+    tokens.get(id).map { token =>
       spendOrReceive match {
         case Left(iEb) => token.spendBox(iEb, Some(history)) // spend box
         case Right(iEb) => token.addBox(iEb) // receive box
       }
       return
-    })
+    }
     history.typedExtraIndexById[IndexedToken](uniqueId(id)) match { // token not found in last saveLimit modifiers
       case Some(x) =>
         spendOrReceive match {
@@ -187,7 +187,7 @@ trait ExtraIndexerBase extends ScorexLogging {
     // perform segmentation on big addresses and save their internal segment buffer
     trees.values.foreach { tree =>
       if(tree.buffer.nonEmpty) {
-        tree.buffer.foreach(seg => segments.put(seg.id, seg))
+        tree.buffer.values.foreach(seg => segments.put(seg.id, seg))
         tree.buffer.clear()
       }
       if(tree.txs.length > segmentTreshold || tree.boxes.length > segmentTreshold)
@@ -197,7 +197,7 @@ trait ExtraIndexerBase extends ScorexLogging {
     // perform segmentation on big tokens and save their internal segment buffer
     tokens.values.foreach { token =>
       if(token.buffer.nonEmpty) {
-        token.buffer.foreach(seg => segments.put(seg.id, seg))
+        token.buffer.values.foreach(seg => segments.put(seg.id, seg))
         token.buffer.clear()
       }
       if(token.boxes.length > segmentTreshold)
@@ -273,7 +273,13 @@ trait ExtraIndexerBase extends ScorexLogging {
         cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { j =>
           if (!inputTokens.exists(x => x._1 == iEb.box.additionalTokens(j)._1)) {
             val token = IndexedToken.fromBox(iEb, j)
-            tokens.put(token.tokenId, token)
+            tokens.get(token.tokenId) match {
+              case Some(t) => // same new token created in multiple boxes -> add amounts
+                val newToken = IndexedToken(t.tokenId, t.boxId, t.amount + token.amount, t.name, t.description, t.decimals, t.boxes)
+                newToken.buffer ++= t.buffer
+                tokens.put(token.tokenId, newToken)
+              case None => tokens.put(token.tokenId, token) // new token
+            }
           }
           findAndUpdateToken(iEb.box.additionalTokens(j)._1.toModifierId, Right(iEb))
         }
@@ -359,12 +365,12 @@ trait ExtraIndexerBase extends ScorexLogging {
 
         val address = history.typedExtraIndexById[IndexedErgoAddress](hashErgoTree(iEb.box.ergoTree)).get.addBox(iEb, record = false)
         address.findAndModBox(iEb.globalIndex, history)
-        historyStorage.insertExtra(Array.empty, Array[ExtraIndex](iEb, address) ++ address.buffer)
+        historyStorage.insertExtra(Array.empty, Array[ExtraIndex](iEb, address) ++ address.buffer.values)
 
         cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { i =>
           history.typedExtraIndexById[IndexedToken](IndexedToken.fromBox(iEb, i).id).map { token =>
             token.findAndModBox(iEb.globalIndex, history)
-            historyStorage.insertExtra(Array.empty, Array[ExtraIndex](token) ++ token.buffer)
+            historyStorage.insertExtra(Array.empty, Array[ExtraIndex](token) ++ token.buffer.values)
           }
         }
       }
@@ -508,7 +514,7 @@ object ExtraIndexer {
   /**
    * Current newest database schema version. Used to force extra database resync.
    */
-  val NewestVersion: Int = 3
+  val NewestVersion: Int = 4
   val NewestVersionBytes: Array[Byte] = ByteBuffer.allocate(4).putInt(NewestVersion).array
 
   val IndexedHeightKey: Array[Byte] = Algos.hash("indexed height")
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
index deef2da190..c5626e9875 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -12,6 +12,7 @@ import spire.implicits.cfor
 
 import scala.collection.mutable.ArrayBuffer
 import java.lang.Math.abs
+import scala.collection.mutable
 import scala.reflect.ClassTag
 
 /**
@@ -38,7 +39,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
   /**
    * Internal segment buffer
    */
-  private[extra] val buffer: ArrayBuffer[T] = ArrayBuffer.empty[T]
+  private[extra] val buffer: mutable.HashMap[ModifierId,T] = new mutable.HashMap[ModifierId,T]
 
   /**
    * Number of segments in database containing box numbers
@@ -60,21 +61,6 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
    */
   def txCount(implicit segmentTreshold: Int): Long = segmentTreshold * txSegmentCount + txs.length
 
-  /**
-   * Retrieve segment with specified id from buffer or database
-   *
-   * @param history - database handle to search, if segment is not found in buffer
-   * @param searchId- segment id to search for
-   * @return
-   */
-  private def getSegmentFromBufferOrHistroy(history: ErgoHistoryReader, searchId: ModifierId): Int = {
-    cfor(buffer.length - 1)(_ >= 0, _ - 1) { i =>
-      if(buffer(i).id.equals(searchId)) return i
-    }
-    buffer += history.typedExtraIndexById[T](idMod(searchId)).get
-    buffer.length - 1
-  }
-
   /**
    * Locate which segment the given box number is in and change its sign, meaning it spends unspent boxes and vice versa.
    *
@@ -85,28 +71,34 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
     val boxNumAbs = abs(boxNum)
     val inCurrent: Int = binarySearch(boxes, boxNumAbs)
     if(inCurrent >= 0) { // box found in current box array
-      boxes(inCurrent) = -boxes(inCurrent)
+      boxes.update(inCurrent, -boxes(inCurrent))
     } else { // box is in another segment, use binary search to locate
-      var n = 0
+      var segmentId: ModifierId = ModifierId @@ ""
       var low = 0
       var high = boxSegmentCount - 1
-      while (low <= high) {
+      while(low <= high) {
         val mid = (low + high) >>> 1
-        n = getSegmentFromBufferOrHistroy(history, boxSegmentId(parentId, mid))
-        if (abs(buffer(n).boxes.head) < boxNumAbs &&
-          abs(buffer(n).boxes.last) < boxNumAbs)
-          low = mid + 1
-        else if (abs(buffer(n).boxes.head) > boxNumAbs &&
-          abs(buffer(n).boxes.last) > boxNumAbs)
-          high = mid - 1
-        else
-          low = high + 1 // break
+        segmentId = boxSegmentId(parentId, mid)
+        buffer.get(segmentId).orElse(history.typedExtraIndexById[T](idMod(segmentId))).foreach { segment =>
+          if(abs(segment.boxes.head) < boxNumAbs &&
+            abs(segment.boxes.last) < boxNumAbs)
+            low = mid + 1
+          else if(abs(segment.boxes.head) > boxNumAbs &&
+            abs(segment.boxes.last) > boxNumAbs)
+            high = mid - 1
+          else {
+            low = high + 1 // break
+            buffer.put(segmentId, segment)
+          }
+        }
+      }
+      buffer.get(segmentId) match {
+        case Some(segment) =>
+          val i: Int = binarySearch(segment.boxes, boxNumAbs)
+          segment.boxes(i) = -segment.boxes(i)
+        case None =>
+          log.warn(s"Box $boxNum not found in any segment of parent when trying to spend")
       }
-      val i: Int = binarySearch(buffer(n).boxes, boxNumAbs)
-      if (i >= 0)
-        buffer(n).boxes(i) = -buffer(n).boxes(i)
-      else
-        log.warn(s"Box $boxNum not found in any segment of parent when trying to spend")
     }
   }
 
@@ -411,9 +403,9 @@ object SegmentSerializer {
 
   def parse(r: Reader, s: Segment[_]): Unit = {
     val txnsLen: Long = r.getUInt()
-    cfor(0)(_ < txnsLen, _ + 1) { _ => s.txs += r.getLong() }
+    cfor(0)(_ < txnsLen, _ + 1) { _ => s.txs.+=(r.getLong()) }
     val boxesLen: Long = r.getUInt()
-    cfor(0)(_ < boxesLen, _ + 1) { _ => s.boxes += r.getLong() }
+    cfor(0)(_ < boxesLen, _ + 1) { _ => s.boxes.+=(r.getLong()) }
     s.boxSegmentCount = r.getInt()
     s.txSegmentCount = r.getInt()
   }
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala
index 3385322366..2b4559af84 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala
@@ -4,7 +4,7 @@ import com.github.benmanes.caffeine.cache.Caffeine
 import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId}
 import org.ergoplatform.modifiers.history.HistoryModifierSerializer
 import org.ergoplatform.modifiers.history.header.Header
-import org.ergoplatform.nodeView.history.extra.{ExtraIndex, ExtraIndexSerializer, IndexedErgoAddress}
+import org.ergoplatform.nodeView.history.extra.{ExtraIndex, ExtraIndexSerializer, Segment}
 import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings}
 import scorex.core.utils.ScorexEncoding
 import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore}
@@ -94,8 +94,8 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e
       ExtraIndexSerializer.parseBytesTry(bytes) match {
         case Success(pm) =>
           log.trace(s"Cache miss for existing index $id")
-          if(pm.isInstanceOf[IndexedErgoAddress]){
-            extraCache.put(pm.id, pm) // only cache addresses
+          if(pm.isInstanceOf[Segment[_]]){
+            extraCache.put(pm.id, pm) // cache all segment type objects
           }
           Some(pm)
         case Failure(_) =>
@@ -151,7 +151,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e
   def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])],
                   objectsToInsert: Array[ExtraIndex]): Unit = {
     extraStore.insert(
-      objectsToInsert.map(mod => (mod.serializedId)),
+      objectsToInsert.map(mod => mod.serializedId),
       objectsToInsert.map(mod => ExtraIndexSerializer.toBytes(mod))
     )
     cfor(0)(_ < objectsToInsert.length, _ + 1) { i => val ei = objectsToInsert(i); extraCache.put(ei.id, ei)}

From 8f9a83fad329e95b0433e17fe707af0a54658098 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Fri, 29 Sep 2023 04:48:18 +0200
Subject: [PATCH 42/58] Added API endpoints

---
 src/main/resources/api/openapi.yaml           | 117 ++++++++++++++++++
 .../http/api/BlockchainApiRoute.scala         |  37 +++++-
 2 files changed, 152 insertions(+), 2 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 563abdb14e..280f46fd27 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -6019,6 +6019,123 @@ paths:
               schema:
                 $ref: '#/components/schemas/ApiError'
 
+  /blockchain/box/byTokenId/{tokenId}:
+    get:
+      summary: Retrieve boxes by an associated token id
+      operationId: getBoxesByTokenId
+      tags:
+        - blockchain
+      parameters:
+        - in: path
+          name: tokenId
+          required: true
+          description: id of the token
+          schema:
+            $ref: '#/components/schemas/ModifierId'
+        - in: query
+          name: offset
+          required: false
+          description: amount of elements to skip from the start
+          schema:
+            type: integer
+            format: int32
+            default: 0
+        - in: query
+          name: limit
+          required: false
+          description: amount of elements to retrieve
+          schema:
+            type: integer
+            format: int32
+            default: 5
+      responses:
+        '200':
+          description: boxes associated with wanted token
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  items:
+                    type: array
+                    description: Array of boxes
+                    items:
+                      $ref: '#/components/schemas/IndexedErgoBox'
+                  total:
+                    type: integer
+                    description: Total number of retreived boxes
+        '404':
+          description: No boxes found for wanted token
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+        default:
+          description: Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+
+  /blockchain/box/unspent/byTokenId/{tokenId}:
+    get:
+      summary: Retrieve unspent boxes by an associated token id
+      operationId: getBoxesByTokenIdUnspent
+      tags:
+        - blockchain
+      parameters:
+        - in: path
+          name: tokenId
+          required: true
+          description: id of the token
+          schema:
+            $ref: '#/components/schemas/ModifierId'
+        - in: query
+          name: offset
+          required: false
+          description: amount of elements to skip from the start
+          schema:
+            type: integer
+            format: int32
+            default: 0
+        - in: query
+          name: limit
+          required: false
+          description: amount of elements to retrieve
+          schema:
+            type: integer
+            format: int32
+            default: 5
+        - in: query
+          name: sortDirection
+          required: false
+          description: desc = new boxes first ; asc = old boxes first
+          schema:
+            type: string
+            default: desc
+      responses:
+        '200':
+          description: unspent boxes associated with wanted token
+          content:
+            application/json:
+              schema:
+                type: array
+                description: Array of boxes
+                items:
+                  $ref: '#/components/schemas/IndexedErgoBox'
+        '404':
+          description: No unspent boxes found for wanted token
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+        default:
+          description: Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+
   /blockchain/box/byAddress:
     post:
       summary: Retrieve boxes by their associated address
diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
index 408f7a0364..c0d173689d 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
@@ -13,6 +13,7 @@ import org.ergoplatform.nodeView.history.ErgoHistoryReader
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.ReceivableMessages.GetSegmentTreshold
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{GlobalBoxIndexKey, GlobalTxIndexKey, getIndex}
 import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree
+import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId
 import org.ergoplatform.nodeView.history.extra._
 import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
 import org.ergoplatform.settings.ErgoSettings
@@ -72,6 +73,8 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
       getTxRangeR ~
       getBoxByIdR ~
       getBoxByIndexR ~
+      getBoxesByTokenIdR ~
+      getBoxesByTokenIdUnspentR ~
       getBoxesByAddressR ~
       getBoxesByAddressGetRoute ~
       getBoxesByAddressUnspentR ~
@@ -316,7 +319,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     if(limit > MaxItems) {
       BadRequest(s"No more than $MaxItems boxes can be requested")
     }else if (dir == SortDirection.INVALID) {
-      BadRequest("Invalid parameter for sort direction, valid values are \"ASC\" and \"DESC\"")
+      BadRequest("Invalid parameter for sort direction, valid values are 'ASC' and 'DESC'")
     }else {
       ApiResponse(getBoxesByErgoTreeUnspent(tree, offset, limit, dir))
     }
@@ -324,7 +327,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
 
   private def getTokenInfoById(id: ModifierId): Future[Option[IndexedToken]] = {
     getHistory.map { history =>
-      history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.uniqueId(id))
+      history.typedExtraIndexById[IndexedToken](uniqueId(id))
     }
   }
 
@@ -332,6 +335,36 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     ApiResponse(getTokenInfoById(id))
   }
 
+  private def getBoxesByTokenId(id: ModifierId, offset: Int, limit: Int): Future[(Seq[IndexedErgoBox],Long)] =
+    getHistory.map { history =>
+      history.typedExtraIndexById[IndexedToken](uniqueId(id)) match {
+        case Some(token) => (token.retrieveBoxes(history, offset, limit), token.boxCount)
+        case None        => (Seq.empty[IndexedErgoBox], 0L)
+      }
+    }
+
+  private def getBoxesByTokenIdR: Route = (get & pathPrefix("box" / "byTokenId") & modifierId & paging) { (id, offset, limit) =>
+    ApiResponse(getBoxesByTokenId(id, offset, limit))
+  }
+
+  private def getBoxesByTokenIdUnspent(id: ModifierId, offset: Int, limit: Int, sortDir: Direction): Future[Seq[IndexedErgoBox]] =
+    getHistory.map { history =>
+      history.typedExtraIndexById[IndexedToken](uniqueId(id)) match {
+        case Some(token) => token.retrieveUtxos(history, offset, limit, sortDir)
+        case None        => Seq.empty[IndexedErgoBox]
+      }
+    }
+
+  private def getBoxesByTokenIdUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byTokenId") & modifierId & paging & sortDir) { (id, offset, limit, dir) =>
+    if (limit > MaxItems) {
+      BadRequest(s"No more than $MaxItems boxes can be requested")
+    } else if (dir == SortDirection.INVALID) {
+      BadRequest("Invalid parameter for sort direction, valid values are 'ASC' and 'DESC'")
+    } else {
+      ApiResponse(getBoxesByTokenIdUnspent(id, offset, limit, dir))
+    }
+  }
+
   private def getUnconfirmedForAddress(address: ErgoAddress)(mempool: ErgoMemPoolReader): BalanceInfo = {
     val bal: BalanceInfo = BalanceInfo()
     mempool.getAll.map(_.transaction).foreach(tx => {

From aa9cb7bac44feec44d9d9bd78b38e8b5ded7bbe9 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Mon, 2 Oct 2023 19:29:32 +0300
Subject: [PATCH 43/58] fixing test - remove transactions which become invalid

---
 src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala
index 5096dec09d..f018569c5e 100644
--- a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala
+++ b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala
@@ -54,7 +54,7 @@ class MempoolAuditorSpec extends AnyFlatSpec with NodeViewTestOps with ErgoTestH
     val boxes = ErgoState.newBoxes(genesis.transactions).find(_.ergoTree == Constants.TrueLeaf)
     boxes.nonEmpty shouldBe true
 
-    val script = s"{sigmaProp(HEIGHT == ${genesis.height})}"
+    val script = s"{sigmaProp(HEIGHT == ${genesis.height} + 1)}"
     val compiler = new SigmaCompiler(ErgoAddressEncoder.MainnetNetworkPrefix)
     val prop = compiler.compile(emptyEnv, script).buildTree
     val tree = ErgoTree.fromProposition(prop.asSigmaProp)

From 1590842186b1581a0171a174cc1ef85e2ad27854 Mon Sep 17 00:00:00 2001
From: Alexander Slesarenko <avslesarenko@gmail.com>
Date: Mon, 2 Oct 2023 20:49:28 +0200
Subject: [PATCH 44/58] with-sigma-v5.0.12: updated to v5.0.12

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index b784648934..a72c075a74 100644
--- a/build.sbt
+++ b/build.sbt
@@ -37,7 +37,7 @@ val circeVersion = "0.13.0"
 val akkaVersion = "2.6.10"
 val akkaHttpVersion = "10.2.4"
 
-val sigmaStateVersion = "5.0.11-43-321b3b04-SNAPSHOT"
+val sigmaStateVersion = "5.0.12"
 
 // for testing current sigmastate build (see sigmastate-ergo-it jenkins job)
 val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion)

From 10bba1e1dcb725d94cf5c427c5132b1cf05151bd Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Tue, 3 Oct 2023 00:48:39 +0300
Subject: [PATCH 45/58] #2037 Add bulk endpoint for  mem pool box fetch

---
 src/main/resources/api/openapi.yaml           | 34 ++++++++++
 .../ergoplatform/http/api/UtxoApiRoute.scala  | 66 ++++++++++++-------
 .../http/routes/UtxoApiRouteSpec.scala        | 35 +++++++---
 3 files changed, 103 insertions(+), 32 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 563abdb14e..40d11aabdf 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -5106,6 +5106,40 @@ paths:
               schema:
                 $ref: '#/components/schemas/ApiError'
 
+  /utxo/withPool/byIdsList:
+    get:
+      summary: Get box contents for a list of boxes by a unique identifier, from UTXO set and also the mempool.
+      operationId: getBoxWithPoolByIdsList
+      tags:
+        - utxo
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                type: string
+      responses:
+        '200':
+          description: Box object
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErgoTransactionOutput'
+        '404':
+          description: Any boxes from the list with this id doesn't exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+        default:
+          description: Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+
   /utxo/withPool/byIdBinary/{boxId}:
     get:
       summary: Get serialized box in Base16 encoding by an identifier, considering also the mempool.
diff --git a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
index 129d0bf809..3a2bfc47c8 100644
--- a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
@@ -6,7 +6,11 @@ import akka.pattern.ask
 import org.ergoplatform.ErgoBox
 import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers}
 import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
-import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoSetSnapshotPersistence, UtxoStateReader}
+import org.ergoplatform.nodeView.state.{
+  ErgoStateReader,
+  UtxoSetSnapshotPersistence,
+  UtxoStateReader
+}
 import org.ergoplatform.wallet.boxes.ErgoBoxSerializer
 import scorex.core.api.http.ApiResponse
 import scorex.core.settings.RESTApiSettings
@@ -15,16 +19,19 @@ import scorex.util.encode.Base16
 
 import scala.concurrent.Future
 
-case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiSettings)
-                       (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute with ApiCodecs {
+case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiSettings)(
+  implicit val context: ActorRefFactory
+) extends ErgoBaseApiRoute
+  with ApiCodecs {
 
-  private def getState: Future[ErgoStateReader] = (readersHolder ? GetReaders).mapTo[Readers].map(_.s)
+  private def getState: Future[ErgoStateReader] =
+    (readersHolder ? GetReaders).mapTo[Readers].map(_.s)
 
   private def getStateAndPool: Future[(ErgoStateReader, ErgoMemPoolReader)] =
     (readersHolder ? GetReaders).mapTo[Readers].map(rs => (rs.s, rs.m))
 
   override val route: Route = pathPrefix("utxo") {
-    byId ~ serializedById ~ genesis ~ withPoolById ~ withPoolSerializedById ~ getBoxesBinaryProof ~ getSnapshotsInfo
+    byId ~ serializedById ~ genesis ~ withPoolById ~ withPoolByIdsList ~ withPoolSerializedById ~ getBoxesBinaryProof ~ getSnapshotsInfo
   }
 
   def withPoolById: Route = (get & path("withPool" / "byId" / Segment)) { id =>
@@ -35,18 +42,28 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS
     })
   }
 
-  def withPoolSerializedById: Route = (get & path("withPool" / "byIdBinary" / Segment)) { id =>
-    ApiResponse(
-      getStateAndPool.map {
+  def withPoolByIdsList: Route =
+    (get & path("withPool" / "byIdsList") & entity(as[Seq[String]])) { ids =>
+      ApiResponse(getStateAndPool.map {
         case (usr: UtxoStateReader, mp) =>
-          usr.withMempool(mp).boxById(ADKey @@ Base16.decode(id).get).map { box =>
-            val bytes = ErgoBoxSerializer.toBytes(box)
-            val boxBytes = Base16.encode(bytes)
-            Map("boxId" -> id, "bytes" -> boxBytes)
-          }
-        case _ => None
-      }
-    )
+          ids.flatMap(id => usr.withMempool(mp).boxById(ADKey @@ Base16.decode(id).get))
+        case _ => Seq.empty
+      })
+    }
+
+  def withPoolSerializedById: Route = (get & path("withPool" / "byIdBinary" / Segment)) {
+    id =>
+      ApiResponse(
+        getStateAndPool.map {
+          case (usr: UtxoStateReader, mp) =>
+            usr.withMempool(mp).boxById(ADKey @@ Base16.decode(id).get).map { box =>
+              val bytes    = ErgoBoxSerializer.toBytes(box)
+              val boxBytes = Base16.encode(bytes)
+              Map("boxId" -> id, "bytes" -> boxBytes)
+            }
+          case _ => None
+        }
+      )
   }
 
   def byId: Route = (get & path("byId" / Segment)) { id =>
@@ -62,7 +79,7 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS
       getState.map {
         case usr: UtxoStateReader =>
           usr.boxById(ADKey @@ Base16.decode(id).get).map { box =>
-            val bytes = ErgoBoxSerializer.toBytes(box)
+            val bytes    = ErgoBoxSerializer.toBytes(box)
             val boxBytes = Base16.encode(bytes)
             Map("boxId" -> id, "bytes" -> boxBytes)
           }
@@ -75,13 +92,14 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS
     ApiResponse(getState.map(_.genesisBoxes))
   }
 
-  def getBoxesBinaryProof: Route = (post & path("getBoxesBinaryProof") & entity(as[Seq[ErgoBox.BoxId]])) { boxes =>
-    ApiResponse(getState.map {
-      case usr: UtxoStateReader =>
-        Some(Base16.encode(usr.generateBatchProofForBoxes(boxes)))
-      case _ => None
-    })
-  }
+  def getBoxesBinaryProof: Route =
+    (post & path("getBoxesBinaryProof") & entity(as[Seq[ErgoBox.BoxId]])) { boxes =>
+      ApiResponse(getState.map {
+        case usr: UtxoStateReader =>
+          Some(Base16.encode(usr.generateBatchProofForBoxes(boxes)))
+        case _ => None
+      })
+    }
 
   /**
     * Handler for /utxo/getSnapshotsInfo API call which is providing list of
diff --git a/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
index 39d2ccf129..29a47a6aac 100644
--- a/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
@@ -14,7 +14,8 @@ import org.scalatest.matchers.should.Matchers
 import scorex.crypto.hash.Blake2b256
 import scorex.util.encode.Base16
 
-class UtxoApiRouteSpec extends AnyFlatSpec
+class UtxoApiRouteSpec
+  extends AnyFlatSpec
   with Matchers
   with ScalatestRouteTest
   with Stubs
@@ -23,10 +24,11 @@ class UtxoApiRouteSpec extends AnyFlatSpec
 
   val prefix = "/utxo"
 
-  val route: Route = UtxoApiRoute(utxoReadersRef, utxoSettings.scorexSettings.restApi).route
+  val route: Route =
+    UtxoApiRoute(utxoReadersRef, utxoSettings.scorexSettings.restApi).route
 
   it should "get utxo box with /byId" in {
-    val box = utxoState.takeBoxes(1).head
+    val box   = utxoState.takeBoxes(1).head
     val boxId = Base16.encode(box.id)
     Get(prefix + s"/byId/$boxId") ~> route ~> check {
       status shouldBe StatusCodes.OK
@@ -36,7 +38,7 @@ class UtxoApiRouteSpec extends AnyFlatSpec
   }
 
   it should "get mempool box with withPool/byId" in {
-    val box = memPool.getAll.map(utx => utx.transaction).flatMap(_.outputs).head
+    val box   = memPool.getAll.map(utx => utx.transaction).flatMap(_.outputs).head
     val boxId = Base16.encode(box.id)
     Get(prefix + s"/byId/$boxId") ~> route ~> check {
       status shouldBe StatusCodes.NotFound
@@ -48,6 +50,19 @@ class UtxoApiRouteSpec extends AnyFlatSpec
     }
   }
 
+  it should "get all mempool boxes with withPool/byIdsList" in {
+    val boxes = memPool.getAll.map(utx => utx.transaction).flatMap(_.outputs)
+    val boxesEncoded = boxes.map(box => Base16.encode(box.id))
+
+    Get(prefix + "/withPool/byIdsList", boxesEncoded.asJson) ~> route ~> check {
+      status shouldBe StatusCodes.OK
+      responseAs[Seq[Json]]
+        .map(_.hcursor.downField("value").as[Long]) shouldEqual boxes.map(x => Right(x.value))
+      responseAs[Seq[Json]]
+        .map(_.hcursor.downField("boxId").as[String]) shouldEqual boxesEncoded.map(x => Right(x))
+    }
+  }
+
   it should "not found utxo box with /byId" in {
     val boxId = Base16.encode(Blake2b256(utxoState.takeBoxes(1).head.id))
     Get(prefix + s"/byId/$boxId") ~> route ~> check {
@@ -56,12 +71,14 @@ class UtxoApiRouteSpec extends AnyFlatSpec
   }
 
   it should "get utxo box with /byIdBinary" in {
-    val box = utxoState.takeBoxes(1).head
+    val box   = utxoState.takeBoxes(1).head
     val boxId = Base16.encode(box.id)
     Get(prefix + s"/byIdBinary/$boxId") ~> route ~> check {
       status shouldBe StatusCodes.OK
       responseAs[Json].hcursor.downField("boxId").as[String] shouldEqual Right(boxId)
-      val bytes = Base16.decode(responseAs[Json].hcursor.downField("bytes").as[String].toOption.get).get
+      val bytes = Base16
+        .decode(responseAs[Json].hcursor.downField("bytes").as[String].toOption.get)
+        .get
       val boxRestored = ErgoBoxSerializer.parseBytes(bytes)
       box shouldEqual boxRestored
     }
@@ -75,7 +92,7 @@ class UtxoApiRouteSpec extends AnyFlatSpec
   }
 
   it should "get pool box with /withPool/byIdBinary" in {
-    val box = memPool.getAll.map(utx => utx.transaction).flatMap(_.outputs).head
+    val box   = memPool.getAll.map(utx => utx.transaction).flatMap(_.outputs).head
     val boxId = Base16.encode(box.id)
     Get(prefix + s"/byIdBinary/$boxId") ~> route ~> check {
       status shouldBe StatusCodes.NotFound
@@ -83,7 +100,9 @@ class UtxoApiRouteSpec extends AnyFlatSpec
     Get(prefix + s"/withPool/byIdBinary/$boxId") ~> route ~> check {
       status shouldBe StatusCodes.OK
       responseAs[Json].hcursor.downField("boxId").as[String] shouldEqual Right(boxId)
-      val bytes = Base16.decode(responseAs[Json].hcursor.downField("bytes").as[String].toOption.get).get
+      val bytes = Base16
+        .decode(responseAs[Json].hcursor.downField("bytes").as[String].toOption.get)
+        .get
       val boxRestored = ErgoBoxSerializer.parseBytes(bytes)
       box shouldEqual boxRestored
     }

From 55e0f0d49ce0b122d738ca2bff3f901686494d8b Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Tue, 3 Oct 2023 00:50:08 +0300
Subject: [PATCH 46/58] #2037 Add bulk endpoint for  mem pool box fetch

---
 src/main/resources/api/openapi.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 40d11aabdf..0eae75d54c 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -5126,7 +5126,9 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ErgoTransactionOutput'
+                type: array
+                items:
+                  $ref: '#/components/schemas/ErgoTransactionOutput'
         '404':
           description: Any boxes from the list with this id doesn't exist
           content:

From 304f13a9e86d226626f4a6904072c99718570b12 Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Tue, 3 Oct 2023 01:56:43 +0300
Subject: [PATCH 47/58] #2037 Add bulk endpoint for  mem pool box fetch

---
 src/main/resources/api/openapi.yaml                             | 2 +-
 src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala     | 2 +-
 .../scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 0eae75d54c..0dc7b40fe6 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -5107,7 +5107,7 @@ paths:
                 $ref: '#/components/schemas/ApiError'
 
   /utxo/withPool/byIdsList:
-    get:
+    post:
       summary: Get box contents for a list of boxes by a unique identifier, from UTXO set and also the mempool.
       operationId: getBoxWithPoolByIdsList
       tags:
diff --git a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
index 3a2bfc47c8..87b876a7a7 100644
--- a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
@@ -43,7 +43,7 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS
   }
 
   def withPoolByIdsList: Route =
-    (get & path("withPool" / "byIdsList") & entity(as[Seq[String]])) { ids =>
+    (post & path("withPool" / "byIdsList") & entity(as[Seq[String]])) { ids =>
       ApiResponse(getStateAndPool.map {
         case (usr: UtxoStateReader, mp) =>
           ids.flatMap(id => usr.withMempool(mp).boxById(ADKey @@ Base16.decode(id).get))
diff --git a/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
index 29a47a6aac..07091634c3 100644
--- a/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
@@ -54,7 +54,7 @@ class UtxoApiRouteSpec
     val boxes = memPool.getAll.map(utx => utx.transaction).flatMap(_.outputs)
     val boxesEncoded = boxes.map(box => Base16.encode(box.id))
 
-    Get(prefix + "/withPool/byIdsList", boxesEncoded.asJson) ~> route ~> check {
+    Post(prefix + "/withPool/byIdsList", boxesEncoded.asJson) ~> route ~> check {
       status shouldBe StatusCodes.OK
       responseAs[Seq[Json]]
         .map(_.hcursor.downField("value").as[Long]) shouldEqual boxes.map(x => Right(x.value))

From 6ea3736a4fa467eced84ca6e71d16275f59e4417 Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Tue, 3 Oct 2023 03:41:20 +0300
Subject: [PATCH 48/58] #2039 Add bulk endpoint for full block by header ids
 fetch

---
 src/main/resources/api/openapi.yaml           | 36 +++++++++++
 .../http/api/BlocksApiRoute.scala             | 10 +++
 .../http/api/ErgoBaseApiRoute.scala           | 63 ++++++++++++++-----
 .../http/routes/BlocksApiRouteSpec.scala      | 55 +++++++++++++---
 4 files changed, 139 insertions(+), 25 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 563abdb14e..41736f5393 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -2721,6 +2721,42 @@ paths:
               schema:
                 $ref: '#/components/schemas/ApiError'
 
+  /blocks/headerIdsList:
+    get:
+      summary: Get the list of full block info by given header ids
+      operationId: getFullBlockByIdsList
+      tags:
+        - blocks
+      requestBody:
+          required: true
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+      responses:
+        '200':
+          description: List of block objects representing the full block data
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/FullBlock'
+        '404':
+          description: Blocks with this ids doesn't exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+        default:
+          description: Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiError'
+
   /blocks/{headerId}/header:
     get:
       summary: Get the block header info by a given header id
diff --git a/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala
index 6a1e9160e8..751d8246bd 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala
@@ -38,6 +38,7 @@ case class BlocksApiRoute(viewHolderRef: ActorRef, readersHolder: ActorRef, ergo
       getChainSliceR ~
       getBlockIdsAtHeightR ~
       getBlockHeaderByHeaderIdR ~
+      getFullBlockByHeaderIdsListR ~
       getBlockTransactionsByHeaderIdR ~
       getProofForTxR ~
       getFullBlockByHeaderIdR ~
@@ -69,6 +70,11 @@ case class BlocksApiRoute(viewHolderRef: ActorRef, readersHolder: ActorRef, ergo
       history.typedModifierById[Header](headerId).flatMap(history.getFullBlock)
     }
 
+  private def getFullBlockByHeaderIdsList(headerIds: Seq[ModifierId]): Future[Seq[ErgoFullBlock]] =
+    getHistory.map { history =>
+      headerIds.flatMap(headerId => history.typedModifierById[Header](headerId).flatMap(history.getFullBlock))
+    }
+
   private def getModifierById(modifierId: ModifierId): Future[Option[BlockSection]] =
     getHistory.map(_.modifierById(modifierId))
 
@@ -177,4 +183,8 @@ case class BlocksApiRoute(viewHolderRef: ActorRef, readersHolder: ActorRef, ergo
     ApiResponse(getFullBlockByHeaderId(id))
   }
 
+  def getFullBlockByHeaderIdsListR: Route = (post & path("headerIdsList") & modifierIds) { ids =>
+    ApiResponse(getFullBlockByHeaderIdsList(ids))
+  }
+
 }
diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
index 498e73fa91..2354787c25 100644
--- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala
@@ -9,7 +9,7 @@ import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
 import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader}
 import org.ergoplatform.settings.{Algos, ErgoSettings}
 import scorex.core.api.http.{ApiError, ApiRoute}
-import scorex.util.{ModifierId, bytesToId}
+import scorex.util.{bytesToId, ModifierId}
 import akka.pattern.ask
 import io.circe.syntax.EncoderOps
 import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction
@@ -20,7 +20,7 @@ import sigmastate.Values.ErgoTree
 import sigmastate.serialization.ErgoTreeSerializer
 
 import scala.concurrent.{ExecutionContextExecutor, Future}
-import scala.util.{Success, Try}
+import scala.util.{Failure, Success, Try}
 
 trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
 
@@ -28,13 +28,32 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
 
   val modifierId: Directive1[ModifierId] = pathPrefix(Segment).flatMap(handleModifierId)
 
+  val modifierIds: Directive1[Seq[ModifierId]] =
+    entity(as[Seq[String]]).flatMap(handleModifierIds)
+
   val modifierIdGet: Directive1[ModifierId] = parameters("id".as[String])
     .flatMap(handleModifierId)
 
   private def handleModifierId(value: String): Directive1[ModifierId] = {
     Algos.decode(value) match {
       case Success(bytes) => provide(bytesToId(bytes))
-      case _ => reject(ValidationRejection("Wrong modifierId format"))
+      case _              => reject(ValidationRejection("Wrong modifierId format"))
+    }
+  }
+
+  private def handleModifierIds(values: Seq[String]): Directive1[Seq[ModifierId]] = {
+    val acc = collection.mutable.Buffer.empty[ModifierId]
+    val err = collection.mutable.Buffer.empty[String]
+    for (value <- values) {
+      Algos.decode(value) match {
+        case Success(bytes) => acc += bytesToId(bytes)
+        case Failure(e)     => err += e.getMessage
+      }
+    }
+    if (err.nonEmpty) {
+      reject(ValidationRejection(s"Wrong modifierId format for: ${err.mkString(",")}"))
+    } else {
+      provide(acc)
     }
   }
 
@@ -49,13 +68,16 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
 
   private def handleErgoTree(value: String): Directive1[ErgoTree] = {
     Base16.decode(fromJsonOrPlain(value)) match {
-      case Success(bytes) => provide(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(bytes))
+      case Success(bytes) =>
+        provide(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(bytes))
       case _ => reject(ValidationRejection("Invalid hex data"))
     }
 
   }
 
-  private def getStateAndPool(readersHolder: ActorRef): Future[(ErgoStateReader, ErgoMemPoolReader)] = {
+  private def getStateAndPool(
+    readersHolder: ActorRef
+  ): Future[(ErgoStateReader, ErgoMemPoolReader)] = {
     (readersHolder ? GetReaders).mapTo[Readers].map { rs =>
       (rs.s, rs.m)
     }
@@ -65,14 +87,18 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
     * Send local transaction to ErgoNodeViewHolder
     * @return Transaction Id with status OK(200), or BadRequest(400)
     */
-  protected def sendLocalTransactionRoute(nodeViewActorRef: ActorRef, unconfirmedTx: UnconfirmedTransaction): Route = {
+  protected def sendLocalTransactionRoute(
+    nodeViewActorRef: ActorRef,
+    unconfirmedTx: UnconfirmedTransaction
+  ): Route = {
     val resultFuture =
       (nodeViewActorRef ? LocallyGeneratedTransaction(unconfirmedTx))
         .mapTo[ProcessingOutcome]
         .flatMap {
           case _: Accepted => Future.successful(unconfirmedTx.transaction.id)
-          case _: DoubleSpendingLoser => Future.failed(new IllegalArgumentException("Double spending attempt"))
-          case d: Declined => Future.failed(d.e)
+          case _: DoubleSpendingLoser =>
+            Future.failed(new IllegalArgumentException("Double spending attempt"))
+          case d: Declined    => Future.failed(d.e)
           case i: Invalidated => Future.failed(i.e)
         }
     completeOrRecoverWith(resultFuture) { ex =>
@@ -87,21 +113,28 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
     * Used in /transactions (POST /transactions and /transactions/check methods) and /wallet (/wallet/payment/send
     * and /wallet/transaction/send) API methods to check submitted or generated transaction
     */
-  protected def verifyTransaction(tx: ErgoTransaction,
-                                  readersHolder: ActorRef,
-                                  ergoSettings: ErgoSettings): Future[Try[UnconfirmedTransaction]] = {
+  protected def verifyTransaction(
+    tx: ErgoTransaction,
+    readersHolder: ActorRef,
+    ergoSettings: ErgoSettings
+  ): Future[Try[UnconfirmedTransaction]] = {
     val now: Long = System.currentTimeMillis()
-    val bytes = Some(tx.bytes)
+    val bytes     = Some(tx.bytes)
 
     getStateAndPool(readersHolder)
       .map {
         case (utxo: UtxoStateReader, mp: ErgoMemPoolReader) =>
           val maxTxCost = ergoSettings.nodeSettings.maxTransactionCost
-          utxo.withMempool(mp)
+          utxo
+            .withMempool(mp)
             .validateWithCost(tx, maxTxCost)
-            .map(cost => new UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None))
+            .map(cost =>
+              new UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None)
+            )
         case _ =>
-          tx.statelessValidity().map(_ => new UnconfirmedTransaction(tx, None, now, now, bytes, source = None))
+          tx.statelessValidity()
+            .map(_ => new UnconfirmedTransaction(tx, None, now, now, bytes, source = None)
+            )
       }
   }
 
diff --git a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
index ed0e328ed4..5e4d824292 100644
--- a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
@@ -3,6 +3,7 @@ package org.ergoplatform.http.routes
 import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes, UniversalEntity}
 import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.testkit.ScalatestRouteTest
+import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport
 import io.circe.syntax._
 import org.ergoplatform.http.api.BlocksApiRoute
 import org.ergoplatform.modifiers.ErgoFullBlock
@@ -13,9 +14,11 @@ import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import scorex.util.ModifierId
 
-class BlocksApiRouteSpec extends AnyFlatSpec
+class BlocksApiRouteSpec
+  extends AnyFlatSpec
   with Matchers
   with ScalatestRouteTest
+  with FailFastCirceSupport
   with Stubs {
 
   val prefix = "/blocks"
@@ -23,19 +26,24 @@ class BlocksApiRouteSpec extends AnyFlatSpec
   val route: Route = BlocksApiRoute(nodeViewRef, digestReadersRef, settings).route
 
   val headerIdBytes: ModifierId = history.lastHeaders(1).headers.head.id
-  val headerIdString: String = Algos.encode(headerIdBytes)
+  val headerIdString: String    = Algos.encode(headerIdBytes)
 
   it should "get last blocks" in {
     Get(prefix) ~> route ~> check {
       status shouldBe StatusCodes.OK
-      history.headerIdsAt(0, 50).map(Algos.encode).asJson.toString() shouldEqual responseAs[String]
+      history
+        .headerIdsAt(0, 50)
+        .map(Algos.encode)
+        .asJson
+        .toString() shouldEqual responseAs[String]
     }
   }
 
   it should "post block correctly" in {
-    val (st, bh) = createUtxoState(settings)
+    val (st, bh)             = createUtxoState(settings)
     val block: ErgoFullBlock = validFullBlock(parentOpt = None, st, bh)
-    val blockJson: UniversalEntity = HttpEntity(block.asJson.toString).withContentType(ContentTypes.`application/json`)
+    val blockJson: UniversalEntity =
+      HttpEntity(block.asJson.toString).withContentType(ContentTypes.`application/json`)
     Post(prefix, blockJson) ~> route ~> check {
       status shouldBe StatusCodes.OK
     }
@@ -44,14 +52,23 @@ class BlocksApiRouteSpec extends AnyFlatSpec
   it should "get last headers" in {
     Get(prefix + "/lastHeaders/1") ~> route ~> check {
       status shouldBe StatusCodes.OK
-      history.lastHeaders(1).headers.map(_.asJson).asJson.toString() shouldEqual responseAs[String]
+      history
+        .lastHeaders(1)
+        .headers
+        .map(_.asJson)
+        .asJson
+        .toString() shouldEqual responseAs[String]
     }
   }
 
   it should "get block at height" in {
     Get(prefix + "/at/0") ~> route ~> check {
       status shouldBe StatusCodes.OK
-      history.headerIdsAtHeight(0).map(Algos.encode).asJson.toString() shouldEqual responseAs[String]
+      history
+        .headerIdsAtHeight(0)
+        .map(Algos.encode)
+        .asJson
+        .toString() shouldEqual responseAs[String]
     }
   }
 
@@ -69,7 +86,8 @@ class BlocksApiRouteSpec extends AnyFlatSpec
   it should "get block by header id" in {
     Get(prefix + "/" + headerIdString) ~> route ~> check {
       status shouldBe StatusCodes.OK
-      val expected = history.typedModifierById[Header](headerIdBytes)
+      val expected = history
+        .typedModifierById[Header](headerIdBytes)
         .flatMap(history.getFullBlock)
         .map(_.asJson)
         .get
@@ -78,6 +96,23 @@ class BlocksApiRouteSpec extends AnyFlatSpec
     }
   }
 
+  it should "get blocks by header ids" in {
+    val headerIdsBytes               = history.lastHeaders(10).headers
+    val headerIdsString: Seq[String] = headerIdsBytes.map(h => Algos.encode(h.id))
+
+    Post(prefix + "/headerIdsList", headerIdsString.asJson) ~> route ~> check {
+      status shouldBe StatusCodes.OK
+
+      val expected = headerIdsBytes
+        .map(_.id)
+        .flatMap(headerId =>
+          history.typedModifierById[Header](headerId).flatMap(history.getFullBlock)
+        )
+
+      responseAs[Seq[ErgoFullBlock]] shouldEqual expected
+    }
+  }
+
   it should "get header by header id" in {
     Get(prefix + "/" + headerIdString + "/header") ~> route ~> check {
       status shouldBe StatusCodes.OK
@@ -94,9 +129,9 @@ class BlocksApiRouteSpec extends AnyFlatSpec
   it should "get transactions by header id" in {
     Get(prefix + "/" + headerIdString + "/transactions") ~> route ~> check {
       status shouldBe StatusCodes.OK
-      val header = history.typedModifierById[Header](headerIdBytes).value
+      val header    = history.typedModifierById[Header](headerIdBytes).value
       val fullBlock = history.getFullBlock(header).value
-      val expected = fullBlock.blockTransactions.asJson.toString
+      val expected  = fullBlock.blockTransactions.asJson.toString
       responseAs[String] shouldEqual expected
     }
   }

From 5bfc075cdf96170720ff3c19df5e82f490cf7829 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Tue, 3 Oct 2023 11:38:52 +0200
Subject: [PATCH 49/58] Fixed incorrect request type

---
 .../scala/org/ergoplatform/http/api/BlockchainApiRoute.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
index c0d173689d..71f54cc1ea 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
@@ -355,7 +355,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
       }
     }
 
-  private def getBoxesByTokenIdUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byTokenId") & modifierId & paging & sortDir) { (id, offset, limit, dir) =>
+  private def getBoxesByTokenIdUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byTokenId") & modifierId & paging & sortDir) { (id, offset, limit, dir) =>
     if (limit > MaxItems) {
       BadRequest(s"No more than $MaxItems boxes can be requested")
     } else if (dir == SortDirection.INVALID) {

From 3c995c2c827c1069aba9e8c915fe7f52bf8baa6d Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Tue, 3 Oct 2023 19:15:39 +0200
Subject: [PATCH 50/58] Fixed incorrect id usage in segment lookup

---
 .../org/ergoplatform/nodeView/history/extra/Segment.scala   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
index c5626e9875..3cb4c6879e 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -201,7 +201,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
       val data: ArrayBuffer[Long] = ArrayBuffer.empty[Long]
       getSegmentsForRange(offset, limit).map(n => math.max(segmentCount - n, 0)).distinct.foreach { num =>
         arraySelector(
-          history.typedExtraIndexById[T](idMod(idOf(id, num))).get
+          history.typedExtraIndexById[T](idMod(idOf(parentId, num))).get
         ) ++=: data
       }
       data ++= (if(offset < array.length) array else Nil)
@@ -249,14 +249,14 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
         var segment: Int = boxSegmentCount
         while(data.length < (limit + offset) && segment > 0) {
           segment -= 1
-          history.typedExtraIndexById[T](idMod(boxSegmentId(id, segment))).get.boxes
+          history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes
             .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) ++=: data
         }
         data.reverse.slice(offset, offset + limit).toArray
       case ASC =>
         var segment: Int = 0
         while(data.length < (limit + offset) && segment < boxSegmentCount) {
-          data ++= history.typedExtraIndexById[T](idMod(boxSegmentId(id, segment))).get.boxes
+          data ++= history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes
             .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
           segment += 1
         }

From 07a68f779699883f8af9c15bb8bd310012723d90 Mon Sep 17 00:00:00 2001
From: jellymlg <erikpanta@gmail.com>
Date: Tue, 3 Oct 2023 20:18:58 +0200
Subject: [PATCH 51/58] Added includeUnconfirmed option to Token API

---
 src/main/resources/api/openapi.yaml           |  7 +++
 .../http/api/BlockchainApiRoute.scala         | 10 ++---
 .../history/extra/IndexedErgoAddress.scala    | 16 +++++--
 .../nodeView/history/extra/IndexedToken.scala | 12 +++++-
 .../nodeView/history/extra/Segment.scala      | 43 +++++++++++++++----
 5 files changed, 69 insertions(+), 19 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index c2da56410f..bc8d683851 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -6130,6 +6130,13 @@ paths:
           schema:
             type: string
             default: desc
+        - in: query
+          name: includeUnconfirmed
+          required: false
+          description: if true include unconfirmed transactions from mempool
+          schema:
+            type: boolean
+            default: false
       responses:
         '200':
           description: unspent boxes associated with wanted token
diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
index bec04b9373..3547f20d50 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
@@ -350,21 +350,21 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     ApiResponse(getBoxesByTokenId(id, offset, limit))
   }
 
-  private def getBoxesByTokenIdUnspent(id: ModifierId, offset: Int, limit: Int, sortDir: Direction): Future[Seq[IndexedErgoBox]] =
-    getHistory.map { history =>
+  private def getBoxesByTokenIdUnspent(id: ModifierId, offset: Int, limit: Int, sortDir: Direction, unconfirmed: Boolean): Future[Seq[IndexedErgoBox]] =
+    getHistoryWithMempool.map { case (history, mempool) =>
       history.typedExtraIndexById[IndexedToken](uniqueId(id)) match {
-        case Some(token) => token.retrieveUtxos(history, offset, limit, sortDir)
+        case Some(token) => token.retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed)
         case None        => Seq.empty[IndexedErgoBox]
       }
     }
 
-  private def getBoxesByTokenIdUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byTokenId") & modifierId & paging & sortDir) { (id, offset, limit, dir) =>
+  private def getBoxesByTokenIdUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byTokenId") & modifierId & paging & sortDir & unconfirmed) { (id, offset, limit, dir, unconfirmed) =>
     if (limit > MaxItems) {
       BadRequest(s"No more than $MaxItems boxes can be requested")
     } else if (dir == SortDirection.INVALID) {
       BadRequest("Invalid parameter for sort direction, valid values are 'ASC' and 'DESC'")
     } else {
-      ApiResponse(getBoxesByTokenIdUnspent(id, offset, limit, dir))
+      ApiResponse(getBoxesByTokenIdUnspent(id, offset, limit, dir, unconfirmed))
     }
   }
 
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
index 2b0d7e6dde..f7408a497a 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala
@@ -1,11 +1,9 @@
 package org.ergoplatform.nodeView.history.extra
 
-import org.ergoplatform.ErgoAddressEncoder
+import org.ergoplatform.{ErgoAddressEncoder, ErgoBox}
 import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes}
-import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getFromSegments, getTxs}
-import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, hashErgoTree, txSegmentId}
-import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
+import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree
 import org.ergoplatform.settings.Algos
 import scorex.core.serialization.ErgoSerializer
 import scorex.util.{ModifierId, bytesToId}
@@ -93,6 +91,16 @@ case class IndexedErgoAddress(treeHash: ModifierId,
 
     toRemove.toArray
   }
+
+  /**
+   * Filter mempool boxes if API call requires it
+   *
+   * @param boxes - all boxes in mempool
+   * @return associated boxes
+   */
+  override private[extra] def filterMempool(boxes: Seq[ErgoBox]): Seq[ErgoBox] =
+    boxes.filter(box => hashErgoTree(box.ergoTree) == treeHash)
+
 }
 
 object IndexedErgoAddressSerializer extends ErgoSerializer[IndexedErgoAddress] {
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
index 725b068fda..061bdf207e 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala
@@ -1,6 +1,6 @@
 package org.ergoplatform.nodeView.history.extra
 
-import org.ergoplatform.ErgoAddressEncoder
+import org.ergoplatform.{ErgoAddressEncoder, ErgoBox}
 import org.ergoplatform.ErgoBox.{R4, R5, R6}
 import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes}
@@ -90,6 +90,16 @@ case class IndexedToken(tokenId: ModifierId,
       findAndModBox(iEb.globalIndex, historyOpt.get)
     this
   }
+
+  /**
+   * Filter mempool boxes if API call requires it
+   *
+   * @param boxes - all boxes in mempool
+   * @return associated boxes
+   */
+  override private[extra] def filterMempool(boxes: Seq[ErgoBox]): Seq[ErgoBox] =
+    boxes.filter(_.additionalTokens.exists(_._1.toModifierId == tokenId))
+
 }
 
 object IndexedTokenSerializer extends ErgoSerializer[IndexedToken] {
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
index 3cb4c6879e..e1f82c7d8b 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala
@@ -1,10 +1,11 @@
 package org.ergoplatform.nodeView.history.extra
 
-import org.ergoplatform.ErgoAddressEncoder
+import org.ergoplatform.{ErgoAddressEncoder, ErgoBox}
 import org.ergoplatform.http.api.SortDirection.{ASC, DESC, Direction}
 import org.ergoplatform.nodeView.history.extra.ExtraIndexer.fastIdToBytes
 import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader}
 import org.ergoplatform.nodeView.history.extra.SegmentSerializer._
+import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
 import org.ergoplatform.sdk.JavaHelpers.Algos
 import scorex.util.serialization.{Reader, Writer}
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
@@ -235,15 +236,22 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
   /**
    * Get a range of the boxes associated with the parent that are NOT spent
    *
-   * @param history - history to use
-   * @param offset  - items to skip from the start
-   * @param limit   - items to retrieve
-   * @param sortDir - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]])
+   * @param history     - history to use
+   * @param mempool     - mempool to use, if unconfirmed is true
+   * @param offset      - items to skip from the start
+   * @param limit       - items to retrieve
+   * @param sortDir     - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]])
+   * @param unconfirmed - whether to include unconfirmed boxes
    * @return array of unspent boxes
    */
-  def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int, sortDir: Direction): Array[IndexedErgoBox] = {
+  def retrieveUtxos(history: ErgoHistoryReader,
+                    mempool: ErgoMemPoolReader,
+                    offset: Int,
+                    limit: Int,
+                    sortDir: Direction,
+                    unconfirmed: Boolean): Seq[IndexedErgoBox] = {
     val data: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox]
-    sortDir match {
+    val confirmedBoxes: Seq[IndexedErgoBox] = sortDir match {
       case DESC =>
         data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
         var segment: Int = boxSegmentCount
@@ -252,7 +260,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
           history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes
             .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) ++=: data
         }
-        data.reverse.slice(offset, offset + limit).toArray
+        data.reverse.slice(offset, offset + limit)
       case ASC =>
         var segment: Int = 0
         while(data.length < (limit + offset) && segment < boxSegmentCount) {
@@ -262,8 +270,17 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
         }
         if (data.length < (limit + offset))
           data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get)
-        data.slice(offset, offset + limit).toArray
+        data.slice(offset, offset + limit)
     }
+    if(unconfirmed) {
+      val mempoolBoxes = filterMempool(mempool.getAll.flatMap(_.transaction.outputs))
+      val unconfirmedBoxes = mempoolBoxes.map(new IndexedErgoBox(0, None, None, _, 0))
+      sortDir match {
+        case DESC => unconfirmedBoxes ++ confirmedBoxes
+        case ASC => confirmedBoxes ++ unconfirmedBoxes
+      }
+    } else
+      confirmedBoxes
   }
 
   /**
@@ -349,6 +366,14 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
    */
   private[extra] def spendBox(iEb: IndexedErgoBox, historyOpt: Option[ErgoHistoryReader] = None)(implicit ae: ErgoAddressEncoder): T
 
+  /**
+   * Filter mempool boxes if API call requires it
+   *
+   * @param boxes - all boxes in mempool
+   * @return associated boxes
+   */
+  private[extra] def filterMempool(boxes: Seq[ErgoBox]): Seq[ErgoBox]
+
 }
 
 object SegmentSerializer {

From 4fe244f05181527c28908856fbfe48e70e4317d6 Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Wed, 4 Oct 2023 16:05:31 +0300
Subject: [PATCH 52/58] #2037 Add bulk endpoint for mem pool box fetch

---
 src/main/resources/api/openapi.yaml                         | 6 +++---
 src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala | 6 +++---
 .../org/ergoplatform/http/routes/UtxoApiRouteSpec.scala     | 4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 0dc7b40fe6..d657c45841 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -5106,10 +5106,10 @@ paths:
               schema:
                 $ref: '#/components/schemas/ApiError'
 
-  /utxo/withPool/byIdsList:
+  /utxo/withPool/byIds:
     post:
-      summary: Get box contents for a list of boxes by a unique identifier, from UTXO set and also the mempool.
-      operationId: getBoxWithPoolByIdsList
+      summary: Get boxes JSONs for ids in array provided, from UTXO or the mempool.
+      operationId: getBoxWithPoolByIds
       tags:
         - utxo
       requestBody:
diff --git a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
index 87b876a7a7..4dac6ef738 100644
--- a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala
@@ -31,7 +31,7 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS
     (readersHolder ? GetReaders).mapTo[Readers].map(rs => (rs.s, rs.m))
 
   override val route: Route = pathPrefix("utxo") {
-    byId ~ serializedById ~ genesis ~ withPoolById ~ withPoolByIdsList ~ withPoolSerializedById ~ getBoxesBinaryProof ~ getSnapshotsInfo
+    byId ~ serializedById ~ genesis ~ withPoolById ~ withPoolByIds ~ withPoolSerializedById ~ getBoxesBinaryProof ~ getSnapshotsInfo
   }
 
   def withPoolById: Route = (get & path("withPool" / "byId" / Segment)) { id =>
@@ -42,8 +42,8 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS
     })
   }
 
-  def withPoolByIdsList: Route =
-    (post & path("withPool" / "byIdsList") & entity(as[Seq[String]])) { ids =>
+  def withPoolByIds: Route =
+    (post & path("withPool" / "byIds") & entity(as[Seq[String]])) { ids =>
       ApiResponse(getStateAndPool.map {
         case (usr: UtxoStateReader, mp) =>
           ids.flatMap(id => usr.withMempool(mp).boxById(ADKey @@ Base16.decode(id).get))
diff --git a/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
index 07091634c3..b5a2fa391b 100644
--- a/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/UtxoApiRouteSpec.scala
@@ -50,11 +50,11 @@ class UtxoApiRouteSpec
     }
   }
 
-  it should "get all mempool boxes with withPool/byIdsList" in {
+  it should "get all mempool boxes with withPool/byIds" in {
     val boxes = memPool.getAll.map(utx => utx.transaction).flatMap(_.outputs)
     val boxesEncoded = boxes.map(box => Base16.encode(box.id))
 
-    Post(prefix + "/withPool/byIdsList", boxesEncoded.asJson) ~> route ~> check {
+    Post(prefix + "/withPool/byIds", boxesEncoded.asJson) ~> route ~> check {
       status shouldBe StatusCodes.OK
       responseAs[Seq[Json]]
         .map(_.hcursor.downField("value").as[Long]) shouldEqual boxes.map(x => Right(x.value))

From 29b8b0fd1895c6dbc3f164172bdc05151692cd14 Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Wed, 4 Oct 2023 15:59:54 +0300
Subject: [PATCH 53/58] #2039 Add bulk endpoint for full block by header ids
 fetch

---
 src/main/resources/api/openapi.yaml                       | 6 +++---
 .../scala/org/ergoplatform/http/api/BlocksApiRoute.scala  | 8 ++++----
 .../org/ergoplatform/http/routes/BlocksApiRouteSpec.scala | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 236698d71e..b9d86e56aa 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -2703,10 +2703,10 @@ paths:
               schema:
                 $ref: '#/components/schemas/ApiError'
 
-  /blocks/headerIdsList:
+  /blocks/headerIds:
     get:
       summary: Get the list of full block info by given header ids
-      operationId: getFullBlockByIdsList
+      operationId: getFullBlockByIds
       tags:
         - blocks
       requestBody:
@@ -2719,7 +2719,7 @@ paths:
                   type: string
       responses:
         '200':
-          description: List of block objects representing the full block data
+          description: Full blocks corresponding to ids provided
           content:
             application/json:
               schema:
diff --git a/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala
index 751d8246bd..ebf82586d8 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala
@@ -38,7 +38,7 @@ case class BlocksApiRoute(viewHolderRef: ActorRef, readersHolder: ActorRef, ergo
       getChainSliceR ~
       getBlockIdsAtHeightR ~
       getBlockHeaderByHeaderIdR ~
-      getFullBlockByHeaderIdsListR ~
+      getFullBlockByHeaderIdsR ~
       getBlockTransactionsByHeaderIdR ~
       getProofForTxR ~
       getFullBlockByHeaderIdR ~
@@ -70,7 +70,7 @@ case class BlocksApiRoute(viewHolderRef: ActorRef, readersHolder: ActorRef, ergo
       history.typedModifierById[Header](headerId).flatMap(history.getFullBlock)
     }
 
-  private def getFullBlockByHeaderIdsList(headerIds: Seq[ModifierId]): Future[Seq[ErgoFullBlock]] =
+  private def getFullBlockByHeaderIds(headerIds: Seq[ModifierId]): Future[Seq[ErgoFullBlock]] =
     getHistory.map { history =>
       headerIds.flatMap(headerId => history.typedModifierById[Header](headerId).flatMap(history.getFullBlock))
     }
@@ -183,8 +183,8 @@ case class BlocksApiRoute(viewHolderRef: ActorRef, readersHolder: ActorRef, ergo
     ApiResponse(getFullBlockByHeaderId(id))
   }
 
-  def getFullBlockByHeaderIdsListR: Route = (post & path("headerIdsList") & modifierIds) { ids =>
-    ApiResponse(getFullBlockByHeaderIdsList(ids))
+  def getFullBlockByHeaderIdsR: Route = (post & path("headerIds") & modifierIds) { ids =>
+    ApiResponse(getFullBlockByHeaderIds(ids))
   }
 
 }
diff --git a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
index 5e4d824292..ab7efade11 100644
--- a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
@@ -100,7 +100,7 @@ class BlocksApiRouteSpec
     val headerIdsBytes               = history.lastHeaders(10).headers
     val headerIdsString: Seq[String] = headerIdsBytes.map(h => Algos.encode(h.id))
 
-    Post(prefix + "/headerIdsList", headerIdsString.asJson) ~> route ~> check {
+    Post(prefix + "/headerIds", headerIdsString.asJson) ~> route ~> check {
       status shouldBe StatusCodes.OK
 
       val expected = headerIdsBytes

From 542963357af95eeccc7992e41897ce60fc8ccc37 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Wed, 4 Oct 2023 16:01:29 +0300
Subject: [PATCH 54/58] pre 5.0.12 sigma import fix

---
 .../org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
index e257e0365d..f0b0e4e34c 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala
@@ -16,7 +16,7 @@ import org.ergoplatform.nodeView.history.storage.HistoryStorage
 import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings}
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
 import sigmastate.Values.ErgoTree
-import special.collection.Extensions._
+import sigma.Extensions._
 
 import java.nio.ByteBuffer
 import scala.collection.mutable.ArrayBuffer

From 890bc6f7376eef5ae3208f5ed48f06f8c59de352 Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Wed, 4 Oct 2023 18:26:42 +0300
Subject: [PATCH 55/58] Fixed BlocksApiRouteSpec Fixed request type

---
 src/main/resources/api/openapi.yaml           |  2 +-
 .../http/routes/BlocksApiRouteSpec.scala      | 26 +++++++++----------
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index b9d86e56aa..aca16042ea 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -2704,7 +2704,7 @@ paths:
                 $ref: '#/components/schemas/ApiError'
 
   /blocks/headerIds:
-    get:
+    post:
       summary: Get the list of full block info by given header ids
       operationId: getFullBlockByIds
       tags:
diff --git a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
index ab7efade11..7edef059f5 100644
--- a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
+++ b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala
@@ -4,6 +4,7 @@ import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes, Universa
 import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.testkit.ScalatestRouteTest
 import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport
+import io.circe.Json
 import io.circe.syntax._
 import org.ergoplatform.http.api.BlocksApiRoute
 import org.ergoplatform.modifiers.ErgoFullBlock
@@ -34,8 +35,7 @@ class BlocksApiRouteSpec
       history
         .headerIdsAt(0, 50)
         .map(Algos.encode)
-        .asJson
-        .toString() shouldEqual responseAs[String]
+        .asJson shouldEqual responseAs[Json]
     }
   }
 
@@ -56,8 +56,7 @@ class BlocksApiRouteSpec
         .lastHeaders(1)
         .headers
         .map(_.asJson)
-        .asJson
-        .toString() shouldEqual responseAs[String]
+        .asJson shouldEqual responseAs[Json]
     }
   }
 
@@ -67,19 +66,18 @@ class BlocksApiRouteSpec
       history
         .headerIdsAtHeight(0)
         .map(Algos.encode)
-        .asJson
-        .toString() shouldEqual responseAs[String]
+        .asJson shouldEqual responseAs[Json]
     }
   }
 
   it should "get chain slice" in {
     Get(prefix + "/chainSlice?fromHeight=0") ~> route ~> check {
       status shouldBe StatusCodes.OK
-      chain.map(_.header).asJson.toString() shouldEqual responseAs[String]
+      chain.map(_.header).asJson shouldEqual responseAs[Json]
     }
     Get(prefix + "/chainSlice?fromHeight=2&toHeight=4") ~> route ~> check {
       status shouldBe StatusCodes.OK
-      chain.slice(2, 4).map(_.header).asJson.toString() shouldEqual responseAs[String]
+      chain.slice(2, 4).map(_.header).asJson shouldEqual responseAs[Json]
     }
   }
 
@@ -91,8 +89,8 @@ class BlocksApiRouteSpec
         .flatMap(history.getFullBlock)
         .map(_.asJson)
         .get
-        .toString
-      responseAs[String] shouldEqual expected
+
+      responseAs[Json] shouldEqual expected
     }
   }
 
@@ -121,8 +119,8 @@ class BlocksApiRouteSpec
         .flatMap(history.getFullBlock)
         .map(_.header.asJson)
         .get
-        .toString
-      responseAs[String] shouldEqual expected
+
+      responseAs[Json] shouldEqual expected
     }
   }
 
@@ -131,8 +129,8 @@ class BlocksApiRouteSpec
       status shouldBe StatusCodes.OK
       val header    = history.typedModifierById[Header](headerIdBytes).value
       val fullBlock = history.getFullBlock(header).value
-      val expected  = fullBlock.blockTransactions.asJson.toString
-      responseAs[String] shouldEqual expected
+      val expected  = fullBlock.blockTransactions.asJson
+      responseAs[Json] shouldEqual expected
     }
   }
 

From 0ebae161e65e4514963b7fa70a7edafbf7c57c86 Mon Sep 17 00:00:00 2001
From: ccellado <taikatytto@disroot.org>
Date: Fri, 6 Oct 2023 04:23:19 +0300
Subject: [PATCH 56/58] Fixed serializing Add activityTime update on handshake

---
 src/main/scala/scorex/core/network/ConnectedPeer.scala     | 1 -
 src/main/scala/scorex/core/network/NetworkController.scala | 5 +++--
 src/main/scala/scorex/core/network/peer/PeerInfo.scala     | 6 ++++--
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/main/scala/scorex/core/network/ConnectedPeer.scala b/src/main/scala/scorex/core/network/ConnectedPeer.scala
index 711e6b89b6..b8e28ccd86 100644
--- a/src/main/scala/scorex/core/network/ConnectedPeer.scala
+++ b/src/main/scala/scorex/core/network/ConnectedPeer.scala
@@ -10,7 +10,6 @@ import scorex.core.network.peer.PeerInfo
   *
   * @param connectionId - connection address
   * @param handlerRef   - reference to PeerConnectionHandler that is responsible for communication with this peer
-  * @param lastMessage  - timestamp of last received message
   * @param peerInfo     - information about this peer. May be None if peer is connected, but is not handshaked yet
   */
 case class ConnectedPeer(connectionId: ConnectionId,
diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala
index bee88fb2a1..4f31089b11 100644
--- a/src/main/scala/scorex/core/network/NetworkController.scala
+++ b/src/main/scala/scorex/core/network/NetworkController.scala
@@ -425,9 +425,10 @@ class NetworkController(ergoSettings: ErgoSettings,
         peerManagerRef ! RemovePeer(peerAddress)
         connections -= connectedPeer.connectionId.remoteAddress
       } else {
-        peerManagerRef ! AddOrUpdatePeer(peerInfo)
+        val newPeerInfo = peerInfo.copy(lastStoredActivityTime = time())
+        peerManagerRef ! AddOrUpdatePeer(newPeerInfo)
 
-        val updatedConnectedPeer = connectedPeer.copy(peerInfo = Some(peerInfo))
+        val updatedConnectedPeer = connectedPeer.copy(peerInfo = Some(newPeerInfo))
         connections += remoteAddress -> updatedConnectedPeer
         context.system.eventStream.publish(HandshakedPeer(updatedConnectedPeer))
       }
diff --git a/src/main/scala/scorex/core/network/peer/PeerInfo.scala b/src/main/scala/scorex/core/network/peer/PeerInfo.scala
index d37bbc2242..b7ba46f025 100644
--- a/src/main/scala/scorex/core/network/peer/PeerInfo.scala
+++ b/src/main/scala/scorex/core/network/peer/PeerInfo.scala
@@ -37,7 +37,7 @@ object PeerInfo {
     */
   def fromAddress(address: InetSocketAddress): PeerInfo = {
     val peerSpec = PeerSpec("unknown", Version.initial, s"unknown-$address", Some(address), Seq())
-    PeerInfo(peerSpec, 0L, None)
+    PeerInfo(peerSpec, 0L, None, 0L)
   }
 
 }
@@ -51,12 +51,14 @@ object PeerInfoSerializer extends ErgoSerializer[PeerInfo] {
     w.putLong(obj.lastHandshake)
     w.putOption(obj.connectionType)((w,d) => w.putBoolean(d.isIncoming))
     PeerSpecSerializer.serialize(obj.peerSpec, w)
+    w.putLong(obj.lastStoredActivityTime)
   }
 
    override def parse(r: Reader): PeerInfo = {
      val lastHandshake = r.getLong()
      val connectionType = r.getOption(if (r.getUByte() != 0) Incoming else Outgoing)
      val peerSpec = PeerSpecSerializer.parse(r)
-     PeerInfo(peerSpec, lastHandshake, connectionType)
+     val lastStoredTime = r.getLong()
+     PeerInfo(peerSpec, lastHandshake, connectionType, lastStoredTime)
    }
 }

From 9ab35f6f7a7e4c236dfc701bae7195d3d077edd4 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Wed, 11 Oct 2023 18:46:28 +0300
Subject: [PATCH 57/58] version increased in openapi-ai

---
 src/main/resources/api/openapi-ai.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/api/openapi-ai.yaml b/src/main/resources/api/openapi-ai.yaml
index 392309a6d1..472b4753e6 100644
--- a/src/main/resources/api/openapi-ai.yaml
+++ b/src/main/resources/api/openapi-ai.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.2"
 
 info:
-  version: "5.0.14"
+  version: "5.0.15"
   title: Ergo Node API
   description: Specification of Ergo Node API for ChatGPT plugin.
     The following endpoints supported 

From 4de322a59ade5f5f6875f80688039b1857b5aa33 Mon Sep 17 00:00:00 2001
From: Alexander Chepurnoy <alex.chepurnoy@iohk.io>
Date: Tue, 17 Oct 2023 23:46:58 +0300
Subject: [PATCH 58/58] lastSeen serialization fix, minor polishing

---
 src/main/resources/api/openapi.yaml                       | 8 ++++----
 .../org/ergoplatform/http/api/BlockchainApiRoute.scala    | 4 ++--
 .../ergoplatform/nodeView/history/extra/BalanceInfo.scala | 2 +-
 .../scala/scorex/core/network/NetworkController.scala     | 2 +-
 src/main/scala/scorex/core/network/peer/PeerInfo.scala    | 4 +---
 5 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml
index 24afaa165c..d7fb5d1859 100644
--- a/src/main/resources/api/openapi.yaml
+++ b/src/main/resources/api/openapi.yaml
@@ -2705,7 +2705,7 @@ paths:
 
   /blocks/headerIds:
     post:
-      summary: Get the list of full block info by given header ids
+      summary: Get full blocks by given header ids
       operationId: getFullBlockByIds
       tags:
         - blocks
@@ -2727,7 +2727,7 @@ paths:
                 items:
                   $ref: '#/components/schemas/FullBlock'
         '404':
-          description: Blocks with this ids doesn't exist
+          description: No block exist for every id provided
           content:
             application/json:
               schema:
@@ -5126,7 +5126,7 @@ paths:
 
   /utxo/withPool/byIds:
     post:
-      summary: Get boxes JSONs for ids in array provided, from UTXO or the mempool.
+      summary: Get boxes for ids provided, from UTXO or the mempool.
       operationId: getBoxWithPoolByIds
       tags:
         - utxo
@@ -5148,7 +5148,7 @@ paths:
                 items:
                   $ref: '#/components/schemas/ErgoTransactionOutput'
         '404':
-          description: Any boxes from the list with this id doesn't exist
+          description: No any box exists for every id provided
           content:
             application/json:
               schema:
diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
index 3547f20d50..e5cb090761 100644
--- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
+++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala
@@ -382,9 +382,9 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     getHistoryWithMempool.map { case (history, mempool) =>
       getAddress(address)(history) match {
         case Some(addr) =>
-          (addr.balanceInfo.get.retreiveAdditionalTokenInfo(history), getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history))
+          (addr.balanceInfo.get.retrieveAdditionalTokenInfo(history), getUnconfirmedForAddress(address)(mempool).retrieveAdditionalTokenInfo(history))
         case None =>
-          (BalanceInfo(), getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history))
+          (BalanceInfo(), getUnconfirmedForAddress(address)(mempool).retrieveAdditionalTokenInfo(history))
       }
     }
   }
diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala
index 67a6fb1a1d..adbd35eecc 100644
--- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala
+++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala
@@ -27,7 +27,7 @@ case class BalanceInfo() extends ScorexLogging {
     * @param history - database handle
     * @return this object
     */
-  def retreiveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = {
+  def retrieveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = {
     additionalTokenInfo ++= tokens.map(token => {
       history.typedExtraIndexById[IndexedToken](uniqueId(token._1)) match {
         case Some(iT) => (token._1,(iT.name,iT.decimals))
diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala
index 4f31089b11..16995e7e69 100644
--- a/src/main/scala/scorex/core/network/NetworkController.scala
+++ b/src/main/scala/scorex/core/network/NetworkController.scala
@@ -301,7 +301,7 @@ class NetworkController(ergoSettings: ErgoSettings,
             val lastSeen = peerInfo.lastStoredActivityTime
             val timeout = networkSettings.inactiveConnectionDeadline.toMillis
             val delta = now - lastSeen
-            if (delta > timeout) {
+            if (delta > timeout && lastSeen > 0) {
               log.info(s"Dropping connection with ${peerInfo}, last seen ${delta / 1000.0} seconds ago")
               cp.handlerRef ! CloseConnection
             }
diff --git a/src/main/scala/scorex/core/network/peer/PeerInfo.scala b/src/main/scala/scorex/core/network/peer/PeerInfo.scala
index b7ba46f025..b95bafa938 100644
--- a/src/main/scala/scorex/core/network/peer/PeerInfo.scala
+++ b/src/main/scala/scorex/core/network/peer/PeerInfo.scala
@@ -51,14 +51,12 @@ object PeerInfoSerializer extends ErgoSerializer[PeerInfo] {
     w.putLong(obj.lastHandshake)
     w.putOption(obj.connectionType)((w,d) => w.putBoolean(d.isIncoming))
     PeerSpecSerializer.serialize(obj.peerSpec, w)
-    w.putLong(obj.lastStoredActivityTime)
   }
 
    override def parse(r: Reader): PeerInfo = {
      val lastHandshake = r.getLong()
      val connectionType = r.getOption(if (r.getUByte() != 0) Incoming else Outgoing)
      val peerSpec = PeerSpecSerializer.parse(r)
-     val lastStoredTime = r.getLong()
-     PeerInfo(peerSpec, lastHandshake, connectionType, lastStoredTime)
+     PeerInfo(peerSpec, lastHandshake, connectionType, lastStoredActivityTime = 0L)
    }
 }