Revert "Decoupling mongo db dependencies to allow adding postgreSQL (b…

…lock-core#213)"

README.md

@@ -66,8 +66,8 @@ The default configurations are read from [appsetting.json](src/Blockcore.Indexer
       // Syncing parameters
       "SyncBlockchain": true,
       "SyncMemoryPool": true,
-      "DbBatchSize": 10000000,
-      "DbBatchCount" : 10000,
+      "MongoBatchSize": 10000000,
+      "MongoBatchCount" : 10000,
       "ParallelRequestsToTransactionRpc": 50,
       "DetailedTrace": 0,
       "MaxItemsInQueue": 10,
@@ -226,4 +226,4 @@ Blockcore.Indexer.Client.BitcoinCommunicationException: Daemon Failed Url = 'htt
  ---> System.Net.Sockets.SocketException (111): Connection refused
 ```
 
-Issue: This happens when the DNS name is not accessible.
+Issue: This happens when the DNS name is not accessible.

src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs

@@ -1,4 +1,4 @@
-using Blockcore.Indexer.Core.Storage.Types;
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
 
 namespace Blockcore.Indexer.Angor.Storage.Mongo.Types;
 

src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs

@@ -3,7 +3,6 @@
 using Blockcore.Indexer.Angor.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Settings;
 using Blockcore.Indexer.Core.Storage.Mongo.Types;
-using Blockcore.Indexer.Core.Storage.Types;
 using Blockcore.Indexer.Core.Sync.SyncTasks;
 using Blockcore.NBitcoin.DataEncoders;
 using Microsoft.Extensions.Options;

src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs

@@ -6,7 +6,6 @@
 using Blockcore.Indexer.Core.Operations.Types;
 using Blockcore.Indexer.Core.Settings;
 using Blockcore.Indexer.Core.Storage.Mongo.Types;
-using Blockcore.Indexer.Core.Storage.Types;
 using Blockcore.Indexer.Core.Sync.SyncTasks;
 using Blockcore.NBitcoin;
 using Blockcore.NBitcoin.BIP32;

src/Blockcore.Indexer.Angor/appsettings.json

@@ -18,7 +18,6 @@
       }
    },
    "Indexer": {
-      "DbType": "MongoDb",
       "ConnectionString": "mongodb://{Symbol}-mongo",
       "DatabaseNameSubfix": true,
 
@@ -35,8 +34,8 @@
       // Syncing parameters
       "SyncBlockchain": true,
       "SyncMemoryPool": true,
-      "DbBatchSize": 10000000,
-      "DbBatchCount" : 10000,
+      "MongoBatchSize": 10000000,
+      "MongoBatchCount" : 10000,
       "ParallelRequestsToTransactionRpc": 50,
       "DetailedTrace": 0,
       "MaxItemsInQueue": 10,

src/Blockcore.Indexer.Cirrus/CirrusStartup.cs

@@ -61,7 +61,6 @@ public void ConfigureServices(IServiceCollection services)
 
          services.AddSingleton<ICirrusStorage, CirrusMongoData>();
          services.AddSingleton<ICirrusMongoDb, CirrusMongoDb>();
-         services.Replace(new ServiceDescriptor(typeof(IStorageBatchFactory), typeof(CirrusStorageBatchFactory),ServiceLifetime.Singleton));
 
          services.AddControllers()
             .AddApplicationPart(typeof(Startup).Assembly)

src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs

@@ -4,10 +4,11 @@
 
 namespace Blockcore.Indexer.Cirrus.Operations.Types
 {
-   public class CirrusStorageBatch : MongoStorageBatch
+   public class CirrusStorageBatch
    {
-      public List<CirrusContractTable> CirrusContractTable { get; set; } = new ();
+      public List<CirrusContractTable> CirrusContractTable { get; set; } = new List<CirrusContractTable>();
+
+      public List<CirrusContractCodeTable> CirrusContractCodeTable { get; set; } = new List<CirrusContractCodeTable>();
 
-      public List<CirrusContractCodeTable> CirrusContractCodeTable { get; set; } = new ();
    }
 }

src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs

@@ -4,7 +4,7 @@
 using Blockcore.Indexer.Cirrus.Storage.Mongo.Types;
 using Blockcore.Indexer.Cirrus.Storage.Types;
 using Blockcore.Indexer.Core.Client.Types;
-using Blockcore.Indexer.Core.Storage.Mongo;
+using Blockcore.Indexer.Core.Storage;
 using Blockcore.Indexer.Core.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Storage.Types;
 using Blockcore.NBitcoin;

src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs

@@ -28,6 +28,7 @@ public class CirrusMongoStorageOperations : MongoStorageOperations
       public CirrusMongoStorageOperations(
          SyncConnection syncConnection,
          IStorage storage,
+         IUtxoCache utxoCache,
          IOptions<IndexerSettings> configuration,
          GlobalState globalState,
          IMapMongoBlockToStorageBlock mongoBlockToStorageBlock,
@@ -37,6 +38,7 @@ public CirrusMongoStorageOperations(
          base(
              syncConnection,
              db,
+             utxoCache,
              configuration,
              globalState,
              mongoBlockToStorageBlock,
@@ -50,7 +52,12 @@ public CirrusMongoStorageOperations(
 
       protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item)
       {
-         CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch;
+         storageBatch.ExtraData ??= new CirrusStorageBatch();
+
+         if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch))
+         {
+            throw new ArgumentNullException(nameof(cirrusStorageBatch));
+         }
 
          foreach (Transaction transaction in item.Transactions)
          {
@@ -118,7 +125,10 @@ protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlock
 
       protected override void OnPushStorageBatch(StorageBatch storageBatch)
       {
-         CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch;
+         if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch))
+         {
+            throw new ArgumentNullException(nameof(cirrusStorageBatch));
+         }
 
          var t1 = Task.Run(() =>
          {

src/Blockcore.Indexer.Cirrus/appsettings.json

@@ -18,7 +18,7 @@
       }
    },
    "Indexer": {
-      "DbType": "MongoDb",
+      // "ConnectionString": "mongodb://localhost",
       "ConnectionString": "mongodb://{Symbol}-mongo",
       "DatabaseNameSubfix": true,
 
@@ -34,8 +34,8 @@
       // Syncing parameters
       "SyncBlockchain": true,
       "SyncMemoryPool": true,
-      "DbBatchSize": 10000000,
-      "DbBatchCount" : 10000,
+      "MongoBatchSize": 10000000,
+      "MongoBatchCount" : 10000,
       "ParallelRequestsToTransactionRpc": 50,
       "DetailedTrace": 0,
       "MaxItemsInQueue": 10,

src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
+using MongoDB.Bson.Serialization.Attributes;
 using Newtonsoft.Json;
 
 namespace Blockcore.Indexer.Core.Client.Types
 {
+   [BsonIgnoreExtraElements]
    public class PeerInfo
    {
       public string Addr { get; set; }

src/Blockcore.Indexer.Core/Controllers/InsightController.cs

@@ -172,7 +172,7 @@ private Supply CalculateSupply()
       /// <returns></returns>
       private List<Wallet> RetrieveWallets()
       {
-         // TODO: Funds should be stored in the DB, and be editable by individual chains and not hard-coded.
+         // TODO: Funds should be stored in MongoDB, and be editable by individual chains and not hard-coded.
          var funds = new List<Wallet>();
 
          List<Wallet> wallets = insightConfiguration.Wallets;
@@ -181,7 +181,7 @@ private List<Wallet> RetrieveWallets()
          {
             if (wallet.Address != null && wallet.Address.Length > 0)
             {
-               var balances = storage.AddressBalances(wallet.Address);
+               List<Storage.Mongo.Types.RichlistTable> balances = storage.AddressBalances(wallet.Address);
                long balance = balances.Sum(b => b.Balance);
                wallet.Balance = balance;
             }
@@ -207,7 +207,7 @@ private decimal CalculateCirculatingSupply()
             }
          }
 
-         var balances = storage.AddressBalances(listOfAddress);
+         List<Storage.Mongo.Types.RichlistTable> balances = storage.AddressBalances(listOfAddress);
          long walletBalances = balances.Sum(b => b.Balance);
 
          long circulatingSupply = totalBalance - walletBalances;

src/Blockcore.Indexer.Core/Controllers/QueryController.cs

@@ -79,7 +79,7 @@ public IActionResult GetAddressTransactions([MinLength(4)][MaxLength(100)] strin
       [Route("address/{address}/transactions/unspent")]
       public async Task<IActionResult> GetAddressTransactionsUnspent([MinLength(30)][MaxLength(100)] string address, long confirmations = 0, [Range(0, int.MaxValue)] int offset = 0, [Range(1, 50)] int limit = 10)
       {
-         var result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit);
+         QueryResult<Storage.Mongo.Types.OutputTable> result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit);
 
          return OkPaging(result);
       }

src/Blockcore.Indexer.Core/Controllers/StatsController.cs

@@ -1,11 +1,10 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
-using Blockcore.Indexer.Core.Client.Types;
 using Blockcore.Indexer.Core.Handlers;
 using Blockcore.Indexer.Core.Models;
 using Blockcore.Indexer.Core.Storage;
+using Blockcore.Indexer.Core.Storage.Mongo;
 using Microsoft.AspNetCore.Mvc;
 
 namespace Blockcore.Indexer.Core.Controllers
@@ -19,15 +18,15 @@ public class StatsController : ControllerBase
    {
       private readonly StatsHandler statsHandler;
 
-      private readonly IStorage storage;
+      private readonly MongoData storage;
 
       /// <summary>
       /// Initializes a new instance of the <see cref="StatsController"/> class.
       /// </summary>
       public StatsController(StatsHandler statsHandler, IStorage storage)
       {
          this.statsHandler = statsHandler;
-         this.storage = storage;
+         this.storage = storage as MongoData;
       }
 
       [HttpGet]
@@ -84,9 +83,7 @@ public async Task<IActionResult> Peers()
       [Route("peers/{date}")]
       public IActionResult Peers(DateTime date)
       {
-         List<PeerInfo> list = storage.GetPeerFromDate(date)
-            .Select(x => x as PeerInfo)
-            .ToList();
+         List<Client.Types.PeerInfo> list = storage.GetPeerFromDate(date);
          return Ok(list);
       }
 

src/Blockcore.Indexer.Core/Models/QueryBlock.cs

@@ -1,9 +1,16 @@
+using System.Collections.Generic;
 
 namespace Blockcore.Indexer.Core.Models
 {
-   public class QueryBlock
+   public class QueryBlockResults
    {
+      public IEnumerable<QueryBlock> Blocks { get; set; }
+
+      public int Total { get; set; }
+   }
 
+   public class QueryBlock
+   {
       /// <summary>
       /// Gets or sets the Symbol.
       /// </summary>
@@ -53,16 +60,16 @@ public class QueryBlock
 
       public string Bits { get; set; }
 
-      public double Difficulty { get; set; }
-
-      public string ChainWork { get; set; }
-
       public string Merkleroot { get; set; }
 
       public long Nonce { get; set; }
 
       public long Version { get; set; }
 
+      public double Difficulty { get; set; }
+
+      public string ChainWork { get; set; }
+
       public string PosBlockSignature { get; set; }
 
       public string PosModifierv2 { get; set; }
@@ -74,5 +81,10 @@ public class QueryBlock
       public string PosBlockTrust { get; set; }
 
       public string PosChainTrust { get; set; }
+
+      /// <summary>
+      /// Gets or sets the transactions.
+      /// </summary>
+      public IEnumerable<QueryTransaction> Transactions { get; set; }
    }
 }

src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs

@@ -1,5 +1,6 @@
 using System;
-using Blockcore.Indexer.Core.Storage.Types;
+using System.Collections.Generic;
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
 
 namespace Blockcore.Indexer.Core.Models
 {
@@ -8,6 +9,6 @@ public class QueryOrphanBlock
       public DateTime Created { get; set; }
       public uint BlockIndex { get; set; }
       public string BlockHash { get; set; }
-      public QueryBlock Block { get; set; }
+      public BlockTable Block { get; set; }
    }
 }

src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs

@@ -13,6 +13,6 @@ public interface IStorageOperations
 
       SyncBlockInfo PushStorageBatch(StorageBatch storageBatch);
 
-      void InsertMempoolTransactions(SyncBlockTransactionsOperation item);
+      InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item);
    }
 }

src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using Blockcore.Indexer.Core.Operations.Types;
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
+
+namespace Blockcore.Indexer.Core.Operations
+{
+   /// <summary>
+   /// Maintain a cache of unspent outputs
+   /// </summary>
+   public interface IUtxoCache
+   {
+      int CacheSize { get; }
+
+      UtxoCacheItem GetOne(string outpoint);
+      void AddToCache(IEnumerable<OutputTable> outputs);
+
+      void RemoveFromCache(IEnumerable<InputTable> inputs);
+   }
+}

src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs

@@ -1,3 +1,6 @@
+using System.Collections.Generic;
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
+
 namespace Blockcore.Indexer.Core.Operations.Types
 {
    #region Using Directives
@@ -20,5 +23,10 @@ public class InsertStats
       /// Gets or sets the outputs.
       /// </summary>
       public int InputsOutputs { get; set; }
+
+      /// <summary>
+      /// Gets or sets the items.
+      /// </summary>
+      public List<MempoolTable> Items { get; set; }
    }
 }