From 5cf9bd1637ea78601e3ea72a5b0840eb337085c6 Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 10 Jun 2024 13:13:43 +0100 Subject: [PATCH 01/16] Moved mongo service registration into mongo namespace --- src/Blockcore.Indexer.Angor/appsettings.json | 1 + .../Storage/CirrusBlockMapping.cs | 2 +- src/Blockcore.Indexer.Cirrus/appsettings.json | 2 +- .../Settings/IndexerSettings.cs | 2 + src/Blockcore.Indexer.Core/Startup.cs | 34 ++++++----------- .../Storage/DatabaseRegistration.cs | 38 +++++++++++++++++++ .../IMapMongoBlockToStorageBlock.cs | 2 +- .../MapMongoBlockToStorageBlock.cs | 2 +- src/Blockcore.Indexer/appsettings.json | 1 + 9 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs rename src/Blockcore.Indexer.Core/Storage/{ => Mongo}/IMapMongoBlockToStorageBlock.cs (85%) rename src/Blockcore.Indexer.Core/Storage/{ => Mongo}/MapMongoBlockToStorageBlock.cs (97%) diff --git a/src/Blockcore.Indexer.Angor/appsettings.json b/src/Blockcore.Indexer.Angor/appsettings.json index 41b5bdd5..ab84dcac 100644 --- a/src/Blockcore.Indexer.Angor/appsettings.json +++ b/src/Blockcore.Indexer.Angor/appsettings.json @@ -18,6 +18,7 @@ } }, "Indexer": { + "DbType": "MongoDb", "ConnectionString": "mongodb://{Symbol}-mongo", "DatabaseNameSubfix": true, diff --git a/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs b/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs index aaed4980..dd1430aa 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs @@ -4,7 +4,7 @@ using Blockcore.Indexer.Cirrus.Storage.Mongo.Types; using Blockcore.Indexer.Cirrus.Storage.Types; using Blockcore.Indexer.Core.Client.Types; -using Blockcore.Indexer.Core.Storage; +using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; using Blockcore.NBitcoin; diff --git a/src/Blockcore.Indexer.Cirrus/appsettings.json b/src/Blockcore.Indexer.Cirrus/appsettings.json index 7e6e450b..557f0660 100644 --- a/src/Blockcore.Indexer.Cirrus/appsettings.json +++ b/src/Blockcore.Indexer.Cirrus/appsettings.json @@ -18,7 +18,7 @@ } }, "Indexer": { - // "ConnectionString": "mongodb://localhost", + "DbType": "MongoDb", "ConnectionString": "mongodb://{Symbol}-mongo", "DatabaseNameSubfix": true, diff --git a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs index 6d4eea37..dced12c1 100644 --- a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs +++ b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs @@ -44,5 +44,7 @@ public class IndexerSettings public int MaxItemsInHistoryQueue { get; set; } public int IndexCountForBlockIndexProperty { get;set; } + + public string DbType { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Startup.cs b/src/Blockcore.Indexer.Core/Startup.cs index 2e58db94..687ccaa1 100644 --- a/src/Blockcore.Indexer.Core/Startup.cs +++ b/src/Blockcore.Indexer.Core/Startup.cs @@ -15,17 +15,13 @@ using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Sync; using Blockcore.Indexer.Core.Sync.SyncTasks; -using Blockcore.Utilities; -using ConcurrentCollections; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.ApplicationModels; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; using Microsoft.Extensions.PlatformAbstractions; using Microsoft.OpenApi.Models; -using MongoDB.Driver; using Newtonsoft.Json; namespace Blockcore.Indexer.Core @@ -39,30 +35,24 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio services.Configure(configuration.GetSection("Indexer")); services.Configure(configuration.GetSection("Insight")); + IndexerSettings indexer = new IndexerSettings(); - services.AddSingleton(_ => - { - var indexerConfiguration = _.GetService(typeof(IOptions))as IOptions ;// configuration.GetSection("Indexer") as IndexerSettings; - var chainConfiguration = _.GetService(typeof(IOptions)) as IOptions;// configuration.GetSection("Chain") as ChainSettings; - - var mongoClient = new MongoClient(indexerConfiguration.Value.ConnectionString.Replace("{Symbol}", - chainConfiguration.Value.Symbol.ToLower())); - - string dbName = indexerConfiguration.Value.DatabaseNameSubfix - ? $"Blockchain{chainConfiguration.Value.Symbol}" - : "Blockchain"; + configuration.GetSection("Indexer").Bind(indexer); - return mongoClient.GetDatabase(dbName); - }); + switch (indexer.DbType) + { + case "MongoDb": + services.AddMongoDatabase(); + break; + default: throw new InvalidOperationException(); + } // services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + services.AddTransient(); services.AddSingleton(); services.AddSingleton(); @@ -142,7 +132,7 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio .AllowAnyHeader(); })); - services.AddTransient(); + services.AddSingleton(); services.AddSingleton(); diff --git a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs new file mode 100644 index 00000000..3d8d5388 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs @@ -0,0 +1,38 @@ +using Blockcore.Indexer.Core.Operations; +using Blockcore.Indexer.Core.Settings; +using Blockcore.Indexer.Core.Storage.Mongo; +using Blockcore.Indexer.Core.Sync.SyncTasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using MongoDB.Driver; + +namespace Blockcore.Indexer.Core.Storage; + +public static class DatabaseRegistration +{ + public static IServiceCollection AddMongoDatabase( this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddTransient(); + + services.AddSingleton(_ => + { + var indexerConfiguration = _.GetService(typeof(IOptions))as IOptions ;// configuration.GetSection("Indexer") as IndexerSettings; + var chainConfiguration = _.GetService(typeof(IOptions)) as IOptions;// configuration.GetSection("Chain") as ChainSettings; + + var mongoClient = new MongoClient(indexerConfiguration.Value.ConnectionString.Replace("{Symbol}", + chainConfiguration.Value.Symbol.ToLower())); + + string dbName = indexerConfiguration.Value.DatabaseNameSubfix + ? $"Blockchain{chainConfiguration.Value.Symbol}" + : "Blockchain"; + + return mongoClient.GetDatabase(dbName); + }); + + return services; + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/IMapMongoBlockToStorageBlock.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/IMapMongoBlockToStorageBlock.cs similarity index 85% rename from src/Blockcore.Indexer.Core/Storage/IMapMongoBlockToStorageBlock.cs rename to src/Blockcore.Indexer.Core/Storage/Mongo/IMapMongoBlockToStorageBlock.cs index 8d93a850..b1060a39 100644 --- a/src/Blockcore.Indexer.Core/Storage/IMapMongoBlockToStorageBlock.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/IMapMongoBlockToStorageBlock.cs @@ -2,7 +2,7 @@ using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; -namespace Blockcore.Indexer.Core.Storage +namespace Blockcore.Indexer.Core.Storage.Mongo { public interface IMapMongoBlockToStorageBlock { diff --git a/src/Blockcore.Indexer.Core/Storage/MapMongoBlockToStorageBlock.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MapMongoBlockToStorageBlock.cs similarity index 97% rename from src/Blockcore.Indexer.Core/Storage/MapMongoBlockToStorageBlock.cs rename to src/Blockcore.Indexer.Core/Storage/Mongo/MapMongoBlockToStorageBlock.cs index a4604ada..6a4b2b6f 100644 --- a/src/Blockcore.Indexer.Core/Storage/MapMongoBlockToStorageBlock.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MapMongoBlockToStorageBlock.cs @@ -3,7 +3,7 @@ using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; -namespace Blockcore.Indexer.Core.Storage +namespace Blockcore.Indexer.Core.Storage.Mongo { public class MapMongoBlockToStorageBlock : IMapMongoBlockToStorageBlock { diff --git a/src/Blockcore.Indexer/appsettings.json b/src/Blockcore.Indexer/appsettings.json index c67bc07e..8b699a70 100644 --- a/src/Blockcore.Indexer/appsettings.json +++ b/src/Blockcore.Indexer/appsettings.json @@ -18,6 +18,7 @@ } }, "Indexer": { + "DbType": "MongoDb", "ConnectionString": "mongodb://{Symbol}-mongo", "DatabaseNameSubfix": true, From 8e956c1aa024d053ca10ef2d8760a4bed851c974 Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 10 Jun 2024 13:33:14 +0100 Subject: [PATCH 02/16] Removed mongo dependency from storage operations class --- .../Models/BalanceForAddress.cs | 7 +++++ .../Storage/IStorage.cs | 8 +++--- .../Storage/Mongo/MongoData.cs | 27 ++++++++++--------- .../Sync/SyncOperations.cs | 15 ++++------- 4 files changed, 32 insertions(+), 25 deletions(-) create mode 100644 src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs diff --git a/src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs b/src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs new file mode 100644 index 00000000..53191459 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs @@ -0,0 +1,7 @@ +namespace Blockcore.Indexer.Core.Models; + +public class BalanceForAddress +{ + public string Address { get; set; } + public long Balance { get; set; } +} diff --git a/src/Blockcore.Indexer.Core/Storage/IStorage.cs b/src/Blockcore.Indexer.Core/Storage/IStorage.cs index 3f21e375..05ff3d3a 100644 --- a/src/Blockcore.Indexer.Core/Storage/IStorage.cs +++ b/src/Blockcore.Indexer.Core/Storage/IStorage.cs @@ -43,9 +43,7 @@ Task> QuickBalancesLookupForAddressesWithHistoryCheckA ReorgBlockTable OrphanBlockByHash(string blockHash); - QueryResult Richlist(int offset, int limit); - - RichlistTable RichlistBalance(string address); + QueryResult Richlist(int offset, int limit); List AddressBalances(IEnumerable addresses); @@ -58,5 +56,9 @@ Task> QuickBalancesLookupForAddressesWithHistoryCheckA public List GetIndexesBuildProgress(); public List GetBlockIndexIndexes(); + + public List GetMempoolTransactionIds(); + + public bool DeleteTransactionsFromMempool(List transactionIds); } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs index 4fd8c3e7..3ce9be17 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs @@ -83,6 +83,18 @@ public List GetBlockIndexIndexes() return indexNames.Where(w => w.Contains("BlockIndex")).ToList(); } + public List GetMempoolTransactionIds() => mongoDb.Mempool.AsQueryable().Select(x => x.TransactionId).ToList(); + + public bool DeleteTransactionsFromMempool(List transactionIds) + { + FilterDefinitionBuilder builder = Builders.Filter; + FilterDefinition filter = builder.In(mempoolItem => mempoolItem.TransactionId, transactionIds); + + var result = mongoDb.Mempool.DeleteMany(filter); + + return result.IsAcknowledged; //TODO should we change this to count == count of transaction ids? + } + public List GetIndexesBuildProgress() { IMongoDatabase db = mongoDatabase.Client.GetDatabase("admin"); @@ -474,7 +486,7 @@ public SyncTransactionItems TransactionItemsGet(string transactionId, Transactio return ret; } - public QueryResult Richlist(int offset, int limit) + public QueryResult Richlist(int offset, int limit) { FilterDefinitionBuilder filterBuilder = Builders.Filter; FilterDefinition filter = filterBuilder.Empty; @@ -502,17 +514,8 @@ public QueryResult Richlist(int offset, int limit) .Limit(limit) .ToList(); - return new QueryResult { Items = list, Total = total, Offset = offset, Limit = limit }; - } - - public RichlistTable RichlistBalance(string address) - { - FilterDefinitionBuilder filterBuilder = Builders.Filter; - FilterDefinition filter = filterBuilder.Eq(m => m.Address, address); - - RichlistTable table = mongoDb.RichlistTable.Find(filter).SingleOrDefault(); - - return table; + return new QueryResult { Items = list.Select(x => new BalanceForAddress + { Address = x.Address,Balance = x.Balance }), Total = total, Offset = offset, Limit = limit }; } public List AddressBalances(IEnumerable addresses) diff --git a/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs b/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs index 19038a39..dfae0d66 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs @@ -26,7 +26,6 @@ namespace Blockcore.Indexer.Core.Sync public class SyncOperations : ISyncOperations { private readonly IStorage storage; - private readonly IMongoDb db; private readonly ILogger log; @@ -50,7 +49,7 @@ public SyncOperations( IOptions configuration, IMemoryCache cache, GlobalState globalState, ICryptoClientFactory clientFactory, - ISyncBlockTransactionOperationBuilder blockInfoEnrichment, IMongoDb db) + ISyncBlockTransactionOperationBuilder blockInfoEnrichment) { this.configuration = configuration.Value; log = logger; @@ -59,7 +58,6 @@ public SyncOperations( this.globalState = globalState; this.clientFactory = clientFactory; transactionOperationBuilder = blockInfoEnrichment; - this.db = db; // Register the cold staking template. StandardScripts.RegisterStandardScriptTemplate(ColdStakingScriptTemplate.Instance); @@ -69,11 +67,11 @@ public SyncOperations( public void InitializeMmpool() { - var allitems = db.Mempool.AsQueryable().ToList(); + var mempoolTransactionIds = storage.GetMempoolTransactionIds(); - foreach (MempoolTable allitem in allitems) + foreach (var transactionId in mempoolTransactionIds) { - globalState.LocalMempoolView.TryAdd(allitem.TransactionId, string.Empty); + globalState.LocalMempoolView.TryAdd(transactionId, string.Empty); } } @@ -169,10 +167,7 @@ private SyncPoolTransactions FindPoolInternal(SyncConnection connection) { List toRemoveFromMempool = deleteTransaction; - FilterDefinitionBuilder builder = Builders.Filter; - FilterDefinition filter = builder.In(mempoolItem => mempoolItem.TransactionId, toRemoveFromMempool); - - db.Mempool.DeleteMany(filter); + storage.DeleteTransactionsFromMempool(toRemoveFromMempool); foreach (string mempooltrx in toRemoveFromMempool) globalState.LocalMempoolView.Remove(mempooltrx, out _); From 3703b909b8a9a9c0f5e11e72cbf0840d609dd1c6 Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 10 Jun 2024 15:30:29 +0100 Subject: [PATCH 03/16] Moved the rich list scanning into mongo (this is actually only relevant for address based blockchains unlike BTC) --- src/Blockcore.Indexer.Core/Startup.cs | 1 - .../Storage/DatabaseRegistration.cs | 2 ++ .../Mongo/RichListScanning.cs} | 12 +++++------- 3 files changed, 7 insertions(+), 8 deletions(-) rename src/Blockcore.Indexer.Core/{Sync/SyncTasks/RichListSync.cs => Storage/Mongo/RichListScanning.cs} (95%) diff --git a/src/Blockcore.Indexer.Core/Startup.cs b/src/Blockcore.Indexer.Core/Startup.cs index 687ccaa1..2d770c36 100644 --- a/src/Blockcore.Indexer.Core/Startup.cs +++ b/src/Blockcore.Indexer.Core/Startup.cs @@ -72,7 +72,6 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio services.AddScoped(); services.AddScoped(); - services.AddScoped(); // TODO: Verify that it is OK we add this to shared Startup for Blockcore and Cirrus. services.AddScoped(); diff --git a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs index 3d8d5388..34323d11 100644 --- a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs +++ b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs @@ -17,6 +17,8 @@ public static IServiceCollection AddMongoDatabase( this IServiceCollection servi services.AddSingleton(); services.AddSingleton(); services.AddTransient(); + //TODO add this for address driven blockchains + //services.AddScoped(); services.AddSingleton(_ => { diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/RichListSync.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/RichListScanning.cs similarity index 95% rename from src/Blockcore.Indexer.Core/Sync/SyncTasks/RichListSync.cs rename to src/Blockcore.Indexer.Core/Storage/Mongo/RichListScanning.cs index cf800103..2dcaa23b 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/RichListSync.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/RichListScanning.cs @@ -4,21 +4,19 @@ using System.Linq; using System.Threading.Tasks; using Blockcore.Indexer.Core.Settings; -using Blockcore.Indexer.Core.Storage; -using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Sync.SyncTasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -using MongoDB.Driver.Core.Operations; -namespace Blockcore.Indexer.Core.Sync.SyncTasks +namespace Blockcore.Indexer.Core.Storage.Mongo { - public class RichListSync : TaskRunner + public class RichListScanning : TaskRunner { private readonly IMongoDb db; - private readonly ILogger log; + private readonly ILogger log; private readonly IndexerSettings indexerSettings; private readonly Stopwatch watch; @@ -26,7 +24,7 @@ public class RichListSync : TaskRunner private bool syncInProgress; DateTime lastSync; - public RichListSync(IOptions configuration, ILogger logger,IMongoDb data) + public RichListScanning(IOptions configuration, ILogger logger,IMongoDb data) : base(configuration, logger) { db = data; From 0f7c93cd1fece01b398ff4b87994e95dec78016c Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 10 Jun 2024 15:41:50 +0100 Subject: [PATCH 04/16] The block indexing sync task is dedicated to mongo db and has been moved into the mongo namespace --- src/Blockcore.Indexer.Core/Startup.cs | 2 -- .../Storage/DatabaseRegistration.cs | 2 ++ .../Mongo/SyncTasks/MongoDbBlockIndexer.cs} | 16 ++++++---------- .../Mongo/{ => SyncTasks}/RichListScanning.cs | 2 +- 4 files changed, 9 insertions(+), 13 deletions(-) rename src/Blockcore.Indexer.Core/{Sync/SyncTasks/BlockIndexer.cs => Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs} (96%) rename src/Blockcore.Indexer.Core/Storage/Mongo/{ => SyncTasks}/RichListScanning.cs (99%) diff --git a/src/Blockcore.Indexer.Core/Startup.cs b/src/Blockcore.Indexer.Core/Startup.cs index 2d770c36..151d985a 100644 --- a/src/Blockcore.Indexer.Core/Startup.cs +++ b/src/Blockcore.Indexer.Core/Startup.cs @@ -71,8 +71,6 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio services.AddScoped(); services.AddScoped(); - services.AddScoped(); - // TODO: Verify that it is OK we add this to shared Startup for Blockcore and Cirrus. services.AddScoped(); services.AddSingleton(); diff --git a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs index 34323d11..78ced171 100644 --- a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs +++ b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs @@ -1,6 +1,7 @@ using Blockcore.Indexer.Core.Operations; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo; +using Blockcore.Indexer.Core.Storage.Mongo.SyncTasks; using Blockcore.Indexer.Core.Sync.SyncTasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -17,6 +18,7 @@ public static IServiceCollection AddMongoDatabase( this IServiceCollection servi services.AddSingleton(); services.AddSingleton(); services.AddTransient(); + services.AddScoped(); //TODO add this for address driven blockchains //services.AddScoped(); diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs similarity index 96% rename from src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs rename to src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs index 80051f76..c393ca3f 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs @@ -4,23 +4,19 @@ using System.Text; using System.Threading.Tasks; using Blockcore.Indexer.Core.Extensions; -using Blockcore.Indexer.Core.Operations; -using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; -using Blockcore.Indexer.Core.Storage; -using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Sync.SyncTasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using MongoDB.Bson; using MongoDB.Driver; -namespace Blockcore.Indexer.Core.Sync.SyncTasks +namespace Blockcore.Indexer.Core.Storage.Mongo.SyncTasks { - public class BlockIndexer : TaskRunner + public class MongoDbBlockIndexer : TaskRunner { private readonly IndexerSettings config; - private readonly ILogger log; + private readonly ILogger log; readonly IStorage data; private readonly System.Diagnostics.Stopwatch watch; @@ -31,9 +27,9 @@ public class BlockIndexer : TaskRunner Task indexingCompletTask; bool initialized; - public BlockIndexer( + public MongoDbBlockIndexer( IOptions configuration, - ILogger logger, + ILogger logger, IStorage data, IMongoDb db) : base(configuration, logger) { diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/RichListScanning.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/RichListScanning.cs similarity index 99% rename from src/Blockcore.Indexer.Core/Storage/Mongo/RichListScanning.cs rename to src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/RichListScanning.cs index 2dcaa23b..b0730f06 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/RichListScanning.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/RichListScanning.cs @@ -11,7 +11,7 @@ using MongoDB.Bson; using MongoDB.Driver; -namespace Blockcore.Indexer.Core.Storage.Mongo +namespace Blockcore.Indexer.Core.Storage.Mongo.SyncTasks { public class RichListScanning : TaskRunner { From c9368004995c1f2c62a252c61c5b502d8a64f265 Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 10 Jun 2024 16:09:32 +0100 Subject: [PATCH 05/16] Moved Mongo driver dependency into mongo namespace --- src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs | 2 -- .../Storage/Mongo/MongoBuilder.cs | 10 ++++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs b/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs index dadd19a8..09954cc7 100644 --- a/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs +++ b/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs @@ -1,11 +1,9 @@ using System; using System.Collections.Generic; -using MongoDB.Bson.Serialization.Attributes; using Newtonsoft.Json; namespace Blockcore.Indexer.Core.Client.Types { - [BsonIgnoreExtraElements] public class PeerInfo { public string Addr { get; set; } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs index 286777de..810bd21f 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; @@ -143,6 +144,15 @@ public override Task OnExecute() }); } + if (!MongoDB.Bson.Serialization.BsonClassMap.IsClassMapRegistered(typeof(PeerInfo))) + { + MongoDB.Bson.Serialization.BsonClassMap.RegisterClassMap(cm => + { + cm.AutoMap(); + cm.SetIgnoreExtraElements(true); + }); + } + mongoDb.UnspentOutputTable.Indexes .CreateOne(new CreateIndexModel(Builders .IndexKeys.Hashed(trxBlk => trxBlk.Outpoint))); From 451391ba58a25de3a93c240dbafd56db2e08931b Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 10 Jun 2024 16:52:10 +0100 Subject: [PATCH 06/16] Removed stats syncer and stats controller dependency on mongo namespace --- .../Controllers/StatsController.cs | 11 ++++--- .../Storage/IStorage.cs | 4 +++ .../Storage/Mongo/IMongoDb.cs | 3 +- .../Storage/Mongo/MongoBuilder.cs | 6 ++-- .../Storage/Mongo/MongoData.cs | 9 ++---- .../Storage/Mongo/MongoDb.cs | 8 ++--- .../Storage/Types/PeerDetails.cs | 8 +++++ .../Sync/SyncTasks/StatsSyncer.cs | 29 ++++++++++++++++--- 8 files changed, 55 insertions(+), 23 deletions(-) create mode 100644 src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs diff --git a/src/Blockcore.Indexer.Core/Controllers/StatsController.cs b/src/Blockcore.Indexer.Core/Controllers/StatsController.cs index 4b5b2b13..43813688 100644 --- a/src/Blockcore.Indexer.Core/Controllers/StatsController.cs +++ b/src/Blockcore.Indexer.Core/Controllers/StatsController.cs @@ -1,10 +1,11 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Handlers; using Blockcore.Indexer.Core.Models; using Blockcore.Indexer.Core.Storage; -using Blockcore.Indexer.Core.Storage.Mongo; using Microsoft.AspNetCore.Mvc; namespace Blockcore.Indexer.Core.Controllers @@ -18,7 +19,7 @@ public class StatsController : ControllerBase { private readonly StatsHandler statsHandler; - private readonly MongoData storage; + private readonly IStorage storage; /// /// Initializes a new instance of the class. @@ -26,7 +27,7 @@ public class StatsController : ControllerBase public StatsController(StatsHandler statsHandler, IStorage storage) { this.statsHandler = statsHandler; - this.storage = storage as MongoData; + this.storage = storage; } [HttpGet] @@ -83,7 +84,9 @@ public async Task Peers() [Route("peers/{date}")] public IActionResult Peers(DateTime date) { - List list = storage.GetPeerFromDate(date); + List list = storage.GetPeerFromDate(date) + .Select(x => x as PeerInfo) + .ToList(); return Ok(list); } diff --git a/src/Blockcore.Indexer.Core/Storage/IStorage.cs b/src/Blockcore.Indexer.Core/Storage/IStorage.cs index 05ff3d3a..20c7e1a6 100644 --- a/src/Blockcore.Indexer.Core/Storage/IStorage.cs +++ b/src/Blockcore.Indexer.Core/Storage/IStorage.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Threading.Tasks; using Blockcore.Indexer.Core.Models; @@ -60,5 +61,8 @@ Task> QuickBalancesLookupForAddressesWithHistoryCheckA public List GetMempoolTransactionIds(); public bool DeleteTransactionsFromMempool(List transactionIds); + + List GetPeerFromDate(DateTime date); + Task InsertPeer(PeerDetails info); } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs index feb49a64..b11dfc92 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs @@ -1,5 +1,6 @@ using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using MongoDB.Driver; namespace Blockcore.Indexer.Core.Storage.Mongo; @@ -15,7 +16,7 @@ public interface IMongoDb IMongoCollection TransactionTable { get; } IMongoCollection BlockTable { get; } IMongoCollection RichlistTable { get; } - IMongoCollection Peer { get; } + IMongoCollection Peer { get; } IMongoCollection Mempool { get; } IMongoCollection ReorgBlock { get; } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs index 810bd21f..32f89829 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs @@ -1,7 +1,7 @@ using System.Threading.Tasks; -using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -144,9 +144,9 @@ public override Task OnExecute() }); } - if (!MongoDB.Bson.Serialization.BsonClassMap.IsClassMapRegistered(typeof(PeerInfo))) + if (!MongoDB.Bson.Serialization.BsonClassMap.IsClassMapRegistered(typeof(PeerDetails))) { - MongoDB.Bson.Serialization.BsonClassMap.RegisterClassMap(cm => + MongoDB.Bson.Serialization.BsonClassMap.RegisterClassMap(cm => { cm.AutoMap(); cm.SetIgnoreExtraElements(true); diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs index 3ce9be17..dac7faaf 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs @@ -5,9 +5,7 @@ using Blockcore.Consensus.ScriptInfo; using Blockcore.Consensus.TransactionInfo; using Blockcore.Indexer.Core.Client; -using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Crypto; -using Blockcore.Indexer.Core.Extensions; using Blockcore.Indexer.Core.Models; using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; @@ -19,7 +17,6 @@ using MongoDB.Bson; using MongoDB.Driver; using Blockcore.NBitcoin.DataEncoders; -using Blockcore.Utilities; namespace Blockcore.Indexer.Core.Storage.Mongo { @@ -304,7 +301,7 @@ public ReorgBlockTable OrphanBlockByHash(string blockHash) /// /// /// - public async Task InsertPeer(PeerInfo info) + public async Task InsertPeer(PeerDetails info) { // Always update the LastSeen. info.LastSeen = DateTime.UtcNow; @@ -314,9 +311,9 @@ public async Task InsertPeer(PeerInfo info) return replaceOneResult.ModifiedCount; } - public List GetPeerFromDate(DateTime date) + public List GetPeerFromDate(DateTime date) { - FilterDefinition filter = Builders.Filter.Gt(addr => addr.LastSeen, date); + FilterDefinition filter = Builders.Filter.Gt(addr => addr.LastSeen, date); return mongoDb.Peer.Find(filter).ToList(); } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs index c7e02fda..a27d5d5b 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs @@ -1,9 +1,7 @@ using Blockcore.Indexer.Core.Client.Types; -using Blockcore.Indexer.Core.Operations.Types; -using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using MongoDB.Driver; namespace Blockcore.Indexer.Core.Storage.Mongo @@ -93,11 +91,11 @@ public IMongoCollection RichlistTable } } - public IMongoCollection Peer + public IMongoCollection Peer { get { - return mongoDatabase.GetCollection("Peer"); + return mongoDatabase.GetCollection("Peer"); } } diff --git a/src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs b/src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs new file mode 100644 index 00000000..0e4e79ec --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs @@ -0,0 +1,8 @@ +using Blockcore.Indexer.Core.Client.Types; + +namespace Blockcore.Indexer.Core.Storage.Types; + +public class PeerDetails : PeerInfo +{ + +} diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs index f3b89689..1aa00581 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs @@ -8,7 +8,7 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage; -using Blockcore.Indexer.Core.Storage.Mongo; +using Blockcore.Indexer.Core.Storage.Types; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -25,7 +25,7 @@ public class StatsSyncer : TaskRunner private readonly IStorageOperations storageOperations; - private readonly MongoData data; + private readonly IStorage data; private readonly System.Diagnostics.Stopwatch watch; @@ -43,7 +43,7 @@ public StatsSyncer( { log = logger; - data = (MongoData)storage; + data = storage; this.statsHandler = statsHandler; this.storageOperations = storageOperations; @@ -67,7 +67,28 @@ public override async Task OnExecute() // TODO: Look into potential optimization on updates, not replacing the whole document for all connected nodes all the time. foreach (PeerInfo peer in peers) { - await data.InsertPeer(peer); + await data.InsertPeer(new PeerDetails + { + Addr = peer.Addr, + AddrLocal = peer.AddrLocal, + Version = peer.Version, + Inbound = peer.Inbound, + Services = peer.Services, + BanScore = peer.BanScore, + BytesRecv = peer.BytesRecv, + BytesSent = peer.BytesSent, + ConnTime = peer.ConnTime, + PingTime = peer.PingTime, + InFlight = peer.InFlight, + LastRecv = peer.LastRecv, + LastSeen = peer.LastSeen, + LastSend = peer.LastSend, + StartingHeight = peer.StartingHeight, + SubVer = peer.SubVer, + SyncedBlocks = peer.SyncedBlocks, + SyncedHeaders = peer.SyncedHeaders, + WhiteListed = peer.WhiteListed + }); } log.LogInformation($"Time taken to update peers in database: {watch.Elapsed.TotalSeconds}."); From f97b1f3415b62544e5d69c882b463b3ccce3175e Mon Sep 17 00:00:00 2001 From: TheDude Date: Wed, 12 Jun 2024 14:53:08 +0100 Subject: [PATCH 07/16] Removed mongo dependency from insight controller --- .../Controllers/InsightController.cs | 4 ++-- src/Blockcore.Indexer.Core/Storage/IStorage.cs | 2 +- src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs | 9 +++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Blockcore.Indexer.Core/Controllers/InsightController.cs b/src/Blockcore.Indexer.Core/Controllers/InsightController.cs index 4eaf7872..990be93a 100644 --- a/src/Blockcore.Indexer.Core/Controllers/InsightController.cs +++ b/src/Blockcore.Indexer.Core/Controllers/InsightController.cs @@ -181,7 +181,7 @@ private List RetrieveWallets() { if (wallet.Address != null && wallet.Address.Length > 0) { - List balances = storage.AddressBalances(wallet.Address); + var balances = storage.AddressBalances(wallet.Address); long balance = balances.Sum(b => b.Balance); wallet.Balance = balance; } @@ -207,7 +207,7 @@ private decimal CalculateCirculatingSupply() } } - List balances = storage.AddressBalances(listOfAddress); + var balances = storage.AddressBalances(listOfAddress); long walletBalances = balances.Sum(b => b.Balance); long circulatingSupply = totalBalance - walletBalances; diff --git a/src/Blockcore.Indexer.Core/Storage/IStorage.cs b/src/Blockcore.Indexer.Core/Storage/IStorage.cs index 20c7e1a6..319a3b9f 100644 --- a/src/Blockcore.Indexer.Core/Storage/IStorage.cs +++ b/src/Blockcore.Indexer.Core/Storage/IStorage.cs @@ -46,7 +46,7 @@ Task> QuickBalancesLookupForAddressesWithHistoryCheckA QueryResult Richlist(int offset, int limit); - List AddressBalances(IEnumerable addresses); + List AddressBalances(IEnumerable addresses); long TotalBalance(); diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs index dac7faaf..260eb082 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs @@ -515,14 +515,15 @@ public QueryResult Richlist(int offset, int limit) { Address = x.Address,Balance = x.Balance }), Total = total, Offset = offset, Limit = limit }; } - public List AddressBalances(IEnumerable addresses) + public List AddressBalances(IEnumerable addresses) { FilterDefinitionBuilder filterBuilder = Builders.Filter; FilterDefinition filter = filterBuilder.Where(s => addresses.Contains(s.Address)); - List document = mongoDb.RichlistTable.Find(filter).ToList(); - - return document; + return mongoDb.RichlistTable.Find(filter) + .ToList() + .Select(x => new BalanceForAddress { Balance = x.Balance, Address = x.Address }) + .ToList(); } public long TotalBalance() From a10d5c71a505005bb1cd1d9ca3c9cb0a7461c066 Mon Sep 17 00:00:00 2001 From: TheDude Date: Wed, 12 Jun 2024 15:22:52 +0100 Subject: [PATCH 08/16] Removed mongo dependency from query controller --- .../Storage/Mongo/Types/Investment.cs | 2 +- .../SyncTasks/ProjectTransactionsSyncRunner.cs | 1 + .../Sync/SyncTasks/ProjectsSyncRunner.cs | 1 + .../Controllers/QueryController.cs | 2 +- .../Storage/Mongo/BlockRewindOperation.cs | 1 + .../Mongo/Types/AddressUtxoComputedTable.cs | 2 ++ .../Storage/Mongo/Types/InputTable.cs | 2 ++ .../Storage/Mongo/Types/MempoolTable.cs | 1 + .../Storage/Mongo/Types/OutputTable.cs | 12 +++--------- .../Storage/Mongo/Types/UnspentOutputTable.cs | 2 ++ .../Storage/{Mongo => }/Types/Outpoint.cs | 2 +- src/Blockcore.Indexer.Core/Storage/Types/Output.cs | 14 ++++++++++++++ .../Storage/Mongo/MongoDataTests.cs | 1 + .../Storage/Mongo/MongoStorageOperationsTests.cs | 1 + 14 files changed, 32 insertions(+), 12 deletions(-) rename src/Blockcore.Indexer.Core/Storage/{Mongo => }/Types/Outpoint.cs (81%) create mode 100644 src/Blockcore.Indexer.Core/Storage/Types/Output.cs diff --git a/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs b/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs index d82d9ca8..23e44732 100644 --- a/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs +++ b/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs @@ -1,4 +1,4 @@ -using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; namespace Blockcore.Indexer.Angor.Storage.Mongo.Types; diff --git a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs index 68f8233b..4b6a864d 100644 --- a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs +++ b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs @@ -3,6 +3,7 @@ using Blockcore.Indexer.Angor.Storage.Mongo.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; using Blockcore.NBitcoin.DataEncoders; using Microsoft.Extensions.Options; diff --git a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs index 3cd749b8..6becda85 100644 --- a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs +++ b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs @@ -6,6 +6,7 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; using Blockcore.NBitcoin; using Blockcore.NBitcoin.BIP32; diff --git a/src/Blockcore.Indexer.Core/Controllers/QueryController.cs b/src/Blockcore.Indexer.Core/Controllers/QueryController.cs index 44ceabf3..64f7a3fe 100644 --- a/src/Blockcore.Indexer.Core/Controllers/QueryController.cs +++ b/src/Blockcore.Indexer.Core/Controllers/QueryController.cs @@ -79,7 +79,7 @@ public IActionResult GetAddressTransactions([MinLength(4)][MaxLength(100)] strin [Route("address/{address}/transactions/unspent")] public async Task GetAddressTransactionsUnspent([MinLength(30)][MaxLength(100)] string address, long confirmations = 0, [Range(0, int.MaxValue)] int offset = 0, [Range(1, 50)] int limit = 10) { - QueryResult result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit); + var result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit); return OkPaging(result); } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs index a3636188..74fe76c4 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs @@ -4,6 +4,7 @@ using System.Linq.Expressions; using System.Threading.Tasks; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using MongoDB.Bson; using MongoDB.Driver; diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs index b3a3d5d8..e3278b1d 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs @@ -1,3 +1,5 @@ +using Blockcore.Indexer.Core.Storage.Types; + namespace Blockcore.Indexer.Core.Storage.Mongo.Types { /// diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs index d65bbc60..a8ccfb04 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs @@ -1,3 +1,5 @@ +using Blockcore.Indexer.Core.Storage.Types; + namespace Blockcore.Indexer.Core.Storage.Mongo.Types { public class InputTable diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs index 72e45f1b..768b7888 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using Blockcore.Indexer.Core.Storage.Types; namespace Blockcore.Indexer.Core.Storage.Mongo.Types { diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs index f4fac3fc..03829feb 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs @@ -1,15 +1,9 @@ +using Blockcore.Indexer.Core.Storage.Types; + namespace Blockcore.Indexer.Core.Storage.Mongo.Types { - public class OutputTable + public class OutputTable : Output { - public Outpoint Outpoint { get; set; } - - public string Address { get; set; } - public string ScriptHex { get; set; } - public long Value { get; set; } - public uint BlockIndex { get; set; } - public bool CoinBase { get; set; } - public bool CoinStake { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs index 973c8302..b532027c 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs @@ -1,3 +1,5 @@ +using Blockcore.Indexer.Core.Storage.Types; + namespace Blockcore.Indexer.Core.Storage.Mongo.Types; public class UnspentOutputTable diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/Outpoint.cs b/src/Blockcore.Indexer.Core/Storage/Types/Outpoint.cs similarity index 81% rename from src/Blockcore.Indexer.Core/Storage/Mongo/Types/Outpoint.cs rename to src/Blockcore.Indexer.Core/Storage/Types/Outpoint.cs index 8ae6ff15..1bf86263 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/Outpoint.cs +++ b/src/Blockcore.Indexer.Core/Storage/Types/Outpoint.cs @@ -1,4 +1,4 @@ -namespace Blockcore.Indexer.Core.Storage.Mongo.Types +namespace Blockcore.Indexer.Core.Storage.Types { public class Outpoint { diff --git a/src/Blockcore.Indexer.Core/Storage/Types/Output.cs b/src/Blockcore.Indexer.Core/Storage/Types/Output.cs new file mode 100644 index 00000000..d313766b --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Types/Output.cs @@ -0,0 +1,14 @@ +namespace Blockcore.Indexer.Core.Storage.Types; + +public class Output +{ + public Outpoint Outpoint { get; set; } + + public string Address { get; set; } + + public string ScriptHex { get; set; } + public long Value { get; set; } + public uint BlockIndex { get; set; } + public bool CoinBase { get; set; } + public bool CoinStake { get; set; } +} diff --git a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs index 81ed5aef..cf9fdb75 100644 --- a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs +++ b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs @@ -11,6 +11,7 @@ using Blockcore.Indexer.Core.Storage; using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Networks; using Microsoft.Extensions.Options; using Moq; diff --git a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs index 99b7c37c..a00004fa 100644 --- a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs +++ b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs @@ -14,6 +14,7 @@ using Blockcore.Indexer.Core.Storage; using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; using Blockcore.NBitcoin; using Blockcore.Networks; using FluentAssertions; From 1ff847e86e94611c9602153b3e85c36e45ff4898 Mon Sep 17 00:00:00 2001 From: TheDude Date: Wed, 12 Jun 2024 15:45:58 +0100 Subject: [PATCH 09/16] Removed mongo dependency from QueryOrphanBlock.cs --- .../Models/QueryBlock.cs | 22 ++++---------- .../Models/QueryBlockResults.cs | 10 +++++++ .../Models/QueryOrphanBlock.cs | 5 ++-- .../Storage/Mongo/MongoData.cs | 30 ++++++++++++++++++- .../Sync/SyncOperations.cs | 6 +--- 5 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs diff --git a/src/Blockcore.Indexer.Core/Models/QueryBlock.cs b/src/Blockcore.Indexer.Core/Models/QueryBlock.cs index 8c50a81d..b48136d5 100644 --- a/src/Blockcore.Indexer.Core/Models/QueryBlock.cs +++ b/src/Blockcore.Indexer.Core/Models/QueryBlock.cs @@ -1,16 +1,9 @@ -using System.Collections.Generic; namespace Blockcore.Indexer.Core.Models { - public class QueryBlockResults - { - public IEnumerable Blocks { get; set; } - - public int Total { get; set; } - } - public class QueryBlock { + /// /// Gets or sets the Symbol. /// @@ -60,16 +53,16 @@ public class QueryBlock public string Bits { get; set; } + public double Difficulty { get; set; } + + public string ChainWork { get; set; } + public string Merkleroot { get; set; } public long Nonce { get; set; } public long Version { get; set; } - public double Difficulty { get; set; } - - public string ChainWork { get; set; } - public string PosBlockSignature { get; set; } public string PosModifierv2 { get; set; } @@ -81,10 +74,5 @@ public class QueryBlock public string PosBlockTrust { get; set; } public string PosChainTrust { get; set; } - - /// - /// Gets or sets the transactions. - /// - public IEnumerable Transactions { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs b/src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs new file mode 100644 index 00000000..2935610f --- /dev/null +++ b/src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; + +namespace Blockcore.Indexer.Core.Models; + +public class QueryBlockResults +{ + public IEnumerable Blocks { get; set; } + + public int Total { get; set; } +} diff --git a/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs b/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs index 502986fe..dc8019b1 100644 --- a/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs +++ b/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs @@ -1,6 +1,5 @@ using System; -using System.Collections.Generic; -using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Types; namespace Blockcore.Indexer.Core.Models { @@ -9,6 +8,6 @@ public class QueryOrphanBlock public DateTime Created { get; set; } public uint BlockIndex { get; set; } public string BlockHash { get; set; } - public BlockTable Block { get; set; } + public QueryBlock Block { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs index 260eb082..165189f3 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs @@ -281,7 +281,7 @@ public QueryResult OrphanBlocks(int? offset, int limit) BlockHash = s.BlockHash, BlockIndex = s.BlockIndex, Created = s.Created, - Block = s.Block + Block = MapQueryBlock(s.Block) }), Total = total, Offset = itemsToSkip, @@ -289,6 +289,34 @@ public QueryResult OrphanBlocks(int? offset, int limit) }; } + private static QueryBlock MapQueryBlock(BlockTable blockTable) + { + return new QueryBlock() + { + BlockHash = blockTable.BlockHash, + BlockIndex = blockTable.BlockIndex, + BlockSize = blockTable.BlockSize, + BlockTime = blockTable.BlockTime, + NextBlockHash = blockTable.NextBlockHash, + PreviousBlockHash = blockTable.PreviousBlockHash, + Confirmations = blockTable.Confirmations, + Bits = blockTable.Bits, + Difficulty = blockTable.Difficulty, + ChainWork = blockTable.ChainWork, + Merkleroot = blockTable.Merkleroot, + Nonce = blockTable.Nonce, + Version = blockTable.Version, + Synced = blockTable.SyncComplete, + TransactionCount = blockTable.TransactionCount, + PosBlockSignature = blockTable.PosBlockSignature, + PosModifierv2 = blockTable.PosModifierv2, + PosFlags = blockTable.PosFlags, + PosHashProof = blockTable.PosHashProof, + PosBlockTrust = blockTable.PosBlockTrust, + PosChainTrust = blockTable.PosChainTrust, + }; + } + public ReorgBlockTable OrphanBlockByHash(string blockHash) { FilterDefinition filter = Builders.Filter.Eq(info => info.BlockHash, blockHash); diff --git a/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs b/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs index dfae0d66..98838c9b 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Blockcore.Consensus.BlockInfo; using Blockcore.Consensus.ScriptInfo; using Blockcore.Consensus.TransactionInfo; using Blockcore.Indexer.Core.Client; @@ -10,13 +9,10 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage; -using Blockcore.Indexer.Core.Storage.Mongo; -using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using MongoDB.Driver; namespace Blockcore.Indexer.Core.Sync { @@ -231,7 +227,7 @@ private SyncBlockTransactionsOperation SyncBlockInternal(SyncConnection connecti string hex = client.GetBlockHex(block.Hash); - var blockItem = Block.Parse(hex, connection.Network.Consensus.ConsensusFactory); + var blockItem = Consensus.BlockInfo.Block.Parse(hex, connection.Network.Consensus.ConsensusFactory); foreach (Transaction blockItemTransaction in blockItem.Transactions) { From 752147b74a213f634d01e640b9d6484db6a43128 Mon Sep 17 00:00:00 2001 From: TheDude Date: Wed, 12 Jun 2024 15:55:34 +0100 Subject: [PATCH 10/16] Removed mongo dependency from InsertStats class --- .../Operations/IStorageOperations.cs | 2 +- .../Operations/Types/InsertStats.cs | 8 -------- .../Storage/Mongo/MongoStorageOperations.cs | 4 +--- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs b/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs index ad437454..01a4c055 100644 --- a/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs +++ b/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs @@ -13,6 +13,6 @@ public interface IStorageOperations SyncBlockInfo PushStorageBatch(StorageBatch storageBatch); - InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item); + void InsertMempoolTransactions(SyncBlockTransactionsOperation item); } } diff --git a/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs b/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs index 0631a30b..ad148d41 100644 --- a/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs +++ b/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs @@ -1,6 +1,3 @@ -using System.Collections.Generic; -using Blockcore.Indexer.Core.Storage.Mongo.Types; - namespace Blockcore.Indexer.Core.Operations.Types { #region Using Directives @@ -23,10 +20,5 @@ public class InsertStats /// Gets or sets the outputs. /// public int InputsOutputs { get; set; } - - /// - /// Gets or sets the items. - /// - public List Items { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs index 0891f3da..ffb4014d 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs @@ -271,7 +271,7 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) return block; } - public InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item) + public void InsertMempoolTransactions(SyncBlockTransactionsOperation item) { var mempool = new List(); var inputs = new Dictionary(); @@ -349,8 +349,6 @@ public InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item foreach (MempoolTable mempooltrx in mempool) globalState.LocalMempoolView.TryAdd(mempooltrx.TransactionId, string.Empty); - - return new InsertStats { Items = mempool }; } protected virtual void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) From 1e0c71e60738c3ee56f4b2c44a28e243771f712b Mon Sep 17 00:00:00 2001 From: TheDude Date: Wed, 12 Jun 2024 19:51:18 +0100 Subject: [PATCH 11/16] Commented out unused code that has a dependency on mongo namespace --- .../Operations/IUtxoCache.cs | 18 ++-- .../Operations/Types/UtxoCache.cs | 88 +++++++++---------- .../Mongo/MongoStorageOperationsTests.cs | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs b/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs index 57a5089f..5e872a51 100644 --- a/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs +++ b/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs @@ -1,6 +1,6 @@ -using System.Collections.Generic; -using Blockcore.Indexer.Core.Operations.Types; -using Blockcore.Indexer.Core.Storage.Mongo.Types; +// using System.Collections.Generic; +// using Blockcore.Indexer.Core.Operations.Types; +// using Blockcore.Indexer.Core.Storage.Mongo.Types; namespace Blockcore.Indexer.Core.Operations { @@ -9,11 +9,11 @@ namespace Blockcore.Indexer.Core.Operations /// public interface IUtxoCache { - int CacheSize { get; } - - UtxoCacheItem GetOne(string outpoint); - void AddToCache(IEnumerable outputs); - - void RemoveFromCache(IEnumerable inputs); + // int CacheSize { get; } + // + // UtxoCacheItem GetOne(string outpoint); + // void AddToCache(IEnumerable outputs); + // + // void RemoveFromCache(IEnumerable inputs); } } diff --git a/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs b/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs index f4636bc6..4740f508 100644 --- a/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs +++ b/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs @@ -1,8 +1,8 @@ -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Microsoft.Extensions.Logging; +// using System.Collections.Concurrent; +// using System.Collections.Generic; +// using System.Linq; +// using Blockcore.Indexer.Core.Storage.Mongo.Types; +// using Microsoft.Extensions.Logging; namespace Blockcore.Indexer.Core.Operations.Types { @@ -14,44 +14,44 @@ public class UtxoCacheItem public class UtxoCache : IUtxoCache { - private readonly ILogger logger; - private readonly ConcurrentDictionary cache; - - private readonly int maxItemInCache = 0; - - public UtxoCache(ILogger logger) - { - this.logger = logger; - cache = new ConcurrentDictionary(); - } - - public int CacheSize { get { return cache.Count; } } - - public UtxoCacheItem GetOne(string outpoint) - { - if (cache.TryGetValue(outpoint, out UtxoCacheItem utxo)) - { - return new UtxoCacheItem { Value = utxo.Value, Address = utxo.Address }; - } - - return null; - } - - public void AddToCache(IEnumerable outputs) - { - int maxToAdd = maxItemInCache - cache.Count; - foreach (OutputTable output in outputs.Take(maxToAdd)) - { - cache.TryAdd($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", new UtxoCacheItem { Value = output.Value, Address = output.Address }); - } - } - - public void RemoveFromCache(IEnumerable inputs) - { - foreach (InputTable output in inputs) - { - cache.TryRemove($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", out _); - } - } + // private readonly ILogger logger; + // private readonly ConcurrentDictionary cache; + // + // private readonly int maxItemInCache = 0; + // + // public UtxoCache(ILogger logger) + // { + // this.logger = logger; + // cache = new ConcurrentDictionary(); + // } + // + // public int CacheSize { get { return cache.Count; } } + // + // public UtxoCacheItem GetOne(string outpoint) + // { + // if (cache.TryGetValue(outpoint, out UtxoCacheItem utxo)) + // { + // return new UtxoCacheItem { Value = utxo.Value, Address = utxo.Address }; + // } + // + // return null; + // } + // + // public void AddToCache(IEnumerable outputs) + // { + // int maxToAdd = maxItemInCache - cache.Count; + // foreach (OutputTable output in outputs.Take(maxToAdd)) + // { + // cache.TryAdd($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", new UtxoCacheItem { Value = output.Value, Address = output.Address }); + // } + // } + // + // public void RemoveFromCache(IEnumerable inputs) + // { + // foreach (InputTable output in inputs) + // { + // cache.TryRemove($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", out _); + // } + // } } } diff --git a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs index a00004fa..89209d26 100644 --- a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs +++ b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs @@ -87,7 +87,7 @@ public MongoStorageOperationsTests() sut = new MongoStorageOperations(syncConnection, mongodbMock.MongoDbObject, - new UtxoCache(null), + new UtxoCache(), indexSettingsMock.Object, globalState, new MapMongoBlockToStorageBlock(), From 382c3096438cf1098137aa1f3e8317a65bd44ef7 Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 17 Jun 2024 13:49:09 +0100 Subject: [PATCH 12/16] Renamed settings properties from prefix mongo to db --- README.md | 6 +++--- src/Blockcore.Indexer.Angor/appsettings.json | 4 ++-- src/Blockcore.Indexer.Cirrus/appsettings.json | 4 ++-- src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs | 4 ++-- src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs | 4 ++-- src/Blockcore.Indexer/appsettings.json | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 799fe26d..3c01a21a 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ The default configurations are read from [appsetting.json](src/Blockcore.Indexer // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "MongoBatchSize": 10000000, - "MongoBatchCount" : 10000, + "DbBatchSize": 10000000, + "DbBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10, @@ -226,4 +226,4 @@ Blockcore.Indexer.Client.BitcoinCommunicationException: Daemon Failed Url = 'htt ---> System.Net.Sockets.SocketException (111): Connection refused ``` -Issue: This happens when the DNS name is not accessible. \ No newline at end of file +Issue: This happens when the DNS name is not accessible. diff --git a/src/Blockcore.Indexer.Angor/appsettings.json b/src/Blockcore.Indexer.Angor/appsettings.json index ab84dcac..7ff113db 100644 --- a/src/Blockcore.Indexer.Angor/appsettings.json +++ b/src/Blockcore.Indexer.Angor/appsettings.json @@ -35,8 +35,8 @@ // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "MongoBatchSize": 10000000, - "MongoBatchCount" : 10000, + "DbBatchSize": 10000000, + "DbBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10, diff --git a/src/Blockcore.Indexer.Cirrus/appsettings.json b/src/Blockcore.Indexer.Cirrus/appsettings.json index 557f0660..be7b749c 100644 --- a/src/Blockcore.Indexer.Cirrus/appsettings.json +++ b/src/Blockcore.Indexer.Cirrus/appsettings.json @@ -34,8 +34,8 @@ // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "MongoBatchSize": 10000000, - "MongoBatchCount" : 10000, + "DbBatchSize": 10000000, + "DbBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10, diff --git a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs index dced12c1..e511f1ad 100644 --- a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs +++ b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs @@ -32,8 +32,8 @@ public class IndexerSettings public int NotifyBatchCount { get; set; } - public int MongoBatchSize { get; set; } - public int MongoBatchCount { get; set; } + public int DbBatchSize { get; set; } + public int DbBatchCount { get; set; } public int AverageInterval { get; set; } diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs index ccdbf2f5..bdd38a5e 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs @@ -211,8 +211,8 @@ private void ProcessFromBlockingCollection() bool bip30Issue = bip30Blocks.Contains(block.BlockInfo.Height + 1); if (ibd && !bip30Issue && - currentStorageBatch.BlockTable.Count < config.MongoBatchCount && - currentStorageBatch.TotalSize <= config.MongoBatchSize) + currentStorageBatch.BlockTable.Count < config.DbBatchCount && + currentStorageBatch.TotalSize <= config.DbBatchSize) { continue; } diff --git a/src/Blockcore.Indexer/appsettings.json b/src/Blockcore.Indexer/appsettings.json index 8b699a70..b646ac7a 100644 --- a/src/Blockcore.Indexer/appsettings.json +++ b/src/Blockcore.Indexer/appsettings.json @@ -34,8 +34,8 @@ // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "MongoBatchSize": 10000000, - "MongoBatchCount" : 10000, + "DbBatchSize": 10000000, + "DbBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10, From 65b9f0813e99a858daa49fde062bff4912900dbe Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 17 Jun 2024 14:00:58 +0100 Subject: [PATCH 13/16] Removed mongo dependency from core tasks --- .../Sync/SyncTasks/BlockStartup.cs | 7 +++---- .../Sync/SyncTasks/BlockStore.cs | 12 ++++-------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs index 44befc85..34df35c7 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs @@ -7,7 +7,6 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage; -using Blockcore.Indexer.Core.Storage.Mongo; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -25,7 +24,7 @@ public class BlockStartup : TaskStarter private readonly SyncConnection connection; private readonly IStorageOperations storageOperations; readonly ICryptoClientFactory clientFactory; - private readonly MongoData mongoData; + private readonly IStorage storageData; readonly IOptions indexerSettings; /// @@ -48,7 +47,7 @@ public BlockStartup( this.syncOperations = syncOperations; log = logger; - mongoData = (MongoData)data; + storageData = data; } @@ -67,7 +66,7 @@ public override async Task OnExecute() { IBlockchainClient client = clientFactory.Create(connection); - List allIndexes = mongoData.GetBlockIndexIndexes(); + List allIndexes = storageData.GetBlockIndexIndexes(); if (allIndexes.Count == indexerSettings.Value.IndexCountForBlockIndexProperty) { diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs index 3446dc2c..df61b976 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs @@ -6,7 +6,6 @@ using Blockcore.Indexer.Core.Operations; using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; -using Blockcore.Indexer.Core.Storage.Mongo.Types; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -23,7 +22,6 @@ public class BlockStore : TaskRunner private readonly ISyncOperations syncOperations; private readonly SyncConnection syncConnection; - readonly IUtxoCache utxoCache; private readonly System.Diagnostics.Stopwatch watch; @@ -35,14 +33,12 @@ public BlockStore( ILogger logger, IStorageOperations storageOperations, ISyncOperations syncOperations, - SyncConnection syncConnection, - IUtxoCache utxoCache) + SyncConnection syncConnection) : base(configuration, logger) { this.storageOperations = storageOperations; this.syncOperations = syncOperations; this.syncConnection = syncConnection; - this.utxoCache = utxoCache; log = logger; watch = Stopwatch.Start(); } @@ -50,7 +46,7 @@ public BlockStore( /// public override async Task OnExecute() { - if (Runner.GlobalState.ReorgMode == true) + if (Runner.GlobalState.ReorgMode) { // null the store tip so the document count will be taken form disk Runner.GlobalState.StoreTip = null; @@ -89,7 +85,7 @@ public override async Task OnExecute() log.LogInformation($"Store - blocks={batch.BlockTable.Count}, outputs={batch.OutputTable.Count}, inputs={batch.InputTable.Count}, trx={batch.TransactionBlockTable.Count}, total Size = {((decimal)batch.TotalSize / 1000000):0.00}mb, tip={Runner.GlobalState.StoreTip.BlockIndex}, Seconds = {watch.Elapsed.TotalSeconds}, inserts = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b)"); - foreach (BlockTable mapBlocksValue in batch.BlockTable.Values) + foreach (var mapBlocksValue in batch.BlockTable.Values) syncConnection.RecentItems.Add((DateTime.UtcNow, TimeSpan.FromSeconds(blocksPerSecond), mapBlocksValue.BlockSize)); var notifications = new AddressNotifications { Addresses = new List() };// count.Items.Where(ad => ad.Addresses != null).SelectMany(s => s.Addresses).Distinct().ToList() }; @@ -103,7 +99,7 @@ void ValidateBatch(StorageBatch item) { // check all blocks are consecutive and start from the last block in store. string prevHash = Runner.GlobalState.StoreTip.BlockHash; - foreach (BlockTable mapBlock in item.BlockTable.Values.OrderBy(b => b.BlockIndex)) + foreach (var mapBlock in item.BlockTable.Values.OrderBy(b => b.BlockIndex)) { if (mapBlock.PreviousBlockHash != prevHash) { From 1a6be60bee73d0e27e4e57f7b953dfa8db28d898 Mon Sep 17 00:00:00 2001 From: TheDude Date: Mon, 17 Jun 2024 15:33:35 +0100 Subject: [PATCH 14/16] Fixed text in controller --- src/Blockcore.Indexer.Core/Controllers/InsightController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Blockcore.Indexer.Core/Controllers/InsightController.cs b/src/Blockcore.Indexer.Core/Controllers/InsightController.cs index 990be93a..aa224ae8 100644 --- a/src/Blockcore.Indexer.Core/Controllers/InsightController.cs +++ b/src/Blockcore.Indexer.Core/Controllers/InsightController.cs @@ -172,7 +172,7 @@ private Supply CalculateSupply() /// private List RetrieveWallets() { - // TODO: Funds should be stored in MongoDB, and be editable by individual chains and not hard-coded. + // TODO: Funds should be stored in the DB, and be editable by individual chains and not hard-coded. var funds = new List(); List wallets = insightConfiguration.Wallets; From 2851a26e9b750f120deaf9dc46163336cfafcd29 Mon Sep 17 00:00:00 2001 From: TheDude Date: Tue, 18 Jun 2024 13:10:13 +0100 Subject: [PATCH 15/16] Decoupled the storage batch from Mongo db dependency --- src/Blockcore.Indexer.Cirrus/CirrusStartup.cs | 1 + .../Operations/Types/CirrusStorageBatch.cs | 7 ++- .../Mongo/CirrusMongoStorageOperations.cs | 12 +---- .../Mongo/CirrusStorageBatchFactory.cs | 10 ++++ .../Operations/Types/StorageBatch.cs | 17 +++--- .../Storage/DatabaseRegistration.cs | 1 + .../Storage/IStorageBatchFactory.cs | 8 +++ .../Storage/Mongo/MongoStorageBatchFactory.cs | 9 ++++ .../Storage/Mongo/MongoStorageOperations.cs | 54 ++++++++++--------- .../Storage/Mongo/Types/MongoStorageBatch.cs | 43 +++++++++++++++ .../Sync/SyncTasks/BlockPuller.cs | 19 ++++--- .../Sync/SyncTasks/BlockStartup.cs | 7 ++- .../Sync/SyncTasks/BlockStore.cs | 26 +++------ .../Mongo/MongoStorageOperationsTests.cs | 40 +++++++------- 14 files changed, 157 insertions(+), 97 deletions(-) create mode 100644 src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs diff --git a/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs b/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs index 099ec747..c10ed857 100644 --- a/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs +++ b/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs @@ -61,6 +61,7 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); + services.Replace(new ServiceDescriptor(typeof(IStorageBatchFactory), typeof(CirrusStorageBatchFactory),ServiceLifetime.Singleton)); services.AddControllers() .AddApplicationPart(typeof(Startup).Assembly) diff --git a/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs b/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs index b249d72a..607e54a4 100644 --- a/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs +++ b/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs @@ -4,11 +4,10 @@ namespace Blockcore.Indexer.Cirrus.Operations.Types { - public class CirrusStorageBatch + public class CirrusStorageBatch : MongoStorageBatch { - public List CirrusContractTable { get; set; } = new List(); - - public List CirrusContractCodeTable { get; set; } = new List(); + public List CirrusContractTable { get; set; } = new (); + public List CirrusContractCodeTable { get; set; } = new (); } } diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs index bab42737..590bf16d 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs @@ -52,12 +52,7 @@ public CirrusMongoStorageOperations( protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) { - storageBatch.ExtraData ??= new CirrusStorageBatch(); - - if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch)) - { - throw new ArgumentNullException(nameof(cirrusStorageBatch)); - } + CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch; foreach (Transaction transaction in item.Transactions) { @@ -125,10 +120,7 @@ protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlock protected override void OnPushStorageBatch(StorageBatch storageBatch) { - if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch)) - { - throw new ArgumentNullException(nameof(cirrusStorageBatch)); - } + CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch; var t1 = Task.Run(() => { diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs new file mode 100644 index 00000000..15cc8869 --- /dev/null +++ b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs @@ -0,0 +1,10 @@ +using Blockcore.Indexer.Cirrus.Operations.Types; +using Blockcore.Indexer.Core.Operations.Types; +using Blockcore.Indexer.Core.Storage; + +namespace Blockcore.Indexer.Cirrus.Storage.Mongo; + +public class CirrusStorageBatchFactory : IStorageBatchFactory +{ + public StorageBatch GetStorageBatch() => new CirrusStorageBatch(); +} diff --git a/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs b/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs index f6a5084e..057866d4 100644 --- a/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs +++ b/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using Blockcore.Indexer.Core.Storage.Mongo.Types; namespace Blockcore.Indexer.Core.Operations.Types { @@ -7,15 +6,15 @@ namespace Blockcore.Indexer.Core.Operations.Types #endregion Using Directives - public class StorageBatch + public abstract class StorageBatch { - public long TotalSize { get; set; } - public List TransactionBlockTable { get; set; } = new(); - public Dictionary BlockTable { get; set; } = new(); - public List TransactionTable { get; set; } = new(); - public Dictionary OutputTable { get; set; } = new(); - public List InputTable { get; set; } = new(); + public abstract int GetBlockCount(); + public abstract int GetOutputCount(); + public abstract int GetInputCount(); + public abstract int GetTransactionCount(); + public abstract long GetBatchSize(); - public object ExtraData { get; set; } + public abstract IEnumerable GetBlockSizes(); + public abstract bool ValidateBatch(string prevBlockHash); } } diff --git a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs index 78ced171..d9337fc0 100644 --- a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs +++ b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs @@ -19,6 +19,7 @@ public static IServiceCollection AddMongoDatabase( this IServiceCollection servi services.AddSingleton(); services.AddTransient(); services.AddScoped(); + services.AddSingleton(); //TODO add this for address driven blockchains //services.AddScoped(); diff --git a/src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs b/src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs new file mode 100644 index 00000000..106aed35 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs @@ -0,0 +1,8 @@ +using Blockcore.Indexer.Core.Operations.Types; + +namespace Blockcore.Indexer.Core.Storage; + +public interface IStorageBatchFactory +{ + StorageBatch GetStorageBatch(); +} diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs new file mode 100644 index 00000000..27046920 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs @@ -0,0 +1,9 @@ +using Blockcore.Indexer.Core.Operations.Types; +using Blockcore.Indexer.Core.Storage.Mongo.Types; + +namespace Blockcore.Indexer.Core.Storage.Mongo; + +public class MongoStorageBatchFactory : IStorageBatchFactory +{ + public StorageBatch GetStorageBatch() => new MongoStorageBatch(); +} diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs index ffb4014d..bdbae284 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs @@ -48,15 +48,17 @@ public MongoStorageOperations( public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) { - storageBatch.TotalSize += item.BlockInfo.Size; - storageBatch.BlockTable.Add(item.BlockInfo.Height, mongoBlockToStorageBlock.Map(item.BlockInfo)); + var mongoStorageBatch = storageBatch as MongoStorageBatch; + + mongoStorageBatch.TotalSize += item.BlockInfo.Size; + mongoStorageBatch.BlockTable.Add(item.BlockInfo.Height, mongoBlockToStorageBlock.Map(item.BlockInfo)); int transactionIndex = 0; foreach (Transaction trx in item.Transactions) { string trxHash = trx.GetHash().ToString(); - storageBatch.TransactionBlockTable.Add( + mongoStorageBatch.TransactionBlockTable.Add( new TransactionBlockTable { BlockIndex = item.BlockInfo.HeightAsUint32, @@ -67,7 +69,7 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp if (configuration.StoreRawTransactions) { - storageBatch.TransactionTable.Add(new TransactionTable + mongoStorageBatch.TransactionTable.Add(new TransactionTable { TransactionId = trxHash, RawTransaction = trx.ToBytes(syncConnection.Network.Consensus.ConsensusFactory) @@ -84,7 +86,7 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp var outpoint = new Outpoint { TransactionId = trxHash, OutputIndex = outputIndex++ }; - storageBatch.OutputTable.Add(outpoint.ToString(), new OutputTable + mongoStorageBatch.OutputTable.Add(outpoint.ToString(), new OutputTable { Address = addr, Outpoint = outpoint, @@ -106,9 +108,9 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp TransactionId = input.PrevOut.Hash.ToString(), OutputIndex = (int)input.PrevOut.N }; - storageBatch.OutputTable.TryGetValue(outpoint.ToString(), out OutputTable output); + mongoStorageBatch.OutputTable.TryGetValue(outpoint.ToString(), out OutputTable output); - storageBatch.InputTable.Add(new InputTable() + mongoStorageBatch.InputTable.Add(new InputTable() { Outpoint = outpoint, TrxHash = trxHash, @@ -126,13 +128,15 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) { + var mongoStorageBatch = storageBatch as MongoStorageBatch; + if (globalState.IndexModeCompleted) { if (globalState.IbdMode() == false) { if (globalState.LocalMempoolView.Any()) { - var toRemoveFromMempool = storageBatch.TransactionBlockTable.Select(s => s.TransactionId).ToList(); + var toRemoveFromMempool = mongoStorageBatch.TransactionBlockTable.Select(s => s.TransactionId).ToList(); FilterDefinitionBuilder builder = Builders.Filter; FilterDefinition filter = builder.In(mempoolItem => mempoolItem.TransactionId, @@ -146,24 +150,24 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) } } - var blockTableTask = storageBatch.BlockTable.Values.Any() - ? db.BlockTable.InsertManyAsync(storageBatch.BlockTable.Values, new InsertManyOptions { IsOrdered = false }) + var blockTableTask = mongoStorageBatch.BlockTable.Values.Any() + ? db.BlockTable.InsertManyAsync(mongoStorageBatch.BlockTable.Values, new InsertManyOptions { IsOrdered = false }) : Task.CompletedTask; - var transactionBlockTableTask = storageBatch.TransactionBlockTable.Any() - ? db.TransactionBlockTable.InsertManyAsync(storageBatch.TransactionBlockTable, new InsertManyOptions { IsOrdered = false }) + var transactionBlockTableTask = mongoStorageBatch.TransactionBlockTable.Any() + ? db.TransactionBlockTable.InsertManyAsync(mongoStorageBatch.TransactionBlockTable, new InsertManyOptions { IsOrdered = false }) : Task.CompletedTask; - var outputTableTask = storageBatch.OutputTable.Any() - ? db.OutputTable.InsertManyAsync(storageBatch.OutputTable.Values, new InsertManyOptions { IsOrdered = false }) + var outputTableTask = mongoStorageBatch.OutputTable.Any() + ? db.OutputTable.InsertManyAsync(mongoStorageBatch.OutputTable.Values, new InsertManyOptions { IsOrdered = false }) : Task.CompletedTask; Task transactionTableTask = Task.Run(() => { try { - if (storageBatch.TransactionTable.Any()) - db.TransactionTable.InsertMany(storageBatch.TransactionTable, new InsertManyOptions {IsOrdered = false}); + if (mongoStorageBatch.TransactionTable.Any()) + db.TransactionTable.InsertMany(mongoStorageBatch.TransactionTable, new InsertManyOptions {IsOrdered = false}); } catch (MongoBulkWriteException mbwex) { @@ -176,9 +180,9 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) } }); - var utxos = new List(storageBatch.OutputTable.Values.Count); + var utxos = new List(mongoStorageBatch.OutputTable.Values.Count); - foreach (OutputTable outputTable in storageBatch.OutputTable.Values) + foreach (OutputTable outputTable in mongoStorageBatch.OutputTable.Values) { if (outputTable.Address.Equals(OpReturnAddress)) continue; @@ -201,14 +205,14 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) : Task.CompletedTask; var inputTableTask = Task.CompletedTask; - if (storageBatch.InputTable.Any()) + if (mongoStorageBatch.InputTable.Any()) { var utxosLookups = FetchUtxos( - storageBatch.InputTable + mongoStorageBatch.InputTable .Where(_ => _.Address == null) .Select(_ => _.Outpoint)); - foreach (InputTable input in storageBatch.InputTable) + foreach (InputTable input in mongoStorageBatch.InputTable) { if (input.Address != null) continue; @@ -217,17 +221,17 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) input.Value = utxosLookups[key].Value; } - inputTableTask = db.InputTable.InsertManyAsync(storageBatch.InputTable, new InsertManyOptions { IsOrdered = false }); + inputTableTask = db.InputTable.InsertManyAsync(mongoStorageBatch.InputTable, new InsertManyOptions { IsOrdered = false }); } Task.WaitAll(blockTableTask, transactionBlockTableTask, outputTableTask, inputTableTask, transactionTableTask, unspentOutputTableTask); - if (storageBatch.InputTable.Any()) + if (mongoStorageBatch.InputTable.Any()) { // TODO: if earlier we filtered out outputs that are already spent and not pushed to the utxo table // now we do not need to try and delete such outputs becuase they where never pushed to the store. - var outpointsFromNewInput = storageBatch.InputTable + var outpointsFromNewInput = mongoStorageBatch.InputTable .Select(_ => _.Outpoint) .ToList(); @@ -246,7 +250,7 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) string lastBlockHash = null; long blockIndex = 0; var markBlocksAsComplete = new List>(); - foreach (BlockTable mapBlock in storageBatch.BlockTable.Values.OrderBy(b => b.BlockIndex)) + foreach (BlockTable mapBlock in mongoStorageBatch.BlockTable.Values.OrderBy(b => b.BlockIndex)) { FilterDefinition filter = Builders.Filter.Eq(block => block.BlockIndex, mapBlock.BlockIndex); diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs new file mode 100644 index 00000000..255f8011 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; +using System.Linq; +using Blockcore.Indexer.Core.Operations.Types; + +namespace Blockcore.Indexer.Core.Storage.Mongo.Types; + +public class MongoStorageBatch : StorageBatch +{ + public long TotalSize { get; set; } + public List TransactionBlockTable { get; set; } = new(); + public Dictionary BlockTable { get; set; } = new(); + public List TransactionTable { get; set; } = new(); + public Dictionary OutputTable { get; set; } = new(); + public List InputTable { get; set; } = new(); + + public override int GetBlockCount() => BlockTable.Count; + + public override int GetOutputCount() => OutputTable.Count; + + public override int GetInputCount() => InputTable.Count; + + public override int GetTransactionCount() => TransactionBlockTable.Count; + + public override long GetBatchSize() => TotalSize; + + public override IEnumerable GetBlockSizes() => BlockTable.Values.Select(x => x.BlockSize).ToList(); + + public override bool ValidateBatch(string prevBlockHash) + { + string prevHash = prevBlockHash; + foreach (var mapBlock in BlockTable.Values.OrderBy(b => b.BlockIndex)) + { + if (mapBlock.PreviousBlockHash != prevHash) + { + return false; + } + + prevHash = mapBlock.BlockHash; + } + + return true; + } +} diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs index bdd38a5e..fe0b32e2 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs @@ -9,6 +9,7 @@ using Blockcore.Indexer.Core.Operations; using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; +using Blockcore.Indexer.Core.Storage; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -35,6 +36,8 @@ public class BlockPuller : TaskRunner readonly ICryptoClientFactory clientFactory; + readonly IStorageBatchFactory StorageBatchFactory; + private readonly IEnumerable bip30Blocks = new List {91842 , 91880 }; private readonly BlockingCollection pendingBlocksToAddToStorage = new(); @@ -52,12 +55,14 @@ public BlockPuller( SyncConnection syncConnection, ILogger logger, IStorageOperations storageOperations, - ICryptoClientFactory clientFactory) + ICryptoClientFactory clientFactory, + IStorageBatchFactory storageBatchFactory) : base(configuration, logger) { log = logger; this.storageOperations = storageOperations; this.clientFactory = clientFactory; + StorageBatchFactory = storageBatchFactory; this.syncConnection = syncConnection; this.syncOperations = syncOperations; config = configuration.Value; @@ -108,7 +113,7 @@ public override async Task OnExecute() { // start pulling blocks form this tip Runner.GlobalState.PullingTip = await clientFactory.Create(syncConnection).GetBlockAsync(Runner.GlobalState.StoreTip.BlockHash); - currentStorageBatch = new StorageBatch(); + currentStorageBatch = StorageBatchFactory.GetStorageBatch(); log.LogInformation($"Fetching block started at block {Runner.GlobalState.PullingTip.Height}({Runner.GlobalState.PullingTip.Hash})"); } @@ -211,21 +216,21 @@ private void ProcessFromBlockingCollection() bool bip30Issue = bip30Blocks.Contains(block.BlockInfo.Height + 1); if (ibd && !bip30Issue && - currentStorageBatch.BlockTable.Count < config.DbBatchCount && - currentStorageBatch.TotalSize <= config.DbBatchSize) + currentStorageBatch.GetBlockCount() < config.DbBatchCount && + currentStorageBatch.GetBatchSize() <= config.DbBatchSize) { continue; } - long totalBlocks = currentStorageBatch.BlockTable.Count; + long totalBlocks = currentStorageBatch.GetBlockCount(); double totalSeconds = watchBatch.Elapsed.TotalSeconds; double blocksPerSecond = totalBlocks / totalSeconds; double secondsPerBlock = totalSeconds / totalBlocks; - log.LogInformation($"Puller - blocks={currentStorageBatch.BlockTable.Count}, height = {block.BlockInfo.Height}, batch size = {((decimal)currentStorageBatch.TotalSize / 1000000):0.00}mb, Seconds = {watchBatch.Elapsed.TotalSeconds}, fetchs = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b). ({pendingBlocksToAddToStorage.Count})"); + log.LogInformation($"Puller - blocks={currentStorageBatch.GetBlockCount()}, height = {block.BlockInfo.Height}, batch size = {((decimal)currentStorageBatch.GetBatchSize() / 1000000):0.00}mb, Seconds = {watchBatch.Elapsed.TotalSeconds}, fetchs = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b). ({pendingBlocksToAddToStorage.Count})"); Runner.Get().Enqueue(currentStorageBatch); - currentStorageBatch = new StorageBatch(); + currentStorageBatch = StorageBatchFactory.GetStorageBatch(); watchBatch.Restart(); } diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs index 34df35c7..f2b818cc 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs @@ -26,6 +26,7 @@ public class BlockStartup : TaskStarter readonly ICryptoClientFactory clientFactory; private readonly IStorage storageData; readonly IOptions indexerSettings; + readonly IStorageBatchFactory StorageBatchFactory; /// /// Initializes a new instance of the class. @@ -37,13 +38,15 @@ public BlockStartup( IStorageOperations storageOperations, ICryptoClientFactory clientFactory, IStorage data, - IOptions indexerSettings) + IOptions indexerSettings, + IStorageBatchFactory storageBatchFactory) : base(logger) { connection = syncConnection; this.storageOperations = storageOperations; this.clientFactory = clientFactory; this.indexerSettings = indexerSettings; + StorageBatchFactory = storageBatchFactory; this.syncOperations = syncOperations; log = logger; @@ -91,7 +94,7 @@ public override async Task OnExecute() BlockInfo genesisBlock = await client.GetBlockAsync(genesisHash); SyncBlockTransactionsOperation block = syncOperations.FetchFullBlock(connection, genesisBlock); - StorageBatch genesisBatch = new StorageBatch(); + StorageBatch genesisBatch = StorageBatchFactory.GetStorageBatch(); storageOperations.AddToStorageBatch(genesisBatch, block); Runner.GlobalState.StoreTip = storageOperations.PushStorageBatch(genesisBatch); } diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs index df61b976..36f38d02 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs @@ -67,7 +67,8 @@ public override async Task OnExecute() if (!TryDequeue(out StorageBatch batch)) return await Task.FromResult(false); - ValidateBatch(batch); + if (batch.ValidateBatch(Runner.GlobalState.StoreTip.BlockHash) == false) + throw new ApplicationException("None consecutive block received"); watch.Restart(); @@ -78,15 +79,15 @@ public override async Task OnExecute() if (Runner.GlobalState.StoreTip == null) throw new ApplicationException("Store tip was not persisted"); - long totalBlocks = batch.BlockTable.Count;// insertStats.Sum((tuple => tuple.count)); + long totalBlocks = batch.GetBlockCount();// insertStats.Sum((tuple => tuple.count)); double totalSeconds = watch.Elapsed.TotalSeconds;// insertStats.Sum((tuple => tuple.seconds)); double blocksPerSecond = totalBlocks / totalSeconds; double secondsPerBlock = totalSeconds / totalBlocks; - log.LogInformation($"Store - blocks={batch.BlockTable.Count}, outputs={batch.OutputTable.Count}, inputs={batch.InputTable.Count}, trx={batch.TransactionBlockTable.Count}, total Size = {((decimal)batch.TotalSize / 1000000):0.00}mb, tip={Runner.GlobalState.StoreTip.BlockIndex}, Seconds = {watch.Elapsed.TotalSeconds}, inserts = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b)"); + log.LogInformation($"Store - blocks={batch.GetBlockCount()}, outputs={batch.GetOutputCount()}, inputs={batch.GetInputCount()}, trx={batch.GetTransactionCount()}, total Size = {((decimal)batch.GetBatchSize() / 1000000):0.00}mb, tip={Runner.GlobalState.StoreTip.BlockIndex}, Seconds = {watch.Elapsed.TotalSeconds}, inserts = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b)"); - foreach (var mapBlocksValue in batch.BlockTable.Values) - syncConnection.RecentItems.Add((DateTime.UtcNow, TimeSpan.FromSeconds(blocksPerSecond), mapBlocksValue.BlockSize)); + foreach (var blockSiz in batch.GetBlockSizes()) + syncConnection.RecentItems.Add((DateTime.UtcNow, TimeSpan.FromSeconds(blocksPerSecond), blockSiz)); var notifications = new AddressNotifications { Addresses = new List() };// count.Items.Where(ad => ad.Addresses != null).SelectMany(s => s.Addresses).Distinct().ToList() }; Runner.Get().Enqueue(notifications); @@ -94,20 +95,5 @@ public override async Task OnExecute() return await Task.FromResult(true); } - - void ValidateBatch(StorageBatch item) - { - // check all blocks are consecutive and start from the last block in store. - string prevHash = Runner.GlobalState.StoreTip.BlockHash; - foreach (var mapBlock in item.BlockTable.Values.OrderBy(b => b.BlockIndex)) - { - if (mapBlock.PreviousBlockHash != prevHash) - { - throw new ApplicationException("None consecutive block received"); - } - - prevHash = mapBlock.BlockHash; - } - } } } diff --git a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs index 89209d26..4584b19a 100644 --- a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs +++ b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs @@ -155,9 +155,9 @@ static SyncBlockTransactionsOperation WithRandomSyncBlockTransactionsOperation() return item; } - StorageBatch WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully() + MongoStorageBatch WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var block = NewRandomBlockTable; batch.BlockTable.Add(block.BlockIndex, block); @@ -168,7 +168,7 @@ StorageBatch WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully() return batch; } - void WithASuccessfulDeleteManyOnUnspentOutputTable(StorageBatch batch) + void WithASuccessfulDeleteManyOnUnspentOutputTable(MongoStorageBatch batch) { var utxo = batch.InputTable.Select(_ => _.Outpoint).ToList(); @@ -182,7 +182,7 @@ void WithASuccessfulDeleteManyOnUnspentOutputTable(StorageBatch batch) [Fact] public void AddToStorageBatchSetsTheTotalSizeFromBlockInfo() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -194,7 +194,7 @@ public void AddToStorageBatchSetsTheTotalSizeFromBlockInfo() [Fact] public void AddToStorageBatchSetsTheBlockTableFromBlockInfo() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -216,7 +216,7 @@ public void AddToStorageBatchSetsTheBlockTableFromBlockInfo() [Fact] public void AddToStorageBatchSetsTheTransactionBlockTableFromTransactions() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -233,7 +233,7 @@ public void AddToStorageBatchSetsTheTransactionBlockTableFromTransactions() [Fact] public void AddToStorageBatchWithStoreRawTransactionsTrueSetsTransactionTable() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -254,7 +254,7 @@ public void AddToStorageBatchWithStoreRawTransactionsTrueSetsTransactionTable() [Fact] public void AddToStorageBatchSetsTheOutputsInTheTransactionToTheOutputTable() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var valueMoney = new Money(NewRandomInt32); var script = new Script(NewRandomString.Replace('-','1')); var item = WithRandomSyncBlockTransactionsOperation(); @@ -300,7 +300,7 @@ public void AddToStorageBatchSetsTheOutputsInTheTransactionToTheOutputTable() [Fact] public void AddToStorageBatchSetsTheInputsInInputTableWithoutAddress() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var hash = new uint256($"{NewRandomInt64}{NewRandomInt64}{NewRandomInt64}{NewRandomInt64}".Substring(0,64)); int n = NewRandomInt32; var expectedOutpoint = new OutPoint @@ -340,7 +340,7 @@ public void AddToStorageBatchSetsTheInputsInInputTableWithoutAddress() [Fact] public void AddToStorageBatchSetsTheInputsInInputTableWithAddressAndValue() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var valueMoney = new Money(NewRandomInt32); var script = new Script(NewRandomString.Replace('-','1')); var transaction = new Transaction { Outputs = { { new TxOut { Value = valueMoney, ScriptPubKey = script } } } }; @@ -371,7 +371,7 @@ public void AddToStorageBatchSetsTheInputsInInputTableWithAddressAndValue() [Fact] public void PushStorageBatchAddsBlockTableFromBatchToMongoDb() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var block = NewRandomBlockTable; batch.BlockTable.Add(block.BlockIndex, block); @@ -388,7 +388,7 @@ public void PushStorageBatchAddsBlockTableFromBatchToMongoDb() [Fact] public void PushStorageBatchAddsTransactionBlockTableFromBatchToMongoDb() { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var blockTable = new TransactionBlockTable { @@ -407,7 +407,7 @@ public void PushStorageBatchAddsTransactionBlockTableFromBatchToMongoDb() [Fact] public void PushStorageBatchAddsOutputTableFromBatchToMongoDb() { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var outputTable = new OutputTable() { @@ -431,7 +431,7 @@ public void PushStorageBatchAddsOutputTableFromBatchToMongoDb() [Fact] public void PushStorageBatchAddsInputTableAddsToMongodb() { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); //We must have an unspent output for an input that is being processed var outpoint = new Outpoint { OutputIndex = NewRandomInt32, TransactionId = NewRandomString }; @@ -467,7 +467,7 @@ public void PushStorageBatchAddsInputTableAddsToMongodb() [Fact] public void PushStorageBatchAddsTransactionTableDataToMongodb() { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var transactionTable = new TransactionTable() { @@ -487,7 +487,7 @@ public void PushStorageBatchAddsTransactionTableDataToMongodb() public void PushStorageBatchIgnoresDuplicatsOnInputTableForDuplicateKeyException() #pragma warning restore xUnit1013 // Public method should be marked as test { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var transactionTable = new TransactionTable() { @@ -514,7 +514,7 @@ public void PushStorageBatchIgnoresDuplicatsOnInputTableForDuplicateKeyException [Fact] public void PushStorageBatchAddsUnspendOutputToMongodbForEachOutputTableItem() { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var outputTable = new OutputTable { @@ -542,7 +542,7 @@ public void PushStorageBatchAddsUnspendOutputToMongodbForEachOutputTableItem() [Fact] public void PushStorageBatchAddsDeletsFromUnspentOutputTableOnMongoDbForAllInputTableItems() { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); //We must have an unspent output for an input that is being processed var outpoint = new Outpoint { OutputIndex = NewRandomInt32, TransactionId = NewRandomString }; @@ -579,7 +579,7 @@ public void PushStorageBatchAddsDeletsFromUnspentOutputTableOnMongoDbForAllInput [Fact] public void PushStorageBatchUpdatesBlockTableItemsWithSyncCompleteTrueInMongodb() { - StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); sut.PushStorageBatch(batch); @@ -603,7 +603,7 @@ public void PushStorageBatchUpdatesBlockTableItemsWithSyncCompleteTrueInMongodb( [Fact] public void PushStorageBatchThrowsWhenTheBlockInertedIsNotTheSameHashAsTopBlockInMongodb() { - var batch = new StorageBatch(); + var batch = new MongoStorageBatch(); var dbBlock = NewRandomBlockTable; var storageBlock = NewRandomBlockTable; From fe8932eefe34513c76e1ff1eba26e50836e3c562 Mon Sep 17 00:00:00 2001 From: TheDude Date: Fri, 21 Jun 2024 17:13:07 +0100 Subject: [PATCH 16/16] Deleted the utxo cache service that was not in use --- .../Mongo/CirrusMongoStorageOperations.cs | 2 - .../Operations/IUtxoCache.cs | 19 ------- .../Operations/Types/UtxoCache.cs | 57 ------------------- src/Blockcore.Indexer.Core/Startup.cs | 2 - .../Storage/Mongo/MongoStorageOperations.cs | 1 - .../Mongo/MongoStorageOperationsTests.cs | 1 - 6 files changed, 82 deletions(-) delete mode 100644 src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs delete mode 100644 src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs index 590bf16d..1ad3faf7 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs @@ -28,7 +28,6 @@ public class CirrusMongoStorageOperations : MongoStorageOperations public CirrusMongoStorageOperations( SyncConnection syncConnection, IStorage storage, - IUtxoCache utxoCache, IOptions configuration, GlobalState globalState, IMapMongoBlockToStorageBlock mongoBlockToStorageBlock, @@ -38,7 +37,6 @@ public CirrusMongoStorageOperations( base( syncConnection, db, - utxoCache, configuration, globalState, mongoBlockToStorageBlock, diff --git a/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs b/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs deleted file mode 100644 index 5e872a51..00000000 --- a/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs +++ /dev/null @@ -1,19 +0,0 @@ -// using System.Collections.Generic; -// using Blockcore.Indexer.Core.Operations.Types; -// using Blockcore.Indexer.Core.Storage.Mongo.Types; - -namespace Blockcore.Indexer.Core.Operations -{ - /// - /// Maintain a cache of unspent outputs - /// - public interface IUtxoCache - { - // int CacheSize { get; } - // - // UtxoCacheItem GetOne(string outpoint); - // void AddToCache(IEnumerable outputs); - // - // void RemoveFromCache(IEnumerable inputs); - } -} diff --git a/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs b/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs deleted file mode 100644 index 4740f508..00000000 --- a/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs +++ /dev/null @@ -1,57 +0,0 @@ -// using System.Collections.Concurrent; -// using System.Collections.Generic; -// using System.Linq; -// using Blockcore.Indexer.Core.Storage.Mongo.Types; -// using Microsoft.Extensions.Logging; - -namespace Blockcore.Indexer.Core.Operations.Types -{ - public class UtxoCacheItem - { - public string Address { get; set; } - public long Value { get; set; } - } - - public class UtxoCache : IUtxoCache - { - // private readonly ILogger logger; - // private readonly ConcurrentDictionary cache; - // - // private readonly int maxItemInCache = 0; - // - // public UtxoCache(ILogger logger) - // { - // this.logger = logger; - // cache = new ConcurrentDictionary(); - // } - // - // public int CacheSize { get { return cache.Count; } } - // - // public UtxoCacheItem GetOne(string outpoint) - // { - // if (cache.TryGetValue(outpoint, out UtxoCacheItem utxo)) - // { - // return new UtxoCacheItem { Value = utxo.Value, Address = utxo.Address }; - // } - // - // return null; - // } - // - // public void AddToCache(IEnumerable outputs) - // { - // int maxToAdd = maxItemInCache - cache.Count; - // foreach (OutputTable output in outputs.Take(maxToAdd)) - // { - // cache.TryAdd($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", new UtxoCacheItem { Value = output.Value, Address = output.Address }); - // } - // } - // - // public void RemoveFromCache(IEnumerable inputs) - // { - // foreach (InputTable output in inputs) - // { - // cache.TryRemove($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", out _); - // } - // } - } -} diff --git a/src/Blockcore.Indexer.Core/Startup.cs b/src/Blockcore.Indexer.Core/Startup.cs index 151d985a..3fcc0a8c 100644 --- a/src/Blockcore.Indexer.Core/Startup.cs +++ b/src/Blockcore.Indexer.Core/Startup.cs @@ -51,8 +51,6 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddTransient(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs index bdbae284..2c758c9a 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs @@ -30,7 +30,6 @@ public class MongoStorageOperations : IStorageOperations public MongoStorageOperations( SyncConnection syncConnection, IMongoDb storage, - IUtxoCache utxoCache, IOptions configuration, GlobalState globalState, IMapMongoBlockToStorageBlock mongoBlockToStorageBlock, diff --git a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs index 4584b19a..70522640 100644 --- a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs +++ b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs @@ -87,7 +87,6 @@ public MongoStorageOperationsTests() sut = new MongoStorageOperations(syncConnection, mongodbMock.MongoDbObject, - new UtxoCache(), indexSettingsMock.Object, globalState, new MapMongoBlockToStorageBlock(),