From 1bd6ea174e9bbf07c98761e45663fce7bb7c0d98 Mon Sep 17 00:00:00 2001 From: TheDude Date: Wed, 24 Apr 2024 12:02:26 +0100 Subject: [PATCH] Fixed performance issue with investmnet scanning (#208) * Changed the lookup to avoid index scan * fixing slow queries from mongo db * Fixed bug with wrong outpoint output index and added block index on project * Fixed stage lookup filter --- .../Storage/Mongo/AngorMongoBuilder.cs | 6 ++- .../Storage/Mongo/AngorMongoData.cs | 4 +- .../ProjectTransactionsSyncRunner.cs | 53 ++++++++++++------- .../Sync/SyncTasks/ProjectsSyncRunner.cs | 9 ++-- src/Blockcore.Indexer.Angor/appsettings.json | 2 +- .../Storage/Mongo/BlockRewindOperation.cs | 4 +- .../Storage/Mongo/MongoStorageOperations.cs | 1 + .../Mongo/Types/TransactionBlockTable.cs | 4 ++ 8 files changed, 53 insertions(+), 30 deletions(-) diff --git a/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoBuilder.cs b/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoBuilder.cs index c5f3674e..a4c03a91 100644 --- a/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoBuilder.cs +++ b/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoBuilder.cs @@ -44,10 +44,14 @@ public override Task OnExecute() .CreateOne(new CreateIndexModel(Builders .IndexKeys.Hashed(_ => _.FounderKey))); + AngorMongoDb.ProjectTable.Indexes + .CreateOne(new CreateIndexModel(Builders + .IndexKeys.Descending(x => x.BlockIndex))); + //TODO move this to the block indexer task runner, but we'll need to move the indexes in there to a different class for each project/blockchain AngorMongoDb.InvestmentTable.Indexes .CreateOne(new CreateIndexModel(Builders - .IndexKeys.Hashed(_ => _.AngorKey))); + .IndexKeys.Ascending(x => x.AngorKey))); //TODO move this to the block indexer task runner, but we'll need to move the indexes in there to a different class for each project/blockchain AngorMongoDb.InvestmentTable.Indexes diff --git a/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoData.cs b/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoData.cs index 01a67645..6d0689f9 100644 --- a/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoData.cs +++ b/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoData.cs @@ -62,13 +62,13 @@ public AngorMongoData(ILogger dbLogger, SyncConnection connection, if (projectExists) { - var total = await mongoDb.InvestmentTable.CountDocumentsAsync(Builders.Filter.Eq(_ => _.AngorKey, projectId)); + var total = await mongoDb.InvestmentTable.CountDocumentsAsync(Builders.Filter.Eq(_ => _.AngorKey, projectId)).ConfigureAwait(false); var sum = mongoDb.InvestmentTable.AsQueryable() .Where(_ => _.AngorKey == projectId) .Sum(s => s.AmountSats); - var spendingSummery = await GetSpentProjectFundsSplitToFounderAndPenalty(projectId); + var spendingSummery = await GetSpentProjectFundsSplitToFounderAndPenalty(projectId).ConfigureAwait(false); return new ProjectStats { diff --git a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs index 72ddccf0..2252d730 100644 --- a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs +++ b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs @@ -2,12 +2,14 @@ using Blockcore.Indexer.Angor.Storage.Mongo; using Blockcore.Indexer.Angor.Storage.Mongo.Types; using Blockcore.Indexer.Core.Settings; +using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; using Blockcore.NBitcoin.DataEncoders; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.Linq; +using static System.String; namespace Blockcore.Indexer.Angor.Sync.SyncTasks; @@ -50,34 +52,29 @@ public override async Task OnExecute() await Task.WhenAll(investmentTasks); var investments = investmentTasks.AsEnumerable() + .Where(x => x is { Result: not null, IsCompletedSuccessfully: true }) .Select(x => x.Result) - .Where(x => x != null) .OrderBy(x => x!.BlockIndex) - .Distinct() .ToList(); - if (!investments.Any()) + if (investments.Count == 0) return false; - await angorMongoDb.InvestmentTable.InsertManyAsync(investments); + await angorMongoDb.InvestmentTable.InsertManyAsync(investments, new InsertManyOptions { IsOrdered = true }) + .ConfigureAwait(false); return true; } + const string ErrorMssage = "Multiple transactions with the same investor public key {0} for the same project {1}"; async Task ValidateAndCreateInvestmentAsync(BsonDocument investmentOutput) { - var allOutputsOnInvestmentTransaction = await angorMongoDb.OutputTable.AsQueryable() - .Where(output => output.BlockIndex == investmentOutput["OutputBlockIndex"] && - output.Outpoint.TransactionId == investmentOutput["OutputTransactionId"]) - .ToListAsync(); - - if (allOutputsOnInvestmentTransaction.All(x => - x.Address != "none")) //TODO replace with a better indicator of stage investments - return null; + var feeOutput = await angorMongoDb.OutputTable.AsQueryable().SingleAsync(x => + x.Outpoint == new Outpoint{ TransactionId = investmentOutput["OutputTransactionId"].ToString(), OutputIndex = 0}); - var feeOutput = allOutputsOnInvestmentTransaction.Single(x => x.Outpoint.OutputIndex == 0); - var projectDataOutput = allOutputsOnInvestmentTransaction.Single(x => x.Outpoint.OutputIndex == 1); + var projectDataOutput = await angorMongoDb.OutputTable.AsQueryable().SingleAsync(x => + x.Outpoint == new Outpoint{ TransactionId = investmentOutput["OutputTransactionId"].ToString(), OutputIndex = 1}); var projectInfoScript = Script.FromHex(projectDataOutput.ScriptHex); @@ -86,8 +83,7 @@ public override async Task OnExecute() if (angorMongoDb.InvestmentTable.AsQueryable() .Any(_ => _.InvestorPubKey == investorPubKey)) //Investor key is the _id of that document { - logger.LogInformation( - $"Multiple transactions with the same investor public key {investorPubKey} for the same project {feeOutput.ScriptHex}"); + logger.LogDebug(ErrorMssage,investorPubKey,feeOutput.ScriptHex); return null; } @@ -101,9 +97,28 @@ public override async Task OnExecute() var hashOfSecret = projectInfoScript.ToOps().Count == 3 ? Encoders.Hex.EncodeData(projectInfoScript.ToOps()[2].PushData) - : string.Empty; + : Empty; + + int outpointIndex = 2; + OutputTable? stage; + List stages = new(); + do + { + stage = await angorMongoDb.OutputTable.AsQueryable() + .Where(output => output.Outpoint == new Outpoint{ + TransactionId = investmentOutput["OutputTransactionId"].ToString(), OutputIndex = outpointIndex} && + output.Address == "none") + .SingleOrDefaultAsync(); - var stages = allOutputsOnInvestmentTransaction.Where(_ => _.Address == "none"); + outpointIndex += 1; + + if (stage != null) + stages.Add(stage); + + } while (stage != null); + + if (stages.Count == 0) + return null; return new Investment { @@ -156,7 +171,7 @@ private BsonDocument[] MongoDbLookupForInvestments() { "TransactionId", 1 }, { "projectMaxBlockScanned", - new BsonDocument("$ifNull", new BsonArray { "$joinedData.projectMaxBlockScanned", 0 }) + new BsonDocument("$ifNull", new BsonArray { "$joinedData.projectMaxBlockScanned", $"$BlockIndex" }) } }), //Inner join with output on the indexed address and greater than block index and filter by trensaction id diff --git a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs index d46e1629..ac6f5137 100644 --- a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs +++ b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs @@ -57,7 +57,7 @@ public override async Task OnExecute() if (!CanRunProjectSync()) return false; - var blockIndexed = await AngorMongoDb.ProjectTable.EstimatedDocumentCountAsync() > 0 + var blockIndexed = await AngorMongoDb.ProjectTable.AsQueryable().AnyAsync() ? AngorMongoDb.ProjectTable.AsQueryable().Max(p => p.BlockIndex) : 0; @@ -123,10 +123,9 @@ public override async Task OnExecute() .AsQueryable() .Where(_ => //Outpoint with both parameters is the id of the table - _.Outpoint.TransactionId == output.Outpoint.TransactionId && - _.Outpoint.OutputIndex == 0 && - //direct lookup for the exiting key - _.ScriptHex == angorKey.WitHash.ScriptPubKey.ToHex()) + _.Outpoint == new Outpoint{TransactionId = output.Outpoint.TransactionId , OutputIndex = 0} && + //direct lookup for the exiting key + _.ScriptHex == angorKey.WitHash.ScriptPubKey.ToHex()) .FirstOrDefaultAsync(); if (angorFeeOutput == null) return null; diff --git a/src/Blockcore.Indexer.Angor/appsettings.json b/src/Blockcore.Indexer.Angor/appsettings.json index 3b62c542..41b5bdd5 100644 --- a/src/Blockcore.Indexer.Angor/appsettings.json +++ b/src/Blockcore.Indexer.Angor/appsettings.json @@ -46,6 +46,6 @@ "StoreRawTransactions": true, "NumberOfPullerTasksForIBD" : 5, "MaxItemsInBlockingCollection" : 1000, - "IndexCountForBlockIndexProperty": 6 + "IndexCountForBlockIndexProperty": 7 } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs index d9766062..a3636188 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs @@ -149,8 +149,8 @@ private static async Task MergeRewindInputsToUnspentTransactionsAsync(IMongoDb s var filteredUnspentOutputs = duplicates.Any() ? unspentOutputs.Where(_ => !duplicates - .Exists(d => d.Outpoint.TransactionId == _.Outpoint.TransactionId && - d.Outpoint.OutputIndex == _.Outpoint.OutputIndex)) + .Exists(d => d.Outpoint == new Outpoint + {TransactionId = _.Outpoint.TransactionId, OutputIndex = _.Outpoint.OutputIndex})) .ToList() : unspentOutputs; diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs index 5c044519..0891f3da 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs @@ -62,6 +62,7 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp BlockIndex = item.BlockInfo.HeightAsUint32, TransactionId = trxHash, TransactionIndex = transactionIndex++, + NumberOfOutputs = (short) trx.Outputs.Count }); if (configuration.StoreRawTransactions) diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/TransactionBlockTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/TransactionBlockTable.cs index 1a60c14f..d97ebd48 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/TransactionBlockTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/TransactionBlockTable.cs @@ -1,3 +1,5 @@ +using System; + namespace Blockcore.Indexer.Core.Storage.Mongo.Types { public class TransactionBlockTable @@ -7,5 +9,7 @@ public class TransactionBlockTable public string TransactionId { get; set; } public int TransactionIndex { get; set; } + + public short NumberOfOutputs { get; set; } } }