Fixed performance issue with investmnet scanning (#208)

* Changed the lookup to avoid index scan

* fixing slow queries from mongo db

* Fixed bug with wrong outpoint output index and added block index on project

* Fixed stage lookup filter

src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoBuilder.cs

@@ -44,10 +44,14 @@ public override Task OnExecute()
          .CreateOne(new CreateIndexModel<Project>(Builders<Project>
             .IndexKeys.Hashed(_ => _.FounderKey)));
 
+      AngorMongoDb.ProjectTable.Indexes
+         .CreateOne(new CreateIndexModel<Project>(Builders<Project>
+            .IndexKeys.Descending(x => x.BlockIndex)));
+
       //TODO move this to the block indexer task runner, but we'll need to move the indexes in there to a different class for each project/blockchain
       AngorMongoDb.InvestmentTable.Indexes
          .CreateOne(new CreateIndexModel<Investment>(Builders<Investment>
-            .IndexKeys.Hashed(_ => _.AngorKey)));
+            .IndexKeys.Ascending(x => x.AngorKey)));
 
       //TODO move this to the block indexer task runner, but we'll need to move the indexes in there to a different class for each project/blockchain
       AngorMongoDb.InvestmentTable.Indexes

src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoData.cs

@@ -62,13 +62,13 @@ public AngorMongoData(ILogger<AngorMongoDb> dbLogger, SyncConnection connection,
 
       if (projectExists)
       {
-         var total = await mongoDb.InvestmentTable.CountDocumentsAsync(Builders<Investment>.Filter.Eq(_ => _.AngorKey, projectId));
+         var total = await mongoDb.InvestmentTable.CountDocumentsAsync(Builders<Investment>.Filter.Eq(_ => _.AngorKey, projectId)).ConfigureAwait(false);
 
          var sum = mongoDb.InvestmentTable.AsQueryable()
             .Where(_ => _.AngorKey == projectId)
             .Sum(s => s.AmountSats);
 
-         var spendingSummery = await GetSpentProjectFundsSplitToFounderAndPenalty(projectId);
+         var spendingSummery = await GetSpentProjectFundsSplitToFounderAndPenalty(projectId).ConfigureAwait(false);
 
          return new ProjectStats
          {

src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs

@@ -2,12 +2,14 @@
 using Blockcore.Indexer.Angor.Storage.Mongo;
 using Blockcore.Indexer.Angor.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Settings;
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Sync.SyncTasks;
 using Blockcore.NBitcoin.DataEncoders;
 using Microsoft.Extensions.Options;
 using MongoDB.Bson;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
+using static System.String;
 
 namespace Blockcore.Indexer.Angor.Sync.SyncTasks;
 
@@ -50,34 +52,29 @@ public override async Task<bool> OnExecute()
       await Task.WhenAll(investmentTasks);
 
       var investments = investmentTasks.AsEnumerable()
+         .Where(x => x is { Result: not null, IsCompletedSuccessfully: true })
          .Select(x => x.Result)
-         .Where(x => x != null)
          .OrderBy(x => x!.BlockIndex)
-         .Distinct()
          .ToList();
 
-      if (!investments.Any())
+      if (investments.Count == 0)
          return false;
 
-      await angorMongoDb.InvestmentTable.InsertManyAsync(investments);
+      await angorMongoDb.InvestmentTable.InsertManyAsync(investments, new InsertManyOptions { IsOrdered = true })
+         .ConfigureAwait(false);
 
       return true;
 
    }
 
+   const string ErrorMssage = "Multiple transactions with the same investor public key {0} for the same project {1}";
    async Task<Investment?> ValidateAndCreateInvestmentAsync(BsonDocument investmentOutput)
    {
-      var allOutputsOnInvestmentTransaction = await angorMongoDb.OutputTable.AsQueryable()
-         .Where(output => output.BlockIndex == investmentOutput["OutputBlockIndex"] &&
-                          output.Outpoint.TransactionId == investmentOutput["OutputTransactionId"])
-         .ToListAsync();
-
-      if (allOutputsOnInvestmentTransaction.All(x =>
-             x.Address != "none")) //TODO replace with a better indicator of stage investments
-         return null;
+      var feeOutput = await angorMongoDb.OutputTable.AsQueryable().SingleAsync(x =>
+         x.Outpoint == new Outpoint{ TransactionId = investmentOutput["OutputTransactionId"].ToString(), OutputIndex = 0});
 
-      var feeOutput = allOutputsOnInvestmentTransaction.Single(x => x.Outpoint.OutputIndex == 0);
-      var projectDataOutput = allOutputsOnInvestmentTransaction.Single(x => x.Outpoint.OutputIndex == 1);
+      var projectDataOutput = await angorMongoDb.OutputTable.AsQueryable().SingleAsync(x =>
+         x.Outpoint == new Outpoint{ TransactionId = investmentOutput["OutputTransactionId"].ToString(), OutputIndex = 1});
 
       var projectInfoScript = Script.FromHex(projectDataOutput.ScriptHex);
 
@@ -86,8 +83,7 @@ public override async Task<bool> OnExecute()
       if (angorMongoDb.InvestmentTable.AsQueryable()
              .Any(_ => _.InvestorPubKey == investorPubKey)) //Investor key is the _id of that document
       {
-         logger.LogInformation(
-            $"Multiple transactions with the same investor public key {investorPubKey} for the same project {feeOutput.ScriptHex}");
+         logger.LogDebug(ErrorMssage,investorPubKey,feeOutput.ScriptHex);
          return null;
       }
 
@@ -101,9 +97,28 @@ public override async Task<bool> OnExecute()
 
       var hashOfSecret = projectInfoScript.ToOps().Count == 3
          ? Encoders.Hex.EncodeData(projectInfoScript.ToOps()[2].PushData)
-         : string.Empty;
+         : Empty;
+
+      int outpointIndex = 2;
+      OutputTable? stage;
+      List<OutputTable> stages = new();
+      do
+      {
+         stage = await angorMongoDb.OutputTable.AsQueryable()
+            .Where(output => output.Outpoint == new Outpoint{
+               TransactionId = investmentOutput["OutputTransactionId"].ToString(), OutputIndex = outpointIndex} &&
+                             output.Address == "none")
+            .SingleOrDefaultAsync();
 
-      var stages = allOutputsOnInvestmentTransaction.Where(_ => _.Address == "none");
+         outpointIndex += 1;
+
+         if (stage != null)
+            stages.Add(stage);
+
+      } while (stage != null);
+
+      if (stages.Count == 0)
+         return null;
 
       return new Investment
       {
@@ -156,7 +171,7 @@ private BsonDocument[] MongoDbLookupForInvestments()
                { "TransactionId", 1 },
                {
                   "projectMaxBlockScanned",
-                  new BsonDocument("$ifNull", new BsonArray { "$joinedData.projectMaxBlockScanned", 0 })
+                  new BsonDocument("$ifNull", new BsonArray { "$joinedData.projectMaxBlockScanned", $"$BlockIndex" })
                }
             }),
          //Inner join with output on the indexed address and greater than block index and filter by trensaction id

src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs

@@ -57,7 +57,7 @@ public override async Task<bool> OnExecute()
       if (!CanRunProjectSync())
          return false;
 
-      var blockIndexed = await AngorMongoDb.ProjectTable.EstimatedDocumentCountAsync() > 0
+      var blockIndexed = await AngorMongoDb.ProjectTable.AsQueryable().AnyAsync()
          ? AngorMongoDb.ProjectTable.AsQueryable().Max(p => p.BlockIndex)
          : 0;
 
@@ -123,10 +123,9 @@ public override async Task<bool> OnExecute()
          .AsQueryable()
          .Where(_ =>
             //Outpoint with both parameters is the id of the table
-            _.Outpoint.TransactionId == output.Outpoint.TransactionId &&
-            _.Outpoint.OutputIndex == 0 &&
-            //direct lookup for the exiting key
-            _.ScriptHex == angorKey.WitHash.ScriptPubKey.ToHex())
+            _.Outpoint == new Outpoint{TransactionId = output.Outpoint.TransactionId , OutputIndex = 0} &&
+                                         //direct lookup for the exiting key
+                                         _.ScriptHex == angorKey.WitHash.ScriptPubKey.ToHex())
          .FirstOrDefaultAsync();
 
       if (angorFeeOutput == null) return null;

src/Blockcore.Indexer.Angor/appsettings.json

@@ -46,6 +46,6 @@
       "StoreRawTransactions": true,
       "NumberOfPullerTasksForIBD" : 5,
       "MaxItemsInBlockingCollection" : 1000,
-      "IndexCountForBlockIndexProperty": 6
+      "IndexCountForBlockIndexProperty": 7
    }
 }

src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs

@@ -149,8 +149,8 @@ private static async Task MergeRewindInputsToUnspentTransactionsAsync(IMongoDb s
 
          var filteredUnspentOutputs = duplicates.Any()
             ? unspentOutputs.Where(_ => !duplicates
-                  .Exists(d => d.Outpoint.TransactionId == _.Outpoint.TransactionId &&
-                               d.Outpoint.OutputIndex == _.Outpoint.OutputIndex))
+                  .Exists(d => d.Outpoint == new Outpoint
+                  {TransactionId = _.Outpoint.TransactionId, OutputIndex = _.Outpoint.OutputIndex}))
                .ToList()
             : unspentOutputs;
 

src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs

@@ -62,6 +62,7 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp
                   BlockIndex = item.BlockInfo.HeightAsUint32,
                   TransactionId = trxHash,
                   TransactionIndex = transactionIndex++,
+                  NumberOfOutputs = (short) trx.Outputs.Count
                });
 
             if (configuration.StoreRawTransactions)

src/Blockcore.Indexer.Core/Storage/Mongo/Types/TransactionBlockTable.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace Blockcore.Indexer.Core.Storage.Mongo.Types
 {
    public class TransactionBlockTable
@@ -7,5 +9,7 @@ public class TransactionBlockTable
       public string TransactionId { get; set; }
 
       public int TransactionIndex { get; set; }
+
+      public short NumberOfOutputs { get; set; }
    }
 }