From bbe77b09ee52a3d36afb07d743c17d177dc13d8d Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali <119250923+theDRB123@users.noreply.github.com> Date: Mon, 17 Jun 2024 00:30:14 +0530 Subject: [PATCH 1/2] Added Database context and Postgres Types --- CHANGELOG.md | 15 ++- package.json | 2 +- .../Blockcore.Indexer.Core.csproj | 3 + src/Blockcore.Indexer.Core/Startup.cs | 10 +- .../Storage/Postgres/PostgresDBContext.cs | 99 +++++++++++++++++++ .../Postgres/Types/AddressComputedTable.cs | 34 +++++++ .../Types/AddressHistoryComputedTable.cs | 23 +++++ .../Types/AddressUtxoComputedTable.cs | 17 ++++ .../Storage/Postgres/Types/Block.cs | 52 ++++++++++ .../Storage/Postgres/Types/Input.cs | 16 +++ .../Postgres/Types/MempoolTransaction.cs | 32 ++++++ .../Storage/Postgres/Types/Outpoint.cs | 14 +++ .../Storage/Postgres/Types/Output.cs | 16 +++ .../Storage/Postgres/Types/ReorgBlockTable.cs | 13 +++ .../Storage/Postgres/Types/RichlistTable.cs | 11 +++ .../Storage/Postgres/Types/Transaction.cs | 20 ++++ .../Storage/Postgres/Types/UnspentOutput.cs | 12 +++ src/Directory.Build.props | 2 +- 18 files changed, 387 insertions(+), 4 deletions(-) create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/PostgresDBContext.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressComputedTable.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressHistoryComputedTable.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressUtxoComputedTable.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/Block.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/Input.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/MempoolTransaction.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/Outpoint.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/Output.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/ReorgBlockTable.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/RichlistTable.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/Transaction.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/Types/UnspentOutput.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index dc02524..7646863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,23 @@ +## 0.2.66 (2024-05-24) + +* Add mainnet key ([1ed7da0](https://github.com/block-core/blockcore-indexer/commit/1ed7da0)) +* bump version ([c2ed435](https://github.com/block-core/blockcore-indexer/commit/c2ed435)) +* update mainnet key ([c6a1054](https://github.com/block-core/blockcore-indexer/commit/c6a1054)) + + + ## 0.2.65 (2024-05-01) * bump version ([c2e047e](https://github.com/block-core/blockcore-indexer/commit/c2e047e)) +* Fix typo ([d6a73b9](https://github.com/block-core/blockcore-indexer/commit/d6a73b9)) + + + +## 0.2.64 (2024-04-26) + * bump version ([04665ca](https://github.com/block-core/blockcore-indexer/commit/04665ca)) * Changed base image to dotnet 8 (#211) ([37d0fb4](https://github.com/block-core/blockcore-indexer/commit/37d0fb4)), closes [#211](https://github.com/block-core/blockcore-indexer/issues/211) * fix the default null ([2df38ab](https://github.com/block-core/blockcore-indexer/commit/2df38ab)) -* Fix typo ([d6a73b9](https://github.com/block-core/blockcore-indexer/commit/d6a73b9)) diff --git a/package.json b/package.json index 04cdc66..c890313 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "blockcoreindexer", - "version": "0.2.66", + "version": "0.2.67", "license": "MIT", "author": { "name": "Blockcore", diff --git a/src/Blockcore.Indexer.Core/Blockcore.Indexer.Core.csproj b/src/Blockcore.Indexer.Core/Blockcore.Indexer.Core.csproj index 4f19570..889db51 100644 --- a/src/Blockcore.Indexer.Core/Blockcore.Indexer.Core.csproj +++ b/src/Blockcore.Indexer.Core/Blockcore.Indexer.Core.csproj @@ -2,14 +2,17 @@ + + + diff --git a/src/Blockcore.Indexer.Core/Startup.cs b/src/Blockcore.Indexer.Core/Startup.cs index 2e58db9..e32441a 100644 --- a/src/Blockcore.Indexer.Core/Startup.cs +++ b/src/Blockcore.Indexer.Core/Startup.cs @@ -20,6 +20,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.ApplicationModels; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -55,6 +56,8 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio return mongoClient.GetDatabase(dbName); }); + services.AddDbContext(); + // services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -150,7 +153,7 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio services.AddTransient(); } - public static void Configure(IApplicationBuilder app, IWebHostEnvironment env) + public static void Configure(IApplicationBuilder app, IWebHostEnvironment env, PostgresDbContext db) { app.UseExceptionHandler("/error"); @@ -160,6 +163,11 @@ public static void Configure(IApplicationBuilder app, IWebHostEnvironment env) app.UseResponseCompression(); //app.UseMvc(); + db.Database.Migrate(); + + using (var client = new PostgresDbContext()){ + client.Database.EnsureCreated(); + } app.UseDefaultFiles(); diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/PostgresDBContext.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/PostgresDBContext.cs new file mode 100644 index 0000000..a56454b --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/PostgresDBContext.cs @@ -0,0 +1,99 @@ +using System.Globalization; +using Blockcore.Indexer.Core.Storage.Postgres.Types; +using Blockcore.NBitcoin.Protocol; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; + +public class PostgresDbContext : DbContext +{ + public PostgresDbContext() : base() + { + + } + protected override void OnConfiguring(DbContextOptionsBuilder options) + { + options.UseNpgsql("Host=127.0.0.1;Port=5432;Database=IndexerBenchmark;Username=postgres;Password=drb;"); + } + + public DbSet Blocks { get; set; } + public DbSet Transactions { get; set; } + public DbSet Inputs { get; set; } + public DbSet Outputs { get; set; } + public DbSet mempoolTransactions { get; set; } + public DbSet mempoolInputs { get; set; } + public DbSet mempoolOutputs { get; set; } + public DbSet unspentOutputs { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + + modelBuilder.Entity() + .HasKey(b => b.BlockHash); + + modelBuilder.Entity() + .HasKey(t => t.Txid); + + modelBuilder.Entity() + .HasKey(i => new { i.Txid, i.Vout }); + + modelBuilder.Entity() + .HasKey(o => new { o.outpoint.Txid, o.outpoint.Vout }); + + modelBuilder.Entity() + .HasKey(t => t.Txid); + + modelBuilder.Entity() + .HasKey(i => new { i.Txid, i.Vout }); + + modelBuilder.Entity() + .HasKey(o => new { o.outpoint.Txid, o.outpoint.Vout }); + + modelBuilder.Entity() + .HasKey(uo => new { uo.Outpoint.Txid, uo.Outpoint.Vout }); + + + modelBuilder.Entity() + .HasOne(t => t.Block) + .WithMany(b => b.Transactions) + .HasForeignKey(t => t.BlockIndex); + + modelBuilder.Entity() + .HasOne(i => i.Transaction) + .WithMany(t => t.Inputs) + .HasForeignKey(i => i.Txid); + + modelBuilder.Entity() + .HasOne(i => i.Transaction) + .WithMany(t => t.Inputs) + .HasForeignKey(i => i.outpoint.Txid); + + modelBuilder.Entity() + .HasOne(o => o.Transaction) + .WithMany(t => t.Outputs) + .HasForeignKey(o => o.outpoint.Txid); + + modelBuilder.Entity() + .HasOne(i => i.Transaction) + .WithMany(t => t.Inputs) + .HasForeignKey(i => i.Txid); + + modelBuilder.Entity() + .HasOne(o => o.Transaction) + .WithMany(t => t.Outputs) + .HasForeignKey(o => o.outpoint.Txid); + + // modelBuilder.Entity() .HasOne(t => t.) + + modelBuilder.Entity() + .HasIndex(b => b.BlockHash) + .HasMethod("hash"); + + modelBuilder.Entity() + .HasIndex(b => b.BlockIndex) + .IsDescending(); + + modelBuilder.Entity() + .HasIndex(t => t.Txid) + .HasMethod("hash"); + } +} \ No newline at end of file diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressComputedTable.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressComputedTable.cs new file mode 100644 index 0000000..68a310a --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressComputedTable.cs @@ -0,0 +1,34 @@ +using System.ComponentModel.DataAnnotations; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class AddressComputedTable + { + [Key] + public string Id { get; set; } + + public string Address { get; set; } + + public long Available { get; set; } + + public long Received { get; set; } + + public long Sent { get; set; } + + public long Staked { get; set; } + + public long Mined { get; set; } + + public long ComputedBlockIndex { get; set; } + + public long CountReceived { get; set; } + + public long CountSent { get; set; } + + public long CountStaked { get; set; } + + public long CountMined { get; set; } + + public long CountUtxo { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressHistoryComputedTable.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressHistoryComputedTable.cs new file mode 100644 index 0000000..de44a48 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressHistoryComputedTable.cs @@ -0,0 +1,23 @@ +using System.ComponentModel.DataAnnotations; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class AddressHistoryComputedTable + { + [Key] + public string Id { get; set; } + public string Address { get; set; } + + public string EntryType { get; set; } + + public long AmountInInputs { get; set; } + + public long AmountInOutputs { get; set; } + + public string TransactionId { get; set; } + + public long Position { get; set; } + + public uint BlockIndex { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressUtxoComputedTable.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressUtxoComputedTable.cs new file mode 100644 index 0000000..ab7ec73 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/AddressUtxoComputedTable.cs @@ -0,0 +1,17 @@ +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + /// + /// This table is not used anymore to store utxo data in mongodb, + /// however its used in the computation table calculate the utxo count. + /// + public class AddressUtxoComputedTable + { + public Outpoint Outpoint { get; set; } + public string Address { get; set; } + public string ScriptHex { get; set; } + public long Value { get; set; } + public long BlockIndex { get; set; } + public bool CoinBase { get; set; } + public bool CoinStake { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Block.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Block.cs new file mode 100644 index 0000000..9012340 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Block.cs @@ -0,0 +1,52 @@ +using System.Collections; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class Block + { + public string BlockHash { get; set; } + [Key] + public long BlockIndex { get; set; } + + public long BlockSize { get; set; } + + public long BlockTime { get; set; } + + public string NextBlockHash { get; set; } + + public string PreviousBlockHash { get; set; } + + public long Confirmations { get; set; } + + public string Bits { get; set; } + + public double Difficulty { get; set; } + + public string ChainWork { get; set; } + + public string Merkleroot { get; set; } + + public long Nonce { get; set; } + + public long Version { get; set; } + + public bool SyncComplete { get; set; } + + public int TransactionCount { get; set; } + + public string PosBlockSignature { get; set; } + + public string PosModifierv2 { get; set; } + + public string PosFlags { get; set; } + + public string PosHashProof { get; set; } + + public string PosBlockTrust { get; set; } + + public string PosChainTrust { get; set; } + public ICollection Transactions { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Input.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Input.cs new file mode 100644 index 0000000..6f3b170 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Input.cs @@ -0,0 +1,16 @@ +using System.ComponentModel.DataAnnotations; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class Input + { + + public Outpoint outpoint { get; set; } + public string Address { get; set; } + public long Value { get; set; } + public string Txid { get; set; } + public uint Vout { get; set; } + public long BlockIndex { get; set; } + public virtual Transaction Transaction { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/MempoolTransaction.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/MempoolTransaction.cs new file mode 100644 index 0000000..e13afb6 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/MempoolTransaction.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class MempoolInput + { + public Outpoint outpoint { get; set; } + public string Address { get; set; } + public long Value { get; set; } + public string Txid { get; set; } + public uint Vout { get; set; } + public virtual MempoolTransaction Transaction { get; set; } + } + public class MempoolOutput + { + public Outpoint outpoint { get; set; } + public string Address { get; set; } + public string ScriptHex { get; set; } + public long Value { get; set; } + public long BlockIndex { get; set; } + public bool CoinBase { get; set; } + public bool CoinStake { get; set; } + public virtual MempoolTransaction Transaction { get; set; } + } + public class MempoolTransaction + { + public long FirstSeen { get; set; } + public string Txid { get; set; } + public ICollection Inputs { get; set; } + public ICollection Outputs { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Outpoint.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Outpoint.cs new file mode 100644 index 0000000..08d2e79 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Outpoint.cs @@ -0,0 +1,14 @@ +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class Outpoint + { + public string Txid { get; set; } + + public int Vout { get; set; } + + public override string ToString() + { + return Txid + "-" + Vout; + } + } +} \ No newline at end of file diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Output.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Output.cs new file mode 100644 index 0000000..a90b80a --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Output.cs @@ -0,0 +1,16 @@ +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class Output + { + // public string Txid { get; set; } //foreign key + // public uint Vout { get; set; } + public Outpoint outpoint { get; set; } + public string Address { get; set; } + public string ScriptHex { get; set; } + public long Value { get; set; } + public uint BlockIndex { get; set; } + public bool CoinBase { get; set; } + public bool CoinStake { get; set; } + public virtual Transaction Transaction { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/ReorgBlockTable.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/ReorgBlockTable.cs new file mode 100644 index 0000000..8ceb472 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/ReorgBlockTable.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types; + +public class ReorgBlockTable +{ + public DateTime Created { get; set; } + public uint BlockIndex { get; set; } + public string BlockHash { get; set; } + public Block Block { get; set; } + +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/RichlistTable.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/RichlistTable.cs new file mode 100644 index 0000000..3867a76 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/RichlistTable.cs @@ -0,0 +1,11 @@ +using System.ComponentModel.DataAnnotations; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class RichlistTable + { + [Key] + public string Address { get; set; } + public long Balance { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Transaction.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Transaction.cs new file mode 100644 index 0000000..14aec26 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/Transaction.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using NLog; + +namespace Blockcore.Indexer.Core.Storage.Postgres.Types +{ + public class Transaction + { + public byte[] RawTransaction { get; set; } + [Key] + public string Txid { get; set; } + public long BlockIndex { get; set; } + public int TransactionIndex { get; set; } + public short NumberOfOutputs { get; set; } + public ICollection Inputs { get; set; } + public ICollection Outputs { get; set; } + public virtual Block Block { get; set; } + } +} + diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/Types/UnspentOutput.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/UnspentOutput.cs new file mode 100644 index 0000000..6d66a37 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/Types/UnspentOutput.cs @@ -0,0 +1,12 @@ +namespace Blockcore.Indexer.Core.Storage.Postgres.Types; + +public class UnspentOutput +{ + public Outpoint Outpoint { get; set; } + + public string Address { get; set; } + + public long Value { get; set; } + + public uint BlockIndex { get; set; } +} diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 98e5c3a..146b220 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,6 +1,6 @@ - 0.2.66 + 0.2.67 net8.0 Blockcore MIT From 2901039cc96a9a1f18f62f1688c7ee8a3fe3458a Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali <119250923+theDRB123@users.noreply.github.com> Date: Mon, 17 Jun 2024 01:31:35 +0530 Subject: [PATCH 2/2] Added PostgresStorageOperations, changed StorageBatch(temp) --- .../Operations/Types/StorageBatch.cs | 13 +- .../Postgres/IMapPgBlockToStorageBlock.cs | 13 + .../Postgres/MapPgBlockToStorageBlock.cs | 95 +++++++ .../Postgres/PostgresStorageOperations.cs | 250 ++++++++++++++++++ 4 files changed, 365 insertions(+), 6 deletions(-) create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/IMapPgBlockToStorageBlock.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/MapPgBlockToStorageBlock.cs create mode 100644 src/Blockcore.Indexer.Core/Storage/Postgres/PostgresStorageOperations.cs diff --git a/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs b/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs index f6a5084..e76730b 100644 --- a/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs +++ b/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs @@ -1,5 +1,5 @@ using System.Collections.Generic; -using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Blockcore.Indexer.Core.Storage.Postgres.Types; namespace Blockcore.Indexer.Core.Operations.Types { @@ -10,11 +10,12 @@ namespace Blockcore.Indexer.Core.Operations.Types public class StorageBatch { public long TotalSize { get; set; } - public List TransactionBlockTable { get; set; } = new(); - public Dictionary BlockTable { get; set; } = new(); - public List TransactionTable { get; set; } = new(); - public Dictionary OutputTable { get; set; } = new(); - public List InputTable { get; set; } = new(); + // public List TransactionBlockTable { get; set; } = new(); + // public Dictionary BlockTable { get; set; } = new(); + // public List TransactionTable { get; set; } = new(); + public List Inputs { get; set; } = []; + public Dictionary Blocks { get; set; } = []; + public Dictionary Outputs { get; set; } = []; public object ExtraData { get; set; } } diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/IMapPgBlockToStorageBlock.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/IMapPgBlockToStorageBlock.cs new file mode 100644 index 0000000..6907e17 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/IMapPgBlockToStorageBlock.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using Blockcore.Indexer.Core.Client.Types; +using Blockcore.Indexer.Core.Storage.Postgres.Types; +using Blockcore.Indexer.Core.Storage.Types; + +namespace Blockcore.Indexer.Core.Storage +{ + public interface IMapPgBlockToStorageBlock + { + SyncBlockInfo Map(Block block); + Block Map(BlockInfo blockInfo); + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/MapPgBlockToStorageBlock.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/MapPgBlockToStorageBlock.cs new file mode 100644 index 0000000..a695c1b --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/MapPgBlockToStorageBlock.cs @@ -0,0 +1,95 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net.Http.Headers; +using Blockcore.Indexer.Core.Client.Types; +using Blockcore.Indexer.Core.Storage.Postgres.Types; +using Blockcore.Indexer.Core.Storage.Types; + +namespace Blockcore.Indexer.Core.Storage +{ + public class MapPgBlockToStorageBlock : IMapPgBlockToStorageBlock + { + public SyncBlockInfo Map(Block block) => new SyncBlockInfo + { + BlockIndex = block.BlockIndex, + BlockSize = block.BlockSize, + BlockHash = block.BlockHash, + BlockTime = block.BlockTime, + NextBlockHash = block.NextBlockHash, + PreviousBlockHash = block.PreviousBlockHash, + TransactionCount = block.TransactionCount, + Nonce = block.Nonce, + ChainWork = block.ChainWork, + Difficulty = block.Difficulty, + Merkleroot = block.Merkleroot, + PosModifierv2 = block.PosModifierv2, + PosHashProof = block.PosHashProof, + PosFlags = block.PosFlags, + PosChainTrust = block.PosChainTrust, + PosBlockTrust = block.PosBlockTrust, + PosBlockSignature = block.PosBlockSignature, + Confirmations = block.Confirmations, + Bits = block.Bits, + Version = block.Version, + SyncComplete = block.SyncComplete + }; + + + public Block Map(BlockInfo block, List transactions) => + + new Block + { + BlockIndex = block.Height, + BlockHash = block.Hash, + BlockSize = block.Size, + BlockTime = block.Time, + NextBlockHash = block.NextBlockHash, + PreviousBlockHash = block.PreviousBlockHash, + TransactionCount = block.Transactions.Count(), + Bits = block.Bits, + Confirmations = block.Confirmations, + Merkleroot = block.Merkleroot, + Nonce = block.Nonce, + ChainWork = block.ChainWork, + Difficulty = block.Difficulty, + PosBlockSignature = block.PosBlockSignature, + PosBlockTrust = block.PosBlockTrust, + PosChainTrust = block.PosChainTrust, + PosFlags = block.PosFlags, + PosHashProof = block.PosHashProof, + PosModifierv2 = block.PosModifierv2, + Version = block.Version, + SyncComplete = false, + Transactions = transactions + }; + public Block Map(BlockInfo block) + { + Block output = new Block + { + BlockIndex = block.Height, + BlockHash = block.Hash, + BlockSize = block.Size, + BlockTime = block.Time, + NextBlockHash = block.NextBlockHash, + PreviousBlockHash = block.PreviousBlockHash, + TransactionCount = block.Transactions.Count(), + Bits = block.Bits, + Confirmations = block.Confirmations, + Merkleroot = block.Merkleroot, + Nonce = block.Nonce, + ChainWork = block.ChainWork, + Difficulty = block.Difficulty, + PosBlockSignature = block.PosBlockSignature, + PosBlockTrust = block.PosBlockTrust, + PosChainTrust = block.PosChainTrust, + PosFlags = block.PosFlags, + PosHashProof = block.PosHashProof, + PosModifierv2 = block.PosModifierv2, + Version = block.Version, + SyncComplete = false, + Transactions = [], + }; + return output; + } + } +} diff --git a/src/Blockcore.Indexer.Core/Storage/Postgres/PostgresStorageOperations.cs b/src/Blockcore.Indexer.Core/Storage/Postgres/PostgresStorageOperations.cs new file mode 100644 index 0000000..195ea23 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Storage/Postgres/PostgresStorageOperations.cs @@ -0,0 +1,250 @@ + +using Blockcore.Indexer.Core.Crypto; +using Blockcore.Indexer.Core.Operations; +using Blockcore.Indexer.Core.Operations.Types; +using Blockcore.Indexer.Core.Settings; +using Blockcore.Indexer.Core.Storage.Types; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; +using Blockcore.Indexer.Core.Storage.Postgres.Types; +using Blockcore.NBitcoin; +using Blockcore.Consensus.TransactionInfo; +using Transaction = Blockcore.Indexer.Core.Storage.Postgres.Types.Transaction; +using System.Linq; +using System.Collections.Generic; +using System; +using System.Threading.Tasks; + +namespace Blockcore.Indexer.Core.Storage.Postgres +{ + public class PostgresStorageOperations : IStorageOperations + { + const string OpReturnAddress = "TX_NULL_DATA"; + protected readonly SyncConnection syncConnection; + protected readonly GlobalState globalState; + protected readonly IScriptInterpreter scriptInterpeter; + protected readonly IndexerSettings configuration; + protected readonly PostgresDbContext db; + protected readonly IStorage storage; + //todo -> change to generic interface + protected readonly IMapPgBlockToStorageBlock pgBlockToStorageBlock; + + public PostgresStorageOperations( + SyncConnection connection, + IStorage storage, + IUtxoCache utxoCache, + IOptions configuration, + GlobalState globalState, + IMapPgBlockToStorageBlock pgBlockToStorageBlock, + IScriptInterpreter scriptInterpeter, + PostgresDbContext context) + { + syncConnection = connection; + // this.storage = storage; + this.globalState = globalState; + this.pgBlockToStorageBlock = pgBlockToStorageBlock; + this.scriptInterpeter = scriptInterpeter; + db = context; + this.configuration = configuration.Value; + } + + public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) + { + storageBatch.TotalSize += item.BlockInfo.Size; + Block block = pgBlockToStorageBlock.Map(item.BlockInfo); + + int transactionIndex = 0; + foreach (var trx in item.Transactions) + { + string txid = trx.GetHash().ToString(); + Transaction transaction = new() + { + BlockIndex = item.BlockInfo.HeightAsUint32, + Txid = txid, + TransactionIndex = transactionIndex++, + RawTransaction = configuration.StoreRawTransactions ? trx.ToBytes(syncConnection.Network.Consensus.ConsensusFactory) : null, + Inputs = [], + Outputs = [] + }; + + int outputIndex = 0; + foreach (TxOut output in trx.Outputs) + { + ScriptOutputInfo res = scriptInterpeter.InterpretScript(syncConnection.Network, output.ScriptPubKey); + string addr = res != null + ? (res?.Addresses != null && res.Addresses.Any()) ? res.Addresses.First() : res.ScriptType + : "none"; + + var outpoint = new Outpoint { Txid = txid, Vout = outputIndex++ }; + Output storageOutput = new Output + { + outpoint = outpoint, + Address = addr, + BlockIndex = item.BlockInfo.HeightAsUint32, + Value = output.Value, + ScriptHex = output.ScriptPubKey.ToHex(), + CoinBase = trx.IsCoinBase, + CoinStake = syncConnection.Network.Consensus.IsProofOfStake && trx.IsCoinStake + }; + + transaction.Outputs.Add(storageOutput); + storageBatch.Outputs.Add(outpoint.ToString(), storageOutput); + } + + if (trx.IsCoinBase) + continue; + + int inputIndex = 0; + foreach (TxIn input in trx.Inputs) + { + var outpoint = new Outpoint { Txid = input.PrevOut.Hash.ToString(), Vout = (int)input.PrevOut.N }; + storageBatch.Outputs.TryGetValue(outpoint.ToString(), out Output output); + + Input storageInput = new Input + { + outpoint = outpoint, + Txid = txid, + BlockIndex = item.BlockInfo.HeightAsUint32, + Value = output?.Value ?? 0, + }; + transaction.Inputs.Add(storageInput); + storageBatch.Inputs.Add(storageInput); + } + + block.Transactions.Add(transaction); + block.TransactionCount++; + } + OnAddtoStorageBatch(storageBatch, item); + storageBatch.Blocks.Add(item.BlockInfo.Height, pgBlockToStorageBlock.Map(item.BlockInfo)); + } + + private void OnAddtoStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) => throw new NotImplementedException(); + public InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item) => throw new System.NotImplementedException(); + public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) + { + if (globalState.IndexModeCompleted) + { + if (globalState.IbdMode() == false) + { + if (globalState.LocalMempoolView.Any()) + { + var toRemoveFromMempool = storageBatch.Blocks.Values.SelectMany(b => b.Transactions.Select(t => t.Txid)); + Task deleteFromMempoolTask = Task.Run(async () => + { + await db.mempoolTransactions.Where(mt => toRemoveFromMempool.Contains(mt.Txid)).ExecuteDeleteAsync(); + }); + // db.BulkDelete(storageBatch.Blocks.Values.SelectMany(b => b.Transactions)); + + deleteFromMempoolTask.Wait(); + + foreach (string mempooltrx in toRemoveFromMempool) + { + globalState.LocalMempoolView.Remove(mempooltrx, out _); + } + } + } + } + + + var utxos = new List(storageBatch.Outputs.Values.Count); + + foreach (Output output in storageBatch.Outputs.Values) + { + if (output.Address.Equals(OpReturnAddress)) + continue; + + // TODO: filter out outputs that are already spent in the storageBatch.InputTable table + // such inputs will get deleted anyway in the next operation of UnspentOutputTable.DeleteMany + // this means we should probably make the storageBatch.InputTable a dictionary as well. + + utxos.Add(new UnspentOutput + { + Address = output.Address, + Outpoint = output.outpoint, + Value = output.Value, + BlockIndex = output.BlockIndex, + }); + } + + Task utxoInsertTask = utxos.Any() ? Task.Run(async () => await db.BulkInsertAsync(utxos)) + : Task.CompletedTask; + + if (storageBatch.Inputs.Any()) + { + var utxoLookups = FetchUtxos(storageBatch.Inputs + .Where(_ => _.Address == null) + .Select(_ => _.outpoint)); + + foreach (Input input in storageBatch.Inputs) + { + if (input.Address != null) + { + continue; + } + + string key = input.outpoint.ToString(); + input.Address = utxoLookups[key].Address; + input.Value = utxoLookups[key].Value; + } + } + + Task pushBatchToPostgresTask = Task.Run(async () => + { + await db.BulkInsertAsync(storageBatch.Blocks.Values, options => { options.IncludeGraph = true; }); + }); + + Task.WaitAll(pushBatchToPostgresTask, utxoInsertTask); + + if (storageBatch.Inputs.Any()) + { + // TODO: if earlier we filtered out outputs that are already spent and not pushed to the utxo table + // now we do not need to try and delete such outputs becuase they where never pushed to the store. + var outpointsFromNewInput = storageBatch.Inputs.Select(_ => _.outpoint).ToList(); + + int rowsDeleted = db.unspentOutputs.Where(uo => outpointsFromNewInput.Contains(uo.Outpoint)).ExecuteDelete(); + + if (rowsDeleted != outpointsFromNewInput.Count) + { + throw new ApplicationException($"Delete of unspent outputs did not complete successfully : {rowsDeleted} deleted but {outpointsFromNewInput.Count} expected"); + } + } + + OnPushStorageBatch(storageBatch); + + string lastBlockHash = null; + long blockIndex = 0; + List markBlocksAsComplete = []; + foreach (Block mapBlock in storageBatch.Blocks.Values.OrderBy(b => b.BlockIndex)) + { + mapBlock.SyncComplete = true; + markBlocksAsComplete.Add(mapBlock); + lastBlockHash = mapBlock.BlockHash; + blockIndex = mapBlock.BlockIndex; + } + + db.BulkUpdate(markBlocksAsComplete); + + SyncBlockInfo block = storage.BlockByIndex(blockIndex); + + if (block.BlockHash != lastBlockHash) + { + throw new ArgumentException($"Expected hash {blockIndex} for block {lastBlockHash} but was {block.BlockHash}"); + } + + return block; + } + + private void OnPushStorageBatch(StorageBatch storageBatch) => throw new NotImplementedException(); + + private Dictionary FetchUtxos(IEnumerable outputs) + { + var outpoints = outputs.Select(o => o.ToString()).ToList(); + + var res = db.unspentOutputs + .Where(utxo => outpoints.Contains(utxo.Outpoint.ToString())) + .ToDictionary(_ => _.Outpoint.ToString()); + + return res; + } + } +} \ No newline at end of file