From 23fc3a48561b9304fa823a671977364dba36995e Mon Sep 17 00:00:00 2001 From: Priojeet Das Priyom Date: Mon, 20 Nov 2023 16:58:49 +0100 Subject: [PATCH 1/6] Fix wrong transaction count in new.block event and new.transactions events not being fired --- services/blockchain-indexer/events/blockchain.js | 16 ++++++++++++---- .../shared/dataService/blocks.js | 4 ++-- .../shared/messageProcessor.js | 7 +++++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/services/blockchain-indexer/events/blockchain.js b/services/blockchain-indexer/events/blockchain.js index e01a13c40d..89f23003ca 100644 --- a/services/blockchain-indexer/events/blockchain.js +++ b/services/blockchain-indexer/events/blockchain.js @@ -17,7 +17,6 @@ const { Logger, Signals } = require('lisk-service-framework'); const { reloadAllPendingTransactions, - getTransactionsByBlockID, reloadGeneratorsCache, getGenerators, } = require('../shared/dataService'); @@ -64,12 +63,21 @@ module.exports = [ try { if (payload && Array.isArray(payload.data)) { const [block] = payload.data; - if (block.numberOfTransactions > 0) { + const { numberOfTransactions } = block; + + if (numberOfTransactions > 0) { logger.debug( `Block (${block.id}) arrived containing ${block.numberOfTransactions} new transactions`, ); - const transactionData = await getTransactionsByBlockID(block.id); - callback(transactionData); + const transactionsPayload = { + data: block.transactions, + meta: { + count: numberOfTransactions, + offset: 0, + total: numberOfTransactions, + }, + }; + callback(transactionsPayload); } } else { const payloadStr = JSON.stringify(payload); diff --git a/services/blockchain-indexer/shared/dataService/blocks.js b/services/blockchain-indexer/shared/dataService/blocks.js index 8d1d38e2a8..b6577334e2 100644 --- a/services/blockchain-indexer/shared/dataService/blocks.js +++ b/services/blockchain-indexer/shared/dataService/blocks.js @@ -79,14 +79,14 @@ const getBlocksTotal = async (params, blocksResponse) => { return total; }; -const formatBlock = async (header, isDeletedBlock = false) => { +const formatBlock = async (blockInfo, isDeletedBlock = false) => { const blocksResponse = { data: [], meta: {}, }; const response = await business.formatBlock( - { header, assets: [], transactions: [] }, + { header: blockInfo.header, assets: [], transactions: blockInfo.transactions || [] }, isDeletedBlock, ); blocksResponse.data.push(response); diff --git a/services/blockchain-indexer/shared/messageProcessor.js b/services/blockchain-indexer/shared/messageProcessor.js index 208e660217..a343797a08 100644 --- a/services/blockchain-indexer/shared/messageProcessor.js +++ b/services/blockchain-indexer/shared/messageProcessor.js @@ -36,6 +36,7 @@ const { reloadValidatorCache, getGenerators, getNumberOfGenerators, + getTransactionsByBlockID, } = require('./dataService'); const { accountAddrUpdateQueue } = require('./indexer/accountIndex'); @@ -77,8 +78,10 @@ const initQueueStatus = async () => { const newBlockProcessor = async header => { logger.debug(`New block (${header.id}) received at height ${header.height}.`); - const response = await formatBlock(header); + const { data: transactions } = await getTransactionsByBlockID(header.id); + const response = await formatBlock({ header, transactions }); const [newBlock] = response.data; + await indexNewBlock(newBlock); await performLastBlockUpdate(newBlock); Signals.get('newBlock').dispatch(response); @@ -92,7 +95,7 @@ const deleteBlockProcessor = async header => { logger.debug( `Scheduling the delete block (${header.id}) event for the block at height ${header.height}.`, ); - const response = await formatBlock(header, true); + const response = await formatBlock({ header }, true); await scheduleBlockDeletion(header); Signals.get('deleteBlock').dispatch(response); logger.info( From 96d5d54309aeffb9db011d6477fc708e5694e7d9 Mon Sep 17 00:00:00 2001 From: Priojeet Das Priyom Date: Tue, 21 Nov 2023 13:59:23 +0100 Subject: [PATCH 2/6] Apply suggestions --- services/blockchain-indexer/shared/constants.js | 2 ++ .../shared/dataService/blocks.js | 7 ++++++- .../shared/messageProcessor.js | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/services/blockchain-indexer/shared/constants.js b/services/blockchain-indexer/shared/constants.js index 52710a7e26..b55f7b6228 100644 --- a/services/blockchain-indexer/shared/constants.js +++ b/services/blockchain-indexer/shared/constants.js @@ -156,6 +156,7 @@ const LENGTH_BYTE_SIGNATURE = 64; const LENGTH_BYTE_ID = 32; const DEFAULT_NUM_OF_SIGNATURES = 1; const LENGTH_ID = LENGTH_BYTE_ID * 2; // Each byte is represented with 2 nibbles +const EMPTY_TREE_ROOT = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; const MAX_COMMISSION = BigInt('10000'); @@ -249,4 +250,5 @@ module.exports = { LENGTH_ID, DEFAULT_NUM_OF_SIGNATURES, CCM_SENT_FAILED_ERROR_MESSAGE, + EMPTY_TREE_ROOT, }; diff --git a/services/blockchain-indexer/shared/dataService/blocks.js b/services/blockchain-indexer/shared/dataService/blocks.js index b6577334e2..0a9eb89748 100644 --- a/services/blockchain-indexer/shared/dataService/blocks.js +++ b/services/blockchain-indexer/shared/dataService/blocks.js @@ -86,7 +86,12 @@ const formatBlock = async (blockInfo, isDeletedBlock = false) => { }; const response = await business.formatBlock( - { header: blockInfo.header, assets: [], transactions: blockInfo.transactions || [] }, + { + header: {}, + assets: [], + transactions: [], + ...blockInfo, + }, isDeletedBlock, ); blocksResponse.data.push(response); diff --git a/services/blockchain-indexer/shared/messageProcessor.js b/services/blockchain-indexer/shared/messageProcessor.js index a343797a08..5e49177ce8 100644 --- a/services/blockchain-indexer/shared/messageProcessor.js +++ b/services/blockchain-indexer/shared/messageProcessor.js @@ -21,7 +21,7 @@ const config = require('../config'); const logger = Logger(); -const { initNodeConstants } = require('./constants'); +const { initNodeConstants, EMPTY_TREE_ROOT } = require('./constants'); const { addHeightToIndexBlocksQueue, @@ -36,7 +36,7 @@ const { reloadValidatorCache, getGenerators, getNumberOfGenerators, - getTransactionsByBlockID, + getBlockByID, } = require('./dataService'); const { accountAddrUpdateQueue } = require('./indexer/accountIndex'); @@ -78,8 +78,16 @@ const initQueueStatus = async () => { const newBlockProcessor = async header => { logger.debug(`New block (${header.id}) received at height ${header.height}.`); - const { data: transactions } = await getTransactionsByBlockID(header.id); - const response = await formatBlock({ header, transactions }); + let transactions = []; + let assets = []; + + if (header.transactionRoot !== EMPTY_TREE_ROOT || header.assetRoot !== EMPTY_TREE_ROOT) { + const block = await getBlockByID(header.id); + transactions = block.transactions; + assets = block.assets; + } + + const response = await formatBlock({ header, assets, transactions }); const [newBlock] = response.data; await indexNewBlock(newBlock); From 717626e622b719fcd72e497a892385705b9cfce5 Mon Sep 17 00:00:00 2001 From: Priojeet Das Priyom Date: Tue, 21 Nov 2023 16:26:01 +0100 Subject: [PATCH 3/6] Apply suggestions --- services/blockchain-indexer/shared/constants.js | 4 ++-- .../shared/dataService/business/blocks.js | 10 ++++++++-- services/blockchain-indexer/shared/messageProcessor.js | 7 +++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/services/blockchain-indexer/shared/constants.js b/services/blockchain-indexer/shared/constants.js index b55f7b6228..7c2c34ad8a 100644 --- a/services/blockchain-indexer/shared/constants.js +++ b/services/blockchain-indexer/shared/constants.js @@ -156,7 +156,7 @@ const LENGTH_BYTE_SIGNATURE = 64; const LENGTH_BYTE_ID = 32; const DEFAULT_NUM_OF_SIGNATURES = 1; const LENGTH_ID = LENGTH_BYTE_ID * 2; // Each byte is represented with 2 nibbles -const EMPTY_TREE_ROOT = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; +const EMPTY_TREE_ROOT_HASH = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; const MAX_COMMISSION = BigInt('10000'); @@ -250,5 +250,5 @@ module.exports = { LENGTH_ID, DEFAULT_NUM_OF_SIGNATURES, CCM_SENT_FAILED_ERROR_MESSAGE, - EMPTY_TREE_ROOT, + EMPTY_TREE_ROOT_HASH, }; diff --git a/services/blockchain-indexer/shared/dataService/business/blocks.js b/services/blockchain-indexer/shared/dataService/business/blocks.js index b73a4caa40..19ab81f0cb 100644 --- a/services/blockchain-indexer/shared/dataService/business/blocks.js +++ b/services/blockchain-indexer/shared/dataService/business/blocks.js @@ -27,7 +27,7 @@ const { const logger = Logger(); const { getEventsByHeight, getEventsByBlockID } = require('./events'); -const { getFinalizedHeight, MODULE, EVENT } = require('../../constants'); +const { getFinalizedHeight, MODULE, EVENT, getGenesisHeight } = require('../../constants'); const blocksTableSchema = require('../../database/schema/blocks'); const { getIndexedAccountInfo } = require('../utils/account'); @@ -75,7 +75,13 @@ const normalizeBlock = async (originalBlock, isDeletedBlock = false) => { block.isFinal = block.height <= (await getFinalizedHeight()); block.numberOfTransactions = block.transactions.length; - block.numberOfAssets = block.assets.length; + block.numberOfAssets = + block.height !== (await getGenesisHeight()) + ? block.assets.length + : await (async () => { + const response = await requestConnector('getGenesisAssetsLength'); + return Object.entries(response).length; + })(); const { numberOfEvents, reward } = await (async () => { const [dbResponse] = await blocksTable.find({ height: block.height, limit: 1 }, [ diff --git a/services/blockchain-indexer/shared/messageProcessor.js b/services/blockchain-indexer/shared/messageProcessor.js index 5e49177ce8..92553ffefa 100644 --- a/services/blockchain-indexer/shared/messageProcessor.js +++ b/services/blockchain-indexer/shared/messageProcessor.js @@ -21,7 +21,7 @@ const config = require('../config'); const logger = Logger(); -const { initNodeConstants, EMPTY_TREE_ROOT } = require('./constants'); +const { initNodeConstants, EMPTY_TREE_ROOT_HASH } = require('./constants'); const { addHeightToIndexBlocksQueue, @@ -81,7 +81,10 @@ const newBlockProcessor = async header => { let transactions = []; let assets = []; - if (header.transactionRoot !== EMPTY_TREE_ROOT || header.assetRoot !== EMPTY_TREE_ROOT) { + if ( + header.transactionRoot !== EMPTY_TREE_ROOT_HASH || + header.assetRoot !== EMPTY_TREE_ROOT_HASH + ) { const block = await getBlockByID(header.id); transactions = block.transactions; assets = block.assets; From 0970422823e7b77fd3b5d5ccc081738fd519a3b6 Mon Sep 17 00:00:00 2001 From: Priojeet Das Priyom Date: Wed, 22 Nov 2023 14:02:24 +0100 Subject: [PATCH 4/6] Fetch block in connector before newBlock event emission --- .../events/controller/blockchain.js | 30 +++++++++++++++++-- .../shared/eventsScheduler.js | 9 +++--- .../blockchain-indexer/shared/constants.js | 2 -- .../shared/messageProcessor.js | 27 +++++------------ 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/services/blockchain-connector/events/controller/blockchain.js b/services/blockchain-connector/events/controller/blockchain.js index 713c78c055..00b4282bb3 100644 --- a/services/blockchain-connector/events/controller/blockchain.js +++ b/services/blockchain-connector/events/controller/blockchain.js @@ -14,8 +14,11 @@ * */ const { Signals } = require('lisk-service-framework'); +const { getBlockByID } = require('../../shared/sdk/endpoints'); const { formatBlock: formatBlockFromFormatter } = require('../../shared/sdk/formatter'); +const EMPTY_TREE_ROOT_HASH = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; + const appReadyController = async cb => { const appReadyListener = async payload => cb(payload); Signals.get('appReady').add(appReadyListener); @@ -54,12 +57,33 @@ const chainValidatorsChangeController = async cb => { const formatBlock = payload => formatBlockFromFormatter({ header: payload.blockHeader, - assets: [], - transactions: [], + assets: payload.assets || [], + transactions: payload.transactions || [], }); const chainNewBlockController = async cb => { - const chainNewBlockListener = async payload => cb(formatBlock(payload)); + const chainNewBlockListener = async payload => { + const { blockHeader } = payload; + let transactions = []; + let assets = []; + + if ( + blockHeader.transactionRoot !== EMPTY_TREE_ROOT_HASH || + blockHeader.assetRoot !== EMPTY_TREE_ROOT_HASH + ) { + const block = await getBlockByID(blockHeader.id); + transactions = block.transactions; + assets = block.assets; + } + + cb( + formatBlock({ + blockHeader, + assets, + transactions, + }), + ); + }; Signals.get('chainNewBlock').add(chainNewBlockListener); }; diff --git a/services/blockchain-coordinator/shared/eventsScheduler.js b/services/blockchain-coordinator/shared/eventsScheduler.js index ef13255071..bdc11149a0 100644 --- a/services/blockchain-coordinator/shared/eventsScheduler.js +++ b/services/blockchain-coordinator/shared/eventsScheduler.js @@ -24,11 +24,10 @@ const eventMessageQueue = new MessageQueue(config.queue.event.name, config.endpo defaultJobOptions: config.queue.defaultJobOptions, }); -const scheduleUpdatesOnNewBlock = async payload => { - const { header } = payload; - logger.debug(`Scheduling indexing new block at height: ${header.height}.`); - await eventMessageQueue.add({ header, isNewBlock: true }); - logger.info(`Finished scheduling indexing new block at height: ${header.height}.`); +const scheduleUpdatesOnNewBlock = async block => { + logger.debug(`Scheduling indexing new block at height: ${block.height}.`); + await eventMessageQueue.add({ block, isNewBlock: true }); + logger.info(`Finished scheduling indexing new block at height: ${block.height}.`); }; const scheduleDeleteBlock = async payload => { diff --git a/services/blockchain-indexer/shared/constants.js b/services/blockchain-indexer/shared/constants.js index 7c2c34ad8a..52710a7e26 100644 --- a/services/blockchain-indexer/shared/constants.js +++ b/services/blockchain-indexer/shared/constants.js @@ -156,7 +156,6 @@ const LENGTH_BYTE_SIGNATURE = 64; const LENGTH_BYTE_ID = 32; const DEFAULT_NUM_OF_SIGNATURES = 1; const LENGTH_ID = LENGTH_BYTE_ID * 2; // Each byte is represented with 2 nibbles -const EMPTY_TREE_ROOT_HASH = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; const MAX_COMMISSION = BigInt('10000'); @@ -250,5 +249,4 @@ module.exports = { LENGTH_ID, DEFAULT_NUM_OF_SIGNATURES, CCM_SENT_FAILED_ERROR_MESSAGE, - EMPTY_TREE_ROOT_HASH, }; diff --git a/services/blockchain-indexer/shared/messageProcessor.js b/services/blockchain-indexer/shared/messageProcessor.js index 92553ffefa..08cb0b7289 100644 --- a/services/blockchain-indexer/shared/messageProcessor.js +++ b/services/blockchain-indexer/shared/messageProcessor.js @@ -21,7 +21,7 @@ const config = require('../config'); const logger = Logger(); -const { initNodeConstants, EMPTY_TREE_ROOT_HASH } = require('./constants'); +const { initNodeConstants } = require('./constants'); const { addHeightToIndexBlocksQueue, @@ -36,7 +36,6 @@ const { reloadValidatorCache, getGenerators, getNumberOfGenerators, - getBlockByID, } = require('./dataService'); const { accountAddrUpdateQueue } = require('./indexer/accountIndex'); @@ -76,28 +75,16 @@ const initQueueStatus = async () => { await queueStatus(eventMessageQueue); }; -const newBlockProcessor = async header => { - logger.debug(`New block (${header.id}) received at height ${header.height}.`); - let transactions = []; - let assets = []; - - if ( - header.transactionRoot !== EMPTY_TREE_ROOT_HASH || - header.assetRoot !== EMPTY_TREE_ROOT_HASH - ) { - const block = await getBlockByID(header.id); - transactions = block.transactions; - assets = block.assets; - } - - const response = await formatBlock({ header, assets, transactions }); +const newBlockProcessor = async block => { + logger.debug(`New block (${block.id}) received at height ${block.height}.`); + const response = await formatBlock(block); const [newBlock] = response.data; await indexNewBlock(newBlock); await performLastBlockUpdate(newBlock); Signals.get('newBlock').dispatch(response); logger.info( - `Finished scheduling new block (${header.id}) event for the block at height ${header.height}.`, + `Finished scheduling new block (${block.id}) event for the block at height ${block.height}.`, ); }; @@ -155,8 +142,8 @@ const initMessageProcessors = async () => { const { isNewBlock, isDeleteBlock, isNewRound } = job.data; if (isNewBlock) { - const { header } = job.data; - await newBlockProcessor(header); + const { block } = job.data; + await newBlockProcessor(block); } else if (isDeleteBlock) { try { const { header } = job.data; From 28a7e0076ea0ebccfaa6a367994dbfee076bbcf0 Mon Sep 17 00:00:00 2001 From: Priojeet Das Priyom Date: Wed, 22 Nov 2023 15:00:51 +0100 Subject: [PATCH 5/6] Fix total is null for first newBlock event --- services/blockchain-indexer/shared/dataService/blocks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/blockchain-indexer/shared/dataService/blocks.js b/services/blockchain-indexer/shared/dataService/blocks.js index 0a9eb89748..12f2d47086 100644 --- a/services/blockchain-indexer/shared/dataService/blocks.js +++ b/services/blockchain-indexer/shared/dataService/blocks.js @@ -73,7 +73,7 @@ const getBlocksTotal = async (params, blocksResponse) => { ) { total = blocksResponse.meta.total; } else { - total = await getTotalNumberOfBlocks(); + total = (await getTotalNumberOfBlocks()) || blocksResponse.data.length; } return total; From 9d26b671e10e98b9ffa6ac8d92e14ccc42ae6571 Mon Sep 17 00:00:00 2001 From: Priojeet Das Priyom Date: Wed, 22 Nov 2023 15:07:17 +0100 Subject: [PATCH 6/6] :art: Nitpick --- services/blockchain-indexer/shared/dataService/blocks.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/blockchain-indexer/shared/dataService/blocks.js b/services/blockchain-indexer/shared/dataService/blocks.js index 12f2d47086..2f309f2fca 100644 --- a/services/blockchain-indexer/shared/dataService/blocks.js +++ b/services/blockchain-indexer/shared/dataService/blocks.js @@ -88,9 +88,8 @@ const formatBlock = async (blockInfo, isDeletedBlock = false) => { const response = await business.formatBlock( { header: {}, - assets: [], - transactions: [], - ...blockInfo, + assets: blockInfo.assets || [], + transactions: blockInfo.transactions || [], }, isDeletedBlock, );