Merge pull request #1953 from LiskHQ/1952-Event-API-takes-a-long-time…

…-to-return-result-when-queried-with-topics

Event API takes a long time to return result when queried with topics

README.md

@@ -106,8 +106,8 @@ The following system requirements are recommended to start Lisk Service:
 
 #### Memory
 
-- Machines with a minimum of 8 GB RAM for the Mainnet.
-- Machines with a minimum of 8 GB RAM for the Testnet.
+- Machines with a minimum of 16 GB RAM for the Mainnet.
+- Machines with a minimum of 16 GB RAM for the Testnet.
 
 #### Storage
 

docker/example.env

@@ -89,7 +89,7 @@ ENABLE_PERSIST_EVENTS=false
 # INDEX_MISSING_BLOCKS_SKIP_THRESHOLD=1000
 # INDEX_MISSING_BLOCKS_MAX_SCHEDULE=25000
 # JOB_INTERVAL_INDEX_MISSING_BLOCKS=0
-# JOB_SCHEDULE_INDEX_MISSING_BLOCKS='*/5 * * * *'
+# JOB_SCHEDULE_INDEX_MISSING_BLOCKS='* * * * *'
 
 ## Lisk Service Fee Estimator
 ENABLE_FEE_ESTIMATOR_QUICK=true

docs/antora/modules/ROOT/pages/configuration/index.adoc

@@ -537,8 +537,8 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_INDEX_MISSING_BLOCKS`
 | string
 | Job run cron schedule to index missing blocks.
-By default, it is set to run every 5 minutes.
-| */5 * * * *
+By default, it is set to run every minute.
+| * * * * *
 |===
 
 .Sample Configurations
@@ -564,7 +564,7 @@ module.exports = {
         // SERVICE_LOG_FILE: false,
         // DOCKER_HOST: 'local',
         // JOB_INTERVAL_INDEX_MISSING_BLOCKS: 0,
-        // JOB_SCHEDULE_INDEX_MISSING_BLOCKS: '*/5 * * * *',
+        // JOB_SCHEDULE_INDEX_MISSING_BLOCKS: '* * * * *',
       },
     },
     // Configuration for other microservices.

docs/antora/modules/ROOT/pages/index.adoc

@@ -456,11 +456,15 @@ Lisk Testnet::
 // 	});
 // };
 
-// subscribe('update.block');
+// subscribe('new.block');
+// subscribe('new.transactions');
+// subscribe('delete.block');
+// subscribe('delete.transactions');
 // subscribe('update.round');
-// subscribe('update.forgers');
-// subscribe('update.transactions.confirmed');
+// subscribe('update.generators');
 // subscribe('update.fee_estimates');
+// subscribe('update.metadata');
+// subscribe('update.index.status');
 
 // To log all events
 // [

docs/antora/modules/ROOT/pages/setup/docker.adoc

@@ -57,9 +57,9 @@ Node.js::
 The following system requirements are recommended:
 
 Memory::
-* Machines with a minimum of 8 GB RAM for the Mainnet.
+* Machines with a minimum of 16 GB RAM for the Mainnet.
 
-* Machines with a minimum of 8 GB RAM for the Testnet.
+* Machines with a minimum of 16 GB RAM for the Testnet.
 
 Storage::
 * Machines with a minimum of 40 GB HDD.

docs/antora/modules/ROOT/pages/setup/source.adoc

@@ -64,9 +64,9 @@ Node.js::
 The following system requirements are recommended:
 
 Memory::
-* Machines with a minimum of 8 GB RAM for the Mainnet.
+* Machines with a minimum of 16 GB RAM for the Mainnet.
 
-* Machines with a minimum of 8 GB RAM for the Testnet.
+* Machines with a minimum of 16 GB RAM for the Testnet.
 
 Storage::
 * Machines with a minimum of 40 GB HDD.

docs/api/version3.md

@@ -758,7 +758,7 @@ _Supports pagination._
 | --------- | ---- | ---------- | ------- | ------- |
 | transactionID | String | `/^\b(?:[A-Fa-f0-9]){64}\b$/` | *(empty)* |  |
 | senderAddress | String | `/^lsk[a-hjkm-z2-9]{38}$/` | *(empty)* |  |
-| topic | String | `/^\b(?:(?:04\|05)?[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38})(?:,(?:(?:04\|05)?[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38}))*\b$/` | *(empty)* | Can be expressed as a CSV. |
+| topic | String | `/^\b(?:(?:04\|05)?[0-9a-fA-F]{64}\|lsk[a-hjkm-z2-9]{38})(?:,(?:(?:04\|05)?[0-9a-fA-F]{64}\|lsk[a-hjkm-z2-9]{38}))*\b$/` | *(empty)* | Can be expressed as a CSV. For performance reasons, we do not allow users to query default topics such as `03`. |
 | blockID | String | `/^\b(?:[A-Fa-f0-9]){64}\b$/` | *(empty)* |  |
 | height | String | `/^(?:(?:\d+)\|(?::(?:\d+))\|(?:(?:\d+):(?:\d+)?))$/` | *(empty)* | Query by height or a height range. Can be expressed as an interval i.e. `1:20` or `1:` or `:20`. Specified values are inclusive. |
 | timestamp | String | `/^(?:(?:\d+)\|(?::(?:\d+))\|(?:(?:\d+):(?:\d+)?))$/` | *(empty)* | Query by timestamp or a timestamp range. Can be expressed as an interval i.e. `1000000:2000000` or `1000000:` or `:2000000`. Specified values are inclusive. |

ecosystem.config.js

@@ -223,7 +223,7 @@ module.exports = {
 				// INDEX_MISSING_BLOCKS_SKIP_THRESHOLD: 1000,
 				// INDEX_MISSING_BLOCKS_MAX_SCHEDULE: 25000,
 				// JOB_INTERVAL_INDEX_MISSING_BLOCKS: 0,
-				// JOB_SCHEDULE_INDEX_MISSING_BLOCKS: '*/5 * * * *',
+				// JOB_SCHEDULE_INDEX_MISSING_BLOCKS: '* * * * *',
 			},
 		},
 		{

framework/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "lisk-service-framework",
-	"version": "1.6.9",
+	"version": "1.6.10",
 	"description": "Lisk Service Framework",
 	"keywords": [
 		"lisk",

framework/src/database/util.js

@@ -320,18 +320,34 @@ const getTableInstance = (tableConfig, knex) => {
 		}
 
 		if (params.leftOuterJoin) {
-			const { targetTable, leftColumn, rightColumn } = params.leftOuterJoin;
-			query.leftOuterJoin(targetTable, leftColumn, rightColumn);
+			params.leftOuterJoin = Array.isArray(params.leftOuterJoin)
+				? params.leftOuterJoin
+				: [params.leftOuterJoin];
+
+			params.leftOuterJoin.forEach(join => {
+				const { targetTable, leftColumn, rightColumn } = join;
+				query.leftOuterJoin(targetTable, leftColumn, rightColumn);
+			});
 		}
 
 		if (params.rightOuterJoin) {
-			const { targetTable, leftColumn, rightColumn } = params.rightOuterJoin;
-			query.rightOuterJoin(targetTable, leftColumn, rightColumn);
+			params.rightOuterJoin = Array.isArray(params.rightOuterJoin)
+				? params.rightOuterJoin
+				: [params.rightOuterJoin];
+
+			params.rightOuterJoin.forEach(join => {
+				const { targetTable, leftColumn, rightColumn } = join;
+				query.rightOuterJoin(targetTable, leftColumn, rightColumn);
+			});
 		}
 
 		if (params.innerJoin) {
-			const { targetTable, leftColumn, rightColumn } = params.innerJoin;
-			query.innerJoin(targetTable, leftColumn, rightColumn);
+			params.innerJoin = Array.isArray(params.innerJoin) ? params.innerJoin : [params.innerJoin];
+
+			params.innerJoin.forEach(join => {
+				const { targetTable, leftColumn, rightColumn } = join;
+				query.innerJoin(targetTable, leftColumn, rightColumn);
+			});
 		}
 
 		if (params.search) {

services/blockchain-app-registry/package.json

@@ -31,7 +31,7 @@
 	},
 	"dependencies": {
 		"bluebird": "^3.7.2",
-		"lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/68b71e08b61b49a40434a1cf1f1d5a9cdc9053a3/framework/dist/lisk-service-framework-1.6.9.tgz",
+		"lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/bea05ce084df5446738846964b4d6c30b5088b69/framework/dist/lisk-service-framework-1.6.10.tgz",
 		"lodash": "^4.17.21",
 		"octokit": "^2.0.4",
 		"tar": "^6.1.11"

services/blockchain-app-registry/yarn.lock

@@ -2920,9 +2920,39 @@ lines-and-columns@^1.1.6:
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
   integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
 
-"lisk-service-framework@https://github.com/LiskHQ/lisk-service/raw/68b71e08b61b49a40434a1cf1f1d5a9cdc9053a3/framework/dist/lisk-service-framework-1.6.9.tgz":
-  version "1.6.9"
-  resolved "https://github.com/LiskHQ/lisk-service/raw/68b71e08b61b49a40434a1cf1f1d5a9cdc9053a3/framework/dist/lisk-service-framework-1.6.9.tgz#47db5a32c2747052b4f697e8f2f2ec005842b6cb"
+"lisk-service-framework@https://github.com/LiskHQ/lisk-service/raw/21c265fab4cfde3c21b703453a55159b5d0e960a/framework/dist/lisk-service-framework-1.6.10.tgz":
+  version "1.6.10"
+  resolved "https://github.com/LiskHQ/lisk-service/raw/21c265fab4cfde3c21b703453a55159b5d0e960a/framework/dist/lisk-service-framework-1.6.10.tgz#7e1516ecc6c5c185a742396231cbac5a83482919"
+  dependencies:
+    "@keyv/redis" "^2.1.2"
+    "@log4js-node/gelf" "github:MichalTuleja/log4js-node-gelf#89d9933"
+    axios "^1.6.0"
+    better-sqlite3 "^8.5.0"
+    bull "^3.29.3"
+    debug "^4.3.1"
+    fastest-validator "^1.10.1"
+    http-status-codes "^1.4.0"
+    json-colorizer "^2.2.2"
+    keyv "^4.0.3"
+    keyv-lru "^3.0.4"
+    knex "^2.5.1"
+    log4js "^6.5.2"
+    moleculer "^0.14.21"
+    moleculer-web "^0.10.4"
+    moment "^2.29.4"
+    mysql2 "^3.5.2"
+    nats "^1.4.12"
+    node-cron "^2.0.3"
+    prettyjson "^1.2.1"
+    require-all "^3.0.0"
+    signals "^1.0.0"
+    socket.io "^4.4.1"
+    socket.io-client "^4.0.1"
+    stack-trace "0.0.10"
+
+"lisk-service-framework@https://github.com/LiskHQ/lisk-service/raw/bea05ce084df5446738846964b4d6c30b5088b69/framework/dist/lisk-service-framework-1.6.10.tgz":
+  version "1.6.10"
+  resolved "https://github.com/LiskHQ/lisk-service/raw/bea05ce084df5446738846964b4d6c30b5088b69/framework/dist/lisk-service-framework-1.6.10.tgz#97bf69581c35c6c3e9318ebb54853b890543f7a1"
   dependencies:
     "@keyv/redis" "^2.1.2"
     "@log4js-node/gelf" "github:MichalTuleja/log4js-node-gelf#89d9933"

services/blockchain-connector/package.json

@@ -39,7 +39,7 @@
 		"big-json": "^3.1.0",
 		"bluebird": "^3.7.2",
 		"knex": "^2.4.0",
-		"lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/68b71e08b61b49a40434a1cf1f1d5a9cdc9053a3/framework/dist/lisk-service-framework-1.6.9.tgz",
+		"lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/bea05ce084df5446738846964b4d6c30b5088b69/framework/dist/lisk-service-framework-1.6.10.tgz",
 		"moment": "^2.29.4",
 		"signals": "^1.0.0",
 		"tar": "^6.1.11"

services/blockchain-connector/yarn.lock

@@ -3889,9 +3889,39 @@ lines-and-columns@^1.1.6:
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
   integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
 
-"lisk-service-framework@https://github.com/LiskHQ/lisk-service/raw/68b71e08b61b49a40434a1cf1f1d5a9cdc9053a3/framework/dist/lisk-service-framework-1.6.9.tgz":
-  version "1.6.9"
-  resolved "https://github.com/LiskHQ/lisk-service/raw/68b71e08b61b49a40434a1cf1f1d5a9cdc9053a3/framework/dist/lisk-service-framework-1.6.9.tgz#47db5a32c2747052b4f697e8f2f2ec005842b6cb"
+"lisk-service-framework@https://github.com/LiskHQ/lisk-service/raw/21c265fab4cfde3c21b703453a55159b5d0e960a/framework/dist/lisk-service-framework-1.6.10.tgz":
+  version "1.6.10"
+  resolved "https://github.com/LiskHQ/lisk-service/raw/21c265fab4cfde3c21b703453a55159b5d0e960a/framework/dist/lisk-service-framework-1.6.10.tgz#7e1516ecc6c5c185a742396231cbac5a83482919"
+  dependencies:
+    "@keyv/redis" "^2.1.2"
+    "@log4js-node/gelf" "github:MichalTuleja/log4js-node-gelf#89d9933"
+    axios "^1.6.0"
+    better-sqlite3 "^8.5.0"
+    bull "^3.29.3"
+    debug "^4.3.1"
+    fastest-validator "^1.10.1"
+    http-status-codes "^1.4.0"
+    json-colorizer "^2.2.2"
+    keyv "^4.0.3"
+    keyv-lru "^3.0.4"
+    knex "^2.5.1"
+    log4js "^6.5.2"
+    moleculer "^0.14.21"
+    moleculer-web "^0.10.4"
+    moment "^2.29.4"
+    mysql2 "^3.5.2"
+    nats "^1.4.12"
+    node-cron "^2.0.3"
+    prettyjson "^1.2.1"
+    require-all "^3.0.0"
+    signals "^1.0.0"
+    socket.io "^4.4.1"
+    socket.io-client "^4.0.1"
+    stack-trace "0.0.10"
+
+"lisk-service-framework@https://github.com/LiskHQ/lisk-service/raw/bea05ce084df5446738846964b4d6c30b5088b69/framework/dist/lisk-service-framework-1.6.10.tgz":
+  version "1.6.10"
+  resolved "https://github.com/LiskHQ/lisk-service/raw/bea05ce084df5446738846964b4d6c30b5088b69/framework/dist/lisk-service-framework-1.6.10.tgz#97bf69581c35c6c3e9318ebb54853b890543f7a1"
   dependencies:
     "@keyv/redis" "^2.1.2"
     "@log4js-node/gelf" "github:MichalTuleja/log4js-node-gelf#89d9933"

services/blockchain-coordinator/README.md

@@ -29,7 +29,7 @@ A list of the most commonly used environment variables is presented below:
 - `SERVICE_BROKER`: URL of the microservice message broker (NATS or Redis).
 - `SERVICE_MESSAGE_QUEUE_REDIS`: URL of the job queue to schedule the indexing jobs (Redis).
 - `JOB_INTERVAL_INDEX_MISSING_BLOCKS`: Job run interval to index missing blocks. By default, it is set to 0.
-- `JOB_SCHEDULE_INDEX_MISSING_BLOCKS`: Job run cron schedule to index missing blocks. By default, it is set to run every 5 minutes (`*/5 * * * *`).
+- `JOB_SCHEDULE_INDEX_MISSING_BLOCKS`: Job run cron schedule to index missing blocks. By default, it is set to run every minute (`* * * * *`).
 
 > **Note**: `interval` takes priority over `schedule` and must be greater than 0 to be valid for all the moleculer job configurations.
 

services/blockchain-coordinator/config.js

@@ -61,6 +61,11 @@ config.queue = {
 		attempts: 5,
 		timeout: 5 * 60 * 1000, // millisecs
 		removeOnComplete: true,
+		removeOnFail: {
+			age: 1 * 60 * 60, // 1 hr in secs
+			count: 100000,
+		},
+		stackTraceLimit: 0,
 	},
 
 	// Inter-microservice message queues
@@ -73,7 +78,7 @@ config.job = {
 	// Interval takes priority over schedule and must be greater than 0 to be valid
 	indexMissingBlocks: {
 		interval: Number(process.env.JOB_INTERVAL_INDEX_MISSING_BLOCKS) || 0,
-		schedule: process.env.JOB_SCHEDULE_INDEX_MISSING_BLOCKS || '*/5 * * * *',
+		schedule: process.env.JOB_SCHEDULE_INDEX_MISSING_BLOCKS || '* * * * *',
 		skipThreshold: Number(process.env.INDEX_MISSING_BLOCKS_SKIP_THRESHOLD) || 1000,
 		maxBlocksToSchedule: Number(process.env.INDEX_MISSING_BLOCKS_MAX_SCHEDULE) || 25000,
 	},

services/blockchain-coordinator/package.json

@@ -31,7 +31,7 @@
 	"dependencies": {
 		"bluebird": "^3.7.2",
 		"bull": "^4.8.1",
-		"lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/68b71e08b61b49a40434a1cf1f1d5a9cdc9053a3/framework/dist/lisk-service-framework-1.6.9.tgz"
+		"lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/bea05ce084df5446738846964b4d6c30b5088b69/framework/dist/lisk-service-framework-1.6.10.tgz"
 	},
 	"devDependencies": {
 		"@babel/preset-env": "^7.14.0",

services/blockchain-coordinator/shared/scheduler.js

@@ -22,6 +22,7 @@ const logger = Logger();
 
 const { initEventsScheduler } = require('./eventsScheduler');
 const {
+	getIndexStatus,
 	getMissingBlocks,
 	getIndexVerifiedHeight,
 	isGenesisBlockIndexed,
@@ -32,6 +33,7 @@ const { getAllPosValidators } = require('./sources/connector');
 
 const { getCurrentHeight, getGenesisHeight, initNodeConstants } = require('./constants');
 
+const { range } = require('./utils/array');
 const delay = require('./utils/delay');
 const config = require('../config');
 
@@ -236,7 +238,7 @@ const scheduleMissingBlocksIndexing = async () => {
 	const jobCount = await getLiveIndexingJobCount();
 	if (jobCount > config.job.indexMissingBlocks.skipThreshold) {
 		logger.info(
-			`Skipping missing blocks job run. ${jobCount} indexing jobs (current threshold: ${config.job.indexMissingBlocks.skipThreshold}) already in the queue.`,
+			`Skipping missing blocks job run. ${jobCount} indexing jobs already in the queue. Current threshold set at: ${config.job.indexMissingBlocks.skipThreshold}.`,
 		);
 		return;
 	}
@@ -279,6 +281,18 @@ const scheduleMissingBlocksIndexing = async () => {
 			}
 		}
 
+		// Re-check for tiny gaps and schedule jobs accordingly
+		const indexStatus = await getIndexStatus();
+		if (indexStatus) {
+			const { chainLength, numBlocksIndexed, lastBlockHeight } = indexStatus.data;
+			const numStillMissingJobs = chainLength - numBlocksIndexed - missingBlocksByHeight.length;
+			if (numStillMissingJobs > 0 && numStillMissingJobs <= 10) {
+				missingBlocksByHeight.push(
+					...range(lastBlockHeight - numStillMissingJobs + 1, lastBlockHeight + 1),
+				);
+			}
+		}
+
 		if (missingBlocksByHeight.length === 0) {
 			logger.info(
 				`No missing blocks found in range ${blockIndexLowerRange} - ${blockIndexHigherRange}. Setting index verified height to ${blockIndexHigherRange}.`,

services/blockchain-coordinator/shared/sources/indexer.js

@@ -25,6 +25,8 @@ const isGenesisBlockIndexed = async () => {
 	return isGenesisBlockIndexedFlag;
 };
 
+const getIndexStatus = async () => requestIndexer('index.status').catch(() => null);
+
 const getMissingBlocks = async (from, to) => requestIndexer('getMissingBlocks', { from, to });
 
 const getIndexVerifiedHeight = async () =>
@@ -38,6 +40,7 @@ const getLiveIndexingJobCount = async () =>
 
 module.exports = {
 	isGenesisBlockIndexed,
+	getIndexStatus,
 	getMissingBlocks,
 	getIndexVerifiedHeight,
 	getLiveIndexingJobCount,

services/blockchain-coordinator/shared/utils/array.js

@@ -0,0 +1,29 @@
+/*
+ * LiskHQ/lisk-service
+ * Copyright © 2023 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ *
+ */
+const range = (start = 0, end, step = 1) => {
+	if (!end) {
+		end = start;
+		start = 0;
+	}
+	const arrSize = Math.floor((end - start) / step);
+
+	// 'end' is non-inclusive
+	return new Array(arrSize).fill().map((_, index) => start + index * step);
+};
+
+module.exports = {
+	range,
+};