Merge branch 'release/0.7.0' into 1840-update-lisk-sdk-to-v6-0-0-rc-0

docker/example.env

@@ -42,8 +42,6 @@
 
 # Lisk ecosystem configuration
 LISK_APP_WS=ws://host.docker.internal:7887
-# USE_LISK_IPC_CLIENT=false
-# LISK_APP_DATA_PATH='~/.lisk/lisk-core'
 
 # Lisk Service geolocation backend (empty = disabled)
 GEOIP_JSON='https://geoip.lisk.com/json'

framework/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lisk-service-framework",
-  "version": "1.4.22",
+  "version": "1.4.23",
   "description": "Lisk Service Framework",
   "keywords": [
     "lisk",
@@ -47,7 +47,7 @@
     "moleculer": "^0.14.21",
     "moleculer-web": "^0.10.4",
     "moment": "^2.29.4",
-    "mysql2": "^2.3.3",
+    "mysql2": "^3.5.2",
     "nats": "^1.4.12",
     "node-cron": "^2.0.3",
     "prettyjson": "^1.2.1",

framework/yarn.lock

@@ -1295,7 +1295,7 @@ denque@^1.1.0:
   resolved "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf"
   integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
 
-denque@^2.0.1, denque@^2.1.0:
+denque@^2.1.0:
   version "2.1.0"
   resolved "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
   integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==
@@ -2840,10 +2840,10 @@ log4js@^6.5.2:
     rfdc "^1.3.0"
     streamroller "^3.1.5"
 
-long@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.npmjs.org/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
-  integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==
+long@^5.2.1:
+  version "5.2.3"
+  resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1"
+  integrity sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==
 
 lru-cache@^5.1.1:
   version "5.1.1"
@@ -2864,6 +2864,11 @@ lru-cache@^7.14.1:
   resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89"
   integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==
 
+lru-cache@^8.0.0:
+  version "8.0.5"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-8.0.5.tgz#983fe337f3e176667f8e567cfcce7cb064ea214e"
+  integrity sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==
+
 make-dir@^4.0.0:
   version "4.0.0"
   resolved "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e"
@@ -3005,23 +3010,23 @@ ms@2.1.3, ms@^2.1.1:
   resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
-mysql2@^2.3.3:
-  version "2.3.3"
-  resolved "https://registry.npmjs.org/mysql2/-/mysql2-2.3.3.tgz#944f3deca4b16629052ff8614fbf89d5552545a0"
-  integrity sha512-wxJUev6LgMSgACDkb/InIFxDprRa6T95+VEoR+xPvtngtccNH2dGjEB/fVZ8yg1gWv1510c9CvXuJHi5zUm0ZA==
+mysql2@^3.5.2:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.6.0.tgz#d54ef4867655daf2f65ced1dbdb63852314f039f"
+  integrity sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==
   dependencies:
-    denque "^2.0.1"
+    denque "^2.1.0"
     generate-function "^2.3.1"
     iconv-lite "^0.6.3"
-    long "^4.0.0"
-    lru-cache "^6.0.0"
-    named-placeholders "^1.1.2"
+    long "^5.2.1"
+    lru-cache "^8.0.0"
+    named-placeholders "^1.1.3"
     seq-queue "^0.0.5"
     sqlstring "^2.3.2"
 
-named-placeholders@^1.1.2:
+named-placeholders@^1.1.3:
   version "1.1.3"
-  resolved "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz#df595799a36654da55dda6152ba7a137ad1d9351"
+  resolved "https://registry.yarnpkg.com/named-placeholders/-/named-placeholders-1.1.3.tgz#df595799a36654da55dda6152ba7a137ad1d9351"
   integrity sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==
   dependencies:
     lru-cache "^7.14.1"

services/blockchain-app-registry/package.json

@@ -33,7 +33,7 @@
   },
   "dependencies": {
     "bluebird": "^3.7.2",
-    "lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/d63fb9be0e01a28ea8dc5f87a2d1b70a61650dda/framework/dist/lisk-service-framework-1.4.22.tgz",
+    "lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/b3646b1215a5abcca3e75f6b36074438ae42b238/framework/dist/lisk-service-framework-1.4.23.tgz",
     "lodash": "^4.17.21",
     "octokit": "^2.0.4",
     "tar": "^6.1.11"

services/blockchain-app-registry/shared/database/init.js

@@ -52,6 +52,7 @@ const initializeSearchIndex = async () => {
 };
 
 const truncateAllTables = async () => {
+	// Truncate is a DDL statement, so NO explicit DB trx required
 	logger.info('Truncating all the tables.');
 	await BluebirdPromise.map(
 		Object.values(tableSchemas),

services/blockchain-app-registry/shared/init.js

@@ -13,20 +13,40 @@
  * Removal or modification of this copyright notice is prohibited.
  *
  */
-const logger = require('lisk-service-framework').Logger();
+const {
+	Logger,
+	DB: {
+		MySQL: {
+			getDBConnection,
+			startDBTransaction,
+			commitDBTransaction,
+			rollbackDBTransaction,
+		},
+	},
+} = require('lisk-service-framework');
 
+const logger = Logger();
 const { indexAllBlockchainAppsMeta } = require('./metadataIndex');
 const { downloadRepositoryToFS } = require('./utils/downloadRepository');
 
+const config = require('../config');
+
+const MYSQL_ENDPOINT = config.endpoints.mysql;
+
 const init = async () => {
+	const dbConnection = await getDBConnection(MYSQL_ENDPOINT);
+	const dbTrx = await startDBTransaction(dbConnection);
+
 	try {
-		await downloadRepositoryToFS();
-		await indexAllBlockchainAppsMeta();
+		await downloadRepositoryToFS(dbTrx);
+		await indexAllBlockchainAppsMeta(dbTrx);
+		await commitDBTransaction(dbTrx);
 	} catch (error) {
+		await rollbackDBTransaction(dbTrx);
 		const errorMsg = Array.isArray(error)
 			? error.map(e => e.message).join('\n')
 			: error.message;
-		logger.error(`Unable to initialize metadata information due to: ${errorMsg}`);
+		logger.error(`Unable to initialize metadata information due to: ${errorMsg}.`);
 	}
 };
 

services/blockchain-app-registry/shared/metadataIndex.js

@@ -24,10 +24,6 @@ const {
 	DB: {
 		MySQL: {
 			getTableInstance,
-			getDBConnection,
-			startDBTransaction,
-			commitDBTransaction,
-			rollbackDBTransaction,
 		},
 	},
 } = require('lisk-service-framework');
@@ -182,7 +178,7 @@ const deleteIndexedMetadataFromFile = async (filePath, dbTrx) => {
 	logger.info(`Finished deleting metadata information for the app: ${app} (${network}) filename: ${filename}.`);
 };
 
-const indexAllBlockchainAppsMeta = async () => {
+const indexAllBlockchainAppsMeta = async (dbTrx) => {
 	const dataDirectory = config.dataDir;
 	const repo = config.gitHub.appRegistryRepoName;
 	const repoPath = path.join(dataDirectory, repo);
@@ -205,18 +201,7 @@ const indexAllBlockchainAppsMeta = async () => {
 							const filename = file.split('/').pop();
 							// Only process the known config files
 							if (KNOWN_CONFIG_FILES.includes(filename)) {
-								const connection = await getDBConnection(MYSQL_ENDPOINT);
-								const dbTrx = await startDBTransaction(connection);
-
-								try {
-									logger.debug('Created new MySQL transaction to index blockchain metadata information.');
-									await indexMetadataFromFile(file, dbTrx);
-									await commitDBTransaction(dbTrx);
-									logger.debug('Committed MySQL transaction to index blockchain metadata information.');
-								} catch (error) {
-									await rollbackDBTransaction(dbTrx);
-									logger.debug(`Rolled back MySQL transaction to index blockchain metadata information.\nError: ${error}`);
-								}
+								await indexMetadataFromFile(file, dbTrx);
 							}
 						},
 						{ concurrency: allFilesFromApp.length },

services/blockchain-app-registry/shared/utils/downloadRepository.js

@@ -72,9 +72,8 @@ const GITHUB_FILE_STATUS = Object.freeze({
 	UNCHANGED: 'unchanged',
 });
 
-const getRepoInfoFromURL = (url) => {
-	const urlInput = url || '';
-	const [, , , owner, repo] = urlInput.split('/');
+const getRepoInfoFromURL = (url = '') => {
+	const [, , , owner, repo] = url.split('/');
 	return { owner, repo };
 };
 
@@ -100,8 +99,11 @@ const getLatestCommitHash = async () => {
 	}
 };
 
-const getCommitInfo = async () => {
-	const lastSyncedCommitHash = await keyValueTable.get(KV_STORE_KEY.COMMIT_HASH_UNTIL_LAST_SYNC);
+const getCommitInfo = async (dbTrx = null) => {
+	const lastSyncedCommitHash = await keyValueTable.get(
+		KV_STORE_KEY.COMMIT_HASH_UNTIL_LAST_SYNC,
+		dbTrx,
+	);
 	const latestCommitHash = await getLatestCommitHash();
 	return { lastSyncedCommitHash, latestCommitHash };
 };
@@ -262,18 +264,24 @@ const getModifiedFileNames = (groupedFiles) => {
 	return fileNames;
 };
 
-const syncWithRemoteRepo = async () => {
-	const connection = await getDBConnection(MYSQL_ENDPOINT);
-	const dbTrx = await startDBTransaction(connection);
+const syncWithRemoteRepo = async (_dbTrx = null) => {
+	let isCustomDBTrx = false;
+	const dbTrx = _dbTrx || await (async () => {
+		const connection = await getDBConnection(MYSQL_ENDPOINT);
+		const newDBTrx = await startDBTransaction(connection);
+		isCustomDBTrx = true;
+		return newDBTrx;
+	})();
 
 	try {
 		const dataDirectory = config.dataDir;
 		const appDirPath = path.join(dataDirectory, repo);
-		const { lastSyncedCommitHash, latestCommitHash } = await getCommitInfo();
+		const { lastSyncedCommitHash, latestCommitHash } = await getCommitInfo(dbTrx);
 
 		// Skip if there is no new commit
 		if (lastSyncedCommitHash === latestCommitHash) {
 			logger.info('Database is already up-to-date.');
+			if (isCustomDBTrx) await rollbackDBTransaction(dbTrx);
 			return;
 		}
 
@@ -356,7 +364,7 @@ const syncWithRemoteRepo = async () => {
 		);
 
 		await keyValueTable.set(KV_STORE_KEY.COMMIT_HASH_UNTIL_LAST_SYNC, latestCommitHash, dbTrx);
-		await commitDBTransaction(dbTrx);
+		if (isCustomDBTrx) await commitDBTransaction(dbTrx);
 
 		// Delete files which are removed from remote
 		await BluebirdPromise.map(
@@ -385,22 +393,26 @@ const syncWithRemoteRepo = async () => {
 			Signals.get('metadataUpdated').dispatch(eventPayload);
 		}
 	} catch (error) {
-		await rollbackDBTransaction(dbTrx);
+		if (isCustomDBTrx) await rollbackDBTransaction(dbTrx);
 		let errorMsg = error.message;
 		if (Array.isArray(error)) errorMsg = error.map(e => e.message).join('\n');
 		logger.error(`Unable to sync changes due to: ${errorMsg}.`);
 		throw error;
 	}
 };
 
-const downloadRepositoryToFS = async () => {
+const downloadRepositoryToFS = async (dbTrx) => {
 	const dataDirectory = config.dataDir;
 	const appDirPath = path.join(dataDirectory, repo);
-	const lastSyncedCommitHash = await keyValueTable.get(KV_STORE_KEY.COMMIT_HASH_UNTIL_LAST_SYNC);
+
+	const lastSyncedCommitHash = await keyValueTable.get(
+		KV_STORE_KEY.COMMIT_HASH_UNTIL_LAST_SYNC,
+		dbTrx,
+	);
 
 	if (lastSyncedCommitHash && await exists(appDirPath)) {
 		logger.trace('Synchronizing with the remote repository.');
-		await syncWithRemoteRepo();
+		await syncWithRemoteRepo(dbTrx);
 		logger.info('Finished synchronizing with the remote repository successfully.');
 	} else {
 		if (!(await exists(dataDirectory))) {
@@ -419,7 +431,7 @@ const downloadRepositoryToFS = async () => {
 		await mv(oldDir, appDirPath);
 
 		const latestCommitHash = await getLatestCommitHash();
-		await keyValueTable.set(KV_STORE_KEY.COMMIT_HASH_UNTIL_LAST_SYNC, latestCommitHash);
+		await keyValueTable.set(KV_STORE_KEY.COMMIT_HASH_UNTIL_LAST_SYNC, latestCommitHash, dbTrx);
 	}
 };
 
@@ -440,3 +452,5 @@ module.exports = {
 	buildEventPayload,
 	getModifiedFileNames,
 };
+
+// deleteIndexedMetadataFromFile

services/blockchain-app-registry/tests/unit/shared/metadataIndex.test.js

@@ -322,7 +322,9 @@ describe('Test indexAllBlockchainAppsMeta method', () => {
 				DB: {
 					...actualLiskServiceFramework.DB,
 					MySQL: {
-						getTableInstance: jest.fn(),
+						getTableInstance: () => ({
+							upsert: jest.fn(),
+						}),
 						getDBConnection: jest.fn(),
 						startDBTransaction: jest.fn(),
 						commitDBTransaction: jest.fn(),

services/blockchain-app-registry/tests/unit/shared/utils/downloadRepository.test.js

@@ -85,12 +85,8 @@ describe('Test getRepoInfoFromURL method', () => {
 		});
 	});
 
-	it('should return proper response when url is null', async () => {
-		const response = getRepoInfoFromURL(null);
-		expect(response).toMatchObject({
-			owner: undefined,
-			repo: undefined,
-		});
+	it('should throw error when url is null', async () => {
+		expect(async () => getRepoInfoFromURL(null)).rejects.toThrow();
 	});
 });