From 51801c402eaf1242c26e04ac94d7dad6e4f06e38 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Fri, 5 Jan 2024 20:13:50 +0300 Subject: [PATCH 01/17] test set max_allowed_packet --- lib/sqlAnalyzer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index c244d20..f504f50 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -59,6 +59,7 @@ function cleanUP(sql) { // eslint-disable-next-line no-param-reassign while (sql.startsWith('--') || sql.startsWith('\r\n') || sql.startsWith('\n')) sql = removeFirstLine(sql); } + if (sql.startsWith('SET max_allowed_packet')) sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); return sql; } /** From 14add5560d7c779045c8376d4bba948449fe762f Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Fri, 5 Jan 2024 20:27:40 +0300 Subject: [PATCH 02/17] chore: testing --- lib/sqlAnalyzer.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index f504f50..c6d4082 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -59,7 +59,8 @@ function cleanUP(sql) { // eslint-disable-next-line no-param-reassign while (sql.startsWith('--') || sql.startsWith('\r\n') || sql.startsWith('\n')) sql = removeFirstLine(sql); } - if (sql.startsWith('SET max_allowed_packet')) sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); + // eslint-disable-next-line no-param-reassign + if (sql.startsWith('SET max_allowed_packet')) sql = sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); return sql; } /** From fc5fc5b15546d7306205b4388ba9c5f97a6ce313 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Fri, 5 Jan 2024 20:44:22 +0300 Subject: [PATCH 03/17] chore: test --- lib/sqlAnalyzer.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index c6d4082..3225823 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -54,6 +54,9 @@ function removeFirstLine(str) { * @param {string} sql sql query */ function cleanUP(sql) { + if (sql.startsWith('SET max_allowed_packet')) { + return sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); + } if (sql.startsWith('/*') && sql.endsWith('*/') && !sql.startsWith('/*!')) return false; if (sql.startsWith('--')) { // eslint-disable-next-line no-param-reassign @@ -61,6 +64,7 @@ function cleanUP(sql) { } // eslint-disable-next-line no-param-reassign if (sql.startsWith('SET max_allowed_packet')) sql = sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); + return sql; } /** From f97ff69a4b2710ebd649b8bcf2e4f293c30c7df6 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 15:22:33 +0300 Subject: [PATCH 04/17] test --- lib/sqlAnalyzer.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index 3225823..20e9b0b 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -62,9 +62,6 @@ function cleanUP(sql) { // eslint-disable-next-line no-param-reassign while (sql.startsWith('--') || sql.startsWith('\r\n') || sql.startsWith('\n')) sql = removeFirstLine(sql); } - // eslint-disable-next-line no-param-reassign - if (sql.startsWith('SET max_allowed_packet')) sql = sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); - return sql; } /** @@ -86,7 +83,7 @@ function analyzeSql(sql, options) { const output = querySplitter.splitQuery(sql, sqlOptions); const analyzedArray = []; for (let i = 0; i < output.length; i += 1) { - const tempSql = cleanUP(output[i]); + const tempSql = cleanUP(output[i].trim()); if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); } return analyzedArray; From 22f63add12499dec1efb15a3e259890e467904c2 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 15:39:41 +0300 Subject: [PATCH 05/17] extra logs --- lib/log.js | 2 +- lib/sqlAnalyzer.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/log.js b/lib/log.js index e121e30..746cc4e 100644 --- a/lib/log.js +++ b/lib/log.js @@ -53,6 +53,6 @@ module.exports = { writeToFile(args, 'debug.txt'); }, query(...args) { - writeToFile(args, `query_${args[2]}.txt`); + writeToFile(args, 'query.txt'); }, }; diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index 20e9b0b..870ebe6 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -86,6 +86,7 @@ function analyzeSql(sql, options) { const tempSql = cleanUP(output[i].trim()); if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); } + log.query(JSON.stringify(analyzedArray)); return analyzedArray; } From 6b6aa7529ec62d5fa2de718a7284860b34af1b1f Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 15:48:07 +0300 Subject: [PATCH 06/17] fix query logs --- ClusterOperator/server.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index bfe1ca7..a5359cf 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -142,6 +142,9 @@ function startUI() { case 'debug': logFile = 'debug.txt'; break; + case 'query': + logFile = 'query.txt'; + break; default: logFile = 'errors.txt'; break; From 76ef99b77b38fde3c0b72d8cb514daedfd1f1f93 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 15:50:38 +0300 Subject: [PATCH 07/17] minor change --- ClusterOperator/server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index a5359cf..4117141 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -149,6 +149,7 @@ function startUI() { logFile = 'errors.txt'; break; } + if (whiteList.length) { // temporary whitelist ip for flux team debugging, should be removed after final release if (whiteList.includes(remoteIp) || remoteIp === '167.235.234.45' || remoteIp === '45.89.52.198') { From 4b168e5aa86aeb2eba47c96a407e0485bad0a83b Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 16:01:07 +0300 Subject: [PATCH 08/17] fix logs --- ClusterOperator/server.js | 3 ++- lib/sqlAnalyzer.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 4117141..d955ea7 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -95,6 +95,7 @@ function startUI() { fs.writeFileSync('errors.txt', `version: ${config.version}
`); fs.writeFileSync('warnings.txt', `version: ${config.version}
`); fs.writeFileSync('info.txt', `version: ${config.version}
`); + fs.writeFileSync('query.txt', `version: ${config.version}
`); fs.appendFileSync('debug.txt', `------------------------------------------------------
version: ${config.version}
`); app.options('/*', (req, res, next) => { @@ -149,7 +150,7 @@ function startUI() { logFile = 'errors.txt'; break; } - + if (whiteList.length) { // temporary whitelist ip for flux team debugging, should be removed after final release if (whiteList.includes(remoteIp) || remoteIp === '167.235.234.45' || remoteIp === '45.89.52.198') { diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index 870ebe6..d2e7a9d 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -87,6 +87,7 @@ function analyzeSql(sql, options) { if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); } log.query(JSON.stringify(analyzedArray)); + log.info(JSON.stringify(analyzedArray), 'yellow'); return analyzedArray; } From 423b8e13a5e621510c9b1ad3d987b400eea2780a Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 16:27:57 +0300 Subject: [PATCH 09/17] adjust logs --- lib/log.js | 2 +- lib/sqlAnalyzer.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/log.js b/lib/log.js index 746cc4e..f104d54 100644 --- a/lib/log.js +++ b/lib/log.js @@ -19,7 +19,7 @@ function getFilesizeInBytes(filename) { function writeToFile(args, file) { const size = getFilesizeInBytes(file); let flag = 'a+'; - if (size > (12 * 1024 * 1024)) { // 12MB + if (size > (10 * 1024 * 1024)) { // 10MB flag = 'w'; // rewrite file } const stream = fs.createWriteStream(file, { flags: flag }); diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index d2e7a9d..870ebe6 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -87,7 +87,6 @@ function analyzeSql(sql, options) { if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); } log.query(JSON.stringify(analyzedArray)); - log.info(JSON.stringify(analyzedArray), 'yellow'); return analyzedArray; } From c2b42e58ef7b2be03d1403ff2672dc50a08c0d67 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 16:43:24 +0300 Subject: [PATCH 10/17] adjust logs --- lib/sqlAnalyzer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index 870ebe6..3bdceec 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -86,6 +86,7 @@ function analyzeSql(sql, options) { const tempSql = cleanUP(output[i].trim()); if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); } + sql.query(sql, 'yellow'); log.query(JSON.stringify(analyzedArray)); return analyzedArray; } From 94f32dae4b064ef0f0e7ca10ab4ec824e88cb30b Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 16:43:40 +0300 Subject: [PATCH 11/17] minor fix --- lib/sqlAnalyzer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index 3bdceec..ec74801 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -86,7 +86,7 @@ function analyzeSql(sql, options) { const tempSql = cleanUP(output[i].trim()); if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); } - sql.query(sql, 'yellow'); + log.query(sql, 'yellow'); log.query(JSON.stringify(analyzedArray)); return analyzedArray; } From ad80c1f1ad26d32afbe83662ad5dbde01c01c9d7 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 16:52:17 +0300 Subject: [PATCH 12/17] possible fix for commented queries --- lib/sqlAnalyzer.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index ec74801..bcf85fb 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -54,10 +54,10 @@ function removeFirstLine(str) { * @param {string} sql sql query */ function cleanUP(sql) { - if (sql.startsWith('SET max_allowed_packet')) { - return sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); - } - if (sql.startsWith('/*') && sql.endsWith('*/') && !sql.startsWith('/*!')) return false; + // if (sql.startsWith('SET max_allowed_packet')) { + // return sql.replace('SET max_allowed_packet', 'SET GLOBAL max_allowed_packet'); + // } + // if (sql.startsWith('/*') && sql.endsWith('*/') && !sql.startsWith('/*!')) return false; if (sql.startsWith('--')) { // eslint-disable-next-line no-param-reassign while (sql.startsWith('--') || sql.startsWith('\r\n') || sql.startsWith('\n')) sql = removeFirstLine(sql); From 8ce1f2caedffa6df24d7a4ff93f1f36aab742e73 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 17:27:54 +0300 Subject: [PATCH 13/17] disable querySplitter --- lib/sqlAnalyzer.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index bcf85fb..3277100 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -80,12 +80,12 @@ function analyzeSql(sql, options) { } else if (options === 'postgre') { sqlOptions = querySplitter.postgreSplitterOptions; } else return []; - const output = querySplitter.splitQuery(sql, sqlOptions); + // const output = querySplitter.splitQuery(sql, sqlOptions); const analyzedArray = []; - for (let i = 0; i < output.length; i += 1) { - const tempSql = cleanUP(output[i].trim()); - if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); - } + // for (let i = 0; i < output.length; i += 1) { + const tempSql = cleanUP(sql.trim()); + if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); + // } log.query(sql, 'yellow'); log.query(JSON.stringify(analyzedArray)); return analyzedArray; From 399d95616b054ef280d2b5181948b3b37c6e7624 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 18:08:25 +0300 Subject: [PATCH 14/17] remove dbgate-query-splitter library --- lib/sqlAnalyzer.js | 28 +++------------------------- package-lock.json | 11 ----------- package.json | 1 - 3 files changed, 3 insertions(+), 37 deletions(-) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index 3277100..9f5a526 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -3,10 +3,8 @@ /** * [This Library helps analyze sql queries and seperate read/write queries for mysql, mssql and postgre] */ -const querySplitter = require('dbgate-query-splitter'); const log = require('./log'); -let prevQuery = ''; /** * [getQueryType analyzes a single query and returns r or w for query type] * @param {string} sql sql query @@ -22,24 +20,9 @@ function getQueryType(sql) { // log.warn('wp transient query'); return 'r'; } - /* - if (sql.includes('`wp_options`') && sql.includes("`option_name` = 'cron'")) { - log.warn('wp cron query'); - return 'r'; - } - */ if (sql.toLowerCase().startsWith('set session')) { return 's'; } - /* - if (sql === prevQuery && (sql.includes('`wp_options`'))) { - log.warn(`duplicate write query ${sql}`); - prevQuery = sql; - return 'r'; - } - */ - prevQuery = sql; - // end fix for wordpress fuckery // it's a write query return 'w'; @@ -71,21 +54,16 @@ function cleanUP(sql) { * @return {array} [array of queries] */ function analyzeSql(sql, options) { - let sqlOptions = null; if (options === 'mysql') { - sqlOptions = querySplitter.mysqlSplitterOptions; - sqlOptions.multilineComments = true; + // } else if (options === 'mssql') { - sqlOptions = querySplitter.mssqlSplitterOptions; + // } else if (options === 'postgre') { - sqlOptions = querySplitter.postgreSplitterOptions; + // } else return []; - // const output = querySplitter.splitQuery(sql, sqlOptions); const analyzedArray = []; - // for (let i = 0; i < output.length; i += 1) { const tempSql = cleanUP(sql.trim()); if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); - // } log.query(sql, 'yellow'); log.query(JSON.stringify(analyzedArray)); return analyzedArray; diff --git a/package-lock.json b/package-lock.json index 8f81b83..5b91e14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "body-parser": "^1.20.2", "cookie-parser": "^1.4.6", "cors": "^2.8.5", - "dbgate-query-splitter": "^4.9.0", "express": "^4.18.2", "express-fileupload": "^1.4.1", "express-rate-limit": "^7.1.1", @@ -1368,11 +1367,6 @@ "node": "*" } }, - "node_modules/dbgate-query-splitter": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/dbgate-query-splitter/-/dbgate-query-splitter-4.9.0.tgz", - "integrity": "sha512-POifNiMDkeksA9YXaC82u5O6krYC21xyROoNjDh3ouKI4xeB37DG+cP/D4IdICWHYZudlgKiziQ4v3W+5+O1DA==" - }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -5616,11 +5610,6 @@ "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==" }, - "dbgate-query-splitter": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/dbgate-query-splitter/-/dbgate-query-splitter-4.9.0.tgz", - "integrity": "sha512-POifNiMDkeksA9YXaC82u5O6krYC21xyROoNjDh3ouKI4xeB37DG+cP/D4IdICWHYZudlgKiziQ4v3W+5+O1DA==" - }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", diff --git a/package.json b/package.json index 064d569..0ccde1c 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,6 @@ "body-parser": "^1.20.2", "cookie-parser": "^1.4.6", "cors": "^2.8.5", - "dbgate-query-splitter": "^4.9.0", "express": "^4.18.2", "express-fileupload": "^1.4.1", "express-rate-limit": "^7.1.1", From 2cd68d609584d2b1e50d2d2733f14c1d0fef2f0f Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 9 Jan 2024 19:54:32 +0300 Subject: [PATCH 15/17] remove query logs --- lib/sqlAnalyzer.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/sqlAnalyzer.js b/lib/sqlAnalyzer.js index 9f5a526..d0c3a89 100644 --- a/lib/sqlAnalyzer.js +++ b/lib/sqlAnalyzer.js @@ -64,8 +64,7 @@ function analyzeSql(sql, options) { const analyzedArray = []; const tempSql = cleanUP(sql.trim()); if (tempSql) analyzedArray.push([tempSql, getQueryType(tempSql)]); - log.query(sql, 'yellow'); - log.query(JSON.stringify(analyzedArray)); + // log.query(sql, 'yellow'); return analyzedArray; } From bb963dd67dd4eced0f1c55050794e3bd3eaea7f6 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Mon, 22 Jan 2024 12:49:44 +0300 Subject: [PATCH 16/17] testing --- ClusterOperator/Operator.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ClusterOperator/Operator.js b/ClusterOperator/Operator.js index 7a2048e..76e3638 100644 --- a/ClusterOperator/Operator.js +++ b/ClusterOperator/Operator.js @@ -594,7 +594,7 @@ class Operator { log.info('previous master is in the node list. continue...'); } else { log.info('previous master is NOT in the node list.'); - while (ipList.length < this.nodeInstances) { + while (ipList.length < this.nodeInstances / 2) { log.info(`Waiting for all nodes to spawn ${ipList.length}/${this.nodeInstances}...`); await timer.setTimeout(10000); ipList = await fluxAPI.getApplicationIP(config.DBAppName); @@ -602,7 +602,7 @@ class Operator { } } else { log.info('no master node defined before.'); - while (ipList.length < this.nodeInstances) { + while (ipList.length < this.nodeInstances / 2) { log.info(`Waiting for all nodes to spawn ${ipList.length}/${this.nodeInstances}...`); await timer.setTimeout(10000); ipList = await fluxAPI.getApplicationIP(config.DBAppName); From f31e79b6886979da566a03c3cced4bf01f49feac Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Mon, 29 Jan 2024 11:59:45 +0300 Subject: [PATCH 17/17] bump version --- ClusterOperator/config.js | 2 +- ui/index.html | 2 +- ui/login.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ClusterOperator/config.js b/ClusterOperator/config.js index 569df7b..cc47a93 100644 --- a/ClusterOperator/config.js +++ b/ClusterOperator/config.js @@ -16,7 +16,7 @@ module.exports = { containerApiPort: String(process.env.API_PORT || 33950).trim(), DBAppName: process.env.DB_APPNAME || 'wordpressonflux', AppName: process.env.CLIENT_APPNAME || 'explorer', - version: '1.2.2', + version: '1.2.3', whiteListedIps: process.env.WHITELIST || '127.0.0.1', debugMode: true, authMasterOnly: process.env.AUTH_MASTER_ONLY || false, diff --git a/ui/index.html b/ui/index.html index d8aa672..36a8375 100644 --- a/ui/index.html +++ b/ui/index.html @@ -276,7 +276,7 @@
Active Nodes:
© 2022-2023 InFlux Technologies Beta - v1.2.2 + v1.2.3
diff --git a/ui/login.html b/ui/login.html index 737e361..61c5cc8 100644 --- a/ui/login.html +++ b/ui/login.html @@ -87,7 +87,7 @@

Flux Decentralized Database

© 2022-2023 InFlux Technologies Beta - v1.2.2 + v1.2.3