From 67613a3bb5e69dc06304a6a5441b997e52e7b5f1 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 17 Oct 2023 13:21:04 +0300 Subject: [PATCH] fix(accountCounters): added counters object that contains cumulative counter of all account specific triggered events --- lib/account.js | 27 +++++++++++++++++++++++++-- lib/connection.js | 13 +++++++------ lib/schemas.js | 7 ++++++- server.js | 24 ++++++++++++++++++++---- workers/api.js | 14 +++++++++++++- 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/lib/account.js b/lib/account.js index 7fcc4f9d..49c70015 100644 --- a/lib/account.js +++ b/lib/account.js @@ -91,10 +91,14 @@ class Account { ? accountData.oauth2.provider : undefined, state: accountData.state, - syncTime: accountData.sync, + webhooks: accountData.webhooks || undefined, proxy: accountData.proxy || undefined, smtpEhloName: accountData.smtpEhloName || undefined, + + counters: accountData.counters, + + syncTime: accountData.sync, lastError: accountData.state === 'connected' || !accountData.lastErrorState || !Object.keys(accountData.lastErrorState).length ? null @@ -139,9 +143,26 @@ class Account { } unserializeAccountData(accountData) { - let result = {}; + const result = {}; + + const counters = {}; Object.keys(accountData).forEach(key => { + let countMatch = key.match(/^stats:count:([^:]+):([^:]+)/); + if (countMatch) { + const [, type, counter] = countMatch; + if (!counters[type]) { + counters[type] = {}; + } + + if (!counters[type][counter]) { + counters[type][counter] = 0; + } + + counters[type][counter] = Number(accountData[key]); + return; + } + switch (key) { case 'notifyFrom': case 'syncFrom': @@ -251,6 +272,8 @@ class Account { result.account = null; } + result.counters = counters; + return result; } diff --git a/lib/connection.js b/lib/connection.js index 1fa30ec8..71468c33 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -71,13 +71,14 @@ const settings = require('./settings'); const { redis } = require('./db'); const { addTrackers } = require('./add-trackers'); -async function metrics(logger, key, method, ...args) { +async function metricsMeta(meta, logger, key, method, ...args) { try { parentPort.postMessage({ cmd: 'metrics', key, method, - args + args, + meta: meta || {} }); } catch (err) { logger.error({ msg: 'Failed to post metrics to parent', err }); @@ -698,7 +699,7 @@ class Connection { extraOpts = extraOpts || {}; const { skipWebhook, canSync = true } = extraOpts; - metrics(this.logger, 'events', 'inc', { + metricsMeta({ account: this.account }, this.logger, 'events', 'inc', { event }); @@ -1073,13 +1074,13 @@ class Connection { }); this.imapClient.on('response', data => { - metrics(this.logger, 'imapResponses', 'inc', data); + metricsMeta({}, this.logger, 'imapResponses', 'inc', data); // update byte counters as well let imapStats = this.imapClient.stats(true); - metrics(this.logger, 'imapBytesSent', 'inc', imapStats.sent); - metrics(this.logger, 'imapBytesReceived', 'inc', imapStats.received); + metricsMeta({}, this.logger, 'imapBytesSent', 'inc', imapStats.sent); + metricsMeta({}, this.logger, 'imapBytesReceived', 'inc', imapStats.received); }); try { diff --git a/lib/schemas.js b/lib/schemas.js index 02d7325b..994d7900 100644 --- a/lib/schemas.js +++ b/lib/schemas.js @@ -1234,6 +1234,10 @@ const ipSchema = Joi.string() const accountIdSchema = Joi.string().empty('').trim().max(256).example('example').description('Account ID'); +const accountCountersSchema = Joi.object({ + events: Joi.object().unknown().description('Lifetime event counters').label('AcountCountersEvents').example({ messageNew: 30, messageDeleted: 5 }) +}).label('AccountCounters'); + module.exports = { ADDRESS_STRATEGIES, @@ -1262,7 +1266,8 @@ module.exports = { oauthCreateSchema, tokenRestrictionsSchema, accountIdSchema, - ipSchema + ipSchema, + accountCountersSchema }; /* diff --git a/server.js b/server.js index 3e73b0eb..121263d8 100644 --- a/server.js +++ b/server.js @@ -783,6 +783,9 @@ let spawnWorker = async type => { switch (message.cmd) { case 'metrics': { let statUpdateKey = false; + let accountUpdateKey = false; + + let { account } = message.meta || {}; switch (message.key) { // gather for dashboard counter @@ -798,6 +801,10 @@ let spawnWorker = async type => { case 'events': { let { event } = message.args[0] || {}; + if (account) { + accountUpdateKey = `${message.key}:${event}`; + } + switch (event) { case MESSAGE_NEW_NOTIFY: case MESSAGE_DELETED_NOTIFY: @@ -844,14 +851,23 @@ let spawnWorker = async type => { let hkey = `${REDIS_PREFIX}stats:${statUpdateKey}:${dateStr}`; - redis + let update = redis .multi() .hincrby(hkey, timeStr, 1) .sadd(`${REDIS_PREFIX}stats:keys`, statUpdateKey) // keep alive at most 2 days - .expire(hkey, MAX_DAYS_STATS + 1 * 24 * 3600) - .exec() - .catch(() => false); + .expire(hkey, MAX_DAYS_STATS + 1 * 24 * 3600); + + if (account && accountUpdateKey) { + // increment account specific counter + let accountKey = `${REDIS_PREFIX}iad:${account}`; + update = update.hincrby(accountKey, `stats:count:${account}`, 1); + } + + update.exec().catch(() => false); + } else if (account && accountUpdateKey) { + let accountKey = `${REDIS_PREFIX}iad:${account}`; + redis.hincrby(accountKey, `stats:count:${accountUpdateKey}`, 1).catch(() => false); } if (message.key && metrics[message.key] && typeof metrics[message.key][message.method] === 'function') { diff --git a/workers/api.js b/workers/api.js index 9a5c4146..9ba06dc4 100644 --- a/workers/api.js +++ b/workers/api.js @@ -148,7 +148,8 @@ const { oauthCreateSchema, tokenRestrictionsSchema, accountIdSchema, - ipSchema + ipSchema, + accountCountersSchema } = require('../lib/schemas'); const FLAG_SORT_ORDER = ['\\Inbox', '\\Flagged', '\\Sent', '\\Drafts', '\\All', '\\Archive', '\\Junk', '\\Trash']; @@ -2665,6 +2666,9 @@ When making API calls remember that requests against the same account are queued .description('Account-specific webhook URL'), proxy: settingsSchema.proxyUrl, smtpEhloName: settingsSchema.smtpEhloName, + + counters: accountCountersSchema, + syncTime: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('Last sync time'), lastError: lastErrorSchema.allow(null) }).label('AccountResponseItem') @@ -2767,6 +2771,10 @@ When making API calls remember that requests against the same account are queued result.lastError = accountData.state === 'connected' ? null : accountData.lastErrorState; } + if (accountData.counters) { + result.counters = accountData.counters; + } + return result; } catch (err) { request.logger.error({ msg: 'API request failed', err }); @@ -2855,6 +2863,10 @@ When making API calls remember that requests against the same account are queued .required(), app: Joi.string().max(256).example('AAABhaBPHscAAAAH').description('OAuth2 application ID'), + counters: accountCountersSchema, + + syncTime: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('Last sync time'), + lastError: lastErrorSchema.allow(null) }).label('AccountResponse'), failAction: 'log'