Skip to content

Commit ea6cde7

Browse files
CookiezaursCookiezaurs
committed
[SER-1344] Cahce beautified collection names to speed up loading dbviewer.
1 parent a693c09 commit ea6cde7

File tree

2 files changed

+157
-46
lines changed

2 files changed

+157
-46
lines changed

‎api/utils/rights.js

Lines changed: 147 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -488,19 +488,22 @@ function wrapCallback(params, callback, callbackParam, func) {
488488
}
489489

490490
/**
491-
* Get events data
492-
* A helper function for db access check
493-
* @param {object} params - {@link params} object
494-
* @param {array} apps - array with each element being app document
495-
* @param {function} callback - callback method
496-
**/
497-
function dbLoadEventsData(params, apps, callback) {
491+
* Function to load and cache data
492+
* @param {object} apps - apps
493+
* @param {function} callback - callback function
494+
*/
495+
function loadAndCacheEventsData(apps, callback) {
498496
const appIds = [];
499497
const appNamesById = {};
500-
498+
var anyNameMissing = false;
501499
apps.forEach((app) => {
500+
cachedSchema[app._id + ''] = cachedSchema[app._id + ''] || {};
501+
cachedSchema[app._id + ''].loading = true;
502502
appIds.push(common.db.ObjectID(app._id + ''));
503503
appNamesById[app._id + ''] = app.name;
504+
if (!appNamesById[app._id + '']) {
505+
anyNameMissing = true;
506+
}
504507
});
505508

506509
/**
@@ -510,13 +513,12 @@ function dbLoadEventsData(params, apps, callback) {
510513
* @param {function} cb - callback method
511514
**/
512515
function getEvents(appColl, cb) {
513-
var result = {};
514516
common.db.collection('events').find({'_id': { $in: appColl.appIds }}).toArray(function(err, events) {
515517
if (!err && events) {
516518
for (let h = 0; h < events.length; h++) {
517519
if (events[h].list) {
518520
for (let i = 0; i < events[h].list.length; i++) {
519-
result[crypto.createHash('sha1').update(events[h].list[i] + events[h]._id + "").digest('hex')] = "(" + appColl.appNamesById[events[h]._id + ''] + ": " + events[h].list[i] + ")";
521+
collectionMap[crypto.createHash('sha1').update(events[h].list[i] + events[h]._id + "").digest('hex')] = {"n": true, "a": events[h]._id + "", "e": events[h].list[i], "name": "(" + appNamesById[events[h]._id + ''] + ": " + events[h].list[i] + ")"};
520522
}
521523
}
522524
}
@@ -525,18 +527,17 @@ function dbLoadEventsData(params, apps, callback) {
525527
appColl.appIds.forEach((appId) => {
526528
if (plugins.internalDrillEvents) {
527529
for (let i = 0; i < plugins.internalDrillEvents.length; i++) {
528-
result[crypto.createHash('sha1').update(plugins.internalDrillEvents[i] + appId + "").digest('hex')] = "(" + appColl.appNamesById[appId + ''] + ": " + plugins.internalDrillEvents[i] + ")";
530+
collectionMap[crypto.createHash('sha1').update(plugins.internalDrillEvents[i] + appId + "").digest('hex')] = {"n": true, "a": appId + "", "e": plugins.internalDrillEvents[i], "name": "(" + appColl.appNamesById[appId + ''] + ": " + plugins.internalDrillEvents[i] + ")"};
529531
}
530532
}
531533

532534
if (plugins.internalEvents) {
533535
for (let i = 0; i < plugins.internalEvents.length; i++) {
534-
result[crypto.createHash('sha1').update(plugins.internalEvents[i] + appId + "").digest('hex')] = "(" + appColl.appNamesById[appId + ''] + ": " + plugins.internalEvents[i] + ")";
536+
collectionMap[crypto.createHash('sha1').update(plugins.internalEvents[i] + appId + "").digest('hex')] = {"n": true, "a": appId + "", "e": plugins.internalEvents[i], "name": "(" + appColl.appNamesById[appId + ''] + ": " + plugins.internalEvents[i] + ")"};
535537
}
536538
}
537539
});
538-
539-
cb(null, result);
540+
cb(null, true);
540541
});
541542
}
542543

@@ -553,35 +554,140 @@ function dbLoadEventsData(params, apps, callback) {
553554
for (let idx = 0; idx < viewDocs.length; idx++) {
554555
if (viewDocs[idx].segments) {
555556
for (var segkey in viewDocs[idx].segments) {
556-
result["app_viewdata" + crypto.createHash('sha1').update(segkey + viewDocs[idx]._id + '').digest('hex')] = "(" + appColl.appNamesById[viewDocs[idx]._id + ''] + ": " + segkey + ")";
557+
collectionMap["app_viewdata" + crypto.createHash('sha1').update(segkey + viewDocs[idx]._id + '').digest('hex')] = {"n": true, "a": viewDocs[idx]._id + '', "vs": segkey, "name": "(" + appColl.appNamesById[viewDocs[idx]._id + ''] + ": " + segkey + ")"};
557558
}
558559
}
559560
}
560561
}
561-
562562
appColl.appIds.forEach((appId) => {
563-
result["app_viewdata" + crypto.createHash('sha1').update("" + appId).digest('hex')] = "(" + appColl.appNamesById[appId + ''] + ": no-segment)";
563+
collectionMap["app_viewdata" + crypto.createHash('sha1').update("" + appId).digest('hex')] = {"n": true, "a": "" + appId, "vs": "", "name": "(" + appColl.appNamesById[appId + ''] + ": no-segment)"};
564564
});
565+
cb(null, true);
566+
});
567+
}
565568

566-
cb(null, result);
569+
if (anyNameMissing) { //We do not have name for APPs, so we need to fetch them
570+
common.db.collection('apps').find({'_id': { $in: appIds }}, {'name': 1}).toArray(function(err, newapps) {
571+
if (err) {
572+
log.e(err);
573+
callback(err);
574+
}
575+
else {
576+
for (var i = 0; i < newapps.length; i++) {
577+
newapps[i].name = newapps[i].name || "Unknown";
578+
}
579+
loadAndCacheEventsData(newapps, callback);
580+
}
581+
});
582+
}
583+
else {
584+
getEvents({ appIds, appNamesById }, function(err) {
585+
if (err) {
586+
log.e(err);
587+
}
588+
getViews({ appIds, appNamesById }, function(err1) {
589+
if (err1) {
590+
log.e(err1);
591+
}
592+
for (var item in collectionMap) {
593+
if (appNamesById[collectionMap[item].a]) {
594+
if (!collectionMap[item].n) {
595+
delete collectionMap[item];
596+
}
597+
else {
598+
delete collectionMap[item].n;
599+
}
600+
}
601+
}
602+
apps.forEach((app) => {
603+
cachedSchema[app._id + ''].ts = Date.now();
604+
cachedSchema[app._id + ''].loading = false;
605+
});
606+
common.cachedSchema = cachedSchema;
607+
common.collectionMap = collectionMap;
608+
callback(err || err1);
609+
});
567610
});
568611
}
569612

613+
614+
}
615+
/**
616+
* Get events data
617+
* A helper function for db access check
618+
* @param {object} params - {@link params} object
619+
* @param {array} apps - array with each element being app document
620+
* @param {function} callback - callback method
621+
**/
622+
function dbLoadEventsData(params, apps, callback) {
623+
var events = {};
624+
var views = {};
625+
var callCalculate = [];
626+
var appMap = {};
627+
for (var a in apps) {
628+
if (!cachedSchema[apps[a]._id + ''] || (cachedSchema[apps[a]._id + ''] && !cachedSchema[apps[a]._id + ''].loading && (Date.now() - cachedSchema[apps[a]._id + ''].ts) > 10 * 60 * 1000)) {
629+
callCalculate.push(apps[a]);
630+
}
631+
appMap[apps[a]._id + ''] = true;
632+
}
633+
570634
if (params.member.eventList) {
571635
callback(null, params.member.eventList, params.member.viewList);
636+
if (callCalculate.length > 0) {
637+
loadAndCacheEventsData(callCalculate, function(err) {
638+
if (err) {
639+
log.e(err);
640+
}
641+
});
642+
}
572643
}
573-
else {
574-
getEvents({ appIds, appNamesById }, function(err, events) {
644+
else if (callCalculate.length > 0) {
645+
loadAndCacheEventsData(callCalculate, function(err) {
646+
if (err) {
647+
log.e(err);
648+
}
649+
for (var key in collectionMap) {
650+
if (appMap[collectionMap[key].a]) {
651+
if (collectionMap[key].e) {
652+
events[key] = collectionMap[key].name;
653+
}
654+
else if (collectionMap[key].vs) {
655+
views[key] = collectionMap[key].name;
656+
}
657+
}
658+
}
575659
params.member.eventList = events;
576-
getViews({ appIds, appNamesById }, function(err1, views) {
577-
params.member.viewList = views;
578-
callback(err, events, views);
579-
});
660+
params.member.viewList = views;
661+
callback(null, events, views);
580662
});
581663
}
664+
else {
665+
for (var key in collectionMap) {
666+
if (appMap[collectionMap[key].a]) {
667+
if (collectionMap[key].e) {
668+
events[key] = collectionMap[key].name;
669+
}
670+
else if (collectionMap[key].vs) {
671+
views[key] = collectionMap[key].name;
672+
}
673+
}
674+
}
675+
params.member.eventList = events;
676+
params.member.viewList = views;
677+
callback(null, events, views);
678+
}
582679
}
583680
exports.dbLoadEventsData = dbLoadEventsData;
584681

682+
exports.getCollectionName = function(hashValue) {
683+
if (collectionMap[hashValue]) {
684+
return collectionMap[hashValue].name;
685+
}
686+
else {
687+
return hashValue;
688+
}
689+
};
690+
585691
/**
586692
* Check user has access to collection
587693
* @param {object} params - {@link params} object
@@ -601,7 +707,7 @@ exports.dbUserHasAccessToCollection = function(params, collection, app_id, callb
601707
}
602708
var apps = [];
603709
var userApps = module.exports.getUserApps(params.member);
604-
710+
var hashValue = "";
605711
//use whatever user has permission for
606712
apps = userApps || [];
607713
// also check for app based restrictions
@@ -622,17 +728,20 @@ exports.dbUserHasAccessToCollection = function(params, collection, app_id, callb
622728
appList.push({_id: apps[i]});
623729
}
624730
}
625-
dbLoadEventsData(params, appList, function(err, eventList/*, viewList*/) {
731+
hashValue = collection.replace("drill_events", "").replace("events", "");
732+
dbLoadEventsData(params, appList, function(err) {
626733
if (err) {
627734
log.e("[rights.js].dbUserHasAccessToCollection() failed at dbLoadEventsData (events) callback.", err);
628735
return callback(false);
629736
}
630-
for (let i in eventList) {
631-
if (collection.indexOf(i, collection.length - i.length) !== -1) {
737+
else {
738+
if (collectionMap[hashValue] && apps.length > 0 && apps.indexOf(collectionMap[hashValue].a) !== -1) {
632739
return callback(true);
633740
}
741+
else {
742+
return callback(false);
743+
}
634744
}
635-
return callback(false);
636745
});
637746
}
638747
else if (collection.indexOf("app_viewdata") === 0) {
@@ -641,18 +750,23 @@ exports.dbUserHasAccessToCollection = function(params, collection, app_id, callb
641750
appList.push({_id: apps[i]});
642751
}
643752
}
753+
hashValue = collection;//we keep app_viewdata
644754

645-
dbLoadEventsData(params, appList, function(err, eventList, viewList) {
755+
dbLoadEventsData(params, appList, function(err) {
646756
if (err) {
647757
log.e("[rights.js].dbUserHasAccessToCollection() failed at dbLoadEventsData (app_viewdata) callback.", err);
648758
return callback(false);
649759
}
650-
for (let i in viewList) {
651-
if (collection.indexOf(i, collection.length - i.length) !== -1) {
760+
else {
761+
if (collectionMap[hashValue] && apps.length > 0 && apps.indexOf(collectionMap[hashValue].a) !== -1) {
652762
return callback(true);
653763
}
764+
else {
765+
return callback(false);
766+
}
767+
654768
}
655-
return callback(false);
769+
656770
});
657771
}
658772
else {

‎plugins/dbviewer/api/api.js

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ var common = require('../../../api/utils/common.js'),
55
countlyFs = require('../../../api/utils/countlyFs.js'),
66
_ = require('underscore'),
77
taskManager = require('../../../api/utils/taskmanager.js'),
8-
{ dbUserHasAccessToCollection, dbLoadEventsData, validateUser, getUserApps, validateGlobalAdmin, hasReadRight } = require('../../../api/utils/rights.js'),
8+
{ getCollectionName, dbUserHasAccessToCollection, dbLoadEventsData, validateUser, getUserApps, validateGlobalAdmin, hasReadRight } = require('../../../api/utils/rights.js'),
99
exported = {};
1010
const { EJSON } = require('bson');
1111

@@ -251,7 +251,10 @@ var spawn = require('child_process').spawn,
251251
lookup[apps[i]._id + ""] = apps[i].name;
252252
}
253253

254-
dbLoadEventsData(params, apps, function(err, eventList, viewList) {
254+
dbLoadEventsData(params, apps, function(err) {
255+
if (err) {
256+
log.e(err);
257+
}
255258
async.map(Object.keys(dbs), getCollections, function(error, results) {
256259
if (error) {
257260
console.error(error);
@@ -276,7 +279,7 @@ var spawn = require('child_process').spawn,
276279
if (col.collectionName.indexOf("system.indexes") === -1 && col.collectionName.indexOf("sessions_") === -1) {
277280
userHasAccess(params, col.collectionName, params.qstring.app_id, function(hasAccess) {
278281
if (hasAccess) {
279-
ob = parseCollectionName(col.collectionName, lookup, eventList, viewList);
282+
ob = parseCollectionName(col.collectionName, lookup);
280283
db.collections[ob.pretty] = ob.name;
281284
}
282285
done();
@@ -604,7 +607,7 @@ var spawn = require('child_process').spawn,
604607

605608
}
606609

607-
var parseCollectionName = function parseCollectionName(name, apps, events, views) {
610+
var parseCollectionName = function parseCollectionName(name, apps) {
608611
var pretty = name;
609612

610613
let isEvent = false;
@@ -619,17 +622,11 @@ var spawn = require('child_process').spawn,
619622
isEvent = true;
620623
}
621624
else if (name.indexOf("app_viewdata") === 0) {
625+
isView = true;
622626

623-
let hash = name.substring(12);
624-
if (views["app_viewdata" + hash]) {
625-
isView = true;
626-
}
627627
}
628628
if (isView) {
629-
let hash = name.substring(12);
630-
if (views["app_viewdata" + hash]) {
631-
pretty = name.replace(hash, views["app_viewdata" + hash]);
632-
}
629+
pretty = "app_viewdata" + getCollectionName(name);
633630
}
634631
else if (!isEvent) {
635632
for (let i in apps) {
@@ -645,7 +642,7 @@ var spawn = require('child_process').spawn,
645642
pretty = name;
646643
}
647644
else {
648-
const targetEntry = events[eventHash];
645+
var targetEntry = getCollectionName(eventHash);
649646
if (!_.isUndefined(targetEntry)) {
650647
pretty = name.replace(eventHash, targetEntry);
651648
}

0 commit comments

Comments
 (0)