Merge pull request Countly#5307 from Countly/next

Next

.editorconfig

@@ -4,6 +4,6 @@ root = true
 end_of_line = lf
 insert_final_newline = false
 
-[*.{js,html}]
+[*.{js,html,scss,json}]
 indent_style = space
 indent_size = 4

.eslintrc.json

@@ -130,9 +130,12 @@
                 "frontend/express/public/core/*/javascripts/countly.*.js",
                 "plugins/*/frontend/public/javascripts/countly.*.js"
             ],
+            "parserOptions": {
+                "ecmaVersion": 2020
+            },
             "env": {
                 "browser": true,
-                "es6": true
+                "es2020": true
             },
             "rules": {
                 "valid-jsdoc": [
@@ -205,7 +208,7 @@
             ],
             "excludedFiles": "plugins/*/frontend/public/**/*.js",
             "env": {
-                "es6": true,
+                "es2020": true,
                 "node": true
             },
             "parserOptions": {
@@ -285,7 +288,7 @@
             ],
             "excludedFiles": "plugins/*/tests.js",
             "env": {
-                "es6": true,
+                "es2020": true,
                 "node": true
             },
             "parserOptions": {
@@ -302,7 +305,7 @@
                 "plugins/*/tests.js"
             ],
             "env": {
-                "es6": true,
+                "es2020": true,
                 "node": true,
                 "mocha": true
             },
@@ -321,7 +324,7 @@
                 "ui-tests/**/*.js"
             ],
             "env": {
-                "es6": true,
+                "es2020": true,
                 "node": true,
                 "mocha": true
             },

.github/workflows/main.yml

@@ -77,6 +77,7 @@ jobs:
       image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
+        COUNTLY_CONFIG_API_PREVENT_JOBS: true
 
     steps:
       - uses: actions/checkout@v3
@@ -133,6 +134,7 @@ jobs:
       image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
+        COUNTLY_CONFIG_API_PREVENT_JOBS: true
 
     steps:
       - uses: actions/checkout@v3
@@ -179,6 +181,7 @@ jobs:
       image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
+        COUNTLY_CONFIG_API_PREVENT_JOBS: true
 
     steps:
       - uses: actions/checkout@v3
@@ -212,8 +215,8 @@ jobs:
           npm install
           /sbin/my_init &
           node bin/scripts/test.connection.js
-          npx grunt mochaTest
-  ui-test:
+          npx grunt mochaTest       
+  ui-test-dashboard:
     runs-on: ubuntu-latest
 
     services:
@@ -231,6 +234,7 @@ jobs:
       image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
+        COUNTLY_CONFIG_API_PREVENT_JOBS: true
 
     steps:
       - uses: actions/checkout@v2
@@ -277,7 +281,80 @@ jobs:
           /sbin/my_init &
           cd ui-tests
           npm install
-          npm run cy:run
+          npm run cy:run:dashboard
+
+      - name: Upload UI tests artifacts
+        if: ${{ failure() }}
+        shell: bash
+        working-directory: /opt/countly/ui-tests/cypress
+        run: |
+          ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}.tar.gz"
+          mkdir -p screenshots videos
+          tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos
+          curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME"
+
+  ui-test-onboarding:
+    runs-on: ubuntu-latest
+
+    services:
+      mongodb:
+        image: mongo:6.0
+        options: >-
+          --health-cmd mongosh
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 27017:27017
+
+    container:
+      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      env:
+        COUNTLY_CONFIG__MONGODB_HOST: mongodb
+        COUNTLY_CONFIG_API_PREVENT_JOBS: true
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Chrome
+        shell: bash
+        run: |
+          apt update
+          apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb wget
+          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O /tmp/chrome.deb
+          apt install -y /tmp/chrome.deb
+
+      - name: Copy code
+        shell: bash
+        run: cp -rf ./* /opt/countly
+
+      - name: Prepare files to use correct MongoDB host
+        shell: bash
+        run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
+
+      - name: Prepare environment
+        shell: bash
+        working-directory: /opt/countly
+        run: |
+          sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js
+          cp "./plugins/plugins.default.json" "/opt/countly/plugins/plugins.json"
+          npm install
+          sudo countly task dist-all
+          bash bin/scripts/countly.prepare.ce.tests.sh
+          cd ui-tests
+          echo '{"username": "${{ secrets.CYPRESS_USER_USERNAME }}","email": "${{ secrets.CYPRESS_USER_EMAIL }}","password": "${{ secrets.CYPRESS_USER_PASSWORD }}"}' > cypress/fixtures/user.json
+          sed -i 's/00000000-0000-0000-0000-000000000000/${{ secrets.CYPRESS_KEY }}/g' package.json
+          cp cypress.config.sample.js cypress.config.js
+          sed -i 's/000000/${{ secrets.CYPRESS_PROJECT_ID }}/g' cypress.config.js
+
+      - name: Run UI tests
+        shell: bash
+        working-directory: /opt/countly
+        run: |
+          /sbin/my_init &
+          cd ui-tests
+          npm install
+          npm run cy:run:onboarding
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}

Gruntfile.js

@@ -150,6 +150,8 @@ module.exports = function(grunt) {
                     'frontend/express/public/javascripts/countly/vue/components/drawer.js',
                     'frontend/express/public/core/notes/javascripts/countly.models.js',
                     'frontend/express/public/core/notes/javascripts/countly.common.notes.js',
+                    'frontend/express/public/core/date-presets/javascripts/countly.models.js',
+                    'frontend/express/public/core/date-presets/javascripts/countly.views.js',
                     'frontend/express/public/javascripts/countly/vue/components/vis.js',
                     'frontend/express/public/javascripts/countly/vue/components/helpers.js',
                     'frontend/express/public/javascripts/countly/vue/components/sidebar.js',
@@ -504,4 +506,4 @@ module.exports = function(grunt) {
 
     grunt.registerTask('dist-all', ['dist', 'plugins', 'locales']);
 
-};
+};

api/api.js

@@ -173,7 +173,7 @@ plugins.connectToAllDatabases().then(function() {
     *  Handle exit events for gracefull close
     */
     ['SIGHUP', 'SIGINT', 'SIGQUIT', 'SIGILL', 'SIGTRAP', 'SIGABRT',
-        'SIGBUS', 'SIGFPE', 'SIGUSR1', 'SIGSEGV', 'SIGUSR2', 'SIGTERM'
+        'SIGBUS', 'SIGFPE', 'SIGSEGV', 'SIGTERM',
     ].forEach(function(sig) {
         process.on(sig, async function() {
             storeBatchedData(sig);
@@ -280,7 +280,14 @@ plugins.connectToAllDatabases().then(function() {
             : os.cpus().length;
 
         for (let i = 0; i < workerCount; i++) {
-            const worker = cluster.fork();
+            // there's no way to define inspector port of a worker in the code. So if we don't
+            // pick a unique port for each worker, they conflict with each other.
+            let nodeOptions = {};
+            if (countlyConfig?.symlinked !== true) { // countlyConfig.symlinked is passed when running in a symlinked setup
+                const inspectorPort = i + 1 + (common?.config?.masterInspectorPort || 9229);
+                nodeOptions = { NODE_OPTIONS: "--inspect-port=" + inspectorPort };
+            }
+            const worker = cluster.fork(nodeOptions);
             workers.push(worker);
         }
 

api/configextender.js

@@ -52,7 +52,8 @@ const OVERRIDES = {
     FILESTORAGE: 'fileStorage',
     RELOADCONFIGAFTER: 'reloadConfigAfter',
     PREVENTJOBS: 'preventJobs',
-    SHARED_CONNECTION: 'shared_connection'
+    SHARED_CONNECTION: 'shared_connection',
+    PREVENT_JOBS: 'preventJobs'
 };
 
 /**

api/lib/countly.common.js

@@ -384,10 +384,16 @@ function getPeriodObject(prmPeriod, bucket) {
 
     var period = prmPeriod || _period;
 
-    endTimestamp = _currMoment.clone().endOf("day");
+    var excludeCurrentDay = period.excludeCurrentDay || false;
+
+    if (period.period) {
+        period = period.period;
+    }
+
+    endTimestamp = excludeCurrentDay ? _currMoment.clone().subtract(1, 'days').endOf('day') : _currMoment.clone().endOf('day');
 
     if (period.since) {
-        period = [period.since, Date.now()];
+        period = [period.since, endTimestamp.clone().valueOf()];
     }
 
     if (period && typeof period === 'string' && period.indexOf(",") !== -1) {
@@ -518,6 +524,24 @@ function getPeriodObject(prmPeriod, bucket) {
             previousPeriod: yesterday.clone().subtract(1, "day").format("YYYY.M.D")
         });
     }
+    else if (/([1-9][0-9]*)minutes/.test(period)) {
+        let nMinutes = parseInt(/([1-9][0-9]*)minutes/.exec(period)[1]);
+        startTimestamp = _currMoment.clone().startOf("minute").subtract(nMinutes - 1, "minutes");
+        cycleDuration = moment.duration(nMinutes, "minutes");
+        Object.assign(periodObject, {
+            dateString: "HH:mm",
+            isSpecialPeriod: true
+        });
+    }
+    else if (/([1-9][0-9]*)hours/.test(period)) {
+        let nHours = parseInt(/([1-9][0-9]*)hours/.exec(period)[1]);
+        startTimestamp = _currMoment.clone().startOf("hour").subtract(nHours - 1, "hours");
+        cycleDuration = moment.duration(nHours, "hours");
+        Object.assign(periodObject, {
+            dateString: "HH:mm",
+            isSpecialPeriod: true
+        });
+    }
     else if (/([1-9][0-9]*)days/.test(period)) {
         let nDays = parseInt(/([1-9][0-9]*)days/.exec(period)[1]);
         startTimestamp = _currMoment.clone().startOf("day").subtract(nDays - 1, "days");
@@ -741,6 +765,46 @@ function getPeriodObject(prmPeriod, bucket) {
     return periodObject;
 }
 
+/**
+ * Checks if the period parameter is valid
+ * @param {string} period - period parameter
+ * @returns {boolean} true if period is valid, false if not
+*/
+countlyCommon.isValidPeriodParam = function(period) {
+
+    if (period && typeof period === 'string' && period.indexOf(",") !== -1) {
+        try {
+            period = JSON.parse(period);
+        }
+        catch (SyntaxError) {
+            return false;
+        }
+    }
+
+    if (Array.isArray(period)) {
+        return period.length === 2;
+    }
+
+    if (typeof period === 'object') {
+        if (Object.prototype.hasOwnProperty.call(period, 'period')) {
+            return countlyCommon.isValidPeriodParam(period.period);
+        }
+        else {
+            return Object.prototype.hasOwnProperty.call(period, 'since');
+        }
+    }
+
+    return period === 'month' ||
+        period === 'prevMonth' ||
+        period === 'day' ||
+        period === 'yesterday' ||
+        period === 'hour' ||
+        /([1-9][0-9]*)days/.test(period) ||
+        /([1-9][0-9]*)weeks/.test(period) ||
+        /([1-9][0-9]*)months/.test(period) ||
+        /([1-9][0-9]*)years/.test(period);
+};
+
 // Public Properties
 /**
 * Currently selected period

api/parts/data/fetch.js

@@ -2019,8 +2019,26 @@ fetch.alljobs = async function(metric, params) {
     ]).toArray();
     total = total.length > 0 ? total[0].total : 0;
     const pipeline = [
+        {
+            $addFields: {
+                sortKey: {
+                    $cond: {
+                        if: { $eq: ["$status", 0] },
+                        then: 0,
+                        else: {
+                            $cond: {
+                                if: { $eq: ["$status", 7] },
+                                then: 1,
+                                else: 2
+                            }
+                        }
+                    }
+                }
+            }
+        },
         {
             $sort: {
+                sortKey: 1,
                 finished: -1
             }
         },