Merge pull request #891 from 18F/develop

[Feature] Update screen size report to screen resolution -- Staging

bin/analytics

@@ -12,7 +12,10 @@
  * --only: only run one or more named reports.
  * --slim: Where supported, use totals only (omit the `data` array).
  *         Only applies to JSON, and reports where "slim": true.
- * --csv: CSV instead of JSON.
+ * --csv: Use format CSV for reports. If CSV and JSON are both specified, will
+ *        create reports for both formats.
+ * --json: Use format JSON for reports. If CSV and JSON are both specified, will
+ *         create reports for both formats.
  * --frequency: Limit to reports with this 'frequency' value.
  * --debug: print debug details on STDOUT
  */

bin/analytics-publisher

@@ -14,7 +14,10 @@
  * --only: only run one or more named reports.
  * --slim: Where supported, use totals only (omit the `data` array).
  *         Only applies to JSON, and reports where "slim": true.
- * --csv: CSV instead of JSON.
+ * --csv: Use format CSV for reports. If CSV and JSON are both specified, will
+ *        create reports for both formats.
+ * --json: Use format JSON for reports. If CSV and JSON are both specified, will
+ *         create reports for both formats.
  * --frequency: Limit to reports with this 'frequency' value.
  * --debug: print debug details on STDOUT
  * --agenciesFile: The file path of a JSON array of agency objects. Reports will

deploy/daily.sh

@@ -2,6 +2,5 @@
 
 export ANALYTICS_SCRIPT_NAME=daily.sh
 
-$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=daily --slim --debug --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json
-$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=daily --slim --debug --csv --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json
+$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=daily --slim --debug --csv --json --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json
 

deploy/hourly.sh

@@ -2,4 +2,4 @@
 
 export ANALYTICS_SCRIPT_NAME=hourly.sh
 
-$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=hourly --slim --debug --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json
+$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=hourly --slim --debug --json --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json

deploy/realtime.sh

@@ -2,5 +2,4 @@
 
 export ANALYTICS_SCRIPT_NAME=realtime.sh
 
-$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=realtime --slim --debug --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json
-$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=realtime --slim --debug --csv --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json
+$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=realtime --slim --debug --csv --json --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json

index.js

@@ -13,8 +13,8 @@ const Processor = require("./src/processor");
  *
  * @param {object} options an object with options to be used when processing
  * all reports.
- * @param {string} options.format the format of the analytics data produced.
- * Accepted formats are "csv" or "json"
+ * @param {boolean} options.csv if true, format report data to CSV
+ * @param {boolean} options.json if true, format report data to JSON
  * @param {string} options.output a string filepath where the analytics data
  * will be written to disk after processing.
  * @param {boolean} options.publish if true, the analytics data will be written
@@ -86,8 +86,8 @@ async function _processReport(appConfig, context, reportConfig, processor) {
  *
  * @param {object} options an object with options to be used when processing
  * all reports.
- * @param {string} options.format the format of the analytics data produced.
- * Accepted formats are "csv" or "json"
+ * @param {boolean} options.csv if true, format report data to CSV
+ * @param {boolean} options.json if true, format report data to JSON
  * @param {string} options.output a string filepath where the analytics data
  * will be written to disk after processing.
  * @param {boolean} options.publish if true, the analytics data will be written
@@ -114,8 +114,8 @@ async function runQueuePublish(options = {}) {
   const queueClient = await _initQueueClient(appConfig, appLogger);
   const queue = "analytics-reporter-job-queue";
 
-  for (const reportConfig of reportConfigs) {
-    for (const agency of agencies) {
+  for (const agency of agencies) {
+    for (const reportConfig of reportConfigs) {
       process.env.AGENCY_NAME = agency.agencyName;
       const reportLogger = Logger.initialize(appConfig, reportConfig);
       try {
@@ -129,7 +129,7 @@ async function runQueuePublish(options = {}) {
           ),
           {
             priority: _messagePriority(reportConfig),
-            singletonKey: `${appConfig.scriptName}-${appConfig.agency}-${reportConfig.name}-${appConfig.format}`,
+            singletonKey: `${appConfig.scriptName}-${appConfig.agency}-${reportConfig.name}`,
           },
         );
         if (jobId) {

migrations/20240819182419_add_jsonb_data_index_ua.js

@@ -0,0 +1,19 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = function (knex) {
+  return knex.schema.raw(
+    "CREATE INDEX analytics_data_ua_gin_jsonb ON analytics_data USING gin(data jsonb_path_ops)",
+  );
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function (knex) {
+  return knex.schema.table("analytics_data", (table) => {
+    table.dropIndex("analytics_data_gin_jsonb");
+  });
+};

reports/usa.json

@@ -54,10 +54,10 @@
       }
     },
     {
-      "name": "screen-size",
+      "name": "screen-resolution",
       "frequency": "daily",
       "slim": true,
-      "sumVisitsByColumns": [
+      "sumUsersByColumns": [
         "screen_resolution"
       ],
       "query": {
@@ -71,7 +71,7 @@
         ],
         "metrics": [
           {
-            "name": "sessions"
+            "name": "totalUsers"
           }
         ],
         "dateRanges": [
@@ -86,12 +86,18 @@
               "dimensionName": "date"
             },
             "desc": true
+          },
+          {
+            "metric": {
+              "metricName": "totalUsers"
+            },
+            "desc": true
           }
         ]
       },
       "meta": {
         "name": "Screen Resolution",
-        "description": "90 days of Screen Resolution visits for all sites."
+        "description": "90 days of screen resolution demographics by user for all sites."
       }
     },
     {

src/actions/format_processed_analytics_data.js

@@ -15,13 +15,17 @@ class FormatProcessedAnalyticsData extends Action {
    */
   async executeStrategy(context) {
     context.logger.debug("Formatting analytics data");
-    context.formattedAnalyticsData = await ResultFormatter.formatResult(
-      context.processedAnalyticsData,
-      {
-        format: context.appConfig.format,
-        slim: context.appConfig.slim && context.reportConfig.slim,
-      },
-    );
+    const formattedAnalyticsData = {};
+    for (const format of context.appConfig.formats) {
+      formattedAnalyticsData[format] = await ResultFormatter.formatResult(
+        context.processedAnalyticsData,
+        {
+          format,
+          slim: context.appConfig.slim && context.reportConfig.slim,
+        },
+      );
+    }
+    context.formattedAnalyticsData = formattedAnalyticsData;
   }
 }
 

src/actions/publish_analytics_data_to_disk.js

@@ -25,11 +25,14 @@ class PublishAnalyticsDataToDisk extends Action {
    */
   async executeStrategy(context) {
     context.logger.debug("Publishing analytics data to disk");
-    await DiskPublisher.publish(
-      context.reportConfig,
-      context.formattedAnalyticsData,
-      context.appConfig,
-    );
+    for (const format of context.appConfig.formats) {
+      await DiskPublisher.publish({
+        name: context.reportConfig.name,
+        data: context.formattedAnalyticsData[format],
+        format,
+        directory: context.appConfig.output,
+      });
+    }
   }
 }
 

src/actions/publish_analytics_data_to_s3.js

@@ -34,16 +34,20 @@ class PublishAnalyticsDataToS3 extends Action {
    */
   async executeStrategy(context) {
     const appConfig = context.appConfig;
-    context.logger.debug("Publishing analytics data to S3");
-    await this.#s3Service.publish(
-      {
-        name: context.reportConfig.name,
-        bucket: appConfig.aws.bucket,
-        path: appConfig.aws.path,
-        format: appConfig.format,
-      },
-      context.formattedAnalyticsData,
-    );
+    for (const format of context.appConfig.formats) {
+      context.logger.debug(
+        `Publishing ${format.toUpperCase()} analytics data to S3`,
+      );
+      await this.#s3Service.publish(
+        {
+          name: context.reportConfig.name,
+          bucket: appConfig.aws.bucket,
+          path: appConfig.aws.path,
+          format,
+        },
+        context.formattedAnalyticsData[format],
+      );
+    }
   }
 }
 

src/app_config.js

@@ -10,8 +10,8 @@ class AppConfig {
   /**
    * @param {object} options an object with options to be used when processing
    * all reports.
-   * @param {string} options.format the format of the analytics data produced.
-   * Accepted formats are "csv" or "json"
+   * @param {boolean} options.csv if true, format report data to CSV
+   * @param {boolean} options.json if true, format report data to JSON
    * @param {string} options.output a string filepath where the analytics data
    * will be written to disk after processing.
    * @param {boolean} options.publish if true, the analytics data will be written
@@ -35,6 +35,17 @@ class AppConfig {
     return this.#options.csv ? "csv" : "json";
   }
 
+  get formats() {
+    const formats = [];
+    if (this.#options.csv) {
+      formats.push("csv");
+    }
+    if (this.#options.json) {
+      formats.push("json");
+    }
+    return formats;
+  }
+
   get output() {
     return this.#options.output;
   }

src/process_results/analytics_data_processor.js

@@ -82,6 +82,7 @@ class AnalyticsDataProcessor {
 
     result.totals = ResultTotalsCalculator.calculateTotals(result, {
       sumVisitsByColumns: report.sumVisitsByColumns,
+      sumUsersByColumns: report.sumUsersByColumns,
     });
 
     return result;

src/process_results/result_totals_calculator.js

@@ -23,7 +23,22 @@ const calculateTotals = (result, options = {}) => {
 
   if (options.sumVisitsByColumns && Array.isArray(options.sumVisitsByColumns)) {
     for (const column of options.sumVisitsByColumns) {
-      totals[`by_${column}`] = _sumVisitsByColumn({ column, result });
+      totals[`by_${column}`] = _sumMetricByColumn({
+        metric: "visits",
+        column,
+        result,
+      });
+      totals[`by_${column}`] = _sortObjectByValues(totals[`by_${column}`]);
+    }
+  }
+
+  if (options.sumUsersByColumns && Array.isArray(options.sumUsersByColumns)) {
+    for (const column of options.sumUsersByColumns) {
+      totals[`by_${column}`] = _sumMetricByColumn({
+        metric: "users",
+        column,
+        result,
+      });
     }
   }
 
@@ -73,15 +88,19 @@ const _sumColumn = ({ result, column }) => {
   }, 0);
 };
 
-const _sumVisitsByColumn = ({ result, column }) => {
+const _sumMetricByColumn = ({ metric, result, column }) => {
   return result.data.reduce((categories, row) => {
     const category = row[column];
-    const visits = parseInt(row.visits);
-    categories[category] = (categories[category] || 0) + visits;
-    return categories;
+    const count = parseInt(row[metric]);
+    categories[category] = (categories[category] || 0) + count;
+    return _sortObjectByValues(categories);
   }, {});
 };
 
+const _sortObjectByValues = (object) => {
+  return Object.fromEntries(Object.entries(object).sort((a, b) => b[1] - a[1]));
+};
+
 const _sumVisitsByCategoryWithDimension = ({ result, column, dimension }) => {
   return result.data.reduce((categories, row) => {
     const parentCategory = row[column];

src/publish/disk.js

@@ -6,12 +6,14 @@ const path = require("path");
  *
  * @param {string} name the name of the file to create.
  * @param {string} data the data to write to the file.
+ * @param {string} format the file extension to use
+ * @param {string} directory the path for the directory to use for the file
  * @param {import('../app_config')} appConfig application config instance. Sets the file
  * extension and the path of the file to create.
  */
-const publish = async ({ name }, data, appConfig) => {
-  const filename = `${name}.${appConfig.format}`;
-  const filepath = path.join(appConfig.output, filename);
+const publish = async ({ name, data, format, directory }) => {
+  const filename = `${name}.${format}`;
+  const filepath = path.join(directory, filename);
   await fs.writeFile(filepath, data);
 };