NMS-16318: User Data Collection

Author: synqotik

.gitignore

@@ -34,4 +34,8 @@ node_modules
 
 # WebStorm
 .idea
-dist
+dist
+
+# This file should NOT be checked in, it will be added in production
+# environments with credentials populated
+config/production.json

README.md

@@ -48,6 +48,20 @@ curl -v -X POST --location "http://localhost:9200/opennms_system/_count?pretty"
 
 ## Running in the production environment
 
+Add a file `config/production.json` which is a copy of `config/default.json` but with actual production values populated.
+
+* Note: make sure not to check in `config/production.json` as it may contain credentials.
+
+Then:
+
+```shell
+npm run start:prod
+```
+
+If you are running this is in a different way (e.g. as part of a service), you can run `npm build-only` to build. Make sure to set the `NODE_ENV` environment variable to `production` (`export NODE_ENV=production`), then run the `dist/app.js` file:
+
 ```shell
-npm start  
+export NODE_ENV=production
+node dist/app.js
 ```
+

config/default.json

@@ -0,0 +1,9 @@
+{
+  "crmConfig": {
+    "crmBaseUrl": "http://localhost",
+    "crmUrlPath": "/crm",
+    "portalId": "123",
+    "formGuid": "abc",
+    "subscriptionTypeId": "111"
+  }
+}

package.json

@@ -6,9 +6,11 @@
   "dependencies": {
     "axios": "^1.6.5",
     "axios-retry": "^4.0.0",
+    "config": "^3.3.10",
     "express": "^4.17.1"
   },
   "devDependencies": {
+    "@types/config": "^3.3.3",
     "@types/express": "^4.17.13",
     "@types/node": "^16.7.10",
     "@typescript-eslint/eslint-plugin": "^6.19.1",
@@ -17,7 +19,9 @@
     "typescript": "^5.3.3"
   },
   "scripts": {
+    "build-only": "tsc -p .",
     "start": "tsc -p . && node dist/app.js",
+    "start:prod": "tsc -p . && export NODE_ENV=production && node dist/app.js",
     "test": "tsc --noEmit --strict -p . && eslint ."
   },
   "author": "Jesse White <jesse@opennms.org>",

src/app.ts

@@ -1,6 +1,9 @@
-import express from "express";
-import type {ErrorRequestHandler} from "express";
-import {Elastic} from "./elastic";
+import express from 'express';
+import type { ErrorRequestHandler } from 'express';
+import { Elastic } from './elastic';
+import config from 'config'
+import { postCrmData, validateConfig } from './lib/userDataCollection/service';
+import { CrmConfig } from './lib/userDataCollection/types';
 
 const port = 3542;
 const app = express();
@@ -15,7 +18,6 @@ const errorHandler: ErrorRequestHandler = (err, req, res, next) => {
     res.writeHead(500, headers);
     res.end("");
     next();
-
 }
 app.use(errorHandler);
 
@@ -40,8 +42,34 @@ app.post("/hs-usage-report", async (req, res) => {
     res.end();
 });
 
+/**
+ * Receive user data from OpenNMS product (e.g. Horizon), send to CRM.
+ */
+app.post("/user-data-collection", async (req, res) => {
+    const crmConfig = config.get<CrmConfig>('crmConfig');
+
+    if (!validateConfig(crmConfig)) {
+      console.error('Invalid CRM configuration', crmConfig);
+      res.writeHead(500, headers);
+      res.end();
+    }
+
+    const data = req.body;
+    console.log("User data collection data received: ", data);
+
+    let statusCode = 200;
+
+    try {
+      await postCrmData(crmConfig, data);
+    } catch (e) {
+      console.error('Error received posting CRM data: ', e);
+      statusCode = 500;
+    }
+
+    res.writeHead(statusCode, headers);
+    res.end();
+});
+
 app.listen(port, () => {
     console.log(`server is listening on ${port}`);
 })
-
-

src/lib/userDataCollection/parser.ts

@@ -0,0 +1,65 @@
+import {
+  CrmDataField,
+  CrmJsonData,
+  FormContext,
+  FormData,
+  LegalConsentCommunication,
+  LegalConsentOptions
+} from './types'
+
+const CONSENT_TEXT = 'I agree to receive email communications from OpenNMS';
+const LEGAL_CONSENT_COMMUNICATION_TEXT = 'If you consent to us contacting you, please opt in below. We will maintain your data until you request us to delete it from our systems. You may opt out of receiving communications from us at any time.';
+const CONTACT_OBJECT_ID = '0-1';
+
+export const parseCrmData = (formData: FormData, subscriptionTypeId: string): CrmJsonData => {
+  const context = {
+    pageName: 'OpenNMS Web Console',
+    pageUri: 'opennms/index.jsp'
+  } as FormContext;
+
+  const communication = {
+    value: true,
+    subscriptionTypeId,
+    text: LEGAL_CONSENT_COMMUNICATION_TEXT
+  } as LegalConsentCommunication;
+
+  const legalConsentOptions = {
+    consent: {
+      consentToProcess: true,
+      text: CONSENT_TEXT,
+      communications: [communication]
+    }
+  } as LegalConsentOptions;
+
+  const fieldValues = [
+    { name: 'firstname', value: formData.firstName },
+    { name: 'lastname', value: formData.lastName },
+    { name: 'email', value: formData.email },
+    { name: 'company', value: formData.company },
+    { name: 'horizon_system_id', value: formData.systemId },
+    { name: 'utm_campaign', value: 'Horizon' },
+    { name: 'utm_medium', value: 'direct' },
+    { name: 'utm_content', value: 'Horizon IForm' },
+    { name: 'utm_term', value: '' }
+  ];
+
+  // UTC epoch milliseconds
+  const submittedAt = Date.now();
+
+  const jsonData = {
+    fields: fieldValues.map(({ name, value }) => createField(name, value)),
+    submittedAt,
+    context,
+    legalConsentOptions
+  } as CrmJsonData;
+
+  return jsonData;
+}
+
+const createField = (name: string, value: string) => {
+  return {
+    objectTypeId: CONTACT_OBJECT_ID,
+    name,
+    value
+  } as CrmDataField;
+}

src/lib/userDataCollection/service.ts

@@ -0,0 +1,51 @@
+import { CrmConfig, CrmJsonData, FormData } from './types';
+import { parseCrmData } from './parser';
+import { Httpclient } from '../../httpclient';
+
+export const postCrmData = async (config: CrmConfig, data: FormData) => {
+  if (!validateFormData(data)) {
+    console.error('Invalid form data received', data);
+    throw new Error('Invalid form data');
+  }  
+
+  const crmData: CrmJsonData = parseCrmData(data, config.subscriptionTypeId);
+  const client = new Httpclient(config.crmBaseUrl);
+  const url = `${config.crmUrlPath}/${config.portalId}/${config.formGuid}`
+
+  try {
+      await client.post(url, crmData);
+
+      console.log(`Successfully posted User Data Collection CRM data to url '${config.crmBaseUrl}${url}'`);
+  } catch (e) {
+      console.error(`Failed to post User Data Collection CRM data report to url '${config.crmBaseUrl}${url}'`, e);
+      throw e;
+  }
+}
+
+const validateFormData = (data: FormData): boolean => {
+  if (!data ||
+    !data.consent ||
+    !data.email ||
+    !data.product ||
+    data.product.toLowerCase() !== 'horizon') {
+      return false;
+    }
+
+  return true;
+}
+
+export const validateConfig = (config?: CrmConfig): boolean => {
+  if (!config) {
+    return false;
+  }
+
+  if (!config.crmBaseUrl ||
+    !config.crmUrlPath ||
+    !config.formGuid ||
+    !config.portalId ||
+    !config.subscriptionTypeId) {
+      return false;
+    }
+
+    return true;
+}

src/lib/userDataCollection/types.ts

@@ -0,0 +1,57 @@
+/**
+ * User data received from an OpenNMS product instance.
+ */
+export interface FormData {
+  product: string; // for now this should be 'Horizon', but could potentially be used for other products
+  consent: boolean;
+  firstName: string;
+  lastName: string;
+  email: string;
+  company: string;
+  systemId: string;
+}
+
+export interface CrmDataField {
+  objectTypeId: string;
+  name: string;
+  value: string;
+}
+
+export interface FormContext {
+  pageUri: string;
+  pageName: string;
+}
+
+export interface LegalConsentCommunication {
+  value: boolean;
+  subscriptionTypeId: string;
+  text: string;
+}
+
+export interface LegalConsent {
+  communications: LegalConsentCommunication[];
+  consentToProcess: boolean;
+  text: string;
+}
+
+export interface LegalConsentOptions {
+  consent: LegalConsent;
+}
+
+/**
+ * Data to post to the CRM endpoint.
+ */
+export interface CrmJsonData {
+  fields: CrmDataField[];
+  submittedAt: number; // timestamp, epoch in ms
+  context: FormContext;
+  legalConsentOptions: LegalConsentOptions;
+}
+
+export interface CrmConfig {
+  crmBaseUrl: string;
+  crmUrlPath: string;
+  portalId: string;
+  formGuid: string;
+  subscriptionTypeId: string;
+}

test/user-data-collection-test.http

@@ -0,0 +1,14 @@
+POST http://localhost:3542/user-data-collection
+Content-Type: application/json
+
+{
+  "product": "Horizon",
+  "consent": true,
+  "firstName": "First",
+  "lastName": "Last",
+  "email": "someone@example.com",
+  "company": "OpenNMS Test",
+  "systemId": "444"
+}
+
+###