From 171114af1f093b0d40a605bc813b31535f761cdf Mon Sep 17 00:00:00 2001
From: darnjo <josh@reso.org>
Date: Mon, 25 Sep 2023 21:19:35 -0700
Subject: [PATCH] #54: Added basic availability report scoring and
 serialization

---
 lib/replication/index.js | 54 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 2 deletions(-)

diff --git a/lib/replication/index.js b/lib/replication/index.js
index a9ea544..8975b7e 100644
--- a/lib/replication/index.js
+++ b/lib/replication/index.js
@@ -2,6 +2,49 @@
 
 const { replicationIterator, REPLICATION_STRATEGIES } = require('./replication-iterator');
 const humanizeDuration = require('humanize-duration');
+const { writeFile } = require('fs/promises');
+
+const scorePayload = (records = [], availabilityMap = {}) => {
+  records.forEach(r => {
+    Object.entries(r).forEach(([k, v]) => {
+      if (!availabilityMap?.[k]) {
+        availabilityMap[k] = 0;
+      }
+
+      if (!!v) {
+        availabilityMap[k]++;
+      }
+    });
+  });
+
+  return availabilityMap;
+};
+
+const consolidateResults = (availabilityMap = {}, totalRecordCount = 0) => {
+  return {
+    totalRecordCount,
+    fields: Object.entries(availabilityMap ?? {}).map(([k, v]) => {
+      return { fieldName: k, frequency: v };
+    })
+  };
+};
+
+const writeDataAvailabilityReport = async (availabilityMap = {}, totalRecordCount = 0) => {
+  const AVAILABILITY_REPORT_FILENAME = 'data-availability-report.json';
+
+  try {
+    await writeFile(AVAILABILITY_REPORT_FILENAME, JSON.stringify({
+      description: 'RESO Data Availability Report',
+      version: '1.7',
+      generatedOn: new Date().toISOString(),
+      fields: consolidateResults(availabilityMap, totalRecordCount)
+    }, null, '  '));
+
+    console.log(`Results written to ${AVAILABILITY_REPORT_FILENAME}`);
+  } catch (err) {
+    console.error(err);
+  }
+};
 
 const replicate = async ({ url: requestUri, strategy, bearerToken, expansions }) => {
   if (!Object.values(REPLICATION_STRATEGIES).includes(strategy)) {
@@ -20,17 +63,22 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, expansions })
   const startTime = Date.now(),
     responseTimes = [];
 
-  let recordsFetched = 0;
+  let recordsFetched = 0,
+    availabilityMap = {};
 
   for await (const data of replicationIterator(config)) {
     if (data?.hasResults) {
       console.log('Data fetched!');
       responseTimes.push(data?.responseTimeMs ?? 0);
       recordsFetched = data?.recordsFetched ?? 0;
+
+      if (data?.response?.value?.length) {
+        availabilityMap = scorePayload(data.response.value, availabilityMap);
+      }
     }
   }
 
-  console.log(`\nReplication completed in ${humanizeDuration(Date.now() - startTime)}!`);
+  console.log(`\nReplication completed in ~${humanizeDuration(Date.now() - startTime, { round: true })}!`);
   console.log(
     `Total requests: ${responseTimes?.length}, Average response time: ${parseInt(
       responseTimes?.reduce((acc, item) => {
@@ -41,6 +89,8 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, expansions })
       }, 0) / responseTimes.length
     )}ms, Total records: ${recordsFetched}\n`
   );
+
+  await writeDataAvailabilityReport(availabilityMap, recordsFetched);
   return;
 };