#54: Update availability percentage calculation for expansions (#55)

* #54: update tests to fail the current process * #54: use numExpandedRecordsFetched for expanded items * #54: add optional chaining * #54: update tests to fail * #54: change numExpandedRecordsFetched to numUniqueRecordsFetched * #54: update tests * #54: add new param 'useNewReportFormat' we use 'numUniqueRecordsFetched' for counts with new format otherwise we will use 'numRecordsFetched'
RESOStandards · Apr 15, 2024 · a02515a · a02515a
1 parent b3331d6
commit a02515a
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 26 deletions.
diff --git a/lib/process-data-availability.js b/lib/process-data-availability.js
@@ -195,7 +195,7 @@ const getAvailabilityAggregatesByResourceTemplate = () => {
  * @param {Object} availabilityReport the RESO availability report JSON to process.
  * @returns a JSON availability report with the appropriate roll-ups and aggregates.
  */
-const processDataAvailability = async availabilityReport => {
+const processDataAvailability = async (availabilityReport, useNewReportFormat = true) => {
   //iterate over each field and lookup and compute their availabilities
   const { resources, fields, lookups, lookupValues, ...rest } = availabilityReport;
 
@@ -204,13 +204,15 @@ const processDataAvailability = async availabilityReport => {
 
   const { resourceCounts, expandedResourcesCounts } = resources.reduce(
     (acc, curr) => {
-      const { resourceName, numRecordsFetched, expansions } = curr;
-      acc.resourceCounts[resourceName] = numRecordsFetched;
+      const { resourceName, numRecordsFetched, expansions, numUniqueRecordsFetched } = curr;
+      const totalFetchedRecords = useNewReportFormat ? numUniqueRecordsFetched : numRecordsFetched;
+      acc.resourceCounts[resourceName] = totalFetchedRecords;
       expansions?.forEach(expansion => {
+        const totalFetchedExpandedRecords = useNewReportFormat ? expansion.numUniqueRecordsFetched : expansion.numRecordsFetched;
         if (!acc.expandedResourcesCounts[resourceName]) {
           acc.expandedResourcesCounts[resourceName] = {};
         }
-        acc.expandedResourcesCounts[resourceName][expansion.resourceName] = expansion.numRecordsFetched;
+        acc.expandedResourcesCounts[resourceName][expansion.resourceName] = totalFetchedExpandedRecords;
       });
       return acc;
     },
@@ -380,7 +382,7 @@ const processDataAvailability = async availabilityReport => {
     }
 
     const lookupsForField = isExpansion
-      ? lookupValueCache[parentResourceName].expansions?.[resourceName]?.[fieldName]
+      ? lookupValueCache[parentResourceName]?.expansions?.[resourceName]?.[fieldName]
       : lookupValueCache[resourceName]?.[fieldName];
 
     //only process if there are lookups for this field

diff --git a/test/sample-data-availability-reports/availability-report.js b/test/sample-data-availability-reports/availability-report.js
@@ -10,6 +10,7 @@ const daReport = {
       resourceName: 'Office',
       recordCount: 1000,
       numRecordsFetched: 1000,
+      numUniqueRecordsFetched: 1000,
       numSamples: 10,
       pageSize: 100,
       averageResponseBytes: 100000,
@@ -23,6 +24,7 @@ const daReport = {
       resourceName: 'Property',
       recordCount: 1000,
       numRecordsFetched: 1000,
+      numUniqueRecordsFetched: 1000,
       numSamples: 10,
       pageSize: 100,
       averageResponseBytes: 100000,
@@ -44,7 +46,9 @@ const daReport = {
           dateField: 'ModificationTimestamp',
           dateLow: '2021-03-15T10:12:32.319Z',
           dateHigh: '2022-03-15T10:12:32.319Z',
-          keyFields: ['MemberKey']
+          keyFields: ['MemberKey'],
+          numExpandedRecordsFetched: 2000,
+          numUniqueRecordsFetched: 2000
         }
       ]
     }

diff --git a/test/sample-data-availability-reports/expected-availability-report.json b/test/sample-data-availability-reports/expected-availability-report.json
@@ -37,21 +37,21 @@
     {
       "resourceName": "Media",
       "fieldName": "ListOfficeKey",
-      "availability": 1,
+      "availability": 0.5,
       "parentResourceName": "Property",
       "frequency": 1000
     },
     {
       "resourceName": "Media",
       "fieldName": "ResourceRecordKey",
-      "availability": 0.5,
+      "availability": 0.25,
       "parentResourceName": "Property",
       "frequency": 500
     },
     {
       "resourceName": "Media",
       "fieldName": "MediaModificationTimestamp",
-      "availability": 2,
+      "availability": 1,
       "parentResourceName": "Property",
       "frequency": 2000
     }
@@ -118,6 +118,7 @@
       "resourceName": "Office",
       "recordCount": 1000,
       "numRecordsFetched": 1000,
+      "numUniqueRecordsFetched": 1000,
       "numSamples": 10,
       "pageSize": 100,
       "averageResponseBytes": 100000,
@@ -131,6 +132,7 @@
       "resourceName": "Property",
       "recordCount": 1000,
       "numRecordsFetched": 1000,
+      "numUniqueRecordsFetched": 1000,
       "numSamples": 10,
       "pageSize": 100,
       "averageResponseBytes": 100000,
@@ -152,17 +154,19 @@
           "dateField": "ModificationTimestamp",
           "dateLow": "2021-03-15T10:12:32.319Z",
           "dateHigh": "2022-03-15T10:12:32.319Z",
-          "keyFields": ["MemberKey"]
+          "keyFields": ["MemberKey"],
+          "numExpandedRecordsFetched": 2000,
+          "numUniqueRecordsFetched": 2000
         }
       ]
     }
   ],
   "availability": {
     "fields": {
-      "total": { "eqZero": 0, "gtZero": 8, "gte25": 8, "gte50": 8, "gte75": 6, "eq100": 3 },
-      "reso": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 6, "gte75": 5, "eq100": 2 },
-      "idx": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 6, "gte75": 5, "eq100": 2 },
-      "local": { "eqZero": 0, "gtZero": 2, "gte25": 2, "gte50": 2, "gte75": 1, "eq100": 1 }
+      "total": { "eqZero": 0, "gtZero": 8, "gte25": 8, "gte50": 7, "gte75": 5, "eq100": 3 },
+      "reso": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 5, "gte75": 5, "eq100": 3 },
+      "idx": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 5, "gte75": 5, "eq100": 3 },
+      "local": { "eqZero": 0, "gtZero": 2, "gte25": 2, "gte50": 2, "gte75": 0, "eq100": 0 }
     },
     "lookups": {
       "total": { "eqZero": 0, "gtZero": 7, "gte25": 2, "gte50": 2, "gte75": 2, "eq100": 0 },
@@ -186,10 +190,10 @@
         "expansions": {
           "Media": {
             "fields": {
-              "total": 1.1666666666666667,
-              "reso": 1.25,
-              "local": 1,
-              "idx": 1.25
+              "total": 0.5833333333333334,
+              "reso": 0.625,
+              "local": 0.5,
+              "idx": 0.625
             },
             "lookups": {
               "total": 0,
@@ -250,33 +254,33 @@
                 "eqZero": 0,
                 "gtZero": 3,
                 "gte25": 3,
-                "gte50": 3,
-                "gte75": 2,
+                "gte50": 2,
+                "gte75": 1,
                 "eq100": 1
               },
               "reso": {
                 "eqZero": 0,
                 "gtZero": 2,
                 "gte25": 2,
-                "gte50": 2,
+                "gte50": 1,
                 "gte75": 1,
-                "eq100": 0
+                "eq100": 1
               },
               "idx": {
                 "eqZero": 0,
                 "gtZero": 2,
                 "gte25": 2,
-                "gte50": 2,
+                "gte50": 1,
                 "gte75": 1,
-                "eq100": 0
+                "eq100": 1
               },
               "local": {
                 "eqZero": 0,
                 "gtZero": 1,
                 "gte25": 1,
                 "gte50": 1,
-                "gte75": 1,
-                "eq100": 1
+                "gte75": 0,
+                "eq100": 0
               }
             },
             "lookups": {