diff --git a/lib/process-data-availability.js b/lib/process-data-availability.js index 0363b5a..8965549 100644 --- a/lib/process-data-availability.js +++ b/lib/process-data-availability.js @@ -195,7 +195,7 @@ const getAvailabilityAggregatesByResourceTemplate = () => { * @param {Object} availabilityReport the RESO availability report JSON to process. * @returns a JSON availability report with the appropriate roll-ups and aggregates. */ -const processDataAvailability = async availabilityReport => { +const processDataAvailability = async (availabilityReport, useNewReportFormat = true) => { //iterate over each field and lookup and compute their availabilities const { resources, fields, lookups, lookupValues, ...rest } = availabilityReport; @@ -204,13 +204,15 @@ const processDataAvailability = async availabilityReport => { const { resourceCounts, expandedResourcesCounts } = resources.reduce( (acc, curr) => { - const { resourceName, numRecordsFetched, expansions } = curr; - acc.resourceCounts[resourceName] = numRecordsFetched; + const { resourceName, numRecordsFetched, expansions, numUniqueRecordsFetched } = curr; + const totalFetchedRecords = useNewReportFormat ? numUniqueRecordsFetched : numRecordsFetched; + acc.resourceCounts[resourceName] = totalFetchedRecords; expansions?.forEach(expansion => { + const totalFetchedExpandedRecords = useNewReportFormat ? expansion.numUniqueRecordsFetched : expansion.numRecordsFetched; if (!acc.expandedResourcesCounts[resourceName]) { acc.expandedResourcesCounts[resourceName] = {}; } - acc.expandedResourcesCounts[resourceName][expansion.resourceName] = expansion.numRecordsFetched; + acc.expandedResourcesCounts[resourceName][expansion.resourceName] = totalFetchedExpandedRecords; }); return acc; }, @@ -380,7 +382,7 @@ const processDataAvailability = async availabilityReport => { } const lookupsForField = isExpansion - ? lookupValueCache[parentResourceName].expansions?.[resourceName]?.[fieldName] + ? lookupValueCache[parentResourceName]?.expansions?.[resourceName]?.[fieldName] : lookupValueCache[resourceName]?.[fieldName]; //only process if there are lookups for this field diff --git a/test/sample-data-availability-reports/availability-report.js b/test/sample-data-availability-reports/availability-report.js index 1a38693..6b2c4e3 100644 --- a/test/sample-data-availability-reports/availability-report.js +++ b/test/sample-data-availability-reports/availability-report.js @@ -10,6 +10,7 @@ const daReport = { resourceName: 'Office', recordCount: 1000, numRecordsFetched: 1000, + numUniqueRecordsFetched: 1000, numSamples: 10, pageSize: 100, averageResponseBytes: 100000, @@ -23,6 +24,7 @@ const daReport = { resourceName: 'Property', recordCount: 1000, numRecordsFetched: 1000, + numUniqueRecordsFetched: 1000, numSamples: 10, pageSize: 100, averageResponseBytes: 100000, @@ -44,7 +46,9 @@ const daReport = { dateField: 'ModificationTimestamp', dateLow: '2021-03-15T10:12:32.319Z', dateHigh: '2022-03-15T10:12:32.319Z', - keyFields: ['MemberKey'] + keyFields: ['MemberKey'], + numExpandedRecordsFetched: 2000, + numUniqueRecordsFetched: 2000 } ] } diff --git a/test/sample-data-availability-reports/expected-availability-report.json b/test/sample-data-availability-reports/expected-availability-report.json index 69a4287..3d0ed04 100644 --- a/test/sample-data-availability-reports/expected-availability-report.json +++ b/test/sample-data-availability-reports/expected-availability-report.json @@ -37,21 +37,21 @@ { "resourceName": "Media", "fieldName": "ListOfficeKey", - "availability": 1, + "availability": 0.5, "parentResourceName": "Property", "frequency": 1000 }, { "resourceName": "Media", "fieldName": "ResourceRecordKey", - "availability": 0.5, + "availability": 0.25, "parentResourceName": "Property", "frequency": 500 }, { "resourceName": "Media", "fieldName": "MediaModificationTimestamp", - "availability": 2, + "availability": 1, "parentResourceName": "Property", "frequency": 2000 } @@ -118,6 +118,7 @@ "resourceName": "Office", "recordCount": 1000, "numRecordsFetched": 1000, + "numUniqueRecordsFetched": 1000, "numSamples": 10, "pageSize": 100, "averageResponseBytes": 100000, @@ -131,6 +132,7 @@ "resourceName": "Property", "recordCount": 1000, "numRecordsFetched": 1000, + "numUniqueRecordsFetched": 1000, "numSamples": 10, "pageSize": 100, "averageResponseBytes": 100000, @@ -152,17 +154,19 @@ "dateField": "ModificationTimestamp", "dateLow": "2021-03-15T10:12:32.319Z", "dateHigh": "2022-03-15T10:12:32.319Z", - "keyFields": ["MemberKey"] + "keyFields": ["MemberKey"], + "numExpandedRecordsFetched": 2000, + "numUniqueRecordsFetched": 2000 } ] } ], "availability": { "fields": { - "total": { "eqZero": 0, "gtZero": 8, "gte25": 8, "gte50": 8, "gte75": 6, "eq100": 3 }, - "reso": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 6, "gte75": 5, "eq100": 2 }, - "idx": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 6, "gte75": 5, "eq100": 2 }, - "local": { "eqZero": 0, "gtZero": 2, "gte25": 2, "gte50": 2, "gte75": 1, "eq100": 1 } + "total": { "eqZero": 0, "gtZero": 8, "gte25": 8, "gte50": 7, "gte75": 5, "eq100": 3 }, + "reso": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 5, "gte75": 5, "eq100": 3 }, + "idx": { "eqZero": 0, "gtZero": 6, "gte25": 6, "gte50": 5, "gte75": 5, "eq100": 3 }, + "local": { "eqZero": 0, "gtZero": 2, "gte25": 2, "gte50": 2, "gte75": 0, "eq100": 0 } }, "lookups": { "total": { "eqZero": 0, "gtZero": 7, "gte25": 2, "gte50": 2, "gte75": 2, "eq100": 0 }, @@ -186,10 +190,10 @@ "expansions": { "Media": { "fields": { - "total": 1.1666666666666667, - "reso": 1.25, - "local": 1, - "idx": 1.25 + "total": 0.5833333333333334, + "reso": 0.625, + "local": 0.5, + "idx": 0.625 }, "lookups": { "total": 0, @@ -250,33 +254,33 @@ "eqZero": 0, "gtZero": 3, "gte25": 3, - "gte50": 3, - "gte75": 2, + "gte50": 2, + "gte75": 1, "eq100": 1 }, "reso": { "eqZero": 0, "gtZero": 2, "gte25": 2, - "gte50": 2, + "gte50": 1, "gte75": 1, - "eq100": 0 + "eq100": 1 }, "idx": { "eqZero": 0, "gtZero": 2, "gte25": 2, - "gte50": 2, + "gte50": 1, "gte75": 1, - "eq100": 0 + "eq100": 1 }, "local": { "eqZero": 0, "gtZero": 1, "gte25": 1, "gte50": 1, - "gte75": 1, - "eq100": 1 + "gte75": 0, + "eq100": 0 } }, "lookups": {