Skip to content

Commit

Permalink
#54: Added basic availability report scoring and serialization
Browse files Browse the repository at this point in the history
  • Loading branch information
darnjo committed Sep 26, 2023
1 parent b487919 commit 171114a
Showing 1 changed file with 52 additions and 2 deletions.
54 changes: 52 additions & 2 deletions lib/replication/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,49 @@

const { replicationIterator, REPLICATION_STRATEGIES } = require('./replication-iterator');
const humanizeDuration = require('humanize-duration');
const { writeFile } = require('fs/promises');

const scorePayload = (records = [], availabilityMap = {}) => {
records.forEach(r => {
Object.entries(r).forEach(([k, v]) => {
if (!availabilityMap?.[k]) {
availabilityMap[k] = 0;
}

if (!!v) {
availabilityMap[k]++;
}
});
});

return availabilityMap;
};

const consolidateResults = (availabilityMap = {}, totalRecordCount = 0) => {
return {
totalRecordCount,
fields: Object.entries(availabilityMap ?? {}).map(([k, v]) => {
return { fieldName: k, frequency: v };
})
};
};

const writeDataAvailabilityReport = async (availabilityMap = {}, totalRecordCount = 0) => {
const AVAILABILITY_REPORT_FILENAME = 'data-availability-report.json';

try {
await writeFile(AVAILABILITY_REPORT_FILENAME, JSON.stringify({
description: 'RESO Data Availability Report',
version: '1.7',
generatedOn: new Date().toISOString(),
fields: consolidateResults(availabilityMap, totalRecordCount)
}, null, ' '));

console.log(`Results written to ${AVAILABILITY_REPORT_FILENAME}`);
} catch (err) {
console.error(err);
}
};

const replicate = async ({ url: requestUri, strategy, bearerToken, expansions }) => {
if (!Object.values(REPLICATION_STRATEGIES).includes(strategy)) {
Expand All @@ -20,17 +63,22 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, expansions })
const startTime = Date.now(),
responseTimes = [];

let recordsFetched = 0;
let recordsFetched = 0,
availabilityMap = {};

for await (const data of replicationIterator(config)) {
if (data?.hasResults) {
console.log('Data fetched!');
responseTimes.push(data?.responseTimeMs ?? 0);
recordsFetched = data?.recordsFetched ?? 0;

if (data?.response?.value?.length) {
availabilityMap = scorePayload(data.response.value, availabilityMap);
}
}
}

console.log(`\nReplication completed in ${humanizeDuration(Date.now() - startTime)}!`);
console.log(`\nReplication completed in ~${humanizeDuration(Date.now() - startTime, { round: true })}!`);
console.log(
`Total requests: ${responseTimes?.length}, Average response time: ${parseInt(
responseTimes?.reduce((acc, item) => {
Expand All @@ -41,6 +89,8 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, expansions })
}, 0) / responseTimes.length
)}ms, Total records: ${recordsFetched}\n`
);

await writeDataAvailabilityReport(availabilityMap, recordsFetched);
return;
};

Expand Down

0 comments on commit 171114a

Please sign in to comment.