#54: Added improved error handling and updated readme

RESOStandards · Oct 1, 2023 · d1bd8c5 · d1bd8c5
1 parent 90cfeef
commit d1bd8c5
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 59 deletions.
diff --git a/index.js b/index.js
@@ -44,7 +44,7 @@ if (require?.main === module) {
     .option('-b, --bearerToken <string>', 'Bearer token to use for authorization')
     .option('-p, --pathToConfigFile', 'Path to config containing credentials')
     .option('-r, --resourceName <string>', 'Resource name to replicate data from')
-    .option('-x, --expansions [string...]', 'Items to expand during the query process, e.g. Media')
+    .option('-x, --expansions <items>', 'Comma-separated list of items to expand during the query process, e.g. Media,OpenHouse')
     .option('-m, --metadataReportPath <string>', 'Path to metadata report to use for replication')
     .option('-o, --outputPath <string>', 'Name of directory for results')
     .option('-l, --limit <number>', 'Limit for total number of records')

diff --git a/lib/replication/README.md b/lib/replication/README.md
@@ -12,18 +12,32 @@ Usage: reso-certification-utils replicate [options]
 Replicates data from a given resource with expansions.
 
 Options:
-  -s, --strategy <string>      One of TopAndSkip, ModificationTimestampAsc, ModificationTimestampDesc, or NextLink
-  -u, --url <string>           The URL to start replicating from
-  -b, --bearerToken <string>   Bearer token to use for authorization
-  -p, --pathToConfigFile       Path to config containing credentials
-  -r, --resourceName <string>  Resource name to replicate data from
-  -x, --expansions <string>    Items to expand during the query process, e.g. Media
-  -m, --metadataReportPath     Path to metadata report to use for replication
-  -h, --help                   display help for command
+  -s, --strategy <string>            One of TopAndSkip, ModificationTimestampAsc, ModificationTimestampDesc, or NextLink
+  -u, --url <string>                 The URL to start replicating from
+  -b, --bearerToken <string>         Bearer token to use for authorization
+  -p, --pathToConfigFile             Path to config containing credentials
+  -r, --resourceName <string>        Resource name to replicate data from
+  -x, --expansions <items>           Comma-separated list of items to expand during the query process, e.g. Media,OpenHouse
+  -m, --metadataReportPath <string>  Path to metadata report to use for replication
+  -o, --outputPath <string>          Name of directory for results
+  -l, --limit <number>               Limit for total number of records
+  -h, --help                         display help for command
 ```
 
 ## Example: Replicate Data from a URL Using `TopAndSkip`
 Replicating from `https://some.api.com/Property` can be done as follows:
 ```
 $ reso-certification-utils replicate -s TopAndSkip -u https://some.api.com/Property -b <your test token>
 ```
+
+## Example: Replicate Data from the Property Resource Using `NextLink` with a Media and OpenHouse Expansion
+Replicating from `https://some.api.com/Property` can be done as follows:
+```
+$ reso-certification-utils replicate -s NextLink -u https://some.api.com/Property -b <your test token> -x Media,OpenHouse
+```
+
+You can also use the expand query directly, without the `-x` option. In that case, the dollar sign in `$expand` needs to be escaped.
+```
+$ reso-certification-utils replicate -s NextLink -u https://some.api.com/Property?\$expand=Media,OpenHouse -b <your test token>
+```
+Note the `\` before the `$expand`. Shells sometimes require escape sequences for special characters like `$`.
diff --git a/lib/replication/index.js b/lib/replication/index.js
@@ -8,7 +8,15 @@ const humanizeDuration = require('humanize-duration');
 
 const REPLICATION_DIRECTORY_NAME = 'reso-replication-output';
 
-// need to get the last part of the URL before the querystring
+/**
+ *
+ * Parses the OData resource name from the given URI
+ *
+ * Example: https://some.api.com/v2/Property
+ *
+ * @param {String} requestUri the string for the OData request URI
+ * @returns OData resource name or null
+ */
 const parseResourceNameFromODataRequestUri = (requestUri = '') => {
   try {
     const [resourceName = null] = new URL(requestUri).pathname.split('/').slice(-1);
@@ -19,19 +27,30 @@ const parseResourceNameFromODataRequestUri = (requestUri = '') => {
   }
 };
 
+/**
+ * Builds a path to use when saving results
+ *
+ * @param {String} outputPath the target directory in which to save results (current by default)
+ * @param {*} resourceName the name of the Data Dictionary resource whose files are being saved
+ * @returns an operating system dependent path to save replication data with
+ */
 const buildOutputFilePath = (outputPath, resourceName) =>
   join(outputPath, REPLICATION_DIRECTORY_NAME, resourceName, new Date().toISOString().replaceAll(':', '-'));
 
-const replicate = async ({ url: requestUri, strategy, bearerToken, outputPath, limit, expansions = ['Media' /* TODO */ ] }) => {
+/**
+ * Replicates data from the given OData request URL using the given strategy, credentials, and options
+ *
+ * @param {Object} args this function takes multiple parameters
+ * @returns this function has no return value, but will produce side effects if outputPath is used (will write files)
+ */
+const replicate = async ({ url: requestUri, strategy, bearerToken, outputPath, limit, expansions = ['Media' /* TODO */] }) => {
   if (!Object.values(REPLICATION_STRATEGIES).includes(strategy)) {
     throw new Error(`Unknown strategy: '${strategy}'!`);
   }
 
   const resourceName = parseResourceNameFromODataRequestUri(requestUri),
     shouldSaveResults = !!outputPath,
-    resultsPath = shouldSaveResults
-      ? buildOutputFilePath(outputPath, resourceName)
-      : null;
+    resultsPath = shouldSaveResults ? buildOutputFilePath(outputPath, resourceName) : null;
 
   const startTime = Date.now(),
     responseTimes = [],
@@ -50,14 +69,19 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, outputPath, l
       stopTime = 0,
       totalRecordsFetched = 0,
       pagesFetched = 0
-    } of replicationIterator({ requestUri, strategy, authInfo: { bearerToken }})) {
-
+    } of replicationIterator({ requestUri, strategy, authInfo: { bearerToken } })) {
       if (hasError) {
         const { statusCode, message, errorType, error: errorData } = error;
         if (errorType === 'http') {
           console.error(`HTTP request error! Status code: ${statusCode}, message: '${message}'`);
         } else {
-          throw new Error(`${errorType} error occurred! Error data: ${JSON.stringify(errorData, null, '  ')}`);
+          let errorString = null;
+          try {
+            errorString = JSON.stringify(JSON.parse(errorData));
+          } catch (err) {
+            errorString = err?.toString ? err.toString() : '<unknown>';
+          }
+          throw new Error(`${errorType} error occurred! Error: ${errorString}`);
         }
       }
 
@@ -79,7 +103,7 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, outputPath, l
             await writeFile(join(resultsPath, `page-${pagesFetched}.json`), JSON.stringify(response));
           }
         }
-        
+
         totalRecordCount = totalRecordsFetched;
         responseTimes.push(responseTimeMs);
       }
@@ -94,14 +118,16 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, outputPath, l
   } finally {
     console.log(`\nReplication completed in ${humanizeDuration(Date.now() - startTime, { round: false })}!`);
     console.log(
-      `Total requests: ${responseTimes?.length || 0}, Average response time: ${humanizeDuration(parseInt(
-        responseTimes?.reduce((acc, item) => {
-          if (item) {
-            acc += item;
-          }
-          return acc;
-        }, 0) / (responseTimes.length || 1)
-      ))}, Total records: ${totalRecordCount}\n`
+      `Total requests: ${responseTimes?.length || 0}, Average response time: ${humanizeDuration(
+        parseInt(
+          responseTimes?.reduce((acc, item) => {
+            if (item) {
+              acc += item;
+            }
+            return acc;
+          }, 0) / (responseTimes.length || 1)
+        )
+      )}, Total records: ${totalRecordCount}\n`
     );
   }
   return;

diff --git a/lib/replication/replication-iterator.js b/lib/replication/replication-iterator.js
@@ -15,7 +15,7 @@ const ODATA_PREFER_HEADER_NAME = 'Prefer', ODATA_MAX_PAGE_SIZE_HEADER_NAME = 'od
 
 // See: https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#_Toc31358888
 // The value of the Preference-Applied header is a comma-separated list of preferences applied in the response. 
-const ODATA_PREFERENCE_APPLIED_HEADER_NAME = 'Preference-Applied';
+// const ODATA_PREFERENCE_APPLIED_HEADER_NAME = 'Preference-Applied';
 
 const REPLICATION_STRATEGIES = Object.freeze({
   TOP_AND_SKIP: 'TopAndSkip',
@@ -24,9 +24,15 @@ const REPLICATION_STRATEGIES = Object.freeze({
   NEXT_LINK: 'NextLink'
 });
 
+/**
+ * Creates a bearer token auth header, i.e. "Authorization: Bearer <token>"
+ * 
+ * @param {String} token bearer token to be used for a given HTTP request 
+ * @returns a header constructed from the given token, or an empty object if the token is invalid
+ */
 const getBearerTokenAuthHeader = (token = '') => (token?.length ? { Authorization: `Bearer ${token}` } : {});
 
-const buildRequestUri = ({ requestUri, strategy, totalRecordsFetched = 0, pageSize, lastIsoTimestamp, nextLink }) => {
+const buildRequestUri = ({ requestUri, strategy, totalRecordsFetched = 0, pageSize, /* TODO lastIsoTimestamp, */ nextLink }) => {
   const [baseUri = null, query = null] = requestUri.split('?');
 
   const queryParams = query !== null ? queryString.parse(query) : {};
@@ -50,10 +56,20 @@ const buildRequestUri = ({ requestUri, strategy, totalRecordsFetched = 0, pageSi
   }
 };
 
+/**
+ * Replication iterator which maintains some internal state during runtime.
+ * 
+ * Will keep iterating until there are no more records or the max number of errors has been reached.
+ *
+ * It's the client's responsibility to determine when to stop iterating
+ * 
+ * @param {Object} config configuration for the replication iterator
+ * @returns yields with a number of parameters relevant to replication
+ */
 async function* replicationIterator(config = {}) {
   const { requestUri: initialRequestUri = '', maxErrorCount = 3, authInfo = {}, strategy } = config;
 
-  const { bearerToken, clientCredentials } = authInfo;
+  const { bearerToken, /* TODO clientCredentials */ } = authInfo;
 
   let
     pageSize = DEFAULT_PAGE_SIZE,
@@ -64,7 +80,7 @@ async function* replicationIterator(config = {}) {
   //GET https://api.reso.org/Property
   let requestUri = initialRequestUri,
     lastRequestUri = null,
-    lastIsoTimestamp = null,
+    // TODO lastIsoTimestamp = null,
     nextLink = null;
 
   // TODO: handle client credentials auth
@@ -88,7 +104,7 @@ async function* replicationIterator(config = {}) {
       strategy,
       totalRecordsFetched,
       pageSize,
-      lastIsoTimestamp,
+      // TODO lastIsoTimestamp,
       nextLink
     });
 

diff --git a/lib/replication/utils.js b/lib/replication/utils.js
@@ -2,6 +2,13 @@
 
 const { writeFile } = require('fs/promises');
 
+/**
+ * Scores a payload with the given data
+ * @param {Object} data options to be extracted from the caller
+ * 
+ * Note that this function mutates the resourceAvailabilityMap, which 
+ * the caller would pass in and have declared in the calling function
+ */
 const scorePayload = ({
   requestUri = '',
   records = [],
@@ -100,6 +107,13 @@ const scorePayload = ({
   });
 };
 
+/**
+ * Processes data, keyed by resources, fields, and enumerations, into its 
+ * first round of aggregation
+ * 
+ * @param {Map} resourceAvailabilityMap map containing availability data
+ * @returns consolidated availability data set in canonical resources, fields, lookups format
+ */
 const consolidateResults = (resourceAvailabilityMap = {}) =>
   Object.values(resourceAvailabilityMap ?? {}).reduce(
     (acc, resourceData) => {
@@ -119,34 +133,11 @@ const consolidateResults = (resourceAvailabilityMap = {}) =>
     }
   );
 
-/*
-  {
-    "description": "RESO Data Availability Report",
-    "version": "1.7",
-    "generatedOn": "2023-09-11T17:37:06.066Z",
-    "resources": [
-      {
-        "resourceName": "Office",
-        "recordCount": 1751,
-        "numRecordsFetched": 1709,
-        "numSamples": 18,
-        "pageSize": 100,
-        "averageResponseBytes": 106953,
-        "averageResponseTimeMs": 547,
-        "dateField": "ModificationTimestamp",
-        "dateLow": "2019-08-14T13:59:06Z",
-        "dateHigh": "2023-08-28T13:46:24Z",
-        "keyFields": [
-          "OfficeKey"  //TODO: key fields needs to be in the metadata report instead
-        ]
-      },
-    "fields": [
-
-    ], 
-    "lookups": [],
-    "lookupValues": []
-  }
-*/
+/**
+ * Writes a data-availability-report.json file for the given version and availability data
+ * 
+ * @param {Object} params 
+ */
 const writeDataAvailabilityReport = async ({ version, resourceAvailabilityMap = {} }) => {
   const AVAILABILITY_REPORT_FILENAME = 'data-availability-report.json';
 
@@ -233,6 +224,12 @@ const writeDataAvailabilityReport = async ({ version, resourceAvailabilityMap =
   }
 
 */
+
+/**
+ * Processes an HTTP error response from the Fetch API
+ * @param {Response} response the HTTP error response from the Fetch API
+ * @returns relevant error data
+ */
 const processHttpErrorResponse = ({ status, statusText }) => {
   return {
     statusCode: status,