#54: Added additional error handling and logging

RESOStandards · Oct 1, 2023 · 5e62676 · 5e62676
1 parent d1bd8c5
commit 5e62676
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 53 deletions.
diff --git a/lib/replication/index.js b/lib/replication/index.js
@@ -1,7 +1,7 @@
 'use strict';
 
 const { replicationIterator, REPLICATION_STRATEGIES } = require('./replication-iterator');
-const { scorePayload, writeDataAvailabilityReport } = require('./utils');
+const { scorePayload, writeDataAvailabilityReport, ERROR_TYPES } = require('./utils');
 const { writeFile, mkdir } = require('fs/promises');
 const { join } = require('path');
 const humanizeDuration = require('humanize-duration');
@@ -70,52 +70,65 @@ const replicate = async ({ url: requestUri, strategy, bearerToken, outputPath, l
       totalRecordsFetched = 0,
       pagesFetched = 0
     } of replicationIterator({ requestUri, strategy, authInfo: { bearerToken } })) {
-      if (hasError) {
-        const { statusCode, message, errorType, error: errorData } = error;
-        if (errorType === 'http') {
-          console.error(`HTTP request error! Status code: ${statusCode}, message: '${message}'`);
-        } else {
-          let errorString = null;
-          try {
-            errorString = JSON.stringify(JSON.parse(errorData));
-          } catch (err) {
-            errorString = err?.toString ? err.toString() : '<unknown>';
+      try {
+        if (hasError) {
+          const { statusCode, message, errorType, error: errorData } = error;
+          if (errorType === ERROR_TYPES.HTTP) {
+            console.error(`HTTP request error! Status code: ${statusCode}, message: '${message}'`);
+          } else {
+            let errorString = null;
+            try {
+              errorString = JSON.stringify(JSON.parse(errorData));
+            } catch (err) {
+              errorString = err?.toString ? err.toString() : err.cause || '<unknown>';
+            }
+            throw new Error(`${errorType} error occurred! Error: ${errorString}`);
           }
-          throw new Error(`${errorType} error occurred! Error: ${errorString}`);
         }
-      }
 
-      if (hasResults) {
-        if (response?.value?.length) {
-          scorePayload({
-            requestUri,
-            records: response.value,
-            resourceAvailabilityMap, // mutated on each call
-            resourceName,
-            expansions,
-            responseTimeMs,
-            startTime,
-            stopTime
-          });
-
-          if (shouldSaveResults) {
-            await mkdir(resultsPath, { recursive: true });
-            await writeFile(join(resultsPath, `page-${pagesFetched}.json`), JSON.stringify(response));
+        if (hasResults) {
+          if (response?.value?.length) {
+            scorePayload({
+              requestUri,
+              records: response.value,
+              resourceAvailabilityMap, // mutated on each call
+              resourceName,
+              expansions,
+              responseTimeMs,
+              startTime,
+              stopTime
+            });
+
+            if (shouldSaveResults) {
+              await mkdir(resultsPath, { recursive: true });
+              await writeFile(join(resultsPath, `page-${pagesFetched}.json`), JSON.stringify(response));
+            }
           }
-        }
 
-        totalRecordCount = totalRecordsFetched;
-        responseTimes.push(responseTimeMs);
+          totalRecordCount = totalRecordsFetched;
+          responseTimes.push(responseTimeMs);
+        }
+      } catch (err) {
+        //TODO: add logic to allow fast fail, in which case we'd return here
+        console.error(err);
       }
 
       if (!!limit && totalRecordsFetched >= limit) {
         break;
       }
     }
-    await writeDataAvailabilityReport({ version: '1.7', resourceAvailabilityMap });
   } catch (err) {
+    //TODO: add logic to allow fast fail, in which case we'd return here
     console.error(err);
   } finally {
+
+    // try writing report
+    try {
+      await writeDataAvailabilityReport({ version: '1.7', resourceAvailabilityMap });
+    } catch (err) {
+      console.error(`Could not write data availability report! ${err}`);
+    }
+
     console.log(`\nReplication completed in ${humanizeDuration(Date.now() - startTime, { round: false })}!`);
     console.log(
       `Total requests: ${responseTimes?.length || 0}, Average response time: ${humanizeDuration(

diff --git a/lib/replication/replication-iterator.js b/lib/replication/replication-iterator.js
@@ -1,20 +1,21 @@
 'use strict';
 
 const queryString = require('node:querystring');
-const { processHttpErrorResponse } = require('./utils');
+const { processHttpErrorResponse, ERROR_TYPES } = require('./utils');
 const humanizeDuration = require('humanize-duration');
 
 const MAX_RECORD_COUNT_DEFAULT = 100000,
   DEFAULT_PAGE_SIZE = 100,
   ODATA_VALUE_PROPERTY_NAME = 'value',
   ODATA_NEXT_LINK_PROPERTY_NAME = '@odata.nextLink';
-  
+
 // See: https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_HeaderPrefer
 // The value of the Prefer header is a comma-separated list of preferences.
-const ODATA_PREFER_HEADER_NAME = 'Prefer', ODATA_MAX_PAGE_SIZE_HEADER_NAME = 'odata.maxpagesize';
+const ODATA_PREFER_HEADER_NAME = 'Prefer',
+  ODATA_MAX_PAGE_SIZE_HEADER_NAME = 'odata.maxpagesize';
 
 // See: https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#_Toc31358888
-// The value of the Preference-Applied header is a comma-separated list of preferences applied in the response. 
+// The value of the Preference-Applied header is a comma-separated list of preferences applied in the response.
 // const ODATA_PREFERENCE_APPLIED_HEADER_NAME = 'Preference-Applied';
 
 const REPLICATION_STRATEGIES = Object.freeze({
@@ -26,8 +27,8 @@ const REPLICATION_STRATEGIES = Object.freeze({
 
 /**
  * Creates a bearer token auth header, i.e. "Authorization: Bearer <token>"
- * 
- * @param {String} token bearer token to be used for a given HTTP request 
+ *
+ * @param {String} token bearer token to be used for a given HTTP request
  * @returns a header constructed from the given token, or an empty object if the token is invalid
  */
 const getBearerTokenAuthHeader = (token = '') => (token?.length ? { Authorization: `Bearer ${token}` } : {});
@@ -58,25 +59,24 @@ const buildRequestUri = ({ requestUri, strategy, totalRecordsFetched = 0, pageSi
 
 /**
  * Replication iterator which maintains some internal state during runtime.
- * 
+ *
  * Will keep iterating until there are no more records or the max number of errors has been reached.
  *
  * It's the client's responsibility to determine when to stop iterating
- * 
+ *
  * @param {Object} config configuration for the replication iterator
  * @returns yields with a number of parameters relevant to replication
  */
 async function* replicationIterator(config = {}) {
   const { requestUri: initialRequestUri = '', maxErrorCount = 3, authInfo = {}, strategy } = config;
 
-  const { bearerToken, /* TODO clientCredentials */ } = authInfo;
+  const { bearerToken /* TODO clientCredentials */ } = authInfo;
 
-  let
-    pageSize = DEFAULT_PAGE_SIZE,
+  let pageSize = DEFAULT_PAGE_SIZE,
     pagesFetched = 0,
     numErrors = 0,
     totalRecordsFetched = 0;
-    
+
   //GET https://api.reso.org/Property
   let requestUri = initialRequestUri,
     lastRequestUri = null,
@@ -115,20 +115,20 @@ async function* replicationIterator(config = {}) {
     }
 
     let responseTimeMs = 0,
-      startTime, stopTime;
+      startTime,
+      stopTime;
 
     try {
-
       //request records
       console.log(`Fetching records from '${requestUri}'...`);
 
       const startTimeMs = new Date();
       const response = await fetch(requestUri, { headers });
       const stopTimeMs = new Date();
-    
+
       startTime = startTimeMs.toISOString();
       stopTime = stopTimeMs.toISOString();
-    
+
       //TODO: legacy property - deprecate
       responseTimeMs = stopTimeMs - startTimeMs;
 
@@ -153,7 +153,7 @@ async function* replicationIterator(config = {}) {
         }
         pagesFetched++;
       } else {
-        stopTime = new Date().toISOString;
+        stopTime = new Date().toISOString();
         error = {
           errorType: 'http',
           ...processHttpErrorResponse(response)
@@ -164,7 +164,7 @@ async function* replicationIterator(config = {}) {
       console.error(err);
       numErrors++;
       error = {
-        errorType: 'general',
+        errorType: ERROR_TYPES.GENERAL,
         ...err
       };
     }

diff --git a/lib/replication/utils.js b/lib/replication/utils.js
@@ -2,6 +2,11 @@
 
 const { writeFile } = require('fs/promises');
 
+const ERROR_TYPES = {
+  HTTP: 'http',
+  GENERAL: 'general'
+};
+
 /**
  * Scores a payload with the given data
  * @param {Object} data options to be extracted from the caller
@@ -224,7 +229,6 @@ const writeDataAvailabilityReport = async ({ version, resourceAvailabilityMap =
   }
 
 */
-
 /**
  * Processes an HTTP error response from the Fetch API
  * @param {Response} response the HTTP error response from the Fetch API
@@ -240,5 +244,6 @@ const processHttpErrorResponse = ({ status, statusText }) => {
 module.exports = {
   scorePayload,
   writeDataAvailabilityReport,
-  processHttpErrorResponse
+  processHttpErrorResponse,
+  ERROR_TYPES
 };