#54: Added rate limiting and general non-fatal HTTP error handling su…

…pport
RESOStandards · Oct 5, 2023 · 0cbf39f · 0cbf39f
1 parent d63c478
commit 0cbf39f
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 15 deletions.
diff --git a/common.js b/common.js
@@ -277,7 +277,11 @@ const createResoScriptClientCredentialsConfig = ({ serviceRootUri, clientCredent
   '  </ClientSettings>' +
   '</OutputScript>';
 
+const sleep = async (ms = 500) => new Promise(resolve => setTimeout(resolve, ms));
+
 module.exports = {
+  CURRENT_DATA_DICTIONARY_VERSION,
+  CURRENT_WEB_API_CORE_VERSION,
   endorsements,
   availableVersions,
   isValidEndorsement,
@@ -290,6 +294,5 @@ module.exports = {
   archiveEndorsement,
   getCurrentVersion,
   getPreviousVersion,
-  CURRENT_DATA_DICTIONARY_VERSION,
-  CURRENT_WEB_API_CORE_VERSION
+  sleep
 };
diff --git a/lib/misc/data-access/cert-api-client.js b/lib/misc/data-access/cert-api-client.js
@@ -3,6 +3,7 @@
 const axios = require('axios');
 require('dotenv').config();
 const { CERTIFICATION_API_KEY, ORGS_DATA_URL, SYSTEMS_DATA_URL } = process.env;
+const { sleep } = require('../../../common');
 
 const API_DEBOUNCE_SECONDS = 0.1;
 
@@ -72,8 +73,6 @@ const postDataAvailabilityResultsToApi = async ({ url, reportId, dataAvailabilit
   }
 };
 
-const sleep = async ms => new Promise(resolve => setTimeout(resolve, ms));
-
 const processDataDictionaryResults = async ({
   url,
   providerUoi,
@@ -161,6 +160,5 @@ module.exports = {
   processDataDictionaryResults,
   getOrgsMap,
   getOrgSystemsMap,
-  findDataDictionaryReport,
-  sleep
+  findDataDictionaryReport
 };
diff --git a/lib/orgs-exporter/data-access.js b/lib/orgs-exporter/data-access.js
@@ -1,6 +1,6 @@
 const https = require('https');
 
-const { sleep } = require('./utils');
+const { sleep } = require('../../common');
 
 const REQUEST_TIMEOUT_MS = (process.env.REQUEST_TIMEOUT_S || 30) * 1000;
 const { API_KEY, SERVER_URL, ENDORSEMENTS_PATH, UOI_GOOGLE_SHEET_URL } = process.env;

diff --git a/lib/orgs-exporter/utils.js b/lib/orgs-exporter/utils.js
@@ -21,9 +21,6 @@ const writeDataToS3 = async ({
   await s3.putObject(params).promise();
 };
 
-const sleep = async (ms = 500) => new Promise(resolve => setTimeout(resolve, ms));
-
 module.exports = {
-  writeDataToS3,
-  sleep
+  writeDataToS3
 };
diff --git a/lib/replication/index.js b/lib/replication/index.js
@@ -33,7 +33,8 @@ const replicate = async ({
   pathToMetadataReportJson = '',
   filter,
   top,
-  orderby
+  orderby,
+  rateLimitedWaitTimeMinutes = 60
 }) => {
   if (!Object.values(REPLICATION_STRATEGIES).includes(strategy)) {
     throw new Error(`Unknown strategy: '${strategy}'!`);
@@ -57,9 +58,11 @@ const replicate = async ({
     shouldSaveResults = !!outputPath,
     resourceAvailabilityMap = {};
 
+  // Each resource and expansion will have its separate set of requests
   for await (const request of requests) {
     const { requestUri: initialRequestUri } = request;
 
+    // each item queried has its own set of requests
     try {
       for await (const {
         hasResults = false,
@@ -76,7 +79,8 @@ const replicate = async ({
         try {
           //handle errors
           if (hasError) {
-            handleError(error);
+            // some errors, like HTTP 429, might be able to be handled
+            await handleError({ error, rateLimitedWaitTimeMinutes });
           }
 
           //process results

diff --git a/lib/replication/utils.js b/lib/replication/utils.js
@@ -3,6 +3,7 @@
 const { readFile, writeFile } = require('fs/promises');
 const { join } = require('path');
 const humanizeDuration = require('humanize-duration');
+const { sleep } = require('../../common');
 
 const REPLICATION_DIRECTORY_NAME = 'reso-replication-output';
 
@@ -11,6 +12,12 @@ const ERROR_TYPES = {
   GENERAL: 'general'
 };
 
+const ERROR_CODES = {
+  HTTP: {
+    RATE_LIMITED: 429
+  }
+};
+
 const NOT_OK = 1;
 
 const DEFAULT_DD_VERSION = '1.7',
@@ -478,11 +485,21 @@ const calculateMedian = (numbers = []) => {
  * Handles error objects
  * @param {Object} error the error to process
  */
-const handleError = error => {
+const handleError = async ({ error = {}, rateLimitedWaitTimeMinutes = 60 }) => {
   const { statusCode, message, errorType, error: errorData } = error;
   if (errorType === ERROR_TYPES.HTTP) {
+    // some HTTP errors can potentially be handled
     console.error(`HTTP request error! Status code: ${statusCode}, message: '${message}'`);
+
+    if (parseInt(statusCode) === ERROR_CODES.HTTP.RATE_LIMITED) {
+      console.warn(`${Date.now().toISOString()} - HTTP ${ERROR_CODES.HTTP.RATE_LIMITED} error!`);
+      console.warn(`\t -> Waiting ${rateLimitedWaitTimeMinutes}m before making the next request...`);
+      await sleep(rateLimitedWaitTimeMinutes * 1000);
+    }
+
+    return { errorType, statusCode, message };
   } else {
+    // throw an error for all other errors
     let errorString = null;
     try {
       errorString = JSON.stringify(JSON.parse(errorData));
@@ -519,7 +536,7 @@ const createRequestsFromMetadataReport = ({ serviceRootUri, metadataReportJson =
 
   return requests.map(({ resourceName, fieldName, isExpansion, typeName }) => {
     const expansions = isExpansion ? [fieldName] : undefined;
-    const expandedFields = isExpansion ? [ { fieldName, typeName } ] : undefined;
+    const expandedFields = isExpansion ? [{ fieldName, typeName }] : undefined;
     return {
       requestUri: createODataRequestUri({ serviceRootUri, resourceName, expansions, filter, top, orderby }),
       resourceName,