#28: sync with s3 when restoring raw reports

RESOStandards · Aug 23, 2023 · 6ea6551 · 6ea6551
1 parent 12fecfa
commit 6ea6551
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 7 deletions.
diff --git a/data-access/cert-api-client.js b/data-access/cert-api-client.js
@@ -1,8 +1,10 @@
 'use strict';
 
 const axios = require('axios');
+const chalk = require('chalk');
 require('dotenv').config();
 const { CERTIFICATION_API_KEY, ORGS_DATA_URL, SYSTEMS_DATA_URL } = process.env;
+const AWS = require('aws-sdk');
 
 const API_DEBOUNCE_SECONDS = 0.1;
 
@@ -107,7 +109,7 @@ const processDataDictionaryResults = async ({
 };
 
 const getOrgsMap = async () => {
-  const { Data: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data;
+  const { Organizations: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data;
 
   if (!orgs?.length) throw new Error('ERROR: could not fetch Org data!');
 
@@ -157,10 +159,58 @@ const getOrgSystemsMap = async () => {
   }, {});
 };
 
+const postToS3 = async ({ metadataReport, dataAvailabilityReport, s3Path, bucketName }) => {
+  if (!s3Path) {
+    throw new Error('Invalid S3 path');
+  }
+  if (!bucketName) {
+    throw new Error('Invalid bucket name');
+  }
+
+  AWS.config.update({
+    accessKeyId: process.env.ACCESS_KEY,
+    secretAccessKey: process.env.YOUR_SECRET_KEY,
+    region: process.env.AWS_REGION
+  });
+
+  const s3 = new AWS.S3();
+
+  const promises = [];
+
+  if (metadataReport) {
+    const params = {
+      Bucket: bucketName,
+      Key: `${s3Path}/metadata-report.json`,
+      Body: metadataReport
+    };
+    promises.push(s3.upload(params).promise());
+  }
+
+  if (dataAvailabilityReport) {
+    const params = {
+      Bucket: bucketName,
+      Key: `${s3Path}/data-availability-report.json`,
+      Body: dataAvailabilityReport
+    };
+    promises.push(s3.upload(params).promise());
+  }
+
+  const response = await Promise.allSettled(promises);
+
+  response.forEach(r => {
+    if (r.status === 'fulfilled') {
+      console.log(chalk.green(`File uploaded successfully. ${r.value.Location}`));
+    } else {
+      console.error(chalk.redBright.bold(`Error uploading file to S3: ${r.reason}`));
+    }
+  });
+};
+
 module.exports = {
   processDataDictionaryResults,
   getOrgsMap,
   getOrgSystemsMap,
   findDataDictionaryReport,
-  sleep
+  sleep,
+  postToS3
 };
diff --git a/index.js b/index.js
@@ -12,6 +12,7 @@ program
   .command('restore')
   .option('-p, --pathToResults <string>', 'Path to test results')
   .option('-u, --url <string>', 'URL of Certification API')
+  .option('-b, --bucketName <string>', 'Name of the backup S3 bucket')
   .description('Restores local or S3 results to a RESO Certification API instance')
   .action(restore);
 

diff --git a/utils/restore-utils/index.js b/utils/restore-utils/index.js
@@ -7,7 +7,8 @@ const { resolve, join } = require('path');
 const {
   getOrgsMap,
   getOrgSystemsMap,
-  processDataDictionaryResults
+  processDataDictionaryResults,
+  postToS3
 } = require('../../data-access/cert-api-client');
 
 const { processLookupResourceMetadataFiles } = require('reso-certification-etl');
@@ -104,7 +105,7 @@ const restore = async (options = {}) => {
     missingResultsFilePaths: []
   };
 
-  const { pathToResults, url } = options;
+  const { pathToResults, url, bucketName = 'reso-backup' } = options;
 
   if (isS3Path(pathToResults)) {
     console.log(
@@ -254,6 +255,18 @@ const restore = async (options = {}) => {
                     metadataReport,
                     dataAvailabilityReport
                   });
+
+                  await postToS3({
+                    metadataReport: Object.keys(metadataReport).length
+                      ? JSON.stringify(metadataReport)
+                      : null,
+                    dataAvailabilityReport: Object.keys(dataAvailabilityReport).length
+                      ? JSON.stringify(dataAvailabilityReport)
+                      : null,
+                    s3Path: `data_dictionary-${metadataReport.version}/${providerUoi}-${providerUsi}/${recipientUoi}`,
+                    bucketName
+                  });
+
                   console.log(chalk.bold(`Done! Result: ${result ? 'Succeeded!' : 'Failed!'}`));
                 } catch (err) {
                   console.log(chalk.bgRed.bold(err));
@@ -283,9 +296,7 @@ const restore = async (options = {}) => {
   console.log(chalk.bold(`Processing complete! Time Taken: ~${timeTaken}s`));
   console.log(chalk.magentaBright.bold('------------------------------------------------------------'));
 
-  console.log(
-    chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`)
-  );
+  console.log(chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`));
   STATS.processed.forEach(item => console.log(chalk.bold(`\t * ${item}`)));
 
   console.log(chalk.bold(`\nProvider UOI Paths Skipped: ${STATS.skippedProviderUoiPaths.length}`));