diff --git a/README.md b/README.md index fb42d32..463f789 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,12 @@ add package `@reso/reso-certification-utils` and yarn install --check-files. Aft ## Batch Test Runner Runs a batch of tests using a provided configuration file. -[**MORE INFO**](./utils/batch-test-runner/README.md) +[**MORE INFO**](./lib/batch-test-runner/README.md) ## Restore Certification Server Restores a RESO Certification API Server using a set of existing results in a given directory structure. -[**MORE INFO**](./utils/restore-utils/README.md) +[**MORE INFO**](./lib/restore-utils/README.md) ## Tests diff --git a/common.js b/common.js index 1742719..f8fc995 100644 --- a/common.js +++ b/common.js @@ -44,14 +44,10 @@ const availableVersions = { }; const getCurrentVersion = endorsementName => - endorsementName && - availableVersions[endorsementName] && - availableVersions[endorsementName].currentVersion; + endorsementName && availableVersions[endorsementName] && availableVersions[endorsementName].currentVersion; const getPreviousVersion = endorsementName => - endorsementName && - availableVersions[endorsementName] && - availableVersions[endorsementName].previousVersion; + endorsementName && availableVersions[endorsementName] && availableVersions[endorsementName].previousVersion; /** * Determines whether the given endorsementName is valid. @@ -60,8 +56,7 @@ const getPreviousVersion = endorsementName => * @returns true if the endorsementName is valid, false otherwise. * @throws error if parameters aren't valid */ -const isValidEndorsement = endorsementName => - endorsementName && !!availableVersions[endorsementName]; +const isValidEndorsement = endorsementName => endorsementName && !!availableVersions[endorsementName]; /** * Determines whether the version is valid for the given endorsement. @@ -90,9 +85,7 @@ const getEndorsementMetadata = (endorsementName, version) => { } if (!isValidVersion(endorsementName, version)) { - throw new Error( - `Invalid endorsement version! endorsementKey: ${endorsementName}, version: ${version}` - ); + throw new Error(`Invalid endorsement version! endorsementKey: ${endorsementName}, version: ${version}`); } const ddVersion = version || CURRENT_DATA_DICTIONARY_VERSION, @@ -104,10 +97,7 @@ const getEndorsementMetadata = (endorsementName, version) => { version: `${ddVersion}`, /* TODO: add versions to JSON results file names in the Commander */ jsonResultsFiles: [METADATA_REPORT_JSON, DATA_AVAILABILITY_REPORT_JSON], - htmlReportFiles: [ - `data-dictionary-${ddVersion}.html`, - `data-availability.dd-${ddVersion}.html` - ], + htmlReportFiles: [`data-dictionary-${ddVersion}.html`, `data-availability.dd-${ddVersion}.html`], logFileName: COMMANDER_LOG_FILE_NAME }; } @@ -117,11 +107,7 @@ const getEndorsementMetadata = (endorsementName, version) => { directoryName: `${endorsements.DATA_DICTIONARY_WITH_IDX}`, version: `${version}`, /* TODO: add versions to JSON results file names in the Commander */ - jsonResultsFiles: [ - METADATA_REPORT_JSON, - DATA_AVAILABILITY_REPORT_JSON, - IDX_DIFFERENCE_REPORT_JSON - ], + jsonResultsFiles: [METADATA_REPORT_JSON, DATA_AVAILABILITY_REPORT_JSON, IDX_DIFFERENCE_REPORT_JSON], htmlReportFiles: [ `data-dictionary-${ddVersion}.html`, `data-availability.dd-${ddVersion}.html`, @@ -191,15 +177,10 @@ const buildRecipientEndorsementPath = ({ if (!endorsementName) throw Error('endorsementName is required!'); if (!version) throw Error('version is required!'); - if (!isValidEndorsement(endorsementName)) - throw new Error(`Invalid endorsementName: ${endorsementName}`); + if (!isValidEndorsement(endorsementName)) throw new Error(`Invalid endorsementName: ${endorsementName}`); if (!isValidVersion(endorsementName, version)) throw new Error(`Invalid version: ${version}`); - return path.join( - `${providerUoi}-${providerUsi}`, - recipientUoi, - currentOrArchived - ); + return path.join(`${providerUoi}-${providerUsi}`, recipientUoi, currentOrArchived); }; /** @@ -212,13 +193,7 @@ const buildRecipientEndorsementPath = ({ * @param {String} providerUsi * @param {String} recipientUoi */ -const archiveEndorsement = ({ - providerUoi, - providerUsi, - recipientUoi, - endorsementName, - version -} = {}) => { +const archiveEndorsement = ({ providerUoi, providerUsi, recipientUoi, endorsementName, version } = {}) => { const currentRecipientPath = buildRecipientEndorsementPath({ providerUoi, providerUsi, @@ -270,13 +245,20 @@ const createResoScriptClientCredentialsConfig = ({ serviceRootUri, clientCredent ` ${clientCredentials.clientSecret}` + ` ${clientCredentials.tokenUri}` + ` ${ - clientCredentials.scope - ? '' + clientCredentials.scope + '' - : EMPTY_STRING + clientCredentials.scope ? '' + clientCredentials.scope + '' : EMPTY_STRING }` + ' ' + ''; +const checkFileExists = async filePath => { + try { + await fs.promises.access(filePath); + return true; + } catch (err) { + return false; + } +}; + module.exports = { endorsements, availableVersions, @@ -291,5 +273,6 @@ module.exports = { getCurrentVersion, getPreviousVersion, CURRENT_DATA_DICTIONARY_VERSION, - CURRENT_WEB_API_CORE_VERSION + CURRENT_WEB_API_CORE_VERSION, + checkFileExists }; diff --git a/index.js b/index.js index c6b153f..11c8665 100755 --- a/index.js +++ b/index.js @@ -1,7 +1,8 @@ #! /usr/bin/env node const { program } = require('commander'); -const { restore } = require('./utils/restore-utils'); -const { runTests } = require('./utils/batch-test-runner'); +const { restore } = require('./lib/restore-utils/data-dictionary'); +const { runTests } = require('./lib/batch-test-runner'); +const { syncWebApi } = require('./lib/restore-utils/web-api-core'); program .name('reso-certification-utils') @@ -12,13 +13,27 @@ program .command('restore') .option('-p, --pathToResults ', 'Path to test results') .option('-u, --url ', 'URL of Certification API') + .option('--deleteExistingReport', 'Flag to delete existing report') .description('Restores local or S3 results to a RESO Certification API instance') .action(restore); +program + .command('syncWebApiResults') + .option('-p, --pathToResults ', 'Path to test results') + .option('-u, --url ', 'URL of Certification API') + .option('-r, --recipients ', 'Comma-separated list of recipient orgs') + .option('-i, --system ', 'Unique system identifier') + .option('--deleteExistingReport', 'Flag to delete existing report') + .description('Restores local or S3 Web API results to a RESO Certification API instance') + .action(syncWebApi); + program .command('runDDTests') .requiredOption('-p, --pathToConfigFile ', 'Path to config file') - .option('-a, --runAvailability', 'Flag to run data availability tests, otherwise only metadata tests are run') + .option( + '-a, --runAvailability', + 'Flag to run data availability tests, otherwise only metadata tests are run' + ) .description('Runs Data Dictionary tests') .action(runTests); diff --git a/utils/batch-test-runner/README.md b/lib/batch-test-runner/README.md similarity index 100% rename from utils/batch-test-runner/README.md rename to lib/batch-test-runner/README.md diff --git a/utils/batch-test-runner/index.js b/lib/batch-test-runner/index.js similarity index 100% rename from utils/batch-test-runner/index.js rename to lib/batch-test-runner/index.js diff --git a/utils/batch-test-runner/sample-dd-config.json b/lib/batch-test-runner/sample-dd-config.json similarity index 100% rename from utils/batch-test-runner/sample-dd-config.json rename to lib/batch-test-runner/sample-dd-config.json diff --git a/data-access/cert-api-client.js b/lib/misc/data-access/cert-api-client.js similarity index 58% rename from data-access/cert-api-client.js rename to lib/misc/data-access/cert-api-client.js index 7c9c7c6..f1c4de5 100644 --- a/data-access/cert-api-client.js +++ b/lib/misc/data-access/cert-api-client.js @@ -46,6 +46,66 @@ const postDataDictionaryResultsToApi = async ({ } }; +const postWebAPIResultsToApi = async ({ + url, + providerUoi, + providerUsi, + recipientUoi, + webAPIReport = {} +} = {}) => { + if (!url) throw new Error('url is required!'); + if (!providerUoi) throw new Error('providerUoi is required!'); + if (!providerUsi) throw new Error('providerUsi is required!'); + if (!recipientUoi) throw new Error('recipientUoi is required!'); + if (!Object.keys(webAPIReport)?.length) throw new Error('web API report is empty!'); + + try { + const { id: reportId = null } = + ( + await axios.post( + `${url}/api/v1/certification_reports/web_api_server_core/${providerUoi}`, + webAPIReport, + { + maxContentLength: Infinity, + maxBodyLength: Infinity, + headers: { + Authorization: `ApiKey ${CERTIFICATION_API_KEY}`, + recipientUoi, + 'Content-Type': 'application/json', + providerUsi + } + } + ) + ).data || {}; + + if (!reportId) throw new Error('Did not receive the required id parameter from the response!'); + + return reportId; + } catch (err) { + throw new Error(`Could not post web API results to API! ${err}`); + } +}; + +const deleteExistingDDOrWebAPIReport = async ({ url, reportId } = {}) => { + if (!url) throw new Error('url is required!'); + if (!reportId) throw new Error('reportId is required!'); + + try { + const { deletedReport } = + ( + await axios.delete(`${url}/api/v1/certification_reports/${reportId}`, { + headers: { + Authorization: `ApiKey ${CERTIFICATION_API_KEY}` + } + }) + ).data || {}; + + return deletedReport; + } catch (err) { + throw new Error(err); + } +}; + const postDataAvailabilityResultsToApi = async ({ url, reportId, dataAvailabilityReport = {} } = {}) => { if (!url) throw new Error('url is required!'); if (!reportId) throw new Error('reportId is required!'); @@ -106,8 +166,27 @@ const processDataDictionaryResults = async ({ } }; +const processWebAPIResults = async ({ url, providerUoi, providerUsi, recipientUoi, webAPIReport = {} }) => { + try { + //wait for the dust to settle to avoid thrashing the server + await sleep(API_DEBOUNCE_SECONDS * 1000); + + const reportId = await postWebAPIResultsToApi({ + url, + providerUoi, + providerUsi, + recipientUoi, + webAPIReport + }); + + return reportId; + } catch (err) { + throw new Error(`Could not process webAPI results! ${err}`); + } +}; + const getOrgsMap = async () => { - const { Data: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data; + const { Organizations: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data; if (!orgs?.length) throw new Error('ERROR: could not fetch Org data!'); @@ -149,6 +228,29 @@ const findDataDictionaryReport = async ({ serverUrl, providerUoi, providerUsi, r } }; +const findWebAPIReport = async ({ serverUrl, providerUoi, providerUsi, recipientUoi } = {}) => { + const url = `${serverUrl}/api/v1/certification_reports/summary/${recipientUoi}`, + config = { + headers: { + Authorization: `ApiKey ${CERTIFICATION_API_KEY}` + } + }; + + try { + const { data = [] } = await axios.get(url, config); + + return data.find( + item => + item?.type === 'web_api_server_core' && + item?.providerUoi === providerUoi && + //provider USI isn't in the data set at the moment, only filter if it's present + (item?.providerUsi ? item.providerUsi === providerUsi : true) + ); + } catch (err) { + throw new Error(`Could not connect to ${url}`); + } +}; + const getOrgSystemsMap = async () => { return (await axios.get(SYSTEMS_DATA_URL))?.data?.values.slice(1).reduce((acc, [providerUoi, , usi]) => { if (!acc[providerUoi]) acc[providerUoi] = []; @@ -157,10 +259,22 @@ const getOrgSystemsMap = async () => { }, {}); }; +const getSystemsMap = async () => { + return (await axios.get(SYSTEMS_DATA_URL))?.data?.values.slice(1).reduce((acc, [providerUoi, , usi]) => { + acc[usi] = providerUoi; + return acc; + }, {}); +}; + module.exports = { processDataDictionaryResults, getOrgsMap, getOrgSystemsMap, findDataDictionaryReport, - sleep + sleep, + findWebAPIReport, + postWebAPIResultsToApi, + processWebAPIResults, + getSystemsMap, + deleteExistingDDOrWebAPIReport }; diff --git a/orgs-exporter/data-access.js b/lib/orgs-exporter/data-access.js similarity index 100% rename from orgs-exporter/data-access.js rename to lib/orgs-exporter/data-access.js diff --git a/orgs-exporter/index.js b/lib/orgs-exporter/index.js similarity index 100% rename from orgs-exporter/index.js rename to lib/orgs-exporter/index.js diff --git a/orgs-exporter/utils.js b/lib/orgs-exporter/utils.js similarity index 100% rename from orgs-exporter/utils.js rename to lib/orgs-exporter/utils.js diff --git a/utils/restore-utils/README.md b/lib/restore-utils/README.md similarity index 100% rename from utils/restore-utils/README.md rename to lib/restore-utils/README.md diff --git a/utils/restore-utils/index.js b/lib/restore-utils/data-dictionary.js similarity index 84% rename from utils/restore-utils/index.js rename to lib/restore-utils/data-dictionary.js index 0d31b20..eff0a3c 100644 --- a/utils/restore-utils/index.js +++ b/lib/restore-utils/data-dictionary.js @@ -7,13 +7,18 @@ const { resolve, join } = require('path'); const { getOrgsMap, getOrgSystemsMap, - processDataDictionaryResults -} = require('../../data-access/cert-api-client'); + processDataDictionaryResults, + findDataDictionaryReport, + deleteExistingDDOrWebAPIReport, + sleep +} = require('../misc/data-access/cert-api-client'); const { processLookupResourceMetadataFiles } = require('reso-certification-etl'); +const { checkFileExists } = require('../../common'); const CERTIFICATION_RESULTS_DIRECTORY = 'current', - PATH_DATA_SEPARATOR = '-'; + PATH_DATA_SEPARATOR = '-', + OVERWRITE_DELAY_S = 10; const CERTIFICATION_FILES = { METADATA_REPORT: 'metadata-report.json', @@ -89,7 +94,9 @@ const isValidUrl = url => { /** * Restores a RESO Certification Server from either a local or S3 path. - * @param {String} path + * @param {Object} options + * @param {string} options.pathToResults An absolute local path to the DD results or a valid S3 path. + * @param {string} options.url Cert API base URL. * @throws Error if path is not a valid S3 or local path */ const restore = async (options = {}) => { @@ -104,7 +111,21 @@ const restore = async (options = {}) => { missingResultsFilePaths: [] }; - const { pathToResults, url } = options; + const { pathToResults, url, deleteExistingReport = false } = options; + + if (deleteExistingReport) { + console.log( + chalk.bgYellowBright.bold( + 'WARNING: --deleteExistingReport flag is set! This can delete an already certified report!' + ) + ); + console.log( + chalk.bold( + `Waiting ${OVERWRITE_DELAY_S} seconds before proceeding...use to exit if this was unintended!` + ) + ); + await sleep(OVERWRITE_DELAY_S * 1000); + } if (isS3Path(pathToResults)) { console.log( @@ -221,11 +242,13 @@ const restore = async (options = {}) => { pathToOutputFile = resolve( join(currentResultsPath, CERTIFICATION_FILES.PROCESSED_METADATA_REPORT) ); - await processLookupResourceMetadataFiles( - pathToMetadataReport, - pathToLookupResourceData, - pathToOutputFile - ); + if (!(await checkFileExists(pathToOutputFile))) { + await processLookupResourceMetadataFiles( + pathToMetadataReport, + pathToLookupResourceData, + pathToOutputFile + ); + } } const metadataReport = @@ -245,6 +268,25 @@ const restore = async (options = {}) => { await readFile(join(currentResultsPath, CERTIFICATION_FILES.DATA_AVAILABILITY_REPORT)) ) || {}; + if (deleteExistingReport) { + const report = + (await findDataDictionaryReport({ + serverUrl: url, + providerUoi, + providerUsi, + recipientUoi + })) || {}; + + const { id: reportId = null } = report; + if (reportId) { + console.log(chalk.yellowBright.bold('Deleting existing report...')); + await deleteExistingDDOrWebAPIReport({ + url, + reportId + }); + } + } + console.log('Ingesting results...'); const result = await processDataDictionaryResults({ url, @@ -270,7 +312,6 @@ const restore = async (options = {}) => { } } } - console.log(); } else { console.log(chalk.bgRedBright.bold(`Invalid path: ${pathToResults}! \nMust be valid S3 or local path`)); } @@ -283,9 +324,7 @@ const restore = async (options = {}) => { console.log(chalk.bold(`Processing complete! Time Taken: ~${timeTaken}s`)); console.log(chalk.magentaBright.bold('------------------------------------------------------------')); - console.log( - chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`) - ); + console.log(chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`)); STATS.processed.forEach(item => console.log(chalk.bold(`\t * ${item}`))); console.log(chalk.bold(`\nProvider UOI Paths Skipped: ${STATS.skippedProviderUoiPaths.length}`)); diff --git a/lib/restore-utils/web-api-core.js b/lib/restore-utils/web-api-core.js new file mode 100644 index 0000000..1d408d9 --- /dev/null +++ b/lib/restore-utils/web-api-core.js @@ -0,0 +1,203 @@ +//const conf = new (require('conf'))(); +const chalk = require('chalk'); +const { promises: fs } = require('fs'); +const { checkFileExists } = require('../../common'); +const { + getOrgsMap, + getOrgSystemsMap, + findWebAPIReport, + processWebAPIResults, + getSystemsMap, + deleteExistingDDOrWebAPIReport, + sleep +} = require('../misc/data-access/cert-api-client'); + +const OVERWRITE_DELAY_S = 10; + +const readFile = async filePath => { + try { + return await fs.readFile(filePath); + } catch (err) { + console.error(`Could not read file from path '${filePath}'! Error: ${err}`); + } +}; + +/** + * Determines whether the given path is an S3 path + * @param {String} path the path to test + * @returns true if S3 path, false otherwise + */ +const isS3Path = (path = '') => path.trim().toLowerCase().startsWith('s3://'); + +/** + * Determines whether the given path is a valid local file path + * @param {String} path the path to test + * @returns true if valid local path, false otherwise + */ +const isLocalPath = (path = '') => !isS3Path(path); + +const fetchOrgData = async () => { + //fetch org data + console.log(chalk.cyanBright.bold('\nFetching org data...')); + const orgMap = (await getOrgsMap()) || {}; + if (!Object.keys(orgMap)?.length) throw new Error('Error: could not fetch orgs!'); + console.log(chalk.cyanBright.bold('Done!')); + return orgMap; +}; + +const fetchSystemData = async () => { + //fetch system data + console.log(chalk.cyanBright.bold('\nFetching system data...')); + const orgSystemMap = (await getOrgSystemsMap()) || {}; + if (!Object.keys(orgSystemMap)?.length) throw new Error('Error: could not fetch systems!'); + console.log(chalk.cyanBright.bold('Done!')); + return orgSystemMap; +}; + +const isValidUrl = url => { + try { + new URL(url); + return true; + } catch (err) { + console.log(chalk.redBright.bold(`Error: Cannot parse given url: ${url}`)); + return false; + } +}; + +/** + * @param {Object} options + * @param {string} options.pathToResults An absolute local path to the WebAPI results or a valid S3 path. + * @param {string} options.url Cert API base URL. + * @param {string} options.recipients Comma seperated string of recipient uoi's. + * @param {string} options.system System name for the provider. + * @param {boolean} options.deleteExistingReport Flag to force delete an existing web API report + * @throws Error if path is not a valid S3 or local path + */ +const syncWebApi = async (options = {}) => { + const START_TIME = new Date(); + + const { pathToResults, url, recipients = '', system = '', deleteExistingReport = false } = options; + + if (deleteExistingReport) { + console.log( + chalk.bgYellowBright.bold( + 'WARNING: --deleteExistingReport flag is set! This can delete an already certified report!' + ) + ); + console.log( + chalk.bold( + `Waiting ${OVERWRITE_DELAY_S} seconds before proceeding...use to exit if this was unintended!` + ) + ); + await sleep(OVERWRITE_DELAY_S * 1000); + } + + if (isS3Path(pathToResults)) { + console.log( + chalk.yellowBright.bold(`S3 path provided but not supported at this time!\nPath: ${pathToResults}`) + ); + return; + } + + if (!isValidUrl(url)) return; + + const recipientsList = recipients.split(','); + if (!recipientsList.length || !recipientsList[0]) { + console.log(chalk.redBright.bold(`Error: The recipient string '${recipients}' is invalid`)); + return; + } + console.log(chalk.bold(`\nCertification API URL: ${url}`)); + console.log(chalk.bold(`Path to results: ${pathToResults}`)); + console.log(chalk.bold(`Recipients: ${recipientsList}`)); + + if (isLocalPath(pathToResults)) { + const orgMap = await fetchOrgData(); + const orgSystemMap = await fetchSystemData(); + const systemMap = await getSystemsMap(); + + console.log(chalk.greenBright.bold('\nRestore process starting...\n')); + + const providerUoi = systemMap[system]; + + //is provider UOI valid? + if (!orgMap[providerUoi]) { + console.warn(chalk.redBright.bold(`Error: Could not find providerUoi '${providerUoi}'! Exiting...`)); + return; + } + + //is provider USI valid? + const systems = orgSystemMap[providerUoi] || []; + if (!systems?.length) { + console.warn( + chalk.redBright.bold(`Error: Could not find systems for providerUoi '${providerUoi}'! Exiting...`) + ); + return; + } + + if (!systems?.includes(system)) { + console.log(`Error: Could not find system ${system} for providerUoi '${providerUoi}'! Exiting...`); + return; + } + + const fileExists = await checkFileExists(pathToResults); + + if (!fileExists) { + console.log(chalk.redBright.bold(`Error: Could not find file in path '${pathToResults}'`)); + return; + } + + const webAPIReport = JSON.parse(await readFile(pathToResults)) || {}; + + for await (const recipientUoi of recipientsList) { + try { + if (deleteExistingReport) { + //search for existing results + const report = + (await findWebAPIReport({ + serverUrl: url, + providerUoi, + providerUsi: system, + recipientUoi + })) || {}; + + const { id: reportId = null } = report; + if (reportId) { + console.log(chalk.yellowBright.bold('Deleting existing report...')); + await deleteExistingDDOrWebAPIReport({ + url, + reportId + }); + } + } + + console.log('Ingesting results...'); + const result = await processWebAPIResults({ + url, + providerUoi, + providerUsi: system, + recipientUoi, + webAPIReport: webAPIReport + }); + + console.log(chalk.bold(`Result: ${result ? 'Succeeded!' : 'Failed!'}`)); + } catch (err) { + console.log(chalk.bgRed.bold(err)); + } + } + } else { + console.log(chalk.bgRedBright.bold(`Invalid path: ${pathToResults}! \nMust be valid S3 or local path`)); + } + + const timeTaken = Math.round((new Date() - START_TIME) / 1000); + + console.log(chalk.magentaBright.bold('------------------------------------------------------------')); + + console.log(chalk.bold(`Time Taken: ~${timeTaken}s`)); + console.log(chalk.magentaBright.bold('------------------------------------------------------------')); + + console.log(chalk.bold('\nRestore complete! Exiting...\n')); +}; + +module.exports = { + syncWebApi +};