From fcc675d73fcd5716c2c7af24f4c5035aee32394d Mon Sep 17 00:00:00 2001 From: chulanovskyi Date: Thu, 18 Jul 2024 13:41:50 +0300 Subject: [PATCH] fix: handle the missing Type of the column --- forward_engineering/api.js | 5 ++-- reverse_engineering/api.js | 23 +++++++++++++------ .../helpers/connectionHelper.js | 4 ++-- .../helpers/tablePropertiesHelper.js | 21 +++++++++++++++-- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index 5de3cfe..5c7a2c7 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -1,6 +1,6 @@ 'use strict'; -const { GlueClient, CreateDatabaseCommand, CreateTableCommand } = require('@aws-sdk/client-glue'); +const { GlueClient, CreateDatabaseCommand, CreateTableCommand, GetDatabasesCommand } = require('@aws-sdk/client-glue'); const { getDatabaseStatement } = require('./helpers/databaseHelper'); const { getTableStatement } = require('./helpers/tableHelper'); const { getIndexes } = require('./helpers/indexHelper'); @@ -169,7 +169,8 @@ module.exports = { const glueInstance = getGlueInstance(connectionInfo, app); try { - await glueInstance.getDatabases().promise(); + const command = new GetDatabasesCommand(); + await glueInstance.send(command); callback(); } catch (err) { logger.log('error', { message: err.message, stack: err.stack, error: err }, 'Connection failed'); diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index 02d9b9e..02d3ef7 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -28,7 +28,7 @@ module.exports = { logInfo('Test connection', connectionInfo, logger); const connection = await this.connect(connectionInfo); - const instance = connectionHelper.createInstance(connection, dependencies.lodash); + const instance = connectionHelper.createInstance({ connection, _: dependencies.lodash, logger }); try { await instance.getDatabases(); @@ -46,7 +46,7 @@ module.exports = { try { const connection = await this.connect(connectionInfo); - const instance = connectionHelper.createInstance(connection, dependencies.lodash); + const instance = connectionHelper.createInstance({ connection, _: dependencies.lodash, logger }); const { databaseList, isFullyUploaded } = await instance.getDatabases(); const dbsCollections = databaseList.map(async db => { const dbCollections = await instance.getTables(db.Name); @@ -89,7 +89,7 @@ module.exports = { try { const connection = await this.connect(data); - const instance = connectionHelper.createInstance(connection, dependencies.lodash); + const instance = connectionHelper.createInstance({ connection, _: dependencies.lodash, logger }); const tablesDataPromise = databases.map(async dbName => { const dbDescription = await instance.getDatabaseDescription(dbName); @@ -102,7 +102,10 @@ module.exports = { }); const tableData = await instance.getTable(dbName, tableName); - const jsonSchema = getColumnsSchema([...tableData.columns, ...tableData.partitionKeys]); + const jsonSchema = getColumnsSchema({ + columns: [...tableData.columns, ...tableData.partitionKeys], + logger, + }); return { dbName, @@ -191,10 +194,16 @@ const handleErrorObject = (error, title) => { return { title, ...errorProperties }; }; -const getColumnsSchema = columns => { +const getColumnsSchema = ({ columns, logger }) => { return columns.reduce((acc, item) => { - const sanitizedTypeString = item.type.replace(/\s/g, ''); - let columnSchema = schemaHelper.getJsonSchema(sanitizedTypeString); + if (!item) { + return acc; + } + if (!item.type) { + logger.log('info', item, 'Column Type is missing, fallback to string'); + } + const sanitizedTypeString = item.type?.replace(/\s/g, '') || 'string'; + const columnSchema = schemaHelper.getJsonSchema(sanitizedTypeString); schemaHelper.setProperty(item.name, columnSchema, acc); return acc; }, {}); diff --git a/reverse_engineering/helpers/connectionHelper.js b/reverse_engineering/helpers/connectionHelper.js index 99539dd..6d7806d 100644 --- a/reverse_engineering/helpers/connectionHelper.js +++ b/reverse_engineering/helpers/connectionHelper.js @@ -100,7 +100,7 @@ const close = () => { } }; -const createInstance = (connection, _) => { +const createInstance = ({ connection, _ = {}, logger = {} }) => { const getDatabases = async () => { const command = new GetDatabasesCommand({ MaxResults: MAX_RESULTS, NextToken: databaseLoadContinuationToken }); const dbsData = await connection.send(command); @@ -141,7 +141,7 @@ const createInstance = (connection, _) => { const rawTableData = await connection.send(command); - return mapTableData(rawTableData, _); + return mapTableData({ tableData: rawTableData, _, logger }); }; return { diff --git a/reverse_engineering/helpers/tablePropertiesHelper.js b/reverse_engineering/helpers/tablePropertiesHelper.js index 602bf14..e2d9640 100644 --- a/reverse_engineering/helpers/tablePropertiesHelper.js +++ b/reverse_engineering/helpers/tablePropertiesHelper.js @@ -59,7 +59,24 @@ const getNumBuckets = (numBuckets = 0) => { return numBuckets < 1 ? undefined : numBuckets; }; -const mapTableData = (tableData, _) => { +const mapColumns = ({ columns = [], logger = {} }) => { + let hasErrors = false; + + const mapped = columns.map(({ Type, Name }) => { + if (!Type || !Name) { + hasErrors = true; + } + return { name: Name, type: Type }; + }); + + if (hasErrors) { + logger.log('info', columns, 'Some columns are missing required Type or Name'); + } + + return mapped; +}; + +const mapTableData = ({ tableData, _, logger }) => { const partitionKeys = tableData.Table.PartitionKeys || []; return { @@ -83,7 +100,7 @@ const mapTableData = (tableData, _) => { classification: getClassification(tableData.Table.Parameters), }, partitionKeys: tableData.Table.PartitionKeys || [], - columns: tableData.Table.StorageDescriptor.Columns.map(({ Type, Name }) => ({ name: Name, type: Type })), + columns: mapColumns({ columns: tableData.Table.StorageDescriptor.Columns, logger }), }; };