HCK-7285: missing column type

forward_engineering/api.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { GlueClient, CreateDatabaseCommand, CreateTableCommand } = require('@aws-sdk/client-glue');
+const { GlueClient, CreateDatabaseCommand, CreateTableCommand, GetDatabasesCommand } = require('@aws-sdk/client-glue');
 const { getDatabaseStatement } = require('./helpers/databaseHelper');
 const { getTableStatement } = require('./helpers/tableHelper');
 const { getIndexes } = require('./helpers/indexHelper');
@@ -169,7 +169,8 @@ module.exports = {
 		const glueInstance = getGlueInstance(connectionInfo, app);
 
 		try {
-			await glueInstance.getDatabases().promise();
+			const command = new GetDatabasesCommand();
+			await glueInstance.send(command);
 			callback();
 		} catch (err) {
 			logger.log('error', { message: err.message, stack: err.stack, error: err }, 'Connection failed');

reverse_engineering/api.js

@@ -28,7 +28,7 @@ module.exports = {
 		logInfo('Test connection', connectionInfo, logger);
 
 		const connection = await this.connect(connectionInfo);
-		const instance = connectionHelper.createInstance(connection, dependencies.lodash);
+		const instance = connectionHelper.createInstance({ connection, _: dependencies.lodash, logger });
 
 		try {
 			await instance.getDatabases();
@@ -46,7 +46,7 @@ module.exports = {
 
 		try {
 			const connection = await this.connect(connectionInfo);
-			const instance = connectionHelper.createInstance(connection, dependencies.lodash);
+			const instance = connectionHelper.createInstance({ connection, _: dependencies.lodash, logger });
 			const { databaseList, isFullyUploaded } = await instance.getDatabases();
 			const dbsCollections = databaseList.map(async db => {
 				const dbCollections = await instance.getTables(db.Name);
@@ -89,7 +89,7 @@ module.exports = {
 
 		try {
 			const connection = await this.connect(data);
-			const instance = connectionHelper.createInstance(connection, dependencies.lodash);
+			const instance = connectionHelper.createInstance({ connection, _: dependencies.lodash, logger });
 
 			const tablesDataPromise = databases.map(async dbName => {
 				const dbDescription = await instance.getDatabaseDescription(dbName);
@@ -102,7 +102,10 @@ module.exports = {
 					});
 
 					const tableData = await instance.getTable(dbName, tableName);
-					const jsonSchema = getColumnsSchema([...tableData.columns, ...tableData.partitionKeys]);
+					const jsonSchema = getColumnsSchema({
+						columns: [...tableData.columns, ...tableData.partitionKeys],
+						logger,
+					});
 
 					return {
 						dbName,
@@ -191,10 +194,16 @@ const handleErrorObject = (error, title) => {
 	return { title, ...errorProperties };
 };
 
-const getColumnsSchema = columns => {
+const getColumnsSchema = ({ columns, logger }) => {
 	return columns.reduce((acc, item) => {
-		const sanitizedTypeString = item.type.replace(/\s/g, '');
-		let columnSchema = schemaHelper.getJsonSchema(sanitizedTypeString);
+		if (!item) {
+			return acc;
+		}
+		if (!item.type) {
+			logger.log('info', item, 'Column Type is missing, fallback to string');
+		}
+		const sanitizedTypeString = item.type?.replace(/\s/g, '') || 'string';
+		const columnSchema = schemaHelper.getJsonSchema(sanitizedTypeString);
 		schemaHelper.setProperty(item.name, columnSchema, acc);
 		return acc;
 	}, {});

reverse_engineering/helpers/connectionHelper.js

@@ -100,7 +100,7 @@ const close = () => {
 	}
 };
 
-const createInstance = (connection, _) => {
+const createInstance = ({ connection, _ = {}, logger = {} }) => {
 	const getDatabases = async () => {
 		const command = new GetDatabasesCommand({ MaxResults: MAX_RESULTS, NextToken: databaseLoadContinuationToken });
 		const dbsData = await connection.send(command);
@@ -141,7 +141,7 @@ const createInstance = (connection, _) => {
 
 		const rawTableData = await connection.send(command);
 
-		return mapTableData(rawTableData, _);
+		return mapTableData({ tableData: rawTableData, _, logger });
 	};
 
 	return {

reverse_engineering/helpers/tablePropertiesHelper.js

@@ -59,7 +59,24 @@ const getNumBuckets = (numBuckets = 0) => {
 	return numBuckets < 1 ? undefined : numBuckets;
 };
 
-const mapTableData = (tableData, _) => {
+const mapColumns = ({ columns = [], logger = {} }) => {
+	let hasErrors = false;
+
+	const mapped = columns.map(({ Type, Name }) => {
+		if (!Type || !Name) {
+			hasErrors = true;
+		}
+		return { name: Name, type: Type };
+	});
+
+	if (hasErrors) {
+		logger.log('info', columns, 'Some columns are missing required Type or Name');
+	}
+
+	return mapped;
+};
+
+const mapTableData = ({ tableData, _, logger }) => {
 	const partitionKeys = tableData.Table.PartitionKeys || [];
 
 	return {
@@ -83,7 +100,7 @@ const mapTableData = (tableData, _) => {
 			classification: getClassification(tableData.Table.Parameters),
 		},
 		partitionKeys: tableData.Table.PartitionKeys || [],
-		columns: tableData.Table.StorageDescriptor.Columns.map(({ Type, Name }) => ({ name: Name, type: Type })),
+		columns: mapColumns({ columns: tableData.Table.StorageDescriptor.Columns, logger }),
 	};
 };