Skip to content

Commit

Permalink
Added poam import functionality for XLS, XLSM, XLXS files. Minor chan…
Browse files Browse the repository at this point in the history 
…ges to UI.
  • Loading branch information
@crodriguez6497
crodriguez6497 committed Jan 5, 2024
1 parent 8596312 commit e6a990e
Show file tree
Hide file tree
Showing 13 changed files with 882 additions and 176 deletions.
139 changes: 92 additions & 47 deletions Api/Controllers/poamUpload.controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
!########################################################################
*/

const readXlsxFile = require('read-excel-file/node');
const ExcelJS = require('exceljs');
const { db } = require('../utils/sequelize.js');
const { poamAsset, Poam } = require('../utils/sequelize.js');

Expand All @@ -24,7 +24,7 @@ const excelColumnToDbColumnMapping = {
"Source Identifying Vulnerability ": "vulnerabilitySource",
"Status": "emassStatus",
"Comments": "notes",
"Raw Severity": "rawSeverity",
" Raw Severity": "rawSeverity",
"Devices Affected": "devicesAffected",
"Mitigations (in-house and in conjunction with the Navy CSSP)": "mitigations",
"Predisposing Conditions": "predisposingConditions",
Expand All @@ -39,70 +39,115 @@ const excelColumnToDbColumnMapping = {
"Resulting Residual Risk after Proposed Mitigations": "adjSeverity"
};

function convertToMySQLDate(excelDate) {
if (!excelDate || typeof excelDate !== 'string' || !/^\d{2}\/\d{2}\/\d{4}$/.test(excelDate)) {
console.log(`Invalid date format: ${excelDate}`);
return null;
}

const [month, day, year] = excelDate.split('/');
const convertedDate = `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`;
const date = new Date(convertedDate);

if (isNaN(date.getTime())) {
console.log(`Invalid date conversion: ${excelDate} to ${convertedDate}`);
return null;
}

return convertedDate;
}


exports.uploadPoamFile = async (req, res) => {
if (!req.file) {
return res.status(400).send({ message: "Please upload an Excel file!" });
}

const lastCollectionAccessedId = req.body.lastCollectionAccessedId;

if (!lastCollectionAccessedId) {
return res.status(400).send({ message: "lastCollectionAccessedId is required" });
}

const workbook = new ExcelJS.Workbook();
try {
const rows = await readXlsxFile(req.file.buffer);
const headers = rows[6]; // Headers from row 7
const dataRows = rows.slice(7); // Data starts from row 8

const poamData = dataRows.map(row => {
const poamEntry = {};
row.forEach((value, index) => {
const dbColumn = headers[index] ? excelColumnToDbColumnMapping[headers[index]] : null;
if (dbColumn) {
poamEntry[dbColumn] = value;
await workbook.xlsx.load(req.file.buffer); // Load the workbook
if (workbook.worksheets.length === 0) {
throw new Error('No worksheets found in the workbook');
}
const worksheet = workbook.worksheets[0]; // Get the first sheet

let headers;
const poamData = [];

// Start reading from the 7th row for headers and the 8th row for data
worksheet.eachRow({ includeEmpty: true }, (row, rowNumber) => {
if (rowNumber === 7) { // Headers are on the 7th row
headers = row.values;
headers.shift(); // Remove the first element which is undefined
} else if (rowNumber > 7) { // Data starts from the 8th row
const poamEntry = {};
let isEmptyRow = true;

row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
colNumber--; // Adjust for zero-based indexing
const dbColumn = headers[colNumber] ? excelColumnToDbColumnMapping[headers[colNumber]] : null;

if (dbColumn) {
const cellValue = cell.text && cell.text.trim();
if (dbColumn === 'scheduledCompletionDate' && cellValue) {
poamEntry[dbColumn] = convertToMySQLDate(cellValue);
} else {
poamEntry[dbColumn] = cellValue;
}

if (cellValue) {
isEmptyRow = false;
}
}
});

if (!isEmptyRow) {
poamEntry.collectionId = lastCollectionAccessedId; // Set collectionId to lastCollectionAccessedId
poamData.push(poamEntry);
}
});
return poamEntry;
}
});

// Bulk insert POAM data
const createdPoams = await Poam.bulkCreate(poamData, { returning: true });
const batchSize = 500;
const createdPoams = [];
for (let i = 0; i < poamData.length; i += batchSize) {
const batch = poamData.slice(i, i + batchSize);
const createdBatch = await Poam.bulkCreate(batch, { returning: true });
createdPoams.push(...createdBatch);
}

// Process devicesAffected for each poamEntry...
// Process devicesAffected for each createdPoam...
for (const poamEntry of createdPoams) {
// Now poamEntry includes poamId generated by the database
if (!poamEntry || !poamEntry.poamId) {
console.error('Invalid poamEntry or missing poamId:', poamEntry);
continue;
}

const poamId = poamEntry.poamId;
const devicesString = poamEntry.devicesAffected && poamEntry.devicesAffected.toString();
const devices = devicesString ? devicesString.split('\n') : [];

// Check if devicesAffected is defined and is either a string or a number
if (poamEntry.devicesAffected !== undefined && (typeof poamEntry.devicesAffected === 'string' || typeof poamEntry.devicesAffected === 'number')) {
// Convert to string if it's a number
let devicesString = poamEntry.devicesAffected.toString();

// Split by line breaks
const devices = devicesString.split('\n');
for (const deviceName of devices) {
// Trim the device name to remove extra spaces
const trimmedDeviceName = deviceName.trim();

// Check if the device name is not empty
if (trimmedDeviceName) {
const existingAsset = await poamAsset.findOne({
where: { assetId: trimmedDeviceName }
});

if (existingAsset) {
// Update existing asset with new poamId
await existingAsset.update({ poamId: poamId });
} else {
// Create new asset with both assetId and poamId if it doesn't exist
await poamAsset.create({ assetId: trimmedDeviceName, poamId: poamId });
}
for (const deviceName of devices) {
const trimmedDeviceName = deviceName.trim();
if (trimmedDeviceName) {
const existingAsset = await poamAsset.findOne({ where: { assetId: trimmedDeviceName } });
if (existingAsset) {
await existingAsset.update({ poamId });
} else {
await poamAsset.create({ assetId: trimmedDeviceName, poamId });
}
}
} else {
// Handle cases where devicesAffected is neither a string nor a number
console.log('devicesAffected is neither a string nor a number:', poamEntry.devicesAffected);
}
}

res.status(200).send({ message: "Uploaded the file successfully: " + req.file.originalname });
}
catch (error) {
} catch (error) {
console.error("Error during file upload and processing: ", error);
res.status(500).send({
message: "Could not process the file: " + req.file.originalname,
Expand Down
94 changes: 61 additions & 33 deletions Api/Models/poam.model.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,53 +6,78 @@ module.exports = (sequelize, DataTypes) => {
primaryKey: true,
autoIncrement: true
},
collectionId: {
type: DataTypes.INTEGER,
defaultValue: 0
},
vulnerabilitySource: {
type: DataTypes.STRING(255),
allowNull: false
defaultValue: ''
},
aaPackage: {
type: DataTypes.STRING(50),
defaultValue: ''
},
vulnerabilityId: {
type: DataTypes.STRING(255),
allowNull: true
defaultValue: ''
},
description: {
type: DataTypes.STRING(255),
allowNull: true
defaultValue: ''
},
rawSeverity: {
type: DataTypes.STRING(3),
allowNull: false
type: DataTypes.STRING(10),
defaultValue: ''
},
adjSeverity: {
type: DataTypes.STRING(6),
allowNull: true
type: DataTypes.STRING(10),
defaultValue: ''
},
scheduledCompletionDate: {
type: DataTypes.DATE,
allowNull: true
type: DataTypes.DATEONLY,
defaultValue: '1900-01-01'
},
ownerId: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0
},
mitigations: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
requiredResources: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
milestones: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
residualRisk: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
businessImpact: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
notes: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
status: {
type: DataTypes.STRING(10),
allowNull: false,
defaultValue: 'Draft'
},
poamType: {
type: DataTypes.STRING(10),
allowNull: false,
defaultValue: ''
},
vulnIdRestricted: {
type: DataTypes.STRING(255),
defaultValue: ''
},
submittedDate: {
type: DataTypes.DATEONLY,
defaultValue: '1900-01-01'
},
poamitemid: {
type: DataTypes.INTEGER,
Expand All @@ -61,53 +86,56 @@ module.exports = (sequelize, DataTypes) => {
},
securityControlNumber: {
type: DataTypes.STRING(25),
allowNull: true
defaultValue: ''
},
officeOrg: {
type: DataTypes.STRING(100),
allowNull: true
defaultValue: ''
},
emassStatus: {
type: DataTypes.STRING(15),
allowNull: false,
defaultValue: 'Ongoing'
},
predisposingConditions: {
type: DataTypes.STRING(2000),
allowNull: true
defaultValue: ''
},
severity: {
type: DataTypes.STRING(15),
allowNull: false
allowNull: false,
defaultValue: ''
},
relevanceOfThreat: {
type: DataTypes.STRING(15),
allowNull: false
allowNull: false,
defaultValue: ''
},
threatDescription: {
type: DataTypes.STRING(255),
allowNull: true
defaultValue: ''
},
likelihood: {
type: DataTypes.STRING(15),
allowNull: false
allowNull: false,
defaultValue: ''
},
impactDescription: {
type: DataTypes.STRING(2000),
allowNull: true
defaultValue: ''
},
recommendations: {
type: DataTypes.STRING(2000),
allowNull: true
defaultValue: ''
},
devicesAffected: {
type: DataTypes.STRING(255),
allowNull: false
allowNull: false,
defaultValue: ''
},
}, {
freezeTableName: true,
timestamps: false,
});

return Poam;
};
};
2 changes: 1 addition & 1 deletion Api/Models/poamAsset.model.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ module.exports = (sequelize, DataTypes) => {
primaryKey: true
},
assetId: {
type: DataTypes.INTEGER,
type: DataTypes.STRING(50),
allowNull: false,
primaryKey: true
},
Expand Down
Loading

0 comments on commit e6a990e

Please sign in to comment.