Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chris dev #17

Merged
merged 2 commits into from
Jan 5, 2024
Merged

Chris dev #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8596312
Update to `poam` table config
crodriguez6497 Jan 4, 2024
e6a990e
Added poam import functionality for XLS, XLSM, XLXS files. Minor chan…
crodriguez6497 Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 92 additions & 47 deletions Api/Controllers/poamUpload.controller.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
!########################################################################
*/

const readXlsxFile = require('read-excel-file/node');
const ExcelJS = require('exceljs');
const { db } = require('../utils/sequelize.js');
const { poamAsset, Poam } = require('../utils/sequelize.js');

Expand All @@ -24,7 +24,7 @@ const excelColumnToDbColumnMapping = {
"Source Identifying Vulnerability ": "vulnerabilitySource",
"Status": "emassStatus",
"Comments": "notes",
"Raw Severity": "rawSeverity",
" Raw Severity": "rawSeverity",
"Devices Affected": "devicesAffected",
"Mitigations (in-house and in conjunction with the Navy CSSP)": "mitigations",
"Predisposing Conditions": "predisposingConditions",
Expand All @@ -39,70 +39,115 @@ const excelColumnToDbColumnMapping = {
"Resulting Residual Risk after Proposed Mitigations": "adjSeverity"
};

function convertToMySQLDate(excelDate) {
if (!excelDate || typeof excelDate !== 'string' || !/^\d{2}\/\d{2}\/\d{4}$/.test(excelDate)) {
console.log(`Invalid date format: ${excelDate}`);
return null;
}

const [month, day, year] = excelDate.split('/');
const convertedDate = `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`;
const date = new Date(convertedDate);

if (isNaN(date.getTime())) {
console.log(`Invalid date conversion: ${excelDate} to ${convertedDate}`);
return null;
}

return convertedDate;
}


exports.uploadPoamFile = async (req, res) => {
if (!req.file) {
return res.status(400).send({ message: "Please upload an Excel file!" });
}

const lastCollectionAccessedId = req.body.lastCollectionAccessedId;

if (!lastCollectionAccessedId) {
return res.status(400).send({ message: "lastCollectionAccessedId is required" });
}

const workbook = new ExcelJS.Workbook();
try {
const rows = await readXlsxFile(req.file.buffer);
const headers = rows[6]; // Headers from row 7
const dataRows = rows.slice(7); // Data starts from row 8

const poamData = dataRows.map(row => {
const poamEntry = {};
row.forEach((value, index) => {
const dbColumn = headers[index] ? excelColumnToDbColumnMapping[headers[index]] : null;
if (dbColumn) {
poamEntry[dbColumn] = value;
await workbook.xlsx.load(req.file.buffer); // Load the workbook
if (workbook.worksheets.length === 0) {
throw new Error('No worksheets found in the workbook');
}
const worksheet = workbook.worksheets[0]; // Get the first sheet

let headers;
const poamData = [];

// Start reading from the 7th row for headers and the 8th row for data
worksheet.eachRow({ includeEmpty: true }, (row, rowNumber) => {
if (rowNumber === 7) { // Headers are on the 7th row
headers = row.values;
headers.shift(); // Remove the first element which is undefined
} else if (rowNumber > 7) { // Data starts from the 8th row
const poamEntry = {};
let isEmptyRow = true;

row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
colNumber--; // Adjust for zero-based indexing
const dbColumn = headers[colNumber] ? excelColumnToDbColumnMapping[headers[colNumber]] : null;

if (dbColumn) {
const cellValue = cell.text && cell.text.trim();
if (dbColumn === 'scheduledCompletionDate' && cellValue) {
poamEntry[dbColumn] = convertToMySQLDate(cellValue);
} else {
poamEntry[dbColumn] = cellValue;
}

if (cellValue) {
isEmptyRow = false;
}
}
});

if (!isEmptyRow) {
poamEntry.collectionId = lastCollectionAccessedId; // Set collectionId to lastCollectionAccessedId
poamData.push(poamEntry);
}
});
return poamEntry;
}
});

// Bulk insert POAM data
const createdPoams = await Poam.bulkCreate(poamData, { returning: true });
const batchSize = 500;
const createdPoams = [];
for (let i = 0; i < poamData.length; i += batchSize) {
const batch = poamData.slice(i, i + batchSize);
const createdBatch = await Poam.bulkCreate(batch, { returning: true });
createdPoams.push(...createdBatch);
}

// Process devicesAffected for each poamEntry...
// Process devicesAffected for each createdPoam...
for (const poamEntry of createdPoams) {
// Now poamEntry includes poamId generated by the database
if (!poamEntry || !poamEntry.poamId) {
console.error('Invalid poamEntry or missing poamId:', poamEntry);
continue;
}

const poamId = poamEntry.poamId;
const devicesString = poamEntry.devicesAffected && poamEntry.devicesAffected.toString();
const devices = devicesString ? devicesString.split('\n') : [];

// Check if devicesAffected is defined and is either a string or a number
if (poamEntry.devicesAffected !== undefined && (typeof poamEntry.devicesAffected === 'string' || typeof poamEntry.devicesAffected === 'number')) {
// Convert to string if it's a number
let devicesString = poamEntry.devicesAffected.toString();

// Split by line breaks
const devices = devicesString.split('\n');
for (const deviceName of devices) {
// Trim the device name to remove extra spaces
const trimmedDeviceName = deviceName.trim();

// Check if the device name is not empty
if (trimmedDeviceName) {
const existingAsset = await poamAsset.findOne({
where: { assetId: trimmedDeviceName }
});

if (existingAsset) {
// Update existing asset with new poamId
await existingAsset.update({ poamId: poamId });
} else {
// Create new asset with both assetId and poamId if it doesn't exist
await poamAsset.create({ assetId: trimmedDeviceName, poamId: poamId });
}
for (const deviceName of devices) {
const trimmedDeviceName = deviceName.trim();
if (trimmedDeviceName) {
const existingAsset = await poamAsset.findOne({ where: { assetId: trimmedDeviceName } });
if (existingAsset) {
await existingAsset.update({ poamId });
} else {
await poamAsset.create({ assetId: trimmedDeviceName, poamId });
}
}
} else {
// Handle cases where devicesAffected is neither a string nor a number
console.log('devicesAffected is neither a string nor a number:', poamEntry.devicesAffected);
}
}

res.status(200).send({ message: "Uploaded the file successfully: " + req.file.originalname });
}
catch (error) {
} catch (error) {
console.error("Error during file upload and processing: ", error);
res.status(500).send({
message: "Could not process the file: " + req.file.originalname,
Expand Down
94 changes: 61 additions & 33 deletions Api/Models/poam.model.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,53 +6,78 @@ module.exports = (sequelize, DataTypes) => {
primaryKey: true,
autoIncrement: true
},
collectionId: {
type: DataTypes.INTEGER,
defaultValue: 0
},
vulnerabilitySource: {
type: DataTypes.STRING(255),
allowNull: false
defaultValue: ''
},
aaPackage: {
type: DataTypes.STRING(50),
defaultValue: ''
},
vulnerabilityId: {
type: DataTypes.STRING(255),
allowNull: true
defaultValue: ''
},
description: {
type: DataTypes.STRING(255),
allowNull: true
defaultValue: ''
},
rawSeverity: {
type: DataTypes.STRING(3),
allowNull: false
type: DataTypes.STRING(10),
defaultValue: ''
},
adjSeverity: {
type: DataTypes.STRING(6),
allowNull: true
type: DataTypes.STRING(10),
defaultValue: ''
},
scheduledCompletionDate: {
type: DataTypes.DATE,
allowNull: true
type: DataTypes.DATEONLY,
defaultValue: '1900-01-01'
},
ownerId: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0
},
mitigations: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
requiredResources: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
milestones: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
residualRisk: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
businessImpact: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
notes: {
type: DataTypes.TEXT,
allowNull: true
type: DataTypes.TEXT
},
status: {
type: DataTypes.STRING(10),
allowNull: false,
defaultValue: 'Draft'
},
poamType: {
type: DataTypes.STRING(10),
allowNull: false,
defaultValue: ''
},
vulnIdRestricted: {
type: DataTypes.STRING(255),
defaultValue: ''
},
submittedDate: {
type: DataTypes.DATEONLY,
defaultValue: '1900-01-01'
},
poamitemid: {
type: DataTypes.INTEGER,
Expand All @@ -61,53 +86,56 @@ module.exports = (sequelize, DataTypes) => {
},
securityControlNumber: {
type: DataTypes.STRING(25),
allowNull: true
defaultValue: ''
},
officeOrg: {
type: DataTypes.STRING(100),
allowNull: true
defaultValue: ''
},
emassStatus: {
type: DataTypes.STRING(15),
allowNull: false,
defaultValue: 'Ongoing'
},
predisposingConditions: {
type: DataTypes.STRING(2000),
allowNull: true
defaultValue: ''
},
severity: {
type: DataTypes.STRING(15),
allowNull: false
allowNull: false,
defaultValue: ''
},
relevanceOfThreat: {
type: DataTypes.STRING(15),
allowNull: false
allowNull: false,
defaultValue: ''
},
threatDescription: {
type: DataTypes.STRING(255),
allowNull: true
defaultValue: ''
},
likelihood: {
type: DataTypes.STRING(15),
allowNull: false
allowNull: false,
defaultValue: ''
},
impactDescription: {
type: DataTypes.STRING(2000),
allowNull: true
defaultValue: ''
},
recommendations: {
type: DataTypes.STRING(2000),
allowNull: true
defaultValue: ''
},
devicesAffected: {
type: DataTypes.STRING(255),
allowNull: false
allowNull: false,
defaultValue: ''
},
}, {
freezeTableName: true,
timestamps: false,
});

return Poam;
};
};
2 changes: 1 addition & 1 deletion Api/Models/poamAsset.model.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ module.exports = (sequelize, DataTypes) => {
primaryKey: true
},
assetId: {
type: DataTypes.INTEGER,
type: DataTypes.STRING(50),
allowNull: false,
primaryKey: true
},
Expand Down
Loading
Loading