From a229cfef63c4fc6849b5cbd799514e6257164d87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:35:20 +0000 Subject: [PATCH 1/5] Bump fast-xml-parser from 3.21.1 to 4.2.7 Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 3.21.1 to 4.2.7. - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v3.21.1...v4.2.7) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- package-lock.json | 34 ++++++++++++++++++++-------------- package.json | 2 +- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1055038d..4a074bc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@types/lodash": "^4.14.178", "@types/mustache": "^4.2.0", "@types/pretty": "^2.0.1", - "fast-xml-parser": "^3.1.19", + "fast-xml-parser": "^4.2.7", "flat": "^5.0.2", "he": "^1.2.0", "htmlparser2": "^9.0.0", @@ -2724,18 +2724,24 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "3.21.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-3.21.1.tgz", - "integrity": "sha512-FTFVjYoBOZTJekiUsawGsSYV9QL0A+zDYCRj7y34IO6Jg+2IMYEtQa+bbictpdpV8dHxXywqU7C0gRDEOFtBFg==", + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.7.tgz", + "integrity": "sha512-J8r6BriSLO1uj2miOk1NW0YVm8AGOOu3Si2HQp/cSmo6EA4m3fcwu2WKjJ4RK9wMLBtg69y1kS8baDiQBR41Ig==", + "funding": [ + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + }, + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], "dependencies": { - "strnum": "^1.0.4" + "strnum": "^1.0.5" }, "bin": { - "xml2js": "cli.js" - }, - "funding": { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" + "fxparser": "src/cli/cli.js" } }, "node_modules/fastq": { @@ -7181,11 +7187,11 @@ "dev": true }, "fast-xml-parser": { - "version": "3.21.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-3.21.1.tgz", - "integrity": "sha512-FTFVjYoBOZTJekiUsawGsSYV9QL0A+zDYCRj7y34IO6Jg+2IMYEtQa+bbictpdpV8dHxXywqU7C0gRDEOFtBFg==", + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.7.tgz", + "integrity": "sha512-J8r6BriSLO1uj2miOk1NW0YVm8AGOOu3Si2HQp/cSmo6EA4m3fcwu2WKjJ4RK9wMLBtg69y1kS8baDiQBR41Ig==", "requires": { - "strnum": "^1.0.4" + "strnum": "^1.0.5" } }, "fastq": { diff --git a/package.json b/package.json index 31301b89..5b45fbd6 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "@types/lodash": "^4.14.178", "@types/mustache": "^4.2.0", "@types/pretty": "^2.0.1", - "fast-xml-parser": "^3.1.19", + "fast-xml-parser": "^4.2.7", "flat": "^5.0.2", "he": "^1.2.0", "htmlparser2": "^9.0.0", From ac8a6d46f8cbd4b3c1edc9e68e9a23f6ba72871a Mon Sep 17 00:00:00 2001 From: Emily Rodriguez Date: Wed, 23 Aug 2023 14:13:13 -0500 Subject: [PATCH 2/5] updating use of fast-xml-parser, need to fix array settign Signed-off-by: Emily Rodriguez --- src/utilities/xccdf.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/utilities/xccdf.ts b/src/utilities/xccdf.ts index 2be7126e..21f32fd6 100644 --- a/src/utilities/xccdf.ts +++ b/src/utilities/xccdf.ts @@ -1,4 +1,4 @@ -import parser from 'fast-xml-parser' +import { XMLParser } from 'fast-xml-parser' import {toXML} from 'jstoxml'; import * as htmlparser from 'htmlparser2' import _ from 'lodash' @@ -9,13 +9,15 @@ import he from 'he' export function convertEncodedXmlIntoJson( encodedXml: string ): any { - return parser.parse(encodedXml, { + const options = { ignoreAttributes: false, ignoreNameSpace: true, attributeNamePrefix: '@_', stopNodes: ['div', 'p'], - arrayMode: true - }) + arrayMode: true // needs to be updated to isArray https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md#isarray + } + const parser = new XMLParser(options) + return parser.parse(encodedXml) } From 65e199bdb2f74a9d30083dc436bbd61ee257aa07 Mon Sep 17 00:00:00 2001 From: Emily Rodriguez Date: Mon, 18 Sep 2023 15:31:33 -0500 Subject: [PATCH 3/5] trying to resolve the 'title' tag Signed-off-by: Emily Rodriguez --- src/utilities/xccdf.ts | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/utilities/xccdf.ts b/src/utilities/xccdf.ts index 21f32fd6..e9e30a78 100644 --- a/src/utilities/xccdf.ts +++ b/src/utilities/xccdf.ts @@ -1,10 +1,18 @@ -import { XMLParser } from 'fast-xml-parser' +import {XMLParser} from 'fast-xml-parser' import {toXML} from 'jstoxml'; import * as htmlparser from 'htmlparser2' import _ from 'lodash' import {DecodedDescription} from '../types/xccdf' import he from 'he' +// const alwaysArray = ['cci_item', 'reference', 'Group', 'group', 'Benchmark', 'Rule', 'title', 'rule', 'version', 'title', '@_id', 'check']; +// 'title', +const alwaysArray = ['dc-status', 'description','notice', 'front-matter', 'rear-matter', 'reference', 'plain-text', 'platform', 'metadata', 'Benchmark', 'Group', 'Rule', 'TestResult', 'Value', 'Profile', 'check', 'ident', 'rationale']; + +// arrayMode: () => { +// return true; +// }//true // needs to be updated to isArray https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md#isarray + export function convertEncodedXmlIntoJson( encodedXml: string @@ -14,10 +22,16 @@ export function convertEncodedXmlIntoJson( ignoreNameSpace: true, attributeNamePrefix: '@_', stopNodes: ['div', 'p'], - arrayMode: true // needs to be updated to isArray https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md#isarray - } - const parser = new XMLParser(options) - return parser.parse(encodedXml) + isArray: (tagName: string) => { + if (alwaysArray.includes(tagName)) { + return true; + } else { + return false; + } + } + }; + const parser = new XMLParser(options); + return parser.parse(encodedXml); } From 9041ae4b4b315031f682538e7d60d77ccdbd7390 Mon Sep 17 00:00:00 2001 From: Emily Rodriguez Date: Sat, 9 Dec 2023 12:27:20 -0600 Subject: [PATCH 4/5] including log statements that will need removed, but exploring changing types instead of specifying array values Signed-off-by: Emily Rodriguez --- src/parsers/xccdf.ts | 8 ++++++-- src/types/xccdf.d.ts | 2 +- src/utilities/xccdf.ts | 30 ++++++++++++++++-------------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/parsers/xccdf.ts b/src/parsers/xccdf.ts index 5a4a5481..330dfabc 100644 --- a/src/parsers/xccdf.ts +++ b/src/parsers/xccdf.ts @@ -118,8 +118,12 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro default: throw new Error('useRuleId must be one of "group", "rule", or "version"') } + + if(!(_.isArray(rule.title) && rule.title.length === 1)) { + throw new Error("Rule title is not an array of legnth 1."); + } - control.title = removeXMLSpecialCharacters(rule['@_severity'] ? ensureDecodedXMLStringValue(rule.title, 'undefined title') : `[[[MISSING SEVERITY FROM BENCHMARK]]] ${ensureDecodedXMLStringValue(rule.title,'undefined title')}`) + control.title = removeXMLSpecialCharacters(rule['@_severity'] ? ensureDecodedXMLStringValue(rule.title[0], 'undefined title') : `[[[MISSING SEVERITY FROM BENCHMARK]]] ${ensureDecodedXMLStringValue(rule.title[0],'undefined title')}`) if (typeof extractedDescription === 'object' && !Array.isArray(extractedDescription)) { control.desc = extractedDescription.VulnDiscussion?.split('Satisfies: ')[0] || '' @@ -352,7 +356,7 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro } } else { logger.warn('Reference parts of invalid length:') - logger.info(referenceParts) + // logger.info(referenceParts) } } } catch (e) { diff --git a/src/types/xccdf.d.ts b/src/types/xccdf.d.ts index 54303f08..664412da 100644 --- a/src/types/xccdf.d.ts +++ b/src/types/xccdf.d.ts @@ -722,7 +722,7 @@ export interface BenchmarkRule { '@_severity': Severity; '@_weight': string; version: string; - title: string; + title: string[]; description: string; rationale: FrontMatter[]; reference: PurpleReference[]; diff --git a/src/utilities/xccdf.ts b/src/utilities/xccdf.ts index aeb7f055..7fa3fa58 100644 --- a/src/utilities/xccdf.ts +++ b/src/utilities/xccdf.ts @@ -8,15 +8,14 @@ import he from 'he' // const alwaysArray = ['cci_item', 'reference', 'Group', 'group', 'Benchmark', 'Rule', 'title', 'rule', 'version', 'title', '@_id', 'check']; // 'title', //STIG -const alwaysArray = ['title', 'dc-status', 'description','notice', 'front-matter', 'rear-matter', 'reference', 'plain-text', 'platform', 'metadata', 'Benchmark', 'Group', 'Rule', 'TestResult', 'Value', 'Profile', 'check', 'ident', 'rationale']; +// const alwaysArray = ['title', 'dc-status', 'description','notice', 'front-matter', 'rear-matter', 'reference', 'plain-text', 'platform', 'metadata', 'Benchmark', 'Group', 'Rule', 'TestResult', 'Value', 'Profile', 'check', 'ident', 'rationale']; //OVAL -// const alwaysArray = ['object_reference', 'definition', 'affected', 'reference', 'xsd:any', 'platform', 'product', 'note', 'criteria', 'criterion', 'extend_definition', 'oval-def:tests', 'oval-def:objects', 'oval-def:filter', 'oval-def:states', 'oval-def:variables', 'possible_value', 'possible_restriction', 'restriction', 'value', 'field']; +const alwaysArray = ['object_reference', 'oval-def:definition', 'definition', 'affected', 'reference', 'xsd:any', 'platform', 'product', 'note', 'criteria', 'criterion', 'extend_definition', 'oval-def:test', 'oval-def:object', 'oval-def:filter', 'oval-def:state', 'oval-def:variable', 'possible_value', 'possible_restriction', 'restriction', 'value', 'field', 'definitions', 'generator']; // arrayMode: () => { // return true; // }//true // needs to be updated to isArray https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md#isarray - export function convertEncodedXmlIntoJson( encodedXml: string ): any { @@ -25,12 +24,12 @@ export function convertEncodedXmlIntoJson( ignoreNameSpace: true, attributeNamePrefix: '@_', stopNodes: ['div', 'p'], - isArray: (tagName: string) => { - if (alwaysArray.includes(tagName)) { + isArray: (_name: string, _jpath: string, isLeafNode: boolean) => { + // if (isLeafNode) { return true; - } else { - return false; - } + // } else { + // return false; + // } } }; const parser = new XMLParser(options); @@ -43,27 +42,30 @@ export function convertJsonIntoXML(data: any) { } export function removeXMLSpecialCharacters(str: string) { - return he.decode(str) + console.log("Remove special characters: ", JSON.stringify(str, null, 2)); + const result = he.decode(str); + console.log("Result of he.decode: ", JSON.stringify(result)); + return result } export function severityStringToImpact(string: string, id: string): number { - if (string.match(/none|na|n\/a|not[\s()*_|]?applicable/i)?.length) { + if (RegExp(/none|na|n\/a|not[\s()*_|]?applicable/i).exec(string)?.length) { return 0.0 } - if (string.match(/low|cat(egory)?\s*(iii|3)/i)?.length) { + if (RegExp(/low|cat(egory)?\s*(iii|3)/i).exec(string)?.length) { return 0.3 } - if (string.match(/med(ium)?|cat(egory)?\s*(ii|2)/)?.length) { + if (RegExp(/med(ium)?|cat(egory)?\s*(ii|2)/).exec(string)?.length) { return 0.5 } - if (string.match(/high|cat(egory)?\s*(i|1)/)?.length) { + if (RegExp(/high|cat(egory)?\s*(i|1)/).exec(string)?.length) { return 0.7 } - if (string.match(/crit(ical)?|severe/)?.length) { + if (RegExp(/crit(ical)?|severe/).exec(string)?.length) { return 1.0 } From 1dee12e35386e12fe673ca3a61bf53066aafffde Mon Sep 17 00:00:00 2001 From: Emily Rodriguez Date: Wed, 13 Mar 2024 17:03:33 -0500 Subject: [PATCH 5/5] updating a couple of the array types Signed-off-by: Emily Rodriguez --- src/parsers/xccdf.ts | 42 +++++++++++++++++++++++------------------ src/utilities/update.ts | 2 +- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/parsers/xccdf.ts b/src/parsers/xccdf.ts index 330dfabc..86c11120 100644 --- a/src/parsers/xccdf.ts +++ b/src/parsers/xccdf.ts @@ -94,10 +94,10 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro control.id = rule.group['@_id'] break; case 'rule': - if (rule['@_id'].toLowerCase().startsWith('sv')) { - control.id = rule['@_id'].split('r')[0] + if (rule['@_id'][0].toLowerCase().startsWith('sv')) { + control.id = rule['@_id'][0].split('r')[0] } else { - control.id = rule['@_id'] + control.id = rule['@_id'][0] } break; case 'version': @@ -120,9 +120,9 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro } if(!(_.isArray(rule.title) && rule.title.length === 1)) { - throw new Error("Rule title is not an array of legnth 1."); + throw new Error('Rule title is not an array of length 1.'); } - + control.title = removeXMLSpecialCharacters(rule['@_severity'] ? ensureDecodedXMLStringValue(rule.title[0], 'undefined title') : `[[[MISSING SEVERITY FROM BENCHMARK]]] ${ensureDecodedXMLStringValue(rule.title[0],'undefined title')}`) if (typeof extractedDescription === 'object' && !Array.isArray(extractedDescription)) { @@ -143,7 +143,7 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro if (rule.check) { if (rule.check.some((ruleValue) => 'check-content' in ruleValue)) { - control.descs.check = removeXMLSpecialCharacters(rule.check ? rule.check[0]['check-content'] : 'Missing description') + control.descs.check = removeXMLSpecialCharacters(rule.check ? rule.check[0]['check-content'][0] : 'Missing description') control.tags.check_id = rule.check[0]['@_system'] } else if (rule.check.some((ruleValue) => 'check-content-ref' in ruleValue) && ovalDefinitions) { let referenceID: string | null = null; @@ -158,7 +158,7 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro } } if (referenceID && referenceID in ovalDefinitions) { - control.descs.check = removeXMLSpecialCharacters(ovalDefinitions[referenceID].metadata[0].title) + control.descs.check = removeXMLSpecialCharacters(ovalDefinitions[referenceID].metadata[0].title[0]) } else if (referenceID) { logger.warn(`Could not find OVAL definition for ${referenceID}`) } @@ -220,11 +220,11 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro if (_.get(rule.fixtext, '[0]["#text"]')) { control.descs.fix = removeXMLSpecialCharacters(rule.fixtext[0]['#text']) - } else if (typeof rule.fixtext === 'string') { - control.descs.fix = removeXMLSpecialCharacters(rule.fixtext) - } else if (typeof rule.fixtext === 'object') { - if (Array.isArray(rule.fixtext)) { - control.descs.fix = removeXMLSpecialCharacters(pretty(convertJsonIntoXML(rule.fixtext.map((fixtext: any) => { + } else if (typeof rule.fixtext[0] === 'string') { + control.descs.fix = removeXMLSpecialCharacters(rule.fixtext[0]) + } else if (typeof rule.fixtext[0] === 'object') { + if (Array.isArray(rule.fixtext[0])) { + control.descs.fix = removeXMLSpecialCharacters(pretty(convertJsonIntoXML(rule.fixtext[0].map((fixtext: any) => { if (fixtext.div) { return fixtext.div } else { @@ -249,9 +249,8 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro control.tags.rid = rule['@_id'] control.tags.stig_id = rule['version'] - - if (typeof rule.group.title === 'string') { - control.tags.gtitle = removeXMLSpecialCharacters(rule.group.title) + if (typeof rule.group.title[0] === 'string') { + control.tags.gtitle = removeXMLSpecialCharacters(rule.group.title[0]) } else { control.tags.gtitle = removeXMLSpecialCharacters(_.get(rule.group, 'title[0].#text', 'undefined title')) } @@ -280,7 +279,14 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro } control.tags = _.mapValues(_.omitBy(control.tags, (value) => value === undefined), (value) => { - if (typeof value === 'string') { + if (value && Array.isArray(value)) { + if (Array.isArray(value[0])) { + return removeXMLSpecialCharacters(value[0][0] as string) + } + else { + return removeXMLSpecialCharacters(value[0] as string) + } + } else if (typeof value === 'string') { return removeXMLSpecialCharacters(value) } else { return value @@ -291,14 +297,14 @@ export function processXCCDF(xml: string, removeNewlines: false, useRuleId: 'gro if (rule.ident) { rule.ident.forEach((identifier) => { // Get CCIs - if (identifier['@_system'].toLowerCase().includes('cci')) { + if (identifier['@_system'][0].toLowerCase().includes('cci')) { if (!('cci' in control.tags)) { control.tags.cci = [] } control.tags.cci?.push(identifier['#text']) } // Get legacy identifiers - else if (identifier['@_system'].toLowerCase().includes('legacy')) { + else if (identifier['@_system'][0].toLowerCase().includes('legacy')) { if (!('legacy' in control.tags)) { control.tags.legacy = [] } diff --git a/src/utilities/update.ts b/src/utilities/update.ts index 4dc93f9c..626f19a1 100644 --- a/src/utilities/update.ts +++ b/src/utilities/update.ts @@ -244,7 +244,7 @@ export function getExistingDescribeFromControl(control: Control): string { export function findUpdatedControlByAllIdentifiers(existingControl: Control, updatedControls: Control[]): Control | undefined { // Try to match based on IDs let updatedControl = updatedControls.find((updatedControl) => { - return updatedControl.id.toLowerCase() === existingControl.id.toLowerCase() + return updatedControl.id[0].toLowerCase() === existingControl.id[0].toLowerCase() }) if (updatedControl) {