From e18c52eec2cb18dc2590b61d7649de4507f060a7 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Fri, 11 Oct 2024 11:34:12 +0200 Subject: [PATCH] [Cloud Security] Increase retention period on queries related to 3rd party data loading (#195636) ## Summary Increase retention on Cloud Security queries to accommodate a longer retention period on third-party CDR integrations, such as Wiz and AWS SecurityHub. This introduces regression for https://github.com/elastic/kibana/issues/142198 This is meant is a temporary workaround until we find a robust way to get full posture for third-party CDR integrations This change goes together with increasing retention period on Wiz: - https://github.com/elastic/integrations/pull/11393 fixes: - https://github.com/elastic/security-team/issues/10683 ## How to test The CI deployed a serverless project where I installed Wiz and CSP integrations and ingested some data. --- .../constants.ts | 4 + .../src/utils/hooks_utils.ts | 7 +- .../latest_findings/use_latest_findings.ts | 4 +- .../use_latest_findings_grouping.tsx | 4 +- .../hooks/use_latest_vulnerabilities.tsx | 4 +- .../use_latest_vulnerabilities_grouping.tsx | 4 +- .../server/routes/status/status.ts | 5 +- .../page_objects/findings_page.ts | 3 +- .../pages/findings_old_data.ts | 111 ++++++++++++++++-- 9 files changed, 121 insertions(+), 25 deletions(-) diff --git a/x-pack/packages/kbn-cloud-security-posture-common/constants.ts b/x-pack/packages/kbn-cloud-security-posture-common/constants.ts index 7ff50efdd94893..a24d676dc6f884 100644 --- a/x-pack/packages/kbn-cloud-security-posture-common/constants.ts +++ b/x-pack/packages/kbn-cloud-security-posture-common/constants.ts @@ -36,6 +36,10 @@ export const CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN = export const CDR_VULNERABILITIES_INDEX_PATTERN = `${CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN},${CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN}`; export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d'; +// TODO: remove once https://github.com/elastic/security-team/issues/10801 is done +// meant as a temp workaround to get good enough posture view for 3rd party integrations, see https://github.com/elastic/security-team/issues/10683 +export const CDR_3RD_PARTY_RETENTION_POLICY = '90d'; + export const VULNERABILITIES_SEVERITY: Record = { LOW: 'LOW', MEDIUM: 'MEDIUM', diff --git a/x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts b/x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts index c7585adba9ce44..d06f4efbde0260 100644 --- a/x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts +++ b/x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts @@ -9,8 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { CDR_MISCONFIGURATIONS_INDEX_PATTERN, CDR_VULNERABILITIES_INDEX_PATTERN, - LATEST_FINDINGS_RETENTION_POLICY, - LATEST_VULNERABILITIES_RETENTION_POLICY, + CDR_3RD_PARTY_RETENTION_POLICY, } from '@kbn/cloud-security-posture-common'; import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest'; import { buildMutedRulesFilter } from '@kbn/cloud-security-posture-common'; @@ -103,7 +102,7 @@ const buildMisconfigurationsFindingsQueryWithFilters = ( { range: { '@timestamp': { - gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`, + gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`, lte: 'now', }, }, @@ -182,7 +181,7 @@ const buildVulnerabilityFindingsQueryWithFilters = (query: UseCspOptions['query' { range: { '@timestamp': { - gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`, + gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`, lte: 'now', }, }, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts index 955f5a45a9743c..068eb3df1b10fb 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts @@ -15,7 +15,7 @@ import { showErrorToast } from '@kbn/cloud-security-posture'; import { MAX_FINDINGS_TO_LOAD, buildMutedRulesFilter } from '@kbn/cloud-security-posture-common'; import { CDR_MISCONFIGURATIONS_INDEX_PATTERN, - LATEST_FINDINGS_RETENTION_POLICY, + CDR_3RD_PARTY_RETENTION_POLICY, } from '@kbn/cloud-security-posture-common'; import type { CspFinding } from '@kbn/cloud-security-posture-common'; import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest'; @@ -77,7 +77,7 @@ export const getFindingsQuery = ( { range: { '@timestamp': { - gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`, + gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`, lte: 'now', }, }, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx index 6482d864347a10..e009ee966fb966 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx @@ -16,7 +16,7 @@ import { import { useMemo } from 'react'; import { buildEsQuery, Filter } from '@kbn/es-query'; import { - LATEST_FINDINGS_RETENTION_POLICY, + CDR_3RD_PARTY_RETENTION_POLICY, buildMutedRulesFilter, } from '@kbn/cloud-security-posture-common'; import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api'; @@ -249,7 +249,7 @@ export const useLatestFindingsGrouping = ({ additionalFilters: query ? [query, additionalFilters] : [additionalFilters], groupByField: currentSelectedGroup, uniqueValue, - from: `now-${LATEST_FINDINGS_RETENTION_POLICY}`, + from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`, to: 'now', pageNumber: activePageIndex * pageSize, size: pageSize, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx index 0b9cf6978c2580..5f01a4693c8f5f 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx @@ -19,7 +19,7 @@ import { EsHitRecord } from '@kbn/discover-utils/types'; import { MAX_FINDINGS_TO_LOAD, CDR_VULNERABILITIES_INDEX_PATTERN, - LATEST_VULNERABILITIES_RETENTION_POLICY, + CDR_3RD_PARTY_RETENTION_POLICY, } from '@kbn/cloud-security-posture-common'; import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture'; import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest'; @@ -90,7 +90,7 @@ export const getVulnerabilitiesQuery = ( { range: { '@timestamp': { - gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`, + gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`, lte: 'now', }, }, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index 3c52590f8fd80b..d79b4620ec8996 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -15,7 +15,7 @@ import { } from '@kbn/grouping/src'; import { useMemo } from 'react'; import { - LATEST_VULNERABILITIES_RETENTION_POLICY, + CDR_3RD_PARTY_RETENTION_POLICY, VULNERABILITIES_SEVERITY, } from '@kbn/cloud-security-posture-common'; import { buildEsQuery, Filter } from '@kbn/es-query'; @@ -202,7 +202,7 @@ export const useLatestVulnerabilitiesGrouping = ({ additionalFilters: query ? [query, additionalFilters] : [additionalFilters], groupByField: currentSelectedGroup, uniqueValue, - from: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`, + from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`, to: 'now', pageNumber: activePageIndex * pageSize, size: pageSize, diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts index 659b2ed94f43a6..4f5c84b936fb27 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts @@ -15,6 +15,7 @@ import { CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, LATEST_VULNERABILITIES_RETENTION_POLICY, CDR_VULNERABILITIES_INDEX_PATTERN, + CDR_3RD_PARTY_RETENTION_POLICY, } from '@kbn/cloud-security-posture-common'; import type { CspSetupStatus, @@ -218,13 +219,13 @@ export const getCspStatus = async ({ checkIndexHasFindings( esClient, CDR_MISCONFIGURATIONS_INDEX_PATTERN, - LATEST_FINDINGS_RETENTION_POLICY, + CDR_3RD_PARTY_RETENTION_POLICY, logger ), checkIndexHasFindings( esClient, CDR_VULNERABILITIES_INDEX_PATTERN, - LATEST_VULNERABILITIES_RETENTION_POLICY, + CDR_3RD_PARTY_RETENTION_POLICY, logger ), checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, { diff --git a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts index 8e9483ce97b334..f58f798a96df4e 100644 --- a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts +++ b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts @@ -366,8 +366,7 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider }); const isLatestFindingsTableThere = async () => { const table = await testSubjects.findAll('docTable'); - const trueOrFalse = table.length > 0 ? true : false; - return trueOrFalse; + return table.length > 0; }; const getUnprivilegedPrompt = async () => { diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts b/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts index 9586387c028ea2..2811d2427108b9 100644 --- a/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts +++ b/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts @@ -8,16 +8,38 @@ import expect from '@kbn/expect'; import Chance from 'chance'; import type { FtrProviderContext } from '../ftr_provider_context'; +import { vulnerabilitiesLatestMock } from '../mocks/vulnerabilities_latest_mock'; // eslint-disable-next-line import/no-default-export -export default function ({ getPageObjects }: FtrProviderContext) { +export default function ({ getPageObjects, getService }: FtrProviderContext) { + const retry = getService('retry'); const pageObjects = getPageObjects(['common', 'findings', 'header']); const chance = new Chance(); - const hoursToMillisecond = (hours: number) => hours * 60 * 60 * 1000; + const daysToMillisecond = (days: number) => days * 24 * 60 * 60 * 1000; + const RETENTION = 90; const dataOldKspm = [ { - '@timestamp': (Date.now() - hoursToMillisecond(27)).toString(), + '@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(), + resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' }, + result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' }, + rule: { + name: 'Upper case rule name', + section: 'Upper case section', + benchmark: { + id: 'cis_k8s', + posture_type: 'kspm', + name: 'CIS Kubernetes V1.23', + version: 'v1.0.0', + }, + type: 'process', + }, + cluster_id: 'Upper case cluster id', + }, + ]; + const dataWithinRetentionKspm = [ + { + '@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(), resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' }, result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' }, rule: { @@ -37,7 +59,26 @@ export default function ({ getPageObjects }: FtrProviderContext) { const dataOldCspm = [ { - '@timestamp': (Date.now() - hoursToMillisecond(27)).toString(), + '@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(), + resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' }, + result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' }, + rule: { + name: 'Upper case rule name', + section: 'Upper case section', + benchmark: { + id: 'cis_aws', + posture_type: 'cspm', + name: 'CIS AWS V1.23', + version: 'v1.0.0', + }, + type: 'process', + }, + cluster_id: 'Upper case cluster id', + }, + ]; + const dataWithinRetentionCspm = [ + { + '@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(), resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' }, result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' }, rule: { @@ -55,25 +96,41 @@ export default function ({ getPageObjects }: FtrProviderContext) { }, ]; + const dataOldCnvm = [ + { + ...vulnerabilitiesLatestMock[0], + '@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(), + }, + ]; + const dataWithinRetentionCnvm = [ + { + ...vulnerabilitiesLatestMock[0], + '@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(), + }, + ]; + describe('Old Data', function () { this.tags(['cloud_security_posture_findings']); let findings: typeof pageObjects.findings; + let latestFindingsTable: typeof findings.latestFindingsTable; + let latestVulnerabilitiesTable: typeof findings.latestVulnerabilitiesTable; before(async () => { findings = pageObjects.findings; + latestFindingsTable = findings.latestFindingsTable; + latestVulnerabilitiesTable = findings.latestVulnerabilitiesTable; // Before we start any test we must wait for cloud_security_posture plugin to complete its initialization await findings.waitForPluginInitialized(); }); - after(async () => { + afterEach(async () => { await findings.index.remove(); + await findings.vulnerabilitiesIndex.remove(); }); describe('Findings page with old data', () => { it('returns no Findings KSPM', async () => { - // Prepare mocked findings - await findings.index.remove(); await findings.index.add(dataOldKspm); await findings.navigateToLatestFindingsPage(); @@ -81,14 +138,50 @@ export default function ({ getPageObjects }: FtrProviderContext) { expect(await findings.isLatestFindingsTableThere()).to.be(false); }); it('returns no Findings CSPM', async () => { - // Prepare mocked findings - await findings.index.remove(); await findings.index.add(dataOldCspm); await findings.navigateToLatestFindingsPage(); await pageObjects.header.waitUntilLoadingHasFinished(); expect(await findings.isLatestFindingsTableThere()).to.be(false); }); + it('returns no Findings CNVM', async () => { + await findings.vulnerabilitiesIndex.add(dataOldCnvm); + + await findings.navigateToLatestVulnerabilitiesPage(); + await pageObjects.header.waitUntilLoadingHasFinished(); + expect(await findings.isLatestFindingsTableThere()).to.be(false); + }); + it('returns data grid with only data within retention KSPM', async () => { + await findings.index.add([...dataOldKspm, ...dataWithinRetentionKspm]); + + await findings.navigateToLatestFindingsPage(); + await retry.waitFor( + 'Findings table to be loaded', + async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionKspm.length + ); + await pageObjects.header.waitUntilLoadingHasFinished(); + }); + it('returns data grid with only data within retention CSPM', async () => { + await findings.index.add([...dataOldCspm, ...dataWithinRetentionCspm]); + + await findings.navigateToLatestFindingsPage(); + await retry.waitFor( + 'Findings table to be loaded', + async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionCspm.length + ); + await pageObjects.header.waitUntilLoadingHasFinished(); + }); + it('returns data grid with only data within retention CSPM', async () => { + await findings.vulnerabilitiesIndex.add([...dataOldCnvm, ...dataWithinRetentionCnvm]); + + await findings.navigateToLatestVulnerabilitiesPage(); + await retry.waitFor( + 'Findings table to be loaded', + async () => + (await latestVulnerabilitiesTable.getRowsCount()) === dataWithinRetentionCnvm.length + ); + await pageObjects.header.waitUntilLoadingHasFinished(); + }); }); }); }