[Cloud Security] Increase retention period on queries related to 3rd …

…party data loading (elastic#195636) ## Summary Increase retention on Cloud Security queries to accommodate a longer retention period on third-party CDR integrations, such as Wiz and AWS SecurityHub. This introduces regression for elastic#142198 This is meant is a temporary workaround until we find a robust way to get full posture for third-party CDR integrations This change goes together with increasing retention period on Wiz: - elastic/integrations#11393 fixes: - elastic/security-team#10683 ## How to test The CI deployed a serverless project where I installed Wiz and CSP integrations and ingested some data.
js-jankisalvi · Oct 11, 2024 · e18c52e · e18c52e
1 parent fb0a933
commit e18c52e
Show file tree

Hide file tree

Showing 9 changed files with 121 additions and 25 deletions.
diff --git a/x-pack/packages/kbn-cloud-security-posture-common/constants.ts b/x-pack/packages/kbn-cloud-security-posture-common/constants.ts
@@ -36,6 +36,10 @@ export const CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN =
 export const CDR_VULNERABILITIES_INDEX_PATTERN = `${CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN},${CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN}`;
 export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d';
 
+// TODO: remove once https://github.com/elastic/security-team/issues/10801 is done
+// meant as a temp workaround to get good enough posture view for 3rd party integrations, see https://github.com/elastic/security-team/issues/10683
+export const CDR_3RD_PARTY_RETENTION_POLICY = '90d';
+
 export const VULNERABILITIES_SEVERITY: Record<VulnSeverity, VulnSeverity> = {
   LOW: 'LOW',
   MEDIUM: 'MEDIUM',

diff --git a/x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts b/x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts
@@ -9,8 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import {
   CDR_MISCONFIGURATIONS_INDEX_PATTERN,
   CDR_VULNERABILITIES_INDEX_PATTERN,
-  LATEST_FINDINGS_RETENTION_POLICY,
-  LATEST_VULNERABILITIES_RETENTION_POLICY,
+  CDR_3RD_PARTY_RETENTION_POLICY,
 } from '@kbn/cloud-security-posture-common';
 import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
 import { buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
@@ -103,7 +102,7 @@ const buildMisconfigurationsFindingsQueryWithFilters = (
         {
           range: {
             '@timestamp': {
-              gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
+              gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
               lte: 'now',
             },
           },
@@ -182,7 +181,7 @@ const buildVulnerabilityFindingsQueryWithFilters = (query: UseCspOptions['query'
         {
           range: {
             '@timestamp': {
-              gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
+              gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
               lte: 'now',
             },
           },

diff --git a/...cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/...cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts
@@ -15,7 +15,7 @@ import { showErrorToast } from '@kbn/cloud-security-posture';
 import { MAX_FINDINGS_TO_LOAD, buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
 import {
   CDR_MISCONFIGURATIONS_INDEX_PATTERN,
-  LATEST_FINDINGS_RETENTION_POLICY,
+  CDR_3RD_PARTY_RETENTION_POLICY,
 } from '@kbn/cloud-security-posture-common';
 import type { CspFinding } from '@kbn/cloud-security-posture-common';
 import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
@@ -77,7 +77,7 @@ export const getFindingsQuery = (
           {
             range: {
               '@timestamp': {
-                gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
+                gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
                 lte: 'now',
               },
             },

diff --git a/...rity_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/...rity_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx
@@ -16,7 +16,7 @@ import {
 import { useMemo } from 'react';
 import { buildEsQuery, Filter } from '@kbn/es-query';
 import {
-  LATEST_FINDINGS_RETENTION_POLICY,
+  CDR_3RD_PARTY_RETENTION_POLICY,
   buildMutedRulesFilter,
 } from '@kbn/cloud-security-posture-common';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
@@ -249,7 +249,7 @@ export const useLatestFindingsGrouping = ({
     additionalFilters: query ? [query, additionalFilters] : [additionalFilters],
     groupByField: currentSelectedGroup,
     uniqueValue,
-    from: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
+    from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
     to: 'now',
     pageNumber: activePageIndex * pageSize,
     size: pageSize,

diff --git a/.../cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/.../cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx
@@ -19,7 +19,7 @@ import { EsHitRecord } from '@kbn/discover-utils/types';
 import {
   MAX_FINDINGS_TO_LOAD,
   CDR_VULNERABILITIES_INDEX_PATTERN,
-  LATEST_VULNERABILITIES_RETENTION_POLICY,
+  CDR_3RD_PARTY_RETENTION_POLICY,
 } from '@kbn/cloud-security-posture-common';
 import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
 import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
@@ -90,7 +90,7 @@ export const getVulnerabilitiesQuery = (
         {
           range: {
             '@timestamp': {
-              gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
+              gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
               lte: 'now',
             },
           },

diff --git a/...curity_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/...curity_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx
@@ -15,7 +15,7 @@ import {
 } from '@kbn/grouping/src';
 import { useMemo } from 'react';
 import {
-  LATEST_VULNERABILITIES_RETENTION_POLICY,
+  CDR_3RD_PARTY_RETENTION_POLICY,
   VULNERABILITIES_SEVERITY,
 } from '@kbn/cloud-security-posture-common';
 import { buildEsQuery, Filter } from '@kbn/es-query';
@@ -202,7 +202,7 @@ export const useLatestVulnerabilitiesGrouping = ({
     additionalFilters: query ? [query, additionalFilters] : [additionalFilters],
     groupByField: currentSelectedGroup,
     uniqueValue,
-    from: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
+    from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
     to: 'now',
     pageNumber: activePageIndex * pageSize,
     size: pageSize,

diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts
@@ -15,6 +15,7 @@ import {
   CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN,
   LATEST_VULNERABILITIES_RETENTION_POLICY,
   CDR_VULNERABILITIES_INDEX_PATTERN,
+  CDR_3RD_PARTY_RETENTION_POLICY,
 } from '@kbn/cloud-security-posture-common';
 import type {
   CspSetupStatus,
@@ -218,13 +219,13 @@ export const getCspStatus = async ({
     checkIndexHasFindings(
       esClient,
       CDR_MISCONFIGURATIONS_INDEX_PATTERN,
-      LATEST_FINDINGS_RETENTION_POLICY,
+      CDR_3RD_PARTY_RETENTION_POLICY,
       logger
     ),
     checkIndexHasFindings(
       esClient,
       CDR_VULNERABILITIES_INDEX_PATTERN,
-      LATEST_VULNERABILITIES_RETENTION_POLICY,
+      CDR_3RD_PARTY_RETENTION_POLICY,
       logger
     ),
     checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {

diff --git a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
@@ -366,8 +366,7 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
   });
   const isLatestFindingsTableThere = async () => {
     const table = await testSubjects.findAll('docTable');
-    const trueOrFalse = table.length > 0 ? true : false;
-    return trueOrFalse;
+    return table.length > 0;
   };
 
   const getUnprivilegedPrompt = async () => {

diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts b/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts
@@ -8,16 +8,38 @@
 import expect from '@kbn/expect';
 import Chance from 'chance';
 import type { FtrProviderContext } from '../ftr_provider_context';
+import { vulnerabilitiesLatestMock } from '../mocks/vulnerabilities_latest_mock';
 
 // eslint-disable-next-line import/no-default-export
-export default function ({ getPageObjects }: FtrProviderContext) {
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const retry = getService('retry');
   const pageObjects = getPageObjects(['common', 'findings', 'header']);
   const chance = new Chance();
-  const hoursToMillisecond = (hours: number) => hours * 60 * 60 * 1000;
+  const daysToMillisecond = (days: number) => days * 24 * 60 * 60 * 1000;
+  const RETENTION = 90;
 
   const dataOldKspm = [
     {
-      '@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
+      '@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
+      resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
+      result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
+      rule: {
+        name: 'Upper case rule name',
+        section: 'Upper case section',
+        benchmark: {
+          id: 'cis_k8s',
+          posture_type: 'kspm',
+          name: 'CIS Kubernetes V1.23',
+          version: 'v1.0.0',
+        },
+        type: 'process',
+      },
+      cluster_id: 'Upper case cluster id',
+    },
+  ];
+  const dataWithinRetentionKspm = [
+    {
+      '@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
       resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
       result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
       rule: {
@@ -37,7 +59,26 @@ export default function ({ getPageObjects }: FtrProviderContext) {
 
   const dataOldCspm = [
     {
-      '@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
+      '@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
+      resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
+      result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
+      rule: {
+        name: 'Upper case rule name',
+        section: 'Upper case section',
+        benchmark: {
+          id: 'cis_aws',
+          posture_type: 'cspm',
+          name: 'CIS AWS V1.23',
+          version: 'v1.0.0',
+        },
+        type: 'process',
+      },
+      cluster_id: 'Upper case cluster id',
+    },
+  ];
+  const dataWithinRetentionCspm = [
+    {
+      '@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
       resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
       result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
       rule: {
@@ -55,40 +96,92 @@ export default function ({ getPageObjects }: FtrProviderContext) {
     },
   ];
 
+  const dataOldCnvm = [
+    {
+      ...vulnerabilitiesLatestMock[0],
+      '@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
+    },
+  ];
+  const dataWithinRetentionCnvm = [
+    {
+      ...vulnerabilitiesLatestMock[0],
+      '@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
+    },
+  ];
+
   describe('Old Data', function () {
     this.tags(['cloud_security_posture_findings']);
     let findings: typeof pageObjects.findings;
+    let latestFindingsTable: typeof findings.latestFindingsTable;
+    let latestVulnerabilitiesTable: typeof findings.latestVulnerabilitiesTable;
 
     before(async () => {
       findings = pageObjects.findings;
+      latestFindingsTable = findings.latestFindingsTable;
+      latestVulnerabilitiesTable = findings.latestVulnerabilitiesTable;
 
       // Before we start any test we must wait for cloud_security_posture plugin to complete its initialization
       await findings.waitForPluginInitialized();
     });
 
-    after(async () => {
+    afterEach(async () => {
       await findings.index.remove();
+      await findings.vulnerabilitiesIndex.remove();
     });
 
     describe('Findings page with old data', () => {
       it('returns no Findings KSPM', async () => {
-        // Prepare mocked findings
-        await findings.index.remove();
         await findings.index.add(dataOldKspm);
 
         await findings.navigateToLatestFindingsPage();
         await pageObjects.header.waitUntilLoadingHasFinished();
         expect(await findings.isLatestFindingsTableThere()).to.be(false);
       });
       it('returns no Findings CSPM', async () => {
-        // Prepare mocked findings
-        await findings.index.remove();
         await findings.index.add(dataOldCspm);
 
         await findings.navigateToLatestFindingsPage();
         await pageObjects.header.waitUntilLoadingHasFinished();
         expect(await findings.isLatestFindingsTableThere()).to.be(false);
       });
+      it('returns no Findings CNVM', async () => {
+        await findings.vulnerabilitiesIndex.add(dataOldCnvm);
+
+        await findings.navigateToLatestVulnerabilitiesPage();
+        await pageObjects.header.waitUntilLoadingHasFinished();
+        expect(await findings.isLatestFindingsTableThere()).to.be(false);
+      });
+      it('returns data grid with only data within retention KSPM', async () => {
+        await findings.index.add([...dataOldKspm, ...dataWithinRetentionKspm]);
+
+        await findings.navigateToLatestFindingsPage();
+        await retry.waitFor(
+          'Findings table to be loaded',
+          async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionKspm.length
+        );
+        await pageObjects.header.waitUntilLoadingHasFinished();
+      });
+      it('returns data grid with only data within retention CSPM', async () => {
+        await findings.index.add([...dataOldCspm, ...dataWithinRetentionCspm]);
+
+        await findings.navigateToLatestFindingsPage();
+        await retry.waitFor(
+          'Findings table to be loaded',
+          async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionCspm.length
+        );
+        await pageObjects.header.waitUntilLoadingHasFinished();
+      });
+      it('returns data grid with only data within retention CSPM', async () => {
+        await findings.vulnerabilitiesIndex.add([...dataOldCnvm, ...dataWithinRetentionCnvm]);
+
+        await findings.navigateToLatestVulnerabilitiesPage();
+        await retry.waitFor(
+          'Findings table to be loaded',
+          async () =>
+            (await latestVulnerabilitiesTable.getRowsCount()) === dataWithinRetentionCnvm.length
+        );
+        await pageObjects.header.waitUntilLoadingHasFinished();
+      });
     });
   });
 }