[Security Solution][Detection Engine] fixes ES|QL data tier filter fr…

…om adv settings (elastic#196390)

## Summary

- fixes absent data tier filter for ES|QL rule
- followup to elastic#186908

### Demo


https://github.com/user-attachments/assets/a6f1290f-ea77-43bf-8def-42712ca5d1b0



### How to test

Create a deployment with cold and frozen data tiers and use following
commands to create index and ILM


<details>
<summary>Data tiers commands</summary>

```JSON

PUT /_cluster/settings
{
  "persistent": {
    "indices.lifecycle.poll_interval": "1m"
  }
}


PUT /_ilm/policy/filtering_data_tiers
{
  "policy": {
    "phases": {
        "frozen": {
          "min_age": "10m",
          "actions": {
            "searchable_snapshot": {
              "snapshot_repository": "found-snapshots",
              "force_merge_index": true
            }
          }
        },
        "cold": {
          "min_age": "1m",
          "actions": {
            "searchable_snapshot": {
              "snapshot_repository": "found-snapshots",
              "force_merge_index": true
            },
            "set_priority": {
              "priority": 0
            }
          }
        },
        "hot": {
          "min_age": "0ms",
          "actions": {
            "set_priority": {
              "priority": 100
            }
          }
        }
    }
  }
}


PUT /_index_template/filtering_data_tiers_template
{
  "index_patterns": [
    "filtering_data_tiers*"
  ],
  "template": {
    "settings": {
      "index.lifecycle.name": "filtering_data_tiers",
      "index.lifecycle.rollover_alias": "test-filtering_data_tiers"
    },
    "mappings": {
      "_meta": {
        "version": "1.6.0"
      },
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "host": {
          "properties": {
            "name": {
              "type": "keyword",
              "ignore_above": 1024
            }
          }
        }
      }
    }
  }
}

PUT /filtering_data_tiers-000001
{
  "aliases": {
    "filtering_data_tiers": {
      "is_write_index": true
    }
  }
}


POST filtering_data_tiers/_doc
{
  "@timestamp": "2024-07-08T17:00:01.000Z",
  "host.name": "test-0"
}


```

</details>

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts

@@ -30,6 +30,7 @@ import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters';
 import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
 import type { CreateRuleOptions, RunOpts, SignalSource } from '../types';
 import { logEsqlRequest } from '../utils/logged_requests';
+import { getDataTierFilter } from '../utils/get_data_tier_filter';
 import * as i18n from '../translations';
 
 import {
@@ -90,6 +91,10 @@ export const esqlExecutor = async ({
   return withSecuritySpan('esqlExecutor', async () => {
     const result = createSearchAfterReturnType();
     let size = tuple.maxSignals;
+    const dataTiersFilters = await getDataTierFilter({
+      uiSettingsClient: services.uiSettingsClient,
+    });
+
     try {
       while (
         result.createdSignalsCount <= tuple.maxSignals &&
@@ -100,7 +105,7 @@ export const esqlExecutor = async ({
           from: tuple.from.toISOString(),
           to: tuple.to.toISOString(),
           size,
-          filters: [],
+          filters: dataTiersFilters,
           primaryTimestamp,
           secondaryTimestamp,
           exceptionFilter,

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts

@@ -14,6 +14,7 @@ import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/comm
 import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring';
 
 import { getMaxSignalsWarning as getMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils';
+import { EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION } from '@kbn/security-solution-plugin/common/constants';
 import {
   getPreviewAlerts,
   previewRule,
@@ -25,6 +26,7 @@ import {
   scheduleRuleRun,
   stopAllManualRuns,
   waitForBackfillExecuted,
+  setAdvancedSettings,
 } from '../../../../utils';
 import {
   deleteAllRules,
@@ -1428,6 +1430,12 @@ export default ({ getService }: FtrProviderContext) => {
         await indexEnhancedDocuments({ documents: [doc1], interval, id });
       });
 
+      afterEach(async () => {
+        await setAdvancedSettings(supertest, {
+          [EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION]: [],
+        });
+      });
+
       it('should not return requests property when not enabled', async () => {
         const { logs } = await previewRule({
           supertest,
@@ -1463,6 +1471,35 @@ export default ({ getService }: FtrProviderContext) => {
           'POST /ecs_compliant/_search?ignore_unavailable=true'
         );
       });
+      it('should not return requests with any data tier filter', async () => {
+        const { logs } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          enableLoggedRequests: true,
+        });
+
+        const requests = logs[0].requests;
+
+        expect(requests![0].request).not.toContain('data_frozen');
+      });
+      it('should return requests with included data tiers filters from advanced settings', async () => {
+        await setAdvancedSettings(supertest, {
+          [EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION]: ['data_frozen'],
+        });
+        const { logs } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          enableLoggedRequests: true,
+        });
+
+        const requests = logs[0].requests;
+
+        expect(requests![0].request).toMatch(
+          /"must_not":\s*\[\s*{\s*"terms":\s*{\s*"_tier":\s*\[\s*"data_frozen"\s*\]/
+        );
+      });
     });
   });
 };

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts

@@ -24,3 +24,4 @@ export * from './get_stats';
 export * from './get_detection_metrics_from_body';
 export * from './get_stats_url';
 export * from './combine_to_ndjson';
+export * from './set_advanced_settings';

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/set_advanced_settings.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ELASTIC_HTTP_VERSION_HEADER,
+  X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
+} from '@kbn/core-http-common';
+
+import type SuperTest from 'supertest';
+
+export const setAdvancedSettings = async (
+  supertest: SuperTest.Agent,
+  settings: Record<string, string[] | string | number | boolean>
+) => {
+  return supertest
+    .post('/internal/kibana/settings')
+    .set('kbn-xsrf', 'true')
+    .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+    .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+    .send({ changes: settings })
+    .expect(200);
+};