[Rule Tuning] Add KEEP Command to all ES|QL Rules (#4146)

terrancedejesus · web-flow · commit 06319b7a1304 · 2024-10-09T21:08:38.000-04:00
* updating ES|QL rules to include KEEP command

* fixed some ES|QL rules with typos; added validation for KEEP command

* fixed ES|QL errors from missing fields

* fixed flake errors

* updated date

* added best practices to hunt docs
diff --git a/detection_rules/rule.py b/detection_rules/rule.py
@@ -926,6 +926,13 @@ def validates_esql_data(self, data, **kwargs):
                 f" Add 'metadata _id, _version, _index' to the from command or add an aggregate function."
             )
 
+        # Enforce KEEP command for ESQL rules
+        if '| keep' not in query_lower:
+            raise ValidationError(
+                f"Rule: {data['name']} does not contain a 'keep' command ->"
+                f" Add a 'keep' command to the query."
+            )
+
 
 @dataclass(frozen=True)
 class ThreatMatchRuleData(QueryRuleData):
diff --git a/hunting/README.md b/hunting/README.md
@@ -44,6 +44,12 @@ Otherwise, the names do not require the integration, since it is already annotat
   - **mitre**: Reference to applicable MITRE ATT&CK tactics or techniques that the rule addresses, enhancing the contextual understanding of its security implications.
   - **references**: Links to external documents, research papers, or websites that provide additional information or validation for the detection logic.
 
+#### Query Best Practices
+* Use `KEEP` command to select specific fields that are relevant or necessary for `STATS` command
+* Use `LIMIT` command to limit the number of results, depending on expected result volume
+* Filter as much as possible in `WHERE` command to reduce events needed to be processed
+* For `FROM` command for index patterns, be as specific as possible to reduce potential event matches that are irrelevant
+
 ### Field Usage
 Use standardized fields where possible to ensure that queries are compatible across different data environments and sources.
 
diff --git a/rules/cross-platform/execution_potential_widespread_malware_infection.toml b/rules/cross-platform/execution_potential_widespread_malware_infection.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/08"
 maturity = "production"
-updated_date = "2024/05/08"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully"
 min_stack_version = "8.13.0"
 
@@ -34,6 +34,7 @@ type = "esql"
 query = '''
 from logs-endpoint.alerts-*
 | where event.code in ("malicious_file", "memory_signature", "shellcode_thread") and rule.name is not null
+| keep host.id, rule.name, event.code
 | stats hosts = count_distinct(host.id) by rule.name, event.code
 | where hosts >= 3
 '''
diff --git a/rules/integrations/aws/discovery_ec2_multi_region_describe_instances.toml b/rules/integrations/aws/discovery_ec2_multi_region_describe_instances.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/08/26"
 maturity = "production"
-updated_date = "2024/08/26"
+updated_date = "2024/10/09"
 
 [rule]
 author = ["Elastic"]
@@ -41,6 +41,9 @@ from logs-aws.cloudtrail-*
 // truncate the timestamp to a 30-second window
 | eval target_time_window = DATE_TRUNC(30 seconds, @timestamp)
 
+// keep only the relevant fields
+| keep target_time_window, aws.cloudtrail.user_identity.arn, cloud.region
+
 // count the number of unique regions and total API calls within the 30-second window
 | stats region_count = count_distinct(cloud.region), window_count = count(*) by target_time_window, aws.cloudtrail.user_identity.arn
 
diff --git a/rules/integrations/aws/discovery_servicequotas_multi_region_service_quota_requests.toml b/rules/integrations/aws/discovery_servicequotas_multi_region_service_quota_requests.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/08/26"
 maturity = "production"
-updated_date = "2024/10/02"
+updated_date = "2024/10/09"
 
 [rule]
 author = ["Elastic"]
@@ -48,6 +48,9 @@ from logs-aws.cloudtrail-*
 // filter for EC2 service quota L-1216C47A (vCPU on-demand instances)
 | where service_code == "ec2" and quota_code == "L-1216C47A"
 
+// keep only the relevant fields
+| keep target_time_window, aws.cloudtrail.user_identity.arn, cloud.region, service_code, quota_code
+
 // count the number of unique regions and total API calls within the 30-second window
 | stats region_count = count_distinct(cloud.region), window_count = count(*) by target_time_window, aws.cloudtrail.user_identity.arn
 
diff --git a/rules/integrations/aws/impact_aws_s3_bucket_enumeration_or_brute_force.toml b/rules/integrations/aws/impact_aws_s3_bucket_enumeration_or_brute_force.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/01"
 maturity = "production"
-updated_date = "2024/07/06"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully"
 min_stack_version = "8.13.0"
 
@@ -89,6 +89,8 @@ type = "esql"
 query = '''
 from logs-aws.cloudtrail*
 | where event.provider == "s3.amazonaws.com" and aws.cloudtrail.error_code == "AccessDenied"
+// keep only relevant fields
+| keep tls.client.server_name, source.address, cloud.account.id
 | stats failed_requests = count(*) by tls.client.server_name, source.address, cloud.account.id
   // can modify the failed request count or tweak time window to fit environment
   // can add `not cloud.account.id in (KNOWN)` or specify in exceptions
diff --git a/rules/integrations/aws/impact_s3_bucket_object_uploaded_with_ransom_extension.toml b/rules/integrations/aws/impact_s3_bucket_object_uploaded_with_ransom_extension.toml
@@ -4,7 +4,7 @@ integration = ["aws"]
 maturity = "production"
 min_stack_comments = "AWS integration breaking changes, bumping version to ^2.0.0"
 min_stack_version = "8.13.0"
-updated_date = "2024/07/01"
+updated_date = "2024/10/09"
 
 [rule]
 author = ["Elastic"]
@@ -98,6 +98,9 @@ from logs-aws.cloudtrail-*
 | where object_name rlike "(.*)(ransom|lock|crypt|enc|readme|how_to_decrypt|decrypt_instructions|recovery|datarescue)(.*)"
     and not object_name rlike "(.*)(AWSLogs|CloudTrail|access-logs)(.*)"
 
+// keep relevant fields
+| keep tls.client.server_name, aws.cloudtrail.user_identity.arn, object_name
+
 // aggregate by S3 bucket, resource and object name
 | stats note_upload_count = count(*) by tls.client.server_name, aws.cloudtrail.user_identity.arn, object_name
 
diff --git a/rules/integrations/aws/impact_s3_object_encryption_with_external_key.toml b/rules/integrations/aws/impact_s3_object_encryption_with_external_key.toml
@@ -4,7 +4,7 @@ integration = ["aws"]
 maturity = "production"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13"
 min_stack_version = "8.13.0"
-updated_date = "2024/10/02"
+updated_date = "2024/10/09"
 
 [rule]
 author = ["Elastic"]
@@ -95,8 +95,10 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
 // filter for s3 objects whose account id is different from the encryption key's account id
 // add exceptions based on key.account.id or keyId for known external accounts or encryption keys
 | where cloud.account.id != key.account.id
-'''
 
+// keep relevant fields
+| keep @timestamp, aws.cloudtrail.user_identity.arn, cloud.account.id, event.action, target.bucketName, key.account.id, keyId, target.objectName
+'''
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"
diff --git a/rules/integrations/aws/initial_access_signin_console_login_no_mfa.toml b/rules/integrations/aws/initial_access_signin_console_login_no_mfa.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/08/19"
 integration = ['aws']
 maturity = "production"
-updated_date = "2024/10/02"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13"
 min_stack_version = "8.13.0"
 
@@ -44,6 +44,7 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
     and aws.cloudtrail.user_identity.type == "FederatedUser"
 | dissect aws.cloudtrail.additional_eventdata "{%{?mobile_version_key}=%{mobile_version}, %{?mfa_used_key}=%{mfa_used}}"
 | where mfa_used == "No"
+| keep @timestamp, event.action, aws.cloudtrail.event_type, aws.cloudtrail.user_identity.type
 '''
 
 [[rule.threat]]
diff --git a/rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml b/rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/06/13"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/10/02"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13"
 min_stack_version = "8.13.0"
 
@@ -104,6 +104,7 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
     and event.action == "CreateAccessKey"
     and event.outcome == "success"
     and user.name != user.target.name
+| keep @timestamp, event.provider, event.action, event.outcome, user.name, user.target.name
 '''
 
 
diff --git a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_group.toml b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_group.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/05/31"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/10/02"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13."
 min_stack_version = "8.13.0"
 
@@ -104,6 +104,7 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
 | where event.provider == "iam.amazonaws.com" and event.action == "AttachGroupPolicy" and event.outcome == "success"
 | dissect aws.cloudtrail.request_parameters "{%{?policyArn}=%{?arn}:%{?aws}:%{?iam}::%{?aws}:%{?policy}/%{policyName},%{?groupName}=%{group.name}}"
 | where policyName == "AdministratorAccess"
+| keep @timestamp, event.provider, event.action, event.outcome, policyName, group.name
 '''
 
 
diff --git a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_role.toml b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_role.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/05/31"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/10/02"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13."
 min_stack_version = "8.13.0"
 
@@ -103,6 +103,7 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
 | where event.provider == "iam.amazonaws.com" and event.action == "AttachRolePolicy" and event.outcome == "success"
 | dissect aws.cloudtrail.request_parameters "{%{?policyArn}=%{?arn}:%{?aws}:%{?iam}::%{?aws}:%{?policy}/%{policyName},%{?roleName}=%{role.name}}"
 | where policyName == "AdministratorAccess"
+| keep @timestamp, event.provider, event.action, event.outcome, policyName, role.name
 '''
 
 
diff --git a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/05/30"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/10/02"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13."
 min_stack_version = "8.13.0"
 
@@ -103,6 +103,7 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
 | where event.provider == "iam.amazonaws.com" and event.action == "AttachUserPolicy" and event.outcome == "success"
 | dissect aws.cloudtrail.request_parameters "{%{?policyArn}=%{?arn}:%{?aws}:%{?iam}::%{?aws}:%{?policy}/%{policyName},%{?userName}=%{target.userName}}"
 | where policyName == "AdministratorAccess"
+| keep @timestamp, event.provider, event.action, event.outcome, policyName, target.userName
 '''
 
 
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_by_single_user.toml b/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_by_single_user.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/02"
 maturity = "production"
-updated_date = "2024/09/27"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
@@ -48,6 +48,7 @@ type = "esql"
 query = '''
 from logs-aws_bedrock.invocation-*
 | where gen_ai.compliance.violation_detected
+| keep user.id, gen_ai.request.model.id, cloud.account.id
 | stats violations = count(*) by user.id, gen_ai.request.model.id, cloud.account.id
 | where violations > 1
 | sort violations desc
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_in_single_request.toml b/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_in_single_request.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/02"
 maturity = "production"
-updated_date = "2024/05/02"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
@@ -50,6 +50,7 @@ from logs-aws_bedrock.invocation-*
 | where gen_ai.policy.action == "BLOCKED"
 | eval policy_violations = mv_count(gen_ai.policy.name)
 | where policy_violations > 1
+| keep gen_ai.policy.action, policy_violations, user.id, gen_ai.request.model.id, cloud.account.id, user.id
 | stats total_unique_request_violations = count(*) by policy_violations, user.id, gen_ai.request.model.id, cloud.account.id
 | sort total_unique_request_violations desc
 '''
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_high_confidence_misconduct_blocks_detected.toml b/rules/integrations/aws_bedrock/aws_bedrock_high_confidence_misconduct_blocks_detected.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/05"
 maturity = "production"
-updated_date = "2024/05/05"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
@@ -46,6 +46,7 @@ type = "esql"
 query = '''
 from logs-aws_bedrock.invocation-*
 | where gen_ai.policy.confidence == "HIGH" and gen_ai.policy.action == "BLOCKED" and gen_ai.compliance.violation_code == "MISCONDUCT"
+| keep gen_ai.policy.confidence, gen_ai.policy.action, gen_ai.compliance.violation_code, user.id
 | stats high_confidence_blocks = count() by user.id
 | where high_confidence_blocks > 5
 | sort high_confidence_blocks desc
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_high_resource_consumption_detection.toml b/rules/integrations/aws_bedrock/aws_bedrock_high_resource_consumption_detection.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/04"
 maturity = "production"
-updated_date = "2024/05/04"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
@@ -46,6 +46,7 @@ type = "esql"
 
 query = '''
 from logs-aws_bedrock.invocation-*
+| keep user.id, gen_ai.usage.prompt_tokens, gen_ai.usage.completion_tokens
 | stats max_tokens = max(gen_ai.usage.prompt_tokens),
          total_requests = count(*),
          avg_response_size = avg(gen_ai.usage.completion_tokens)
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_multiple_attempts_to_use_denied_models_by_user.toml b/rules/integrations/aws_bedrock/aws_bedrock_multiple_attempts_to_use_denied_models_by_user.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/02"
 maturity = "production"
-updated_date = "2024/05/02"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
@@ -48,6 +48,7 @@ type = "esql"
 query = '''
 from logs-aws_bedrock.invocation-*
 | where gen_ai.response.error_code == "AccessDeniedException"
+| keep user.id, gen_ai.request.model.id, cloud.account.id, gen_ai.response.error_code
 | stats total_denials = count(*) by user.id, gen_ai.request.model.id, cloud.account.id
 | where total_denials > 3
 | sort total_denials desc
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_multiple_validation_exception_errors_by_single_user.toml b/rules/integrations/aws_bedrock/aws_bedrock_multiple_validation_exception_errors_by_single_user.toml
@@ -2,15 +2,15 @@
 creation_date = "2024/09/11"
 integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2024/09/11"
+updated_date = "2024/10/09"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
 [rule]
 author = ["Elastic"]
 description = """
-Identifies multiple validation exeception errors within AWS Bedrock. Validation errors occur when you run the InvokeModel or 
-InvokeModelWithResponseStream APIs on a foundation model that uses an incorrect inference parameter or corresponding value. 
+Identifies multiple validation exeception errors within AWS Bedrock. Validation errors occur when you run the InvokeModel or
+InvokeModelWithResponseStream APIs on a foundation model that uses an incorrect inference parameter or corresponding value.
 These errors also occur when you use an inference parameter for one model with a model that doesn't have the same API parameter.
 This could indicate attempts to bypass limitations of other approved models, or to force an impact on the environment by incurring
 exhorbitant costs.
@@ -54,6 +54,7 @@ from logs-aws_bedrock.invocation-*
 // truncate the timestamp to a 1-minute window
 | eval target_time_window = DATE_TRUNC(1 minutes, @timestamp)
 | where gen_ai.response.error_code == "ValidationException"
+| keep user.id, gen_ai.request.model.id, cloud.account.id, gen_ai.response.error_code, target_time_window
 // count the number of users causing validation errors within a 1 minute window
 | stats total_denials = count(*) by target_time_window, user.id, cloud.account.id
 | where total_denials > 3
diff --git a/rules/integrations/azure/credential_access_entra_signin_brute_force_microsoft_365.toml b/rules/integrations/azure/credential_access_entra_signin_brute_force_microsoft_365.toml
@@ -4,7 +4,7 @@ integration = ["azure"]
 maturity = "production"
 min_stack_comments = "ES|QL not available until 8.13.0 in technical preview."
 min_stack_version = "8.13.0"
-updated_date = "2024/09/06"
+updated_date = "2024/10/09"
 
 [rule]
 author = ["Elastic"]
@@ -58,6 +58,10 @@ from logs-azure.signinlogs*
   and event.outcome != "success"
   // for tuning review azure.signinlogs.properties.status.error_code
   // https://learn.microsoft.com/en-us/entra/identity-platform/reference-error-codes
+
+// keep only relevant fields
+| keep target_time_window, event.dataset, event.category, azure.signinlogs.properties.resource_display_name, azure.signinlogs.category, event.outcome, azure.signinlogs.properties.user_principal_name, source.ip
+
 // count the number of login sources and failed login attempts
 | stats
   login_source_count = count(source.ip),
diff --git a/rules/integrations/azure/credential_access_entra_signin_brute_force_microsoft_365_repeat_source.toml b/rules/integrations/azure/credential_access_entra_signin_brute_force_microsoft_365_repeat_source.toml
@@ -4,7 +4,7 @@ integration = ["azure"]
 maturity = "production"
 min_stack_comments = "ES|QL not available until 8.13.0 in technical preview."
 min_stack_version = "8.13.0"
-updated_date = "2024/09/06"
+updated_date = "2024/10/09"
 
 [rule]
 author = ["Elastic"]
@@ -58,6 +58,9 @@ from logs-azure.signinlogs*
   // For tuning, review azure.signinlogs.properties.status.error_code
   // https://learn.microsoft.com/en-us/entra/identity-platform/reference-error-codes
 
+// keep only relevant fields
+| keep event.dataset, event.category, azure.signinlogs.properties.resource_display_name, azure.signinlogs.category, event.outcome, azure.signinlogs.properties.user_principal_name, source.ip
+
 // Count the number of unique targets per source IP
 | stats
   target_count = count_distinct(azure.signinlogs.properties.user_principal_name) by source.ip
diff --git a/rules/integrations/o365/credential_access_microsoft_365_brute_force_user_account_attempt.toml b/rules/integrations/o365/credential_access_microsoft_365_brute_force_user_account_attempt.toml
@@ -4,7 +4,7 @@ integration = ["o365"]
 maturity = "production"
 min_stack_comments = "ES|QL not available until 8.13.0 in technical preview."
 min_stack_version = "8.13.0"
-updated_date = "2024/09/25"
+updated_date = "2024/10/09"
 
 [rule]
 author = ["Elastic", "Willem D'Haese", "Austin Songer"]
@@ -75,6 +75,9 @@ from logs-o365.audit-*
   // filters only for logins from user or application, ignoring oauth:token
   and to_lower(o365.audit.ExtendedProperties.RequestType) rlike "(.*)login(.*)"
 
+// keep only relevant fields
+| keep event.provider, event.dataset, event.category, o365.audit.UserId, event.action, source.ip, o365.audit.LogonError, o365.audit.ExtendedProperties.RequestType, o365.audit.Target.Type, target_time_window
+
 // count the number of login sources and failed login attempts
 | stats
   login_source_count = count(source.ip),
diff --git a/rules/integrations/okta/credential_access_multiple_device_token_hashes_for_single_okta_session.toml b/rules/integrations/okta/credential_access_multiple_device_token_hashes_for_single_okta_session.toml
diff --git a/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_from_single_source.toml b/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_from_single_source.toml
diff --git a/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_with_the_same_device_token_hash.toml b/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_with_the_same_device_token_hash.toml
diff --git a/rules/integrations/okta/credential_access_okta_multiple_device_token_hashes_for_single_user.toml b/rules/integrations/okta/credential_access_okta_multiple_device_token_hashes_for_single_user.toml
diff --git a/rules/integrations/okta/initial_access_okta_user_sessions_started_from_different_geolocations.toml b/rules/integrations/okta/initial_access_okta_user_sessions_started_from_different_geolocations.toml
