Merge branch 'Azure:master' into master

Author: shishirdw

.script/tests/KqlvalidationsTests/CustomTables/Illumio_Auditable_Events_CL.json

@@ -44,6 +44,14 @@
         {
             "name": "version",
             "type": "int"
+        },
+        {
+            "name": "TenantId",
+            "type": "String"
+        },
+        {
+            "Name": "_ItemId",
+            "Type": "String"
         }
     ]
 }

.script/tests/KqlvalidationsTests/CustomTables/Illumio_Flow_Events_CL.json

@@ -45,6 +45,10 @@
             "name": "un",
             "type": "string"
         },
+        {
+            "name": "sn",
+            "type": "string"
+        },        
         {
             "name": "src_ip",
             "type": "string"
@@ -128,6 +132,18 @@
         {
             "name": "version",
             "type": "int"
-        }
+        },
+        {
+            "name": "icmp_type",
+            "type": "int"
+        },        
+        {
+            "name": "TenantId",
+            "type": "String"
+        },
+        {
+            "Name": "_ItemId",
+            "Type": "String"
+        }        
     ]
 }

.script/tests/asimParsersTest/ingestASimSampleData.py

@@ -267,6 +267,28 @@ def extract_event_vendor_product(parser_query,parser_file):
         print(f'Event Product field not mapped in parser. Please map it in parser query.{parser_file}')
     return event_vendor, event_product ,schema_name   
 
+def convert_data_type(schema_result, data_result):
+    for data in data_result:
+        for schema in schema_result:
+            field_name = schema["name"]
+            field_type = schema["type"]
+            
+            if field_name in data:
+                value = data[field_name]
+                
+                # Handle conversion based on schema type
+                
+                if field_type == "string":
+                    # Convert to string
+                    data[field_name] = str(value)
+                elif field_type == "boolean":
+                    # Convert to boolean
+                    if isinstance(value, str) and value.lower() in ["true", "false"]:
+                        data[field_name] = value.lower() == "true"
+
+    return data_result
+
+
 #main starting point of script
 
 workspace_id = "e9beceee-7d61-429f-a177-ee5e2b7f481a"
@@ -336,7 +358,11 @@ def extract_event_vendor_product(parser_query,parser_file):
         else:
             print(f"::error::An error occurred while trying to get content of Schema file located at {schemaUrl}: {response.text}")
             continue        
-        schema_result = convert_schema_csv_to_json('tempfile.csv') 
+        schema_result = convert_schema_csv_to_json('tempfile.csv')
+        data_result = convert_data_type(schema_result, data_result)
+        # conversion of datatype is needed for boolean and string values because during testing it has been observed that 
+        # boolean values are consider as string and numerical value of type string are consider 
+        # as integer which leds to non ingestion of those value in sentinel    
         # create table 
         request_body, url_to_call , method_to_use = create_table(json.dumps(schema_result, indent=4),table_name)
         response_body=hit_api(url_to_call,request_body,method_to_use)

ASIM/dev/ASimTester/ASimTester.csv

@@ -547,13 +547,13 @@ EventOwner,string,Optional,ProcessEvent,,,
 EventOwner,string,Optional,RegistryEvent,,,
 EventOwner,string,Optional,UserManagement,,,
 EventOwner,string,Optional,WebSession,,,
-EventProduct,string,Mandatory,Authentication,Enumerated,Service Cloud|Auth0|CloudTrail|AAD|ASA|Microsoft Defender for IoT|ISE|M365 Defender for Endpoint|Meraki|Security Events|Okta|PostgreSQL|OpenSSH|su|sudo|Vectra XDR|SentinelOne|WAF|FalconHost|Carbon Black Cloud|Cortex Data Lake|Workspace,
-EventProduct,string,Mandatory,AuditEvent,Enumerated,Azure|WAF|Security Events|Exchange 365|Dataminr Pulse|ISE|XDR|Meraki|FalconHost|SentinelOne|Carbon Black Cloud|BloxOne,
+EventProduct,string,Mandatory,Authentication,Enumerated,Service Cloud|Auth0|CloudTrail|AAD|ASA|Microsoft Defender for IoT|ISE|M365 Defender for Endpoint|Meraki|Security Events|Okta|PostgreSQL|OpenSSH|su|sudo|Vectra XDR|SentinelOne|WAF|FalconHost|Carbon Black Cloud|Cortex Data Lake|Workspace|Core,
+EventProduct,string,Mandatory,AuditEvent,Enumerated,Azure|WAF|Security Events|Exchange 365|Dataminr Pulse|ISE|XDR|Meraki|FalconHost|SentinelOne|Carbon Black Cloud|BloxOne|Core,
 EventProduct,string,Mandatory,Common,,,
 EventProduct,string,Mandatory,DhcpEvent,,BloxOne,
 EventProduct,string,Mandatory,FileEvent,Enumerated,Security Events|Sysmon for Linux|Sysmon|M365 Defender for Endpoint|Azure File Storage|SharePoint|OneDrive|SentinelOne|Carbon Black Cloud|Workspace,
 EventProduct,string,Mandatory,Dns,Enumerated,Umbrella|Azure Firewall|DNS Server|Sysmon|Sysmon for Linux|ZIA DNS|NIOS|Cloud DNS|Zeek|Vectra Stream|SentinelOne|FortiGate|BloxOne,
-EventProduct,string,Mandatory,NetworkSession,Enumerated,Fortigate|IOS|ISE|SDP|Vectra Stream|NSGFlow|Fireware|VPC|Azure Defender for IoT|Azure Firewall|M365 Defender for Endpoint|Sysmon|Sysmon for Linux|Windows Firewall|WireData|ZIA Firewall|CDL|PanOS|VMConnection|Meraki|Zeek|Firewall|ASA|Cynerio|SentinelOne|WAF|Firepower|FalconHost|Carbon Black Cloud|Cortex Data Lake,
+EventProduct,string,Mandatory,NetworkSession,Enumerated,Fortigate|IOS|ISE|SDP|Vectra Stream|NSGFlow|Fireware|VPC|Azure Defender for IoT|Azure Firewall|M365 Defender for Endpoint|Sysmon|Sysmon for Linux|Windows Firewall|WireData|ZIA Firewall|CDL|PanOS|VMConnection|Meraki|Zeek|Firewall|ASA|Cynerio|SentinelOne|WAF|Firepower|FalconHost|Carbon Black Cloud|Cortex Data Lake|Core,
 EventProduct,string,Mandatory,ProcessEvent,Enumerated,M365 Defender for Endpoint|Sysmon for Linux|Sysmon|Azure Defender for IoT|Security Events|SentinelOne|Carbon Black Cloud|Vision One,
 EventProduct,string,Mandatory,RegistryEvent,Enumerated,M365 Defender for Endpoint|Security Events|Sysmon|Windows Event|SentinelOne|Carbon Black Cloud|Vision One,
 EventProduct,string,Mandatory,WebSession,Enumerated,IIS|Squid Proxy|ZIA Proxy|Vectra Stream|PanOS|CDL|Fireware|Meraki|Web Security Gateway|Zeek|Dataminr Pulse|HTTP Server|Fortigate|WAF|ASM|NetScaler|Firepower|Cortex Data Lake|Firewall,
@@ -677,13 +677,13 @@ EventUid,string,Recommended,ProcessEvent,,,
 EventUid,string,Recommended,RegistryEvent,,,
 EventUid,string,Recommended,UserManagement,,,
 EventUid,string,Recommended,WebSession,,,
-EventVendor,string,Mandatory,Authentication,Enumerated,Salesforce|AWS|Barracuda|Cisco|Microsoft|Okta|PostgreSQL|OpenBSD|Linux|Vectra|SentinelOne|CrowdStrike|VMware|Google,
-EventVendor,string,Mandatory,AuditEvent,Enumerated,Microsoft|AWS|Barracuda|Cisco|Dataminr|Vectra|CrowdStrike|SentinelOne|VMware|Infoblox,
+EventVendor,string,Mandatory,Authentication,Enumerated,Salesforce|AWS|Barracuda|Cisco|Microsoft|Okta|PostgreSQL|OpenBSD|Linux|Vectra|SentinelOne|CrowdStrike|VMware|Google|Illumio,
+EventVendor,string,Mandatory,AuditEvent,Enumerated,Microsoft|AWS|Barracuda|Cisco|Dataminr|Vectra|CrowdStrike|SentinelOne|VMware|Infoblox|Illumio,
 EventVendor,string,Mandatory,Common,,,
 EventVendor,string,Mandatory,DhcpEvent,,Infoblox,
 EventVendor,string,Mandatory,FileEvent,Enumerated,Microsoft|SentinelOne|VMware|Google,
 EventVendor,string,Mandatory,Dns,Enumerated,Cisco|Corelight|GCP|Infoblox|Microsoft|Zscaler|Vectra AI|SentinelOne|Fortinet,
-EventVendor,string,Mandatory,NetworkSession,Enumerated,Fortinet|AppGate|Barracuda|Palo Alto|Microsoft|Zscaler|AWS|Vectra AI|WatchGuard|Cisco|Corelight|Check Point|Forcepoint|Cynerio|SentinelOne|CrowdStrike|VMware|SonicWall,
+EventVendor,string,Mandatory,NetworkSession,Enumerated,Fortinet|AppGate|Barracuda|Palo Alto|Microsoft|Zscaler|AWS|Vectra AI|WatchGuard|Cisco|Corelight|Check Point|Forcepoint|Cynerio|SentinelOne|CrowdStrike|VMware|SonicWall|Illumio,
 EventVendor,string,Mandatory,ProcessEvent,Enumerated,Microsoft|SentinelOne|VMware|TrendMicro,
 EventVendor,string,Mandatory,WebSession,Enumerated,Apache|Barracuda|Fortinet|Microsoft|Squid|Zscaler|Vectra AI|Palo Alto|WatchGuard|Cisco|Forcepoint|Corelight|Dataminr|Citrix|F5|SonicWall,
 EventVendor,string,Mandatory,UserManagement,Enumerated,Microsoft|Linux|Cisco|SentinelOne,

DataConnectors/microsoft-sentinel-log-analytics-logstash-output-plugin/CHANGELOG.md

@@ -1,16 +1,13 @@
 ## 1.1.3
-* Replace the library rest-client used for connecting with Azure to excon.
-
+-  Replaces the `rest-client` library used for connecting to Azure with the `excon` library.
+ 
 ## 1.1.1
-* Support China and US Government Azure sovereign clouds.
-
+- Adds support for Azure US Government cloud and Microsoft Azure operated by 21Vianet in China.
+ 
 ## 1.1.0 
-* Increase timeout for read/open connections to 120 seconds.
-* Add error handling for when connection timeout occurs.
-* Upgrade the rest-client dependency minimum version to 2.1.0.
-* Allow setting different proxy values for api connections.
-* Upgrade version for ingestion api to 2023-01-01.
-* Rename the plugin to microsoft-sentinel-log-analytics-logstash-output-plugin.
-
+- Allows setting different proxy values for API connections.
+- Upgrades version for logs ingestion API to 2023-01-01.
+- Renames the plugin to microsoft-sentinel-log-analytics-logstash-output-plugin.
+ 
 ## 1.0.0
-* Initial release for output plugin for logstash to Microsoft Sentinel. This is done with the Log Analytics DCR based API.
+- The initial release for the Logstash output plugin for Microsoft Sentinel. This plugin uses Data Collection Rules (DCRs) with Azure Monitor's Logs Ingestion API.

DataConnectors/microsoft-sentinel-log-analytics-logstash-output-plugin/README.md

@@ -3,8 +3,8 @@
 Microsoft Sentinel provides a new output plugin for Logstash. Use this output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace. This is done with the Log Analytics DCR-based API.
 You may send logs to custom or standard tables.
 
-Plugin version: v1.1.0  
-Released on: 2023-07-23
+Plugin version: v1.1.3  
+Released on: 2024-10-10
 
 This plugin is currently in development and is free to use. We welcome contributions from the open source community on this project, and we request and appreciate feedback from users.
 
@@ -22,12 +22,12 @@ Microsoft Sentinel provides Logstash output plugin to Log analytics workspace us
 
 The plugin is published on [RubyGems](https://rubygems.org/gems/microsoft-sentinel-log-analytics-logstash-output-plugin). To install to an existing logstash installation, run `logstash-plugin install microsoft-sentinel-log-analytics-logstash-output-plugin`.
 
-If you do not have a direct internet connection, you can install the plugin to another logstash installation, and then export and import a plugin bundle to the offline host. For more information, see [Logstash Offline Plugin Management instruction](<https://www.elastic.co/guide/en/logstash/current/offline-plugins.html>). 
+If you do not have a direct internet connection, you can install the plugin to another logstash installation, and then export and import a plugin bundle to the offline host. For more information, see [Logstash Offline Plugin Management instruction](<https://www.elastic.co/guide/en/logstash/current/offline-plugins.html>).  
 
 Microsoft Sentinel's Logstash output plugin supports the following versions
 - 7.0 - 7.17.13
 - 8.0 - 8.9
-- 8.11 - 8.14
+- 8.11 - 8.15
 
 Please note that when using Logstash 8, it is recommended to disable ECS in the pipeline. For more information refer to [Logstash documentation.](<https://www.elastic.co/guide/en/logstash/8.4/ecs-ls.html>)
 
@@ -236,3 +236,23 @@ Which will produce this content in the sample file:
 	}
 ]
 ```
+
+
+## Known issues
+ 
+When using Logstash installed on a Docker image of Lite Ubuntu, the following warning may appear:
+
+```
+java.lang.RuntimeException: getprotobyname_r failed
+```
+
+To resolve it, use the following commands to install the *netbase* package within your Dockerfile:
+```bash
+USER root
+RUN apt install netbase -y
+```
+For more information, see [JNR regression in Logstash 7.17.0 (Docker)](https://github.com/elastic/logstash/issues/13703).
+
+If your environment's event rate is low considering the number of allocated Logstash workers, we recommend increasing the value of *plugin_flush_interval* to 60 or more. This change will allow each worker to batch more events before uploading to the Data Collection Endpoint (DCE).  You can monitor the ingestion payload using [DCR metrics](https://learn.microsoft.com/azure/azure-monitor/essentials/data-collection-monitor#dcr-metrics).
+For more information on *plugin_flush_interval*, see the [Optional Configuration table](https://learn.microsoft.com/azure/sentinel/connect-logstash-data-connection-rules#optional-configuration) mentioned earlier.
+

Detections/CommonSecurityLog/MultiVendor-PossibleDGAContacts.yaml

@@ -4,7 +4,8 @@ description: |
   'Identifies contacts with domains names in CommonSecurityLog that might have been generated by a Domain Generation Algorithm (DGA). DGAs can be used by malware to generate rendezvous points that are difficult to predict in advance.
   This detection uses the Alexa Top 1 million domain names to build a model of what normal domains look like. It uses this to identify domains that may have been randomly generated by an algorithm.
   The triThreshold is set to 500 - increase this to report on domains that are less likely to have been randomly generated, decrease it for more likely.
-  The start time and end time look back over 6 hours of data and the dgaLengthThreshold is set to 8 - meaning domains whose length is 8 or more are reported.'
+  The start time and end time look back over 6 hours of data and the dgaLengthThreshold is set to 8 - meaning domains whose length is 8 or more are reported.
+  NOTE - The top1M csv zip file used in the query is dynamic and may produce different results over various time periods. It's important to cross-check the events against the entities involved in the incident.'
 severity: Medium
 requiredDataConnectors:
   - connectorId: Zscaler
@@ -118,7 +119,7 @@ entityMappings:
     fieldMappings:
       - identifier: DomainName
         columnName: Name
-version: 1.0.5
+version: 1.0.6
 kind: Scheduled
 metadata:
     source: 

Detections/MultipleDataSources/AADHostLoginCorrelation.yaml

@@ -41,7 +41,7 @@ query: |
   let suspicious_signins =
   table(tableName)
   | where ResultType !in ("0", "50125", "50140")
-  | where IPAddress !in ('127.0.0.1', '::1')
+  | where IPAddress !in ('127.0.0.1', '::1', '')
   | summarize count() by IPAddress
   | where count_ > signin_threshold
   | summarize make_set(IPAddress);
@@ -115,7 +115,7 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IpAddress
-version: 1.3.1
+version: 1.3.2
 kind: Scheduled
 metadata:
     source:
@@ -125,4 +125,4 @@ metadata:
     support:
         tier: Community
     categories:
-        domains: [ "Security - Others", "Identity" ]
+        domains: [ "Security - Others", "Identity" ]

Parsers/ASimAuditEvent/Parsers/ASimAuditEvent.yaml

@@ -33,6 +33,7 @@ Parsers:
   - _ASim_AuditEvent_SentinelOne
   - _ASim_AuditEvent_VMwareCarbonBlackCloud
   - _ASim_AuditEvent_InfobloxBloxOne
+  - _ASim_AuditEvent_IllumioSaaSCore
 ParserParams:
   - Name: pack
     Type: bool
@@ -56,5 +57,6 @@ ParserQuery: |
     ASimAuditEventSentinelOne  (BuiltInDisabled or ('ExcludeASimAuditEventSentinelOne' in (DisabledParsers))),
     ASimAuditEventCrowdStrikeFalconHost(BuiltInDisabled or ('ExcludeASimAuditEventCrowdStrikeFalconHost' in (DisabledParsers))),
     ASimAuditEventVMwareCarbonBlackCloud(BuiltInDisabled or ('ExcludeASimAuditEventVMwareCarbonBlackCloud' in (DisabledParsers))),
-    ASimAuditEventInfobloxBloxOne(BuiltInDisabled or ('ExcludeASimAuditEventInfobloxBloxOne' in (DisabledParsers)))
+    ASimAuditEventInfobloxBloxOne(BuiltInDisabled or ('ExcludeASimAuditEventInfobloxBloxOne' in (DisabledParsers))),
+    ASimAuditEventIllumioSaaSCore(BuiltInDisabled or ('ExcludeASimAuditEventIllumioSaaSCore' in (DisabledParsers)))