Adding alias extractions (#100)

* adding docs for alias extractions

* Update README.md

* Update README.md

* making tests more clear

* Update embedded_img_ignore.txt

* implement alias search #91

* Update cases-extraction-type-lookup.md

---------

Co-authored-by: Fadl <chaos@efqr.dev>

README.md

@@ -89,8 +89,8 @@ How the extractions are performed
 * `--relationship_mode` (REQUIRED): either.
 	* `ai`: AI provider must be enabled. extractions performed by either regex or AI for extractions user selected. Rich relationships created from AI provider from extractions.
 	* `standard`: extractions performed by either regex or AI (AI provider must be enabled) for extractions user selected. Basic relationships created from extractions back to master Report object generated.
-* `--ignore_image_refs` (default `true`): images references in documents don't usually need extracting. e.g. `<img src="https://example.com/image.png">` you would not want domain or file extractions extracting `example.com` and `image.png`. Hence these are ignored by default. If you want extractions to consider this data, set it to `false`
-* `--ignore_link_refs` (default `true`): link references in documents don't usually need extracting e.g. `<a href="https://example.com/link.html">Bad Actor</a>` you would only want `Bad actor` to be considered for extraction. Setting this to `false` will also include everything inside the link tag (e.g. `example.com` would extract as a domain)
+* `--ignore_image_refs` (default `true`): images references in documents don't usually need extracting. e.g. `<img src="https://example.com/image.png" alt="something">` you would not want domain or file extractions extracting `example.com` and `image.png`. Hence these are ignored by default (they are removed from text sent to extraction). Note, only the `img src` is ignored, all other values e.g. `alt` are considered. If you want extractions to consider this data, set it to `false`
+* `--ignore_link_refs` (default `true`): link references in documents don't usually need extracting e.g. `<a href="https://example.com/link.html" title="something">Bad Actor</a>` you would only want `Bad actor` to be considered for extraction. Hence these part of the link are ignored by default (they are removed from text sent to extraction). Note, only the `a href` is ignored, all other values e.g. `title` are considered. Setting this to `false` will also include everything inside the link tag (e.g. `example.com` would extract as a domain)
 
 #### AI settings
 

docs/stix-mapping.md

@@ -1714,6 +1714,14 @@ Takes the extracted name and passes it to;
 GET CTIBUTLER_HOST/api/v1/attack-enterprise/objects/?name=<VALUE>
 ```
 
+### stix-mapping: `ctibutler-mitre-attack-enterprise-alias`
+
+Takes the extracted alias and passes it to;
+
+```shell
+GET CTIBUTLER_HOST/api/v1/attack-enterprise/objects/?alias=<VALUE>
+```
+
 ### stix-mapping: `ctibutler-mitre-attack-mobile-id`
 
 Takes the extracted ID and passes it to
@@ -1737,6 +1745,14 @@ Takes the extracted name and passes it to;
 GET CTIBUTLER_HOST/api/v1/attack-mobile/objects/?name=<VALUE>
 ```
 
+### stix-mapping: `ctibutler-mitre-attack-mobile-alias`
+
+Takes the extracted alias and passes it to;
+
+```shell
+GET CTIBUTLER_HOST/api/v1/attack-mobile/objects/?alias=<VALUE>
+```
+
 ### stix-mapping: `ctibutler-mitre-attack-ics-id`
 
 Takes the extracted ID and passes it to
@@ -1762,6 +1778,14 @@ Takes the extracted name and passes it to;
 GET CTIBUTLER_HOST/api/v1/attack-ics/objects/?name=<VALUE>
 ```
 
+### stix-mapping: `ctibutler-mitre-attack-ics-alias`
+
+Takes the extracted alias and passes it to;
+
+```shell
+GET CTIBUTLER_HOST/api/v1/attack-ics/objects/?alias=<VALUE>
+```
+
 ### stix-mapping: `ctibutler-mitre-capec-id`
 
 Takes the extracted ID and passes it to 

includes/extractions/lookup/config.yaml

@@ -43,6 +43,19 @@ lookup_mitre_attack_enterprise_name:
   test_cases: generic_mitre_attack_enterprise_name
   stix_mapping: ctibutler-mitre-attack-enterprise-name
 
+lookup_mitre_attack_enterprise_alias:
+  type: lookup
+  name: 'MITRE ATT&CK Enterprise alias'
+  description: 'Extracts MITRE ATT&CK Enterprise aliases from text. See lookup name for version used.'
+  notes: 'ai_mitre_attack_enterprise also exists but beware of hallucinations'
+  file: 'lookups/mitre_attack_enterprise_aliases_v16_0.txt'
+  created: 2020-01-01
+  modified:  2020-01-01
+  created_by: DOGESEC
+  version: 1.0.0
+  test_cases: generic_mitre_attack_enterprise_aliases
+  stix_mapping: ctibutler-mitre-attack-enterprise-aliases
+
 lookup_mitre_attack_mobile_id:
   type: lookup
   name: 'MITRE ATT&CK Mobile IDs'
@@ -69,6 +82,19 @@ lookup_mitre_attack_mobile_name:
   test_cases: generic_mitre_attack_mobile_name
   stix_mapping: ctibutler-mitre-attack-mobile-name
 
+lookup_mitre_attack_mobile_alias:
+  type: lookup
+  name: 'MITRE ATT&CK Mobile alias'
+  description: 'Extracts MITRE ATT&CK Mobile aliases from text. See lookup name for version used.'
+  notes: 'ai_mitre_attack_mobile also exists but beware of hallucinations'
+  file: 'lookups/mitre_attack_mobile_aliases_v16_0.txt'
+  created: 2020-01-01
+  modified:  2020-01-01
+  created_by: DOGESEC
+  version: 1.0.0
+  test_cases: generic_mitre_attack_mobile_aliases
+  stix_mapping: ctibutler-mitre-attack-mobile-aliases
+
 lookup_mitre_attack_ics_id:
   type: lookup
   name: 'MITRE ATT&CK ICS IDs'
@@ -95,6 +121,19 @@ lookup_mitre_attack_ics_name:
   test_cases: generic_mitre_attack_ics_name
   stix_mapping: ctibutler-mitre-attack-ics-name
 
+lookup_mitre_attack_ics_alias:
+  type: lookup
+  name: 'MITRE ATT&CK ICS alias'
+  description: 'Extracts MITRE ATT&CK ICS aliases from text. See lookup name for version used.'
+  notes: 'ai_mitre_attack_ics also exists but beware of hallucinations'
+  file: 'lookups/mitre_attack_ics_aliases_v16_0.txt'
+  created: 2020-01-01
+  modified:  2020-01-01
+  created_by: DOGESEC
+  version: 1.0.0
+  test_cases: generic_mitre_attack_ics_aliases
+  stix_mapping: ctibutler-mitre-attack-ics-aliases
+
 ####### MITRE CAPEC #######
 
 lookup_mitre_capec_id:

tests/data/extraction_types/ai_mitre_attack_enterprise.txt

@@ -9,7 +9,9 @@ T1040
 TA0003
 Rundll32
 OS Credential Dumping
+Operation Sharpshooter
+Operation Spalax
 
 ====Bad====
 
-None
+

tests/data/extraction_types/ai_mitre_attack_ics.txt

@@ -3,4 +3,9 @@
 TA0111
 Scripting
 Program Upload
+Energetic Bear
+BROMINE
+
+====Bad====
+
 

tests/data/extraction_types/ai_mitre_attack_mobile.txt

@@ -6,7 +6,9 @@ T1630.001
 TA0029
 Impair Defenses
 Call Log
+Storm-0875
+Octo Tempest
 
 ====Bad====
 
-None
+

tests/data/extraction_types/all_cases.txt

@@ -121,6 +121,8 @@ T1040
 TA0003
 Rundll32
 OS Credential Dumping
+Operation Sharpshooter
+Operation Spalax
 T1174
 TA0006
 TA0011
@@ -130,24 +132,34 @@ T1040
 TA0003
 Rundll32
 OS Credential Dumping
+Operation Sharpshooter
+Operation Spalax
 M1013
 S0505
 T1630.001
 TA0029
 Impair Defenses
 Call Log
+Storm-0875
+Octo Tempest
 M1013
 S0505
 T1630.001
 TA0029
 Impair Defenses
 Call Log
+Storm-0875
+Octo Tempest
 TA0111
 Scripting
 Program Upload
+Energetic Bear
+BROMINE
 TA0111
 Scripting
 Program Upload
+Energetic Bear
+BROMINE
 CAPEC-110
 Clickjacking
 Overflow Buffers

tests/data/extraction_types/generic_mitre_attack_enterprise_aliases.txt

@@ -0,0 +1,10 @@
+====Good====
+
+Operation Sharpshooter
+Operation Spalax
+
+====Bad====
+
+TA0006
+Rundll32
+OS Credential Dumping

tests/data/extraction_types/generic_mitre_attack_ics_aliases.txt

@@ -0,0 +1,10 @@
+====Good====
+
+Energetic Bear
+BROMINE
+
+====Bad====
+
+TA0111
+Scripting
+Program Upload

tests/data/extraction_types/generic_mitre_attack_mobile_aliases.txt

@@ -0,0 +1,10 @@
+====Good====
+
+Storm-0875
+Octo Tempest
+
+====Bad====
+
+M1013
+Impair Defenses
+Call Log