From b82cfb186242d3075ee1cf8ea6413557cd6cd609 Mon Sep 17 00:00:00 2001 From: Bracey Summers <35816572+bsummers-tc@users.noreply.github.com> Date: Tue, 12 Mar 2024 10:16:40 -0500 Subject: [PATCH] 4.0.4 Release Commit - APP-4307 - [API] Added support for paginating indicator custom associations - APP-4334 - [API] Fixed issue where transform methods wasn't being called if value was not a string - APP-4380 - [API] Added support for external date fields to TI Transform model - APP-4381 - [Logging] Fixed API logging issues - APP-4383 - [API] Updated TC API module for changes in the V3 API - APP-4400 - [Input] Added support for KeyValue input type with None value - APP-4401 - [API] Fixed issue with Assignee model not calculating appropriate model type --- .gitignore | 1 + release_notes.md | 10 ++- tcex/api/tc/v3/_gen/_gen_abc.py | 12 ++++ tcex/api/tc/v3/_gen/_gen_filter_abc.py | 21 +++++++ tcex/api/tc/v3/_gen/_gen_object_abc.py | 21 ++++++- tcex/api/tc/v3/_gen/model/_property_model.py | 7 +++ .../attribute_types/attribute_type_filter.py | 9 +++ .../tc/v3/case_attributes/case_attribute.py | 2 + .../case_attributes/case_attribute_filter.py | 24 ++++++++ .../case_attributes/case_attribute_model.py | 10 +++ tcex/api/tc/v3/cases/case_filter.py | 10 +++ .../tc/v3/group_attributes/group_attribute.py | 2 + .../group_attribute_filter.py | 24 ++++++++ .../group_attributes/group_attribute_model.py | 10 +++ tcex/api/tc/v3/groups/group_filter.py | 25 ++++++++ tcex/api/tc/v3/groups/group_model.py | 8 +++ .../indicator_attribute.py | 2 + .../indicator_attribute_filter.py | 24 ++++++++ .../indicator_attribute_model.py | 10 +++ tcex/api/tc/v3/indicators/indicator.py | 7 ++- tcex/api/tc/v3/indicators/indicator_filter.py | 61 +++++++++++++++++++ tcex/api/tc/v3/indicators/indicator_model.py | 6 +- .../intel_requirement_filter.py | 25 ++++++++ tcex/api/tc/v3/notes/note_filter.py | 15 +++++ tcex/api/tc/v3/object_abc.py | 18 ++++-- .../api/tc/v3/security/assignee_user_model.py | 4 +- tcex/api/tc/v3/tags/tag.py | 2 + tcex/api/tc/v3/tags/tag_filter.py | 16 +++++ tcex/api/tc/v3/tags/tag_model.py | 9 +++ tcex/api/tc/v3/v3_model_abc.py | 2 +- .../v3/victim_attributes/victim_attribute.py | 2 + .../victim_attribute_filter.py | 24 ++++++++ .../victim_attribute_model.py | 10 +++ tcex/api/tc/v3/victims/victim_filter.py | 10 +++ tcex/app/config | 2 +- tcex/app/key_value_store | 2 +- tcex/input/field_type/key_value.py | 23 +++++++ tcex/logger/api_handler.py | 2 +- tcex/tcex.py | 7 +-- .../api/tc/v2/batch/test_group_interface_1.py | 3 +- .../api/tc/v2/batch/test_group_interface_2.py | 2 +- tests/api/tc/v3/cases/test_case_interface.py | 1 + .../v3/indicators/test_indicator_interface.py | 6 ++ tests/api/tc/v3/v3_helpers.py | 9 ++- 44 files changed, 472 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index c143fecc4..591f4ea8f 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ templates threatconnect* test.log* tests/reports/* +tcex/api/tc/v3/_gen/options_data TODO.md #------------------------------------------------- diff --git a/release_notes.md b/release_notes.md index 7815502e0..5381b52ea 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,8 +1,14 @@ # Release Notes ## 4.0.4 -- APP-4334 - [API] Call method transforms in TI transform models even if value is not a string -- APP-4380 - [API] Add externalDateAdded, externalDateExpires, externalLastModified, firstSeen, and lastSeen fields to TI Transform model + +- APP-4307 - [API] Added support for paginating indicator custom associations +- APP-4334 - [API] Fixed issue where transform methods wasn't being called if value was not a string +- APP-4380 - [API] Added support for external date fields to TI Transform model +- APP-4381 - [Logging] Fixed API logging issues +- APP-4383 - [API] Updated TC API module for changes in the V3 API +- APP-4400 - [Input] Added support for KeyValue input type with None value +- APP-4401 - [API] Fixed issue with Assignee model not calculating appropriate model type ## 4.0.3 diff --git a/tcex/api/tc/v3/_gen/_gen_abc.py b/tcex/api/tc/v3/_gen/_gen_abc.py index cd7f6f2f8..2ba690fe6 100644 --- a/tcex/api/tc/v3/_gen/_gen_abc.py +++ b/tcex/api/tc/v3/_gen/_gen_abc.py @@ -1,8 +1,10 @@ """TcEx Framework Module""" # standard library +import json import os from abc import ABC from collections.abc import Generator +from pathlib import Path from textwrap import TextWrapper # third-party @@ -103,6 +105,11 @@ def _prop_contents(self) -> dict: # print(r.request.method, r.request.url, r.text) if r.ok: _properties = r.json() + + options_data = Path('tcex/api/tc/v3/_gen/options_data') / f'{self.type_}.json' + with options_data.open(mode='w') as fh: + json.dump(r.json(), fh, indent=2) + except (ConnectionError, ProxyError) as ex: Render.panel.failure(f'Failed getting types properties ({ex}).') @@ -143,6 +150,11 @@ def _prop_contents_data(self, properties: dict) -> Generator: # pair is a list (currently the list only contains a single dict). to be safe # we loop over the list and update the type for each item. field_data = field_data[0] + + # # handle special case for customAssociationNames, where the data is an array of + # # string, but the type in the object states 'String'. + # if field_data['type'] == 'String': + # field_data['type'] = 'ListString' else: raise RuntimeError( f'Invalid type properties data: field-name={field_name}, type={self.type_}' diff --git a/tcex/api/tc/v3/_gen/_gen_filter_abc.py b/tcex/api/tc/v3/_gen/_gen_filter_abc.py index a99d9c6fb..f46286bf9 100644 --- a/tcex/api/tc/v3/_gen/_gen_filter_abc.py +++ b/tcex/api/tc/v3/_gen/_gen_filter_abc.py @@ -315,6 +315,25 @@ def _gen_code_has_tag_method(self) -> list: '', ] + def _gen_code_has_all_tags_method(self) -> list: + """Return code for has_tag TQL filter methods.""" + self._add_tql_imports() + return [ + f'{self.i1}@property', + f'{self.i1}def has_all_tags(self):', + f'{self.i2}"""Return **TagFilter** for further filtering."""', + f'{self.i2}# first-party', + f'{self.i2}from tcex.api.tc.v3.tags.tag_filter import TagFilter', + '', + f'{self.i2}tags = TagFilter(Tql())', + ( + f'''{self.i2}self._tql.add_filter('hasAllTags', ''' + '''TqlOperator.EQ, tags, TqlType.SUB_QUERY)''' + ), + f'{self.i2}return tags', + '', + ] + def _gen_code_has_task_method(self) -> list: """Return code for has_task TQL filter methods.""" self._add_tql_imports() @@ -495,6 +514,8 @@ def gen_class_methods(self): _filter_class.extend(self._gen_code_has_note_method()) elif f.keyword.snake_case() == 'has_tag': _filter_class.extend(self._gen_code_has_tag_method()) + elif f.keyword.snake_case() == 'has_all_tags': + _filter_class.extend(self._gen_code_has_all_tags_method()) elif f.keyword.snake_case() == 'has_task': _filter_class.extend(self._gen_code_has_task_method()) elif f.keyword.snake_case() == 'has_attribute': diff --git a/tcex/api/tc/v3/_gen/_gen_object_abc.py b/tcex/api/tc/v3/_gen/_gen_object_abc.py index 7eda65621..47d3ac886 100644 --- a/tcex/api/tc/v3/_gen/_gen_object_abc.py +++ b/tcex/api/tc/v3/_gen/_gen_object_abc.py @@ -609,7 +609,7 @@ def stage_assignee( ) def _gen_code_object_type_property_method( - self, type_: str, model_type: str | None = None + self, type_: str, model_type: str | None = None, custom_associations: bool = False ) -> str: """Return the method code. @@ -670,7 +670,18 @@ def artifacts(self): # Custom logic to ensure that when iterating over the associated indicators or associated # groups then the item currently being iterated over is not included in the results. - if ( + if custom_associations is True: + _code.extend( + [ + ( + f'''{self.i2}yield from self._iterate_over_sublist''' + f'''({model_import_data.get('object_collection_class')}, ''' + '''custom_associations=True)''' + ''' # type: ignore''' + ), + ] + ) + elif ( self.type_ in ['indicators', 'groups', 'artifacts', 'cases'] and model_type == f'associated_{self.type_}' ): @@ -914,6 +925,12 @@ def filter ... 'indicators', 'associated_indicators' ) + # generate custom_associations property method + if 'customAssociations' in add_properties: + _code += self._gen_code_object_type_property_method( + 'indicators', 'custom_associations', custom_associations=True + ) + # generate associated_victim_asset property method if 'associatedVictimAssets' in add_properties: _code += self._gen_code_object_type_property_method( diff --git a/tcex/api/tc/v3/_gen/model/_property_model.py b/tcex/api/tc/v3/_gen/model/_property_model.py index edc1a1bea..1a28e2548 100644 --- a/tcex/api/tc/v3/_gen/model/_property_model.py +++ b/tcex/api/tc/v3/_gen/model/_property_model.py @@ -163,6 +163,7 @@ def __process_bool_types(cls, pm: 'PropertyModel', extra: dict[str, str]): def __process_dict_types(cls, pm: 'PropertyModel', extra: dict[str, str]): """Process standard type.""" types = [ + 'AttackSecurityCoverage', 'AttributeSource', 'DNSResolutions', 'Enrichments', @@ -308,6 +309,12 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: dict[str, str]): 'typing_type': f'list[{cls.__extra_format_type_model(pm.type)}]', } ) + elif pm.name == 'customAssociationNames': + extra.update( + { + 'typing_type': 'list[str]', + } + ) elif pm.type == 'TaskAssignees': extra.update( { diff --git a/tcex/api/tc/v3/attribute_types/attribute_type_filter.py b/tcex/api/tc/v3/attribute_types/attribute_type_filter.py index fb80dabad..48ecd8014 100644 --- a/tcex/api/tc/v3/attribute_types/attribute_type_filter.py +++ b/tcex/api/tc/v3/attribute_types/attribute_type_filter.py @@ -31,6 +31,15 @@ def associated_type(self, operator: Enum, associated_type: list | str): self._tql.add_filter('associatedType', operator, associated_type, TqlType.STRING) + def default(self, operator: Enum, default: bool): + """Filter Displayed based on **default** keyword. + + Args: + operator: The operator enum for the filter. + default: Whether or not the attribute type is displayable on the item. + """ + self._tql.add_filter('default', operator, default, TqlType.BOOLEAN) + def description(self, operator: Enum, description: list | str): """Filter Description based on **description** keyword. diff --git a/tcex/api/tc/v3/case_attributes/case_attribute.py b/tcex/api/tc/v3/case_attributes/case_attribute.py index 71515fa05..89f58cdd6 100644 --- a/tcex/api/tc/v3/case_attributes/case_attribute.py +++ b/tcex/api/tc/v3/case_attributes/case_attribute.py @@ -32,6 +32,8 @@ class CaseAttribute(ObjectABC): the one(s) specified). source (str, kwargs): The attribute source. type (str, kwargs): The attribute type. + update_last_modified_date (bool, kwargs): A flag giving the client the ability to choose if + the attribute last modified date should be changed. value (str, kwargs): The attribute value. """ diff --git a/tcex/api/tc/v3/case_attributes/case_attribute_filter.py b/tcex/api/tc/v3/case_attributes/case_attribute_filter.py index 819aa418c..866cd5ed3 100644 --- a/tcex/api/tc/v3/case_attributes/case_attribute_filter.py +++ b/tcex/api/tc/v3/case_attributes/case_attribute_filter.py @@ -57,6 +57,15 @@ def date_val(self, operator: Enum, date_val: Arrow | datetime | int | str): date_val = self.util.any_to_datetime(date_val).strftime('%Y-%m-%d %H:%M:%S') self._tql.add_filter('dateVal', operator, date_val, TqlType.STRING) + def default(self, operator: Enum, default: bool): + """Filter Default based on **default** keyword. + + Args: + operator: The operator enum for the filter. + default: A flag that is set by an attribute type configuration. + """ + self._tql.add_filter('default', operator, default, TqlType.BOOLEAN) + def displayed(self, operator: Enum, displayed: bool): """Filter Displayed based on **displayed** keyword. @@ -170,6 +179,21 @@ def pinned(self, operator: Enum, pinned: bool): """ self._tql.add_filter('pinned', operator, pinned, TqlType.BOOLEAN) + def short_text(self, operator: Enum, short_text: list | str): + """Filter Text based on **shortText** keyword. + + Args: + operator: The operator enum for the filter. + short_text: The short text of the attribute (only applies to certain types). + """ + if isinstance(short_text, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('shortText', operator, short_text, TqlType.STRING) + def source(self, operator: Enum, source: list | str): """Filter Source based on **source** keyword. diff --git a/tcex/api/tc/v3/case_attributes/case_attribute_model.py b/tcex/api/tc/v3/case_attributes/case_attribute_model.py index 8b1033678..442819bd7 100644 --- a/tcex/api/tc/v3/case_attributes/case_attribute_model.py +++ b/tcex/api/tc/v3/case_attributes/case_attribute_model.py @@ -100,6 +100,16 @@ class CaseAttributeModel( read_only=False, title='type', ) + update_last_modified_date: bool = Field( + None, + description=( + 'A flag giving the client the ability to choose if the attribute last modified date ' + 'should be changed.' + ), + methods=['POST', 'PUT'], + read_only=False, + title='updateLastModifiedDate', + ) value: str | None = Field( None, description='The attribute value.', diff --git a/tcex/api/tc/v3/cases/case_filter.py b/tcex/api/tc/v3/cases/case_filter.py index 33d0e6855..d51126b50 100644 --- a/tcex/api/tc/v3/cases/case_filter.py +++ b/tcex/api/tc/v3/cases/case_filter.py @@ -267,6 +267,16 @@ def description(self, operator: Enum, description: list | str): self._tql.add_filter('description', operator, description, TqlType.STRING) + @property + def has_all_tags(self): + """Return **TagFilter** for further filtering.""" + # first-party + from tcex.api.tc.v3.tags.tag_filter import TagFilter + + tags = TagFilter(Tql()) + self._tql.add_filter('hasAllTags', TqlOperator.EQ, tags, TqlType.SUB_QUERY) + return tags + @property def has_artifact(self): """Return **ArtifactFilter** for further filtering.""" diff --git a/tcex/api/tc/v3/group_attributes/group_attribute.py b/tcex/api/tc/v3/group_attributes/group_attribute.py index baf697937..61b7f7e6c 100644 --- a/tcex/api/tc/v3/group_attributes/group_attribute.py +++ b/tcex/api/tc/v3/group_attributes/group_attribute.py @@ -32,6 +32,8 @@ class GroupAttribute(ObjectABC): the one(s) specified). source (str, kwargs): The attribute source. type (str, kwargs): The attribute type. + update_last_modified_date (bool, kwargs): A flag giving the client the ability to choose if + the attribute last modified date should be changed. value (str, kwargs): The attribute value. """ diff --git a/tcex/api/tc/v3/group_attributes/group_attribute_filter.py b/tcex/api/tc/v3/group_attributes/group_attribute_filter.py index c87c4e091..4f5e461b7 100644 --- a/tcex/api/tc/v3/group_attributes/group_attribute_filter.py +++ b/tcex/api/tc/v3/group_attributes/group_attribute_filter.py @@ -51,6 +51,15 @@ def date_val(self, operator: Enum, date_val: Arrow | datetime | int | str): date_val = self.util.any_to_datetime(date_val).strftime('%Y-%m-%d %H:%M:%S') self._tql.add_filter('dateVal', operator, date_val, TqlType.STRING) + def default(self, operator: Enum, default: bool): + """Filter Associable based on **default** keyword. + + Args: + operator: The operator enum for the filter. + default: A flag to include an attribute in group associations. + """ + self._tql.add_filter('default', operator, default, TqlType.BOOLEAN) + def displayed(self, operator: Enum, displayed: bool): """Filter Displayed based on **displayed** keyword. @@ -189,6 +198,21 @@ def pinned(self, operator: Enum, pinned: bool): """ self._tql.add_filter('pinned', operator, pinned, TqlType.BOOLEAN) + def short_text(self, operator: Enum, short_text: list | str): + """Filter Text based on **shortText** keyword. + + Args: + operator: The operator enum for the filter. + short_text: The short text of the attribute (only applies to certain types). + """ + if isinstance(short_text, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('shortText', operator, short_text, TqlType.STRING) + def source(self, operator: Enum, source: list | str): """Filter Source based on **source** keyword. diff --git a/tcex/api/tc/v3/group_attributes/group_attribute_model.py b/tcex/api/tc/v3/group_attributes/group_attribute_model.py index 5b47d12e8..19ca5ddbb 100644 --- a/tcex/api/tc/v3/group_attributes/group_attribute_model.py +++ b/tcex/api/tc/v3/group_attributes/group_attribute_model.py @@ -107,6 +107,16 @@ class GroupAttributeModel( read_only=False, title='type', ) + update_last_modified_date: bool = Field( + None, + description=( + 'A flag giving the client the ability to choose if the attribute last modified date ' + 'should be changed.' + ), + methods=['POST', 'PUT'], + read_only=False, + title='updateLastModifiedDate', + ) value: str | None = Field( None, description='The attribute value.', diff --git a/tcex/api/tc/v3/groups/group_filter.py b/tcex/api/tc/v3/groups/group_filter.py index 3e456eb91..6c9a32120 100644 --- a/tcex/api/tc/v3/groups/group_filter.py +++ b/tcex/api/tc/v3/groups/group_filter.py @@ -321,6 +321,16 @@ def generated_report(self, operator: Enum, generated_report: bool): """ self._tql.add_filter('generatedReport', operator, generated_report, TqlType.BOOLEAN) + @property + def has_all_tags(self): + """Return **TagFilter** for further filtering.""" + # first-party + from tcex.api.tc.v3.tags.tag_filter import TagFilter + + tags = TagFilter(Tql()) + self._tql.add_filter('hasAllTags', TqlOperator.EQ, tags, TqlType.SUB_QUERY) + return tags + @property def has_artifact(self): """Return **ArtifactFilter** for further filtering.""" @@ -455,6 +465,21 @@ def id(self, operator: Enum, id: int | list): # pylint: disable=redefined-built self._tql.add_filter('id', operator, id, TqlType.INTEGER) + def insights(self, operator: Enum, insights: list | str): + """Filter Insights (Report) based on **insights** keyword. + + Args: + operator: The operator enum for the filter. + insights: The AI generated synopsis of the report. + """ + if isinstance(insights, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('insights', operator, insights, TqlType.STRING) + def is_group(self, operator: Enum, is_group: bool): """Filter isGroup based on **isGroup** keyword. diff --git a/tcex/api/tc/v3/groups/group_model.py b/tcex/api/tc/v3/groups/group_model.py index c2a9ba2c4..216ec115b 100644 --- a/tcex/api/tc/v3/groups/group_model.py +++ b/tcex/api/tc/v3/groups/group_model.py @@ -252,6 +252,14 @@ class GroupModel( read_only=True, title='id', ) + insights: str | None = Field( + None, + allow_mutation=False, + applies_to=['Document', 'Report'], + description='An AI generated synopsis of the document.', + read_only=True, + title='insights', + ) last_modified: datetime | None = Field( None, allow_mutation=False, diff --git a/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py b/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py index f1ad6eded..7bae37980 100644 --- a/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py +++ b/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py @@ -32,6 +32,8 @@ class IndicatorAttribute(ObjectABC): the one(s) specified). source (str, kwargs): The attribute source. type (str, kwargs): The attribute type. + update_last_modified_date (bool, kwargs): A flag giving the client the ability to choose if + the attribute last modified date should be changed. value (str, kwargs): The attribute value. """ diff --git a/tcex/api/tc/v3/indicator_attributes/indicator_attribute_filter.py b/tcex/api/tc/v3/indicator_attributes/indicator_attribute_filter.py index 640b83ac1..d4994b44a 100644 --- a/tcex/api/tc/v3/indicator_attributes/indicator_attribute_filter.py +++ b/tcex/api/tc/v3/indicator_attributes/indicator_attribute_filter.py @@ -51,6 +51,15 @@ def date_val(self, operator: Enum, date_val: Arrow | datetime | int | str): date_val = self.util.any_to_datetime(date_val).strftime('%Y-%m-%d %H:%M:%S') self._tql.add_filter('dateVal', operator, date_val, TqlType.STRING) + def default(self, operator: Enum, default: bool): + """Filter Associable based on **default** keyword. + + Args: + operator: The operator enum for the filter. + default: A flag to include an attribute in indicator associations. + """ + self._tql.add_filter('default', operator, default, TqlType.BOOLEAN) + def displayed(self, operator: Enum, displayed: bool): """Filter Displayed based on **displayed** keyword. @@ -189,6 +198,21 @@ def pinned(self, operator: Enum, pinned: bool): """ self._tql.add_filter('pinned', operator, pinned, TqlType.BOOLEAN) + def short_text(self, operator: Enum, short_text: list | str): + """Filter Text based on **shortText** keyword. + + Args: + operator: The operator enum for the filter. + short_text: The short text of the attribute (only applies to certain types). + """ + if isinstance(short_text, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('shortText', operator, short_text, TqlType.STRING) + def source(self, operator: Enum, source: list | str): """Filter Source based on **source** keyword. diff --git a/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py b/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py index 74b06b3e6..de8edd289 100644 --- a/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py +++ b/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py @@ -107,6 +107,16 @@ class IndicatorAttributeModel( read_only=False, title='type', ) + update_last_modified_date: bool = Field( + None, + description=( + 'A flag giving the client the ability to choose if the attribute last modified date ' + 'should be changed.' + ), + methods=['POST', 'PUT'], + read_only=False, + title='updateLastModifiedDate', + ) value: str | None = Field( None, description='The attribute value.', diff --git a/tcex/api/tc/v3/indicators/indicator.py b/tcex/api/tc/v3/indicators/indicator.py index 273efe7e2..c645f19a0 100644 --- a/tcex/api/tc/v3/indicators/indicator.py +++ b/tcex/api/tc/v3/indicators/indicator.py @@ -49,7 +49,7 @@ class Indicator(ObjectABC): attributes (IndicatorAttributes, kwargs): A list of Attributes corresponding to the Indicator. confidence (int, kwargs): The indicator threat confidence. - custom_association_name (str, kwargs): The custom association name if assigned to this + custom_association_names (array, kwargs): The custom association names assigned to this indicator. custom_associations (Indicators, kwargs): A list of indicators with custom associations to this indicator. @@ -198,6 +198,11 @@ def associated_indicators(self) -> Generator[Self, None, None]: continue yield indicator # type: ignore + @property + def custom_associations(self) -> Generator[Self, None, None]: + """Yield Indicator from Indicators.""" + yield from self._iterate_over_sublist(Indicators, custom_associations=True) # type: ignore + @property def attributes(self) -> Generator['IndicatorAttribute', None, None]: """Yield Attribute from Attributes.""" diff --git a/tcex/api/tc/v3/indicators/indicator_filter.py b/tcex/api/tc/v3/indicators/indicator_filter.py index 8fbe6557b..e47b72fc0 100644 --- a/tcex/api/tc/v3/indicators/indicator_filter.py +++ b/tcex/api/tc/v3/indicators/indicator_filter.py @@ -411,6 +411,16 @@ def first_seen(self, operator: Enum, first_seen: Arrow | datetime | int | str): first_seen = self.util.any_to_datetime(first_seen).strftime('%Y-%m-%d %H:%M:%S') self._tql.add_filter('firstSeen', operator, first_seen, TqlType.STRING) + @property + def has_all_tags(self): + """Return **TagFilter** for further filtering.""" + # first-party + from tcex.api.tc.v3.tags.tag_filter import TagFilter + + tags = TagFilter(Tql()) + self._tql.add_filter('hasAllTags', TqlOperator.EQ, tags, TqlType.SUB_QUERY) + return tags + @property def has_artifact(self): """Return **ArtifactFilter** for further filtering.""" @@ -443,6 +453,23 @@ def has_case(self): self._tql.add_filter('hasCase', TqlOperator.EQ, cases, TqlType.SUB_QUERY) return cases + def has_custom_association(self, operator: Enum, has_custom_association: int | list): + """Filter Associated Indicator based on **hasCustomAssociation** keyword. + + Args: + operator: The operator enum for the filter. + has_custom_association: A nested query for association to other indicators. + """ + if isinstance(has_custom_association, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter( + 'hasCustomAssociation', operator, has_custom_association, TqlType.INTEGER + ) + @property def has_group(self): """Return **GroupFilter** for further filtering.""" @@ -663,6 +690,40 @@ def rating(self, operator: Enum, rating: int | list): self._tql.add_filter('rating', operator, rating, TqlType.INTEGER) + def risk_iq_classification(self, operator: Enum, risk_iq_classification: list | str): + """Filter RiskIQ Classification based on **riskIqClassification** keyword. + + Args: + operator: The operator enum for the filter. + risk_iq_classification: The classification from the RiskIQ enrichment data. + """ + if isinstance(risk_iq_classification, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter( + 'riskIqClassification', operator, risk_iq_classification, TqlType.STRING + ) + + def risk_iq_reputation_score(self, operator: Enum, risk_iq_reputation_score: int | list): + """Filter RiskIQ Reputation Score based on **riskIqReputationScore** keyword. + + Args: + operator: The operator enum for the filter. + risk_iq_reputation_score: The reputation score from the RiskIQ enrichment data. + """ + if isinstance(risk_iq_reputation_score, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter( + 'riskIqReputationScore', operator, risk_iq_reputation_score, TqlType.INTEGER + ) + def security_label(self, operator: Enum, security_label: list | str): """Filter Security Label based on **securityLabel** keyword. diff --git a/tcex/api/tc/v3/indicators/indicator_model.py b/tcex/api/tc/v3/indicators/indicator_model.py index 1bb410b1b..3357ae1d7 100644 --- a/tcex/api/tc/v3/indicators/indicator_model.py +++ b/tcex/api/tc/v3/indicators/indicator_model.py @@ -93,12 +93,12 @@ class IndicatorModel( read_only=False, title='confidence', ) - custom_association_name: str | None = Field( + custom_association_names: list[str] = Field( None, - description='The custom association name if assigned to this indicator.', + description='The custom association names assigned to this indicator.', methods=['POST', 'PUT'], read_only=False, - title='customAssociationName', + title='customAssociationNames', ) custom_associations: 'IndicatorsModel' = Field( None, diff --git a/tcex/api/tc/v3/intel_requirements/intel_requirement_filter.py b/tcex/api/tc/v3/intel_requirements/intel_requirement_filter.py index 8551c8015..928b6ada2 100644 --- a/tcex/api/tc/v3/intel_requirements/intel_requirement_filter.py +++ b/tcex/api/tc/v3/intel_requirements/intel_requirement_filter.py @@ -321,6 +321,16 @@ def generated_report(self, operator: Enum, generated_report: bool): """ self._tql.add_filter('generatedReport', operator, generated_report, TqlType.BOOLEAN) + @property + def has_all_tags(self): + """Return **TagFilter** for further filtering.""" + # first-party + from tcex.api.tc.v3.tags.tag_filter import TagFilter + + tags = TagFilter(Tql()) + self._tql.add_filter('hasAllTags', TqlOperator.EQ, tags, TqlType.SUB_QUERY) + return tags + @property def has_artifact(self): """Return **ArtifactFilter** for further filtering.""" @@ -458,6 +468,21 @@ def id(self, operator: Enum, id: int | list): # pylint: disable=redefined-built self._tql.add_filter('id', operator, id, TqlType.INTEGER) + def insights(self, operator: Enum, insights: list | str): + """Filter Insights (Report) based on **insights** keyword. + + Args: + operator: The operator enum for the filter. + insights: The AI generated synopsis of the report. + """ + if isinstance(insights, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('insights', operator, insights, TqlType.STRING) + def is_group(self, operator: Enum, is_group: bool): """Filter isGroup based on **isGroup** keyword. diff --git a/tcex/api/tc/v3/notes/note_filter.py b/tcex/api/tc/v3/notes/note_filter.py index d2d24ee93..338ddbd34 100644 --- a/tcex/api/tc/v3/notes/note_filter.py +++ b/tcex/api/tc/v3/notes/note_filter.py @@ -67,6 +67,21 @@ def case_id(self, operator: Enum, case_id: int | list): self._tql.add_filter('caseId', operator, case_id, TqlType.INTEGER) + def data(self, operator: Enum, data: list | str): + """Filter Data based on **data** keyword. + + Args: + operator: The operator enum for the filter. + data: Contents of the note. + """ + if isinstance(data, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('data', operator, data, TqlType.STRING) + def date_added(self, operator: Enum, date_added: Arrow | datetime | int | str): """Filter Date Added based on **dateAdded** keyword. diff --git a/tcex/api/tc/v3/object_abc.py b/tcex/api/tc/v3/object_abc.py index 34585e81a..d6051f7a3 100644 --- a/tcex/api/tc/v3/object_abc.py +++ b/tcex/api/tc/v3/object_abc.py @@ -71,7 +71,7 @@ def _calculate_unique_id(self) -> dict[str, int | str]: return {} def _iterate_over_sublist( - self, sublist_type: ObjectCollectionABC + self, sublist_type: ObjectCollectionABC, custom_associations: bool = False ) -> Generator[Self, None, None]: """Iterate over any nested collections.""" sublist = sublist_type(session=self._session) # type: ignore @@ -79,11 +79,19 @@ def _iterate_over_sublist( # determine the filter type and value based on the available object fields. unique_id_data = self._calculate_unique_id() + # id!=2984993+AND+hasGroup(typename="all"+AND+isGroup+=+false+AND+hasIndicator(id=2984993)) # add the filter (e.g., group.has_indicator.id(TqlOperator.EQ, 123)) for the parent object. - getattr( - getattr(sublist.filter, self._nested_filter), # type: ignore - unique_id_data.get('filter'), # type: ignore - )(TqlOperator.EQ, unique_id_data.get('value')) + if custom_associations is True: + sublist.filter.tql = ( + 'id!={id} AND hasGroup(typename="all" ' + 'AND isGroup = false AND hasIndicator(id={id}))' + ).format(id=self.model.id) + sublist.params = {'fields': ['associationName']} + else: + getattr( + getattr(sublist.filter, self._nested_filter), # type: ignore + unique_id_data.get('filter'), # type: ignore + )(TqlOperator.EQ, unique_id_data.get('value')) # return the sub object, injecting the parent data for obj in sublist: diff --git a/tcex/api/tc/v3/security/assignee_user_model.py b/tcex/api/tc/v3/security/assignee_user_model.py index fdc070dc7..3013df24a 100644 --- a/tcex/api/tc/v3/security/assignee_user_model.py +++ b/tcex/api/tc/v3/security/assignee_user_model.py @@ -17,8 +17,8 @@ class AssigneeUserModel( ): """Assignee Model""" - user_name: str | None = Field( - None, + user_name: str = Field( + ..., allow_mutation=False, description='The **user name** for the User.', methods=['POST', 'PUT'], diff --git a/tcex/api/tc/v3/tags/tag.py b/tcex/api/tc/v3/tags/tag.py index 5a098f6aa..102f448f2 100644 --- a/tcex/api/tc/v3/tags/tag.py +++ b/tcex/api/tc/v3/tags/tag.py @@ -18,6 +18,8 @@ class Tag(ObjectABC): description (str, kwargs): A brief description of the Tag. name (str, kwargs): The **name** for the Tag. owner (str, kwargs): The name of the Owner of the Tag. + security_coverage (object, kwargs): For ATT&CK-based tags, this is the security coverage + level assigned to the tag. """ def __init__(self, **kwargs): diff --git a/tcex/api/tc/v3/tags/tag_filter.py b/tcex/api/tc/v3/tags/tag_filter.py index cde0ede03..9e14ed529 100644 --- a/tcex/api/tc/v3/tags/tag_filter.py +++ b/tcex/api/tc/v3/tags/tag_filter.py @@ -222,6 +222,22 @@ def owner_name(self, operator: Enum, owner_name: list | str): self._tql.add_filter('ownerName', operator, owner_name, TqlType.STRING) + def security_coverage(self, operator: Enum, security_coverage: list | str): + """Filter Security Coverage based on **securityCoverage** keyword. + + Args: + operator: The operator enum for the filter. + security_coverage: The security coverage level of an ATT&CK-based tag as it relates to + the user's organization. + """ + if isinstance(security_coverage, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('securityCoverage', operator, security_coverage, TqlType.STRING) + def summary(self, operator: Enum, summary: list | str): """Filter Summary based on **summary** keyword. diff --git a/tcex/api/tc/v3/tags/tag_model.py b/tcex/api/tc/v3/tags/tag_model.py index d23edaee4..35449a220 100644 --- a/tcex/api/tc/v3/tags/tag_model.py +++ b/tcex/api/tc/v3/tags/tag_model.py @@ -96,6 +96,15 @@ class TagModel( read_only=True, title='platforms', ) + security_coverage: dict | None = Field( + None, + description=( + 'For ATT&CK-based tags, this is the security coverage level assigned to the tag.' + ), + methods=['POST', 'PUT'], + read_only=False, + title='securityCoverage', + ) synonymous_tag_names: dict | None = Field( None, allow_mutation=False, diff --git a/tcex/api/tc/v3/v3_model_abc.py b/tcex/api/tc/v3/v3_model_abc.py index 78eab0a62..db02ef7d7 100644 --- a/tcex/api/tc/v3/v3_model_abc.py +++ b/tcex/api/tc/v3/v3_model_abc.py @@ -34,7 +34,7 @@ def default(self, o: Any) -> str: return o -class V3ModelABC(BaseModel, ABC): +class V3ModelABC(BaseModel, ABC, allow_population_by_field_name=True): """V3 Base Model""" _associated_type = PrivateAttr(False) diff --git a/tcex/api/tc/v3/victim_attributes/victim_attribute.py b/tcex/api/tc/v3/victim_attributes/victim_attribute.py index f70210a4e..143b8075c 100644 --- a/tcex/api/tc/v3/victim_attributes/victim_attribute.py +++ b/tcex/api/tc/v3/victim_attributes/victim_attribute.py @@ -31,6 +31,8 @@ class VictimAttribute(ObjectABC): the one(s) specified). source (str, kwargs): The attribute source. type (str, kwargs): The attribute type. + update_last_modified_date (bool, kwargs): A flag giving the client the ability to choose if + the attribute last modified date should be changed. value (str, kwargs): The attribute value. victim_id (int, kwargs): Victim associated with attribute. """ diff --git a/tcex/api/tc/v3/victim_attributes/victim_attribute_filter.py b/tcex/api/tc/v3/victim_attributes/victim_attribute_filter.py index 19dc2f24d..9f304923a 100644 --- a/tcex/api/tc/v3/victim_attributes/victim_attribute_filter.py +++ b/tcex/api/tc/v3/victim_attributes/victim_attribute_filter.py @@ -42,6 +42,15 @@ def date_val(self, operator: Enum, date_val: Arrow | datetime | int | str): date_val = self.util.any_to_datetime(date_val).strftime('%Y-%m-%d %H:%M:%S') self._tql.add_filter('dateVal', operator, date_val, TqlType.STRING) + def default(self, operator: Enum, default: bool): + """Filter Default based on **default** keyword. + + Args: + operator: The operator enum for the filter. + default: A flag that is set by an attribute type configuration. + """ + self._tql.add_filter('default', operator, default, TqlType.BOOLEAN) + def displayed(self, operator: Enum, displayed: bool): """Filter Displayed based on **displayed** keyword. @@ -165,6 +174,21 @@ def pinned(self, operator: Enum, pinned: bool): """ self._tql.add_filter('pinned', operator, pinned, TqlType.BOOLEAN) + def short_text(self, operator: Enum, short_text: list | str): + """Filter Text based on **shortText** keyword. + + Args: + operator: The operator enum for the filter. + short_text: The short text of the attribute (only applies to certain types). + """ + if isinstance(short_text, list) and operator not in self.list_types: + raise RuntimeError( + 'Operator must be CONTAINS, NOT_CONTAINS, IN' + 'or NOT_IN when filtering on a list of values.' + ) + + self._tql.add_filter('shortText', operator, short_text, TqlType.STRING) + def source(self, operator: Enum, source: list | str): """Filter Source based on **source** keyword. diff --git a/tcex/api/tc/v3/victim_attributes/victim_attribute_model.py b/tcex/api/tc/v3/victim_attributes/victim_attribute_model.py index 9b8584ec0..b489ef70c 100644 --- a/tcex/api/tc/v3/victim_attributes/victim_attribute_model.py +++ b/tcex/api/tc/v3/victim_attributes/victim_attribute_model.py @@ -93,6 +93,16 @@ class VictimAttributeModel( read_only=False, title='type', ) + update_last_modified_date: bool = Field( + None, + description=( + 'A flag giving the client the ability to choose if the attribute last modified date ' + 'should be changed.' + ), + methods=['POST', 'PUT'], + read_only=False, + title='updateLastModifiedDate', + ) value: str | None = Field( None, description='The attribute value.', diff --git a/tcex/api/tc/v3/victims/victim_filter.py b/tcex/api/tc/v3/victims/victim_filter.py index 4af0126b7..88786eafe 100644 --- a/tcex/api/tc/v3/victims/victim_filter.py +++ b/tcex/api/tc/v3/victims/victim_filter.py @@ -93,6 +93,16 @@ def description(self, operator: Enum, description: list | str): self._tql.add_filter('description', operator, description, TqlType.STRING) + @property + def has_all_tags(self): + """Return **TagFilter** for further filtering.""" + # first-party + from tcex.api.tc.v3.tags.tag_filter import TagFilter + + tags = TagFilter(Tql()) + self._tql.add_filter('hasAllTags', TqlOperator.EQ, tags, TqlType.SUB_QUERY) + return tags + @property def has_attribute(self): """Return **VictimAttributeFilter** for further filtering.""" diff --git a/tcex/app/config b/tcex/app/config index c374d2414..79ab61680 160000 --- a/tcex/app/config +++ b/tcex/app/config @@ -1 +1 @@ -Subproject commit c374d2414d7603415b865b93f2e8bf405a34e88c +Subproject commit 79ab61680087851df45f17afbc9c898fb4410fbd diff --git a/tcex/app/key_value_store b/tcex/app/key_value_store index 2c8e730f1..76b469431 160000 --- a/tcex/app/key_value_store +++ b/tcex/app/key_value_store @@ -1 +1 @@ -Subproject commit 2c8e730f1ae90adfe2ad6a8642adc1b2bcbef83b +Subproject commit 76b469431cc689fbfb6671a80a9cd43284133309 diff --git a/tcex/input/field_type/key_value.py b/tcex/input/field_type/key_value.py index 74af4b792..a9e0437a1 100644 --- a/tcex/input/field_type/key_value.py +++ b/tcex/input/field_type/key_value.py @@ -48,3 +48,26 @@ class Config: KeyValue.update_forward_refs() + + +def key_value(allow_none=False): + """Return configured instance of KeyValue model.""" + key_value_model = KeyValue + if allow_none is True: + + class _KeyValue(KeyValue): + value: ( + list[KeyValue] + | KeyValue + | list[TCEntity] + | TCEntity + | list[String] + | String + | list[Binary] + | Binary + | Sensitive + | None + ) + + key_value_model = _KeyValue + return key_value_model diff --git a/tcex/logger/api_handler.py b/tcex/logger/api_handler.py index f36c70b9c..a3cc6a763 100644 --- a/tcex/logger/api_handler.py +++ b/tcex/logger/api_handler.py @@ -60,7 +60,7 @@ def handle(self, record: logging.LogRecord): # for the I/O lock from super().handle. if threading.current_thread().name != 'MainThread': with self._entries_lock: - self._entries.append(record) + self._entries.append(self.format(record)) else: super().handle(record) diff --git a/tcex/tcex.py b/tcex/tcex.py index 960a842d0..78d17aeb6 100644 --- a/tcex/tcex.py +++ b/tcex/tcex.py @@ -120,10 +120,9 @@ def logger(self) -> Logger: self._log = _logger.log # add api handler - if ( - self.inputs.contents.get('tc_token') is not None - and self.inputs.contents.get('tc_log_to_api') is True - ): + if self.inputs.contents.get('tc_token') is not None and self.inputs.contents.get( + 'tc_log_to_api' + ) in (True, 'true'): _logger.add_api_handler( session_tc=self.requests_tc.get_session(), level=self.inputs.model_tc.tc_log_level, diff --git a/tests/api/tc/v2/batch/test_group_interface_1.py b/tests/api/tc/v2/batch/test_group_interface_1.py index 323688c86..1cd5855f8 100644 --- a/tests/api/tc/v2/batch/test_group_interface_1.py +++ b/tests/api/tc/v2/batch/test_group_interface_1.py @@ -145,7 +145,8 @@ def test_event(self, name, description, label, tag, tcex: TcEx): """Test event creation""" batch = tcex.api.tc.v2.batch(owner='TCI') xid = batch.generate_xid(['pytest', 'event', name]) - ti = batch.event(name=name, event_data='2008-12-12T12:12:12Z', xid=xid) + ti = batch.event(name=name, event_date='2008-12-12T12:12:12Z', xid=xid) + # ti = batch.event(name=name, event_date='2008-12-12', xid=xid) ti.status = 'Escalated' ti.attribute(attr_type='Description', attr_value=description, displayed=True) ti.security_label(name=label, description='Pytest Label Description', color='ffc0cb') diff --git a/tests/api/tc/v2/batch/test_group_interface_2.py b/tests/api/tc/v2/batch/test_group_interface_2.py index a029ccc34..b7b54e8f4 100644 --- a/tests/api/tc/v2/batch/test_group_interface_2.py +++ b/tests/api/tc/v2/batch/test_group_interface_2.py @@ -163,7 +163,7 @@ def test_event(self, name, description, label, tag, tcex: TcEx): xid = batch.generate_xid(['pytest', 'event', name]) ti = cast( Group, - batch.group(group_type='Event', name=name, event_data='2008-12-12T12:12:12Z', xid=xid), + batch.group(group_type='Event', name=name, event_date='2008-12-12T12:12:12Z', xid=xid), ) ti.status = 'Escalated' ti.attribute(attr_type='Description', attr_value=description, displayed=True) diff --git a/tests/api/tc/v3/cases/test_case_interface.py b/tests/api/tc/v3/cases/test_case_interface.py index aca70dbf4..39dbef4ee 100644 --- a/tests/api/tc/v3/cases/test_case_interface.py +++ b/tests/api/tc/v3/cases/test_case_interface.py @@ -376,6 +376,7 @@ def test_case_all_filters(self, request: FixtureRequest): case_occurrence_time = datetime.now() - timedelta(days=20) assignee = {'type': 'User', 'data': {'user_name': os.getenv('TC_API_ACCESS_ID')}} + # assignee = {'type': 'Group', 'data': {'name': 'TcEx Testing'}} # [Create Testing] define case data case_data = { diff --git a/tests/api/tc/v3/indicators/test_indicator_interface.py b/tests/api/tc/v3/indicators/test_indicator_interface.py index a893ba5f3..16620bbc3 100644 --- a/tests/api/tc/v3/indicators/test_indicator_interface.py +++ b/tests/api/tc/v3/indicators/test_indicator_interface.py @@ -65,6 +65,12 @@ def test_indicator_object_properties_extra(self): # return + # def test_custom_associations(self): + # """.""" + # indicators = self.v3.indicator(id=2984993) + # for indicator in enumerate(indicators.custom_associations, start=1): + # print(indicator.model.json(indent=4, exclude_none=True, exclude_unset=True)) + def test_indicator_create_and_retrieve_nested_types(self): """Test Object Creation diff --git a/tests/api/tc/v3/v3_helpers.py b/tests/api/tc/v3/v3_helpers.py index aa11c6438..9f6037af6 100644 --- a/tests/api/tc/v3/v3_helpers.py +++ b/tests/api/tc/v3/v3_helpers.py @@ -757,12 +757,12 @@ def obj_api_options(self): ] if self.v3_helper.v3_object == 'indicators': + if 'associationName' in names: + # fix discrepancy between /fields and + names = ['customAssociationNames'] if 'genericCustomIndicatorValues' in names: # fix discrepancy between /fields and names = ['value1', 'value2', 'value3'] - if 'whoIs' in names: - # fix discrepancy between /fields and - names = ['whois'] if 'threatAssess' in names: # fix discrepancy between /fields and names = [ @@ -772,6 +772,9 @@ def obj_api_options(self): 'threatAssessScoreFalsePositive', 'threatAssessScoreObserved', ] + if 'whoIs' in names: + # fix discrepancy between /fields and + names = ['whois'] if self.v3_helper.v3_object == 'results': if 'intelRequirementDetails' in names: