Updates for multiple tickets

.gitignore

@@ -53,6 +53,7 @@ templates
 threatconnect*
 test.log*
 tests/reports/*
+tcex/api/tc/v3/_gen/options_data
 TODO.md
 
 #-------------------------------------------------

release_notes.md

@@ -1,8 +1,14 @@
 # Release Notes
 
 ## 4.0.4
--   APP-4334 - [API] Call method transforms in TI transform models even if value is not a string
--   APP-4380 - [API] Add externalDateAdded, externalDateExpires, externalLastModified, firstSeen, and lastSeen fields to TI Transform model
+
+-   APP-4307 - [API] Added support for paginating indicator custom associations
+-   APP-4334 - [API] Fixed issue where transform methods wasn't being called if value was not a string
+-   APP-4380 - [API] Added support for external date fields to TI Transform model
+-   APP-4381 - [Logging] Fixed API logging issues
+-   APP-4383 - [API] Updated TC API module for changes in the V3 API
+-   APP-4400 - [Input] Added support for KeyValue input type with None value
+-   APP-4401 - [API] Fixed issue with Assignee model not calculating appropriate model type
 
 ## 4.0.3
 

tcex/api/tc/v3/_gen/_gen_abc.py

@@ -1,8 +1,10 @@
 """TcEx Framework Module"""
 # standard library
+import json
 import os
 from abc import ABC
 from collections.abc import Generator
+from pathlib import Path
 from textwrap import TextWrapper
 
 # third-party
@@ -103,6 +105,11 @@ def _prop_contents(self) -> dict:
             # print(r.request.method, r.request.url, r.text)
             if r.ok:
                 _properties = r.json()
+
+                options_data = Path('tcex/api/tc/v3/_gen/options_data') / f'{self.type_}.json'
+                with options_data.open(mode='w') as fh:
+                    json.dump(r.json(), fh, indent=2)
+
         except (ConnectionError, ProxyError) as ex:
             Render.panel.failure(f'Failed getting types properties ({ex}).')
 
@@ -143,6 +150,11 @@ def _prop_contents_data(self, properties: dict) -> Generator:
                 # pair is a list (currently the list only contains a single dict). to be safe
                 # we loop over the list and update the type for each item.
                 field_data = field_data[0]
+
+                # # handle special case for customAssociationNames, where the data is an array of
+                # # string, but the type in the object states 'String'.
+                # if field_data['type'] == 'String':
+                #     field_data['type'] = 'ListString'
             else:
                 raise RuntimeError(
                     f'Invalid type properties data: field-name={field_name}, type={self.type_}'

tcex/api/tc/v3/_gen/_gen_filter_abc.py

@@ -315,6 +315,25 @@ def _gen_code_has_tag_method(self) -> list:
             '',
         ]
 
+    def _gen_code_has_all_tags_method(self) -> list:
+        """Return code for has_tag TQL filter methods."""
+        self._add_tql_imports()
+        return [
+            f'{self.i1}@property',
+            f'{self.i1}def has_all_tags(self):',
+            f'{self.i2}"""Return **TagFilter** for further filtering."""',
+            f'{self.i2}# first-party',
+            f'{self.i2}from tcex.api.tc.v3.tags.tag_filter import TagFilter',
+            '',
+            f'{self.i2}tags = TagFilter(Tql())',
+            (
+                f'''{self.i2}self._tql.add_filter('hasAllTags', '''
+                '''TqlOperator.EQ, tags, TqlType.SUB_QUERY)'''
+            ),
+            f'{self.i2}return tags',
+            '',
+        ]
+
     def _gen_code_has_task_method(self) -> list:
         """Return code for has_task TQL filter methods."""
         self._add_tql_imports()
@@ -495,6 +514,8 @@ def gen_class_methods(self):
                 _filter_class.extend(self._gen_code_has_note_method())
             elif f.keyword.snake_case() == 'has_tag':
                 _filter_class.extend(self._gen_code_has_tag_method())
+            elif f.keyword.snake_case() == 'has_all_tags':
+                _filter_class.extend(self._gen_code_has_all_tags_method())
             elif f.keyword.snake_case() == 'has_task':
                 _filter_class.extend(self._gen_code_has_task_method())
             elif f.keyword.snake_case() == 'has_attribute':

tcex/api/tc/v3/_gen/_gen_object_abc.py

@@ -609,7 +609,7 @@ def stage_assignee(
         )
 
     def _gen_code_object_type_property_method(
-        self, type_: str, model_type: str | None = None
+        self, type_: str, model_type: str | None = None, custom_associations: bool = False
     ) -> str:
         """Return the method code.
 
@@ -670,7 +670,18 @@ def artifacts(self):
 
         # Custom logic to ensure that when iterating over the associated indicators or associated
         # groups then the item currently being iterated over is not included in the results.
-        if (
+        if custom_associations is True:
+            _code.extend(
+                [
+                    (
+                        f'''{self.i2}yield from self._iterate_over_sublist'''
+                        f'''({model_import_data.get('object_collection_class')}, '''
+                        '''custom_associations=True)'''
+                        '''  # type: ignore'''
+                    ),
+                ]
+            )
+        elif (
             self.type_ in ['indicators', 'groups', 'artifacts', 'cases']
             and model_type == f'associated_{self.type_}'
         ):
@@ -914,6 +925,12 @@ def filter ...
                 'indicators', 'associated_indicators'
             )
 
+        # generate custom_associations property method
+        if 'customAssociations' in add_properties:
+            _code += self._gen_code_object_type_property_method(
+                'indicators', 'custom_associations', custom_associations=True
+            )
+
         # generate associated_victim_asset property method
         if 'associatedVictimAssets' in add_properties:
             _code += self._gen_code_object_type_property_method(

tcex/api/tc/v3/_gen/model/_property_model.py

@@ -163,6 +163,7 @@ def __process_bool_types(cls, pm: 'PropertyModel', extra: dict[str, str]):
     def __process_dict_types(cls, pm: 'PropertyModel', extra: dict[str, str]):
         """Process standard type."""
         types = [
+            'AttackSecurityCoverage',
             'AttributeSource',
             'DNSResolutions',
             'Enrichments',
@@ -308,6 +309,12 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: dict[str, str]):
                     'typing_type': f'list[{cls.__extra_format_type_model(pm.type)}]',
                 }
             )
+        elif pm.name == 'customAssociationNames':
+            extra.update(
+                {
+                    'typing_type': 'list[str]',
+                }
+            )
         elif pm.type == 'TaskAssignees':
             extra.update(
                 {

tcex/api/tc/v3/attribute_types/attribute_type_filter.py

@@ -31,6 +31,15 @@ def associated_type(self, operator: Enum, associated_type: list | str):
 
         self._tql.add_filter('associatedType', operator, associated_type, TqlType.STRING)
 
+    def default(self, operator: Enum, default: bool):
+        """Filter Displayed based on **default** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            default: Whether or not the attribute type is displayable on the item.
+        """
+        self._tql.add_filter('default', operator, default, TqlType.BOOLEAN)
+
     def description(self, operator: Enum, description: list | str):
         """Filter Description based on **description** keyword.
 

tcex/api/tc/v3/case_attributes/case_attribute.py

@@ -32,6 +32,8 @@ class CaseAttribute(ObjectABC):
             the one(s) specified).
         source (str, kwargs): The attribute source.
         type (str, kwargs): The attribute type.
+        update_last_modified_date (bool, kwargs): A flag giving the client the ability to choose if
+            the attribute last modified date should be changed.
         value (str, kwargs): The attribute value.
     """
 

tcex/api/tc/v3/case_attributes/case_attribute_filter.py

@@ -57,6 +57,15 @@ def date_val(self, operator: Enum, date_val: Arrow | datetime | int | str):
         date_val = self.util.any_to_datetime(date_val).strftime('%Y-%m-%d %H:%M:%S')
         self._tql.add_filter('dateVal', operator, date_val, TqlType.STRING)
 
+    def default(self, operator: Enum, default: bool):
+        """Filter Default based on **default** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            default: A flag that is set by an attribute type configuration.
+        """
+        self._tql.add_filter('default', operator, default, TqlType.BOOLEAN)
+
     def displayed(self, operator: Enum, displayed: bool):
         """Filter Displayed based on **displayed** keyword.
 
@@ -170,6 +179,21 @@ def pinned(self, operator: Enum, pinned: bool):
         """
         self._tql.add_filter('pinned', operator, pinned, TqlType.BOOLEAN)
 
+    def short_text(self, operator: Enum, short_text: list | str):
+        """Filter Text based on **shortText** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            short_text: The short text of the attribute (only applies to certain types).
+        """
+        if isinstance(short_text, list) and operator not in self.list_types:
+            raise RuntimeError(
+                'Operator must be CONTAINS, NOT_CONTAINS, IN'
+                'or NOT_IN when filtering on a list of values.'
+            )
+
+        self._tql.add_filter('shortText', operator, short_text, TqlType.STRING)
+
     def source(self, operator: Enum, source: list | str):
         """Filter Source based on **source** keyword.
 

tcex/api/tc/v3/case_attributes/case_attribute_model.py

@@ -100,6 +100,16 @@ class CaseAttributeModel(
         read_only=False,
         title='type',
     )
+    update_last_modified_date: bool = Field(
+        None,
+        description=(
+            'A flag giving the client the ability to choose if the attribute last modified date '
+            'should be changed.'
+        ),
+        methods=['POST', 'PUT'],
+        read_only=False,
+        title='updateLastModifiedDate',
+    )
     value: str | None = Field(
         None,
         description='The attribute value.',

tcex/api/tc/v3/cases/case_filter.py

@@ -267,6 +267,16 @@ def description(self, operator: Enum, description: list | str):
 
         self._tql.add_filter('description', operator, description, TqlType.STRING)
 
+    @property
+    def has_all_tags(self):
+        """Return **TagFilter** for further filtering."""
+        # first-party
+        from tcex.api.tc.v3.tags.tag_filter import TagFilter
+
+        tags = TagFilter(Tql())
+        self._tql.add_filter('hasAllTags', TqlOperator.EQ, tags, TqlType.SUB_QUERY)
+        return tags
+
     @property
     def has_artifact(self):
         """Return **ArtifactFilter** for further filtering."""

tcex/api/tc/v3/group_attributes/group_attribute.py

@@ -32,6 +32,8 @@ class GroupAttribute(ObjectABC):
             the one(s) specified).
         source (str, kwargs): The attribute source.
         type (str, kwargs): The attribute type.
+        update_last_modified_date (bool, kwargs): A flag giving the client the ability to choose if
+            the attribute last modified date should be changed.
         value (str, kwargs): The attribute value.
     """
 

tcex/api/tc/v3/group_attributes/group_attribute_filter.py

@@ -51,6 +51,15 @@ def date_val(self, operator: Enum, date_val: Arrow | datetime | int | str):
         date_val = self.util.any_to_datetime(date_val).strftime('%Y-%m-%d %H:%M:%S')
         self._tql.add_filter('dateVal', operator, date_val, TqlType.STRING)
 
+    def default(self, operator: Enum, default: bool):
+        """Filter Associable based on **default** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            default: A flag to include an attribute in group associations.
+        """
+        self._tql.add_filter('default', operator, default, TqlType.BOOLEAN)
+
     def displayed(self, operator: Enum, displayed: bool):
         """Filter Displayed based on **displayed** keyword.
 
@@ -189,6 +198,21 @@ def pinned(self, operator: Enum, pinned: bool):
         """
         self._tql.add_filter('pinned', operator, pinned, TqlType.BOOLEAN)
 
+    def short_text(self, operator: Enum, short_text: list | str):
+        """Filter Text based on **shortText** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            short_text: The short text of the attribute (only applies to certain types).
+        """
+        if isinstance(short_text, list) and operator not in self.list_types:
+            raise RuntimeError(
+                'Operator must be CONTAINS, NOT_CONTAINS, IN'
+                'or NOT_IN when filtering on a list of values.'
+            )
+
+        self._tql.add_filter('shortText', operator, short_text, TqlType.STRING)
+
     def source(self, operator: Enum, source: list | str):
         """Filter Source based on **source** keyword.
 

tcex/api/tc/v3/group_attributes/group_attribute_model.py

@@ -107,6 +107,16 @@ class GroupAttributeModel(
         read_only=False,
         title='type',
     )
+    update_last_modified_date: bool = Field(
+        None,
+        description=(
+            'A flag giving the client the ability to choose if the attribute last modified date '
+            'should be changed.'
+        ),
+        methods=['POST', 'PUT'],
+        read_only=False,
+        title='updateLastModifiedDate',
+    )
     value: str | None = Field(
         None,
         description='The attribute value.',

tcex/api/tc/v3/groups/group_filter.py

@@ -321,6 +321,16 @@ def generated_report(self, operator: Enum, generated_report: bool):
         """
         self._tql.add_filter('generatedReport', operator, generated_report, TqlType.BOOLEAN)
 
+    @property
+    def has_all_tags(self):
+        """Return **TagFilter** for further filtering."""
+        # first-party
+        from tcex.api.tc.v3.tags.tag_filter import TagFilter
+
+        tags = TagFilter(Tql())
+        self._tql.add_filter('hasAllTags', TqlOperator.EQ, tags, TqlType.SUB_QUERY)
+        return tags
+
     @property
     def has_artifact(self):
         """Return **ArtifactFilter** for further filtering."""
@@ -455,6 +465,21 @@ def id(self, operator: Enum, id: int | list):  # pylint: disable=redefined-built
 
         self._tql.add_filter('id', operator, id, TqlType.INTEGER)
 
+    def insights(self, operator: Enum, insights: list | str):
+        """Filter Insights (Report) based on **insights** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            insights: The AI generated synopsis of the report.
+        """
+        if isinstance(insights, list) and operator not in self.list_types:
+            raise RuntimeError(
+                'Operator must be CONTAINS, NOT_CONTAINS, IN'
+                'or NOT_IN when filtering on a list of values.'
+            )
+
+        self._tql.add_filter('insights', operator, insights, TqlType.STRING)
+
     def is_group(self, operator: Enum, is_group: bool):
         """Filter isGroup based on **isGroup** keyword.
 

tcex/api/tc/v3/groups/group_model.py

@@ -252,6 +252,14 @@ class GroupModel(
         read_only=True,
         title='id',
     )
+    insights: str | None = Field(
+        None,
+        allow_mutation=False,
+        applies_to=['Document', 'Report'],
+        description='An AI generated synopsis of the document.',
+        read_only=True,
+        title='insights',
+    )
     last_modified: datetime | None = Field(
         None,
         allow_mutation=False,

tcex/api/tc/v3/indicator_attributes/indicator_attribute.py

@@ -32,6 +32,8 @@ class IndicatorAttribute(ObjectABC):
             the one(s) specified).
         source (str, kwargs): The attribute source.
         type (str, kwargs): The attribute type.
+        update_last_modified_date (bool, kwargs): A flag giving the client the ability to choose if
+            the attribute last modified date should be changed.
         value (str, kwargs): The attribute value.
     """