Skip to content

Commit

Permalink
Merge pull request #602 from chisholm/relational-data-sink-fix-dictio…
Browse files Browse the repository at this point in the history 
…nary-property

Fix DictionaryProperty valid_types bugs, and other misc property issues
  • Loading branch information
@rpiazza
rpiazza authored Sep 16, 2024
2 parents 16aaf9c + f0f6d8f commit 92231d9
Show file tree
Hide file tree
Showing 3 changed files with 322 additions and 147 deletions.
224 changes: 140 additions & 84 deletions stix2/properties.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ class Property(object):
"""

def _default_clean(self, value, allow_custom=False):
def _default_clean(self, value, allow_custom=False, strict=False):
if value != self._fixed_value:
raise ValueError("must equal '{}'.".format(self._fixed_value))
return value, False
Expand Down Expand Up @@ -226,7 +226,7 @@ def __init__(self, contained, **kwargs):

super(ListProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom, strict_flag=False):
def clean(self, value, allow_custom=False, strict=False):
try:
iter(value)
except TypeError:
Expand All @@ -240,7 +240,7 @@ def clean(self, value, allow_custom, strict_flag=False):
if isinstance(self.contained, Property):
for item in value:
try:
valid, temp_custom = self.contained.clean(item, allow_custom, strict=strict_flag)
valid, temp_custom = self.contained.clean(item, allow_custom, strict=strict)
except TypeError:
valid, temp_custom = self.contained.clean(item, allow_custom)
result.append(valid)
Expand Down Expand Up @@ -281,11 +281,10 @@ def __init__(self, **kwargs):
super(StringProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom=False, strict=False):
if not isinstance(value, str):
if strict is True:
raise ValueError("Must be a string.")
if strict and not isinstance(value, str):
raise ValueError("Must be a string.")

value = str(value)
value = str(value)
return value, False


Expand Down Expand Up @@ -320,7 +319,7 @@ def __init__(self, min=None, max=None, **kwargs):
super(IntegerProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom=False, strict=False):
if strict is True and not isinstance(value, int):
if strict and not isinstance(value, int):
raise ValueError("must be an integer.")

try:
Expand All @@ -347,7 +346,7 @@ def __init__(self, min=None, max=None, **kwargs):
super(FloatProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom=False, strict=False):
if strict is True and not isinstance(value, float):
if strict and not isinstance(value, float):
raise ValueError("must be a float.")

try:
Expand All @@ -372,6 +371,9 @@ class BooleanProperty(Property):

def clean(self, value, allow_custom=False, strict=False):

if strict and not isinstance(value, bool):
raise ValueError("must be a boolean value.")

if isinstance(value, str):
value = value.lower()

Expand Down Expand Up @@ -403,74 +405,134 @@ class DictionaryProperty(Property):

def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
self.spec_version = spec_version
self.valid_types = self._normalize_valid_types(valid_types or [])

simple_types = [
BinaryProperty, BooleanProperty, FloatProperty, HexProperty, IntegerProperty, StringProperty,
TimestampProperty, ReferenceProperty, EnumProperty,
]
if not valid_types:
valid_types = [Property]
else:
if not isinstance(valid_types, list):
valid_types = [valid_types]
for type_ in valid_types:
if isinstance(type_, ListProperty):
found = False
for simple_type in simple_types:
if isinstance(type_.contained, simple_type):
found = True
if not found:
raise ValueError("Dictionary Property does not support lists of type: ", type_.contained, type(type_.contained))
elif type_ not in simple_types:
raise ValueError("Dictionary Property does not support this value's type: ", type_)
super(DictionaryProperty, self).__init__(**kwargs)

self.valid_types = valid_types
def _normalize_valid_types(self, valid_types):
"""
Normalize valid_types to a list of property instances. Also ensure any
property types given are supported for type enforcement.
super(DictionaryProperty, self).__init__(**kwargs)
:param valid_types: A single or iterable of Property instances or
subclasses
:return: A list of Property instances, or None if none were given
"""
simple_types = (
BinaryProperty, BooleanProperty, FloatProperty, HexProperty,
IntegerProperty, StringProperty, TimestampProperty,
ReferenceProperty, EnumProperty,
)

def clean(self, value, allow_custom=False):
# Normalize single prop instances/classes to lists
try:
iter(valid_types)
except TypeError:
valid_types = [valid_types]

prop_instances = []
for valid_type in valid_types:
if inspect.isclass(valid_type):
# Note: this will fail as of this writing with EnumProperty
# ReferenceProperty, ListProperty. Callers must instantiate
# those with suitable settings themselves.
prop_instance = valid_type()

else:
prop_instance = valid_type

# ListProperty's element type must be one of the supported
# simple types.
if isinstance(prop_instance, ListProperty):
if not isinstance(prop_instance.contained, simple_types):
raise ValueError(
"DictionaryProperty does not support lists of type: "
+ type(prop_instance.contained).__name__
)

elif not isinstance(prop_instance, simple_types):
raise ValueError(
"DictionaryProperty does not support value type: "
+ type(prop_instance).__name__
)

prop_instances.append(prop_instance)

return prop_instances or None

def _check_dict_key(self, k):
if self.spec_version == '2.0':
if len(k) < 3:
raise DictionaryKeyError(k, "shorter than 3 characters")
elif len(k) > 256:
raise DictionaryKeyError(k, "longer than 256 characters")
elif self.spec_version == '2.1':
if len(k) > 250:
raise DictionaryKeyError(k, "longer than 250 characters")
if not re.match(r"^[a-zA-Z0-9_-]+$", k):
msg = (
"contains characters other than lowercase a-z, "
"uppercase A-Z, numerals 0-9, hyphen (-), or "
"underscore (_)"
)
raise DictionaryKeyError(k, msg)

def clean(self, value, allow_custom=False, strict=False):
try:
dictified = _get_dict(value)
except ValueError:
raise ValueError("The dictionary property must contain a dictionary")

for k in dictified.keys():
if self.spec_version == '2.0':
if len(k) < 3:
raise DictionaryKeyError(k, "shorter than 3 characters")
elif len(k) > 256:
raise DictionaryKeyError(k, "longer than 256 characters")
elif self.spec_version == '2.1':
if len(k) > 250:
raise DictionaryKeyError(k, "longer than 250 characters")
if not re.match(r"^[a-zA-Z0-9_-]+$", k):
msg = (
"contains characters other than lowercase a-z, "
"uppercase A-Z, numerals 0-9, hyphen (-), or "
"underscore (_)"
)
raise DictionaryKeyError(k, msg)
has_custom = False
for k, v in dictified.items():

clean = False
for type_ in self.valid_types:
if isinstance(type_, ListProperty):
type_.clean(value=dictified[k], allow_custom=False, strict_flag=True)
clean = True
else:
type_instance = type_()
self._check_dict_key(k)

if self.valid_types:
for type_ in self.valid_types:
try:
type_instance.clean(value=dictified[k], allow_custom=False, strict=True)
clean = True
# ReferenceProperty at least, does check for
# customizations, so we must propagate that
dictified[k], temp_custom = type_.clean(
value=v,
allow_custom=allow_custom,
# Ignore the passed-in value and fix this to True;
# we need strict cleaning to disambiguate value
# types here.
strict=True
)
except CustomContentError:
# Need to propagate these, not treat as a type error
raise
except Exception as e:
# clean failed; value must not conform to type_
# Should be a narrower exception type here, but I don't
# know if it's safe to assume any particular exception
# types...
pass
else:
# clean succeeded; should check the has_custom flag
# just in case. But if allow_custom is False, I expect
# one of the valid_types property instances would have
# already raised an exception.
has_custom = has_custom or temp_custom
if has_custom and not allow_custom:
raise CustomContentError(f'Custom content detected in key "{k}"')

break
except ValueError:
continue
if not clean:
raise ValueError("Dictionary Property does not support this value's type: ", type(dictified[k]))

else:
# clean failed for all properties!
raise ValueError(
f"Invalid value: {v!r}"
)

# else: no valid types given, so we skip the validity check

if len(dictified) < 1:
raise ValueError("must not be empty.")

return dictified, False
return dictified, has_custom


class HashesProperty(DictionaryProperty):
Expand All @@ -488,7 +550,7 @@ def __init__(self, spec_hash_names, spec_version=DEFAULT_VERSION, **kwargs):
if alg:
self.__alg_to_spec_name[alg] = spec_hash_name

def clean(self, value, allow_custom, strict=False):
def clean(self, value, allow_custom=False, strict=False):
# ignore the has_custom return value here; there is no customization
# of DictionaryProperties.
clean_dict, _ = super().clean(value, allow_custom)
Expand Down Expand Up @@ -597,7 +659,7 @@ def __init__(self, valid_types=None, invalid_types=None, spec_version=DEFAULT_VE

super(ReferenceProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
if isinstance(value, _STIXBase):
value = value.id
value = str(value)
Expand Down Expand Up @@ -675,7 +737,7 @@ def clean(self, value, allow_custom):

class SelectorProperty(Property):

def clean(self, value, allow_custom=False):
def clean(self, value, allow_custom=False, strict=False):
if not SELECTOR_REGEX.match(value):
raise ValueError("must adhere to selector syntax.")
return value, False
Expand All @@ -696,7 +758,7 @@ def __init__(self, type, **kwargs):
self.type = type
super(EmbeddedObjectProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
if isinstance(value, dict):
value = self.type(allow_custom=allow_custom, **value)
elif not isinstance(value, self.type):
Expand Down Expand Up @@ -724,7 +786,7 @@ def __init__(self, allowed, **kwargs):
self.allowed = allowed
super(EnumProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom, strict=False):
def clean(self, value, allow_custom=False, strict=False):

cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom, strict)

Expand All @@ -746,26 +808,20 @@ def __init__(self, allowed, **kwargs):
allowed = [allowed]
self.allowed = allowed

def clean(self, value, allow_custom, strict=False):
def clean(self, value, allow_custom=False, strict=False):
cleaned_value, _ = super(OpenVocabProperty, self).clean(
value, allow_custom, strict,
)

# Disabled: it was decided that enforcing this is too strict (might
# break too much user code). Revisit when we have the capability for
# more granular config settings when creating objects.
#
if strict is True:
has_custom = cleaned_value not in self.allowed

if not allow_custom and has_custom:
raise CustomContentError(
"custom value in open vocab: '{}'".format(cleaned_value),
)
# Customization enforcement is disabled: it was decided that enforcing
# it is too strict (might break too much user code). On the other
# hand, we need to lock it down in strict mode. If we are locking it
# down in strict mode, we always throw an exception if a value isn't
# in the vocab list, and never report anything as "custom".
if strict and cleaned_value not in self.allowed:
raise ValueError("not in vocab: " + cleaned_value)

has_custom = False

return cleaned_value, has_custom
return cleaned_value, False


class PatternProperty(StringProperty):
Expand All @@ -780,7 +836,7 @@ def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs):
self.spec_version = spec_version
super(ObservableProperty, self).__init__(*args, **kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
try:
dictified = _get_dict(value)
# get deep copy since we are going modify the dict and might
Expand Down Expand Up @@ -828,7 +884,7 @@ class ExtensionsProperty(DictionaryProperty):
def __init__(self, spec_version=DEFAULT_VERSION, required=False):
super(ExtensionsProperty, self).__init__(spec_version=spec_version, required=required)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
try:
dictified = _get_dict(value)
# get deep copy since we are going modify the dict and might
Expand Down Expand Up @@ -894,7 +950,7 @@ def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs):
self.spec_version = spec_version
super(STIXObjectProperty, self).__init__(*args, **kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
# Any STIX Object (SDO, SRO, or Marking Definition) can be added to
# a bundle with no further checks.
stix2_classes = {'_DomainObject', '_RelationshipObject', 'MarkingDefinition'}
Expand Down
Loading

0 comments on commit 92231d9

Please sign in to comment.