Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix DictionaryProperty valid_types bugs, and other misc property issues #602

Merged
merged 1 commit into from
Sep 16, 2024
Merged

Fix DictionaryProperty valid_types bugs, and other misc property issues #602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
224 changes: 140 additions & 84 deletions stix2/properties.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ class Property(object):

"""

def _default_clean(self, value, allow_custom=False):
def _default_clean(self, value, allow_custom=False, strict=False):
if value != self._fixed_value:
raise ValueError("must equal '{}'.".format(self._fixed_value))
return value, False
Expand Down Expand Up @@ -226,7 +226,7 @@ def __init__(self, contained, **kwargs):

super(ListProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom, strict_flag=False):
def clean(self, value, allow_custom=False, strict=False):
try:
iter(value)
except TypeError:
Expand All @@ -240,7 +240,7 @@ def clean(self, value, allow_custom, strict_flag=False):
if isinstance(self.contained, Property):
for item in value:
try:
valid, temp_custom = self.contained.clean(item, allow_custom, strict=strict_flag)
valid, temp_custom = self.contained.clean(item, allow_custom, strict=strict)
except TypeError:
valid, temp_custom = self.contained.clean(item, allow_custom)
result.append(valid)
Expand Down Expand Up @@ -281,11 +281,10 @@ def __init__(self, **kwargs):
super(StringProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom=False, strict=False):
if not isinstance(value, str):
if strict is True:
raise ValueError("Must be a string.")
if strict and not isinstance(value, str):
raise ValueError("Must be a string.")

value = str(value)
value = str(value)
return value, False


Expand Down Expand Up @@ -320,7 +319,7 @@ def __init__(self, min=None, max=None, **kwargs):
super(IntegerProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom=False, strict=False):
if strict is True and not isinstance(value, int):
if strict and not isinstance(value, int):
raise ValueError("must be an integer.")

try:
Expand All @@ -347,7 +346,7 @@ def __init__(self, min=None, max=None, **kwargs):
super(FloatProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom=False, strict=False):
if strict is True and not isinstance(value, float):
if strict and not isinstance(value, float):
raise ValueError("must be a float.")

try:
Expand All @@ -372,6 +371,9 @@ class BooleanProperty(Property):

def clean(self, value, allow_custom=False, strict=False):

if strict and not isinstance(value, bool):
raise ValueError("must be a boolean value.")

if isinstance(value, str):
value = value.lower()

Expand Down Expand Up @@ -403,74 +405,134 @@ class DictionaryProperty(Property):

def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
self.spec_version = spec_version
self.valid_types = self._normalize_valid_types(valid_types or [])

simple_types = [
BinaryProperty, BooleanProperty, FloatProperty, HexProperty, IntegerProperty, StringProperty,
TimestampProperty, ReferenceProperty, EnumProperty,
]
if not valid_types:
valid_types = [Property]
else:
if not isinstance(valid_types, list):
valid_types = [valid_types]
for type_ in valid_types:
if isinstance(type_, ListProperty):
found = False
for simple_type in simple_types:
if isinstance(type_.contained, simple_type):
found = True
if not found:
raise ValueError("Dictionary Property does not support lists of type: ", type_.contained, type(type_.contained))
elif type_ not in simple_types:
raise ValueError("Dictionary Property does not support this value's type: ", type_)
super(DictionaryProperty, self).__init__(**kwargs)

self.valid_types = valid_types
def _normalize_valid_types(self, valid_types):
"""
Normalize valid_types to a list of property instances. Also ensure any
property types given are supported for type enforcement.

super(DictionaryProperty, self).__init__(**kwargs)
:param valid_types: A single or iterable of Property instances or
subclasses
:return: A list of Property instances, or None if none were given
"""
simple_types = (
BinaryProperty, BooleanProperty, FloatProperty, HexProperty,
IntegerProperty, StringProperty, TimestampProperty,
ReferenceProperty, EnumProperty,
)

def clean(self, value, allow_custom=False):
# Normalize single prop instances/classes to lists
try:
iter(valid_types)
except TypeError:
valid_types = [valid_types]

prop_instances = []
for valid_type in valid_types:
if inspect.isclass(valid_type):
# Note: this will fail as of this writing with EnumProperty
# ReferenceProperty, ListProperty. Callers must instantiate
# those with suitable settings themselves.
prop_instance = valid_type()

else:
prop_instance = valid_type

# ListProperty's element type must be one of the supported
# simple types.
if isinstance(prop_instance, ListProperty):
if not isinstance(prop_instance.contained, simple_types):
raise ValueError(
"DictionaryProperty does not support lists of type: "
+ type(prop_instance.contained).__name__
)

elif not isinstance(prop_instance, simple_types):
raise ValueError(
"DictionaryProperty does not support value type: "
+ type(prop_instance).__name__
)

prop_instances.append(prop_instance)

return prop_instances or None

def _check_dict_key(self, k):
if self.spec_version == '2.0':
if len(k) < 3:
raise DictionaryKeyError(k, "shorter than 3 characters")
elif len(k) > 256:
raise DictionaryKeyError(k, "longer than 256 characters")
elif self.spec_version == '2.1':
if len(k) > 250:
raise DictionaryKeyError(k, "longer than 250 characters")
if not re.match(r"^[a-zA-Z0-9_-]+$", k):
msg = (
"contains characters other than lowercase a-z, "
"uppercase A-Z, numerals 0-9, hyphen (-), or "
"underscore (_)"
)
raise DictionaryKeyError(k, msg)

def clean(self, value, allow_custom=False, strict=False):
try:
dictified = _get_dict(value)
except ValueError:
raise ValueError("The dictionary property must contain a dictionary")

for k in dictified.keys():
if self.spec_version == '2.0':
if len(k) < 3:
raise DictionaryKeyError(k, "shorter than 3 characters")
elif len(k) > 256:
raise DictionaryKeyError(k, "longer than 256 characters")
elif self.spec_version == '2.1':
if len(k) > 250:
raise DictionaryKeyError(k, "longer than 250 characters")
if not re.match(r"^[a-zA-Z0-9_-]+$", k):
msg = (
"contains characters other than lowercase a-z, "
"uppercase A-Z, numerals 0-9, hyphen (-), or "
"underscore (_)"
)
raise DictionaryKeyError(k, msg)
has_custom = False
for k, v in dictified.items():

clean = False
for type_ in self.valid_types:
if isinstance(type_, ListProperty):
type_.clean(value=dictified[k], allow_custom=False, strict_flag=True)
clean = True
else:
type_instance = type_()
self._check_dict_key(k)

if self.valid_types:
for type_ in self.valid_types:
try:
type_instance.clean(value=dictified[k], allow_custom=False, strict=True)
clean = True
# ReferenceProperty at least, does check for
# customizations, so we must propagate that
dictified[k], temp_custom = type_.clean(
value=v,
allow_custom=allow_custom,
# Ignore the passed-in value and fix this to True;
# we need strict cleaning to disambiguate value
# types here.
strict=True
)
except CustomContentError:
# Need to propagate these, not treat as a type error
raise
except Exception as e:
# clean failed; value must not conform to type_
# Should be a narrower exception type here, but I don't
# know if it's safe to assume any particular exception
# types...
pass
else:
# clean succeeded; should check the has_custom flag
# just in case. But if allow_custom is False, I expect
# one of the valid_types property instances would have
# already raised an exception.
has_custom = has_custom or temp_custom
if has_custom and not allow_custom:
raise CustomContentError(f'Custom content detected in key "{k}"')

break
except ValueError:
continue
if not clean:
raise ValueError("Dictionary Property does not support this value's type: ", type(dictified[k]))

else:
# clean failed for all properties!
raise ValueError(
f"Invalid value: {v!r}"
)

# else: no valid types given, so we skip the validity check

if len(dictified) < 1:
raise ValueError("must not be empty.")

return dictified, False
return dictified, has_custom


class HashesProperty(DictionaryProperty):
Expand All @@ -488,7 +550,7 @@ def __init__(self, spec_hash_names, spec_version=DEFAULT_VERSION, **kwargs):
if alg:
self.__alg_to_spec_name[alg] = spec_hash_name

def clean(self, value, allow_custom, strict=False):
def clean(self, value, allow_custom=False, strict=False):
# ignore the has_custom return value here; there is no customization
# of DictionaryProperties.
clean_dict, _ = super().clean(value, allow_custom)
Expand Down Expand Up @@ -597,7 +659,7 @@ def __init__(self, valid_types=None, invalid_types=None, spec_version=DEFAULT_VE

super(ReferenceProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
if isinstance(value, _STIXBase):
value = value.id
value = str(value)
Expand Down Expand Up @@ -675,7 +737,7 @@ def clean(self, value, allow_custom):

class SelectorProperty(Property):

def clean(self, value, allow_custom=False):
def clean(self, value, allow_custom=False, strict=False):
if not SELECTOR_REGEX.match(value):
raise ValueError("must adhere to selector syntax.")
return value, False
Expand All @@ -696,7 +758,7 @@ def __init__(self, type, **kwargs):
self.type = type
super(EmbeddedObjectProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
if isinstance(value, dict):
value = self.type(allow_custom=allow_custom, **value)
elif not isinstance(value, self.type):
Expand Down Expand Up @@ -724,7 +786,7 @@ def __init__(self, allowed, **kwargs):
self.allowed = allowed
super(EnumProperty, self).__init__(**kwargs)

def clean(self, value, allow_custom, strict=False):
def clean(self, value, allow_custom=False, strict=False):

cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom, strict)

Expand All @@ -746,26 +808,20 @@ def __init__(self, allowed, **kwargs):
allowed = [allowed]
self.allowed = allowed

def clean(self, value, allow_custom, strict=False):
def clean(self, value, allow_custom=False, strict=False):
cleaned_value, _ = super(OpenVocabProperty, self).clean(
value, allow_custom, strict,
)

# Disabled: it was decided that enforcing this is too strict (might
# break too much user code). Revisit when we have the capability for
# more granular config settings when creating objects.
#
if strict is True:
has_custom = cleaned_value not in self.allowed

if not allow_custom and has_custom:
raise CustomContentError(
"custom value in open vocab: '{}'".format(cleaned_value),
)
# Customization enforcement is disabled: it was decided that enforcing
# it is too strict (might break too much user code). On the other
# hand, we need to lock it down in strict mode. If we are locking it
# down in strict mode, we always throw an exception if a value isn't
# in the vocab list, and never report anything as "custom".
if strict and cleaned_value not in self.allowed:
raise ValueError("not in vocab: " + cleaned_value)

has_custom = False

return cleaned_value, has_custom
return cleaned_value, False


class PatternProperty(StringProperty):
Expand All @@ -780,7 +836,7 @@ def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs):
self.spec_version = spec_version
super(ObservableProperty, self).__init__(*args, **kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
try:
dictified = _get_dict(value)
# get deep copy since we are going modify the dict and might
Expand Down Expand Up @@ -828,7 +884,7 @@ class ExtensionsProperty(DictionaryProperty):
def __init__(self, spec_version=DEFAULT_VERSION, required=False):
super(ExtensionsProperty, self).__init__(spec_version=spec_version, required=required)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
try:
dictified = _get_dict(value)
# get deep copy since we are going modify the dict and might
Expand Down Expand Up @@ -894,7 +950,7 @@ def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs):
self.spec_version = spec_version
super(STIXObjectProperty, self).__init__(*args, **kwargs)

def clean(self, value, allow_custom):
def clean(self, value, allow_custom=False, strict=False):
# Any STIX Object (SDO, SRO, or Marking Definition) can be added to
# a bundle with no further checks.
stix2_classes = {'_DomainObject', '_RelationshipObject', 'MarkingDefinition'}
Expand Down
Loading
Loading